From 05937ab75b2893173c8211c66dc3ded1601edd86 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 13 Mar 2024 08:24:17 -0500 Subject: [PATCH] exec2: add client support for unveil filesystem isolation mode (#20115) * exec2: add client support for unveil filesystem isolation mode This PR adds support for a new filesystem isolation mode, "Unveil". The mode introduces a "alloc_mounts" directory where tasks have user-owned directory structure which are bind mounts into the real alloc directory structure. This enables a task driver to use landlock (and maybe the real unveil on openbsd one day) to isolate a task to the task owned directory structure, providing sandboxing. * actually create alloc-mounts-dir directory * fix doc strings about alloc mount dir paths --- client/alloc_endpoint.go | 3 +- client/allocdir/alloc_dir.go | 38 +- client/allocdir/alloc_dir_test.go | 234 ++++----- client/allocdir/fs_default.go | 13 + client/allocdir/fs_linux.go | 14 + client/allocdir/task_dir.go | 88 +++- client/allocdir/task_dir_test.go | 109 ++-- client/allocdir/testing.go | 2 +- client/allocrunner/alloc_runner.go | 1 + .../taskrunner/connect_native_hook_test.go | 11 +- .../taskrunner/dispatch_hook_test.go | 13 +- .../taskrunner/envoy_bootstrap_hook_test.go | 13 +- .../taskrunner/envoy_version_hook_test.go | 13 +- .../allocrunner/taskrunner/task_dir_hook.go | 16 +- .../taskrunner/task_runner_linux_test.go | 3 +- .../taskrunner/task_runner_test.go | 2 +- .../taskrunner/template/template_test.go | 2 +- client/allocwatcher/alloc_watcher.go | 2 +- client/allocwatcher/alloc_watcher_test.go | 2 +- client/client.go | 16 +- client/client_test.go | 29 + client/config/config.go | 8 + client/fs_endpoint_test.go | 2 +- command/agent/agent.go | 4 + command/agent/command.go | 18 +- command/agent/command_test.go | 2 +- command/agent/config.go | 6 + command/agent/config_parse_test.go | 11 +- command/agent/consul/int_test.go | 2 +- command/agent/testdata/basic.hcl | 11 +- command/agent/testdata/basic.json | 1 + drivers/docker/config.go | 3 +- drivers/exec/driver.go | 3 +- drivers/java/driver.go | 7 +- drivers/mock/driver.go | 3 +- drivers/mock/driver_test.go | 5 +- drivers/qemu/driver.go | 3 +- drivers/rawexec/driver.go | 3 +- .../shared/executor/executor_linux_test.go | 5 +- drivers/shared/executor/executor_test.go | 9 +- plugins/drivers/client.go | 11 +- plugins/drivers/driver.go | 21 +- plugins/drivers/fsisolation/isolation.go | 25 + plugins/drivers/proto/driver.pb.go | 495 +++++++++--------- plugins/drivers/proto/driver.proto | 1 + plugins/drivers/server.go | 9 +- plugins/drivers/testutils/testing.go | 4 +- 47 files changed, 733 insertions(+), 563 deletions(-) create mode 100644 client/allocdir/fs_default.go create mode 100644 plugins/drivers/fsisolation/isolation.go diff --git a/client/alloc_endpoint.go b/client/alloc_endpoint.go index d063fd54d..f236901be 100644 --- a/client/alloc_endpoint.go +++ b/client/alloc_endpoint.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/nomad/helper/uuid" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" ) // Allocations endpoint is used for interacting with client allocations @@ -286,7 +287,7 @@ func (a *Allocations) execImpl(encoder *codec.Encoder, decoder *codec.Decoder, e } // check node access - if capabilities.FSIsolation == drivers.FSIsolationNone { + if capabilities.FSIsolation == fsisolation.None { exec := aclObj.AllowNsOp(alloc.Namespace, acl.NamespaceCapabilityAllocNodeExec) if !exec { return nil, nstructs.ErrPermissionDenied diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 481e65ea9..6760507ef 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -77,9 +77,6 @@ var ( ) // Interface is implemented by AllocDir. -// -// TODO(shoenig) soon to be implemented by AllocDir2 in support of the exec2 -// driver and perhaps other drivers with landlock or unveil capabilities. type Interface interface { AllocDirFS @@ -110,6 +107,10 @@ type AllocDir struct { // be excluded from chroots and is configured via client.alloc_dir. clientAllocDir string + // clientAllocMountsDir is the client agent's mounts directory. It must be + // excluded from chroots and is configured via client.mounts_dir. + clientAllocMountsDir string + // built is true if Build has successfully run built bool @@ -144,17 +145,18 @@ type AllocDirFS interface { // NewAllocDir initializes the AllocDir struct with allocDir as base path for // the allocation directory. -func NewAllocDir(logger hclog.Logger, clientAllocDir, allocID string) *AllocDir { +func NewAllocDir(logger hclog.Logger, clientAllocDir, clientMountsDir, allocID string) *AllocDir { logger = logger.Named("alloc_dir") allocDir := filepath.Join(clientAllocDir, allocID) shareDir := filepath.Join(allocDir, SharedAllocName) return &AllocDir{ - clientAllocDir: clientAllocDir, - AllocDir: allocDir, - SharedDir: shareDir, - TaskDirs: make(map[string]*TaskDir), - logger: logger, + clientAllocDir: clientAllocDir, + clientAllocMountsDir: clientMountsDir, + AllocDir: allocDir, + SharedDir: shareDir, + TaskDirs: make(map[string]*TaskDir), + logger: logger, } } @@ -163,7 +165,7 @@ func (d *AllocDir) NewTaskDir(name string) *TaskDir { d.mu.Lock() defer d.mu.Unlock() - td := newTaskDir(d.logger, d.clientAllocDir, d.AllocDir, name) + td := d.newTaskDir(name) d.TaskDirs[name] = td return td } @@ -349,6 +351,22 @@ func (d *AllocDir) UnmountAll() error { } } + if pathExists(dir.MountsAllocDir) { + if err := unlinkDir(dir.MountsAllocDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the alloc mounts dir %q: %v", dir.MountsAllocDir, err), + ) + } + } + + if pathExists(dir.MountsTaskDir) { + if err := unlinkDir(dir.MountsTaskDir); err != nil { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("failed to remove the alloc mounts task dir %q: %v", dir.MountsTaskDir, err), + ) + } + } + // Unmount dev/ and proc/ have been mounted. if err := dir.unmountSpecialDirs(); err != nil { mErr.Errors = append(mErr.Errors, err) diff --git a/client/allocdir/alloc_dir_test.go b/client/allocdir/alloc_dir_test.go index e2e1ee8e8..5f7e96fcb 100644 --- a/client/allocdir/alloc_dir_test.go +++ b/client/allocdir/alloc_dir_test.go @@ -19,9 +19,25 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/structs" - "github.com/stretchr/testify/require" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" + "github.com/shoenig/test/must" + "golang.org/x/sys/unix" ) +// copy from testutil to avoid import cycle +func requireNonRoot(t *testing.T) { + if unix.Geteuid() == 0 { + t.Skip("must run as non-root") + } +} + +// copy from testutil to avoid import cycle +func requireRoot(t *testing.T) { + if unix.Geteuid() != 0 { + t.Skip("must run as root") + } +} + var ( t1 = &structs.Task{ Name: "web", @@ -54,13 +70,11 @@ func TestAllocDir_BuildAlloc(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() d.NewTaskDir(t1.Name) d.NewTaskDir(t2.Name) - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + must.NoError(t, d.Build()) // Check that the AllocDir and each of the task directories exist. if _, err := os.Stat(d.AllocDir); os.IsNotExist(err) { @@ -69,17 +83,13 @@ func TestAllocDir_BuildAlloc(t *testing.T) { for _, task := range []*structs.Task{t1, t2} { tDir, ok := d.TaskDirs[task.Name] - if !ok { - t.Fatalf("Task directory not found for %v", task.Name) - } + must.True(t, ok) - if stat, _ := os.Stat(tDir.Dir); stat != nil { - t.Fatalf("Build() created TaskDir %v", tDir.Dir) - } + stat, _ := os.Stat(tDir.Dir) + must.Nil(t, stat) - if stat, _ := os.Stat(tDir.SecretsDir); stat != nil { - t.Fatalf("Build() created secret dir %v", tDir.Dir) - } + stat, _ = os.Stat(tDir.SecretsDir) + must.Nil(t, stat) } } @@ -103,28 +113,21 @@ func TestAllocDir_MountSharedAlloc(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + must.NoError(t, d.Build()) // Build 2 task dirs td1 := d.NewTaskDir(t1.Name) - if err := td1.Build(true, nil); err != nil { - t.Fatalf("error build task=%q dir: %v", t1.Name, err) - } + must.NoError(t, td1.Build(fsisolation.Chroot, nil, "nobody")) + td2 := d.NewTaskDir(t2.Name) - if err := td2.Build(true, nil); err != nil { - t.Fatalf("error build task=%q dir: %v", t2.Name, err) - } + must.NoError(t, td2.Build(fsisolation.Chroot, nil, "nobody")) // Write a file to the shared dir. contents := []byte("foo") const filename = "bar" - if err := os.WriteFile(filepath.Join(d.SharedDir, filename), contents, 0666); err != nil { - t.Fatalf("Couldn't write file to shared directory: %v", err) - } + must.NoError(t, os.WriteFile(filepath.Join(d.SharedDir, filename), contents, 0o666)) // Check that the file exists in the task directories for _, td := range []*TaskDir{td1, td2} { @@ -146,52 +149,37 @@ func TestAllocDir_Snapshot(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + must.NoError(t, d.Build()) // Build 2 task dirs td1 := d.NewTaskDir(t1.Name) - if err := td1.Build(false, nil); err != nil { - t.Fatalf("error build task=%q dir: %v", t1.Name, err) - } + must.NoError(t, td1.Build(fsisolation.None, nil, "nobody")) + td2 := d.NewTaskDir(t2.Name) - if err := td2.Build(false, nil); err != nil { - t.Fatalf("error build task=%q dir: %v", t2.Name, err) - } + must.NoError(t, td2.Build(fsisolation.None, nil, "nobody")) // Write a file to the shared dir. exp := []byte{'f', 'o', 'o'} file := "bar" - if err := os.WriteFile(filepath.Join(d.SharedDir, "data", file), exp, 0666); err != nil { - t.Fatalf("Couldn't write file to shared directory: %v", err) - } + must.NoError(t, os.WriteFile(filepath.Join(d.SharedDir, "data", file), exp, 0o666)) // Write a symlink to the shared dir link := "qux" - if err := os.Symlink("foo", filepath.Join(d.SharedDir, "data", link)); err != nil { - t.Fatalf("Couldn't write symlink to shared directory: %v", err) - } + must.NoError(t, os.Symlink("foo", filepath.Join(d.SharedDir, "data", link))) // Write a file to the task local exp = []byte{'b', 'a', 'r'} file1 := "lol" - if err := os.WriteFile(filepath.Join(td1.LocalDir, file1), exp, 0666); err != nil { - t.Fatalf("couldn't write file to task local directory: %v", err) - } + must.NoError(t, os.WriteFile(filepath.Join(td1.LocalDir, file1), exp, 0o666)) // Write a symlink to the task local link1 := "baz" - if err := os.Symlink("bar", filepath.Join(td1.LocalDir, link1)); err != nil { - t.Fatalf("couldn't write symlink to task local directory :%v", err) - } + must.NoError(t, os.Symlink("bar", filepath.Join(td1.LocalDir, link1))) var b bytes.Buffer - if err := d.Snapshot(&b); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, d.Snapshot(&b)) tr := tar.NewReader(&b) var files []string @@ -211,12 +199,8 @@ func TestAllocDir_Snapshot(t *testing.T) { } } - if len(files) != 2 { - t.Fatalf("bad files: %#v", files) - } - if len(links) != 2 { - t.Fatalf("bad links: %#v", links) - } + must.SliceLen(t, 2, files) + must.SliceLen(t, 2, links) } func TestAllocDir_Move(t *testing.T) { @@ -226,22 +210,16 @@ func TestAllocDir_Move(t *testing.T) { tmp2 := t.TempDir() // Create two alloc dirs - d1 := NewAllocDir(testlog.HCLogger(t), tmp1, "test") - if err := d1.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + d1 := NewAllocDir(testlog.HCLogger(t), tmp1, tmp1, "test") + must.NoError(t, d1.Build()) defer d1.Destroy() - d2 := NewAllocDir(testlog.HCLogger(t), tmp2, "test") - if err := d2.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + d2 := NewAllocDir(testlog.HCLogger(t), tmp2, tmp2, "test") + must.NoError(t, d2.Build()) defer d2.Destroy() td1 := d1.NewTaskDir(t1.Name) - if err := td1.Build(false, nil); err != nil { - t.Fatalf("TaskDir.Build() faild: %v", err) - } + must.NoError(t, td1.Build(fsisolation.None, nil, "nobody")) // Create but don't build second task dir to mimic alloc/task runner // behavior (AllocDir.Move() is called pre-TaskDir.Build). @@ -252,32 +230,24 @@ func TestAllocDir_Move(t *testing.T) { // Write a file to the shared dir. exp1 := []byte("foo") file1 := "bar" - if err := os.WriteFile(filepath.Join(dataDir, file1), exp1, 0666); err != nil { - t.Fatalf("Couldn't write file to shared directory: %v", err) - } + must.NoError(t, os.WriteFile(filepath.Join(dataDir, file1), exp1, 0o666)) // Write a file to the task local exp2 := []byte("bar") file2 := "lol" - if err := os.WriteFile(filepath.Join(td1.LocalDir, file2), exp2, 0666); err != nil { - t.Fatalf("couldn't write to task local directory: %v", err) - } + must.NoError(t, os.WriteFile(filepath.Join(td1.LocalDir, file2), exp2, 0o666)) // Move the d1 allocdir to d2 - if err := d2.Move(d1, []*structs.Task{t1}); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, d2.Move(d1, []*structs.Task{t1})) // Ensure the files in d1 are present in d2 fi, err := os.Stat(filepath.Join(d2.SharedDir, SharedDataDir, file1)) - if err != nil || fi == nil { - t.Fatalf("data dir was not moved") - } + must.NoError(t, err) + must.NotNil(t, fi) fi, err = os.Stat(filepath.Join(d2.TaskDirs[t1.Name].LocalDir, file2)) - if err != nil || fi == nil { - t.Fatalf("task local dir was not moved") - } + must.NoError(t, err) + must.NotNil(t, fi) } func TestAllocDir_EscapeChecking(t *testing.T) { @@ -285,10 +255,8 @@ func TestAllocDir_EscapeChecking(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") + must.NoError(t, d.Build()) defer d.Destroy() // Check that issuing calls that escape the alloc dir returns errors @@ -323,28 +291,23 @@ func TestAllocDir_ReadAt_SecretDir(t *testing.T) { ci.Parallel(t) tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") - err := d.Build() - require.NoError(t, err) - defer func() { - _ = d.Destroy() - }() + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") + must.NoError(t, d.Build()) + defer func() { _ = d.Destroy() }() td := d.NewTaskDir(t1.Name) - err = td.Build(false, nil) - require.NoError(t, err) + must.NoError(t, td.Build(fsisolation.None, nil, "nobody")) // something to write and test reading target := filepath.Join(t1.Name, TaskSecrets, "test_file") // create target file in the task secrets dir full := filepath.Join(d.AllocDir, target) - err = os.WriteFile(full, []byte("hi"), 0600) - require.NoError(t, err) + must.NoError(t, os.WriteFile(full, []byte("hi"), 0o600)) // ReadAt of a file in the task secrets dir should fail - _, err = d.ReadAt(target, 0) - require.EqualError(t, err, "Reading secret file prohibited: web/secrets/test_file") + _, err := d.ReadAt(target, 0) + must.EqError(t, err, "Reading secret file prohibited: web/secrets/test_file") } func TestAllocDir_SplitPath(t *testing.T) { @@ -353,54 +316,39 @@ func TestAllocDir_SplitPath(t *testing.T) { dir := t.TempDir() dest := filepath.Join(dir, "/foo/bar/baz") - if err := os.MkdirAll(dest, os.ModePerm); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, os.MkdirAll(dest, os.ModePerm)) info, err := splitPath(dest) - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) + // Testing that is 6 or more rather than 6 because on osx, the temp dir is // randomized. - if len(info) < 6 { - t.Fatalf("expected more than: %v, actual: %v", 6, len(info)) - } + must.GreaterEq(t, 6, len(info)) } func TestAllocDir_CreateDir(t *testing.T) { + requireRoot(t) + ci.Parallel(t) - if syscall.Geteuid() != 0 { - t.Skip("Must be root to run test") - } dir := t.TempDir() // create a subdir and a file subdir := filepath.Join(dir, "subdir") - if err := os.MkdirAll(subdir, 0760); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, os.MkdirAll(subdir, 0o760)) + subdirMode, err := os.Stat(subdir) - if err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, err) // Create the above hierarchy under another destination dir1 := t.TempDir() - if err := createDir(dir1, subdir); err != nil { - t.Fatalf("err: %v", err) - } + must.NoError(t, createDir(dir1, subdir)) // Ensure that the subdir had the right perm fi, err := os.Stat(filepath.Join(dir1, dir, "subdir")) - if err != nil { - t.Fatalf("err: %v", err) - } - if fi.Mode() != subdirMode.Mode() { - t.Fatalf("wrong file mode: %v, expected: %v", fi.Mode(), subdirMode.Mode()) - } + must.NoError(t, err) + must.Eq(t, fi.Mode(), subdirMode.Mode()) } func TestPathFuncs(t *testing.T) { @@ -410,12 +358,8 @@ func TestPathFuncs(t *testing.T) { missingDir := filepath.Join(dir, "does-not-exist") - if !pathExists(dir) { - t.Errorf("%q exists", dir) - } - if pathExists(missingDir) { - t.Errorf("%q does not exist", missingDir) - } + must.True(t, pathExists(dir)) + must.False(t, pathExists(missingDir)) if empty, err := pathEmpty(dir); err != nil || !empty { t.Errorf("%q is empty and exists. empty=%v error=%v", dir, empty, err) @@ -426,9 +370,7 @@ func TestPathFuncs(t *testing.T) { filename := filepath.Join(dir, "just-some-file") f, err := os.Create(filename) - if err != nil { - t.Fatalf("could not create %q: %v", filename, err) - } + must.NoError(t, err) f.Close() if empty, err := pathEmpty(dir); err != nil || empty { @@ -438,7 +380,6 @@ func TestPathFuncs(t *testing.T) { func TestAllocDir_DetectContentType(t *testing.T) { ci.Parallel(t) - require := require.New(t) inputPath := "input/" var testFiles []string @@ -448,7 +389,7 @@ func TestAllocDir_DetectContentType(t *testing.T) { } return err }) - require.Nil(err) + must.NoError(t, err) expectedEncodings := map[string]string{ "input/happy.gif": "image/gif", @@ -462,9 +403,9 @@ func TestAllocDir_DetectContentType(t *testing.T) { } for _, file := range testFiles { fileInfo, err := os.Stat(file) - require.Nil(err) + must.NoError(t, err) res := detectContentType(fileInfo, file) - require.Equal(expectedEncodings[file], res, "unexpected output for %v", file) + must.Eq(t, expectedEncodings[file], res) } } @@ -482,10 +423,11 @@ func TestAllocDir_SkipAllocDir(t *testing.T) { rootDir := t.TempDir() clientAllocDir := filepath.Join(rootDir, "nomad") - require.NoError(t, os.Mkdir(clientAllocDir, fs.ModeDir|0o777)) + mountAllocDir := filepath.Join(rootDir, "mounts") + must.NoError(t, os.Mkdir(clientAllocDir, fs.ModeDir|0o777)) otherDir := filepath.Join(rootDir, "etc") - require.NoError(t, os.Mkdir(otherDir, fs.ModeDir|0o777)) + must.NoError(t, os.Mkdir(otherDir, fs.ModeDir|0o777)) // chroot contains client.alloc_dir! This could cause infinite // recursion. @@ -493,15 +435,15 @@ func TestAllocDir_SkipAllocDir(t *testing.T) { rootDir: "/", } - allocDir := NewAllocDir(testlog.HCLogger(t), clientAllocDir, "test") + allocDir := NewAllocDir(testlog.HCLogger(t), clientAllocDir, mountAllocDir, "test") taskDir := allocDir.NewTaskDir("testtask") - require.NoError(t, allocDir.Build()) + must.NoError(t, allocDir.Build()) defer allocDir.Destroy() // Build chroot - err := taskDir.Build(true, chroot) - require.NoError(t, err) + err := taskDir.Build(fsisolation.Chroot, chroot, "nobody") + must.NoError(t, err) // Assert other directory *was* embedded embeddedOtherDir := filepath.Join(clientAllocDir, "test", "testtask", "etc") diff --git a/client/allocdir/fs_default.go b/client/allocdir/fs_default.go new file mode 100644 index 000000000..4f36ef3b7 --- /dev/null +++ b/client/allocdir/fs_default.go @@ -0,0 +1,13 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !linux + +package allocdir + +import "os" + +// mountDir bind mounts old to next using the given file mode. +func mountDir(old, next string, uid, gid int, mode os.FileMode) error { + panic("not implemented") +} diff --git a/client/allocdir/fs_linux.go b/client/allocdir/fs_linux.go index 798c1edc7..3dae8c5d0 100644 --- a/client/allocdir/fs_linux.go +++ b/client/allocdir/fs_linux.go @@ -1,6 +1,8 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 +//go:build linux + package allocdir import ( @@ -31,6 +33,18 @@ func linkDir(src, dst string) error { return syscall.Mount(src, dst, "", syscall.MS_BIND, "") } +// mountDir bind mounts old to next using the given file mode. +func mountDir(old, next string, uid, gid int, mode os.FileMode) error { + if err := os.MkdirAll(next, mode); err != nil { + return err + } + opts := unix.MS_BIND | unix.MS_NOSUID | unix.MS_NOATIME + if err := unix.Mount(old, next, "", uintptr(opts), ""); err != nil { + return err + } + return os.Chown(next, uid, gid) +} + // unlinkDir unmounts a bind mounted directory as Linux doesn't support // hardlinking directories. If the dir is already unmounted no error is // returned. diff --git a/client/allocdir/task_dir.go b/client/allocdir/task_dir.go index 9e18a2611..4270b5d4d 100644 --- a/client/allocdir/task_dir.go +++ b/client/allocdir/task_dir.go @@ -8,47 +8,74 @@ import ( "os" "path/filepath" - hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-set/v2" + "github.com/hashicorp/nomad/helper/users/dynamic" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" ) // TaskDir contains all of the paths relevant to a task. All paths are on the // host system so drivers should mount/link into task containers as necessary. type TaskDir struct { - // AllocDir is the path to the alloc directory on the host + // AllocDir is the path to the alloc directory on the host. + // (not to be conflated with client.alloc_dir) + // + // AllocDir string - // Dir is the path to Task directory on the host + // Dir is the path to Task directory on the host. + // + // Dir string + // MountsAllocDir is the path to the alloc directory on the host that has + // been bind mounted under + // + // //alloc -> + MountsAllocDir string + + // MountsTaskDir is the path to the task directory on the host that has been + // bind mounted under + // + // //task -> + MountsTaskDir string + // SharedAllocDir is the path to shared alloc directory on the host + // // /alloc/ SharedAllocDir string // SharedTaskDir is the path to the shared alloc directory linked into // the task directory on the host. + // // /alloc/ SharedTaskDir string // LocalDir is the path to the task's local directory on the host + // // /local/ LocalDir string // LogDir is the path to the task's log directory on the host + // // /alloc/logs/ LogDir string // SecretsDir is the path to secrets/ directory on the host + // // /secrets/ SecretsDir string // PrivateDir is the path to private/ directory on the host + // // /private/ PrivateDir string // skip embedding these paths in chroots. Used for avoiding embedding - // client.alloc_dir recursively. - skip map[string]struct{} + // client.alloc_dir and client.mounts_dir recursively. + skip *set.Set[string] + // logger for this task logger hclog.Logger } @@ -56,32 +83,30 @@ type TaskDir struct { // create paths on disk. // // Call AllocDir.NewTaskDir to create new TaskDirs -func newTaskDir(logger hclog.Logger, clientAllocDir, allocDir, taskName string) *TaskDir { - taskDir := filepath.Join(allocDir, taskName) - - logger = logger.Named("task_dir").With("task_name", taskName) - - // skip embedding client.alloc_dir in chroots - skip := map[string]struct{}{clientAllocDir: {}} +func (d *AllocDir) newTaskDir(taskName string) *TaskDir { + taskDir := filepath.Join(d.AllocDir, taskName) + taskUnique := filepath.Base(d.AllocDir) + "-" + taskName return &TaskDir{ - AllocDir: allocDir, + AllocDir: d.AllocDir, Dir: taskDir, - SharedAllocDir: filepath.Join(allocDir, SharedAllocName), - LogDir: filepath.Join(allocDir, SharedAllocName, LogDirName), + SharedAllocDir: filepath.Join(d.AllocDir, SharedAllocName), + LogDir: filepath.Join(d.AllocDir, SharedAllocName, LogDirName), SharedTaskDir: filepath.Join(taskDir, SharedAllocName), LocalDir: filepath.Join(taskDir, TaskLocal), SecretsDir: filepath.Join(taskDir, TaskSecrets), PrivateDir: filepath.Join(taskDir, TaskPrivate), - skip: skip, - logger: logger, + MountsTaskDir: filepath.Join(d.clientAllocMountsDir, taskUnique, "task"), + MountsAllocDir: filepath.Join(d.clientAllocMountsDir, taskUnique, "alloc"), + skip: set.From[string]([]string{d.clientAllocDir, d.clientAllocMountsDir}), + logger: d.logger.Named("task_dir").With("task_name", taskName), } } // Build default directories and permissions in a task directory. chrootCreated // allows skipping chroot creation if the caller knows it has already been // done. client.alloc_dir will be skipped. -func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { +func (t *TaskDir) Build(fsi fsisolation.Mode, chroot map[string]string, username string) error { if err := os.MkdirAll(t.Dir, 0777); err != nil { return err } @@ -116,7 +141,7 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { // Image based isolation will bind the shared alloc dir in the driver. // If there's no isolation the task will use the host path to the // shared alloc dir. - if createChroot { + if fsi == fsisolation.Chroot { // If the path doesn't exist OR it exists and is empty, link it empty, _ := pathEmpty(t.SharedTaskDir) if !pathExists(t.SharedTaskDir) || empty { @@ -145,12 +170,33 @@ func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error { } // Build chroot if chroot filesystem isolation is going to be used - if createChroot { + if fsi == fsisolation.Chroot { if err := t.buildChroot(chroot); err != nil { return err } } + // Only bind mount the task alloc/task dirs to the client.mounts_dir/ + if fsi == fsisolation.Unveil { + uid, gid, _, err := dynamic.LookupUser(username) + if err != nil { + return fmt.Errorf("Failed to lookup user: %v", err) + } + + // create the task unique directory under the client mounts path + parent := filepath.Dir(t.MountsAllocDir) + if err = os.MkdirAll(parent, 0o710); err != nil { + return fmt.Errorf("Failed to create task mount directory: %v", err) + } + if err = os.Chown(parent, uid, gid); err != nil { + return fmt.Errorf("Failed to chown task mount directory: %v", err) + } + + // create the task and alloc mount points + mountDir(t.AllocDir, t.MountsAllocDir, uid, gid, 0o710) + mountDir(t.Dir, t.MountsTaskDir, uid, gid, 0o710) + } + return nil } @@ -165,7 +211,7 @@ func (t *TaskDir) buildChroot(entries map[string]string) error { func (t *TaskDir) embedDirs(entries map[string]string) error { subdirs := make(map[string]string) for source, dest := range entries { - if _, ok := t.skip[source]; ok { + if t.skip.Contains(source) { // source in skip list continue } diff --git a/client/allocdir/task_dir_test.go b/client/allocdir/task_dir_test.go index d99b2af90..a92a79e5e 100644 --- a/client/allocdir/task_dir_test.go +++ b/client/allocdir/task_dir_test.go @@ -5,11 +5,14 @@ package allocdir import ( "os" + "os/user" "path/filepath" "testing" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" + "github.com/shoenig/test/must" ) // Test that building a chroot will skip nonexistent directories. @@ -18,18 +21,14 @@ func TestTaskDir_EmbedNonexistent(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() td := d.NewTaskDir(t1.Name) - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + must.NoError(t, d.Build()) fakeDir := "/foobarbaz" mapping := map[string]string{fakeDir: fakeDir} - if err := td.embedDirs(mapping); err != nil { - t.Fatalf("embedDirs(%v) should should skip %v since it does not exist", mapping, fakeDir) - } + must.NoError(t, td.embedDirs(mapping)) } // Test that building a chroot copies files from the host into the task dir. @@ -38,12 +37,10 @@ func TestTaskDir_EmbedDirs(t *testing.T) { tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() td := d.NewTaskDir(t1.Name) - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } + must.NoError(t, d.Build()) // Create a fake host directory, with a file, and a subfolder that contains // a file. @@ -51,26 +48,17 @@ func TestTaskDir_EmbedDirs(t *testing.T) { subDirName := "subdir" subDir := filepath.Join(host, subDirName) - if err := os.MkdirAll(subDir, 0777); err != nil { - t.Fatalf("Failed to make subdir %v: %v", subDir, err) - } + must.NoError(t, os.MkdirAll(subDir, 0o777)) file := "foo" subFile := "bar" - if err := os.WriteFile(filepath.Join(host, file), []byte{'a'}, 0777); err != nil { - t.Fatalf("Couldn't create file in host dir %v: %v", host, err) - } - - if err := os.WriteFile(filepath.Join(subDir, subFile), []byte{'a'}, 0777); err != nil { - t.Fatalf("Couldn't create file in host subdir %v: %v", subDir, err) - } + must.NoError(t, os.WriteFile(filepath.Join(host, file), []byte{'a'}, 0o777)) + must.NoError(t, os.WriteFile(filepath.Join(subDir, subFile), []byte{'a'}, 0o777)) // Create mapping from host dir to task dir. taskDest := "bin/test/" mapping := map[string]string{host: taskDest} - if err := td.embedDirs(mapping); err != nil { - t.Fatalf("embedDirs(%v) failed: %v", mapping, err) - } + must.NoError(t, td.embedDirs(mapping)) exp := []string{filepath.Join(td.Dir, taskDest, file), filepath.Join(td.Dir, taskDest, subDirName, subFile)} for _, f := range exp { @@ -82,46 +70,75 @@ func TestTaskDir_EmbedDirs(t *testing.T) { // Test that task dirs for image based isolation don't require root. func TestTaskDir_NonRoot_Image(t *testing.T) { + requireNonRoot(t) + ci.Parallel(t) - if os.Geteuid() == 0 { - t.Skip("test should be run as non-root user") - } + tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() td := d.NewTaskDir(t1.Name) - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } - - if err := td.Build(false, nil); err != nil { - t.Fatalf("TaskDir.Build failed: %v", err) - } + must.NoError(t, d.Build()) + must.NoError(t, td.Build(fsisolation.Image, nil, "nobody")) } // Test that task dirs with no isolation don't require root. func TestTaskDir_NonRoot(t *testing.T) { + requireNonRoot(t) + ci.Parallel(t) - if os.Geteuid() == 0 { - t.Skip("test should be run as non-root user") - } tmp := t.TempDir() - d := NewAllocDir(testlog.HCLogger(t), tmp, "test") + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") defer d.Destroy() td := d.NewTaskDir(t1.Name) - if err := d.Build(); err != nil { - t.Fatalf("Build() failed: %v", err) - } - - if err := td.Build(false, nil); err != nil { - t.Fatalf("TaskDir.Build failed: %v", err) - } + must.NoError(t, d.Build()) + must.NoError(t, td.Build(fsisolation.None, nil, "nobody")) // ${TASK_DIR}/alloc should not exist! if _, err := os.Stat(td.SharedTaskDir); !os.IsNotExist(err) { t.Fatalf("Expected a NotExist error for shared alloc dir in task dir: %q", td.SharedTaskDir) } } + +func TestTaskDir_NonRoot_Unveil(t *testing.T) { + requireNonRoot(t) + + ci.Parallel(t) + + tmp := t.TempDir() + + // non-root, should still work for tasks running as the same user as the + // nomad client agent + u, err := user.Current() + must.NoError(t, err) + + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") + defer d.Destroy() + td := d.NewTaskDir(t1.Name) + must.NoError(t, d.Build()) + must.NoError(t, td.Build(fsisolation.Unveil, nil, u.Username)) + fi, err := os.Stat(td.MountsTaskDir) + must.NoError(t, err) + must.NotNil(t, fi) +} + +func TestTaskDir_Root_Unveil(t *testing.T) { + requireRoot(t) + + ci.Parallel(t) + + tmp := t.TempDir() + + // root, can build task dirs for another user + d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test") + defer d.Destroy() + td := d.NewTaskDir(t1.Name) + must.NoError(t, d.Build()) + must.NoError(t, td.Build(fsisolation.Unveil, nil, "nobody")) + fi, err := os.Stat(td.MountsTaskDir) + must.NoError(t, err) + must.NotNil(t, fi) +} diff --git a/client/allocdir/testing.go b/client/allocdir/testing.go index 382bb631c..6efd22a41 100644 --- a/client/allocdir/testing.go +++ b/client/allocdir/testing.go @@ -18,7 +18,7 @@ func TestAllocDir(t testing.T, l hclog.Logger, prefix, id string) (*AllocDir, fu t.Fatalf("Couldn't create temp dir: %v", err) } - allocDir := NewAllocDir(l, dir, id) + allocDir := NewAllocDir(l, dir, dir, id) cleanup := func() { if err := os.RemoveAll(dir); err != nil { diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index 2fcf7a592..c88b90bc0 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -275,6 +275,7 @@ func NewAllocRunner(config *config.AllocRunnerConfig) (interfaces.AllocRunner, e ar.allocDir = allocdir.NewAllocDir( ar.logger, config.ClientConfig.AllocDir, + config.ClientConfig.AllocMountsDir, alloc.ID, ) diff --git a/client/allocrunner/taskrunner/connect_native_hook_test.go b/client/allocrunner/taskrunner/connect_native_hook_test.go index a3e7615b7..693c859ae 100644 --- a/client/allocrunner/taskrunner/connect_native_hook_test.go +++ b/client/allocrunner/taskrunner/connect_native_hook_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/stretchr/testify/require" ) @@ -280,7 +281,7 @@ func TestTaskRunner_ConnectNativeHook_Noop(t *testing.T) { Task: task, TaskDir: allocDir.NewTaskDir(task.Name), } - require.NoError(t, request.TaskDir.Build(false, nil)) + require.NoError(t, request.TaskDir.Build(fsisolation.None, nil, task.User)) response := new(interfaces.TaskPrestartResponse) @@ -342,7 +343,7 @@ func TestTaskRunner_ConnectNativeHook_Ok(t *testing.T) { TaskDir: allocDir.NewTaskDir(tg.Tasks[0].Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, request.TaskDir.Build(false, nil)) + require.NoError(t, request.TaskDir.Build(fsisolation.None, nil, tg.Tasks[0].User)) response := new(interfaces.TaskPrestartResponse) @@ -404,7 +405,7 @@ func TestTaskRunner_ConnectNativeHook_with_SI_token(t *testing.T) { TaskDir: allocDir.NewTaskDir(tg.Tasks[0].Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, request.TaskDir.Build(false, nil)) + require.NoError(t, request.TaskDir.Build(fsisolation.None, nil, tg.Tasks[0].User)) // Insert service identity token in the secrets directory token := uuid.Generate() @@ -487,7 +488,7 @@ func TestTaskRunner_ConnectNativeHook_shareTLS(t *testing.T) { TaskDir: allocDir.NewTaskDir(tg.Tasks[0].Name), TaskEnv: taskenv.NewEmptyTaskEnv(), // nothing set in env block } - require.NoError(t, request.TaskDir.Build(false, nil)) + require.NoError(t, request.TaskDir.Build(fsisolation.None, nil, tg.Tasks[0].User)) response := new(interfaces.TaskPrestartResponse) response.Env = make(map[string]string) @@ -614,7 +615,7 @@ func TestTaskRunner_ConnectNativeHook_shareTLS_override(t *testing.T) { TaskDir: allocDir.NewTaskDir(tg.Tasks[0].Name), TaskEnv: taskEnv, // env block is configured w/ non-default tls configs } - require.NoError(t, request.TaskDir.Build(false, nil)) + require.NoError(t, request.TaskDir.Build(fsisolation.None, nil, tg.Tasks[0].User)) response := new(interfaces.TaskPrestartResponse) response.Env = make(map[string]string) diff --git a/client/allocrunner/taskrunner/dispatch_hook_test.go b/client/allocrunner/taskrunner/dispatch_hook_test.go index 1b55c27cb..efbe4e056 100644 --- a/client/allocrunner/taskrunner/dispatch_hook_test.go +++ b/client/allocrunner/taskrunner/dispatch_hook_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/stretchr/testify/require" ) @@ -35,10 +36,10 @@ func TestTaskRunner_DispatchHook_NoPayload(t *testing.T) { alloc := mock.BatchAlloc() task := alloc.Job.TaskGroups[0].Tasks[0] - allocDir := allocdir.NewAllocDir(logger, "nomadtest_nopayload", alloc.ID) + allocDir := allocdir.NewAllocDir(logger, "nomadtest_nopayload", "nomadtest_nopayload", alloc.ID) defer allocDir.Destroy() taskDir := allocDir.NewTaskDir(task.Name) - require.NoError(taskDir.Build(false, nil)) + require.NoError(taskDir.Build(fsisolation.None, nil, task.User)) h := newDispatchHook(alloc, logger) @@ -81,10 +82,10 @@ func TestTaskRunner_DispatchHook_Ok(t *testing.T) { File: "out", } - allocDir := allocdir.NewAllocDir(logger, "nomadtest_dispatchok", alloc.ID) + allocDir := allocdir.NewAllocDir(logger, "nomadtest_dispatchok", "nomadtest_dispatchok", alloc.ID) defer allocDir.Destroy() taskDir := allocDir.NewTaskDir(task.Name) - require.NoError(taskDir.Build(false, nil)) + require.NoError(taskDir.Build(fsisolation.None, nil, task.User)) h := newDispatchHook(alloc, logger) @@ -126,10 +127,10 @@ func TestTaskRunner_DispatchHook_Error(t *testing.T) { File: "out", } - allocDir := allocdir.NewAllocDir(logger, "nomadtest_dispatcherr", alloc.ID) + allocDir := allocdir.NewAllocDir(logger, "nomadtest_dispatcherr", "nomadtest_dispatcherr", alloc.ID) defer allocDir.Destroy() taskDir := allocDir.NewTaskDir(task.Name) - require.NoError(taskDir.Build(false, nil)) + require.NoError(taskDir.Build(fsisolation.None, nil, task.User)) h := newDispatchHook(alloc, logger) diff --git a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go index f6caa5082..0e6b32d29 100644 --- a/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) @@ -355,7 +356,7 @@ func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) { TaskDir: allocDir.NewTaskDir(sidecarTask.Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, sidecarTask.User)) // Insert service identity token in the secrets directory token := uuid.Generate() @@ -453,7 +454,7 @@ func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) { TaskDir: allocDir.NewTaskDir(sidecarTask.Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, sidecarTask.User)) resp := &interfaces.TaskPrestartResponse{} @@ -533,7 +534,7 @@ func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) var resp interfaces.TaskPrestartResponse @@ -582,7 +583,7 @@ func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) { Task: task, TaskDir: allocDir.NewTaskDir(task.Name), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, task.User)) resp := &interfaces.TaskPrestartResponse{} @@ -658,7 +659,7 @@ func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) { TaskDir: allocDir.NewTaskDir(sidecarTask.Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, sidecarTask.User)) resp := &interfaces.TaskPrestartResponse{} @@ -743,7 +744,7 @@ func TestTaskRunner_EnvoyBootstrapHook_retryTimeout(t *testing.T) { TaskDir: allocDir.NewTaskDir(sidecarTask.Name), TaskEnv: taskenv.NewEmptyTaskEnv(), } - require.NoError(t, req.TaskDir.Build(false, nil)) + require.NoError(t, req.TaskDir.Build(fsisolation.None, nil, sidecarTask.User)) var resp interfaces.TaskPrestartResponse diff --git a/client/allocrunner/taskrunner/envoy_version_hook_test.go b/client/allocrunner/taskrunner/envoy_version_hook_test.go index 26f45dc81..bd1d1d44a 100644 --- a/client/allocrunner/taskrunner/envoy_version_hook_test.go +++ b/client/allocrunner/taskrunner/envoy_version_hook_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/shoenig/test/must" ) @@ -253,7 +254,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_standard(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse @@ -296,7 +297,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_custom(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse @@ -341,7 +342,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_skip(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse @@ -380,7 +381,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_no_fallback(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse @@ -416,7 +417,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_error(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse @@ -455,7 +456,7 @@ func TestTaskRunner_EnvoyVersionHook_Prestart_restart(t *testing.T) { TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name), TaskEnv: taskEnvDefault, } - must.NoError(t, request.TaskDir.Build(false, nil)) + must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User)) // Prepare a response var response ifs.TaskPrestartResponse diff --git a/client/allocrunner/taskrunner/task_dir_hook.go b/client/allocrunner/taskrunner/task_dir_hook.go index 2f1933389..6ddc72f96 100644 --- a/client/allocrunner/taskrunner/task_dir_hook.go +++ b/client/allocrunner/taskrunner/task_dir_hook.go @@ -5,6 +5,7 @@ package taskrunner import ( "context" + "path/filepath" "strings" log "github.com/hashicorp/go-hclog" @@ -14,7 +15,7 @@ import ( cconfig "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" ) const ( @@ -65,7 +66,7 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart h.runner.EmitEvent(structs.NewTaskEvent(structs.TaskSetup).SetMessage(structs.TaskBuildingTaskDir)) // Build the task directory structure - err := h.runner.taskDir.Build(fsi == drivers.FSIsolationChroot, chroot) + err := h.runner.taskDir.Build(fsi, chroot, req.Task.User) if err != nil { return err } @@ -79,7 +80,7 @@ func (h *taskDirHook) Prestart(ctx context.Context, req *interfaces.TaskPrestart } // setEnvvars sets path and host env vars depending on the FS isolation used. -func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *cconfig.Config) { +func setEnvvars(envBuilder *taskenv.Builder, fsi fsisolation.Mode, taskDir *allocdir.TaskDir, conf *cconfig.Config) { envBuilder.SetClientTaskRoot(taskDir.Dir) envBuilder.SetClientSharedAllocDir(taskDir.SharedAllocDir) @@ -88,7 +89,12 @@ func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *a // Set driver-specific environment variables switch fsi { - case drivers.FSIsolationNone: + case fsisolation.Unveil: + // Use mount paths + envBuilder.SetAllocDir(filepath.Join(taskDir.MountsAllocDir, "alloc")) + envBuilder.SetTaskLocalDir(filepath.Join(taskDir.MountsTaskDir, "local")) + envBuilder.SetSecretsDir(filepath.Join(taskDir.SecretsDir, "secrets")) + case fsisolation.None: // Use host paths envBuilder.SetAllocDir(taskDir.SharedAllocDir) envBuilder.SetTaskLocalDir(taskDir.LocalDir) @@ -101,7 +107,7 @@ func setEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *a } // Set the host environment variables for non-image based drivers - if fsi != drivers.FSIsolationImage { + if fsi != fsisolation.Image { // COMPAT(1.0) using inclusive language, blacklist is kept for backward compatibility. filter := strings.Split(conf.ReadAlternativeDefault( []string{"env.denylist", "env.blacklist"}, diff --git a/client/allocrunner/taskrunner/task_runner_linux_test.go b/client/allocrunner/taskrunner/task_runner_linux_test.go index 3f8148ea8..7a115a221 100644 --- a/client/allocrunner/taskrunner/task_runner_linux_test.go +++ b/client/allocrunner/taskrunner/task_runner_linux_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/shoenig/test/must" ) @@ -48,7 +49,7 @@ func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) { // Remove private dir and write the Vault token to the secrets dir to // simulate an old task. - err = conf.TaskDir.Build(false, nil) + err = conf.TaskDir.Build(fsisolation.None, nil, task.User) must.NoError(t, err) err = syscall.Unmount(conf.TaskDir.PrivateDir, 0) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 53beecef9..e5fe60dc2 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -99,7 +99,7 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri } // Create the alloc dir + task dir - allocDir := allocdir.NewAllocDir(logger, clientConf.AllocDir, alloc.ID) + allocDir := allocdir.NewAllocDir(logger, clientConf.AllocDir, clientConf.AllocMountsDir, alloc.ID) if err := allocDir.Build(); err != nil { cleanup() t.Fatalf("error building alloc dir: %v", err) diff --git a/client/allocrunner/taskrunner/template/template_test.go b/client/allocrunner/taskrunner/template/template_test.go index e4d534f6b..237209b54 100644 --- a/client/allocrunner/taskrunner/template/template_test.go +++ b/client/allocrunner/taskrunner/template/template_test.go @@ -1831,7 +1831,7 @@ func TestTaskTemplateManager_Escapes(t *testing.T) { alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] logger := testlog.HCLogger(t) - allocDir := allocdir.NewAllocDir(logger, clientConf.AllocDir, alloc.ID) + allocDir := allocdir.NewAllocDir(logger, clientConf.AllocDir, clientConf.AllocMountsDir, alloc.ID) taskDir := allocDir.NewTaskDir(task.Name) containerEnv := func() *taskenv.Builder { diff --git a/client/allocwatcher/alloc_watcher.go b/client/allocwatcher/alloc_watcher.go index dcbe364ad..ee0f096ae 100644 --- a/client/allocwatcher/alloc_watcher.go +++ b/client/allocwatcher/alloc_watcher.go @@ -514,7 +514,7 @@ func (p *remotePrevAlloc) getNodeAddr(ctx context.Context, nodeID string) (strin // Destroy on the returned allocdir if no error occurs. func (p *remotePrevAlloc) migrateAllocDir(ctx context.Context, nodeAddr string) (*allocdir.AllocDir, error) { // Create the previous alloc dir - prevAllocDir := allocdir.NewAllocDir(p.logger, p.config.AllocDir, p.prevAllocID) + prevAllocDir := allocdir.NewAllocDir(p.logger, p.config.AllocDir, p.config.AllocMountsDir, p.prevAllocID) if err := prevAllocDir.Build(); err != nil { return nil, fmt.Errorf("error building alloc dir for previous alloc %q: %w", p.prevAllocID, err) } diff --git a/client/allocwatcher/alloc_watcher_test.go b/client/allocwatcher/alloc_watcher_test.go index 4085bd06b..d22dc8fef 100644 --- a/client/allocwatcher/alloc_watcher_test.go +++ b/client/allocwatcher/alloc_watcher_test.go @@ -44,7 +44,7 @@ func newFakeAllocRunner(t *testing.T, logger hclog.Logger) *fakeAllocRunner { return &fakeAllocRunner{ alloc: alloc, - AllocDir: allocdir.NewAllocDir(logger, path, alloc.ID), + AllocDir: allocdir.NewAllocDir(logger, path, path, alloc.ID), Broadcaster: cstructs.NewAllocBroadcaster(logger), } } diff --git a/client/client.go b/client/client.go index 8ac24b5ec..a82189abe 100644 --- a/client/client.go +++ b/client/client.go @@ -692,10 +692,17 @@ func (c *Client) init() error { c.stateDB = db - // Ensure the alloc dir exists if we have one + // Ensure the alloc mounts dir exists if we are configured with a custom path. + if conf.AllocMountsDir != "" { + if err := os.MkdirAll(conf.AllocMountsDir, 0o711); err != nil { + return fmt.Errorf("failed creating alloc mounts dir: %w", err) + } + } + + // Ensure the alloc dir exists if we are configured with a custom path. if conf.AllocDir != "" { - if err := os.MkdirAll(conf.AllocDir, 0711); err != nil { - return fmt.Errorf("failed creating alloc dir: %s", err) + if err := os.MkdirAll(conf.AllocDir, 0o711); err != nil { + return fmt.Errorf("failed creating alloc dir: %w", err) } } else { // Otherwise make a temp directory to use. @@ -710,12 +717,13 @@ func (c *Client) init() error { } // Change the permissions to have the execute bit - if err := os.Chmod(p, 0711); err != nil { + if err := os.Chmod(p, 0o711); err != nil { return fmt.Errorf("failed to change directory permissions for the AllocDir: %v", err) } conf = c.UpdateConfig(func(c *config.Config) { c.AllocDir = p + c.AllocMountsDir = p }) } diff --git a/client/client_test.go b/client/client_test.go index d001dde32..929140042 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -5,6 +5,7 @@ package client import ( "fmt" + "io/fs" "net" "os" "path/filepath" @@ -62,6 +63,34 @@ func TestClient_StartStop(t *testing.T) { } } +func TestClient_alloc_dirs(t *testing.T) { + ci.Parallel(t) + + parent := t.TempDir() + allocs := filepath.Join(parent, "allocs") + mounts := filepath.Join(parent, "mounts") + + client, cleanup := TestClient(t, func(c *config.Config) { + c.AllocDir = allocs + c.AllocMountsDir = mounts + }) + defer cleanup() + + t.Cleanup(func() { + test.NoError(t, client.Shutdown()) + }) + + // assert existence and permissions of alloc-dir + fi, err := os.Stat(allocs) + must.NoError(t, err) + must.Eq(t, 0o711|fs.ModeDir, fi.Mode()) + + // assert existence and permissions of alloc-mounts-dir + fi, err = os.Stat(allocs) + must.NoError(t, err) + must.Eq(t, 0o711|fs.ModeDir, fi.Mode()) +} + // Certain labels for metrics are dependant on client initial setup. This tests // that the client has properly initialized before we assign values to labels func TestClient_BaseLabels(t *testing.T) { diff --git a/client/config/config.go b/client/config/config.go index 14c2a0299..2b32e62d1 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -97,8 +97,16 @@ type Config struct { StateDir string // AllocDir is where we store data for allocations + // + // In a production environment this should be owned by root with file + // mode 0o700. AllocDir string + // AllocMountsDir is where we bind mount paths from AllocDir for tasks making + // use of the unveil file isolation mode. In a production environment this + // should be owned by root with file mode 0o755. + AllocMountsDir string + // Logger provides a logger to the client Logger log.InterceptLogger diff --git a/client/fs_endpoint_test.go b/client/fs_endpoint_test.go index 5ebe1930e..f6494f49b 100644 --- a/client/fs_endpoint_test.go +++ b/client/fs_endpoint_test.go @@ -43,7 +43,7 @@ func tempAllocDir(t testing.TB) *allocdir.AllocDir { require.NoError(t, os.Chmod(dir, 0o777)) - return allocdir.NewAllocDir(testlog.HCLogger(t), dir, "test_allocid") + return allocdir.NewAllocDir(testlog.HCLogger(t), dir, dir, "test_allocid") } type nopWriteCloser struct { diff --git a/command/agent/agent.go b/command/agent/agent.go index f8b39990b..2a64b3c97 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -721,6 +721,7 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { if agentConfig.DataDir != "" { conf.StateDir = filepath.Join(agentConfig.DataDir, "client") conf.AllocDir = filepath.Join(agentConfig.DataDir, "alloc") + conf.AllocMountsDir = filepath.Join(agentConfig.DataDir, "mounts") } if agentConfig.Client.StateDir != "" { conf.StateDir = agentConfig.Client.StateDir @@ -728,6 +729,9 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { if agentConfig.Client.AllocDir != "" { conf.AllocDir = agentConfig.Client.AllocDir } + if agentConfig.Client.AllocMountsDir != "" { + conf.AllocMountsDir = agentConfig.Client.AllocMountsDir + } if agentConfig.Client.NetworkInterface != "" { conf.NetworkInterface = agentConfig.Client.NetworkInterface } diff --git a/command/agent/command.go b/command/agent/command.go index 8cc58f2c3..65011119b 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -110,6 +110,7 @@ func (c *Command) readConfig() *Config { // Client-only options flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "") flags.StringVar(&cmdConfig.Client.AllocDir, "alloc-dir", "", "") + flags.StringVar(&cmdConfig.Client.AllocMountsDir, "alloc-mounts-dir", "", "") flags.StringVar(&cmdConfig.Client.NodeClass, "node-class", "", "") flags.StringVar(&cmdConfig.Client.NodePool, "node-pool", "", "") flags.StringVar(&servers, "servers", "", "") @@ -377,10 +378,11 @@ func (c *Command) IsValidConfig(config, cmdConfig *Config) bool { // Verify the paths are absolute. dirs := map[string]string{ - "data-dir": config.DataDir, - "plugin-dir": config.PluginDir, - "alloc-dir": config.Client.AllocDir, - "state-dir": config.Client.StateDir, + "data-dir": config.DataDir, + "plugin-dir": config.PluginDir, + "alloc-dir": config.Client.AllocDir, + "alloc-mounts-dir": config.Client.AllocMountsDir, + "state-dir": config.Client.StateDir, } for k, dir := range dirs { if dir == "" { @@ -488,8 +490,12 @@ func (c *Command) IsValidConfig(config, cmdConfig *Config) bool { // The config is valid if the top-level data-dir is set or if both // alloc-dir and state-dir are set. if config.Client.Enabled && config.DataDir == "" { - if config.Client.AllocDir == "" || config.Client.StateDir == "" || config.PluginDir == "" { - c.Ui.Error("Must specify the state, alloc dir, and plugin dir if data-dir is omitted.") + missing := config.Client.AllocDir == "" || + config.Client.AllocMountsDir == "" || + config.Client.StateDir == "" || + config.PluginDir == "" + if missing { + c.Ui.Error("Must specify the state, alloc-dir, alloc-mounts-dir and plugin-dir if data-dir is omitted.") return false } } diff --git a/command/agent/command_test.go b/command/agent/command_test.go index ce6cfed3e..014f9a5d4 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -59,7 +59,7 @@ func TestCommand_Args(t *testing.T) { }, { []string{"-client", "-alloc-dir="}, - "Must specify the state, alloc dir, and plugin dir if data-dir is omitted.", + "Must specify the state, alloc-dir, alloc-mounts-dir and plugin-dir if data-dir is omitted.", }, { []string{"-client", "-data-dir=" + tmpDir, "-meta=invalid..key=inaccessible-value"}, diff --git a/command/agent/config.go b/command/agent/config.go index 346a794c9..c0b9b9187 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -223,6 +223,9 @@ type ClientConfig struct { // AllocDir is the directory for storing allocation data AllocDir string `hcl:"alloc_dir"` + // AllocMountsDir is the directory for storing mounts into allocation data + AllocMountsDir string `hcl:"alloc_mounts_dir"` + // Servers is a list of known server addresses. These are as "host:port" Servers []string `hcl:"servers"` @@ -2211,6 +2214,9 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig { if b.AllocDir != "" { result.AllocDir = b.AllocDir } + if b.AllocMountsDir != "" { + result.AllocMountsDir = b.AllocMountsDir + } if b.NodeClass != "" { result.NodeClass = b.NodeClass } diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 6231125b9..6a49782bb 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -44,11 +44,12 @@ var basicConfig = &Config{ Serf: "127.0.0.4", }, Client: &ClientConfig{ - Enabled: true, - StateDir: "/tmp/client-state", - AllocDir: "/tmp/alloc", - Servers: []string{"a.b.c:80", "127.0.0.1:1234"}, - NodeClass: "linux-medium-64bit", + Enabled: true, + StateDir: "/tmp/client-state", + AllocDir: "/tmp/alloc", + AllocMountsDir: "/tmp/mounts", + Servers: []string{"a.b.c:80", "127.0.0.1:1234"}, + NodeClass: "linux-medium-64bit", ServerJoin: &ServerJoin{ RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, RetryInterval: time.Duration(15) * time.Second, diff --git a/command/agent/consul/int_test.go b/command/agent/consul/int_test.go index df6266af8..4ec6e34d7 100644 --- a/command/agent/consul/int_test.go +++ b/command/agent/consul/int_test.go @@ -131,7 +131,7 @@ func TestConsul_Integration(t *testing.T) { logger := testlog.HCLogger(t) logUpdate := &mockUpdater{logger} - allocDir := allocdir.NewAllocDir(logger, conf.AllocDir, alloc.ID) + allocDir := allocdir.NewAllocDir(logger, conf.AllocDir, conf.AllocMountsDir, alloc.ID) if err := allocDir.Build(); err != nil { t.Fatalf("error building alloc dir: %v", err) } diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index d3a27c64b..e60fd04ba 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -42,11 +42,12 @@ advertise { } client { - enabled = true - state_dir = "/tmp/client-state" - alloc_dir = "/tmp/alloc" - servers = ["a.b.c:80", "127.0.0.1:1234"] - node_class = "linux-medium-64bit" + enabled = true + state_dir = "/tmp/client-state" + alloc_dir = "/tmp/alloc" + alloc_mounts_dir = "/tmp/mounts" + servers = ["a.b.c:80", "127.0.0.1:1234"] + node_class = "linux-medium-64bit" meta { foo = "bar" diff --git a/command/agent/testdata/basic.json b/command/agent/testdata/basic.json index 14f07eea0..7b65b08bc 100644 --- a/command/agent/testdata/basic.json +++ b/command/agent/testdata/basic.json @@ -73,6 +73,7 @@ "client": [ { "alloc_dir": "/tmp/alloc", + "alloc_mounts_dir": "/tmp/mounts", "bridge_network_name": "custom_bridge_name", "bridge_network_subnet": "custom_bridge_subnet", "chroot_env": [ diff --git a/drivers/docker/config.go b/drivers/docker/config.go index 699eeffbc..f3f9fb69f 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/shared/hclspec" ) @@ -414,7 +415,7 @@ var ( driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, - FSIsolation: drivers.FSIsolationImage, + FSIsolation: fsisolation.Image, NetIsolationModes: []drivers.NetIsolationMode{ drivers.NetIsolationModeHost, drivers.NetIsolationModeGroup, diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 02e406472..072b430a7 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/nomad/helper/pointer" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/drivers/utils" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" @@ -100,7 +101,7 @@ var ( driverCapabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, - FSIsolation: drivers.FSIsolationChroot, + FSIsolation: fsisolation.Chroot, NetIsolationModes: []drivers.NetIsolationMode{ drivers.NetIsolationModeHost, drivers.NetIsolationModeGroup, diff --git a/drivers/java/driver.go b/drivers/java/driver.go index acaeae651..829f59f65 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -22,6 +22,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/drivers/utils" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" @@ -104,7 +105,7 @@ var ( driverCapabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, - FSIsolation: drivers.FSIsolationNone, + FSIsolation: fsisolation.None, NetIsolationModes: []drivers.NetIsolationMode{ drivers.NetIsolationModeHost, drivers.NetIsolationModeGroup, @@ -117,7 +118,7 @@ var ( func init() { if runtime.GOOS == "linux" { - driverCapabilities.FSIsolation = drivers.FSIsolationChroot + driverCapabilities.FSIsolation = fsisolation.Chroot driverCapabilities.MountConfigs = drivers.MountConfigSupportAll } } @@ -455,7 +456,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", - FSIsolation: driverCapabilities.FSIsolation == drivers.FSIsolationChroot, + FSIsolation: driverCapabilities.FSIsolation == fsisolation.Chroot, Compute: d.nomadConfig.Topology.Compute(), } diff --git a/drivers/mock/driver.go b/drivers/mock/driver.go index 379df1115..ccd8a5ea6 100644 --- a/drivers/mock/driver.go +++ b/drivers/mock/driver.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" ) @@ -63,7 +64,7 @@ var ( configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ "fs_isolation": hclspec.NewDefault( hclspec.NewAttr("fs_isolation", "string", false), - hclspec.NewLiteral(fmt.Sprintf("%q", drivers.FSIsolationNone)), + hclspec.NewLiteral(fmt.Sprintf("%q", fsisolation.None)), ), "shutdown_periodic_after": hclspec.NewDefault( hclspec.NewAttr("shutdown_periodic_after", "bool", false), diff --git a/drivers/mock/driver_test.go b/drivers/mock/driver_test.go index cf999b781..3c7f6fc1e 100644 --- a/drivers/mock/driver_test.go +++ b/drivers/mock/driver_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" basePlug "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" ) @@ -132,13 +133,13 @@ func mkTestAllocDir(t *testing.T, h *dtestutil.DriverHarness, logger hclog.Logge dir, err := os.MkdirTemp("", "nomad_driver_harness-") must.NoError(t, err) - allocDir := allocdir.NewAllocDir(logger, dir, tc.AllocID) + allocDir := allocdir.NewAllocDir(logger, dir, dir, tc.AllocID) must.NoError(t, allocDir.Build()) tc.AllocDir = allocDir.AllocDir taskDir := allocDir.NewTaskDir(tc.Name) - must.NoError(t, taskDir.Build(false, ci.TinyChroot)) + must.NoError(t, taskDir.Build(fsisolation.None, ci.TinyChroot, tc.User)) task := &structs.Task{ Name: tc.Name, diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 4763f2b44..57c99522a 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" ) @@ -102,7 +103,7 @@ var ( capabilities = &drivers.Capabilities{ SendSignals: false, Exec: false, - FSIsolation: drivers.FSIsolationImage, + FSIsolation: fsisolation.Image, NetIsolationModes: []drivers.NetIsolationMode{ drivers.NetIsolationModeHost, drivers.NetIsolationModeGroup, diff --git a/drivers/rawexec/driver.go b/drivers/rawexec/driver.go index 8bcde8b8a..b2efc83eb 100644 --- a/drivers/rawexec/driver.go +++ b/drivers/rawexec/driver.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" ) @@ -91,7 +92,7 @@ var ( capabilities = &drivers.Capabilities{ SendSignals: true, Exec: true, - FSIsolation: drivers.FSIsolationNone, + FSIsolation: fsisolation.None, NetIsolationModes: []drivers.NetIsolationMode{ drivers.NetIsolationModeHost, drivers.NetIsolationModeGroup, diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index cd7661032..70de4b210 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" tu "github.com/hashicorp/nomad/testutil" lconfigs "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/devices" @@ -70,11 +71,11 @@ func testExecutorCommandWithChroot(t *testing.T) *testExecCmd { task := alloc.Job.TaskGroups[0].Tasks[0] taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build() - allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), os.TempDir(), alloc.ID) + allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), os.TempDir(), os.TempDir(), alloc.ID) if err := allocDir.Build(); err != nil { t.Fatalf("AllocDir.Build() failed: %v", err) } - if err := allocDir.NewTaskDir(task.Name).Build(true, chrootEnv); err != nil { + if err := allocDir.NewTaskDir(task.Name).Build(fsisolation.Chroot, chrootEnv, task.User); err != nil { allocDir.Destroy() t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err) } diff --git a/drivers/shared/executor/executor_test.go b/drivers/shared/executor/executor_test.go index 56f3febbd..fa18b0abb 100644 --- a/drivers/shared/executor/executor_test.go +++ b/drivers/shared/executor/executor_test.go @@ -30,6 +30,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" tu "github.com/hashicorp/nomad/testutil" ps "github.com/mitchellh/go-ps" "github.com/shoenig/test/must" @@ -75,11 +76,11 @@ func testExecutorCommand(t *testing.T) *testExecCmd { task := alloc.Job.TaskGroups[0].Tasks[0] taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, "global").Build() - allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), t.TempDir(), alloc.ID) + allocDir := allocdir.NewAllocDir(testlog.HCLogger(t), t.TempDir(), t.TempDir(), alloc.ID) if err := allocDir.Build(); err != nil { t.Fatalf("AllocDir.Build() failed: %v", err) } - if err := allocDir.NewTaskDir(task.Name).Build(false, nil); err != nil { + if err := allocDir.NewTaskDir(task.Name).Build(fsisolation.None, nil, task.User); err != nil { allocDir.Destroy() t.Fatalf("allocDir.NewTaskDir(%q) failed: %v", task.Name, err) } @@ -647,9 +648,9 @@ func TestExecutor_Start_NonExecutableBinaries(t *testing.T) { // need to configure path in chroot with that file if using isolation executor if _, ok := executor.(*UniversalExecutor); !ok { taskName := filepath.Base(testExecCmd.command.TaskDir) - err := allocDir.NewTaskDir(taskName).Build(true, map[string]string{ + err := allocDir.NewTaskDir(taskName).Build(fsisolation.Chroot, map[string]string{ tmpDir: tmpDir, - }) + }, "nobody") require.NoError(err) } diff --git a/plugins/drivers/client.go b/plugins/drivers/client.go index c5447dc7c..cbb27fca1 100644 --- a/plugins/drivers/client.go +++ b/plugins/drivers/client.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/nomad/helper/pluginutils/grpcutils" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/drivers/proto" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" @@ -66,13 +67,15 @@ func (d *driverPluginClient) Capabilities() (*Capabilities, error) { switch resp.Capabilities.FsIsolation { case proto.DriverCapabilities_NONE: - caps.FSIsolation = FSIsolationNone + caps.FSIsolation = fsisolation.None case proto.DriverCapabilities_CHROOT: - caps.FSIsolation = FSIsolationChroot + caps.FSIsolation = fsisolation.Chroot case proto.DriverCapabilities_IMAGE: - caps.FSIsolation = FSIsolationImage + caps.FSIsolation = fsisolation.Image + case proto.DriverCapabilities_UNVEIL: + caps.FSIsolation = fsisolation.Unveil default: - caps.FSIsolation = FSIsolationNone + caps.FSIsolation = fsisolation.None } caps.MountConfigs = MountConfigSupport(resp.Capabilities.MountConfigs) diff --git a/plugins/drivers/driver.go b/plugins/drivers/driver.go index cf2bdd695..cbbf3a4cb 100644 --- a/plugins/drivers/driver.go +++ b/plugins/drivers/driver.go @@ -17,6 +17,7 @@ import ( cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/drivers/proto" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" @@ -132,20 +133,18 @@ type Fingerprint struct { Err error } -// FSIsolation is an enumeration to describe what kind of filesystem isolation -// a driver supports. -type FSIsolation string +// Deprecated: use fsisolation.Mode instead. +type FSIsolation = fsisolation.Mode var ( - // FSIsolationNone means no isolation. The host filesystem is used. - FSIsolationNone = FSIsolation("none") + // Deprecated: use fsisolation.None instead. + FSIsolationNone = fsisolation.None - // FSIsolationChroot means the driver will use a chroot on the host - // filesystem. - FSIsolationChroot = FSIsolation("chroot") + // Deprecated: use fsisolation.Chroot instead. + FSIsolationChroot = fsisolation.Chroot - // FSIsolationImage means the driver uses an image. - FSIsolationImage = FSIsolation("image") + // Deprecated: use fsisolation.Image instead. + FSIsolationImage = fsisolation.Image ) type Capabilities struct { @@ -157,7 +156,7 @@ type Capabilities struct { Exec bool //FSIsolation indicates what kind of filesystem isolation the driver supports. - FSIsolation FSIsolation + FSIsolation fsisolation.Mode //NetIsolationModes lists the set of isolation modes supported by the driver NetIsolationModes []NetIsolationMode diff --git a/plugins/drivers/fsisolation/isolation.go b/plugins/drivers/fsisolation/isolation.go new file mode 100644 index 000000000..417e65759 --- /dev/null +++ b/plugins/drivers/fsisolation/isolation.go @@ -0,0 +1,25 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package fsisolation + +// Mode is an enum to describe what kind of filesystem isolation a +// driver supports. +type Mode string + +const ( + // IsolationNone means no isolation. The host filesystem is used. + None = Mode("none") + + // IsolationChroot means the driver will use a chroot on the host + // filesystem. + Chroot = Mode("chroot") + + // IsolationImage means the driver uses an image. + Image = Mode("image") + + // IsolationUnveil means the driver and client will work together using + // unveil() syscall semantics (i.e. landlock on linux) isolate the host + // filesytem from workloads. + Unveil = Mode("unveil") +) diff --git a/plugins/drivers/proto/driver.pb.go b/plugins/drivers/proto/driver.pb.go index dd6e68720..c7457d515 100644 --- a/plugins/drivers/proto/driver.pb.go +++ b/plugins/drivers/proto/driver.pb.go @@ -118,18 +118,21 @@ const ( DriverCapabilities_NONE DriverCapabilities_FSIsolation = 0 DriverCapabilities_CHROOT DriverCapabilities_FSIsolation = 1 DriverCapabilities_IMAGE DriverCapabilities_FSIsolation = 2 + DriverCapabilities_UNVEIL DriverCapabilities_FSIsolation = 3 ) var DriverCapabilities_FSIsolation_name = map[int32]string{ 0: "NONE", 1: "CHROOT", 2: "IMAGE", + 3: "UNVEIL", } var DriverCapabilities_FSIsolation_value = map[string]int32{ "NONE": 0, "CHROOT": 1, "IMAGE": 2, + "UNVEIL": 3, } func (x DriverCapabilities_FSIsolation) String() string { @@ -3755,252 +3758,252 @@ func init() { } var fileDescriptor_4a8f45747846a74d = []byte{ - // 3910 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0xcd, 0x73, 0x1b, 0xc9, - 0x75, 0xe7, 0xe0, 0x8b, 0xc0, 0x03, 0x09, 0x0e, 0x5b, 0xa4, 0x04, 0x61, 0x9d, 0xac, 0x3c, 0xae, - 0x4d, 0x31, 0xf6, 0x2e, 0xb4, 0xa6, 0x93, 0xd5, 0x6a, 0xad, 0xb5, 0x16, 0x0b, 0x42, 0x22, 0x24, - 0x12, 0x64, 0x1a, 0x60, 0xc9, 0x8a, 0x92, 0x9d, 0x0c, 0x67, 0x5a, 0xe0, 0x88, 0x98, 0x8f, 0x9d, - 0x1e, 0x50, 0xa4, 0x53, 0xa9, 0xa4, 0x9c, 0xaa, 0x94, 0x53, 0x95, 0x54, 0x72, 0xd9, 0xf8, 0x92, - 0x93, 0xab, 0x72, 0x4a, 0xf9, 0x9e, 0x72, 0xca, 0xa7, 0x1c, 0xf2, 0x4f, 0xe4, 0x92, 0x5b, 0xae, - 0xf9, 0x0b, 0x92, 0xea, 0x8f, 0x19, 0xcc, 0x00, 0xd0, 0x6a, 0x00, 0xea, 0x84, 0x79, 0xaf, 0xbb, - 0x7f, 0xfd, 0xf0, 0xde, 0xeb, 0xd7, 0xaf, 0xbb, 0x1f, 0x68, 0xfe, 0x68, 0x3c, 0xb4, 0x5d, 0x7a, - 0xd7, 0x0a, 0xec, 0x0b, 0x12, 0xd0, 0xbb, 0x7e, 0xe0, 0x85, 0x9e, 0xa4, 0x9a, 0x9c, 0x40, 0x1f, - 0x9c, 0x19, 0xf4, 0xcc, 0x36, 0xbd, 0xc0, 0x6f, 0xba, 0x9e, 0x63, 0x58, 0x4d, 0x39, 0xa6, 0x29, - 0xc7, 0x88, 0x6e, 0x8d, 0xdf, 0x1d, 0x7a, 0xde, 0x70, 0x44, 0x04, 0xc2, 0xe9, 0xf8, 0xe5, 0x5d, - 0x6b, 0x1c, 0x18, 0xa1, 0xed, 0xb9, 0xb2, 0xfd, 0xfd, 0xe9, 0xf6, 0xd0, 0x76, 0x08, 0x0d, 0x0d, - 0xc7, 0x97, 0x1d, 0x3e, 0x88, 0x64, 0xa1, 0x67, 0x46, 0x40, 0xac, 0xbb, 0x67, 0xe6, 0x88, 0xfa, - 0xc4, 0x64, 0xbf, 0x3a, 0xfb, 0x90, 0xdd, 0x3e, 0x9c, 0xea, 0x46, 0xc3, 0x60, 0x6c, 0x86, 0x91, - 0xe4, 0x46, 0x18, 0x06, 0xf6, 0xe9, 0x38, 0x24, 0xa2, 0xb7, 0x76, 0x1b, 0x6e, 0x0d, 0x0c, 0x7a, - 0xde, 0xf6, 0xdc, 0x97, 0xf6, 0xb0, 0x6f, 0x9e, 0x11, 0xc7, 0xc0, 0xe4, 0xeb, 0x31, 0xa1, 0xa1, - 0xf6, 0x27, 0x50, 0x9f, 0x6d, 0xa2, 0xbe, 0xe7, 0x52, 0x82, 0xbe, 0x80, 0x02, 0x9b, 0xb2, 0xae, - 0xdc, 0x51, 0x76, 0xaa, 0xbb, 0x1f, 0x36, 0xdf, 0xa4, 0x02, 0x21, 0x43, 0x53, 0x8a, 0xda, 0xec, - 0xfb, 0xc4, 0xc4, 0x7c, 0xa4, 0xb6, 0x0d, 0x37, 0xda, 0x86, 0x6f, 0x9c, 0xda, 0x23, 0x3b, 0xb4, - 0x09, 0x8d, 0x26, 0x1d, 0xc3, 0x56, 0x9a, 0x2d, 0x27, 0xfc, 0x53, 0x58, 0x33, 0x13, 0x7c, 0x39, - 0xf1, 0xfd, 0x66, 0x26, 0xdd, 0x37, 0xf7, 0x38, 0x95, 0x02, 0x4e, 0xc1, 0x69, 0x5b, 0x80, 0x1e, - 0xd9, 0xee, 0x90, 0x04, 0x7e, 0x60, 0xbb, 0x61, 0x24, 0xcc, 0x6f, 0xf3, 0x70, 0x23, 0xc5, 0x96, - 0xc2, 0xbc, 0x02, 0x88, 0xf5, 0xc8, 0x44, 0xc9, 0xef, 0x54, 0x77, 0x9f, 0x64, 0x14, 0x65, 0x0e, - 0x5e, 0xb3, 0x15, 0x83, 0x75, 0xdc, 0x30, 0xb8, 0xc2, 0x09, 0x74, 0xf4, 0x15, 0x94, 0xce, 0x88, - 0x31, 0x0a, 0xcf, 0xea, 0xb9, 0x3b, 0xca, 0x4e, 0x6d, 0xf7, 0xd1, 0x35, 0xe6, 0xd9, 0xe7, 0x40, - 0xfd, 0xd0, 0x08, 0x09, 0x96, 0xa8, 0xe8, 0x23, 0x40, 0xe2, 0x4b, 0xb7, 0x08, 0x35, 0x03, 0xdb, - 0x67, 0x2e, 0x59, 0xcf, 0xdf, 0x51, 0x76, 0x2a, 0x78, 0x53, 0xb4, 0xec, 0x4d, 0x1a, 0x1a, 0x3e, - 0x6c, 0x4c, 0x49, 0x8b, 0x54, 0xc8, 0x9f, 0x93, 0x2b, 0x6e, 0x91, 0x0a, 0x66, 0x9f, 0xe8, 0x31, - 0x14, 0x2f, 0x8c, 0xd1, 0x98, 0x70, 0x91, 0xab, 0xbb, 0x3f, 0x7c, 0x9b, 0x7b, 0x48, 0x17, 0x9d, - 0xe8, 0x01, 0x8b, 0xf1, 0x9f, 0xe5, 0x3e, 0x55, 0xb4, 0xfb, 0x50, 0x4d, 0xc8, 0x8d, 0x6a, 0x00, - 0x27, 0xbd, 0xbd, 0xce, 0xa0, 0xd3, 0x1e, 0x74, 0xf6, 0xd4, 0x15, 0xb4, 0x0e, 0x95, 0x93, 0xde, - 0x7e, 0xa7, 0x75, 0x30, 0xd8, 0x7f, 0xae, 0x2a, 0xa8, 0x0a, 0xab, 0x11, 0x91, 0xd3, 0x2e, 0x01, - 0x61, 0x62, 0x7a, 0x17, 0x24, 0x60, 0x8e, 0x2c, 0xad, 0x8a, 0x6e, 0xc1, 0x6a, 0x68, 0xd0, 0x73, - 0xdd, 0xb6, 0xa4, 0xcc, 0x25, 0x46, 0x76, 0x2d, 0xd4, 0x85, 0xd2, 0x99, 0xe1, 0x5a, 0xa3, 0xb7, - 0xcb, 0x9d, 0x56, 0x35, 0x03, 0xdf, 0xe7, 0x03, 0xb1, 0x04, 0x60, 0xde, 0x9d, 0x9a, 0x59, 0x18, - 0x40, 0x7b, 0x0e, 0x6a, 0x3f, 0x34, 0x82, 0x30, 0x29, 0x4e, 0x07, 0x0a, 0x6c, 0x7e, 0xe9, 0xd1, - 0x8b, 0xcc, 0x29, 0x56, 0x26, 0xe6, 0xc3, 0xb5, 0xff, 0xcd, 0xc1, 0x66, 0x02, 0x5b, 0x7a, 0xea, - 0x33, 0x28, 0x05, 0x84, 0x8e, 0x47, 0x21, 0x87, 0xaf, 0xed, 0x3e, 0xcc, 0x08, 0x3f, 0x83, 0xd4, - 0xc4, 0x1c, 0x06, 0x4b, 0x38, 0xb4, 0x03, 0xaa, 0x18, 0xa1, 0x93, 0x20, 0xf0, 0x02, 0xdd, 0xa1, - 0x43, 0xae, 0xb5, 0x0a, 0xae, 0x09, 0x7e, 0x87, 0xb1, 0x0f, 0xe9, 0x30, 0xa1, 0xd5, 0xfc, 0x35, - 0xb5, 0x8a, 0x0c, 0x50, 0x5d, 0x12, 0xbe, 0xf6, 0x82, 0x73, 0x9d, 0xa9, 0x36, 0xb0, 0x2d, 0x52, - 0x2f, 0x70, 0xd0, 0x4f, 0x32, 0x82, 0xf6, 0xc4, 0xf0, 0x23, 0x39, 0x1a, 0x6f, 0xb8, 0x69, 0x86, - 0xf6, 0x03, 0x28, 0x89, 0x7f, 0xca, 0x3c, 0xa9, 0x7f, 0xd2, 0x6e, 0x77, 0xfa, 0x7d, 0x75, 0x05, - 0x55, 0xa0, 0x88, 0x3b, 0x03, 0xcc, 0x3c, 0xac, 0x02, 0xc5, 0x47, 0xad, 0x41, 0xeb, 0x40, 0xcd, - 0x69, 0xdf, 0x87, 0x8d, 0x67, 0x86, 0x1d, 0x66, 0x71, 0x2e, 0xcd, 0x03, 0x75, 0xd2, 0x57, 0x5a, - 0xa7, 0x9b, 0xb2, 0x4e, 0x76, 0xd5, 0x74, 0x2e, 0xed, 0x70, 0xca, 0x1e, 0x2a, 0xe4, 0x49, 0x10, - 0x48, 0x13, 0xb0, 0x4f, 0xed, 0x35, 0x6c, 0xf4, 0x43, 0xcf, 0xcf, 0xe4, 0xf9, 0x3f, 0x82, 0x55, - 0xb6, 0xdb, 0x78, 0xe3, 0x50, 0xba, 0xfe, 0xed, 0xa6, 0xd8, 0x8d, 0x9a, 0xd1, 0x6e, 0xd4, 0xdc, - 0x93, 0xbb, 0x15, 0x8e, 0x7a, 0xa2, 0x9b, 0x50, 0xa2, 0xf6, 0xd0, 0x35, 0x46, 0x32, 0x5a, 0x48, - 0x4a, 0x43, 0xcc, 0xc9, 0xa3, 0x89, 0xa5, 0xe3, 0xb7, 0x01, 0xed, 0x11, 0x1a, 0x06, 0xde, 0x55, - 0x26, 0x79, 0xb6, 0xa0, 0xf8, 0xd2, 0x0b, 0x4c, 0xb1, 0x10, 0xcb, 0x58, 0x10, 0x6c, 0x51, 0xa5, - 0x40, 0x24, 0xf6, 0x47, 0x80, 0xba, 0x2e, 0xdb, 0x53, 0xb2, 0x19, 0xe2, 0x1f, 0x73, 0x70, 0x23, - 0xd5, 0x5f, 0x1a, 0x63, 0xf9, 0x75, 0xc8, 0x02, 0xd3, 0x98, 0x8a, 0x75, 0x88, 0x8e, 0xa0, 0x24, - 0x7a, 0x48, 0x4d, 0xde, 0x5b, 0x00, 0x48, 0x6c, 0x53, 0x12, 0x4e, 0xc2, 0xcc, 0x75, 0xfa, 0xfc, - 0xbb, 0x75, 0xfa, 0xd7, 0xa0, 0x46, 0xff, 0x83, 0xbe, 0xd5, 0x36, 0x4f, 0xe0, 0x86, 0xe9, 0x8d, - 0x46, 0xc4, 0x64, 0xde, 0xa0, 0xdb, 0x6e, 0x48, 0x82, 0x0b, 0x63, 0xf4, 0x76, 0xbf, 0x41, 0x93, - 0x51, 0x5d, 0x39, 0x48, 0x7b, 0x01, 0x9b, 0x89, 0x89, 0xa5, 0x21, 0x1e, 0x41, 0x91, 0x32, 0x86, - 0xb4, 0xc4, 0xc7, 0x0b, 0x5a, 0x82, 0x62, 0x31, 0x5c, 0xbb, 0x21, 0xc0, 0x3b, 0x17, 0xc4, 0x8d, - 0xff, 0x96, 0xb6, 0x07, 0x9b, 0x7d, 0xee, 0xa6, 0x99, 0xfc, 0x70, 0xe2, 0xe2, 0xb9, 0x94, 0x8b, - 0x6f, 0x01, 0x4a, 0xa2, 0x48, 0x47, 0xbc, 0x82, 0x8d, 0xce, 0x25, 0x31, 0x33, 0x21, 0xd7, 0x61, - 0xd5, 0xf4, 0x1c, 0xc7, 0x70, 0xad, 0x7a, 0xee, 0x4e, 0x7e, 0xa7, 0x82, 0x23, 0x32, 0xb9, 0x16, - 0xf3, 0x59, 0xd7, 0xa2, 0xf6, 0xf7, 0x0a, 0xa8, 0x93, 0xb9, 0xa5, 0x22, 0x99, 0xf4, 0xa1, 0xc5, - 0x80, 0xd8, 0xdc, 0x6b, 0x58, 0x52, 0x92, 0x1f, 0x85, 0x0b, 0xc1, 0x27, 0x41, 0x90, 0x08, 0x47, - 0xf9, 0x6b, 0x86, 0x23, 0x6d, 0x1f, 0xbe, 0x13, 0x89, 0xd3, 0x0f, 0x03, 0x62, 0x38, 0xb6, 0x3b, - 0xec, 0x1e, 0x1d, 0xf9, 0x44, 0x08, 0x8e, 0x10, 0x14, 0x2c, 0x23, 0x34, 0xa4, 0x60, 0xfc, 0x9b, - 0x2d, 0x7a, 0x73, 0xe4, 0xd1, 0x78, 0xd1, 0x73, 0x42, 0xfb, 0xcf, 0x3c, 0xd4, 0x67, 0xa0, 0x22, - 0xf5, 0xbe, 0x80, 0x22, 0x25, 0xe1, 0xd8, 0x97, 0xae, 0xd2, 0xc9, 0x2c, 0xf0, 0x7c, 0xbc, 0x66, - 0x9f, 0x81, 0x61, 0x81, 0x89, 0x86, 0x50, 0x0e, 0xc3, 0x2b, 0x9d, 0xda, 0x3f, 0x8b, 0x12, 0x82, - 0x83, 0xeb, 0xe2, 0x0f, 0x48, 0xe0, 0xd8, 0xae, 0x31, 0xea, 0xdb, 0x3f, 0x23, 0x78, 0x35, 0x0c, - 0xaf, 0xd8, 0x07, 0x7a, 0xce, 0x1c, 0xde, 0xb2, 0x5d, 0xa9, 0xf6, 0xf6, 0xb2, 0xb3, 0x24, 0x14, - 0x8c, 0x05, 0x62, 0xe3, 0x00, 0x8a, 0xfc, 0x3f, 0x2d, 0xe3, 0x88, 0x2a, 0xe4, 0xc3, 0xf0, 0x8a, - 0x0b, 0x55, 0xc6, 0xec, 0xb3, 0xf1, 0x00, 0xd6, 0x92, 0xff, 0x80, 0x39, 0xd2, 0x19, 0xb1, 0x87, - 0x67, 0xc2, 0xc1, 0x8a, 0x58, 0x52, 0xcc, 0x92, 0xaf, 0x6d, 0x4b, 0xa6, 0xac, 0x45, 0x2c, 0x08, - 0xed, 0xdf, 0x72, 0x70, 0x7b, 0x8e, 0x66, 0xa4, 0xb3, 0xbe, 0x48, 0x39, 0xeb, 0x3b, 0xd2, 0x42, - 0xe4, 0xf1, 0x2f, 0x52, 0x1e, 0xff, 0x0e, 0xc1, 0xd9, 0xb2, 0xb9, 0x09, 0x25, 0x72, 0x69, 0x87, - 0xc4, 0x92, 0xaa, 0x92, 0x54, 0x62, 0x39, 0x15, 0xae, 0xbb, 0x9c, 0x0e, 0x61, 0xab, 0x1d, 0x10, - 0x23, 0x24, 0x32, 0x94, 0x47, 0xfe, 0x7f, 0x1b, 0xca, 0xc6, 0x68, 0xe4, 0x99, 0x13, 0xb3, 0xae, - 0x72, 0xba, 0x6b, 0xa1, 0x06, 0x94, 0xcf, 0x3c, 0x1a, 0xba, 0x86, 0x43, 0x64, 0xf0, 0x8a, 0x69, - 0xed, 0x1b, 0x05, 0xb6, 0xa7, 0xf0, 0xa4, 0x15, 0x4e, 0xa1, 0x66, 0x53, 0x6f, 0xc4, 0xff, 0xa0, - 0x9e, 0x38, 0xe1, 0xfd, 0x78, 0xb1, 0xad, 0xa6, 0x1b, 0x61, 0xf0, 0x03, 0xdf, 0xba, 0x9d, 0x24, - 0xb9, 0xc7, 0xf1, 0xc9, 0x2d, 0xb9, 0xd2, 0x23, 0x52, 0xfb, 0x27, 0x05, 0xb6, 0xe5, 0x0e, 0x9f, - 0xfd, 0x8f, 0xce, 0x8a, 0x9c, 0x7b, 0xd7, 0x22, 0x6b, 0x75, 0xb8, 0x39, 0x2d, 0x97, 0x8c, 0xf9, - 0xbf, 0x2e, 0x02, 0x9a, 0x3d, 0x5d, 0xa2, 0xef, 0xc2, 0x1a, 0x25, 0xae, 0xa5, 0x8b, 0xfd, 0x42, - 0x6c, 0x65, 0x65, 0x5c, 0x65, 0x3c, 0xb1, 0x71, 0x50, 0x16, 0x02, 0xc9, 0xa5, 0x94, 0xb6, 0x8c, - 0xf9, 0x37, 0x3a, 0x83, 0xb5, 0x97, 0x54, 0x8f, 0xe7, 0xe6, 0x0e, 0x55, 0xcb, 0x1c, 0xd6, 0x66, - 0xe5, 0x68, 0x3e, 0xea, 0xc7, 0xff, 0x0b, 0x57, 0x5f, 0xd2, 0x98, 0x40, 0xbf, 0x50, 0xe0, 0x56, - 0x94, 0x56, 0x4c, 0xd4, 0xe7, 0x78, 0x16, 0xa1, 0xf5, 0xc2, 0x9d, 0xfc, 0x4e, 0x6d, 0xf7, 0xf8, - 0x1a, 0xfa, 0x9b, 0x61, 0x1e, 0x7a, 0x16, 0xc1, 0xdb, 0xee, 0x1c, 0x2e, 0x45, 0x4d, 0xb8, 0xe1, - 0x8c, 0x69, 0xa8, 0x0b, 0x2f, 0xd0, 0x65, 0xa7, 0x7a, 0x91, 0xeb, 0x65, 0x93, 0x35, 0xa5, 0x7c, - 0x15, 0x9d, 0xc3, 0xba, 0xe3, 0x8d, 0xdd, 0x50, 0x37, 0xf9, 0xf9, 0x87, 0xd6, 0x4b, 0x0b, 0x1d, - 0x8c, 0xe7, 0x68, 0xe9, 0x90, 0xc1, 0x89, 0xd3, 0x14, 0xc5, 0x6b, 0x4e, 0x82, 0x62, 0x86, 0x0c, - 0x88, 0xe3, 0x85, 0x44, 0x67, 0xf1, 0x92, 0xd6, 0x57, 0x85, 0x21, 0x05, 0x8f, 0x85, 0x06, 0x8a, - 0xfe, 0x00, 0x6e, 0x5a, 0x36, 0x35, 0x4e, 0x47, 0x44, 0x1f, 0x79, 0x43, 0x7d, 0x92, 0xe6, 0xd4, - 0xcb, 0xbc, 0xf3, 0x96, 0x6c, 0x3d, 0xf0, 0x86, 0xed, 0xb8, 0x4d, 0x6b, 0x42, 0x35, 0x61, 0x1c, - 0x54, 0x86, 0x42, 0xef, 0xa8, 0xd7, 0x51, 0x57, 0x10, 0x40, 0xa9, 0xbd, 0x8f, 0x8f, 0x8e, 0x06, - 0xe2, 0xac, 0xd1, 0x3d, 0x6c, 0x3d, 0xee, 0xa8, 0x39, 0xad, 0x03, 0x6b, 0x49, 0x31, 0x11, 0x82, - 0xda, 0x49, 0xef, 0x69, 0xef, 0xe8, 0x59, 0x4f, 0x3f, 0x3c, 0x3a, 0xe9, 0x0d, 0xd8, 0x29, 0xa5, - 0x06, 0xd0, 0xea, 0x3d, 0x9f, 0xd0, 0xeb, 0x50, 0xe9, 0x1d, 0x45, 0xa4, 0xd2, 0xc8, 0xa9, 0x8a, - 0xf6, 0x1f, 0x79, 0xd8, 0x9a, 0x67, 0x31, 0x64, 0x41, 0x81, 0x59, 0x5f, 0x9e, 0x13, 0xdf, 0xbd, - 0xf1, 0x39, 0x3a, 0x73, 0x7a, 0xdf, 0x90, 0x1b, 0x43, 0x05, 0xf3, 0x6f, 0xa4, 0x43, 0x69, 0x64, - 0x9c, 0x92, 0x11, 0xad, 0xe7, 0xf9, 0x4d, 0xca, 0xe3, 0xeb, 0xcc, 0x7d, 0xc0, 0x91, 0xc4, 0x35, - 0x8a, 0x84, 0x45, 0x03, 0xa8, 0xb2, 0xd0, 0x47, 0x85, 0xea, 0x64, 0x34, 0xde, 0xcd, 0x38, 0xcb, - 0xfe, 0x64, 0x24, 0x4e, 0xc2, 0x34, 0xee, 0x43, 0x35, 0x31, 0xd9, 0x9c, 0x5b, 0x90, 0xad, 0xe4, - 0x2d, 0x48, 0x25, 0x79, 0xa5, 0xf1, 0x70, 0xd6, 0x06, 0x4c, 0x47, 0xcc, 0x09, 0xf6, 0x8f, 0xfa, - 0x03, 0x71, 0xde, 0x7c, 0x8c, 0x8f, 0x4e, 0x8e, 0x55, 0x85, 0x31, 0x07, 0xad, 0xfe, 0x53, 0x35, - 0x17, 0xfb, 0x48, 0x5e, 0x6b, 0x43, 0x35, 0x21, 0x57, 0x2a, 0xd6, 0x2b, 0xe9, 0x58, 0xcf, 0xa2, - 0xad, 0x61, 0x59, 0x01, 0xa1, 0x54, 0xca, 0x11, 0x91, 0xda, 0x0b, 0xa8, 0xec, 0xf5, 0xfa, 0x12, - 0xa2, 0x0e, 0xab, 0x94, 0x04, 0xec, 0x7f, 0xf3, 0xfb, 0xac, 0x0a, 0x8e, 0x48, 0x06, 0x4e, 0x89, - 0x11, 0x98, 0x67, 0x84, 0xca, 0x0c, 0x21, 0xa6, 0xd9, 0x28, 0x8f, 0xdf, 0x0b, 0x09, 0xdb, 0x55, - 0x70, 0x44, 0x6a, 0xff, 0x57, 0x06, 0x98, 0xdc, 0x51, 0xa0, 0x1a, 0xe4, 0xe2, 0xc8, 0x9d, 0xb3, - 0x2d, 0xe6, 0x07, 0x89, 0x9d, 0x89, 0x7f, 0xa3, 0x5d, 0xd8, 0x76, 0xe8, 0xd0, 0x37, 0xcc, 0x73, - 0x5d, 0x5e, 0x2d, 0x88, 0x05, 0xce, 0xa3, 0xe0, 0x1a, 0xbe, 0x21, 0x1b, 0xe5, 0xfa, 0x15, 0xb8, - 0x07, 0x90, 0x27, 0xee, 0x05, 0x8f, 0x58, 0xd5, 0xdd, 0xcf, 0x16, 0xbe, 0x3b, 0x69, 0x76, 0xdc, - 0x0b, 0xe1, 0x2b, 0x0c, 0x06, 0xe9, 0x00, 0x16, 0xb9, 0xb0, 0x4d, 0xa2, 0x33, 0xd0, 0x22, 0x07, - 0xfd, 0x62, 0x71, 0xd0, 0x3d, 0x8e, 0x11, 0x43, 0x57, 0xac, 0x88, 0x46, 0x3d, 0xa8, 0x04, 0x84, - 0x7a, 0xe3, 0xc0, 0x24, 0x22, 0x6c, 0x65, 0x3f, 0xde, 0xe0, 0x68, 0x1c, 0x9e, 0x40, 0xa0, 0x3d, - 0x28, 0xf1, 0x68, 0xc5, 0xe2, 0x52, 0xfe, 0x5b, 0x2f, 0x62, 0xd3, 0x60, 0x3c, 0x92, 0x60, 0x39, - 0x16, 0x3d, 0x86, 0x55, 0x21, 0x22, 0xad, 0x97, 0x39, 0xcc, 0x47, 0x59, 0x43, 0x29, 0x1f, 0x85, - 0xa3, 0xd1, 0xcc, 0xaa, 0x63, 0x4a, 0x82, 0x7a, 0x45, 0x58, 0x95, 0x7d, 0xa3, 0xf7, 0xa0, 0x22, - 0x76, 0x6e, 0xcb, 0x0e, 0xea, 0x20, 0x9c, 0x93, 0x33, 0xf6, 0xec, 0x00, 0xbd, 0x0f, 0x55, 0x91, - 0xa1, 0xe9, 0x3c, 0x2a, 0x54, 0x79, 0x33, 0x08, 0xd6, 0x31, 0x8b, 0x0d, 0xa2, 0x03, 0x09, 0x02, - 0xd1, 0x61, 0x2d, 0xee, 0x40, 0x82, 0x80, 0x77, 0xf8, 0x3d, 0xd8, 0xe0, 0x79, 0xed, 0x30, 0xf0, - 0xc6, 0xbe, 0xce, 0x7d, 0x6a, 0x9d, 0x77, 0x5a, 0x67, 0xec, 0xc7, 0x8c, 0xdb, 0x63, 0xce, 0x75, - 0x1b, 0xca, 0xaf, 0xbc, 0x53, 0xd1, 0xa1, 0x26, 0xd6, 0xc1, 0x2b, 0xef, 0x34, 0x6a, 0x8a, 0x73, - 0x8b, 0x8d, 0x74, 0x6e, 0xf1, 0x35, 0xdc, 0x9c, 0xdd, 0x24, 0x79, 0x8e, 0xa1, 0x5e, 0x3f, 0xc7, - 0xd8, 0x72, 0xe7, 0xc5, 0xe1, 0x2f, 0x21, 0x6f, 0xb9, 0xb4, 0xbe, 0xb9, 0x90, 0x73, 0xc4, 0xeb, - 0x18, 0xb3, 0xc1, 0x68, 0x1b, 0x4a, 0xec, 0xcf, 0xda, 0x56, 0x1d, 0x89, 0xd0, 0xf3, 0xca, 0x3b, - 0xed, 0x5a, 0xe8, 0x3b, 0x50, 0x61, 0xff, 0x9f, 0xfa, 0x86, 0x49, 0xea, 0x37, 0x78, 0xcb, 0x84, - 0xc1, 0x0c, 0xe5, 0x7a, 0x16, 0x11, 0x2a, 0xda, 0x12, 0x86, 0x62, 0x0c, 0xae, 0xa3, 0x5b, 0xb0, - 0xca, 0x1b, 0x6d, 0xab, 0xbe, 0x2d, 0x8e, 0x0f, 0x8c, 0xec, 0x5a, 0x48, 0x83, 0x75, 0xdf, 0x08, - 0x88, 0x1b, 0xea, 0x72, 0xc6, 0x9b, 0xbc, 0xb9, 0x2a, 0x98, 0x4f, 0xd8, 0xbc, 0x8d, 0x4f, 0xa0, - 0x1c, 0x2d, 0x86, 0x45, 0xc2, 0x64, 0xe3, 0x01, 0xd4, 0xd2, 0x4b, 0x69, 0xa1, 0x20, 0xfb, 0x2f, - 0x39, 0xa8, 0xc4, 0x8b, 0x06, 0xb9, 0x70, 0x83, 0x1b, 0x95, 0xe5, 0x99, 0xfa, 0x64, 0x0d, 0x8a, - 0xec, 0xf6, 0xf3, 0x8c, 0x6a, 0x6e, 0x45, 0x08, 0xf2, 0x98, 0x2d, 0x17, 0x24, 0x8a, 0x91, 0x27, - 0xf3, 0x7d, 0x05, 0x1b, 0x23, 0xdb, 0x1d, 0x5f, 0x26, 0xe6, 0x12, 0x69, 0xe9, 0x1f, 0x66, 0x9c, - 0xeb, 0x80, 0x8d, 0x9e, 0xcc, 0x51, 0x1b, 0xa5, 0x68, 0xb4, 0x0f, 0x45, 0xdf, 0x0b, 0xc2, 0x68, - 0xcf, 0xcc, 0xba, 0x9b, 0x1d, 0x7b, 0x41, 0x78, 0x68, 0xf8, 0x3e, 0x3b, 0x79, 0x09, 0x00, 0xed, - 0x9b, 0x1c, 0xdc, 0x9c, 0xff, 0xc7, 0x50, 0x0f, 0xf2, 0xa6, 0x3f, 0x96, 0x4a, 0x7a, 0xb0, 0xa8, - 0x92, 0xda, 0xfe, 0x78, 0x22, 0x3f, 0x03, 0x42, 0xcf, 0xa0, 0xe4, 0x10, 0xc7, 0x0b, 0xae, 0xa4, - 0x2e, 0x1e, 0x2e, 0x0a, 0x79, 0xc8, 0x47, 0x4f, 0x50, 0x25, 0x1c, 0xc2, 0x50, 0x96, 0x8b, 0x89, - 0xca, 0xb0, 0xbd, 0xe0, 0xdd, 0x58, 0x04, 0x89, 0x63, 0x1c, 0xed, 0x13, 0xd8, 0x9e, 0xfb, 0x57, - 0xd0, 0xef, 0x00, 0x98, 0xfe, 0x58, 0xe7, 0x6f, 0x17, 0xc2, 0x83, 0xf2, 0xb8, 0x62, 0xfa, 0xe3, - 0x3e, 0x67, 0x68, 0x2f, 0xa0, 0xfe, 0x26, 0x79, 0xd9, 0x1a, 0x13, 0x12, 0xeb, 0xce, 0x29, 0xd7, - 0x41, 0x1e, 0x97, 0x05, 0xe3, 0xf0, 0x94, 0x2d, 0xa5, 0xa8, 0xd1, 0xb8, 0x64, 0x1d, 0xf2, 0xbc, - 0x43, 0x55, 0x76, 0x30, 0x2e, 0x0f, 0x4f, 0xb5, 0x5f, 0xe6, 0x60, 0x63, 0x4a, 0x64, 0x76, 0xfe, - 0x14, 0x01, 0x38, 0x3a, 0xd9, 0x0b, 0x8a, 0x45, 0x63, 0xd3, 0xb6, 0xa2, 0x3b, 0x61, 0xfe, 0xcd, - 0xf7, 0x61, 0x5f, 0xde, 0xd7, 0xe6, 0x6c, 0x9f, 0x2d, 0x1f, 0xe7, 0xd4, 0x0e, 0x29, 0x4f, 0x8a, - 0x8a, 0x58, 0x10, 0xe8, 0x39, 0xd4, 0x02, 0xc2, 0xf7, 0x7f, 0x4b, 0x17, 0x5e, 0x56, 0x5c, 0xc8, - 0xcb, 0xa4, 0x84, 0xcc, 0xd9, 0xf0, 0x7a, 0x84, 0xc4, 0x28, 0x8a, 0x9e, 0xc1, 0xba, 0x75, 0xe5, - 0x1a, 0x8e, 0x6d, 0x4a, 0xe4, 0xd2, 0xd2, 0xc8, 0x6b, 0x12, 0x88, 0x03, 0x6b, 0xf7, 0xa1, 0x9a, - 0x68, 0x64, 0x7f, 0x8c, 0x67, 0x7f, 0x52, 0x27, 0x82, 0x48, 0x47, 0x8b, 0xa2, 0x8c, 0x16, 0xda, - 0x29, 0x54, 0x13, 0xeb, 0x62, 0x91, 0xa1, 0x4c, 0x9f, 0xa1, 0xc7, 0xf5, 0x59, 0xc4, 0xb9, 0xd0, - 0x63, 0x71, 0x92, 0x65, 0x5e, 0xba, 0xed, 0x73, 0x8d, 0x56, 0x70, 0x89, 0x91, 0x5d, 0x5f, 0xfb, - 0x4d, 0x0e, 0x6a, 0xe9, 0x25, 0x1d, 0xf9, 0x91, 0x4f, 0x02, 0xdb, 0xb3, 0x12, 0x7e, 0x74, 0xcc, - 0x19, 0xcc, 0x57, 0x58, 0xf3, 0xd7, 0x63, 0x2f, 0x34, 0x22, 0x5f, 0x31, 0xfd, 0xf1, 0x1f, 0x31, - 0x7a, 0xca, 0x07, 0xf3, 0x53, 0x3e, 0x88, 0x3e, 0x04, 0x24, 0x5d, 0x69, 0x64, 0x3b, 0x76, 0xa8, - 0x9f, 0x5e, 0x85, 0x44, 0xd8, 0x38, 0x8f, 0x55, 0xd1, 0x72, 0xc0, 0x1a, 0xbe, 0x64, 0x7c, 0xe6, - 0x78, 0x9e, 0xe7, 0xe8, 0xd4, 0xf4, 0x02, 0xa2, 0x1b, 0xd6, 0x2b, 0x7e, 0xf4, 0xca, 0xe3, 0xaa, - 0xe7, 0x39, 0x7d, 0xc6, 0x6b, 0x59, 0xaf, 0xd8, 0x46, 0x6c, 0xfa, 0x63, 0x4a, 0x42, 0x9d, 0xfd, - 0xf0, 0xdc, 0xa5, 0x82, 0x41, 0xb0, 0xda, 0xfe, 0x98, 0xa2, 0xef, 0xc1, 0x7a, 0xd4, 0x81, 0xef, - 0xc5, 0x32, 0x09, 0x58, 0x93, 0x5d, 0x38, 0x0f, 0x69, 0xb0, 0x76, 0x4c, 0x02, 0x93, 0xb8, 0xe1, - 0xc0, 0x36, 0xcf, 0x29, 0x3f, 0x20, 0x29, 0x38, 0xc5, 0x7b, 0x52, 0x28, 0xaf, 0xaa, 0x65, 0x1c, - 0xcd, 0xe6, 0x10, 0x87, 0x6a, 0xbf, 0x56, 0xa0, 0xc8, 0x53, 0x16, 0xa6, 0x14, 0xbe, 0xdd, 0xf3, - 0x6c, 0x40, 0xa6, 0xba, 0x8c, 0xc1, 0x73, 0x81, 0xf7, 0xa0, 0xc2, 0x95, 0x9f, 0x38, 0x61, 0xf0, - 0x3c, 0x98, 0x37, 0x36, 0xa0, 0x1c, 0x10, 0xc3, 0xf2, 0xdc, 0x51, 0x74, 0xa5, 0x15, 0xd3, 0xe8, - 0xf7, 0x41, 0xf5, 0x03, 0xcf, 0x37, 0x86, 0x93, 0x53, 0xb0, 0x34, 0xdf, 0x46, 0x82, 0xcf, 0x53, - 0xf4, 0xef, 0xc1, 0x3a, 0x25, 0x22, 0xb2, 0x0b, 0x27, 0x29, 0x8a, 0xbf, 0x29, 0x99, 0xfc, 0x44, - 0xa0, 0x7d, 0x0d, 0x25, 0xb1, 0x71, 0x5d, 0x43, 0xde, 0x8f, 0x00, 0x09, 0x45, 0x32, 0x07, 0x71, - 0x6c, 0x4a, 0x65, 0x96, 0xcd, 0xdf, 0x65, 0x45, 0xcb, 0xf1, 0xa4, 0x41, 0xfb, 0x2f, 0x45, 0xe4, - 0xdb, 0xe2, 0xc5, 0x8c, 0x25, 0xe6, 0x6c, 0xd5, 0xb0, 0x43, 0xa8, 0xb8, 0x9a, 0x8b, 0x48, 0xd4, - 0x85, 0x92, 0x4c, 0xab, 0x73, 0xcb, 0x3e, 0x38, 0x4a, 0x80, 0xe8, 0xa2, 0x9e, 0xc8, 0x6b, 0x8a, - 0x45, 0x2f, 0xea, 0x89, 0xb8, 0xa8, 0x27, 0xec, 0x8c, 0x2d, 0x13, 0x7e, 0x01, 0x57, 0xe0, 0xf9, - 0x7e, 0xd5, 0x8a, 0x5f, 0x43, 0x88, 0xf6, 0x3f, 0x4a, 0x1c, 0xf7, 0xa2, 0x57, 0x0b, 0xf4, 0x15, - 0x94, 0x59, 0x08, 0xd1, 0x1d, 0xc3, 0x97, 0x6f, 0xf0, 0xed, 0xe5, 0x1e, 0x44, 0xa2, 0x5d, 0x51, - 0xa4, 0xeb, 0xab, 0xbe, 0xa0, 0x58, 0xfc, 0x64, 0x47, 0xa5, 0x28, 0x7e, 0xb2, 0x6f, 0xf4, 0x01, - 0xd4, 0x8c, 0x71, 0xe8, 0xe9, 0x86, 0x75, 0x41, 0x82, 0xd0, 0xa6, 0x44, 0xfa, 0xd2, 0x3a, 0xe3, - 0xb6, 0x22, 0x66, 0xe3, 0x33, 0x58, 0x4b, 0x62, 0xbe, 0x2d, 0x6f, 0x29, 0x26, 0xf3, 0x96, 0x3f, - 0x03, 0x98, 0xdc, 0x00, 0x32, 0x1f, 0x21, 0x97, 0x76, 0xa8, 0x9b, 0xd1, 0xd9, 0xbc, 0x88, 0xcb, - 0x8c, 0xd1, 0x66, 0xce, 0x98, 0x7e, 0x9e, 0x28, 0x46, 0xcf, 0x13, 0x2c, 0x3a, 0xb0, 0x05, 0x7d, - 0x6e, 0x8f, 0x46, 0xf1, 0xad, 0x64, 0xc5, 0xf3, 0x9c, 0xa7, 0x9c, 0xa1, 0xfd, 0x36, 0x27, 0x7c, - 0x45, 0x3c, 0x34, 0x65, 0x3a, 0x9b, 0xbd, 0x2b, 0x53, 0xdf, 0x07, 0xa0, 0xa1, 0x11, 0xb0, 0x24, - 0xcc, 0x88, 0xee, 0x45, 0x1b, 0x33, 0xef, 0x1b, 0x83, 0xa8, 0xf2, 0x05, 0x57, 0x64, 0xef, 0x56, - 0x88, 0x3e, 0x87, 0x35, 0xd3, 0x73, 0xfc, 0x11, 0x91, 0x83, 0x8b, 0x6f, 0x1d, 0x5c, 0x8d, 0xfb, - 0xb7, 0xc2, 0xc4, 0x6d, 0x6c, 0xe9, 0xba, 0xb7, 0xb1, 0xbf, 0x51, 0xc4, 0x7b, 0x59, 0xf2, 0xb9, - 0x0e, 0x0d, 0xe7, 0xd4, 0x84, 0x3c, 0x5e, 0xf2, 0xed, 0xef, 0xdb, 0x0a, 0x42, 0x1a, 0x9f, 0x67, - 0xa9, 0xc0, 0x78, 0x73, 0x5a, 0xfc, 0xef, 0x79, 0xa8, 0xc4, 0x4f, 0x65, 0x33, 0xb6, 0xff, 0x14, - 0x2a, 0x71, 0xd9, 0x91, 0x0c, 0x10, 0xdf, 0x6a, 0x9e, 0xb8, 0x33, 0x7a, 0x09, 0xc8, 0x18, 0x0e, - 0xe3, 0x74, 0x57, 0x1f, 0x53, 0x63, 0x18, 0x3d, 0x54, 0x7e, 0xba, 0x80, 0x1e, 0xa2, 0xfd, 0xf1, - 0x84, 0x8d, 0xc7, 0xaa, 0x31, 0x1c, 0xa6, 0x38, 0xe8, 0xcf, 0x61, 0x3b, 0x3d, 0x87, 0x7e, 0x7a, - 0xa5, 0xfb, 0xb6, 0x25, 0xef, 0x00, 0xf6, 0x17, 0x7d, 0x2d, 0x6c, 0xa6, 0xe0, 0xbf, 0xbc, 0x3a, - 0xb6, 0x2d, 0xa1, 0x73, 0x14, 0xcc, 0x34, 0x34, 0xfe, 0x12, 0x6e, 0xbd, 0xa1, 0xfb, 0x1c, 0x1b, - 0xf4, 0xd2, 0x55, 0x30, 0xcb, 0x2b, 0x21, 0x61, 0xbd, 0x5f, 0x29, 0xe2, 0x51, 0x33, 0xad, 0x93, - 0x56, 0x32, 0x4f, 0xbf, 0x9b, 0x71, 0x9e, 0xf6, 0xf1, 0x89, 0x80, 0xe7, 0xa9, 0xf9, 0x93, 0xa9, - 0xd4, 0x3c, 0x6b, 0x42, 0x26, 0x32, 0x5c, 0x01, 0x24, 0x11, 0xb4, 0x7f, 0xcd, 0x43, 0x39, 0x42, - 0xe7, 0x27, 0xf8, 0x2b, 0x1a, 0x12, 0x47, 0x8f, 0xaf, 0x17, 0x15, 0x0c, 0x82, 0xc5, 0x77, 0xd4, - 0xf7, 0xa0, 0x32, 0xa6, 0x24, 0x10, 0xcd, 0x39, 0xde, 0x5c, 0x66, 0x0c, 0xde, 0xf8, 0x3e, 0x54, - 0x43, 0x2f, 0x34, 0x46, 0x7a, 0xc8, 0xf3, 0x85, 0xbc, 0x18, 0xcd, 0x59, 0x3c, 0x5b, 0x40, 0x3f, - 0x80, 0xcd, 0xf0, 0x2c, 0xf0, 0xc2, 0x70, 0xc4, 0x72, 0x55, 0x9e, 0x39, 0x89, 0x44, 0xa7, 0x80, - 0xd5, 0xb8, 0x41, 0x64, 0x54, 0x94, 0x45, 0xef, 0x49, 0x67, 0xe6, 0xba, 0x3c, 0x88, 0x14, 0xf0, - 0x7a, 0xcc, 0x65, 0xae, 0xcd, 0x36, 0x4f, 0x5f, 0x64, 0x24, 0x3c, 0x56, 0x28, 0x38, 0x22, 0x91, - 0x0e, 0x1b, 0x0e, 0x31, 0xe8, 0x38, 0x20, 0x96, 0xfe, 0xd2, 0x26, 0x23, 0x4b, 0x5c, 0xbc, 0xd4, - 0x32, 0x1f, 0x37, 0x22, 0xb5, 0x34, 0x1f, 0xf1, 0xd1, 0xb8, 0x16, 0xc1, 0x09, 0x9a, 0x65, 0x0e, - 0xe2, 0x0b, 0x6d, 0x40, 0xb5, 0xff, 0xbc, 0x3f, 0xe8, 0x1c, 0xea, 0x87, 0x47, 0x7b, 0x1d, 0x59, - 0xe8, 0xd4, 0xef, 0x60, 0x41, 0x2a, 0xac, 0x7d, 0x70, 0x34, 0x68, 0x1d, 0xe8, 0x83, 0x6e, 0xfb, - 0x69, 0x5f, 0xcd, 0xa1, 0x6d, 0xd8, 0x1c, 0xec, 0xe3, 0xa3, 0xc1, 0xe0, 0xa0, 0xb3, 0xa7, 0x1f, - 0x77, 0x70, 0xf7, 0x68, 0xaf, 0xaf, 0xe6, 0x11, 0x82, 0xda, 0x84, 0x3d, 0xe8, 0x1e, 0x76, 0xd4, - 0x02, 0xaa, 0xc2, 0xea, 0x71, 0x07, 0xb7, 0x3b, 0xbd, 0x81, 0x5a, 0xd4, 0x7e, 0x99, 0x87, 0x6a, - 0xc2, 0x8a, 0xcc, 0x91, 0x03, 0x2a, 0xce, 0x35, 0x05, 0xcc, 0x3e, 0xf9, 0xc3, 0xac, 0x61, 0x9e, - 0x09, 0xeb, 0x14, 0xb0, 0x20, 0xf8, 0x59, 0xc6, 0xb8, 0x4c, 0xac, 0xf3, 0x02, 0x2e, 0x3b, 0xc6, - 0xa5, 0x00, 0xf9, 0x2e, 0xac, 0x9d, 0x93, 0xc0, 0x25, 0x23, 0xd9, 0x2e, 0x2c, 0x52, 0x15, 0x3c, - 0xd1, 0x65, 0x07, 0x54, 0xd9, 0x65, 0x02, 0x23, 0xcc, 0x51, 0x13, 0xfc, 0xc3, 0x08, 0x6c, 0x0b, - 0x8a, 0xa2, 0x79, 0x55, 0xcc, 0xcf, 0x09, 0xb6, 0x4d, 0xd1, 0xd7, 0x86, 0xcf, 0x73, 0xc8, 0x02, - 0xe6, 0xdf, 0xe8, 0x74, 0xd6, 0x3e, 0x25, 0x6e, 0x9f, 0xfb, 0x8b, 0xbb, 0xf3, 0x9b, 0x4c, 0x74, - 0x16, 0x9b, 0x68, 0x15, 0xf2, 0x38, 0xaa, 0x0e, 0x6a, 0xb7, 0xda, 0xfb, 0xcc, 0x2c, 0xeb, 0x50, - 0x39, 0x6c, 0xfd, 0x54, 0x3f, 0xe9, 0xf3, 0x5b, 0x7b, 0xa4, 0xc2, 0xda, 0xd3, 0x0e, 0xee, 0x75, - 0x0e, 0x24, 0x27, 0x8f, 0xb6, 0x40, 0x95, 0x9c, 0x49, 0xbf, 0x02, 0x43, 0x10, 0x9f, 0x45, 0x54, - 0x86, 0x42, 0xff, 0x59, 0xeb, 0x58, 0x2d, 0x69, 0xff, 0x9d, 0x83, 0x0d, 0xb1, 0x2d, 0xc4, 0x75, - 0x0c, 0x6f, 0x7e, 0xc7, 0x4d, 0xde, 0x62, 0xe5, 0xd2, 0xb7, 0x58, 0x51, 0x12, 0xca, 0x77, 0xf5, - 0xfc, 0x24, 0x09, 0xe5, 0x37, 0x3b, 0xa9, 0x88, 0x5f, 0x58, 0x24, 0xe2, 0xd7, 0x61, 0xd5, 0x21, - 0x34, 0xb6, 0x5b, 0x05, 0x47, 0x24, 0xb2, 0xa1, 0x6a, 0xb8, 0xae, 0x17, 0x1a, 0xe2, 0x6a, 0xb8, - 0xb4, 0xd0, 0x66, 0x38, 0xf5, 0x8f, 0x9b, 0xad, 0x09, 0x92, 0x08, 0xcc, 0x49, 0xec, 0xc6, 0x4f, - 0x40, 0x9d, 0xee, 0xb0, 0xc8, 0x76, 0xf8, 0xfd, 0x1f, 0x4e, 0x76, 0x43, 0xc2, 0xd6, 0x85, 0x7c, - 0x53, 0x51, 0x57, 0x18, 0x81, 0x4f, 0x7a, 0xbd, 0x6e, 0xef, 0xb1, 0xaa, 0x20, 0x80, 0x52, 0xe7, - 0xa7, 0xdd, 0x41, 0x67, 0x4f, 0xcd, 0xed, 0xfe, 0x6a, 0x13, 0x4a, 0x42, 0x48, 0xf4, 0x8d, 0xcc, - 0x04, 0x92, 0x35, 0xb2, 0xe8, 0x27, 0x0b, 0x67, 0xd4, 0xa9, 0xba, 0xdb, 0xc6, 0xc3, 0xa5, 0xc7, - 0xcb, 0x37, 0xc9, 0x15, 0xf4, 0xb7, 0x0a, 0xac, 0xa5, 0xde, 0x23, 0xb3, 0x5e, 0x8d, 0xcf, 0x29, - 0xc9, 0x6d, 0xfc, 0x78, 0xa9, 0xb1, 0xb1, 0x2c, 0xbf, 0x50, 0xa0, 0x9a, 0x28, 0x46, 0x45, 0xf7, - 0x97, 0x29, 0x60, 0x15, 0x92, 0x7c, 0xb6, 0x7c, 0xed, 0xab, 0xb6, 0xf2, 0xb1, 0x82, 0xfe, 0x46, - 0x81, 0x6a, 0xa2, 0x2c, 0x33, 0xb3, 0x28, 0xb3, 0x45, 0xa4, 0x99, 0x45, 0x99, 0x57, 0x05, 0xba, - 0x82, 0xfe, 0x4a, 0x81, 0x4a, 0x5c, 0x62, 0x89, 0xee, 0x2d, 0x5e, 0x94, 0x29, 0x84, 0xf8, 0x74, - 0xd9, 0x6a, 0x4e, 0x6d, 0x05, 0xfd, 0x05, 0x94, 0xa3, 0x7a, 0x44, 0x94, 0x75, 0xf7, 0x9a, 0x2a, - 0x76, 0x6c, 0xdc, 0x5b, 0x78, 0x5c, 0x72, 0xfa, 0xa8, 0x48, 0x30, 0xf3, 0xf4, 0x53, 0xe5, 0x8c, - 0x8d, 0x7b, 0x0b, 0x8f, 0x8b, 0xa7, 0x67, 0x9e, 0x90, 0xa8, 0x25, 0xcc, 0xec, 0x09, 0xb3, 0x45, - 0x8c, 0x99, 0x3d, 0x61, 0x5e, 0xe9, 0xa2, 0x10, 0x24, 0x51, 0x8d, 0x98, 0x59, 0x90, 0xd9, 0x8a, - 0xc7, 0xcc, 0x82, 0xcc, 0x29, 0x7e, 0xd4, 0x56, 0xd0, 0xcf, 0x95, 0xe4, 0xb9, 0xe0, 0xde, 0xc2, - 0x45, 0x77, 0x0b, 0xba, 0xe4, 0x4c, 0xd9, 0x1f, 0x5f, 0xa0, 0x3f, 0x97, 0xb7, 0x18, 0xa2, 0x66, - 0x0f, 0x2d, 0x02, 0x96, 0x2a, 0xf3, 0x6b, 0x7c, 0xb2, 0xdc, 0x66, 0xc3, 0x85, 0xf8, 0x6b, 0x05, - 0x60, 0x52, 0xdd, 0x97, 0x59, 0x88, 0x99, 0xb2, 0xc2, 0xc6, 0xfd, 0x25, 0x46, 0x26, 0x17, 0x48, - 0x54, 0x7d, 0x94, 0x79, 0x81, 0x4c, 0x55, 0x1f, 0x66, 0x5e, 0x20, 0xd3, 0x95, 0x83, 0xda, 0x0a, - 0xfa, 0x67, 0x05, 0x36, 0x67, 0xaa, 0x9f, 0xd0, 0xc3, 0x6b, 0x16, 0xc0, 0x35, 0xbe, 0x58, 0x1e, - 0x20, 0x12, 0x6d, 0x47, 0xf9, 0x58, 0x41, 0x7f, 0xa7, 0xc0, 0x7a, 0xba, 0x2a, 0x24, 0xf3, 0x2e, - 0x35, 0xa7, 0x8e, 0xaa, 0xf1, 0x60, 0xb9, 0xc1, 0xb1, 0xb6, 0xfe, 0x41, 0x81, 0x5a, 0xba, 0x40, - 0x08, 0x3d, 0x58, 0x2c, 0x2c, 0x4c, 0x09, 0xf4, 0xf9, 0x92, 0xa3, 0x23, 0x89, 0xbe, 0x5c, 0xfd, - 0xe3, 0xa2, 0xc8, 0xde, 0x4a, 0xfc, 0xe7, 0x47, 0xff, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x08, 0xce, - 0xb6, 0x69, 0xca, 0x34, 0x00, 0x00, + // 3920 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4f, 0x73, 0xdb, 0x48, + 0x76, 0x17, 0x08, 0x92, 0x22, 0x1f, 0x25, 0x0a, 0x6a, 0x49, 0x36, 0xcd, 0xd9, 0x64, 0xbc, 0xd8, + 0x9a, 0x94, 0xb2, 0x3b, 0x43, 0xcf, 0x6a, 0x93, 0xf1, 0xd8, 0xeb, 0x59, 0x0f, 0x87, 0xa2, 0x2d, + 0xda, 0x12, 0xa5, 0x34, 0xa9, 0x78, 0x1d, 0x27, 0x83, 0x40, 0x40, 0x9b, 0x82, 0x45, 0x02, 0x18, + 0x34, 0x28, 0x4b, 0x9b, 0x4a, 0x25, 0xb5, 0xa9, 0x4a, 0x6d, 0xaa, 0x92, 0x4a, 0x2e, 0x93, 0xbd, + 0xe4, 0xb4, 0x55, 0x39, 0xa5, 0x72, 0x4f, 0x36, 0x35, 0xa7, 0x1c, 0xf2, 0x25, 0x72, 0xc9, 0x2d, + 0xd7, 0x7c, 0x82, 0x6c, 0xf5, 0x1f, 0x80, 0x80, 0x48, 0x8f, 0x41, 0xca, 0x27, 0xe2, 0xbd, 0xee, + 0xfe, 0xf5, 0xe3, 0x7b, 0xaf, 0x5f, 0xbf, 0xee, 0x7e, 0xa0, 0xfb, 0xc3, 0xf1, 0xc0, 0x71, 0xe9, + 0x1d, 0x3b, 0x70, 0xce, 0x49, 0x40, 0xef, 0xf8, 0x81, 0x17, 0x7a, 0x92, 0x6a, 0x70, 0x02, 0x7d, + 0x70, 0x6a, 0xd2, 0x53, 0xc7, 0xf2, 0x02, 0xbf, 0xe1, 0x7a, 0x23, 0xd3, 0x6e, 0xc8, 0x31, 0x0d, + 0x39, 0x46, 0x74, 0xab, 0xff, 0xf6, 0xc0, 0xf3, 0x06, 0x43, 0x22, 0x10, 0x4e, 0xc6, 0x2f, 0xef, + 0xd8, 0xe3, 0xc0, 0x0c, 0x1d, 0xcf, 0x95, 0xed, 0xef, 0x5f, 0x6d, 0x0f, 0x9d, 0x11, 0xa1, 0xa1, + 0x39, 0xf2, 0x65, 0x87, 0x0f, 0x22, 0x59, 0xe8, 0xa9, 0x19, 0x10, 0xfb, 0xce, 0xa9, 0x35, 0xa4, + 0x3e, 0xb1, 0xd8, 0xaf, 0xc1, 0x3e, 0x64, 0xb7, 0x0f, 0xaf, 0x74, 0xa3, 0x61, 0x30, 0xb6, 0xc2, + 0x48, 0x72, 0x33, 0x0c, 0x03, 0xe7, 0x64, 0x1c, 0x12, 0xd1, 0x5b, 0xbf, 0x05, 0x37, 0xfb, 0x26, + 0x3d, 0x6b, 0x79, 0xee, 0x4b, 0x67, 0xd0, 0xb3, 0x4e, 0xc9, 0xc8, 0xc4, 0xe4, 0xab, 0x31, 0xa1, + 0xa1, 0xfe, 0xc7, 0x50, 0x9b, 0x6e, 0xa2, 0xbe, 0xe7, 0x52, 0x82, 0x3e, 0x87, 0x3c, 0x9b, 0xb2, + 0xa6, 0xdc, 0x56, 0xb6, 0x2b, 0x3b, 0x1f, 0x36, 0xde, 0xa4, 0x02, 0x21, 0x43, 0x43, 0x8a, 0xda, + 0xe8, 0xf9, 0xc4, 0xc2, 0x7c, 0xa4, 0xbe, 0x05, 0x1b, 0x2d, 0xd3, 0x37, 0x4f, 0x9c, 0xa1, 0x13, + 0x3a, 0x84, 0x46, 0x93, 0x8e, 0x61, 0x33, 0xcd, 0x96, 0x13, 0xfe, 0x09, 0xac, 0x58, 0x09, 0xbe, + 0x9c, 0xf8, 0x5e, 0x23, 0x93, 0xee, 0x1b, 0xbb, 0x9c, 0x4a, 0x01, 0xa7, 0xe0, 0xf4, 0x4d, 0x40, + 0x8f, 0x1c, 0x77, 0x40, 0x02, 0x3f, 0x70, 0xdc, 0x30, 0x12, 0xe6, 0x1b, 0x15, 0x36, 0x52, 0x6c, + 0x29, 0xcc, 0x2b, 0x80, 0x58, 0x8f, 0x4c, 0x14, 0x75, 0xbb, 0xb2, 0xf3, 0x24, 0xa3, 0x28, 0x33, + 0xf0, 0x1a, 0xcd, 0x18, 0xac, 0xed, 0x86, 0xc1, 0x25, 0x4e, 0xa0, 0xa3, 0x2f, 0xa1, 0x78, 0x4a, + 0xcc, 0x61, 0x78, 0x5a, 0xcb, 0xdd, 0x56, 0xb6, 0xab, 0x3b, 0x8f, 0xae, 0x31, 0xcf, 0x1e, 0x07, + 0xea, 0x85, 0x66, 0x48, 0xb0, 0x44, 0x45, 0x1f, 0x01, 0x12, 0x5f, 0x86, 0x4d, 0xa8, 0x15, 0x38, + 0x3e, 0x73, 0xc9, 0x9a, 0x7a, 0x5b, 0xd9, 0x2e, 0xe3, 0x75, 0xd1, 0xb2, 0x3b, 0x69, 0xa8, 0xfb, + 0xb0, 0x76, 0x45, 0x5a, 0xa4, 0x81, 0x7a, 0x46, 0x2e, 0xb9, 0x45, 0xca, 0x98, 0x7d, 0xa2, 0xc7, + 0x50, 0x38, 0x37, 0x87, 0x63, 0xc2, 0x45, 0xae, 0xec, 0xfc, 0xf0, 0x6d, 0xee, 0x21, 0x5d, 0x74, + 0xa2, 0x07, 0x2c, 0xc6, 0xdf, 0xcf, 0x7d, 0xaa, 0xe8, 0xf7, 0xa0, 0x92, 0x90, 0x1b, 0x55, 0x01, + 0x8e, 0xbb, 0xbb, 0xed, 0x7e, 0xbb, 0xd5, 0x6f, 0xef, 0x6a, 0x4b, 0x68, 0x15, 0xca, 0xc7, 0xdd, + 0xbd, 0x76, 0x73, 0xbf, 0xbf, 0xf7, 0x5c, 0x53, 0x50, 0x05, 0x96, 0x23, 0x22, 0xa7, 0x5f, 0x00, + 0xc2, 0xc4, 0xf2, 0xce, 0x49, 0xc0, 0x1c, 0x59, 0x5a, 0x15, 0xdd, 0x84, 0xe5, 0xd0, 0xa4, 0x67, + 0x86, 0x63, 0x4b, 0x99, 0x8b, 0x8c, 0xec, 0xd8, 0xa8, 0x03, 0xc5, 0x53, 0xd3, 0xb5, 0x87, 0x6f, + 0x97, 0x3b, 0xad, 0x6a, 0x06, 0xbe, 0xc7, 0x07, 0x62, 0x09, 0xc0, 0xbc, 0x3b, 0x35, 0xb3, 0x30, + 0x80, 0xfe, 0x1c, 0xb4, 0x5e, 0x68, 0x06, 0x61, 0x52, 0x9c, 0x36, 0xe4, 0xd9, 0xfc, 0xd2, 0xa3, + 0xe7, 0x99, 0x53, 0xac, 0x4c, 0xcc, 0x87, 0xeb, 0xff, 0x97, 0x83, 0xf5, 0x04, 0xb6, 0xf4, 0xd4, + 0x67, 0x50, 0x0c, 0x08, 0x1d, 0x0f, 0x43, 0x0e, 0x5f, 0xdd, 0x79, 0x98, 0x11, 0x7e, 0x0a, 0xa9, + 0x81, 0x39, 0x0c, 0x96, 0x70, 0x68, 0x1b, 0x34, 0x31, 0xc2, 0x20, 0x41, 0xe0, 0x05, 0xc6, 0x88, + 0x0e, 0xb8, 0xd6, 0xca, 0xb8, 0x2a, 0xf8, 0x6d, 0xc6, 0x3e, 0xa0, 0x83, 0x84, 0x56, 0xd5, 0x6b, + 0x6a, 0x15, 0x99, 0xa0, 0xb9, 0x24, 0x7c, 0xed, 0x05, 0x67, 0x06, 0x53, 0x6d, 0xe0, 0xd8, 0xa4, + 0x96, 0xe7, 0xa0, 0x9f, 0x64, 0x04, 0xed, 0x8a, 0xe1, 0x87, 0x72, 0x34, 0x5e, 0x73, 0xd3, 0x0c, + 0xfd, 0x07, 0x50, 0x14, 0xff, 0x94, 0x79, 0x52, 0xef, 0xb8, 0xd5, 0x6a, 0xf7, 0x7a, 0xda, 0x12, + 0x2a, 0x43, 0x01, 0xb7, 0xfb, 0x98, 0x79, 0x58, 0x19, 0x0a, 0x8f, 0x9a, 0xfd, 0xe6, 0xbe, 0x96, + 0xd3, 0xbf, 0x0f, 0x6b, 0xcf, 0x4c, 0x27, 0xcc, 0xe2, 0x5c, 0xba, 0x07, 0xda, 0xa4, 0xaf, 0xb4, + 0x4e, 0x27, 0x65, 0x9d, 0xec, 0xaa, 0x69, 0x5f, 0x38, 0xe1, 0x15, 0x7b, 0x68, 0xa0, 0x92, 0x20, + 0x90, 0x26, 0x60, 0x9f, 0xfa, 0x6b, 0x58, 0xeb, 0x85, 0x9e, 0x9f, 0xc9, 0xf3, 0x7f, 0x04, 0xcb, + 0x6c, 0xb7, 0xf1, 0xc6, 0xa1, 0x74, 0xfd, 0x5b, 0x0d, 0xb1, 0x1b, 0x35, 0xa2, 0xdd, 0xa8, 0xb1, + 0x2b, 0x77, 0x2b, 0x1c, 0xf5, 0x44, 0x37, 0xa0, 0x48, 0x9d, 0x81, 0x6b, 0x0e, 0x65, 0xb4, 0x90, + 0x94, 0x8e, 0x98, 0x93, 0x47, 0x13, 0x4b, 0xc7, 0x6f, 0x01, 0xda, 0x25, 0x34, 0x0c, 0xbc, 0xcb, + 0x4c, 0xf2, 0x6c, 0x42, 0xe1, 0xa5, 0x17, 0x58, 0x62, 0x21, 0x96, 0xb0, 0x20, 0xd8, 0xa2, 0x4a, + 0x81, 0x48, 0xec, 0x8f, 0x00, 0x75, 0x5c, 0xb6, 0xa7, 0x64, 0x33, 0xc4, 0x3f, 0xe4, 0x60, 0x23, + 0xd5, 0x5f, 0x1a, 0x63, 0xf1, 0x75, 0xc8, 0x02, 0xd3, 0x98, 0x8a, 0x75, 0x88, 0x0e, 0xa1, 0x28, + 0x7a, 0x48, 0x4d, 0xde, 0x9d, 0x03, 0x48, 0x6c, 0x53, 0x12, 0x4e, 0xc2, 0xcc, 0x74, 0x7a, 0xf5, + 0xdd, 0x3a, 0xfd, 0x6b, 0xd0, 0xa2, 0xff, 0x41, 0xdf, 0x6a, 0x9b, 0x27, 0xb0, 0x61, 0x79, 0xc3, + 0x21, 0xb1, 0x98, 0x37, 0x18, 0x8e, 0x1b, 0x92, 0xe0, 0xdc, 0x1c, 0xbe, 0xdd, 0x6f, 0xd0, 0x64, + 0x54, 0x47, 0x0e, 0xd2, 0x5f, 0xc0, 0x7a, 0x62, 0x62, 0x69, 0x88, 0x47, 0x50, 0xa0, 0x8c, 0x21, + 0x2d, 0xf1, 0xf1, 0x9c, 0x96, 0xa0, 0x58, 0x0c, 0xd7, 0x37, 0x04, 0x78, 0xfb, 0x9c, 0xb8, 0xf1, + 0xdf, 0xd2, 0x77, 0x61, 0xbd, 0xc7, 0xdd, 0x34, 0x93, 0x1f, 0x4e, 0x5c, 0x3c, 0x97, 0x72, 0xf1, + 0x4d, 0x40, 0x49, 0x14, 0xe9, 0x88, 0x97, 0xb0, 0xd6, 0xbe, 0x20, 0x56, 0x26, 0xe4, 0x1a, 0x2c, + 0x5b, 0xde, 0x68, 0x64, 0xba, 0x76, 0x2d, 0x77, 0x5b, 0xdd, 0x2e, 0xe3, 0x88, 0x4c, 0xae, 0x45, + 0x35, 0xeb, 0x5a, 0xd4, 0xff, 0x4e, 0x01, 0x6d, 0x32, 0xb7, 0x54, 0x24, 0x93, 0x3e, 0xb4, 0x19, + 0x10, 0x9b, 0x7b, 0x05, 0x4b, 0x4a, 0xf2, 0xa3, 0x70, 0x21, 0xf8, 0x24, 0x08, 0x12, 0xe1, 0x48, + 0xbd, 0x66, 0x38, 0xd2, 0xf7, 0xe0, 0x3b, 0x91, 0x38, 0xbd, 0x30, 0x20, 0xe6, 0xc8, 0x71, 0x07, + 0x9d, 0xc3, 0x43, 0x9f, 0x08, 0xc1, 0x11, 0x82, 0xbc, 0x6d, 0x86, 0xa6, 0x14, 0x8c, 0x7f, 0xb3, + 0x45, 0x6f, 0x0d, 0x3d, 0x1a, 0x2f, 0x7a, 0x4e, 0xe8, 0xff, 0xa5, 0x42, 0x6d, 0x0a, 0x2a, 0x52, + 0xef, 0x0b, 0x28, 0x50, 0x12, 0x8e, 0x7d, 0xe9, 0x2a, 0xed, 0xcc, 0x02, 0xcf, 0xc6, 0x6b, 0xf4, + 0x18, 0x18, 0x16, 0x98, 0x68, 0x00, 0xa5, 0x30, 0xbc, 0x34, 0xa8, 0xf3, 0xb3, 0x28, 0x21, 0xd8, + 0xbf, 0x2e, 0x7e, 0x9f, 0x04, 0x23, 0xc7, 0x35, 0x87, 0x3d, 0xe7, 0x67, 0x04, 0x2f, 0x87, 0xe1, + 0x25, 0xfb, 0x40, 0xcf, 0x99, 0xc3, 0xdb, 0x8e, 0x2b, 0xd5, 0xde, 0x5a, 0x74, 0x96, 0x84, 0x82, + 0xb1, 0x40, 0xac, 0xef, 0x43, 0x81, 0xff, 0xa7, 0x45, 0x1c, 0x51, 0x03, 0x35, 0x0c, 0x2f, 0xb9, + 0x50, 0x25, 0xcc, 0x3e, 0xeb, 0x0f, 0x60, 0x25, 0xf9, 0x0f, 0x98, 0x23, 0x9d, 0x12, 0x67, 0x70, + 0x2a, 0x1c, 0xac, 0x80, 0x25, 0xc5, 0x2c, 0xf9, 0xda, 0xb1, 0x65, 0xca, 0x5a, 0xc0, 0x82, 0xd0, + 0xff, 0x2d, 0x07, 0xb7, 0x66, 0x68, 0x46, 0x3a, 0xeb, 0x8b, 0x94, 0xb3, 0xbe, 0x23, 0x2d, 0x44, + 0x1e, 0xff, 0x22, 0xe5, 0xf1, 0xef, 0x10, 0x9c, 0x2d, 0x9b, 0x1b, 0x50, 0x24, 0x17, 0x4e, 0x48, + 0x6c, 0xa9, 0x2a, 0x49, 0x25, 0x96, 0x53, 0xfe, 0xba, 0xcb, 0xe9, 0x00, 0x36, 0x5b, 0x01, 0x31, + 0x43, 0x22, 0x43, 0x79, 0xe4, 0xff, 0xb7, 0xa0, 0x64, 0x0e, 0x87, 0x9e, 0x35, 0x31, 0xeb, 0x32, + 0xa7, 0x3b, 0x36, 0xaa, 0x43, 0xe9, 0xd4, 0xa3, 0xa1, 0x6b, 0x8e, 0x88, 0x0c, 0x5e, 0x31, 0xad, + 0x7f, 0xad, 0xc0, 0xd6, 0x15, 0x3c, 0x69, 0x85, 0x13, 0xa8, 0x3a, 0xd4, 0x1b, 0xf2, 0x3f, 0x68, + 0x24, 0x4e, 0x78, 0x3f, 0x9e, 0x6f, 0xab, 0xe9, 0x44, 0x18, 0xfc, 0xc0, 0xb7, 0xea, 0x24, 0x49, + 0xee, 0x71, 0x7c, 0x72, 0x5b, 0xae, 0xf4, 0x88, 0xd4, 0xff, 0x51, 0x81, 0x2d, 0xb9, 0xc3, 0x67, + 0xff, 0xa3, 0xd3, 0x22, 0xe7, 0xde, 0xb5, 0xc8, 0x7a, 0x0d, 0x6e, 0x5c, 0x95, 0x4b, 0xc6, 0xfc, + 0x7f, 0x2f, 0x00, 0x9a, 0x3e, 0x5d, 0xa2, 0xef, 0xc2, 0x0a, 0x25, 0xae, 0x6d, 0x88, 0xfd, 0x42, + 0x6c, 0x65, 0x25, 0x5c, 0x61, 0x3c, 0xb1, 0x71, 0x50, 0x16, 0x02, 0xc9, 0x85, 0x94, 0xb6, 0x84, + 0xf9, 0x37, 0x3a, 0x85, 0x95, 0x97, 0xd4, 0x88, 0xe7, 0xe6, 0x0e, 0x55, 0xcd, 0x1c, 0xd6, 0xa6, + 0xe5, 0x68, 0x3c, 0xea, 0xc5, 0xff, 0x0b, 0x57, 0x5e, 0xd2, 0x98, 0x40, 0xbf, 0x50, 0xe0, 0x66, + 0x94, 0x56, 0x4c, 0xd4, 0x37, 0xf2, 0x6c, 0x42, 0x6b, 0xf9, 0xdb, 0xea, 0x76, 0x75, 0xe7, 0xe8, + 0x1a, 0xfa, 0x9b, 0x62, 0x1e, 0x78, 0x36, 0xc1, 0x5b, 0xee, 0x0c, 0x2e, 0x45, 0x0d, 0xd8, 0x18, + 0x8d, 0x69, 0x68, 0x08, 0x2f, 0x30, 0x64, 0xa7, 0x5a, 0x81, 0xeb, 0x65, 0x9d, 0x35, 0xa5, 0x7c, + 0x15, 0x9d, 0xc1, 0xea, 0xc8, 0x1b, 0xbb, 0xa1, 0x61, 0xf1, 0xf3, 0x0f, 0xad, 0x15, 0xe7, 0x3a, + 0x18, 0xcf, 0xd0, 0xd2, 0x01, 0x83, 0x13, 0xa7, 0x29, 0x8a, 0x57, 0x46, 0x09, 0x8a, 0x19, 0x32, + 0x20, 0x23, 0x2f, 0x24, 0x06, 0x8b, 0x97, 0xb4, 0xb6, 0x2c, 0x0c, 0x29, 0x78, 0x2c, 0x34, 0x50, + 0xf4, 0x7b, 0x70, 0xc3, 0x76, 0xa8, 0x79, 0x32, 0x24, 0xc6, 0xd0, 0x1b, 0x18, 0x93, 0x34, 0xa7, + 0x56, 0xe2, 0x9d, 0x37, 0x65, 0xeb, 0xbe, 0x37, 0x68, 0xc5, 0x6d, 0xfa, 0x7d, 0xa8, 0x24, 0x8c, + 0x83, 0x4a, 0x90, 0xef, 0x1e, 0x76, 0xdb, 0xda, 0x12, 0x02, 0x28, 0xb6, 0xf6, 0xf0, 0xe1, 0x61, + 0x5f, 0x9c, 0x35, 0x3a, 0x07, 0xcd, 0xc7, 0x6d, 0x2d, 0xc7, 0xd8, 0xc7, 0xdd, 0x3f, 0x6c, 0x77, + 0xf6, 0x35, 0x55, 0x6f, 0xc3, 0x4a, 0x52, 0x64, 0x84, 0xa0, 0x7a, 0xdc, 0x7d, 0xda, 0x3d, 0x7c, + 0xd6, 0x35, 0x0e, 0x0e, 0x8f, 0xbb, 0x7d, 0x76, 0x62, 0xa9, 0x02, 0x34, 0xbb, 0xcf, 0x27, 0xf4, + 0x2a, 0x94, 0xbb, 0x87, 0x11, 0xa9, 0xd4, 0x73, 0x9a, 0xa2, 0xff, 0xa7, 0x0a, 0x9b, 0xb3, 0xac, + 0x87, 0x6c, 0xc8, 0x33, 0x4f, 0x90, 0x67, 0xc6, 0x77, 0xef, 0x08, 0x1c, 0x9d, 0x2d, 0x00, 0xdf, + 0x94, 0x9b, 0x44, 0x19, 0xf3, 0x6f, 0x64, 0x40, 0x71, 0x68, 0x9e, 0x90, 0x21, 0xad, 0xa9, 0xfc, + 0x56, 0xe5, 0xf1, 0x75, 0xe6, 0xde, 0xe7, 0x48, 0xe2, 0x4a, 0x45, 0xc2, 0xa2, 0x3e, 0x54, 0x58, + 0x18, 0xa4, 0x42, 0x75, 0x32, 0x32, 0xef, 0x64, 0x9c, 0x65, 0x6f, 0x32, 0x12, 0x27, 0x61, 0xea, + 0xf7, 0xa0, 0x92, 0x98, 0x6c, 0xc6, 0x8d, 0xc8, 0x66, 0xf2, 0x46, 0xa4, 0x9c, 0xbc, 0xde, 0x78, + 0x38, 0x6d, 0x03, 0xa6, 0x23, 0xe6, 0x10, 0x7b, 0x87, 0xbd, 0xbe, 0x38, 0x7b, 0x3e, 0xc6, 0x87, + 0xc7, 0x47, 0x9a, 0xc2, 0x98, 0xfd, 0x66, 0xef, 0xa9, 0x96, 0x8b, 0xfd, 0x45, 0xd5, 0x5b, 0x50, + 0x49, 0xc8, 0x95, 0x8a, 0xfb, 0x4a, 0x3a, 0xee, 0xb3, 0xc8, 0x6b, 0xda, 0x76, 0x40, 0x28, 0x95, + 0x72, 0x44, 0xa4, 0xfe, 0x02, 0xca, 0xbb, 0xdd, 0x9e, 0x84, 0xa8, 0xc1, 0x32, 0x25, 0x01, 0xfb, + 0xdf, 0xfc, 0x6e, 0xab, 0x8c, 0x23, 0x92, 0x81, 0x53, 0x62, 0x06, 0xd6, 0x29, 0xa1, 0x32, 0x5b, + 0x88, 0x69, 0x36, 0xca, 0xe3, 0x77, 0x44, 0xc2, 0x76, 0x65, 0x1c, 0x91, 0xfa, 0xff, 0x97, 0x00, + 0x26, 0xf7, 0x15, 0xa8, 0x0a, 0xb9, 0x38, 0x8a, 0xe7, 0x1c, 0x9b, 0xf9, 0x41, 0x62, 0x97, 0xe2, + 0xdf, 0x68, 0x07, 0xb6, 0x46, 0x74, 0xe0, 0x9b, 0xd6, 0x99, 0x21, 0xaf, 0x19, 0xc4, 0x62, 0xe7, + 0x11, 0x71, 0x05, 0x6f, 0xc8, 0x46, 0xb9, 0x96, 0x05, 0xee, 0x3e, 0xa8, 0xc4, 0x3d, 0xe7, 0xd1, + 0xab, 0xb2, 0x73, 0x7f, 0xee, 0x7b, 0x94, 0x46, 0xdb, 0x3d, 0x17, 0xbe, 0xc2, 0x60, 0x90, 0x01, + 0x60, 0x93, 0x73, 0xc7, 0x22, 0x06, 0x03, 0x2d, 0x70, 0xd0, 0xcf, 0xe7, 0x07, 0xdd, 0xe5, 0x18, + 0x31, 0x74, 0xd9, 0x8e, 0x68, 0xd4, 0x85, 0x72, 0x40, 0xa8, 0x37, 0x0e, 0x2c, 0x22, 0x42, 0x58, + 0xf6, 0xa3, 0x0e, 0x8e, 0xc6, 0xe1, 0x09, 0x04, 0xda, 0x85, 0x22, 0x8f, 0x5c, 0x2c, 0x46, 0xa9, + 0xdf, 0x7a, 0x29, 0x9b, 0x06, 0xe3, 0x91, 0x04, 0xcb, 0xb1, 0xe8, 0x31, 0x2c, 0x0b, 0x11, 0x69, + 0xad, 0xc4, 0x61, 0x3e, 0xca, 0x1a, 0x56, 0xf9, 0x28, 0x1c, 0x8d, 0x66, 0x56, 0x1d, 0x53, 0x12, + 0xd4, 0xca, 0xc2, 0xaa, 0xec, 0x1b, 0xbd, 0x07, 0x65, 0xb1, 0x8b, 0xdb, 0x4e, 0x50, 0x03, 0xe1, + 0x9c, 0x9c, 0xb1, 0xeb, 0x04, 0xe8, 0x7d, 0xa8, 0x88, 0x6c, 0xcd, 0xe0, 0x51, 0xa1, 0xc2, 0x9b, + 0x41, 0xb0, 0x8e, 0x58, 0x6c, 0x10, 0x1d, 0x48, 0x10, 0x88, 0x0e, 0x2b, 0x71, 0x07, 0x12, 0x04, + 0xbc, 0xc3, 0xef, 0xc0, 0x1a, 0xcf, 0x71, 0x07, 0x81, 0x37, 0xf6, 0x0d, 0xee, 0x53, 0xab, 0xbc, + 0xd3, 0x2a, 0x63, 0x3f, 0x66, 0xdc, 0x2e, 0x73, 0xae, 0x5b, 0x50, 0x7a, 0xe5, 0x9d, 0x88, 0x0e, + 0x55, 0xb1, 0x0e, 0x5e, 0x79, 0x27, 0x51, 0x53, 0x9c, 0x67, 0xac, 0xa5, 0xf3, 0x8c, 0xaf, 0xe0, + 0xc6, 0xf4, 0x86, 0xc9, 0xf3, 0x0d, 0xed, 0xfa, 0xf9, 0xc6, 0xa6, 0x3b, 0x2b, 0x0e, 0x7f, 0x01, + 0xaa, 0xed, 0xd2, 0xda, 0xfa, 0x5c, 0xce, 0x11, 0xaf, 0x63, 0xcc, 0x06, 0xa3, 0x2d, 0x28, 0xb2, + 0x3f, 0xeb, 0xd8, 0x35, 0x24, 0x42, 0xcf, 0x2b, 0xef, 0xa4, 0x63, 0xa3, 0xef, 0x40, 0x99, 0xfd, + 0x7f, 0xea, 0x9b, 0x16, 0xa9, 0x6d, 0xf0, 0x96, 0x09, 0x83, 0x19, 0xca, 0xf5, 0x6c, 0x22, 0x54, + 0xb4, 0x29, 0x0c, 0xc5, 0x18, 0x5c, 0x47, 0x37, 0x61, 0x99, 0x37, 0x3a, 0x76, 0x6d, 0x4b, 0x1c, + 0x25, 0x18, 0xd9, 0xb1, 0x91, 0x0e, 0xab, 0xbe, 0x19, 0x10, 0x37, 0x34, 0xe4, 0x8c, 0x37, 0x78, + 0x73, 0x45, 0x30, 0x9f, 0xb0, 0x79, 0xeb, 0x9f, 0x40, 0x29, 0x5a, 0x0c, 0xf3, 0x84, 0xc9, 0xfa, + 0x03, 0xa8, 0xa6, 0x97, 0xd2, 0x5c, 0x41, 0xf6, 0x9f, 0x73, 0x50, 0x8e, 0x17, 0x0d, 0x72, 0x61, + 0x83, 0x1b, 0x95, 0xe5, 0x9c, 0xc6, 0x64, 0x0d, 0x8a, 0x4c, 0xf7, 0xb3, 0x8c, 0x6a, 0x6e, 0x46, + 0x08, 0xf2, 0xc8, 0x2d, 0x17, 0x24, 0x8a, 0x91, 0x27, 0xf3, 0x7d, 0x09, 0x6b, 0x43, 0xc7, 0x1d, + 0x5f, 0x24, 0xe6, 0x12, 0x29, 0xea, 0xef, 0x67, 0x9c, 0x6b, 0x9f, 0x8d, 0x9e, 0xcc, 0x51, 0x1d, + 0xa6, 0x68, 0xb4, 0x07, 0x05, 0xdf, 0x0b, 0xc2, 0x68, 0xcf, 0xcc, 0xba, 0x9b, 0x1d, 0x79, 0x41, + 0x78, 0x60, 0xfa, 0x3e, 0x3b, 0x85, 0x09, 0x00, 0xfd, 0xeb, 0x1c, 0xdc, 0x98, 0xfd, 0xc7, 0x50, + 0x17, 0x54, 0xcb, 0x1f, 0x4b, 0x25, 0x3d, 0x98, 0x57, 0x49, 0x2d, 0x7f, 0x3c, 0x91, 0x9f, 0x01, + 0xa1, 0x67, 0x50, 0x1c, 0x91, 0x91, 0x17, 0x5c, 0x4a, 0x5d, 0x3c, 0x9c, 0x17, 0xf2, 0x80, 0x8f, + 0x9e, 0xa0, 0x4a, 0x38, 0x84, 0xa1, 0x24, 0x17, 0x13, 0x95, 0x61, 0x7b, 0xce, 0x7b, 0xb2, 0x08, + 0x12, 0xc7, 0x38, 0xfa, 0x27, 0xb0, 0x35, 0xf3, 0xaf, 0xa0, 0xdf, 0x02, 0xb0, 0xfc, 0xb1, 0xc1, + 0xdf, 0x31, 0x84, 0x07, 0xa9, 0xb8, 0x6c, 0xf9, 0xe3, 0x1e, 0x67, 0xe8, 0x2f, 0xa0, 0xf6, 0x26, + 0x79, 0xd9, 0x1a, 0x13, 0x12, 0x1b, 0xa3, 0x13, 0xae, 0x03, 0x15, 0x97, 0x04, 0xe3, 0xe0, 0x84, + 0x2d, 0xa5, 0xa8, 0xd1, 0xbc, 0x60, 0x1d, 0x54, 0xde, 0xa1, 0x22, 0x3b, 0x98, 0x17, 0x07, 0x27, + 0xfa, 0x2f, 0x73, 0xb0, 0x76, 0x45, 0x64, 0x76, 0x16, 0x15, 0x01, 0x38, 0x3a, 0xe5, 0x0b, 0x8a, + 0x45, 0x63, 0xcb, 0xb1, 0xa3, 0xfb, 0x61, 0xfe, 0xcd, 0xf7, 0x61, 0x5f, 0xde, 0xdd, 0xe6, 0x1c, + 0x9f, 0x2d, 0x9f, 0xd1, 0x89, 0x13, 0x52, 0x9e, 0x14, 0x15, 0xb0, 0x20, 0xd0, 0x73, 0xa8, 0x06, + 0x84, 0xef, 0xff, 0xb6, 0x21, 0xbc, 0xac, 0x30, 0x97, 0x97, 0x49, 0x09, 0x99, 0xb3, 0xe1, 0xd5, + 0x08, 0x89, 0x51, 0x14, 0x3d, 0x83, 0x55, 0xfb, 0xd2, 0x35, 0x47, 0x8e, 0x25, 0x91, 0x8b, 0x0b, + 0x23, 0xaf, 0x48, 0x20, 0x0e, 0xac, 0xdf, 0x83, 0x4a, 0xa2, 0x91, 0xfd, 0x31, 0x9e, 0xfd, 0x49, + 0x9d, 0x08, 0x22, 0x1d, 0x2d, 0x0a, 0x32, 0x5a, 0xe8, 0x27, 0x50, 0x49, 0xac, 0x8b, 0x79, 0x86, + 0x32, 0x7d, 0x86, 0x1e, 0xd7, 0x67, 0x01, 0xe7, 0x42, 0x8f, 0xc5, 0x49, 0x96, 0x79, 0x19, 0x8e, + 0xcf, 0x35, 0x5a, 0xc6, 0x45, 0x46, 0x76, 0x7c, 0xfd, 0xd7, 0x39, 0xa8, 0xa6, 0x97, 0x74, 0xe4, + 0x47, 0x3e, 0x09, 0x1c, 0xcf, 0x4e, 0xf8, 0xd1, 0x11, 0x67, 0x30, 0x5f, 0x61, 0xcd, 0x5f, 0x8d, + 0xbd, 0xd0, 0x8c, 0x7c, 0xc5, 0xf2, 0xc7, 0x7f, 0xc0, 0xe8, 0x2b, 0x3e, 0xa8, 0x5e, 0xf1, 0x41, + 0xf4, 0x21, 0x20, 0xe9, 0x4a, 0x43, 0x67, 0xe4, 0x84, 0xc6, 0xc9, 0x65, 0x48, 0x84, 0x8d, 0x55, + 0xac, 0x89, 0x96, 0x7d, 0xd6, 0xf0, 0x05, 0xe3, 0x33, 0xc7, 0xf3, 0xbc, 0x91, 0x41, 0x2d, 0x2f, + 0x20, 0x86, 0x69, 0xbf, 0xe2, 0xc7, 0x30, 0x15, 0x57, 0x3c, 0x6f, 0xd4, 0x63, 0xbc, 0xa6, 0xfd, + 0x8a, 0x6d, 0xc4, 0x96, 0x3f, 0xa6, 0x24, 0x34, 0xd8, 0x0f, 0xcf, 0x5d, 0xca, 0x18, 0x04, 0xab, + 0xe5, 0x8f, 0x29, 0xfa, 0x1e, 0xac, 0x46, 0x1d, 0xf8, 0x5e, 0x2c, 0x93, 0x80, 0x15, 0xd9, 0x85, + 0xf3, 0x90, 0x0e, 0x2b, 0x47, 0x24, 0xb0, 0x88, 0x1b, 0xf6, 0x1d, 0xeb, 0x8c, 0xf2, 0xc3, 0x92, + 0x82, 0x53, 0xbc, 0x27, 0xf9, 0xd2, 0xb2, 0x56, 0xc2, 0xd1, 0x6c, 0x23, 0x32, 0xa2, 0xfa, 0xbf, + 0x2a, 0x50, 0xe0, 0x29, 0x0b, 0x53, 0x0a, 0xdf, 0xee, 0x79, 0x36, 0x20, 0x53, 0x5d, 0xc6, 0xe0, + 0xb9, 0xc0, 0x7b, 0x50, 0xe6, 0xca, 0x4f, 0x9c, 0x30, 0x78, 0x1e, 0xcc, 0x1b, 0xeb, 0x50, 0x0a, + 0x88, 0x69, 0x7b, 0xee, 0x30, 0xba, 0xde, 0x8a, 0x69, 0xf4, 0xbb, 0xa0, 0xf9, 0x81, 0xe7, 0x9b, + 0x83, 0xc9, 0x89, 0x58, 0x9a, 0x6f, 0x2d, 0xc1, 0xe7, 0x29, 0xfa, 0xf7, 0x60, 0x95, 0x12, 0x11, + 0xd9, 0x85, 0x93, 0x14, 0xc4, 0xdf, 0x94, 0x4c, 0x7e, 0x22, 0xd0, 0xbf, 0x82, 0xa2, 0xd8, 0xb8, + 0xae, 0x21, 0xef, 0x47, 0x80, 0x84, 0x22, 0x99, 0x83, 0x8c, 0x1c, 0x4a, 0x65, 0x96, 0xcd, 0xdf, + 0x68, 0x45, 0xcb, 0xd1, 0xa4, 0x41, 0xff, 0x6f, 0x45, 0xe4, 0xdb, 0xe2, 0xf5, 0x8c, 0x25, 0xe6, + 0x6c, 0xd5, 0xb0, 0x03, 0xa9, 0xb8, 0xa6, 0x8b, 0x48, 0xd4, 0x81, 0xa2, 0x4c, 0xab, 0x73, 0x8b, + 0x3e, 0x3e, 0x4a, 0x80, 0xe8, 0xd2, 0x9e, 0xc8, 0x2b, 0x8b, 0x79, 0x2f, 0xed, 0x89, 0xb8, 0xb4, + 0x27, 0xec, 0xbc, 0x2d, 0x13, 0x7e, 0x01, 0x97, 0xe7, 0xf9, 0x7e, 0xc5, 0x8e, 0x5f, 0x46, 0x88, + 0xfe, 0xbf, 0x4a, 0x1c, 0xf7, 0xa2, 0x17, 0x0c, 0xf4, 0x25, 0x94, 0x58, 0x08, 0x31, 0x46, 0xa6, + 0x2f, 0xdf, 0xe3, 0x5b, 0x8b, 0x3d, 0x8e, 0x44, 0xbb, 0xa2, 0x48, 0xd7, 0x97, 0x7d, 0x41, 0xb1, + 0xf8, 0xc9, 0x8e, 0x4a, 0x51, 0xfc, 0x64, 0xdf, 0xe8, 0x03, 0xa8, 0x9a, 0xe3, 0xd0, 0x33, 0x4c, + 0xfb, 0x9c, 0x04, 0xa1, 0x43, 0x89, 0xf4, 0xa5, 0x55, 0xc6, 0x6d, 0x46, 0xcc, 0xfa, 0x7d, 0x58, + 0x49, 0x62, 0xbe, 0x2d, 0x6f, 0x29, 0x24, 0xf3, 0x96, 0x3f, 0x05, 0x98, 0xdc, 0x06, 0x32, 0x1f, + 0x21, 0x17, 0x4e, 0x68, 0x58, 0xd1, 0xd9, 0xbc, 0x80, 0x4b, 0x8c, 0xd1, 0x62, 0xce, 0x98, 0x7e, + 0xaa, 0x28, 0x44, 0x4f, 0x15, 0x2c, 0x3a, 0xb0, 0x05, 0x7d, 0xe6, 0x0c, 0x87, 0xf1, 0x0d, 0x65, + 0xd9, 0xf3, 0x46, 0x4f, 0x39, 0x43, 0xff, 0x26, 0x27, 0x7c, 0x45, 0x3c, 0x3a, 0x65, 0x3a, 0x9b, + 0xbd, 0x2b, 0x53, 0xdf, 0x03, 0xa0, 0xa1, 0x19, 0xb0, 0x24, 0xcc, 0x8c, 0xee, 0x48, 0xeb, 0x53, + 0x6f, 0x1d, 0xfd, 0xa8, 0x0a, 0x06, 0x97, 0x65, 0xef, 0x66, 0x88, 0x3e, 0x83, 0x15, 0xcb, 0x1b, + 0xf9, 0x43, 0x22, 0x07, 0x17, 0xde, 0x3a, 0xb8, 0x12, 0xf7, 0x6f, 0x86, 0x89, 0x9b, 0xd9, 0xe2, + 0x75, 0x6f, 0x66, 0x7f, 0xad, 0x88, 0xb7, 0xb3, 0xe4, 0xd3, 0x1d, 0x1a, 0xcc, 0xa8, 0x0f, 0x79, + 0xbc, 0xe0, 0x3b, 0xe0, 0xb7, 0x15, 0x87, 0xd4, 0x3f, 0xcb, 0x52, 0x8d, 0xf1, 0xe6, 0xb4, 0xf8, + 0x3f, 0x54, 0x28, 0xc7, 0xcf, 0x66, 0x53, 0xb6, 0xff, 0x14, 0xca, 0x71, 0x09, 0x92, 0x0c, 0x10, + 0xdf, 0x6a, 0x9e, 0xb8, 0x33, 0x7a, 0x09, 0xc8, 0x1c, 0x0c, 0xe2, 0x74, 0xd7, 0x18, 0x53, 0x73, + 0x10, 0x3d, 0x5a, 0x7e, 0x3a, 0x87, 0x1e, 0xa2, 0xfd, 0xf1, 0x98, 0x8d, 0xc7, 0x9a, 0x39, 0x18, + 0xa4, 0x38, 0xe8, 0xcf, 0x60, 0x2b, 0x3d, 0x87, 0x71, 0x72, 0x69, 0xf8, 0x8e, 0x2d, 0xef, 0x00, + 0xf6, 0xe6, 0x7d, 0x39, 0x6c, 0xa4, 0xe0, 0xbf, 0xb8, 0x3c, 0x72, 0x6c, 0xa1, 0x73, 0x14, 0x4c, + 0x35, 0xd4, 0xff, 0x02, 0x6e, 0xbe, 0xa1, 0xfb, 0x0c, 0x1b, 0x74, 0xd3, 0x15, 0x31, 0x8b, 0x2b, + 0x21, 0x61, 0xbd, 0x5f, 0x29, 0xe2, 0x81, 0x33, 0xad, 0x93, 0x66, 0x32, 0x4f, 0xbf, 0x93, 0x71, + 0x9e, 0xd6, 0xd1, 0xb1, 0x80, 0xe7, 0xa9, 0xf9, 0x93, 0x2b, 0xa9, 0x79, 0xd6, 0x84, 0x4c, 0x64, + 0xb8, 0x02, 0x48, 0x22, 0xe8, 0xff, 0xa2, 0x42, 0x29, 0x42, 0xe7, 0x27, 0xf8, 0x4b, 0x1a, 0x92, + 0x91, 0x11, 0x5f, 0x2f, 0x2a, 0x18, 0x04, 0x8b, 0xef, 0xa8, 0xef, 0x41, 0x79, 0x4c, 0x49, 0x20, + 0x9a, 0x73, 0xbc, 0xb9, 0xc4, 0x18, 0xbc, 0xf1, 0x7d, 0xa8, 0x84, 0x5e, 0x68, 0x0e, 0x8d, 0x90, + 0xe7, 0x0b, 0xaa, 0x18, 0xcd, 0x59, 0x3c, 0x5b, 0x40, 0x3f, 0x80, 0xf5, 0xf0, 0x34, 0xf0, 0xc2, + 0x70, 0xc8, 0x72, 0x55, 0x9e, 0x39, 0x89, 0x44, 0x27, 0x8f, 0xb5, 0xb8, 0x41, 0x64, 0x54, 0x94, + 0x45, 0xef, 0x49, 0x67, 0xe6, 0xba, 0x3c, 0x88, 0xe4, 0xf1, 0x6a, 0xcc, 0x65, 0xae, 0xcd, 0x36, + 0x4f, 0x5f, 0x64, 0x24, 0x3c, 0x56, 0x28, 0x38, 0x22, 0x91, 0x01, 0x6b, 0x23, 0x62, 0xd2, 0x71, + 0x40, 0x6c, 0xe3, 0xa5, 0x43, 0x86, 0xb6, 0xb8, 0x78, 0xa9, 0x66, 0x3e, 0x6e, 0x44, 0x6a, 0x69, + 0x3c, 0xe2, 0xa3, 0x71, 0x35, 0x82, 0x13, 0x34, 0xcb, 0x1c, 0xc4, 0x17, 0x5a, 0x83, 0x4a, 0xef, + 0x79, 0xaf, 0xdf, 0x3e, 0x30, 0x0e, 0x0e, 0x77, 0xdb, 0xb2, 0xe8, 0xa9, 0xd7, 0xc6, 0x82, 0x54, + 0x58, 0x7b, 0xff, 0xb0, 0xdf, 0xdc, 0x37, 0xfa, 0x9d, 0xd6, 0xd3, 0x9e, 0x96, 0x43, 0x5b, 0xb0, + 0xde, 0xdf, 0xc3, 0x87, 0xfd, 0xfe, 0x7e, 0x7b, 0xd7, 0x38, 0x6a, 0xe3, 0xce, 0xe1, 0x6e, 0x4f, + 0x53, 0x11, 0x82, 0xea, 0x84, 0xdd, 0xef, 0x1c, 0xb4, 0xb5, 0x3c, 0xaa, 0xc0, 0xf2, 0x51, 0x1b, + 0xb7, 0xda, 0xdd, 0xbe, 0x56, 0xd0, 0x7f, 0xa9, 0x42, 0x25, 0x61, 0x45, 0xe6, 0xc8, 0x01, 0x15, + 0xe7, 0x9a, 0x3c, 0x66, 0x9f, 0xfc, 0x91, 0xd6, 0xb4, 0x4e, 0x85, 0x75, 0xf2, 0x58, 0x10, 0xfc, + 0x2c, 0x63, 0x5e, 0x24, 0xd6, 0x79, 0x1e, 0x97, 0x46, 0xe6, 0x85, 0x00, 0xf9, 0x2e, 0xac, 0x9c, + 0x91, 0xc0, 0x25, 0x43, 0xd9, 0x2e, 0x2c, 0x52, 0x11, 0x3c, 0xd1, 0x65, 0x1b, 0x34, 0xd9, 0x65, + 0x02, 0x23, 0xcc, 0x51, 0x15, 0xfc, 0x83, 0x08, 0x6c, 0x13, 0x0a, 0xa2, 0x79, 0x59, 0xcc, 0xcf, + 0x09, 0xb6, 0x4d, 0xd1, 0xd7, 0xa6, 0xcf, 0x73, 0xc8, 0x3c, 0xe6, 0xdf, 0xe8, 0x64, 0xda, 0x3e, + 0x45, 0x6e, 0x9f, 0x7b, 0xf3, 0xbb, 0xf3, 0x9b, 0x4c, 0x74, 0x1a, 0x9b, 0x68, 0x19, 0x54, 0x1c, + 0x55, 0x0a, 0xb5, 0x9a, 0xad, 0x3d, 0x66, 0x96, 0x55, 0x28, 0x1f, 0x34, 0x7f, 0x6a, 0x1c, 0xf7, + 0xc4, 0x0d, 0xbe, 0x06, 0x2b, 0x4f, 0xdb, 0xb8, 0xdb, 0xde, 0x97, 0x1c, 0x15, 0x6d, 0x82, 0x26, + 0x39, 0x93, 0x7e, 0x79, 0x86, 0x20, 0x3e, 0x0b, 0xa8, 0x04, 0xf9, 0xde, 0xb3, 0xe6, 0x91, 0x56, + 0xd4, 0xff, 0x27, 0x07, 0x6b, 0x62, 0x5b, 0x88, 0x6b, 0x1a, 0xde, 0xfc, 0xa6, 0x9b, 0xbc, 0xc5, + 0xca, 0xa5, 0x6f, 0xb1, 0xa2, 0x24, 0x94, 0xef, 0xea, 0xea, 0x24, 0x09, 0xe5, 0x37, 0x3b, 0xa9, + 0x88, 0x9f, 0x9f, 0x27, 0xe2, 0xd7, 0x60, 0x79, 0x44, 0x68, 0x6c, 0xb7, 0x32, 0x8e, 0x48, 0xe4, + 0x40, 0xc5, 0x74, 0x5d, 0x2f, 0x34, 0xc5, 0xd5, 0x70, 0x71, 0xae, 0xcd, 0xf0, 0xca, 0x3f, 0x6e, + 0x34, 0x27, 0x48, 0x22, 0x30, 0x27, 0xb1, 0xeb, 0x3f, 0x01, 0xed, 0x6a, 0x87, 0x79, 0xb6, 0xc3, + 0xef, 0xff, 0x70, 0xb2, 0x1b, 0x12, 0xb6, 0x2e, 0xe4, 0x9b, 0x8a, 0xb6, 0xc4, 0x08, 0x7c, 0xdc, + 0xed, 0x76, 0xba, 0x8f, 0x35, 0x05, 0x01, 0x14, 0xdb, 0x3f, 0xed, 0xf4, 0xdb, 0xbb, 0x5a, 0x6e, + 0xe7, 0x57, 0xeb, 0x50, 0x14, 0x42, 0xa2, 0xaf, 0x65, 0x26, 0x90, 0xac, 0x97, 0x45, 0x3f, 0x99, + 0x3b, 0xa3, 0x4e, 0xd5, 0xe0, 0xd6, 0x1f, 0x2e, 0x3c, 0x5e, 0xbe, 0x4f, 0x2e, 0xa1, 0xbf, 0x51, + 0x60, 0x25, 0xf5, 0x36, 0x99, 0xf5, 0x6a, 0x7c, 0x46, 0x79, 0x6e, 0xfd, 0xc7, 0x0b, 0x8d, 0x8d, + 0x65, 0xf9, 0x85, 0x02, 0x95, 0x44, 0x61, 0x2a, 0xba, 0xb7, 0x48, 0x31, 0xab, 0x90, 0xe4, 0xfe, + 0xe2, 0x75, 0xb0, 0xfa, 0xd2, 0xc7, 0x0a, 0xfa, 0x6b, 0x05, 0x2a, 0x89, 0x12, 0xcd, 0xcc, 0xa2, + 0x4c, 0x17, 0x94, 0x66, 0x16, 0x65, 0x56, 0x45, 0xe8, 0x12, 0xfa, 0x4b, 0x05, 0xca, 0x71, 0xb9, + 0x25, 0xba, 0x3b, 0x7f, 0x81, 0xa6, 0x10, 0xe2, 0xd3, 0x45, 0x2b, 0x3b, 0xf5, 0x25, 0xf4, 0xe7, + 0x50, 0x8a, 0x6a, 0x13, 0x51, 0xd6, 0xdd, 0xeb, 0x4a, 0xe1, 0x63, 0xfd, 0xee, 0xdc, 0xe3, 0x92, + 0xd3, 0x47, 0x05, 0x83, 0x99, 0xa7, 0xbf, 0x52, 0xda, 0x58, 0xbf, 0x3b, 0xf7, 0xb8, 0x78, 0x7a, + 0xe6, 0x09, 0x89, 0xba, 0xc2, 0xcc, 0x9e, 0x30, 0x5d, 0xd0, 0x98, 0xd9, 0x13, 0x66, 0x95, 0x31, + 0x0a, 0x41, 0x12, 0x95, 0x89, 0x99, 0x05, 0x99, 0xae, 0x7e, 0xcc, 0x2c, 0xc8, 0x8c, 0x42, 0x48, + 0x7d, 0x09, 0xfd, 0x5c, 0x49, 0x9e, 0x0b, 0xee, 0xce, 0x5d, 0x80, 0x37, 0xa7, 0x4b, 0x4e, 0x95, + 0x00, 0xf2, 0x05, 0xfa, 0x73, 0x79, 0x8b, 0x21, 0xea, 0xf7, 0xd0, 0x3c, 0x60, 0xa9, 0x92, 0xbf, + 0xfa, 0x27, 0x8b, 0x6d, 0x36, 0x5c, 0x88, 0xbf, 0x52, 0x00, 0x26, 0x95, 0x7e, 0x99, 0x85, 0x98, + 0x2a, 0x31, 0xac, 0xdf, 0x5b, 0x60, 0x64, 0x72, 0x81, 0x44, 0x95, 0x48, 0x99, 0x17, 0xc8, 0x95, + 0x4a, 0xc4, 0xcc, 0x0b, 0xe4, 0x6a, 0x15, 0xa1, 0xbe, 0x84, 0xfe, 0x49, 0x81, 0xf5, 0xa9, 0x4a, + 0x28, 0xf4, 0xf0, 0x9a, 0xc5, 0x70, 0xf5, 0xcf, 0x17, 0x07, 0x88, 0x44, 0xdb, 0x56, 0x3e, 0x56, + 0xd0, 0xdf, 0x2a, 0xb0, 0x9a, 0xae, 0x10, 0xc9, 0xbc, 0x4b, 0xcd, 0xa8, 0xa9, 0xaa, 0x3f, 0x58, + 0x6c, 0x70, 0xac, 0xad, 0xbf, 0x57, 0xa0, 0x9a, 0x2e, 0x16, 0x42, 0x0f, 0xe6, 0x0b, 0x0b, 0x57, + 0x04, 0xfa, 0x6c, 0xc1, 0xd1, 0x91, 0x44, 0x5f, 0x2c, 0xff, 0x51, 0x41, 0x64, 0x6f, 0x45, 0xfe, + 0xf3, 0xa3, 0xdf, 0x04, 0x00, 0x00, 0xff, 0xff, 0x73, 0xc1, 0x2f, 0xf7, 0xd6, 0x34, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/plugins/drivers/proto/driver.proto b/plugins/drivers/proto/driver.proto index 280933665..13fd22518 100644 --- a/plugins/drivers/proto/driver.proto +++ b/plugins/drivers/proto/driver.proto @@ -371,6 +371,7 @@ message DriverCapabilities { NONE = 0; CHROOT = 1; IMAGE = 2; + UNVEIL= 3; } // FsIsolation indicates what kind of filesystem isolation a driver supports. FSIsolation fs_isolation = 3; diff --git a/plugins/drivers/server.go b/plugins/drivers/server.go index cfd247687..371c06c0a 100644 --- a/plugins/drivers/server.go +++ b/plugins/drivers/server.go @@ -15,6 +15,7 @@ import ( "google.golang.org/grpc/status" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/drivers/proto" dstructs "github.com/hashicorp/nomad/plugins/shared/structs" sproto "github.com/hashicorp/nomad/plugins/shared/structs/proto" @@ -53,12 +54,14 @@ func (b *driverPluginServer) Capabilities(ctx context.Context, req *proto.Capabi } switch caps.FSIsolation { - case FSIsolationNone: + case fsisolation.None: resp.Capabilities.FsIsolation = proto.DriverCapabilities_NONE - case FSIsolationChroot: + case fsisolation.Chroot: resp.Capabilities.FsIsolation = proto.DriverCapabilities_CHROOT - case FSIsolationImage: + case fsisolation.Image: resp.Capabilities.FsIsolation = proto.DriverCapabilities_IMAGE + case fsisolation.Unveil: + resp.Capabilities.FsIsolation = proto.DriverCapabilities_UNVEIL default: resp.Capabilities.FsIsolation = proto.DriverCapabilities_NONE } diff --git a/plugins/drivers/testutils/testing.go b/plugins/drivers/testutils/testing.go index f4dfdc6c9..a89e7b7d3 100644 --- a/plugins/drivers/testutils/testing.go +++ b/plugins/drivers/testutils/testing.go @@ -82,7 +82,7 @@ func (h *DriverHarness) MkAllocDir(t *drivers.TaskConfig, enableLogs bool) func( dir, err := os.MkdirTemp("", "nomad_driver_harness-") must.NoError(h.t, err) - allocDir := allocdir.NewAllocDir(h.logger, dir, t.AllocID) + allocDir := allocdir.NewAllocDir(h.logger, dir, dir, t.AllocID) must.NoError(h.t, allocDir.Build()) t.AllocDir = allocDir.AllocDir @@ -94,7 +94,7 @@ func (h *DriverHarness) MkAllocDir(t *drivers.TaskConfig, enableLogs bool) func( fsi := caps.FSIsolation h.logger.Trace("FS isolation", "fsi", fsi) - must.NoError(h.t, taskDir.Build(fsi == drivers.FSIsolationChroot, ci.TinyChroot)) + must.NoError(h.t, taskDir.Build(fsi, ci.TinyChroot, t.User)) task := &structs.Task{ Name: t.Name,