From 2f16eb9640bc0ad571c3fd58b8129f356d377117 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 30 Sep 2019 11:50:22 -0400 Subject: [PATCH] executor: run exec commands in netns if set --- drivers/shared/executor/exec_utils.go | 3 ++ drivers/shared/executor/executor.go | 13 +++++---- drivers/shared/executor/executor_basic.go | 5 ++++ drivers/shared/executor/executor_linux.go | 28 +++++++------------ .../executor/executor_universal_linux.go | 11 ++++---- 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/drivers/shared/executor/exec_utils.go b/drivers/shared/executor/exec_utils.go index ed3dd2ba8..eefa475ab 100644 --- a/drivers/shared/executor/exec_utils.go +++ b/drivers/shared/executor/exec_utils.go @@ -18,6 +18,9 @@ import ( type execHelper struct { logger hclog.Logger + // networkSpec is the network isolation spec, set when platforms support network namespaces + networkSpec *drivers.NetworkIsolationSpec + // newTerminal function creates a tty appropriate for the command // The returned pty end of tty function is to be called after process start. newTerminal func() (pty func() (*os.File, error), tty *os.File, err error) diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index f2362fc88..ddc698206 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -309,7 +309,7 @@ func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) e.childCmd.Env = e.commandCfg.Env // Start the process - if err = e.start(command); err != nil { + if err = wrapNetns(e.childCmd.Start, command.NetworkIsolation); err != nil { return nil, fmt.Errorf("failed to start command path=%q --- args=%q: %v", path, e.childCmd.Args, err) } @@ -322,13 +322,14 @@ func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) func (e *UniversalExecutor) Exec(deadline time.Time, name string, args []string) ([]byte, int, error) { ctx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() - return ExecScript(ctx, e.childCmd.Dir, e.commandCfg.Env, e.childCmd.SysProcAttr, name, args) + return ExecScript(ctx, e.childCmd.Dir, e.commandCfg.Env, e.childCmd.SysProcAttr, e.commandCfg.NetworkIsolation, name, args) } // ExecScript executes cmd with args and returns the output, exit code, and // error. Output is truncated to drivers/shared/structs.CheckBufSize func ExecScript(ctx context.Context, dir string, env []string, attrs *syscall.SysProcAttr, - name string, args []string) ([]byte, int, error) { + netSpec *drivers.NetworkIsolationSpec, name string, args []string) ([]byte, int, error) { + cmd := exec.CommandContext(ctx, name, args...) // Copy runtime environment from the main command @@ -341,7 +342,7 @@ func ExecScript(ctx context.Context, dir string, env []string, attrs *syscall.Sy cmd.Stdout = buf cmd.Stderr = buf - if err := cmd.Run(); err != nil { + if err := wrapNetns(cmd.Run, netSpec); err != nil { exitErr, ok := err.(*exec.ExitError) if !ok { // Non-exit error, return it and let the caller treat @@ -399,7 +400,9 @@ func (e *UniversalExecutor) ExecStreaming(ctx context.Context, command []string, cmd.Stderr = stderr return nil }, - processStart: cmd.Start, + processStart: func() error { + return wrapNetns(cmd.Start, e.commandCfg.NetworkIsolation) + }, processWait: func() (*os.ProcessState, error) { err := cmd.Wait() return cmd.ProcessState, err diff --git a/drivers/shared/executor/executor_basic.go b/drivers/shared/executor/executor_basic.go index 37726cb3a..04d2e65bb 100644 --- a/drivers/shared/executor/executor_basic.go +++ b/drivers/shared/executor/executor_basic.go @@ -4,6 +4,7 @@ package executor import ( hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/plugins/drivers" ) func NewExecutorWithIsolation(logger hclog.Logger) Executor { @@ -23,3 +24,7 @@ func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) { func (e *UniversalExecutor) start(command *ExecCommand) error { return e.childCmd.Start() } + +func wrapNetns(f func() error, _ *drivers.NetworkIsolationSpec) error { + return f() +} diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index dfe00b0e8..df5c5a6b3 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -15,7 +15,6 @@ import ( "time" "github.com/armon/circbuf" - "github.com/containernetworking/plugins/pkg/ns" "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" @@ -184,23 +183,9 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro l.systemCpuStats = stats.NewCpuStats() // Starts the task - if command.NetworkIsolation != nil && command.NetworkIsolation.Path != "" { - netns, err := ns.GetNS(command.NetworkIsolation.Path) - if err != nil { - return nil, fmt.Errorf("failed to get ns %s: %v", command.NetworkIsolation.Path, err) - } - - // Start the container in the network namespace - err = netns.Do(func(ns.NetNS) error { return container.Run(process) }) - if err != nil { - container.Destroy() - return nil, err - } - } else { - if err := container.Run(process); err != nil { - container.Destroy() - return nil, err - } + if err := container.Run(process); err != nil { + container.Destroy() + return nil, err } pid, err := process.Pid() @@ -622,6 +607,13 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error { {Type: lconfigs.NEWNS}, } + if command.NetworkIsolation != nil { + cfg.Namespaces = append(cfg.Namespaces, lconfigs.Namespace{ + Type: lconfigs.NEWNET, + Path: command.NetworkIsolation.Path, + }) + } + // paths to mask using a bind mount to /dev/null to prevent reading cfg.MaskPaths = []string{ "/proc/kcore", diff --git a/drivers/shared/executor/executor_universal_linux.go b/drivers/shared/executor/executor_universal_linux.go index 3db3b0a87..9b0353091 100644 --- a/drivers/shared/executor/executor_universal_linux.go +++ b/drivers/shared/executor/executor_universal_linux.go @@ -10,6 +10,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/plugins/drivers" "github.com/opencontainers/runc/libcontainer/cgroups" cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" lconfigs "github.com/opencontainers/runc/libcontainer/configs" @@ -173,19 +174,19 @@ func DestroyCgroup(groups *lconfigs.Cgroup, executorPid int) error { return mErrs.ErrorOrNil() } -func (e *UniversalExecutor) start(command *ExecCommand) error { - if command.NetworkIsolation != nil && command.NetworkIsolation.Path != "" { +func wrapNetns(f func() error, spec *drivers.NetworkIsolationSpec) error { + if spec != nil && spec.Path != "" { // Get a handle to the target network namespace - netns, err := ns.GetNS(command.NetworkIsolation.Path) + netns, err := ns.GetNS(spec.Path) if err != nil { return err } // Start the container in the network namespace return netns.Do(func(ns.NetNS) error { - return e.childCmd.Start() + return f() }) } - return e.childCmd.Start() + return f() }