From 13b582fcfb717fa201fcf592de3482fe63b4019b Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 6 Dec 2018 21:13:45 -0500 Subject: [PATCH] executor: merge plugin shim with executor package --- command/executor_plugin.go | 2 +- drivers/exec/driver.go | 7 +- drivers/java/driver.go | 8 +- drivers/qemu/driver.go | 8 +- drivers/rawexec/driver.go | 8 +- drivers/rkt/driver.go | 8 +- .../shared}/executor/client.go | 15 +- .../shared}/executor/executor_plugin.go | 7 +- .../executor/executor_plugin_pre_0_9_0.go | 0 .../shared}/executor/plugins.go | 0 .../shared}/executor/proto/executor.pb.go | 0 .../shared}/executor/proto/executor.proto | 0 .../shared}/executor/server.go | 7 +- drivers/shared/executor/utils.go | 157 ++++++++++++++++++ drivers/shared/executor/utils_unix.go | 18 ++ drivers/shared/executor/utils_windows.go | 6 + plugins/drivers/utils/utils.go | 107 ------------ plugins/drivers/utils/utils_unix.go | 13 -- plugins/drivers/utils/utils_windows.go | 7 - plugins/executor/utils.go | 62 ------- 20 files changed, 210 insertions(+), 230 deletions(-) rename {plugins => drivers/shared}/executor/client.go (83%) rename {plugins => drivers/shared}/executor/executor_plugin.go (68%) rename {plugins => drivers/shared}/executor/executor_plugin_pre_0_9_0.go (100%) rename {plugins => drivers/shared}/executor/plugins.go (100%) rename {plugins => drivers/shared}/executor/proto/executor.pb.go (100%) rename {plugins => drivers/shared}/executor/proto/executor.proto (100%) rename {plugins => drivers/shared}/executor/server.go (94%) create mode 100644 drivers/shared/executor/utils.go create mode 100644 drivers/shared/executor/utils_unix.go create mode 100644 drivers/shared/executor/utils_windows.go delete mode 100644 plugins/executor/utils.go diff --git a/command/executor_plugin.go b/command/executor_plugin.go index 666d1fb47..da75450e0 100644 --- a/command/executor_plugin.go +++ b/command/executor_plugin.go @@ -8,8 +8,8 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/plugins/base" - "github.com/hashicorp/nomad/plugins/executor" ) type ExecutorPluginCommand struct { diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 9353bb695..da9174829 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" - pexecutor "github.com/hashicorp/nomad/plugins/executor" "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" @@ -249,7 +248,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { Reattach: plugRC, } - exec, pluginClient, err := utils.CreateExecutorWithConfig(pluginConfig, os.Stderr) + exec, pluginClient, err := executor.CreateExecutorWithConfig(pluginConfig, os.Stderr) if err != nil { d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to reattach to executor: %v", err) @@ -286,14 +285,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru handle.Config = cfg pluginLogFile := filepath.Join(cfg.TaskDir().Dir, "executor.out") - executorConfig := &pexecutor.ExecutorConfig{ + executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", FSIsolation: true, } // TODO: best way to pass port ranges in from client config - exec, pluginClient, err := utils.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) + exec, pluginClient, err := executor.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create executor: %v", err) } diff --git a/drivers/java/driver.go b/drivers/java/driver.go index 9fbf442c8..18250bad5 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -19,8 +19,6 @@ import ( "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/drivers/utils" - pexecutor "github.com/hashicorp/nomad/plugins/executor" "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" @@ -271,7 +269,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { Reattach: plugRC, } - execImpl, pluginClient, err := utils.CreateExecutorWithConfig(pluginConfig, os.Stderr) + execImpl, pluginClient, err := executor.CreateExecutorWithConfig(pluginConfig, os.Stderr) if err != nil { d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to reattach to executor: %v", err) @@ -320,12 +318,12 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru handle.Config = cfg pluginLogFile := filepath.Join(cfg.TaskDir().Dir, "executor.out") - executorConfig := &pexecutor.ExecutorConfig{ + executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", } - exec, pluginClient, err := utils.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) + exec, pluginClient, err := executor.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create executor: %v", err) } diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 9c498b9ce..8b07ce298 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -21,8 +21,6 @@ import ( "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/drivers/utils" - pexecutor "github.com/hashicorp/nomad/plugins/executor" "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" @@ -270,7 +268,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { Reattach: plugRC, } - execImpl, pluginClient, err := utils.CreateExecutorWithConfig(pluginConfig, os.Stderr) + execImpl, pluginClient, err := executor.CreateExecutorWithConfig(pluginConfig, os.Stderr) if err != nil { d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to reattach to executor: %v", err) @@ -414,12 +412,12 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru d.logger.Debug("starting QemuVM command ", "args", strings.Join(args, " ")) pluginLogFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%s-executor.out", cfg.Name)) - executorConfig := &pexecutor.ExecutorConfig{ + executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", } - execImpl, pluginClient, err := utils.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) + execImpl, pluginClient, err := executor.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) if err != nil { return nil, nil, err } diff --git a/drivers/rawexec/driver.go b/drivers/rawexec/driver.go index c5adb62dd..34bffe1d3 100644 --- a/drivers/rawexec/driver.go +++ b/drivers/rawexec/driver.go @@ -18,8 +18,6 @@ import ( "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/drivers/utils" - pexecutor "github.com/hashicorp/nomad/plugins/executor" "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" @@ -276,7 +274,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { } // Create client for reattached executor - exec, pluginClient, err := utils.CreateExecutorWithConfig(pluginConfig, os.Stderr) + exec, pluginClient, err := executor.CreateExecutorWithConfig(pluginConfig, os.Stderr) if err != nil { d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to reattach to executor: %v", err) @@ -313,12 +311,12 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru handle.Config = cfg pluginLogFile := filepath.Join(cfg.TaskDir().Dir, "executor.out") - executorConfig := &pexecutor.ExecutorConfig{ + executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", } - exec, pluginClient, err := utils.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) + exec, pluginClient, err := executor.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create executor: %v", err) } diff --git a/drivers/rkt/driver.go b/drivers/rkt/driver.go index 99ffea850..a022db9fb 100644 --- a/drivers/rkt/driver.go +++ b/drivers/rkt/driver.go @@ -31,8 +31,6 @@ import ( "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/drivers/utils" - pexecutor "github.com/hashicorp/nomad/plugins/executor" "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" @@ -344,7 +342,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { Reattach: plugRC, } - execImpl, pluginClient, err := utils.CreateExecutorWithConfig(pluginConfig, os.Stderr) + execImpl, pluginClient, err := executor.CreateExecutorWithConfig(pluginConfig, os.Stderr) if err != nil { d.logger.Error("failed to reattach to executor", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to reattach to executor: %v", err) @@ -604,12 +602,12 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru } pluginLogFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%s-executor.out", cfg.Name)) - executorConfig := &pexecutor.ExecutorConfig{ + executorConfig := &executor.ExecutorConfig{ LogFile: pluginLogFile, LogLevel: "debug", } - execImpl, pluginClient, err := utils.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) + execImpl, pluginClient, err := executor.CreateExecutor(os.Stderr, hclog.Debug, d.nomadConfig, executorConfig) if err != nil { return nil, nil, err } diff --git a/plugins/executor/client.go b/drivers/shared/executor/client.go similarity index 83% rename from plugins/executor/client.go rename to drivers/shared/executor/client.go index 089966c43..2a6700b24 100644 --- a/plugins/executor/client.go +++ b/drivers/shared/executor/client.go @@ -8,12 +8,11 @@ import ( "github.com/LK4D4/joincontext" "github.com/golang/protobuf/ptypes" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/drivers/shared/executor" + "github.com/hashicorp/nomad/drivers/shared/executor/proto" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/executor/proto" ) -var _ executor.Executor = (*grpcExecutorClient)(nil) +var _ Executor = (*grpcExecutorClient)(nil) type grpcExecutorClient struct { client proto.ExecutorClient @@ -22,7 +21,7 @@ type grpcExecutorClient struct { doneCtx context.Context } -func (c *grpcExecutorClient) Launch(cmd *executor.ExecCommand) (*executor.ProcessState, error) { +func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) { ctx := context.Background() req := &proto.LaunchRequest{ Cmd: cmd.Cmd, @@ -48,7 +47,7 @@ func (c *grpcExecutorClient) Launch(cmd *executor.ExecCommand) (*executor.Proces return ps, nil } -func (c *grpcExecutorClient) Wait(ctx context.Context) (*executor.ProcessState, error) { +func (c *grpcExecutorClient) Wait(ctx context.Context) (*ProcessState, error) { // Join the passed context and the shutdown context ctx, _ = joincontext.Join(ctx, c.doneCtx) @@ -78,7 +77,7 @@ func (c *grpcExecutorClient) Shutdown(signal string, gracePeriod time.Duration) return nil } -func (c *grpcExecutorClient) UpdateResources(r *executor.Resources) error { +func (c *grpcExecutorClient) UpdateResources(r *Resources) error { ctx := context.Background() req := &proto.UpdateResourcesRequest{Resources: resourcesToProto(r)} if _, err := c.client.UpdateResources(ctx, req); err != nil { @@ -88,13 +87,13 @@ func (c *grpcExecutorClient) UpdateResources(r *executor.Resources) error { return nil } -func (c *grpcExecutorClient) Version() (*executor.ExecutorVersion, error) { +func (c *grpcExecutorClient) Version() (*ExecutorVersion, error) { ctx := context.Background() resp, err := c.client.Version(ctx, &proto.VersionRequest{}) if err != nil { return nil, err } - return &executor.ExecutorVersion{Version: resp.Version}, nil + return &ExecutorVersion{Version: resp.Version}, nil } func (c *grpcExecutorClient) Stats() (*cstructs.TaskResourceUsage, error) { diff --git a/plugins/executor/executor_plugin.go b/drivers/shared/executor/executor_plugin.go similarity index 68% rename from plugins/executor/executor_plugin.go rename to drivers/shared/executor/executor_plugin.go index 905dc6a30..a7a85d5a3 100644 --- a/plugins/executor/executor_plugin.go +++ b/drivers/shared/executor/executor_plugin.go @@ -5,8 +5,7 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-plugin" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/executor/proto" + "github.com/hashicorp/nomad/drivers/shared/executor/proto" "google.golang.org/grpc" ) @@ -19,9 +18,9 @@ type ExecutorPlugin struct { func (p *ExecutorPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { if p.fsIsolation { - proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: executor.NewExecutorWithIsolation(p.logger)}) + proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: NewExecutorWithIsolation(p.logger)}) } else { - proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: executor.NewExecutor(p.logger)}) + proto.RegisterExecutorServer(s, &grpcExecutorServer{impl: NewExecutor(p.logger)}) } return nil } diff --git a/plugins/executor/executor_plugin_pre_0_9_0.go b/drivers/shared/executor/executor_plugin_pre_0_9_0.go similarity index 100% rename from plugins/executor/executor_plugin_pre_0_9_0.go rename to drivers/shared/executor/executor_plugin_pre_0_9_0.go diff --git a/plugins/executor/plugins.go b/drivers/shared/executor/plugins.go similarity index 100% rename from plugins/executor/plugins.go rename to drivers/shared/executor/plugins.go diff --git a/plugins/executor/proto/executor.pb.go b/drivers/shared/executor/proto/executor.pb.go similarity index 100% rename from plugins/executor/proto/executor.pb.go rename to drivers/shared/executor/proto/executor.pb.go diff --git a/plugins/executor/proto/executor.proto b/drivers/shared/executor/proto/executor.proto similarity index 100% rename from plugins/executor/proto/executor.proto rename to drivers/shared/executor/proto/executor.proto diff --git a/plugins/executor/server.go b/drivers/shared/executor/server.go similarity index 94% rename from plugins/executor/server.go rename to drivers/shared/executor/server.go index f6b3a0b24..68c39812b 100644 --- a/plugins/executor/server.go +++ b/drivers/shared/executor/server.go @@ -6,19 +6,18 @@ import ( "github.com/golang/protobuf/ptypes" "github.com/hashicorp/consul-template/signals" - "github.com/hashicorp/nomad/drivers/shared/executor" + "github.com/hashicorp/nomad/drivers/shared/executor/proto" "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/executor/proto" "golang.org/x/net/context" ) type grpcExecutorServer struct { - impl executor.Executor + impl Executor doneCtx context.Context } func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchRequest) (*proto.LaunchResponse, error) { - ps, err := s.impl.Launch(&executor.ExecCommand{ + ps, err := s.impl.Launch(&ExecCommand{ Cmd: req.Cmd, Args: req.Args, Resources: resourcesFromProto(req.Resources), diff --git a/drivers/shared/executor/utils.go b/drivers/shared/executor/utils.go new file mode 100644 index 000000000..30f51b66d --- /dev/null +++ b/drivers/shared/executor/utils.go @@ -0,0 +1,157 @@ +package executor + +import ( + "encoding/json" + "fmt" + "io" + "os/exec" + + "github.com/golang/protobuf/ptypes" + hclog "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/drivers/shared/executor/proto" + "github.com/hashicorp/nomad/helper/discover" + "github.com/hashicorp/nomad/plugins/base" +) + +const ( + // ExecutorDefaultMaxPort is the default max port used by the executor for + // searching for an available port + ExecutorDefaultMaxPort = 14512 + + // ExecutorDefaultMinPort is the default min port used by the executor for + // searching for an available port + ExecutorDefaultMinPort = 14000 +) + +// CreateExecutor launches an executor plugin and returns an instance of the +// Executor interface +func CreateExecutor(w io.Writer, level hclog.Level, driverConfig *base.ClientDriverConfig, + executorConfig *ExecutorConfig) (Executor, *plugin.Client, error) { + + c, err := json.Marshal(executorConfig) + if err != nil { + return nil, nil, fmt.Errorf("unable to create executor config: %v", err) + } + bin, err := discover.NomadExecutable() + if err != nil { + return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err) + } + + config := &plugin.ClientConfig{ + Cmd: exec.Command(bin, "executor", string(c)), + } + config.HandshakeConfig = base.Handshake + config.Plugins = GetPluginMap(w, level, executorConfig.FSIsolation) + config.AllowedProtocols = []plugin.Protocol{plugin.ProtocolGRPC} + + if driverConfig != nil { + config.MaxPort = driverConfig.ClientMaxPort + config.MinPort = driverConfig.ClientMinPort + } else { + config.MaxPort = ExecutorDefaultMaxPort + config.MinPort = ExecutorDefaultMinPort + } + + // setting the setsid of the plugin process so that it doesn't get signals sent to + // the nomad client. + if config.Cmd != nil { + isolateCommand(config.Cmd) + } + + executorClient := plugin.NewClient(config) + rpcClient, err := executorClient.Client() + if err != nil { + return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err) + } + + raw, err := rpcClient.Dispense("executor") + if err != nil { + return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err) + } + executorPlugin := raw.(Executor) + return executorPlugin, executorClient, nil +} + +// CreateExecutorWithConfig launches a plugin with a given plugin config +func CreateExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (Executor, *plugin.Client, error) { + config.HandshakeConfig = base.Handshake + + // Setting this to DEBUG since the log level at the executor server process + // is already set, and this effects only the executor client. + // TODO: Use versioned plugin map to support backwards compatability with + // existing pre-0.9 executors + config.Plugins = GetPluginMap(w, hclog.Debug, false) + config.AllowedProtocols = []plugin.Protocol{plugin.ProtocolGRPC} + + executorClient := plugin.NewClient(config) + rpcClient, err := executorClient.Client() + if err != nil { + return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err) + } + + raw, err := rpcClient.Dispense("executor") + if err != nil { + return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err) + } + executorPlugin, ok := raw.(Executor) + if !ok { + return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw) + } + return executorPlugin, executorClient, nil +} + +func processStateToProto(ps *ProcessState) (*proto.ProcessState, error) { + timestamp, err := ptypes.TimestampProto(ps.Time) + if err != nil { + return nil, err + } + pb := &proto.ProcessState{ + Pid: int32(ps.Pid), + ExitCode: int32(ps.ExitCode), + Signal: int32(ps.Signal), + Time: timestamp, + } + + return pb, nil +} + +func processStateFromProto(pb *proto.ProcessState) (*ProcessState, error) { + timestamp, err := ptypes.Timestamp(pb.Time) + if err != nil { + return nil, err + } + + return &ProcessState{ + Pid: int(pb.Pid), + ExitCode: int(pb.ExitCode), + Signal: int(pb.Signal), + Time: timestamp, + }, nil +} + +func resourcesToProto(r *Resources) *proto.Resources { + if r == nil { + return &proto.Resources{} + } + + return &proto.Resources{ + Cpu: int32(r.CPU), + MemoryMB: int32(r.MemoryMB), + DiskMB: int32(r.DiskMB), + Iops: int32(r.IOPS), + } +} + +func resourcesFromProto(pb *proto.Resources) *Resources { + if pb == nil { + return &Resources{} + } + + return &Resources{ + CPU: int(pb.Cpu), + MemoryMB: int(pb.MemoryMB), + DiskMB: int(pb.DiskMB), + IOPS: int(pb.Iops), + } +} diff --git a/drivers/shared/executor/utils_unix.go b/drivers/shared/executor/utils_unix.go new file mode 100644 index 000000000..ce1a39f86 --- /dev/null +++ b/drivers/shared/executor/utils_unix.go @@ -0,0 +1,18 @@ +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package executor + +import ( + "os/exec" + "syscall" +) + +// isolateCommand sets the setsid flag in exec.Cmd to true so that the process +// becomes the process leader in a new session and doesn't receive signals that +// are sent to the parent process. +func isolateCommand(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setsid = true +} diff --git a/drivers/shared/executor/utils_windows.go b/drivers/shared/executor/utils_windows.go new file mode 100644 index 000000000..76ae317a2 --- /dev/null +++ b/drivers/shared/executor/utils_windows.go @@ -0,0 +1,6 @@ +package executor + +import "os/exec" + +// TODO Figure out if this is needed in Windows +func isolateCommand(cmd *exec.Cmd) {} diff --git a/plugins/drivers/utils/utils.go b/plugins/drivers/utils/utils.go index f36534354..1af2c7e78 100644 --- a/plugins/drivers/utils/utils.go +++ b/plugins/drivers/utils/utils.go @@ -1,34 +1,13 @@ package utils import ( - "encoding/json" - "fmt" - "io" - "os" - "os/exec" "strings" - hclog "github.com/hashicorp/go-hclog" - plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/taskenv" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/plugins/base" - pexecutor "github.com/hashicorp/nomad/plugins/executor" -) - -const ( - // ExecutorDefaultMaxPort is the default max port used by the executor for - // searching for an available port - ExecutorDefaultMaxPort = 14512 - - // ExecutorDefaultMinPort is the default min port used by the executor for - // searching for an available port - ExecutorDefaultMinPort = 14000 ) // SetEnvvars sets path and host env vars depending on the FS isolation used. @@ -60,89 +39,3 @@ func CgroupsMounted(node *structs.Node) bool { _, ok := node.Attributes["unique.cgroup.mountpoint"] return ok } - -// CreateExecutor launches an executor plugin and returns an instance of the -// Executor interface -func CreateExecutor(w io.Writer, level hclog.Level, driverConfig *base.ClientDriverConfig, - executorConfig *pexecutor.ExecutorConfig) (executor.Executor, *plugin.Client, error) { - - c, err := json.Marshal(executorConfig) - if err != nil { - return nil, nil, fmt.Errorf("unable to create executor config: %v", err) - } - bin, err := discover.NomadExecutable() - if err != nil { - return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } - - config := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", string(c)), - } - config.HandshakeConfig = base.Handshake - config.Plugins = pexecutor.GetPluginMap(w, level, executorConfig.FSIsolation) - config.AllowedProtocols = []plugin.Protocol{plugin.ProtocolGRPC} - - if driverConfig != nil { - config.MaxPort = driverConfig.ClientMaxPort - config.MinPort = driverConfig.ClientMinPort - } else { - config.MaxPort = ExecutorDefaultMaxPort - config.MinPort = ExecutorDefaultMinPort - } - - // setting the setsid of the plugin process so that it doesn't get signals sent to - // the nomad client. - if config.Cmd != nil { - isolateCommand(config.Cmd) - } - - executorClient := plugin.NewClient(config) - rpcClient, err := executorClient.Client() - if err != nil { - return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err) - } - - raw, err := rpcClient.Dispense("executor") - if err != nil { - return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err) - } - executorPlugin := raw.(executor.Executor) - return executorPlugin, executorClient, nil -} - -// CreateExecutorWithConfig launches a plugin with a given plugin config -func CreateExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) { - config.HandshakeConfig = base.Handshake - - // Setting this to DEBUG since the log level at the executor server process - // is already set, and this effects only the executor client. - // TODO: Use versioned plugin map to support backwards compatability with - // existing pre-0.9 executors - config.Plugins = pexecutor.GetPluginMap(w, hclog.Debug, false) - config.AllowedProtocols = []plugin.Protocol{plugin.ProtocolGRPC} - - executorClient := plugin.NewClient(config) - rpcClient, err := executorClient.Client() - if err != nil { - return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err) - } - - raw, err := rpcClient.Dispense("executor") - if err != nil { - return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err) - } - executorPlugin, ok := raw.(executor.Executor) - if !ok { - return nil, nil, fmt.Errorf("unexpected executor rpc type: %T", raw) - } - return executorPlugin, executorClient, nil -} - -// KillProcess kills a process with the given pid -func KillProcess(pid int) error { - proc, err := os.FindProcess(pid) - if err != nil { - return err - } - return proc.Kill() -} diff --git a/plugins/drivers/utils/utils_unix.go b/plugins/drivers/utils/utils_unix.go index 85d9fe6c0..d7592be07 100644 --- a/plugins/drivers/utils/utils_unix.go +++ b/plugins/drivers/utils/utils_unix.go @@ -3,22 +3,9 @@ package utils import ( - "os/exec" - "syscall" - "golang.org/x/sys/unix" ) -// isolateCommand sets the setsid flag in exec.Cmd to true so that the process -// becomes the process leader in a new session and doesn't receive signals that -// are sent to the parent process. -func isolateCommand(cmd *exec.Cmd) { - if cmd.SysProcAttr == nil { - cmd.SysProcAttr = &syscall.SysProcAttr{} - } - cmd.SysProcAttr.Setsid = true -} - // IsUnixRoot returns true if system is unix and user running is effectively root func IsUnixRoot() bool { return unix.Geteuid() == 0 diff --git a/plugins/drivers/utils/utils_windows.go b/plugins/drivers/utils/utils_windows.go index 8243e498a..bc322d966 100644 --- a/plugins/drivers/utils/utils_windows.go +++ b/plugins/drivers/utils/utils_windows.go @@ -1,12 +1,5 @@ package utils -import ( - "os/exec" -) - -// TODO Figure out if this is needed in Windows -func isolateCommand(cmd *exec.Cmd) {} - // IsUnixRoot returns true if system is a unix system and the effective uid of user is root func IsUnixRoot() bool { return false diff --git a/plugins/executor/utils.go b/plugins/executor/utils.go deleted file mode 100644 index 222e95727..000000000 --- a/plugins/executor/utils.go +++ /dev/null @@ -1,62 +0,0 @@ -package executor - -import ( - "github.com/golang/protobuf/ptypes" - "github.com/hashicorp/nomad/drivers/shared/executor" - "github.com/hashicorp/nomad/plugins/executor/proto" -) - -func processStateToProto(ps *executor.ProcessState) (*proto.ProcessState, error) { - timestamp, err := ptypes.TimestampProto(ps.Time) - if err != nil { - return nil, err - } - pb := &proto.ProcessState{ - Pid: int32(ps.Pid), - ExitCode: int32(ps.ExitCode), - Signal: int32(ps.Signal), - Time: timestamp, - } - - return pb, nil -} - -func processStateFromProto(pb *proto.ProcessState) (*executor.ProcessState, error) { - timestamp, err := ptypes.Timestamp(pb.Time) - if err != nil { - return nil, err - } - - return &executor.ProcessState{ - Pid: int(pb.Pid), - ExitCode: int(pb.ExitCode), - Signal: int(pb.Signal), - Time: timestamp, - }, nil -} - -func resourcesToProto(r *executor.Resources) *proto.Resources { - if r == nil { - return &proto.Resources{} - } - - return &proto.Resources{ - Cpu: int32(r.CPU), - MemoryMB: int32(r.MemoryMB), - DiskMB: int32(r.DiskMB), - Iops: int32(r.IOPS), - } -} - -func resourcesFromProto(pb *proto.Resources) *executor.Resources { - if pb == nil { - return &executor.Resources{} - } - - return &executor.Resources{ - CPU: int(pb.Cpu), - MemoryMB: int(pb.MemoryMB), - DiskMB: int(pb.DiskMB), - IOPS: int(pb.Iops), - } -}