From 43cd7a71a6073160f19b0fa4beb88c4fef499262 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:06:56 -0400 Subject: [PATCH 01/15] aux: helper method that returns token as well as ACL policy This helper returns the token as well as the ACL policy, to be used in a later commit for logging the token info associated with nomad exec invocation. --- client/acl.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/client/acl.go b/client/acl.go index dc9c21f61..abccdc01f 100644 --- a/client/acl.go +++ b/client/acl.go @@ -70,38 +70,43 @@ func (c *cachedACLValue) Age() time.Duration { // ResolveToken is used to translate an ACL Token Secret ID into // an ACL object, nil if ACLs are disabled, or an error. func (c *Client) ResolveToken(secretID string) (*acl.ACL, error) { + a, _, err := c.resolveTokenAndACL(secretID) + return a, err +} + +func (c *Client) resolveTokenAndACL(secretID string) (*acl.ACL, *structs.ACLToken, error) { // Fast-path if ACLs are disabled if !c.config.ACLEnabled { - return nil, nil + return nil, nil, nil } defer metrics.MeasureSince([]string{"client", "acl", "resolve_token"}, time.Now()) // Resolve the token value token, err := c.resolveTokenValue(secretID) if err != nil { - return nil, err + return nil, nil, err } if token == nil { - return nil, structs.ErrTokenNotFound + return nil, nil, structs.ErrTokenNotFound } // Check if this is a management token if token.Type == structs.ACLManagementToken { - return acl.ManagementACL, nil + return acl.ManagementACL, token, nil } // Resolve the policies policies, err := c.resolvePolicies(token.SecretID, token.Policies) if err != nil { - return nil, err + return nil, nil, err } // Resolve the ACL object aclObj, err := structs.CompileACLObject(c.aclCache, policies) if err != nil { - return nil, err + return nil, nil, err } - return aclObj, nil + return aclObj, token, nil } // resolveTokenValue is used to translate a secret ID into an ACL token with caching From 75349d647e3dcf9299b880acaa3595756340c7e1 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:32:07 -0400 Subject: [PATCH 02/15] test helper for registering jobs with acl Test helper that allows registration of jobs when ACL is activated. --- testutil/wait.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/testutil/wait.go b/testutil/wait.go index d49bf8b47..032df7c78 100644 --- a/testutil/wait.go +++ b/testutil/wait.go @@ -92,11 +92,13 @@ func WaitForLeader(t testing.T, rpc rpcFn) { }) } -func RegisterJob(t testing.T, rpc rpcFn, job *structs.Job) { +func RegisterJobWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) { WaitForResult(func() (bool, error) { args := &structs.JobRegisterRequest{} args.Job = job args.WriteRequest.Region = "global" + args.AuthToken = token + args.Namespace = structs.DefaultNamespace var jobResp structs.JobRegisterResponse err := rpc("Job.Register", args, &jobResp) return err == nil, fmt.Errorf("Job.Register error: %v", err) @@ -107,9 +109,12 @@ func RegisterJob(t testing.T, rpc rpcFn, job *structs.Job) { t.Logf("Job %q registered", job.ID) } -// WaitForRunning runs a job and blocks until all allocs are out of pending. -func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocListStub { - RegisterJob(t, rpc, job) +func RegisterJob(t testing.T, rpc rpcFn, job *structs.Job) { + RegisterJobWithToken(t, rpc, job, "") +} + +func WaitForRunningWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) []*structs.AllocListStub { + RegisterJobWithToken(t, rpc, job, token) var resp structs.JobAllocationsResponse @@ -117,6 +122,8 @@ func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocLi args := &structs.JobSpecificRequest{} args.JobID = job.ID args.QueryOptions.Region = "global" + args.AuthToken = token + args.Namespace = structs.DefaultNamespace err := rpc("Job.Allocations", args, &resp) if err != nil { return false, fmt.Errorf("Job.Allocations error: %v", err) @@ -140,3 +147,8 @@ func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocLi return resp.Allocations } + +// WaitForRunning runs a job and blocks until all allocs are out of pending. +func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocListStub { + return WaitForRunningWithToken(t, rpc, job, "") +} From e79ce1f9d00d806045ab0338d70cd9042a4e3ee5 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:18:52 -0400 Subject: [PATCH 03/15] drivers/mock: extract command related operations Extract command parsing and execution mocking into a separate struct. Also, allow mocking of different fs_isolation for testing. --- drivers/mock/command.go | 93 +++++++++++++++++++++ drivers/mock/driver.go | 180 +++++++++++++++++++++++++--------------- drivers/mock/handle.go | 83 ++---------------- 3 files changed, 214 insertions(+), 142 deletions(-) create mode 100644 drivers/mock/command.go diff --git a/drivers/mock/command.go b/drivers/mock/command.go new file mode 100644 index 000000000..02abfc519 --- /dev/null +++ b/drivers/mock/command.go @@ -0,0 +1,93 @@ +package mock + +import ( + "errors" + "io" + "sync" + "time" + + hclog "github.com/hashicorp/go-hclog" + bstructs "github.com/hashicorp/nomad/plugins/base/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +func runCommand(c Command, stdout, stderr io.WriteCloser, cancelCh <-chan struct{}, pluginExitTimer <-chan time.Time, logger hclog.Logger) *drivers.ExitResult { + errCh := make(chan error, 1) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + runCommandOutput(stdout, c.StdoutString, c.StdoutRepeat, c.stdoutRepeatDuration, cancelCh, logger, errCh) + }() + + wg.Add(1) + go func() { + defer wg.Done() + runCommandOutput(stderr, c.StderrString, c.StderrRepeat, c.stderrRepeatDuration, cancelCh, logger, errCh) + }() + + timer := time.NewTimer(c.runForDuration) + defer timer.Stop() + + select { + case <-timer.C: + logger.Debug("run_for time elapsed; exiting", "run_for", c.RunFor) + case <-cancelCh: + logger.Debug("killed; exiting") + case <-pluginExitTimer: + logger.Debug("exiting plugin") + return &drivers.ExitResult{ + Err: bstructs.ErrPluginShutdown, + } + case err := <-errCh: + logger.Error("error running mock task; exiting", "error", err) + return &drivers.ExitResult{ + Err: err, + } + } + + wg.Wait() + + var exitErr error + if c.ExitErrMsg != "" { + exitErr = errors.New(c.ExitErrMsg) + } + + return &drivers.ExitResult{ + ExitCode: c.ExitCode, + Signal: c.ExitSignal, + Err: exitErr, + } +} + +func runCommandOutput(writer io.WriteCloser, + output string, outputRepeat int, repeatDuration time.Duration, + cancelCh <-chan struct{}, logger hclog.Logger, errCh chan error) { + + defer writer.Close() + + if output == "" { + return + } + + if _, err := io.WriteString(writer, output); err != nil { + logger.Error("failed to write to stdout", "error", err) + errCh <- err + return + } + + for i := 0; i < outputRepeat; i++ { + select { + case <-cancelCh: + logger.Warn("exiting before done writing output", "i", i, "total", outputRepeat) + return + case <-time.After(repeatDuration): + if _, err := io.WriteString(writer, output); err != nil { + logger.Error("failed to write to stdout", "error", err) + errCh <- err + return + } + } + } +} diff --git a/drivers/mock/driver.go b/drivers/mock/driver.go index 0f63d7302..5dced6c85 100644 --- a/drivers/mock/driver.go +++ b/drivers/mock/driver.go @@ -57,6 +57,10 @@ var ( // configSpec is the hcl specification returned by the ConfigSchema RPC configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ + "fs_isolation": hclspec.NewDefault( + hclspec.NewAttr("fs_isolation", "string", false), + hclspec.NewLiteral(fmt.Sprintf("%q", drivers.FSIsolationNone)), + ), "shutdown_periodic_after": hclspec.NewDefault( hclspec.NewAttr("shutdown_periodic_after", "bool", false), hclspec.NewLiteral("false"), @@ -72,26 +76,22 @@ var ( "start_block_for": hclspec.NewAttr("start_block_for", "string", false), "kill_after": hclspec.NewAttr("kill_after", "string", false), "plugin_exit_after": hclspec.NewAttr("plugin_exit_after", "string", false), - "run_for": hclspec.NewAttr("run_for", "string", false), - "exit_code": hclspec.NewAttr("exit_code", "number", false), - "exit_signal": hclspec.NewAttr("exit_signal", "number", false), - "exit_err_msg": hclspec.NewAttr("exit_err_msg", "string", false), - "signal_error": hclspec.NewAttr("signal_error", "string", false), "driver_ip": hclspec.NewAttr("driver_ip", "string", false), "driver_advertise": hclspec.NewAttr("driver_advertise", "bool", false), "driver_port_map": hclspec.NewAttr("driver_port_map", "string", false), - "stdout_string": hclspec.NewAttr("stdout_string", "string", false), - "stdout_repeat": hclspec.NewAttr("stdout_repeat", "number", false), - "stdout_repeat_duration": hclspec.NewAttr("stdout_repeat_duration", "string", false), - }) - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ - SendSignals: true, - Exec: true, - FSIsolation: drivers.FSIsolationNone, - } + "run_for": hclspec.NewAttr("run_for", "string", false), + "exit_code": hclspec.NewAttr("exit_code", "number", false), + "exit_signal": hclspec.NewAttr("exit_signal", "number", false), + "exit_err_msg": hclspec.NewAttr("exit_err_msg", "string", false), + "signal_error": hclspec.NewAttr("signal_error", "string", false), + "stdout_string": hclspec.NewAttr("stdout_string", "string", false), + "stdout_repeat": hclspec.NewAttr("stdout_repeat", "number", false), + "stdout_repeat_duration": hclspec.NewAttr("stdout_repeat_duration", "string", false), + "stderr_string": hclspec.NewAttr("stderr_string", "string", false), + "stderr_repeat": hclspec.NewAttr("stderr_repeat", "number", false), + "stderr_repeat_duration": hclspec.NewAttr("stderr_repeat_duration", "string", false), + }) ) // Driver is a mock DriverPlugin implementation @@ -100,6 +100,10 @@ type Driver struct { // event can be broadcast to all callers eventer *eventer.Eventer + // capabilities is returned by the Capabilities RPC and indicates what + // optional features this driver supports + capabilities *drivers.Capabilities + // config is the driver configuration set by the SetConfig RPC config *Config @@ -133,8 +137,16 @@ type Driver struct { func NewMockDriver(logger hclog.Logger) drivers.DriverPlugin { ctx, cancel := context.WithCancel(context.Background()) logger = logger.Named(pluginName) + + capabilities := &drivers.Capabilities{ + SendSignals: true, + Exec: true, + FSIsolation: drivers.FSIsolationNone, + } + return &Driver{ eventer: eventer.NewEventer(ctx, logger), + capabilities: capabilities, config: &Config{}, tasks: newTaskStore(), ctx: ctx, @@ -145,6 +157,8 @@ func NewMockDriver(logger hclog.Logger) drivers.DriverPlugin { // Config is the configuration for the driver that applies to all tasks type Config struct { + FSIsolation string `codec:"fs_isolation"` + // ShutdownPeriodicAfter is a toggle that can be used during tests to // "stop" a previously-functioning driver, allowing for testing of periodic // drivers and fingerprinters @@ -156,8 +170,52 @@ type Config struct { ShutdownPeriodicDuration time.Duration `codec:"shutdown_periodic_duration"` } +type Command struct { + // RunFor is the duration for which the fake task runs for. After this + // period the MockDriver responds to the task running indicating that the + // task has terminated + RunFor string `codec:"run_for"` + runForDuration time.Duration + + // ExitCode is the exit code with which the MockDriver indicates the task + // has exited + ExitCode int `codec:"exit_code"` + + // ExitSignal is the signal with which the MockDriver indicates the task has + // been killed + ExitSignal int `codec:"exit_signal"` + + // ExitErrMsg is the error message that the task returns while exiting + ExitErrMsg string `codec:"exit_err_msg"` + + // SignalErr is the error message that the task returns if signalled + SignalErr string `codec:"signal_error"` + + // StdoutString is the string that should be sent to stdout + StdoutString string `codec:"stdout_string"` + + // StdoutRepeat is the number of times the output should be sent. + StdoutRepeat int `codec:"stdout_repeat"` + + // StdoutRepeatDur is the duration between repeated outputs. + StdoutRepeatDur string `codec:"stdout_repeat_duration"` + stdoutRepeatDuration time.Duration + + // StderrString is the string that should be sent to stderr + StderrString string `codec:"stderr_string"` + + // StderrRepeat is the number of times the errput should be sent. + StderrRepeat int `codec:"stderr_repeat"` + + // StderrRepeatDur is the duration between repeated errputs. + StderrRepeatDur string `codec:"stderr_repeat_duration"` + stderrRepeatDuration time.Duration +} + // TaskConfig is the driver configuration of a task within a job type TaskConfig struct { + Command + // PluginExitAfter is the duration after which the mock driver indicates the // plugin has exited via the WaitTask call. PluginExitAfter string `codec:"plugin_exit_after"` @@ -179,26 +237,6 @@ type TaskConfig struct { KillAfter string `codec:"kill_after"` killAfterDuration time.Duration - // RunFor is the duration for which the fake task runs for. After this - // period the MockDriver responds to the task running indicating that the - // task has terminated - RunFor string `codec:"run_for"` - runForDuration time.Duration - - // ExitCode is the exit code with which the MockDriver indicates the task - // has exited - ExitCode int `codec:"exit_code"` - - // ExitSignal is the signal with which the MockDriver indicates the task has - // been killed - ExitSignal int `codec:"exit_signal"` - - // ExitErrMsg is the error message that the task returns while exiting - ExitErrMsg string `codec:"exit_err_msg"` - - // SignalErr is the error message that the task returns if signalled - SignalErr string `codec:"signal_error"` - // DriverIP will be returned as the DriverNetwork.IP from Start() DriverIP string `codec:"driver_ip"` @@ -209,16 +247,6 @@ type TaskConfig struct { // DriverPortMap will parse a label:number pair and return it in // DriverNetwork.PortMap from Start(). DriverPortMap string `codec:"driver_port_map"` - - // StdoutString is the string that should be sent to stdout - StdoutString string `codec:"stdout_string"` - - // StdoutRepeat is the number of times the output should be sent. - StdoutRepeat int `codec:"stdout_repeat"` - - // StdoutRepeatDur is the duration between repeated outputs. - StdoutRepeatDur string `codec:"stdout_repeat_duration"` - stdoutRepeatDuration time.Duration } type MockTaskState struct { @@ -245,6 +273,12 @@ func (d *Driver) SetConfig(cfg *base.Config) error { if d.config.ShutdownPeriodicAfter { d.shutdownFingerprintTime = time.Now().Add(d.config.ShutdownPeriodicDuration) } + + isolation := config.FSIsolation + if isolation != "" { + d.capabilities.FSIsolation = drivers.FSIsolation(isolation) + } + return nil } @@ -253,7 +287,7 @@ func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { } func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil + return d.capabilities, nil } func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { @@ -329,6 +363,23 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return nil } +func (c *Command) parseDurations() error { + var err error + if c.runForDuration, err = parseDuration(c.RunFor); err != nil { + return fmt.Errorf("run_for %v not a valid duration: %v", c.RunFor, err) + } + + if c.stdoutRepeatDuration, err = parseDuration(c.StdoutRepeatDur); err != nil { + return fmt.Errorf("stdout_repeat_duration %v not a valid duration: %v", c.stdoutRepeatDuration, err) + } + + if c.stderrRepeatDuration, err = parseDuration(c.StderrRepeatDur); err != nil { + return fmt.Errorf("stderr_repeat_duration %v not a valid duration: %v", c.stderrRepeatDuration, err) + } + + return nil +} + func parseDriverConfig(cfg *drivers.TaskConfig) (*TaskConfig, error) { var driverConfig TaskConfig if err := cfg.DecodeDriverConfig(&driverConfig); err != nil { @@ -340,16 +391,18 @@ func parseDriverConfig(cfg *drivers.TaskConfig) (*TaskConfig, error) { return nil, fmt.Errorf("start_block_for %v not a valid duration: %v", driverConfig.StartBlockFor, err) } - if driverConfig.runForDuration, err = parseDuration(driverConfig.RunFor); err != nil { - return nil, fmt.Errorf("run_for %v not a valid duration: %v", driverConfig.RunFor, err) - } - if driverConfig.pluginExitAfterDuration, err = parseDuration(driverConfig.PluginExitAfter); err != nil { return nil, fmt.Errorf("plugin_exit_after %v not a valid duration: %v", driverConfig.PluginExitAfter, err) } - if driverConfig.stdoutRepeatDuration, err = parseDuration(driverConfig.StdoutRepeatDur); err != nil { - return nil, fmt.Errorf("stdout_repeat_duration %v not a valid duration: %v", driverConfig.stdoutRepeatDuration, err) + if err = driverConfig.parseDurations(); err != nil { + return nil, err + } + + if driverConfig.ExecCommand != nil { + if err = driverConfig.ExecCommand.parseDurations(); err != nil { + return nil, err + } } return &driverConfig, nil @@ -359,26 +412,15 @@ func newTaskHandle(cfg *drivers.TaskConfig, driverConfig *TaskConfig, logger hcl killCtx, killCancel := context.WithCancel(context.Background()) h := &taskHandle{ taskConfig: cfg, - runFor: driverConfig.runForDuration, + command: driverConfig.Command, pluginExitAfter: driverConfig.pluginExitAfterDuration, killAfter: driverConfig.killAfterDuration, - exitCode: driverConfig.ExitCode, - exitSignal: driverConfig.ExitSignal, - stdoutString: driverConfig.StdoutString, - stdoutRepeat: driverConfig.StdoutRepeat, - stdoutRepeatDur: driverConfig.stdoutRepeatDuration, logger: logger.With("task_name", cfg.Name), - waitCh: make(chan struct{}), + waitCh: make(chan interface{}), killCh: killCtx.Done(), kill: killCancel, startedAt: time.Now(), } - if driverConfig.ExitErrMsg != "" { - h.exitErr = errors.New(driverConfig.ExitErrMsg) - } - if driverConfig.SignalErr != "" { - h.signalErr = fmt.Errorf(driverConfig.SignalErr) - } return h } @@ -541,7 +583,11 @@ func (d *Driver) SignalTask(taskID string, signal string) error { return drivers.ErrTaskNotFound } - return h.signalErr + if h.command.SignalErr == "" { + return nil + } + + return errors.New(h.command.SignalErr) } func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { diff --git a/drivers/mock/handle.go b/drivers/mock/handle.go index 7940a6283..69b2f56dc 100644 --- a/drivers/mock/handle.go +++ b/drivers/mock/handle.go @@ -2,13 +2,11 @@ package mock import ( "context" - "io" "sync" "time" hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/lib/fifo" - bstructs "github.com/hashicorp/nomad/plugins/base/structs" "github.com/hashicorp/nomad/plugins/drivers" ) @@ -16,19 +14,13 @@ import ( type taskHandle struct { logger hclog.Logger - runFor time.Duration pluginExitAfter time.Duration killAfter time.Duration - waitCh chan struct{} - exitCode int - exitSignal int - exitErr error - signalErr error - stdoutString string - stdoutRepeat int - stdoutRepeatDur time.Duration + waitCh chan interface{} - taskConfig *drivers.TaskConfig + taskConfig *drivers.TaskConfig + command Command + execCommand *Command // stateLock guards the procState field stateLock sync.RWMutex @@ -81,14 +73,6 @@ func (h *taskHandle) run() { h.procState = drivers.TaskStateRunning h.stateLock.Unlock() - errCh := make(chan error, 1) - - // Setup logging output - go h.handleLogging(errCh) - - timer := time.NewTimer(h.runFor) - defer timer.Stop() - var pluginExitTimer <-chan time.Time if h.pluginExitAfter != 0 { timer := time.NewTimer(h.pluginExitAfter) @@ -96,70 +80,19 @@ func (h *taskHandle) run() { pluginExitTimer = timer.C } - select { - case <-timer.C: - h.logger.Debug("run_for time elapsed; exiting", "run_for", h.runFor) - case <-h.killCh: - h.logger.Debug("killed; exiting") - case <-pluginExitTimer: - h.logger.Debug("exiting plugin") - h.exitResult = &drivers.ExitResult{ - Err: bstructs.ErrPluginShutdown, - } - - return - case err := <-errCh: - h.logger.Error("error running mock task; exiting", "error", err) - h.exitResult = &drivers.ExitResult{ - Err: err, - } - return - } - - h.exitResult = &drivers.ExitResult{ - ExitCode: h.exitCode, - Signal: h.exitSignal, - Err: h.exitErr, - } - return -} - -func (h *taskHandle) handleLogging(errCh chan<- error) { stdout, err := fifo.OpenWriter(h.taskConfig.StdoutPath) if err != nil { h.logger.Error("failed to write to stdout", "error", err) - errCh <- err + h.exitResult = &drivers.ExitResult{Err: err} return } stderr, err := fifo.OpenWriter(h.taskConfig.StderrPath) if err != nil { h.logger.Error("failed to write to stderr", "error", err) - errCh <- err - return - } - defer stderr.Close() - - if h.stdoutString == "" { + h.exitResult = &drivers.ExitResult{Err: err} return } - if _, err := io.WriteString(stdout, h.stdoutString); err != nil { - h.logger.Error("failed to write to stdout", "error", err) - errCh <- err - return - } - - for i := 0; i < h.stdoutRepeat; i++ { - select { - case <-h.waitCh: - h.logger.Warn("exiting before done writing output", "i", i, "total", h.stdoutRepeat) - return - case <-time.After(h.stdoutRepeatDur): - if _, err := io.WriteString(stdout, h.stdoutString); err != nil { - h.logger.Error("failed to write to stdout", "error", err) - errCh <- err - return - } - } - } + h.exitResult = runCommand(h.command, stdout, stderr, h.killCh, pluginExitTimer, h.logger) + return } From 4f7bd68f5ca9f330a894bfb4375b87f786e1e7a3 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 16:45:15 -0400 Subject: [PATCH 04/15] Add ACL capabilities for nomad exec This adds `alloc-exec` capability to allow operator to execute command into a running task. Furthermore, it adds `alloc-node-exec` capability, required when the alloc task is raw_exec or a driver with no FSIsolation. --- acl/policy.go | 6 +++++- acl/policy_test.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/acl/policy.go b/acl/policy.go index eba1204f2..b6efaa642 100644 --- a/acl/policy.go +++ b/acl/policy.go @@ -28,6 +28,8 @@ const ( NamespaceCapabilityDispatchJob = "dispatch-job" NamespaceCapabilityReadLogs = "read-logs" NamespaceCapabilityReadFS = "read-fs" + NamespaceCapabilityAllocExec = "alloc-exec" + NamespaceCapabilityAllocNodeExec = "alloc-node-exec" NamespaceCapabilityAllocLifecycle = "alloc-lifecycle" NamespaceCapabilitySentinelOverride = "sentinel-override" ) @@ -94,7 +96,8 @@ func isNamespaceCapabilityValid(cap string) bool { switch cap { case NamespaceCapabilityDeny, NamespaceCapabilityListJobs, NamespaceCapabilityReadJob, NamespaceCapabilitySubmitJob, NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, - NamespaceCapabilityReadFS, NamespaceCapabilityAllocLifecycle: + NamespaceCapabilityReadFS, NamespaceCapabilityAllocLifecycle, + NamespaceCapabilityAllocExec, NamespaceCapabilityAllocNodeExec: return true // Separate the enterprise-only capabilities case NamespaceCapabilitySentinelOverride: @@ -123,6 +126,7 @@ func expandNamespacePolicy(policy string) []string { NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, NamespaceCapabilityReadFS, + NamespaceCapabilityAllocExec, NamespaceCapabilityAllocLifecycle, } default: diff --git a/acl/policy_test.go b/acl/policy_test.go index 3385370aa..4665d0d45 100644 --- a/acl/policy_test.go +++ b/acl/policy_test.go @@ -80,6 +80,7 @@ func TestParse(t *testing.T) { NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, NamespaceCapabilityReadFS, + NamespaceCapabilityAllocExec, NamespaceCapabilityAllocLifecycle, }, }, From 6d711d054ba9949eb879b61981e17d9925973204 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 16:58:56 -0400 Subject: [PATCH 05/15] add nomad streaming exec core data structures and interfaces In this commit, we add two driver interfaces for supporting `nomad exec` invocation: * A high level `ExecTaskStreamingDriver`, that operates on io reader/writers. Drivers should prefer using this interface * A low level `ExecTaskStreamingRawDriver` that operates on the raw stream of input structs; useful when a driver delegates handling to driver backend (e.g. across RPC/grpc). The interfaces are optional for a driver, as `nomad exec` support is opt-in. Existing drivers continue to compile without exec support, until their maintainer add such support. Furthermore, we create protobuf structures to represent exec stream entities: `ExecTaskStreamingRequest` and `ExecTaskStreamingResponse`. We aim to reuse the protobuf generated code as much as possible, without translation to avoid conversion overhead. `ExecTaskStream` abstract fetching and sending stream entities. It's influenced by the grpc bi-directional stream interface, to avoid needing any adapter. I considered using channels, but the asynchronisity and concurrency makes buffer reuse too complicated, which would put more pressure on GC and slows exec operation. --- plugins/drivers/driver.go | 65 +++ plugins/drivers/proto/driver.pb.go | 767 ++++++++++++++++++++--------- plugins/drivers/proto/driver.proto | 30 ++ 3 files changed, 618 insertions(+), 244 deletions(-) diff --git a/plugins/drivers/driver.go b/plugins/drivers/driver.go index 0af72641a..a4a6c7534 100644 --- a/plugins/drivers/driver.go +++ b/plugins/drivers/driver.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/drivers/proto" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" "github.com/zclconf/go-cty/cty" @@ -56,6 +57,28 @@ type DriverPlugin interface { ExecTask(taskID string, cmd []string, timeout time.Duration) (*ExecTaskResult, error) } +// ExecTaskStreamingDriver marks that a driver supports streaming exec task. This represents a user friendly +// interface to implement, as an alternative to the ExecTaskStreamingRawDriver, the low level interface. +type ExecTaskStreamingDriver interface { + ExecTaskStreaming(ctx context.Context, taskID string, execOptions *ExecOptions) (*ExitResult, error) +} + +type ExecOptions struct { + // Command is command to run + Command []string + + // Tty indicates whether pseudo-terminal is to be allocated + Tty bool + + // streams + Stdin io.ReadCloser + Stdout io.WriteCloser + Stderr io.WriteCloser + + // terminal size channel + ResizeCh <-chan TerminalSize +} + // InternalDriverPlugin is an interface that exposes functions that are only // implemented by internal driver plugins. type InternalDriverPlugin interface { @@ -127,6 +150,11 @@ type Capabilities struct { FSIsolation FSIsolation } +type TerminalSize struct { + Height int + Width int +} + type TaskConfig struct { ID string JobName string @@ -406,3 +434,40 @@ func (d *DriverNetwork) Hash() []byte { } return h.Sum(nil) } + +//// helper types for operating on raw exec operation +// we alias proto instances as much as possible to avoid conversion overhead + +// ExecTaskStreamingRawDriver represents a low-level interface for executing a streaming exec +// call, and is intended to be used when driver instance is to delegate exec handling to another +// backend, e.g. to a executor or a driver behind a grpc/rpc protocol +// +// Nomad client would prefer this interface method over `ExecTaskStreaming` if driver implements it. +type ExecTaskStreamingRawDriver interface { + ExecTaskStreamingRaw( + ctx context.Context, + taskID string, + command []string, + tty bool, + stream ExecTaskStream) error +} + +// ExecTaskStream represents a stream of exec streaming messages, +// and is a handle to get stdin and tty size and send back +// stdout/stderr and exit operations. +// +// The methods are not concurrent safe; callers must ensure that methods are called +// from at most one goroutine. +type ExecTaskStream interface { + // Send relays response message back to API. + // + // The call is synchronous and no references to message is held: once + // method call completes, the message reference can be reused or freed. + Send(*ExecTaskStreamingResponseMsg) error + + // Receive exec streaming messages from API. Returns `io.EOF` on completion of stream. + Recv() (*ExecTaskStreamingRequestMsg, error) +} + +type ExecTaskStreamingRequestMsg = proto.ExecTaskStreamingRequest +type ExecTaskStreamingResponseMsg = proto.ExecTaskStreamingResponse diff --git a/plugins/drivers/proto/driver.pb.go b/plugins/drivers/proto/driver.pb.go index c972e4d67..5473f7eab 100644 --- a/plugins/drivers/proto/driver.pb.go +++ b/plugins/drivers/proto/driver.pb.go @@ -50,7 +50,7 @@ func (x TaskState) String() string { return proto.EnumName(TaskState_name, int32(x)) } func (TaskState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{0} } type FingerprintResponse_HealthState int32 @@ -76,7 +76,7 @@ func (x FingerprintResponse_HealthState) String() string { return proto.EnumName(FingerprintResponse_HealthState_name, int32(x)) } func (FingerprintResponse_HealthState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{5, 0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{5, 0} } type StartTaskResponse_Result int32 @@ -102,7 +102,7 @@ func (x StartTaskResponse_Result) String() string { return proto.EnumName(StartTaskResponse_Result_name, int32(x)) } func (StartTaskResponse_Result) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{9, 0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{9, 0} } type DriverCapabilities_FSIsolation int32 @@ -128,7 +128,7 @@ func (x DriverCapabilities_FSIsolation) String() string { return proto.EnumName(DriverCapabilities_FSIsolation_name, int32(x)) } func (DriverCapabilities_FSIsolation) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{25, 0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{28, 0} } type CPUUsage_Fields int32 @@ -163,7 +163,7 @@ func (x CPUUsage_Fields) String() string { return proto.EnumName(CPUUsage_Fields_name, int32(x)) } func (CPUUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{43, 0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{46, 0} } type MemoryUsage_Fields int32 @@ -201,7 +201,7 @@ func (x MemoryUsage_Fields) String() string { return proto.EnumName(MemoryUsage_Fields_name, int32(x)) } func (MemoryUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{44, 0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{47, 0} } type TaskConfigSchemaRequest struct { @@ -214,7 +214,7 @@ func (m *TaskConfigSchemaRequest) Reset() { *m = TaskConfigSchemaRequest func (m *TaskConfigSchemaRequest) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaRequest) ProtoMessage() {} func (*TaskConfigSchemaRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{0} + return fileDescriptor_driver_8793c97f57ec8d84, []int{0} } func (m *TaskConfigSchemaRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaRequest.Unmarshal(m, b) @@ -246,7 +246,7 @@ func (m *TaskConfigSchemaResponse) Reset() { *m = TaskConfigSchemaRespon func (m *TaskConfigSchemaResponse) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaResponse) ProtoMessage() {} func (*TaskConfigSchemaResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{1} + return fileDescriptor_driver_8793c97f57ec8d84, []int{1} } func (m *TaskConfigSchemaResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaResponse.Unmarshal(m, b) @@ -283,7 +283,7 @@ func (m *CapabilitiesRequest) Reset() { *m = CapabilitiesRequest{} } func (m *CapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*CapabilitiesRequest) ProtoMessage() {} func (*CapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{2} + return fileDescriptor_driver_8793c97f57ec8d84, []int{2} } func (m *CapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesRequest.Unmarshal(m, b) @@ -318,7 +318,7 @@ func (m *CapabilitiesResponse) Reset() { *m = CapabilitiesResponse{} } func (m *CapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*CapabilitiesResponse) ProtoMessage() {} func (*CapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{3} + return fileDescriptor_driver_8793c97f57ec8d84, []int{3} } func (m *CapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesResponse.Unmarshal(m, b) @@ -355,7 +355,7 @@ func (m *FingerprintRequest) Reset() { *m = FingerprintRequest{} } func (m *FingerprintRequest) String() string { return proto.CompactTextString(m) } func (*FingerprintRequest) ProtoMessage() {} func (*FingerprintRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{4} + return fileDescriptor_driver_8793c97f57ec8d84, []int{4} } func (m *FingerprintRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintRequest.Unmarshal(m, b) @@ -398,7 +398,7 @@ func (m *FingerprintResponse) Reset() { *m = FingerprintResponse{} } func (m *FingerprintResponse) String() string { return proto.CompactTextString(m) } func (*FingerprintResponse) ProtoMessage() {} func (*FingerprintResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{5} + return fileDescriptor_driver_8793c97f57ec8d84, []int{5} } func (m *FingerprintResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintResponse.Unmarshal(m, b) @@ -453,7 +453,7 @@ func (m *RecoverTaskRequest) Reset() { *m = RecoverTaskRequest{} } func (m *RecoverTaskRequest) String() string { return proto.CompactTextString(m) } func (*RecoverTaskRequest) ProtoMessage() {} func (*RecoverTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{6} + return fileDescriptor_driver_8793c97f57ec8d84, []int{6} } func (m *RecoverTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskRequest.Unmarshal(m, b) @@ -497,7 +497,7 @@ func (m *RecoverTaskResponse) Reset() { *m = RecoverTaskResponse{} } func (m *RecoverTaskResponse) String() string { return proto.CompactTextString(m) } func (*RecoverTaskResponse) ProtoMessage() {} func (*RecoverTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{7} + return fileDescriptor_driver_8793c97f57ec8d84, []int{7} } func (m *RecoverTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskResponse.Unmarshal(m, b) @@ -529,7 +529,7 @@ func (m *StartTaskRequest) Reset() { *m = StartTaskRequest{} } func (m *StartTaskRequest) String() string { return proto.CompactTextString(m) } func (*StartTaskRequest) ProtoMessage() {} func (*StartTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{8} + return fileDescriptor_driver_8793c97f57ec8d84, []int{8} } func (m *StartTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskRequest.Unmarshal(m, b) @@ -583,7 +583,7 @@ func (m *StartTaskResponse) Reset() { *m = StartTaskResponse{} } func (m *StartTaskResponse) String() string { return proto.CompactTextString(m) } func (*StartTaskResponse) ProtoMessage() {} func (*StartTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{9} + return fileDescriptor_driver_8793c97f57ec8d84, []int{9} } func (m *StartTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskResponse.Unmarshal(m, b) @@ -643,7 +643,7 @@ func (m *WaitTaskRequest) Reset() { *m = WaitTaskRequest{} } func (m *WaitTaskRequest) String() string { return proto.CompactTextString(m) } func (*WaitTaskRequest) ProtoMessage() {} func (*WaitTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{10} + return fileDescriptor_driver_8793c97f57ec8d84, []int{10} } func (m *WaitTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskRequest.Unmarshal(m, b) @@ -684,7 +684,7 @@ func (m *WaitTaskResponse) Reset() { *m = WaitTaskResponse{} } func (m *WaitTaskResponse) String() string { return proto.CompactTextString(m) } func (*WaitTaskResponse) ProtoMessage() {} func (*WaitTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{11} + return fileDescriptor_driver_8793c97f57ec8d84, []int{11} } func (m *WaitTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskResponse.Unmarshal(m, b) @@ -736,7 +736,7 @@ func (m *StopTaskRequest) Reset() { *m = StopTaskRequest{} } func (m *StopTaskRequest) String() string { return proto.CompactTextString(m) } func (*StopTaskRequest) ProtoMessage() {} func (*StopTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{12} + return fileDescriptor_driver_8793c97f57ec8d84, []int{12} } func (m *StopTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskRequest.Unmarshal(m, b) @@ -787,7 +787,7 @@ func (m *StopTaskResponse) Reset() { *m = StopTaskResponse{} } func (m *StopTaskResponse) String() string { return proto.CompactTextString(m) } func (*StopTaskResponse) ProtoMessage() {} func (*StopTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{13} + return fileDescriptor_driver_8793c97f57ec8d84, []int{13} } func (m *StopTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskResponse.Unmarshal(m, b) @@ -821,7 +821,7 @@ func (m *DestroyTaskRequest) Reset() { *m = DestroyTaskRequest{} } func (m *DestroyTaskRequest) String() string { return proto.CompactTextString(m) } func (*DestroyTaskRequest) ProtoMessage() {} func (*DestroyTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{14} + return fileDescriptor_driver_8793c97f57ec8d84, []int{14} } func (m *DestroyTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskRequest.Unmarshal(m, b) @@ -865,7 +865,7 @@ func (m *DestroyTaskResponse) Reset() { *m = DestroyTaskResponse{} } func (m *DestroyTaskResponse) String() string { return proto.CompactTextString(m) } func (*DestroyTaskResponse) ProtoMessage() {} func (*DestroyTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{15} + return fileDescriptor_driver_8793c97f57ec8d84, []int{15} } func (m *DestroyTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskResponse.Unmarshal(m, b) @@ -897,7 +897,7 @@ func (m *InspectTaskRequest) Reset() { *m = InspectTaskRequest{} } func (m *InspectTaskRequest) String() string { return proto.CompactTextString(m) } func (*InspectTaskRequest) ProtoMessage() {} func (*InspectTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{16} + return fileDescriptor_driver_8793c97f57ec8d84, []int{16} } func (m *InspectTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskRequest.Unmarshal(m, b) @@ -940,7 +940,7 @@ func (m *InspectTaskResponse) Reset() { *m = InspectTaskResponse{} } func (m *InspectTaskResponse) String() string { return proto.CompactTextString(m) } func (*InspectTaskResponse) ProtoMessage() {} func (*InspectTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{17} + return fileDescriptor_driver_8793c97f57ec8d84, []int{17} } func (m *InspectTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskResponse.Unmarshal(m, b) @@ -995,7 +995,7 @@ func (m *TaskStatsRequest) Reset() { *m = TaskStatsRequest{} } func (m *TaskStatsRequest) String() string { return proto.CompactTextString(m) } func (*TaskStatsRequest) ProtoMessage() {} func (*TaskStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{18} + return fileDescriptor_driver_8793c97f57ec8d84, []int{18} } func (m *TaskStatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsRequest.Unmarshal(m, b) @@ -1041,7 +1041,7 @@ func (m *TaskStatsResponse) Reset() { *m = TaskStatsResponse{} } func (m *TaskStatsResponse) String() string { return proto.CompactTextString(m) } func (*TaskStatsResponse) ProtoMessage() {} func (*TaskStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{19} + return fileDescriptor_driver_8793c97f57ec8d84, []int{19} } func (m *TaskStatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsResponse.Unmarshal(m, b) @@ -1078,7 +1078,7 @@ func (m *TaskEventsRequest) Reset() { *m = TaskEventsRequest{} } func (m *TaskEventsRequest) String() string { return proto.CompactTextString(m) } func (*TaskEventsRequest) ProtoMessage() {} func (*TaskEventsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{20} + return fileDescriptor_driver_8793c97f57ec8d84, []int{20} } func (m *TaskEventsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskEventsRequest.Unmarshal(m, b) @@ -1112,7 +1112,7 @@ func (m *SignalTaskRequest) Reset() { *m = SignalTaskRequest{} } func (m *SignalTaskRequest) String() string { return proto.CompactTextString(m) } func (*SignalTaskRequest) ProtoMessage() {} func (*SignalTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{21} + return fileDescriptor_driver_8793c97f57ec8d84, []int{21} } func (m *SignalTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskRequest.Unmarshal(m, b) @@ -1156,7 +1156,7 @@ func (m *SignalTaskResponse) Reset() { *m = SignalTaskResponse{} } func (m *SignalTaskResponse) String() string { return proto.CompactTextString(m) } func (*SignalTaskResponse) ProtoMessage() {} func (*SignalTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{22} + return fileDescriptor_driver_8793c97f57ec8d84, []int{22} } func (m *SignalTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskResponse.Unmarshal(m, b) @@ -1193,7 +1193,7 @@ func (m *ExecTaskRequest) Reset() { *m = ExecTaskRequest{} } func (m *ExecTaskRequest) String() string { return proto.CompactTextString(m) } func (*ExecTaskRequest) ProtoMessage() {} func (*ExecTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{23} + return fileDescriptor_driver_8793c97f57ec8d84, []int{23} } func (m *ExecTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskRequest.Unmarshal(m, b) @@ -1250,7 +1250,7 @@ func (m *ExecTaskResponse) Reset() { *m = ExecTaskResponse{} } func (m *ExecTaskResponse) String() string { return proto.CompactTextString(m) } func (*ExecTaskResponse) ProtoMessage() {} func (*ExecTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{24} + return fileDescriptor_driver_8793c97f57ec8d84, []int{24} } func (m *ExecTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskResponse.Unmarshal(m, b) @@ -1291,6 +1291,268 @@ func (m *ExecTaskResponse) GetResult() *ExitResult { return nil } +type ExecTaskStreamingIOOperation struct { + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Close bool `protobuf:"varint,2,opt,name=close,proto3" json:"close,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExecTaskStreamingIOOperation) Reset() { *m = ExecTaskStreamingIOOperation{} } +func (m *ExecTaskStreamingIOOperation) String() string { return proto.CompactTextString(m) } +func (*ExecTaskStreamingIOOperation) ProtoMessage() {} +func (*ExecTaskStreamingIOOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_driver_8793c97f57ec8d84, []int{25} +} +func (m *ExecTaskStreamingIOOperation) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecTaskStreamingIOOperation.Unmarshal(m, b) +} +func (m *ExecTaskStreamingIOOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecTaskStreamingIOOperation.Marshal(b, m, deterministic) +} +func (dst *ExecTaskStreamingIOOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecTaskStreamingIOOperation.Merge(dst, src) +} +func (m *ExecTaskStreamingIOOperation) XXX_Size() int { + return xxx_messageInfo_ExecTaskStreamingIOOperation.Size(m) +} +func (m *ExecTaskStreamingIOOperation) XXX_DiscardUnknown() { + xxx_messageInfo_ExecTaskStreamingIOOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecTaskStreamingIOOperation proto.InternalMessageInfo + +func (m *ExecTaskStreamingIOOperation) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *ExecTaskStreamingIOOperation) GetClose() bool { + if m != nil { + return m.Close + } + return false +} + +type ExecTaskStreamingRequest struct { + Setup *ExecTaskStreamingRequest_Setup `protobuf:"bytes,1,opt,name=setup,proto3" json:"setup,omitempty"` + TtySize *ExecTaskStreamingRequest_TerminalSize `protobuf:"bytes,2,opt,name=tty_size,json=ttySize,proto3" json:"tty_size,omitempty"` + Stdin *ExecTaskStreamingIOOperation `protobuf:"bytes,3,opt,name=stdin,proto3" json:"stdin,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExecTaskStreamingRequest) Reset() { *m = ExecTaskStreamingRequest{} } +func (m *ExecTaskStreamingRequest) String() string { return proto.CompactTextString(m) } +func (*ExecTaskStreamingRequest) ProtoMessage() {} +func (*ExecTaskStreamingRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_driver_8793c97f57ec8d84, []int{26} +} +func (m *ExecTaskStreamingRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecTaskStreamingRequest.Unmarshal(m, b) +} +func (m *ExecTaskStreamingRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecTaskStreamingRequest.Marshal(b, m, deterministic) +} +func (dst *ExecTaskStreamingRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecTaskStreamingRequest.Merge(dst, src) +} +func (m *ExecTaskStreamingRequest) XXX_Size() int { + return xxx_messageInfo_ExecTaskStreamingRequest.Size(m) +} +func (m *ExecTaskStreamingRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ExecTaskStreamingRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecTaskStreamingRequest proto.InternalMessageInfo + +func (m *ExecTaskStreamingRequest) GetSetup() *ExecTaskStreamingRequest_Setup { + if m != nil { + return m.Setup + } + return nil +} + +func (m *ExecTaskStreamingRequest) GetTtySize() *ExecTaskStreamingRequest_TerminalSize { + if m != nil { + return m.TtySize + } + return nil +} + +func (m *ExecTaskStreamingRequest) GetStdin() *ExecTaskStreamingIOOperation { + if m != nil { + return m.Stdin + } + return nil +} + +type ExecTaskStreamingRequest_Setup struct { + TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + Command []string `protobuf:"bytes,2,rep,name=command,proto3" json:"command,omitempty"` + Tty bool `protobuf:"varint,3,opt,name=tty,proto3" json:"tty,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExecTaskStreamingRequest_Setup) Reset() { *m = ExecTaskStreamingRequest_Setup{} } +func (m *ExecTaskStreamingRequest_Setup) String() string { return proto.CompactTextString(m) } +func (*ExecTaskStreamingRequest_Setup) ProtoMessage() {} +func (*ExecTaskStreamingRequest_Setup) Descriptor() ([]byte, []int) { + return fileDescriptor_driver_8793c97f57ec8d84, []int{26, 0} +} +func (m *ExecTaskStreamingRequest_Setup) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecTaskStreamingRequest_Setup.Unmarshal(m, b) +} +func (m *ExecTaskStreamingRequest_Setup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecTaskStreamingRequest_Setup.Marshal(b, m, deterministic) +} +func (dst *ExecTaskStreamingRequest_Setup) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecTaskStreamingRequest_Setup.Merge(dst, src) +} +func (m *ExecTaskStreamingRequest_Setup) XXX_Size() int { + return xxx_messageInfo_ExecTaskStreamingRequest_Setup.Size(m) +} +func (m *ExecTaskStreamingRequest_Setup) XXX_DiscardUnknown() { + xxx_messageInfo_ExecTaskStreamingRequest_Setup.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecTaskStreamingRequest_Setup proto.InternalMessageInfo + +func (m *ExecTaskStreamingRequest_Setup) GetTaskId() string { + if m != nil { + return m.TaskId + } + return "" +} + +func (m *ExecTaskStreamingRequest_Setup) GetCommand() []string { + if m != nil { + return m.Command + } + return nil +} + +func (m *ExecTaskStreamingRequest_Setup) GetTty() bool { + if m != nil { + return m.Tty + } + return false +} + +type ExecTaskStreamingRequest_TerminalSize struct { + Height int32 `protobuf:"varint,1,opt,name=height,proto3" json:"height,omitempty"` + Width int32 `protobuf:"varint,2,opt,name=width,proto3" json:"width,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExecTaskStreamingRequest_TerminalSize) Reset() { *m = ExecTaskStreamingRequest_TerminalSize{} } +func (m *ExecTaskStreamingRequest_TerminalSize) String() string { return proto.CompactTextString(m) } +func (*ExecTaskStreamingRequest_TerminalSize) ProtoMessage() {} +func (*ExecTaskStreamingRequest_TerminalSize) Descriptor() ([]byte, []int) { + return fileDescriptor_driver_8793c97f57ec8d84, []int{26, 1} +} +func (m *ExecTaskStreamingRequest_TerminalSize) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.Unmarshal(m, b) +} +func (m *ExecTaskStreamingRequest_TerminalSize) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.Marshal(b, m, deterministic) +} +func (dst *ExecTaskStreamingRequest_TerminalSize) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.Merge(dst, src) +} +func (m *ExecTaskStreamingRequest_TerminalSize) XXX_Size() int { + return xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.Size(m) +} +func (m *ExecTaskStreamingRequest_TerminalSize) XXX_DiscardUnknown() { + xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize proto.InternalMessageInfo + +func (m *ExecTaskStreamingRequest_TerminalSize) GetHeight() int32 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *ExecTaskStreamingRequest_TerminalSize) GetWidth() int32 { + if m != nil { + return m.Width + } + return 0 +} + +type ExecTaskStreamingResponse struct { + Stdout *ExecTaskStreamingIOOperation `protobuf:"bytes,1,opt,name=stdout,proto3" json:"stdout,omitempty"` + Stderr *ExecTaskStreamingIOOperation `protobuf:"bytes,2,opt,name=stderr,proto3" json:"stderr,omitempty"` + Exited bool `protobuf:"varint,3,opt,name=exited,proto3" json:"exited,omitempty"` + Result *ExitResult `protobuf:"bytes,4,opt,name=result,proto3" json:"result,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ExecTaskStreamingResponse) Reset() { *m = ExecTaskStreamingResponse{} } +func (m *ExecTaskStreamingResponse) String() string { return proto.CompactTextString(m) } +func (*ExecTaskStreamingResponse) ProtoMessage() {} +func (*ExecTaskStreamingResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_driver_8793c97f57ec8d84, []int{27} +} +func (m *ExecTaskStreamingResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ExecTaskStreamingResponse.Unmarshal(m, b) +} +func (m *ExecTaskStreamingResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ExecTaskStreamingResponse.Marshal(b, m, deterministic) +} +func (dst *ExecTaskStreamingResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecTaskStreamingResponse.Merge(dst, src) +} +func (m *ExecTaskStreamingResponse) XXX_Size() int { + return xxx_messageInfo_ExecTaskStreamingResponse.Size(m) +} +func (m *ExecTaskStreamingResponse) XXX_DiscardUnknown() { + xxx_messageInfo_ExecTaskStreamingResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecTaskStreamingResponse proto.InternalMessageInfo + +func (m *ExecTaskStreamingResponse) GetStdout() *ExecTaskStreamingIOOperation { + if m != nil { + return m.Stdout + } + return nil +} + +func (m *ExecTaskStreamingResponse) GetStderr() *ExecTaskStreamingIOOperation { + if m != nil { + return m.Stderr + } + return nil +} + +func (m *ExecTaskStreamingResponse) GetExited() bool { + if m != nil { + return m.Exited + } + return false +} + +func (m *ExecTaskStreamingResponse) GetResult() *ExitResult { + if m != nil { + return m.Result + } + return nil +} + type DriverCapabilities struct { // SendSignals indicates that the driver can send process signals (ex. SIGUSR1) // to the task. @@ -1309,7 +1571,7 @@ func (m *DriverCapabilities) Reset() { *m = DriverCapabilities{} } func (m *DriverCapabilities) String() string { return proto.CompactTextString(m) } func (*DriverCapabilities) ProtoMessage() {} func (*DriverCapabilities) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{25} + return fileDescriptor_driver_8793c97f57ec8d84, []int{28} } func (m *DriverCapabilities) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverCapabilities.Unmarshal(m, b) @@ -1395,7 +1657,7 @@ func (m *TaskConfig) Reset() { *m = TaskConfig{} } func (m *TaskConfig) String() string { return proto.CompactTextString(m) } func (*TaskConfig) ProtoMessage() {} func (*TaskConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{26} + return fileDescriptor_driver_8793c97f57ec8d84, []int{29} } func (m *TaskConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfig.Unmarshal(m, b) @@ -1534,7 +1796,7 @@ func (m *Resources) Reset() { *m = Resources{} } func (m *Resources) String() string { return proto.CompactTextString(m) } func (*Resources) ProtoMessage() {} func (*Resources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{27} + return fileDescriptor_driver_8793c97f57ec8d84, []int{30} } func (m *Resources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Resources.Unmarshal(m, b) @@ -1581,7 +1843,7 @@ func (m *AllocatedTaskResources) Reset() { *m = AllocatedTaskResources{} func (m *AllocatedTaskResources) String() string { return proto.CompactTextString(m) } func (*AllocatedTaskResources) ProtoMessage() {} func (*AllocatedTaskResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{28} + return fileDescriptor_driver_8793c97f57ec8d84, []int{31} } func (m *AllocatedTaskResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedTaskResources.Unmarshal(m, b) @@ -1633,7 +1895,7 @@ func (m *AllocatedCpuResources) Reset() { *m = AllocatedCpuResources{} } func (m *AllocatedCpuResources) String() string { return proto.CompactTextString(m) } func (*AllocatedCpuResources) ProtoMessage() {} func (*AllocatedCpuResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{29} + return fileDescriptor_driver_8793c97f57ec8d84, []int{32} } func (m *AllocatedCpuResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedCpuResources.Unmarshal(m, b) @@ -1671,7 +1933,7 @@ func (m *AllocatedMemoryResources) Reset() { *m = AllocatedMemoryResourc func (m *AllocatedMemoryResources) String() string { return proto.CompactTextString(m) } func (*AllocatedMemoryResources) ProtoMessage() {} func (*AllocatedMemoryResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{30} + return fileDescriptor_driver_8793c97f57ec8d84, []int{33} } func (m *AllocatedMemoryResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedMemoryResources.Unmarshal(m, b) @@ -1714,7 +1976,7 @@ func (m *NetworkResource) Reset() { *m = NetworkResource{} } func (m *NetworkResource) String() string { return proto.CompactTextString(m) } func (*NetworkResource) ProtoMessage() {} func (*NetworkResource) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{31} + return fileDescriptor_driver_8793c97f57ec8d84, []int{34} } func (m *NetworkResource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkResource.Unmarshal(m, b) @@ -1788,7 +2050,7 @@ func (m *NetworkPort) Reset() { *m = NetworkPort{} } func (m *NetworkPort) String() string { return proto.CompactTextString(m) } func (*NetworkPort) ProtoMessage() {} func (*NetworkPort) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{32} + return fileDescriptor_driver_8793c97f57ec8d84, []int{35} } func (m *NetworkPort) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkPort.Unmarshal(m, b) @@ -1848,7 +2110,7 @@ func (m *LinuxResources) Reset() { *m = LinuxResources{} } func (m *LinuxResources) String() string { return proto.CompactTextString(m) } func (*LinuxResources) ProtoMessage() {} func (*LinuxResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{33} + return fileDescriptor_driver_8793c97f57ec8d84, []int{36} } func (m *LinuxResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_LinuxResources.Unmarshal(m, b) @@ -1940,7 +2202,7 @@ func (m *Mount) Reset() { *m = Mount{} } func (m *Mount) String() string { return proto.CompactTextString(m) } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{34} + return fileDescriptor_driver_8793c97f57ec8d84, []int{37} } func (m *Mount) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Mount.Unmarshal(m, b) @@ -2003,7 +2265,7 @@ func (m *Device) Reset() { *m = Device{} } func (m *Device) String() string { return proto.CompactTextString(m) } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{35} + return fileDescriptor_driver_8793c97f57ec8d84, []int{38} } func (m *Device) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Device.Unmarshal(m, b) @@ -2064,7 +2326,7 @@ func (m *TaskHandle) Reset() { *m = TaskHandle{} } func (m *TaskHandle) String() string { return proto.CompactTextString(m) } func (*TaskHandle) ProtoMessage() {} func (*TaskHandle) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{36} + return fileDescriptor_driver_8793c97f57ec8d84, []int{39} } func (m *TaskHandle) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskHandle.Unmarshal(m, b) @@ -2131,7 +2393,7 @@ func (m *NetworkOverride) Reset() { *m = NetworkOverride{} } func (m *NetworkOverride) String() string { return proto.CompactTextString(m) } func (*NetworkOverride) ProtoMessage() {} func (*NetworkOverride) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{37} + return fileDescriptor_driver_8793c97f57ec8d84, []int{40} } func (m *NetworkOverride) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkOverride.Unmarshal(m, b) @@ -2189,7 +2451,7 @@ func (m *ExitResult) Reset() { *m = ExitResult{} } func (m *ExitResult) String() string { return proto.CompactTextString(m) } func (*ExitResult) ProtoMessage() {} func (*ExitResult) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{38} + return fileDescriptor_driver_8793c97f57ec8d84, []int{41} } func (m *ExitResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExitResult.Unmarshal(m, b) @@ -2252,7 +2514,7 @@ func (m *TaskStatus) Reset() { *m = TaskStatus{} } func (m *TaskStatus) String() string { return proto.CompactTextString(m) } func (*TaskStatus) ProtoMessage() {} func (*TaskStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{39} + return fileDescriptor_driver_8793c97f57ec8d84, []int{42} } func (m *TaskStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatus.Unmarshal(m, b) @@ -2327,7 +2589,7 @@ func (m *TaskDriverStatus) Reset() { *m = TaskDriverStatus{} } func (m *TaskDriverStatus) String() string { return proto.CompactTextString(m) } func (*TaskDriverStatus) ProtoMessage() {} func (*TaskDriverStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{40} + return fileDescriptor_driver_8793c97f57ec8d84, []int{43} } func (m *TaskDriverStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskDriverStatus.Unmarshal(m, b) @@ -2372,7 +2634,7 @@ func (m *TaskStats) Reset() { *m = TaskStats{} } func (m *TaskStats) String() string { return proto.CompactTextString(m) } func (*TaskStats) ProtoMessage() {} func (*TaskStats) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{41} + return fileDescriptor_driver_8793c97f57ec8d84, []int{44} } func (m *TaskStats) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStats.Unmarshal(m, b) @@ -2434,7 +2696,7 @@ func (m *TaskResourceUsage) Reset() { *m = TaskResourceUsage{} } func (m *TaskResourceUsage) String() string { return proto.CompactTextString(m) } func (*TaskResourceUsage) ProtoMessage() {} func (*TaskResourceUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{42} + return fileDescriptor_driver_8793c97f57ec8d84, []int{45} } func (m *TaskResourceUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskResourceUsage.Unmarshal(m, b) @@ -2486,7 +2748,7 @@ func (m *CPUUsage) Reset() { *m = CPUUsage{} } func (m *CPUUsage) String() string { return proto.CompactTextString(m) } func (*CPUUsage) ProtoMessage() {} func (*CPUUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{43} + return fileDescriptor_driver_8793c97f57ec8d84, []int{46} } func (m *CPUUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CPUUsage.Unmarshal(m, b) @@ -2574,7 +2836,7 @@ func (m *MemoryUsage) Reset() { *m = MemoryUsage{} } func (m *MemoryUsage) String() string { return proto.CompactTextString(m) } func (*MemoryUsage) ProtoMessage() {} func (*MemoryUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{44} + return fileDescriptor_driver_8793c97f57ec8d84, []int{47} } func (m *MemoryUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MemoryUsage.Unmarshal(m, b) @@ -2672,7 +2934,7 @@ func (m *DriverTaskEvent) Reset() { *m = DriverTaskEvent{} } func (m *DriverTaskEvent) String() string { return proto.CompactTextString(m) } func (*DriverTaskEvent) ProtoMessage() {} func (*DriverTaskEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8c4ef0bcc46739f6, []int{45} + return fileDescriptor_driver_8793c97f57ec8d84, []int{48} } func (m *DriverTaskEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverTaskEvent.Unmarshal(m, b) @@ -2761,6 +3023,11 @@ func init() { proto.RegisterType((*SignalTaskResponse)(nil), "hashicorp.nomad.plugins.drivers.proto.SignalTaskResponse") proto.RegisterType((*ExecTaskRequest)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskRequest") proto.RegisterType((*ExecTaskResponse)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskResponse") + proto.RegisterType((*ExecTaskStreamingIOOperation)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskStreamingIOOperation") + proto.RegisterType((*ExecTaskStreamingRequest)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskStreamingRequest") + proto.RegisterType((*ExecTaskStreamingRequest_Setup)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskStreamingRequest.Setup") + proto.RegisterType((*ExecTaskStreamingRequest_TerminalSize)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskStreamingRequest.TerminalSize") + proto.RegisterType((*ExecTaskStreamingResponse)(nil), "hashicorp.nomad.plugins.drivers.proto.ExecTaskStreamingResponse") proto.RegisterType((*DriverCapabilities)(nil), "hashicorp.nomad.plugins.drivers.proto.DriverCapabilities") proto.RegisterType((*TaskConfig)(nil), "hashicorp.nomad.plugins.drivers.proto.TaskConfig") proto.RegisterMapType((map[string]string)(nil), "hashicorp.nomad.plugins.drivers.proto.TaskConfig.DeviceEnvEntry") @@ -3407,198 +3674,210 @@ var _Driver_serviceDesc = grpc.ServiceDesc{ } func init() { - proto.RegisterFile("plugins/drivers/proto/driver.proto", fileDescriptor_driver_8c4ef0bcc46739f6) + proto.RegisterFile("plugins/drivers/proto/driver.proto", fileDescriptor_driver_8793c97f57ec8d84) } -var fileDescriptor_driver_8c4ef0bcc46739f6 = []byte{ - // 3009 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x59, 0xdb, 0x6f, 0xe3, 0xc6, - 0xd5, 0xb7, 0xae, 0x96, 0x8e, 0x6c, 0x99, 0x3b, 0xbb, 0x9b, 0x28, 0x0a, 0xbe, 0x2f, 0x1b, 0x02, - 0xf9, 0x60, 0x24, 0x59, 0x39, 0x71, 0xf0, 0xed, 0xad, 0xb9, 0x29, 0x12, 0xd7, 0x76, 0xd6, 0x96, - 0xdd, 0x91, 0x8c, 0xcd, 0xb6, 0x4d, 0x58, 0x9a, 0x9c, 0x95, 0xb8, 0xe6, 0x2d, 0xe4, 0xd0, 0x6b, - 0xa3, 0x28, 0x5a, 0xa4, 0x40, 0xd1, 0x3e, 0x14, 0xe8, 0x4b, 0x90, 0xf7, 0xf6, 0xb1, 0x7f, 0x41, - 0x5b, 0xe4, 0x2f, 0x69, 0x5f, 0x5a, 0xa0, 0x40, 0x5f, 0xfb, 0x1f, 0x14, 0x73, 0x21, 0x45, 0xd9, - 0xde, 0x2c, 0xa5, 0xcd, 0x13, 0x79, 0xce, 0xcc, 0xf9, 0xcd, 0x99, 0x39, 0x67, 0xe6, 0x9c, 0x99, - 0x03, 0x6a, 0xe0, 0xc4, 0x63, 0xdb, 0x8b, 0x36, 0xac, 0xd0, 0x3e, 0x21, 0x61, 0xb4, 0x11, 0x84, - 0x3e, 0xf5, 0x25, 0xd5, 0xe1, 0x04, 0x7a, 0x63, 0x62, 0x44, 0x13, 0xdb, 0xf4, 0xc3, 0xa0, 0xe3, - 0xf9, 0xae, 0x61, 0x75, 0xa4, 0x4c, 0x47, 0xca, 0x88, 0x6e, 0xed, 0xff, 0x1d, 0xfb, 0xfe, 0xd8, - 0x21, 0x02, 0xe1, 0x28, 0x7e, 0xbc, 0x61, 0xc5, 0xa1, 0x41, 0x6d, 0xdf, 0x93, 0xed, 0xaf, 0x9d, - 0x6f, 0xa7, 0xb6, 0x4b, 0x22, 0x6a, 0xb8, 0x81, 0xec, 0xf0, 0xf1, 0xd8, 0xa6, 0x93, 0xf8, 0xa8, - 0x63, 0xfa, 0xee, 0x46, 0x3a, 0xe4, 0x06, 0x1f, 0x72, 0x23, 0x51, 0x33, 0x9a, 0x18, 0x21, 0xb1, - 0x36, 0x26, 0xa6, 0x13, 0x05, 0xc4, 0x64, 0x5f, 0x9d, 0xfd, 0x48, 0x84, 0xad, 0xfc, 0x08, 0x11, - 0x0d, 0x63, 0x93, 0x26, 0xf3, 0x35, 0x28, 0x0d, 0xed, 0xa3, 0x98, 0x12, 0x01, 0xa4, 0xbe, 0x02, - 0x2f, 0x8f, 0x8c, 0xe8, 0xb8, 0xe7, 0x7b, 0x8f, 0xed, 0xf1, 0xd0, 0x9c, 0x10, 0xd7, 0xc0, 0xe4, - 0xcb, 0x98, 0x44, 0x54, 0xfd, 0x09, 0xb4, 0x2e, 0x36, 0x45, 0x81, 0xef, 0x45, 0x04, 0x7d, 0x0c, - 0x65, 0xa6, 0x4d, 0xab, 0x70, 0xa3, 0xb0, 0xde, 0xd8, 0x7c, 0xbb, 0xf3, 0xac, 0x85, 0x13, 0x3a, - 0x74, 0xe4, 0x2c, 0x3a, 0xc3, 0x80, 0x98, 0x98, 0x4b, 0xaa, 0xd7, 0xe1, 0x6a, 0xcf, 0x08, 0x8c, - 0x23, 0xdb, 0xb1, 0xa9, 0x4d, 0xa2, 0x64, 0xd0, 0x18, 0xae, 0xcd, 0xb2, 0xe5, 0x80, 0x9f, 0xc3, - 0x8a, 0x99, 0xe1, 0xcb, 0x81, 0xef, 0x76, 0x72, 0x59, 0xac, 0xd3, 0xe7, 0xd4, 0x0c, 0xf0, 0x0c, - 0x9c, 0x7a, 0x0d, 0xd0, 0x7d, 0xdb, 0x1b, 0x93, 0x30, 0x08, 0x6d, 0x8f, 0x26, 0xca, 0x7c, 0x5b, - 0x82, 0xab, 0x33, 0x6c, 0xa9, 0xcc, 0x13, 0x80, 0x74, 0x1d, 0x99, 0x2a, 0xa5, 0xf5, 0xc6, 0xe6, - 0xa7, 0x39, 0x55, 0xb9, 0x04, 0xaf, 0xd3, 0x4d, 0xc1, 0x34, 0x8f, 0x86, 0x67, 0x38, 0x83, 0x8e, - 0xbe, 0x80, 0xea, 0x84, 0x18, 0x0e, 0x9d, 0xb4, 0x8a, 0x37, 0x0a, 0xeb, 0xcd, 0xcd, 0xfb, 0x2f, - 0x30, 0xce, 0x36, 0x07, 0x1a, 0x52, 0x83, 0x12, 0x2c, 0x51, 0xd1, 0x4d, 0x40, 0xe2, 0x4f, 0xb7, - 0x48, 0x64, 0x86, 0x76, 0xc0, 0x1c, 0xb9, 0x55, 0xba, 0x51, 0x58, 0xaf, 0xe3, 0x2b, 0xa2, 0xa5, - 0x3f, 0x6d, 0x68, 0x07, 0xb0, 0x76, 0x4e, 0x5b, 0xa4, 0x40, 0xe9, 0x98, 0x9c, 0x71, 0x8b, 0xd4, - 0x31, 0xfb, 0x45, 0x5b, 0x50, 0x39, 0x31, 0x9c, 0x98, 0x70, 0x95, 0x1b, 0x9b, 0xef, 0x3e, 0xcf, - 0x3d, 0xa4, 0x8b, 0x4e, 0xd7, 0x01, 0x0b, 0xf9, 0x7b, 0xc5, 0x3b, 0x05, 0xf5, 0x2e, 0x34, 0x32, - 0x7a, 0xa3, 0x26, 0xc0, 0xe1, 0xa0, 0xaf, 0x8d, 0xb4, 0xde, 0x48, 0xeb, 0x2b, 0x4b, 0x68, 0x15, - 0xea, 0x87, 0x83, 0x6d, 0xad, 0xbb, 0x3b, 0xda, 0x7e, 0xa4, 0x14, 0x50, 0x03, 0x96, 0x13, 0xa2, - 0xa8, 0x9e, 0x02, 0xc2, 0xc4, 0xf4, 0x4f, 0x48, 0xc8, 0x1c, 0x59, 0x5a, 0x15, 0xbd, 0x0c, 0xcb, - 0xd4, 0x88, 0x8e, 0x75, 0xdb, 0x92, 0x3a, 0x57, 0x19, 0xb9, 0x63, 0xa1, 0x1d, 0xa8, 0x4e, 0x0c, - 0xcf, 0x72, 0x9e, 0xaf, 0xf7, 0xec, 0x52, 0x33, 0xf0, 0x6d, 0x2e, 0x88, 0x25, 0x00, 0xf3, 0xee, - 0x99, 0x91, 0x85, 0x01, 0xd4, 0x47, 0xa0, 0x0c, 0xa9, 0x11, 0xd2, 0xac, 0x3a, 0x1a, 0x94, 0xd9, - 0xf8, 0xd2, 0xa3, 0xe7, 0x19, 0x53, 0xec, 0x4c, 0xcc, 0xc5, 0xd5, 0xff, 0x14, 0xe1, 0x4a, 0x06, - 0x5b, 0x7a, 0xea, 0x43, 0xa8, 0x86, 0x24, 0x8a, 0x1d, 0xca, 0xe1, 0x9b, 0x9b, 0x1f, 0xe5, 0x84, - 0xbf, 0x80, 0xd4, 0xc1, 0x1c, 0x06, 0x4b, 0x38, 0xb4, 0x0e, 0x8a, 0x90, 0xd0, 0x49, 0x18, 0xfa, - 0xa1, 0xee, 0x46, 0x63, 0xbe, 0x6a, 0x75, 0xdc, 0x14, 0x7c, 0x8d, 0xb1, 0xf7, 0xa2, 0x71, 0x66, - 0x55, 0x4b, 0x2f, 0xb8, 0xaa, 0xc8, 0x00, 0xc5, 0x23, 0xf4, 0xa9, 0x1f, 0x1e, 0xeb, 0x6c, 0x69, - 0x43, 0xdb, 0x22, 0xad, 0x32, 0x07, 0xbd, 0x95, 0x13, 0x74, 0x20, 0xc4, 0xf7, 0xa5, 0x34, 0x5e, - 0xf3, 0x66, 0x19, 0xea, 0x5b, 0x50, 0x15, 0x33, 0x65, 0x9e, 0x34, 0x3c, 0xec, 0xf5, 0xb4, 0xe1, - 0x50, 0x59, 0x42, 0x75, 0xa8, 0x60, 0x6d, 0x84, 0x99, 0x87, 0xd5, 0xa1, 0x72, 0xbf, 0x3b, 0xea, - 0xee, 0x2a, 0x45, 0xf5, 0x4d, 0x58, 0x7b, 0x68, 0xd8, 0x34, 0x8f, 0x73, 0xa9, 0x3e, 0x28, 0xd3, - 0xbe, 0xd2, 0x3a, 0x3b, 0x33, 0xd6, 0xc9, 0xbf, 0x34, 0xda, 0xa9, 0x4d, 0xcf, 0xd9, 0x43, 0x81, - 0x12, 0x09, 0x43, 0x69, 0x02, 0xf6, 0xab, 0x3e, 0x85, 0xb5, 0x21, 0xf5, 0x83, 0x5c, 0x9e, 0xff, - 0x1e, 0x2c, 0xb3, 0x18, 0xe5, 0xc7, 0x54, 0xba, 0xfe, 0x2b, 0x1d, 0x11, 0xc3, 0x3a, 0x49, 0x0c, - 0xeb, 0xf4, 0x65, 0x8c, 0xc3, 0x49, 0x4f, 0xf4, 0x12, 0x54, 0x23, 0x7b, 0xec, 0x19, 0x8e, 0x3c, - 0x2d, 0x24, 0xa5, 0x22, 0xe6, 0xe4, 0xc9, 0xc0, 0xd2, 0xf1, 0x7b, 0x80, 0xfa, 0x24, 0xa2, 0xa1, - 0x7f, 0x96, 0x4b, 0x9f, 0x6b, 0x50, 0x79, 0xec, 0x87, 0xa6, 0xd8, 0x88, 0x35, 0x2c, 0x08, 0xb6, - 0xa9, 0x66, 0x40, 0x24, 0xf6, 0x4d, 0x40, 0x3b, 0x1e, 0x8b, 0x29, 0xf9, 0x0c, 0xf1, 0xfb, 0x22, - 0x5c, 0x9d, 0xe9, 0x2f, 0x8d, 0xb1, 0xf8, 0x3e, 0x64, 0x07, 0x53, 0x1c, 0x89, 0x7d, 0x88, 0xf6, - 0xa1, 0x2a, 0x7a, 0xc8, 0x95, 0xbc, 0x3d, 0x07, 0x90, 0x08, 0x53, 0x12, 0x4e, 0xc2, 0x5c, 0xea, - 0xf4, 0xa5, 0xef, 0xd7, 0xe9, 0x9f, 0x82, 0x92, 0xcc, 0x23, 0x7a, 0xae, 0x6d, 0x3e, 0x85, 0xab, - 0xa6, 0xef, 0x38, 0xc4, 0x64, 0xde, 0xa0, 0xdb, 0x1e, 0x25, 0xe1, 0x89, 0xe1, 0x3c, 0xdf, 0x6f, - 0xd0, 0x54, 0x6a, 0x47, 0x0a, 0xa9, 0x3f, 0x86, 0x2b, 0x99, 0x81, 0xa5, 0x21, 0xee, 0x43, 0x25, - 0x62, 0x0c, 0x69, 0x89, 0x77, 0xe6, 0xb4, 0x44, 0x84, 0x85, 0xb8, 0x7a, 0x55, 0x80, 0x6b, 0x27, - 0xc4, 0x4b, 0xa7, 0xa5, 0xf6, 0xe1, 0xca, 0x90, 0xbb, 0x69, 0x2e, 0x3f, 0x9c, 0xba, 0x78, 0x71, - 0xc6, 0xc5, 0xaf, 0x01, 0xca, 0xa2, 0x48, 0x47, 0x3c, 0x83, 0x35, 0xed, 0x94, 0x98, 0xb9, 0x90, - 0x5b, 0xb0, 0x6c, 0xfa, 0xae, 0x6b, 0x78, 0x56, 0xab, 0x78, 0xa3, 0xb4, 0x5e, 0xc7, 0x09, 0x99, - 0xdd, 0x8b, 0xa5, 0xbc, 0x7b, 0x51, 0xfd, 0x5d, 0x01, 0x94, 0xe9, 0xd8, 0x72, 0x21, 0x99, 0xf6, - 0xd4, 0x62, 0x40, 0x6c, 0xec, 0x15, 0x2c, 0x29, 0xc9, 0x4f, 0x8e, 0x0b, 0xc1, 0x27, 0x61, 0x98, - 0x39, 0x8e, 0x4a, 0x2f, 0x78, 0x1c, 0xa9, 0xff, 0x2a, 0x00, 0xba, 0x98, 0x74, 0xa1, 0xd7, 0x61, - 0x25, 0x22, 0x9e, 0xa5, 0x8b, 0x65, 0x14, 0x16, 0xae, 0xe1, 0x06, 0xe3, 0x89, 0xf5, 0x8c, 0x10, - 0x82, 0x32, 0x39, 0x25, 0xa6, 0xdc, 0xf9, 0xfc, 0x1f, 0x4d, 0x60, 0xe5, 0x71, 0xa4, 0xdb, 0x91, - 0xef, 0x18, 0x69, 0x76, 0xd2, 0xdc, 0xd4, 0x16, 0x4e, 0xfe, 0x3a, 0xf7, 0x87, 0x3b, 0x09, 0x18, - 0x6e, 0x3c, 0x8e, 0x52, 0x42, 0xed, 0x40, 0x23, 0xd3, 0x86, 0x6a, 0x50, 0x1e, 0xec, 0x0f, 0x34, - 0x65, 0x09, 0x01, 0x54, 0x7b, 0xdb, 0x78, 0x7f, 0x7f, 0x24, 0x22, 0xc0, 0xce, 0x5e, 0x77, 0x4b, - 0x53, 0x8a, 0xea, 0x9f, 0xab, 0x00, 0xd3, 0x50, 0x8c, 0x9a, 0x50, 0x4c, 0x2d, 0x5d, 0xb4, 0x2d, - 0x36, 0x19, 0xcf, 0x70, 0x89, 0xf4, 0x1e, 0xfe, 0x8f, 0x36, 0xe1, 0xba, 0x1b, 0x8d, 0x03, 0xc3, - 0x3c, 0xd6, 0x65, 0x04, 0x35, 0xb9, 0x30, 0x9f, 0xd5, 0x0a, 0xbe, 0x2a, 0x1b, 0xa5, 0xd6, 0x02, - 0x77, 0x17, 0x4a, 0xc4, 0x3b, 0x69, 0x95, 0x79, 0xa6, 0x79, 0x6f, 0xee, 0x14, 0xa1, 0xa3, 0x79, - 0x27, 0x22, 0xb3, 0x64, 0x30, 0x48, 0x07, 0xb0, 0xc8, 0x89, 0x6d, 0x12, 0x9d, 0x81, 0x56, 0x38, - 0xe8, 0xc7, 0xf3, 0x83, 0xf6, 0x39, 0x46, 0x0a, 0x5d, 0xb7, 0x12, 0x1a, 0x0d, 0xa0, 0x1e, 0x92, - 0xc8, 0x8f, 0x43, 0x93, 0x44, 0xad, 0xea, 0x5c, 0xbb, 0x18, 0x27, 0x72, 0x78, 0x0a, 0x81, 0xfa, - 0x50, 0x75, 0xfd, 0xd8, 0xa3, 0x51, 0x6b, 0x99, 0x2b, 0xfb, 0x76, 0x4e, 0xb0, 0x3d, 0x26, 0x84, - 0xa5, 0x2c, 0xda, 0x82, 0x65, 0xa1, 0x62, 0xd4, 0xaa, 0x71, 0x98, 0x9b, 0x79, 0x1d, 0x88, 0x4b, - 0xe1, 0x44, 0x9a, 0x59, 0x35, 0x8e, 0x48, 0xd8, 0xaa, 0x0b, 0xab, 0xb2, 0x7f, 0xf4, 0x2a, 0xd4, - 0x0d, 0xc7, 0xf1, 0x4d, 0xdd, 0xb2, 0xc3, 0x16, 0xf0, 0x86, 0x1a, 0x67, 0xf4, 0xed, 0x10, 0xbd, - 0x06, 0x0d, 0xb1, 0xf5, 0xf4, 0xc0, 0xa0, 0x93, 0x56, 0x83, 0x37, 0x83, 0x60, 0x1d, 0x18, 0x74, - 0x22, 0x3b, 0x90, 0x30, 0x14, 0x1d, 0x56, 0xd2, 0x0e, 0x24, 0x0c, 0x79, 0x87, 0xff, 0x83, 0x35, - 0x7e, 0x8e, 0x8c, 0x43, 0x3f, 0x0e, 0x74, 0xee, 0x53, 0xab, 0xbc, 0xd3, 0x2a, 0x63, 0x6f, 0x31, - 0xee, 0x80, 0x39, 0xd7, 0x2b, 0x50, 0x7b, 0xe2, 0x1f, 0x89, 0x0e, 0x4d, 0xde, 0x61, 0xf9, 0x89, - 0x7f, 0x94, 0x34, 0x09, 0x0d, 0x6d, 0xab, 0xb5, 0x26, 0x9a, 0x38, 0xbd, 0x63, 0xb5, 0x6f, 0x41, - 0x2d, 0x31, 0xe3, 0x25, 0xd9, 0xfc, 0xb5, 0x6c, 0x36, 0x5f, 0xcf, 0xa4, 0xe6, 0xed, 0xf7, 0xa1, - 0x39, 0xeb, 0x04, 0xf3, 0x48, 0xab, 0x7f, 0x2b, 0x40, 0x3d, 0x35, 0x37, 0xf2, 0xe0, 0x2a, 0x57, - 0xc7, 0xa0, 0xc4, 0xd2, 0xa7, 0xde, 0x23, 0x62, 0xc0, 0x07, 0x39, 0x2d, 0xd5, 0x4d, 0x10, 0xe4, - 0x39, 0x28, 0x5d, 0x09, 0xa5, 0xc8, 0xd3, 0xf1, 0xbe, 0x80, 0x35, 0xc7, 0xf6, 0xe2, 0xd3, 0xcc, - 0x58, 0x22, 0x84, 0xfd, 0x7f, 0xce, 0xb1, 0x76, 0x99, 0xf4, 0x74, 0x8c, 0xa6, 0x33, 0x43, 0xab, - 0x5f, 0x17, 0xe1, 0xa5, 0xcb, 0xd5, 0x41, 0x03, 0x28, 0x99, 0x41, 0x2c, 0xa7, 0xf6, 0xfe, 0xbc, - 0x53, 0xeb, 0x05, 0xf1, 0x74, 0x54, 0x06, 0xc4, 0x92, 0x7c, 0x97, 0xb8, 0x7e, 0x78, 0x26, 0x67, - 0xf0, 0xd1, 0xbc, 0x90, 0x7b, 0x5c, 0x7a, 0x8a, 0x2a, 0xe1, 0x10, 0x86, 0x9a, 0x4c, 0x15, 0x22, - 0x79, 0x4c, 0xcc, 0x99, 0x72, 0x24, 0x90, 0x38, 0xc5, 0x51, 0x6f, 0xc1, 0xf5, 0x4b, 0xa7, 0x82, - 0xfe, 0x07, 0xc0, 0x0c, 0x62, 0x9d, 0x5f, 0x09, 0x85, 0xdd, 0x4b, 0xb8, 0x6e, 0x06, 0xf1, 0x90, - 0x33, 0xd4, 0xdb, 0xd0, 0x7a, 0x96, 0xbe, 0x6c, 0xf3, 0x09, 0x8d, 0x75, 0xf7, 0x88, 0xaf, 0x41, - 0x09, 0xd7, 0x04, 0x63, 0xef, 0x48, 0xfd, 0xa6, 0x08, 0x6b, 0xe7, 0xd4, 0x61, 0x11, 0x50, 0x6c, - 0xe6, 0x24, 0x2a, 0x0b, 0x8a, 0xed, 0x6c, 0xd3, 0xb6, 0x92, 0x34, 0x9a, 0xff, 0xf3, 0x33, 0x3d, - 0x90, 0x29, 0x6e, 0xd1, 0x0e, 0x98, 0x43, 0xbb, 0x47, 0x36, 0x8d, 0xf8, 0xcd, 0xa3, 0x82, 0x05, - 0x81, 0x1e, 0x41, 0x33, 0x24, 0x11, 0x09, 0x4f, 0x88, 0xa5, 0x07, 0x7e, 0x48, 0x93, 0x05, 0xdb, - 0x9c, 0x6f, 0xc1, 0x0e, 0xfc, 0x90, 0xe2, 0xd5, 0x04, 0x89, 0x51, 0x11, 0x7a, 0x08, 0xab, 0xd6, - 0x99, 0x67, 0xb8, 0xb6, 0x29, 0x91, 0xab, 0x0b, 0x23, 0xaf, 0x48, 0x20, 0x0e, 0xcc, 0x6e, 0xd6, - 0x99, 0x46, 0x36, 0x31, 0xc7, 0x38, 0x22, 0x8e, 0x5c, 0x13, 0x41, 0xcc, 0xee, 0xdf, 0x8a, 0xdc, - 0xbf, 0xea, 0x1f, 0x8b, 0xd0, 0x9c, 0xdd, 0x00, 0x89, 0xfd, 0x02, 0x12, 0xda, 0xbe, 0x95, 0xb1, - 0xdf, 0x01, 0x67, 0x30, 0x1b, 0xb1, 0xe6, 0x2f, 0x63, 0x9f, 0x1a, 0x89, 0x8d, 0xcc, 0x20, 0xfe, - 0x21, 0xa3, 0xcf, 0xd9, 0xbe, 0x74, 0xce, 0xf6, 0xe8, 0x6d, 0x40, 0xd2, 0xbe, 0x8e, 0xed, 0xda, - 0x54, 0x3f, 0x3a, 0xa3, 0x44, 0xac, 0x7f, 0x09, 0x2b, 0xa2, 0x65, 0x97, 0x35, 0x7c, 0xc2, 0xf8, - 0x48, 0x85, 0x55, 0xdf, 0x77, 0xf5, 0xc8, 0xf4, 0x43, 0xa2, 0x1b, 0xd6, 0x93, 0x56, 0x85, 0x77, - 0x6c, 0xf8, 0xbe, 0x3b, 0x64, 0xbc, 0xae, 0xf5, 0x84, 0x1d, 0xb8, 0x66, 0x10, 0x47, 0x84, 0xea, - 0xec, 0xc3, 0x63, 0x54, 0x1d, 0x83, 0x60, 0xf5, 0x82, 0x38, 0xca, 0x74, 0x70, 0x89, 0xcb, 0xe2, - 0x4e, 0xa6, 0xc3, 0x1e, 0x71, 0xd9, 0x28, 0x2b, 0x07, 0x24, 0x34, 0x89, 0x47, 0x47, 0xb6, 0x79, - 0xcc, 0x42, 0x4a, 0x61, 0xbd, 0x80, 0x67, 0x78, 0xea, 0xe7, 0x50, 0xe1, 0x21, 0x88, 0x4d, 0x9e, - 0x1f, 0xdf, 0xfc, 0x74, 0x17, 0xcb, 0x5b, 0x63, 0x0c, 0x7e, 0xb6, 0xbf, 0x0a, 0xf5, 0x89, 0x1f, - 0xc9, 0xd8, 0x20, 0x3c, 0xaf, 0xc6, 0x18, 0xbc, 0xb1, 0x0d, 0xb5, 0x90, 0x18, 0x96, 0xef, 0x39, - 0x67, 0x7c, 0x5d, 0x6a, 0x38, 0xa5, 0xd5, 0x2f, 0xa1, 0x2a, 0x8e, 0xdf, 0x17, 0xc0, 0xbf, 0x09, - 0xc8, 0x14, 0x41, 0x25, 0x20, 0xa1, 0x6b, 0x47, 0x91, 0xed, 0x7b, 0x51, 0xf2, 0xfc, 0x23, 0x5a, - 0x0e, 0xa6, 0x0d, 0xea, 0xdf, 0x0b, 0x22, 0xdf, 0x11, 0x17, 0x73, 0x96, 0xc5, 0x32, 0x4f, 0x63, - 0x39, 0x59, 0x81, 0xbb, 0x47, 0x42, 0xb2, 0x5c, 0x52, 0xa6, 0x35, 0xc5, 0x45, 0xdf, 0x35, 0x24, - 0x40, 0x72, 0x1f, 0x20, 0x32, 0xed, 0x9b, 0xf7, 0x3e, 0x40, 0xc4, 0x7d, 0x80, 0xb0, 0xe4, 0x53, - 0x26, 0x5c, 0x02, 0xae, 0xcc, 0xf3, 0xad, 0x86, 0x95, 0x5e, 0xba, 0x88, 0xfa, 0xef, 0x42, 0x7a, - 0x56, 0x24, 0x97, 0x23, 0xf4, 0x05, 0xd4, 0xd8, 0xb6, 0xd3, 0x5d, 0x23, 0x90, 0x4f, 0x7d, 0xbd, - 0xc5, 0xee, 0x5d, 0x1d, 0xb6, 0xcb, 0xf6, 0x8c, 0x40, 0xa4, 0x4b, 0xcb, 0x81, 0xa0, 0xd8, 0x99, - 0x63, 0x58, 0xd3, 0x33, 0x87, 0xfd, 0xa3, 0x37, 0xa0, 0x69, 0xc4, 0xd4, 0xd7, 0x0d, 0xeb, 0x84, - 0x84, 0xd4, 0x8e, 0x88, 0xb4, 0xfd, 0x2a, 0xe3, 0x76, 0x13, 0x66, 0xfb, 0x1e, 0xac, 0x64, 0x31, - 0x9f, 0x17, 0x7d, 0x2b, 0xd9, 0xe8, 0xfb, 0x53, 0x80, 0x69, 0xde, 0xce, 0x7c, 0x84, 0x9c, 0xda, - 0x54, 0x37, 0x7d, 0x8b, 0x48, 0x53, 0xd6, 0x18, 0xa3, 0xe7, 0x5b, 0xe4, 0xdc, 0x2d, 0xa8, 0x92, - 0xdc, 0x82, 0xd8, 0xae, 0x65, 0x1b, 0xed, 0xd8, 0x76, 0x1c, 0x62, 0x49, 0x0d, 0xeb, 0xbe, 0xef, - 0x3e, 0xe0, 0x0c, 0xf5, 0xdb, 0xa2, 0xf0, 0x15, 0x71, 0x9f, 0xcd, 0x95, 0x1b, 0x7f, 0x5f, 0xa6, - 0xbe, 0x0b, 0x10, 0x51, 0x23, 0x64, 0xa9, 0x84, 0x41, 0xe5, 0x13, 0x51, 0xfb, 0xc2, 0x35, 0x6a, - 0x94, 0x3c, 0xcb, 0xe3, 0xba, 0xec, 0xdd, 0xa5, 0xe8, 0x03, 0x58, 0x31, 0x7d, 0x37, 0x70, 0x88, - 0x14, 0xae, 0x3c, 0x57, 0xb8, 0x91, 0xf6, 0xef, 0xd2, 0xcc, 0x1d, 0xaa, 0xfa, 0xa2, 0x77, 0xa8, - 0xbf, 0x14, 0xc4, 0xb5, 0x3c, 0xfb, 0x2a, 0x80, 0xc6, 0x97, 0x3c, 0x3d, 0x6f, 0x2d, 0xf8, 0xc4, - 0xf0, 0x5d, 0xef, 0xce, 0xed, 0x0f, 0xf2, 0x3c, 0xf4, 0x3e, 0x3b, 0xb9, 0xfb, 0x6b, 0x09, 0xea, - 0xe9, 0x8d, 0xfc, 0x82, 0xed, 0xef, 0x40, 0x3d, 0xad, 0x89, 0xc8, 0x03, 0xe2, 0x3b, 0xcd, 0x93, - 0x76, 0x46, 0x8f, 0x01, 0x19, 0xe3, 0x71, 0x9a, 0xb4, 0xe9, 0x71, 0x64, 0x8c, 0x93, 0xf7, 0x90, - 0x3b, 0x73, 0xac, 0x43, 0x12, 0xb7, 0x0e, 0x99, 0x3c, 0x56, 0x8c, 0xf1, 0x78, 0x86, 0x83, 0x7e, - 0x06, 0xd7, 0x67, 0xc7, 0xd0, 0x8f, 0xce, 0xf4, 0xc0, 0xb6, 0xe4, 0x1d, 0x6c, 0x7b, 0xde, 0x47, - 0x89, 0xce, 0x0c, 0xfc, 0x27, 0x67, 0x07, 0xb6, 0x25, 0xd6, 0x1c, 0x85, 0x17, 0x1a, 0xda, 0xbf, - 0x80, 0x97, 0x9f, 0xd1, 0xfd, 0x12, 0x1b, 0x0c, 0x66, 0x1f, 0xdb, 0x17, 0x5f, 0x84, 0x8c, 0xf5, - 0xfe, 0x50, 0x10, 0x6f, 0x27, 0xb3, 0x6b, 0xd2, 0xcd, 0xe6, 0xad, 0x1b, 0x39, 0xc7, 0xe9, 0x1d, - 0x1c, 0x0a, 0x78, 0x9e, 0xaa, 0x7e, 0x7a, 0x2e, 0x55, 0xcd, 0x9b, 0xc4, 0x88, 0x8c, 0x4f, 0x00, - 0x49, 0x04, 0xf5, 0x4f, 0x25, 0xa8, 0x25, 0xe8, 0xfc, 0x06, 0x75, 0x16, 0x51, 0xe2, 0xea, 0x6e, - 0x72, 0x84, 0x15, 0x30, 0x08, 0xd6, 0x1e, 0x3b, 0xc4, 0x5e, 0x85, 0x3a, 0xbb, 0xa8, 0x89, 0xe6, - 0x22, 0x6f, 0xae, 0x31, 0x06, 0x6f, 0x7c, 0x0d, 0x1a, 0xd4, 0xa7, 0x86, 0xa3, 0x53, 0x1e, 0xcb, - 0x4b, 0x42, 0x9a, 0xb3, 0x78, 0x24, 0x47, 0x6f, 0xc1, 0x15, 0x3a, 0x09, 0x7d, 0x4a, 0x1d, 0x96, - 0xdf, 0xf1, 0x8c, 0x46, 0x24, 0x20, 0x65, 0xac, 0xa4, 0x0d, 0x22, 0xd3, 0x89, 0xd8, 0xe9, 0x3d, - 0xed, 0xcc, 0x5c, 0x97, 0x1f, 0x22, 0x65, 0xbc, 0x9a, 0x72, 0x99, 0x6b, 0xb3, 0xe0, 0x19, 0x88, - 0x6c, 0x81, 0x9f, 0x15, 0x05, 0x9c, 0x90, 0x48, 0x87, 0x35, 0x97, 0x18, 0x51, 0x1c, 0x12, 0x4b, - 0x7f, 0x6c, 0x13, 0xc7, 0x12, 0x17, 0xdf, 0x66, 0xee, 0xf4, 0x3b, 0x59, 0x96, 0xce, 0x7d, 0x2e, - 0x8d, 0x9b, 0x09, 0x9c, 0xa0, 0x59, 0xe6, 0x20, 0xfe, 0xd0, 0x1a, 0x34, 0x86, 0x8f, 0x86, 0x23, - 0x6d, 0x4f, 0xdf, 0xdb, 0xef, 0x6b, 0xb2, 0x9e, 0x32, 0xd4, 0xb0, 0x20, 0x0b, 0xac, 0x7d, 0xb4, - 0x3f, 0xea, 0xee, 0xea, 0xa3, 0x9d, 0xde, 0x83, 0xa1, 0x52, 0x44, 0xd7, 0xe1, 0xca, 0x68, 0x1b, - 0xef, 0x8f, 0x46, 0xbb, 0x5a, 0x5f, 0x3f, 0xd0, 0xf0, 0xce, 0x7e, 0x7f, 0xa8, 0x94, 0x10, 0x82, - 0xe6, 0x94, 0x3d, 0xda, 0xd9, 0xd3, 0x94, 0x32, 0x6a, 0xc0, 0xf2, 0x81, 0x86, 0x7b, 0xda, 0x60, - 0xa4, 0x54, 0xd4, 0x6f, 0x4a, 0xd0, 0xc8, 0x58, 0x91, 0x39, 0x72, 0x18, 0x89, 0x3c, 0xbf, 0x8c, - 0xd9, 0x2f, 0x3b, 0x4c, 0x4c, 0xc3, 0x9c, 0x08, 0xeb, 0x94, 0xb1, 0x20, 0x78, 0x6e, 0x6f, 0x9c, - 0x66, 0xf6, 0x79, 0x19, 0xd7, 0x5c, 0xe3, 0x54, 0x80, 0xbc, 0x0e, 0x2b, 0xc7, 0x24, 0xf4, 0x88, - 0x23, 0xdb, 0x85, 0x45, 0x1a, 0x82, 0x27, 0xba, 0xac, 0x83, 0x22, 0xbb, 0x4c, 0x61, 0x84, 0x39, - 0x9a, 0x82, 0xbf, 0x97, 0x80, 0x5d, 0x83, 0x8a, 0x68, 0x5e, 0x16, 0xe3, 0x73, 0x82, 0x85, 0xa9, - 0xe8, 0xa9, 0x11, 0xf0, 0xfc, 0xae, 0x8c, 0xf9, 0x3f, 0x3a, 0xba, 0x68, 0x9f, 0x2a, 0xb7, 0xcf, - 0xdd, 0xf9, 0xdd, 0xf9, 0x59, 0x26, 0x9a, 0xa4, 0x26, 0x5a, 0x86, 0x12, 0x4e, 0x8a, 0x10, 0xbd, - 0x6e, 0x6f, 0x9b, 0x99, 0x65, 0x15, 0xea, 0x7b, 0xdd, 0xcf, 0xf4, 0xc3, 0x21, 0x7f, 0x86, 0x42, - 0x0a, 0xac, 0x3c, 0xd0, 0xf0, 0x40, 0xdb, 0x95, 0x9c, 0x12, 0xba, 0x06, 0x8a, 0xe4, 0x4c, 0xfb, - 0x95, 0x19, 0x82, 0xf8, 0xad, 0xa0, 0x1a, 0x94, 0x87, 0x0f, 0xbb, 0x07, 0x4a, 0x55, 0xfd, 0x47, - 0x11, 0xd6, 0x44, 0x58, 0x48, 0x9f, 0x4b, 0x9f, 0xfd, 0x6e, 0x99, 0x7d, 0x45, 0x28, 0xce, 0xbc, - 0x22, 0xa4, 0x49, 0x28, 0x8f, 0xea, 0xa5, 0x69, 0x12, 0xca, 0x5f, 0x1f, 0x66, 0x4e, 0xfc, 0xf2, - 0x3c, 0x27, 0x7e, 0x0b, 0x96, 0x5d, 0x12, 0xa5, 0x76, 0xab, 0xe3, 0x84, 0x44, 0x36, 0x34, 0x0c, - 0xcf, 0xf3, 0x29, 0x7f, 0xab, 0x4b, 0xae, 0x45, 0x5b, 0x73, 0xbd, 0x0a, 0xa6, 0x33, 0xee, 0x74, - 0xa7, 0x48, 0xe2, 0x60, 0xce, 0x62, 0xb7, 0x3f, 0x04, 0xe5, 0x7c, 0x87, 0x79, 0xc2, 0xe1, 0x9b, - 0xef, 0x4e, 0xa3, 0x21, 0x61, 0xfb, 0xe2, 0x70, 0xf0, 0x60, 0xb0, 0xff, 0x70, 0xa0, 0x2c, 0x31, - 0x02, 0x1f, 0x0e, 0x06, 0x3b, 0x83, 0x2d, 0xa5, 0x80, 0x00, 0xaa, 0xda, 0x67, 0x3b, 0x23, 0xad, - 0xaf, 0x14, 0x37, 0xff, 0xb9, 0x0a, 0x55, 0xa1, 0x24, 0xfa, 0x5a, 0x66, 0x02, 0xd9, 0x52, 0x3c, - 0xfa, 0x70, 0xee, 0x8c, 0x7a, 0xa6, 0xbc, 0xdf, 0xfe, 0x68, 0x61, 0x79, 0xf9, 0xdc, 0xbd, 0x84, - 0x7e, 0x5b, 0x80, 0x95, 0x99, 0xf7, 0xdd, 0xbc, 0x4f, 0x93, 0x97, 0x54, 0xfe, 0xdb, 0x3f, 0x58, - 0x48, 0x36, 0xd5, 0xe5, 0x37, 0x05, 0x68, 0x64, 0x6a, 0xde, 0xe8, 0xee, 0x22, 0x75, 0x72, 0xa1, - 0xc9, 0xbd, 0xc5, 0x4b, 0xec, 0xea, 0xd2, 0x3b, 0x05, 0xf4, 0xeb, 0x02, 0x34, 0x32, 0xd5, 0xdf, - 0xdc, 0xaa, 0x5c, 0xac, 0x55, 0xe7, 0x56, 0xe5, 0xb2, 0x62, 0xf3, 0x12, 0xfa, 0x65, 0x01, 0xea, - 0x69, 0x25, 0x17, 0xdd, 0x9e, 0xbf, 0xf6, 0x2b, 0x94, 0xb8, 0xb3, 0x68, 0xd1, 0x58, 0x5d, 0x42, - 0x3f, 0x87, 0x5a, 0x52, 0xf6, 0x44, 0x79, 0xa3, 0xd7, 0xb9, 0x9a, 0x6a, 0xfb, 0xf6, 0xdc, 0x72, - 0xd9, 0xe1, 0x93, 0x5a, 0x64, 0xee, 0xe1, 0xcf, 0x55, 0x4d, 0xdb, 0xb7, 0xe7, 0x96, 0x4b, 0x87, - 0x67, 0x9e, 0x90, 0x29, 0x59, 0xe6, 0xf6, 0x84, 0x8b, 0xb5, 0xd2, 0xdc, 0x9e, 0x70, 0x59, 0x85, - 0x54, 0x28, 0x92, 0x29, 0x7a, 0xe6, 0x56, 0xe4, 0x62, 0x61, 0x35, 0xb7, 0x22, 0x97, 0xd4, 0x58, - 0xd5, 0x25, 0xf4, 0x55, 0x21, 0x7b, 0x2f, 0xb8, 0x3d, 0x77, 0x6d, 0x6f, 0x4e, 0x97, 0xbc, 0x50, - 0x5d, 0xe4, 0x1b, 0xf4, 0x2b, 0xf9, 0x8a, 0x21, 0x4a, 0x83, 0x68, 0x1e, 0xb0, 0x99, 0x6a, 0x62, - 0xfb, 0xd6, 0x62, 0xc1, 0x86, 0x2b, 0xf1, 0xab, 0x02, 0xc0, 0xb4, 0x88, 0x98, 0x5b, 0x89, 0x0b, - 0xd5, 0xcb, 0xf6, 0xdd, 0x05, 0x24, 0xb3, 0x1b, 0x24, 0xa9, 0x1b, 0xe6, 0xde, 0x20, 0xe7, 0x8a, - 0x9c, 0xb9, 0x37, 0xc8, 0xf9, 0x02, 0xa5, 0xba, 0xf4, 0xc9, 0xf2, 0x8f, 0x2a, 0x22, 0xfa, 0x57, - 0xf9, 0xe7, 0xbd, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x4a, 0xde, 0xd5, 0xff, 0xa7, 0x27, 0x00, - 0x00, +var fileDescriptor_driver_8793c97f57ec8d84 = []byte{ + // 3215 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0xdb, 0x6f, 0x1b, 0xc7, + 0xd5, 0x17, 0xaf, 0x22, 0x0f, 0x25, 0x6a, 0x3d, 0xb6, 0x13, 0x9a, 0xf9, 0xbe, 0x2f, 0xce, 0x02, + 0xf9, 0x20, 0x24, 0x31, 0x95, 0x28, 0xa8, 0x6f, 0xcd, 0x8d, 0xa1, 0x68, 0x49, 0xb1, 0x44, 0xa9, + 0x43, 0x0a, 0x8e, 0xeb, 0x26, 0xdb, 0xd5, 0xee, 0x98, 0x5c, 0x6b, 0x6f, 0xd9, 0x9d, 0x95, 0xa5, + 0x14, 0x45, 0x8b, 0x14, 0x28, 0xda, 0x87, 0x02, 0x7d, 0x09, 0xf2, 0xde, 0x3e, 0xf6, 0x1f, 0xe8, + 0x05, 0x79, 0xee, 0x1f, 0xd1, 0xbe, 0xb4, 0x40, 0x81, 0xbe, 0xf6, 0x3f, 0x28, 0xe6, 0xb2, 0xcb, + 0xa5, 0x28, 0xc7, 0x4b, 0xca, 0x4f, 0xbb, 0xe7, 0xcc, 0xcc, 0x6f, 0xce, 0xcc, 0xb9, 0xcc, 0x99, + 0x0b, 0xa8, 0xbe, 0x1d, 0x0d, 0x2d, 0x37, 0x5c, 0x33, 0x03, 0xeb, 0x98, 0x04, 0xe1, 0x9a, 0x1f, + 0x78, 0xd4, 0x93, 0x54, 0x8b, 0x13, 0xe8, 0xf5, 0x91, 0x1e, 0x8e, 0x2c, 0xc3, 0x0b, 0xfc, 0x96, + 0xeb, 0x39, 0xba, 0xd9, 0x92, 0x6d, 0x5a, 0xb2, 0x8d, 0xa8, 0xd6, 0xfc, 0xbf, 0xa1, 0xe7, 0x0d, + 0x6d, 0x22, 0x10, 0x0e, 0xa3, 0xc7, 0x6b, 0x66, 0x14, 0xe8, 0xd4, 0xf2, 0x5c, 0x59, 0xfe, 0xea, + 0xd9, 0x72, 0x6a, 0x39, 0x24, 0xa4, 0xba, 0xe3, 0xcb, 0x0a, 0x1f, 0x0d, 0x2d, 0x3a, 0x8a, 0x0e, + 0x5b, 0x86, 0xe7, 0xac, 0x25, 0x5d, 0xae, 0xf1, 0x2e, 0xd7, 0x62, 0x31, 0xc3, 0x91, 0x1e, 0x10, + 0x73, 0x6d, 0x64, 0xd8, 0xa1, 0x4f, 0x0c, 0xf6, 0xd5, 0xd8, 0x8f, 0x44, 0xd8, 0xcc, 0x8e, 0x10, + 0xd2, 0x20, 0x32, 0x68, 0x3c, 0x5e, 0x9d, 0xd2, 0xc0, 0x3a, 0x8c, 0x28, 0x11, 0x40, 0xea, 0x35, + 0x78, 0x79, 0xa0, 0x87, 0x47, 0x1d, 0xcf, 0x7d, 0x6c, 0x0d, 0xfb, 0xc6, 0x88, 0x38, 0x3a, 0x26, + 0x5f, 0x44, 0x24, 0xa4, 0xea, 0x8f, 0xa0, 0x31, 0x5d, 0x14, 0xfa, 0x9e, 0x1b, 0x12, 0xf4, 0x11, + 0x14, 0x99, 0x34, 0x8d, 0xdc, 0xf5, 0xdc, 0x6a, 0x6d, 0xfd, 0xad, 0xd6, 0xb3, 0x26, 0x4e, 0xc8, + 0xd0, 0x92, 0xa3, 0x68, 0xf5, 0x7d, 0x62, 0x60, 0xde, 0x52, 0xbd, 0x0a, 0x97, 0x3b, 0xba, 0xaf, + 0x1f, 0x5a, 0xb6, 0x45, 0x2d, 0x12, 0xc6, 0x9d, 0x46, 0x70, 0x65, 0x92, 0x2d, 0x3b, 0xfc, 0x0c, + 0x96, 0x8c, 0x14, 0x5f, 0x76, 0x7c, 0xa7, 0x95, 0x49, 0x63, 0xad, 0x0d, 0x4e, 0x4d, 0x00, 0x4f, + 0xc0, 0xa9, 0x57, 0x00, 0xdd, 0xb3, 0xdc, 0x21, 0x09, 0xfc, 0xc0, 0x72, 0x69, 0x2c, 0xcc, 0xb7, + 0x05, 0xb8, 0x3c, 0xc1, 0x96, 0xc2, 0x3c, 0x01, 0x48, 0xe6, 0x91, 0x89, 0x52, 0x58, 0xad, 0xad, + 0x7f, 0x92, 0x51, 0x94, 0x73, 0xf0, 0x5a, 0xed, 0x04, 0xac, 0xeb, 0xd2, 0xe0, 0x14, 0xa7, 0xd0, + 0xd1, 0xe7, 0x50, 0x1e, 0x11, 0xdd, 0xa6, 0xa3, 0x46, 0xfe, 0x7a, 0x6e, 0xb5, 0xbe, 0x7e, 0xef, + 0x02, 0xfd, 0x6c, 0x71, 0xa0, 0x3e, 0xd5, 0x29, 0xc1, 0x12, 0x15, 0xdd, 0x00, 0x24, 0xfe, 0x34, + 0x93, 0x84, 0x46, 0x60, 0xf9, 0xcc, 0x90, 0x1b, 0x85, 0xeb, 0xb9, 0xd5, 0x2a, 0xbe, 0x24, 0x4a, + 0x36, 0xc6, 0x05, 0x4d, 0x1f, 0x56, 0xce, 0x48, 0x8b, 0x14, 0x28, 0x1c, 0x91, 0x53, 0xae, 0x91, + 0x2a, 0x66, 0xbf, 0x68, 0x13, 0x4a, 0xc7, 0xba, 0x1d, 0x11, 0x2e, 0x72, 0x6d, 0xfd, 0x9d, 0xe7, + 0x99, 0x87, 0x34, 0xd1, 0xf1, 0x3c, 0x60, 0xd1, 0xfe, 0x6e, 0xfe, 0x76, 0x4e, 0xbd, 0x03, 0xb5, + 0x94, 0xdc, 0xa8, 0x0e, 0x70, 0xd0, 0xdb, 0xe8, 0x0e, 0xba, 0x9d, 0x41, 0x77, 0x43, 0x59, 0x40, + 0xcb, 0x50, 0x3d, 0xe8, 0x6d, 0x75, 0xdb, 0x3b, 0x83, 0xad, 0x87, 0x4a, 0x0e, 0xd5, 0x60, 0x31, + 0x26, 0xf2, 0xea, 0x09, 0x20, 0x4c, 0x0c, 0xef, 0x98, 0x04, 0xcc, 0x90, 0xa5, 0x56, 0xd1, 0xcb, + 0xb0, 0x48, 0xf5, 0xf0, 0x48, 0xb3, 0x4c, 0x29, 0x73, 0x99, 0x91, 0xdb, 0x26, 0xda, 0x86, 0xf2, + 0x48, 0x77, 0x4d, 0xfb, 0xf9, 0x72, 0x4f, 0x4e, 0x35, 0x03, 0xdf, 0xe2, 0x0d, 0xb1, 0x04, 0x60, + 0xd6, 0x3d, 0xd1, 0xb3, 0x50, 0x80, 0xfa, 0x10, 0x94, 0x3e, 0xd5, 0x03, 0x9a, 0x16, 0xa7, 0x0b, + 0x45, 0xd6, 0xbf, 0xb4, 0xe8, 0x59, 0xfa, 0x14, 0x9e, 0x89, 0x79, 0x73, 0xf5, 0x3f, 0x79, 0xb8, + 0x94, 0xc2, 0x96, 0x96, 0xfa, 0x00, 0xca, 0x01, 0x09, 0x23, 0x9b, 0x72, 0xf8, 0xfa, 0xfa, 0x87, + 0x19, 0xe1, 0xa7, 0x90, 0x5a, 0x98, 0xc3, 0x60, 0x09, 0x87, 0x56, 0x41, 0x11, 0x2d, 0x34, 0x12, + 0x04, 0x5e, 0xa0, 0x39, 0xe1, 0x90, 0xcf, 0x5a, 0x15, 0xd7, 0x05, 0xbf, 0xcb, 0xd8, 0xbb, 0xe1, + 0x30, 0x35, 0xab, 0x85, 0x0b, 0xce, 0x2a, 0xd2, 0x41, 0x71, 0x09, 0x7d, 0xea, 0x05, 0x47, 0x1a, + 0x9b, 0xda, 0xc0, 0x32, 0x49, 0xa3, 0xc8, 0x41, 0x6f, 0x66, 0x04, 0xed, 0x89, 0xe6, 0x7b, 0xb2, + 0x35, 0x5e, 0x71, 0x27, 0x19, 0xea, 0x9b, 0x50, 0x16, 0x23, 0x65, 0x96, 0xd4, 0x3f, 0xe8, 0x74, + 0xba, 0xfd, 0xbe, 0xb2, 0x80, 0xaa, 0x50, 0xc2, 0xdd, 0x01, 0x66, 0x16, 0x56, 0x85, 0xd2, 0xbd, + 0xf6, 0xa0, 0xbd, 0xa3, 0xe4, 0xd5, 0x37, 0x60, 0xe5, 0x81, 0x6e, 0xd1, 0x2c, 0xc6, 0xa5, 0x7a, + 0xa0, 0x8c, 0xeb, 0x4a, 0xed, 0x6c, 0x4f, 0x68, 0x27, 0xfb, 0xd4, 0x74, 0x4f, 0x2c, 0x7a, 0x46, + 0x1f, 0x0a, 0x14, 0x48, 0x10, 0x48, 0x15, 0xb0, 0x5f, 0xf5, 0x29, 0xac, 0xf4, 0xa9, 0xe7, 0x67, + 0xb2, 0xfc, 0x77, 0x61, 0x91, 0xad, 0x51, 0x5e, 0x44, 0xa5, 0xe9, 0x5f, 0x6b, 0x89, 0x35, 0xac, + 0x15, 0xaf, 0x61, 0xad, 0x0d, 0xb9, 0xc6, 0xe1, 0xb8, 0x26, 0x7a, 0x09, 0xca, 0xa1, 0x35, 0x74, + 0x75, 0x5b, 0x46, 0x0b, 0x49, 0xa9, 0x88, 0x19, 0x79, 0xdc, 0xb1, 0x34, 0xfc, 0x0e, 0xa0, 0x0d, + 0x12, 0xd2, 0xc0, 0x3b, 0xcd, 0x24, 0xcf, 0x15, 0x28, 0x3d, 0xf6, 0x02, 0x43, 0x38, 0x62, 0x05, + 0x0b, 0x82, 0x39, 0xd5, 0x04, 0x88, 0xc4, 0xbe, 0x01, 0x68, 0xdb, 0x65, 0x6b, 0x4a, 0x36, 0x45, + 0xfc, 0x36, 0x0f, 0x97, 0x27, 0xea, 0x4b, 0x65, 0xcc, 0xef, 0x87, 0x2c, 0x30, 0x45, 0xa1, 0xf0, + 0x43, 0xb4, 0x07, 0x65, 0x51, 0x43, 0xce, 0xe4, 0xad, 0x19, 0x80, 0xc4, 0x32, 0x25, 0xe1, 0x24, + 0xcc, 0xb9, 0x46, 0x5f, 0x78, 0xb1, 0x46, 0xff, 0x14, 0x94, 0x78, 0x1c, 0xe1, 0x73, 0x75, 0xf3, + 0x09, 0x5c, 0x36, 0x3c, 0xdb, 0x26, 0x06, 0xb3, 0x06, 0xcd, 0x72, 0x29, 0x09, 0x8e, 0x75, 0xfb, + 0xf9, 0x76, 0x83, 0xc6, 0xad, 0xb6, 0x65, 0x23, 0xf5, 0x11, 0x5c, 0x4a, 0x75, 0x2c, 0x15, 0x71, + 0x0f, 0x4a, 0x21, 0x63, 0x48, 0x4d, 0xbc, 0x3d, 0xa3, 0x26, 0x42, 0x2c, 0x9a, 0xab, 0x97, 0x05, + 0x78, 0xf7, 0x98, 0xb8, 0xc9, 0xb0, 0xd4, 0x0d, 0xb8, 0xd4, 0xe7, 0x66, 0x9a, 0xc9, 0x0e, 0xc7, + 0x26, 0x9e, 0x9f, 0x30, 0xf1, 0x2b, 0x80, 0xd2, 0x28, 0xd2, 0x10, 0x4f, 0x61, 0xa5, 0x7b, 0x42, + 0x8c, 0x4c, 0xc8, 0x0d, 0x58, 0x34, 0x3c, 0xc7, 0xd1, 0x5d, 0xb3, 0x91, 0xbf, 0x5e, 0x58, 0xad, + 0xe2, 0x98, 0x4c, 0xfb, 0x62, 0x21, 0xab, 0x2f, 0xaa, 0xbf, 0xc9, 0x81, 0x32, 0xee, 0x5b, 0x4e, + 0x24, 0x93, 0x9e, 0x9a, 0x0c, 0x88, 0xf5, 0xbd, 0x84, 0x25, 0x25, 0xf9, 0x71, 0xb8, 0x10, 0x7c, + 0x12, 0x04, 0xa9, 0x70, 0x54, 0xb8, 0x60, 0x38, 0x52, 0xb7, 0xe0, 0x7f, 0x62, 0x71, 0xfa, 0x34, + 0x20, 0xba, 0x63, 0xb9, 0xc3, 0xed, 0xbd, 0x3d, 0x9f, 0x08, 0xc1, 0x11, 0x82, 0xa2, 0xa9, 0x53, + 0x5d, 0x0a, 0xc6, 0xff, 0x99, 0xd3, 0x1b, 0xb6, 0x17, 0x26, 0x4e, 0xcf, 0x09, 0xf5, 0xaf, 0x05, + 0x68, 0x4c, 0x41, 0xc5, 0xd3, 0xfb, 0x08, 0x4a, 0x21, 0xa1, 0x91, 0x2f, 0x4d, 0xa5, 0x9b, 0x59, + 0xe0, 0xf3, 0xf1, 0x5a, 0x7d, 0x06, 0x86, 0x05, 0x26, 0x1a, 0x42, 0x85, 0xd2, 0x53, 0x2d, 0xb4, + 0xbe, 0x8c, 0x13, 0x82, 0x9d, 0x8b, 0xe2, 0x0f, 0x48, 0xe0, 0x58, 0xae, 0x6e, 0xf7, 0xad, 0x2f, + 0x09, 0x5e, 0xa4, 0xf4, 0x94, 0xfd, 0xa0, 0x87, 0xcc, 0xe0, 0x4d, 0xcb, 0x95, 0xd3, 0xde, 0x99, + 0xb7, 0x97, 0xd4, 0x04, 0x63, 0x81, 0xd8, 0xdc, 0x81, 0x12, 0x1f, 0xd3, 0x3c, 0x86, 0xa8, 0x40, + 0x81, 0xd2, 0x53, 0x2e, 0x54, 0x05, 0xb3, 0xdf, 0xe6, 0x7b, 0xb0, 0x94, 0x1e, 0x01, 0x33, 0xa4, + 0x11, 0xb1, 0x86, 0x23, 0x61, 0x60, 0x25, 0x2c, 0x29, 0xa6, 0xc9, 0xa7, 0x96, 0x29, 0x53, 0xd6, + 0x12, 0x16, 0x84, 0xfa, 0xc7, 0x3c, 0x5c, 0x3b, 0x67, 0x66, 0xa4, 0xb1, 0x3e, 0x9a, 0x30, 0xd6, + 0x17, 0x34, 0x0b, 0xb1, 0xc5, 0x3f, 0x9a, 0xb0, 0xf8, 0x17, 0x08, 0xce, 0xdc, 0xe6, 0x25, 0x28, + 0x93, 0x13, 0x8b, 0x12, 0x53, 0x4e, 0x95, 0xa4, 0x52, 0xee, 0x54, 0xbc, 0xa8, 0x3b, 0xfd, 0x2b, + 0x07, 0x68, 0x7a, 0x0f, 0x83, 0x5e, 0x83, 0xa5, 0x90, 0xb8, 0xa6, 0x26, 0xa2, 0x92, 0x08, 0x98, + 0x15, 0x5c, 0x63, 0x3c, 0x11, 0x9e, 0x42, 0xe6, 0x68, 0xe4, 0x84, 0x18, 0xd2, 0xa7, 0xf8, 0x3f, + 0x1a, 0xc1, 0xd2, 0xe3, 0x50, 0xb3, 0x42, 0xcf, 0xd6, 0x93, 0x64, 0xbf, 0x9e, 0xd9, 0x79, 0xa6, + 0xe5, 0x68, 0xdd, 0xeb, 0x6f, 0xc7, 0x60, 0xb8, 0xf6, 0x38, 0x4c, 0x08, 0xb5, 0x05, 0xb5, 0x54, + 0x19, 0xaa, 0x40, 0xb1, 0xb7, 0xd7, 0xeb, 0x2a, 0x0b, 0x08, 0xa0, 0xdc, 0xd9, 0xc2, 0x7b, 0x7b, + 0x03, 0x91, 0x50, 0x6d, 0xef, 0xb6, 0x37, 0xbb, 0x4a, 0x5e, 0xfd, 0x53, 0x19, 0x60, 0x9c, 0xd9, + 0xa2, 0x3a, 0xe4, 0x13, 0x7b, 0xcd, 0x5b, 0x26, 0x1b, 0x8c, 0xab, 0x3b, 0x44, 0x06, 0x63, 0xfe, + 0x8f, 0xd6, 0xe1, 0xaa, 0x13, 0x0e, 0x7d, 0xdd, 0x38, 0xd2, 0x64, 0x42, 0x6a, 0xf0, 0xc6, 0x7c, + 0x54, 0x4b, 0xf8, 0xb2, 0x2c, 0x94, 0x52, 0x0b, 0xdc, 0x1d, 0x28, 0x10, 0xf7, 0xb8, 0x51, 0xe4, + 0x1b, 0xb7, 0xbb, 0x33, 0x67, 0xdc, 0xad, 0xae, 0x7b, 0x2c, 0x36, 0x6a, 0x0c, 0x06, 0x69, 0x00, + 0x26, 0x39, 0xb6, 0x0c, 0xa2, 0x31, 0xd0, 0x12, 0x07, 0xfd, 0x68, 0x76, 0xd0, 0x0d, 0x8e, 0x91, + 0x40, 0x57, 0xcd, 0x98, 0x46, 0x3d, 0xa8, 0x06, 0x24, 0xf4, 0xa2, 0xc0, 0x20, 0x61, 0xa3, 0x3c, + 0xd3, 0xa2, 0x88, 0xe3, 0x76, 0x78, 0x0c, 0x81, 0x36, 0xa0, 0xec, 0x78, 0x91, 0x4b, 0xc3, 0xc6, + 0x22, 0x17, 0xf6, 0xad, 0x8c, 0x60, 0xbb, 0xac, 0x11, 0x96, 0x6d, 0xd1, 0x26, 0x2c, 0x0a, 0x11, + 0xc3, 0x46, 0x85, 0xc3, 0xdc, 0xc8, 0x6a, 0x40, 0xbc, 0x15, 0x8e, 0x5b, 0x33, 0xad, 0x46, 0x21, + 0x09, 0x1a, 0x55, 0xa1, 0x55, 0xf6, 0x8f, 0x5e, 0x81, 0xaa, 0x6e, 0xdb, 0x9e, 0xa1, 0x99, 0x56, + 0xd0, 0x00, 0x5e, 0x50, 0xe1, 0x8c, 0x0d, 0x2b, 0x40, 0xaf, 0x42, 0x4d, 0xf8, 0xb5, 0xe6, 0xeb, + 0x74, 0xd4, 0xa8, 0xf1, 0x62, 0x10, 0xac, 0x7d, 0x9d, 0x8e, 0x64, 0x05, 0x12, 0x04, 0xa2, 0xc2, + 0x52, 0x52, 0x81, 0x04, 0x01, 0xaf, 0xf0, 0xff, 0xb0, 0xc2, 0xa3, 0xe1, 0x30, 0xf0, 0x22, 0x5f, + 0xe3, 0x36, 0xb5, 0xcc, 0x2b, 0x2d, 0x33, 0xf6, 0x26, 0xe3, 0xf6, 0x98, 0x71, 0x5d, 0x83, 0xca, + 0x13, 0xef, 0x50, 0x54, 0xa8, 0xf3, 0x0a, 0x8b, 0x4f, 0xbc, 0xc3, 0xb8, 0x48, 0x48, 0x68, 0x99, + 0x8d, 0x15, 0x51, 0xc4, 0xe9, 0x6d, 0xb3, 0x79, 0x13, 0x2a, 0xb1, 0x1a, 0xcf, 0xd9, 0x1c, 0x5f, + 0x49, 0x6f, 0x8e, 0xab, 0xa9, 0x9d, 0x6e, 0xf3, 0x3d, 0xa8, 0x4f, 0x1a, 0xc1, 0x2c, 0xad, 0xd5, + 0xbf, 0xe5, 0xa0, 0x9a, 0xa8, 0x1b, 0xb9, 0x70, 0x99, 0x8b, 0xa3, 0x53, 0x62, 0x6a, 0x63, 0xeb, + 0x11, 0xb1, 0xf5, 0xfd, 0x8c, 0x9a, 0x6a, 0xc7, 0x08, 0x32, 0xad, 0x90, 0xa6, 0x84, 0x12, 0xe4, + 0x71, 0x7f, 0x9f, 0xc3, 0x8a, 0x6d, 0xb9, 0xd1, 0x49, 0xaa, 0x2f, 0x11, 0x6a, 0xbf, 0x97, 0xb1, + 0xaf, 0x1d, 0xd6, 0x7a, 0xdc, 0x47, 0xdd, 0x9e, 0xa0, 0xd5, 0xaf, 0xf3, 0xf0, 0xd2, 0xf9, 0xe2, + 0xa0, 0x1e, 0x14, 0x0c, 0x3f, 0x92, 0x43, 0x7b, 0x6f, 0xd6, 0xa1, 0x75, 0xfc, 0x68, 0xdc, 0x2b, + 0x03, 0x62, 0x7b, 0x66, 0x87, 0x38, 0x5e, 0x70, 0x2a, 0x47, 0xf0, 0xe1, 0xac, 0x90, 0xbb, 0xbc, + 0xf5, 0x18, 0x55, 0xc2, 0x21, 0x0c, 0x15, 0x99, 0x79, 0x87, 0x32, 0x4c, 0xcc, 0x98, 0xc1, 0xc7, + 0x90, 0x38, 0xc1, 0x51, 0x6f, 0xc2, 0xd5, 0x73, 0x87, 0x82, 0xfe, 0x17, 0xc0, 0xf0, 0x23, 0x8d, + 0x9f, 0xb0, 0x08, 0xbd, 0x17, 0x70, 0xd5, 0xf0, 0xa3, 0x3e, 0x67, 0xa8, 0xb7, 0xa0, 0xf1, 0x2c, + 0x79, 0x99, 0xf3, 0x09, 0x89, 0x35, 0xe7, 0x90, 0xcf, 0x41, 0x01, 0x57, 0x04, 0x63, 0xf7, 0x50, + 0xfd, 0x26, 0x0f, 0x2b, 0x67, 0xc4, 0x61, 0x2b, 0xa0, 0x70, 0xe6, 0x38, 0xb7, 0x10, 0x14, 0xf3, + 0x6c, 0xc3, 0x32, 0xe3, 0x5d, 0x29, 0xff, 0xe7, 0x31, 0xdd, 0x97, 0x3b, 0xc6, 0xbc, 0xe5, 0x33, + 0x83, 0x76, 0x0e, 0x2d, 0x1a, 0xf2, 0x45, 0xb2, 0x84, 0x05, 0x81, 0x1e, 0x42, 0x3d, 0x20, 0x21, + 0x09, 0x8e, 0x89, 0xa9, 0xf9, 0x5e, 0x40, 0xe3, 0x09, 0x5b, 0x9f, 0x6d, 0xc2, 0xf6, 0xbd, 0x80, + 0xe2, 0xe5, 0x18, 0x89, 0x51, 0x21, 0x7a, 0x00, 0xcb, 0xe6, 0xa9, 0xab, 0x3b, 0x96, 0x21, 0x91, + 0xcb, 0x73, 0x23, 0x2f, 0x49, 0x20, 0x0e, 0xac, 0xde, 0x81, 0x5a, 0xaa, 0x90, 0x0d, 0xcc, 0xd6, + 0x0f, 0x89, 0x2d, 0xe7, 0x44, 0x10, 0x93, 0xfe, 0x5b, 0x92, 0xfe, 0xab, 0xfe, 0x3e, 0x0f, 0xf5, + 0x49, 0x07, 0x88, 0xf5, 0xe7, 0x93, 0xc0, 0xf2, 0xcc, 0x94, 0xfe, 0xf6, 0x39, 0x83, 0xe9, 0x88, + 0x15, 0x7f, 0x11, 0x79, 0x54, 0x8f, 0x75, 0x64, 0xf8, 0xd1, 0x0f, 0x18, 0x7d, 0x46, 0xf7, 0x85, + 0x33, 0xba, 0x47, 0x6f, 0x01, 0x92, 0xfa, 0xb5, 0x2d, 0xc7, 0xa2, 0xda, 0xe1, 0x29, 0x25, 0x62, + 0xfe, 0x0b, 0x58, 0x11, 0x25, 0x3b, 0xac, 0xe0, 0x63, 0xc6, 0x47, 0x2a, 0x2c, 0x7b, 0x9e, 0xa3, + 0x85, 0x86, 0x17, 0x10, 0x4d, 0x37, 0x9f, 0x34, 0x4a, 0xbc, 0x62, 0xcd, 0xf3, 0x9c, 0x3e, 0xe3, + 0xb5, 0xcd, 0x27, 0x2c, 0xe0, 0x1a, 0x7e, 0x14, 0x12, 0xaa, 0xb1, 0x0f, 0x5f, 0xa3, 0xaa, 0x18, + 0x04, 0xab, 0xe3, 0x47, 0x61, 0xaa, 0x82, 0x43, 0x1c, 0xb6, 0xee, 0xa4, 0x2a, 0xec, 0x12, 0x87, + 0xf5, 0xb2, 0xb4, 0x4f, 0x02, 0x83, 0xb8, 0x74, 0x60, 0x19, 0x47, 0x6c, 0x49, 0xc9, 0xad, 0xe6, + 0xf0, 0x04, 0x4f, 0xfd, 0x0c, 0x4a, 0x7c, 0x09, 0x62, 0x83, 0xe7, 0xe1, 0x9b, 0x47, 0x77, 0x31, + 0xbd, 0x15, 0xc6, 0xe0, 0xb1, 0xfd, 0x15, 0xa8, 0x8e, 0xbc, 0x50, 0xae, 0x0d, 0xc2, 0xf2, 0x2a, + 0x8c, 0xc1, 0x0b, 0x9b, 0x50, 0x09, 0x88, 0x6e, 0x7a, 0xae, 0x1d, 0x27, 0xb6, 0x09, 0xad, 0x7e, + 0x01, 0x65, 0x11, 0x7e, 0x2f, 0x80, 0x7f, 0x03, 0x90, 0x21, 0x16, 0x15, 0x9f, 0x25, 0xca, 0x61, + 0x68, 0x79, 0x6e, 0x18, 0x9f, 0xa6, 0x8a, 0x92, 0xfd, 0x71, 0x81, 0xfa, 0xf7, 0x9c, 0xc8, 0x77, + 0xc4, 0x39, 0x17, 0xcb, 0xc5, 0x99, 0xa5, 0xb1, 0x9c, 0x4c, 0x24, 0xd4, 0x31, 0xc9, 0x72, 0x49, + 0x99, 0xd6, 0xe4, 0xe7, 0x3d, 0x26, 0x94, 0x00, 0xf1, 0xf6, 0x9a, 0xc8, 0xb4, 0x6f, 0xd6, 0xed, + 0x35, 0x11, 0xdb, 0x6b, 0xc2, 0x92, 0x4f, 0x99, 0x70, 0x09, 0xb8, 0x22, 0xcf, 0xb7, 0x6a, 0x66, + 0x72, 0x86, 0x41, 0xd4, 0x7f, 0xe7, 0x92, 0x58, 0x11, 0x9f, 0x35, 0xa0, 0xcf, 0xa1, 0xc2, 0xdc, + 0x4e, 0x73, 0x74, 0x5f, 0x9e, 0x9c, 0x77, 0xe6, 0x3b, 0xc6, 0x68, 0x31, 0x2f, 0xdb, 0xd5, 0x7d, + 0x91, 0x2e, 0x2d, 0xfa, 0x82, 0x62, 0x31, 0x47, 0x37, 0xc7, 0x31, 0x87, 0xfd, 0xa3, 0xd7, 0xa1, + 0xae, 0x47, 0xd4, 0xd3, 0x74, 0xf3, 0x98, 0x04, 0xd4, 0x0a, 0x89, 0xd4, 0xfd, 0x32, 0xe3, 0xb6, + 0x63, 0x66, 0xf3, 0x2e, 0x2c, 0xa5, 0x31, 0x9f, 0xb7, 0xfa, 0x96, 0xd2, 0xab, 0xef, 0x8f, 0x01, + 0xc6, 0x79, 0x3b, 0xb3, 0x11, 0xb6, 0x09, 0xd0, 0x0c, 0xcf, 0x24, 0x52, 0x95, 0x15, 0xc6, 0xe8, + 0x78, 0x26, 0x39, 0x73, 0xa8, 0x50, 0x8a, 0x0f, 0x15, 0x98, 0xd7, 0x32, 0x47, 0x3b, 0xb2, 0x6c, + 0x3b, 0xd9, 0x4b, 0x54, 0x3d, 0xcf, 0xb9, 0xcf, 0x19, 0xea, 0xb7, 0x79, 0x61, 0x2b, 0xe2, 0x78, + 0x28, 0x53, 0x6e, 0xfc, 0xa2, 0x54, 0x7d, 0x07, 0x20, 0xa4, 0x7a, 0xc0, 0x52, 0x09, 0x3d, 0xde, + 0xcd, 0x34, 0xa7, 0x4e, 0x25, 0x06, 0xf1, 0x2d, 0x17, 0xae, 0xca, 0xda, 0x6d, 0x8a, 0xde, 0x87, + 0x25, 0xc3, 0x73, 0x7c, 0x9b, 0xc8, 0xc6, 0xa5, 0xe7, 0x36, 0xae, 0x25, 0xf5, 0xdb, 0x34, 0xb5, + 0x87, 0x2a, 0x5f, 0x74, 0x0f, 0xf5, 0xe7, 0x9c, 0x38, 0xe5, 0x4a, 0x1f, 0xb2, 0xa1, 0xe1, 0x39, + 0x37, 0x39, 0x9b, 0x73, 0x9e, 0xd8, 0x7d, 0xd7, 0x35, 0x4e, 0xf3, 0xfd, 0x2c, 0xf7, 0x26, 0xcf, + 0x4e, 0xee, 0xfe, 0x52, 0x80, 0x6a, 0x72, 0xc0, 0x35, 0xa5, 0xfb, 0xdb, 0x50, 0x4d, 0xae, 0x18, + 0x65, 0x80, 0xf8, 0x4e, 0xf5, 0x24, 0x95, 0xd1, 0x63, 0x40, 0xfa, 0x70, 0x98, 0x24, 0x6d, 0x5a, + 0x14, 0xea, 0xc3, 0xf8, 0x78, 0xf1, 0xf6, 0x0c, 0xf3, 0x10, 0xaf, 0x5b, 0x07, 0xac, 0x3d, 0x56, + 0xf4, 0xe1, 0x70, 0x82, 0x83, 0x7e, 0x02, 0x57, 0x27, 0xfb, 0xd0, 0x0e, 0x4f, 0x35, 0xdf, 0x32, + 0xe5, 0x1e, 0x6c, 0x6b, 0xd6, 0x33, 0xbe, 0xd6, 0x04, 0xfc, 0xc7, 0xa7, 0xfb, 0x96, 0x29, 0xe6, + 0x1c, 0x05, 0x53, 0x05, 0xcd, 0x9f, 0xc1, 0xcb, 0xcf, 0xa8, 0x7e, 0x8e, 0x0e, 0x7a, 0x93, 0x77, + 0x57, 0xf3, 0x4f, 0x42, 0x4a, 0x7b, 0xbf, 0xcb, 0x89, 0xa3, 0xc8, 0xc9, 0x39, 0x69, 0xa7, 0xf3, + 0xd6, 0xb5, 0x8c, 0xfd, 0x74, 0xf6, 0x0f, 0x04, 0x3c, 0x4f, 0x55, 0x3f, 0x39, 0x93, 0xaa, 0x66, + 0x4d, 0x62, 0x44, 0xc6, 0x27, 0x80, 0x24, 0x82, 0xfa, 0x87, 0x02, 0x54, 0x62, 0x74, 0xbe, 0x83, + 0x3a, 0x0d, 0x29, 0x71, 0x34, 0x27, 0x0e, 0x61, 0x39, 0x0c, 0x82, 0xb5, 0xcb, 0x82, 0xd8, 0x2b, + 0x50, 0x65, 0x1b, 0x35, 0x51, 0x9c, 0xe7, 0xc5, 0x15, 0xc6, 0xe0, 0x85, 0xaf, 0x42, 0x8d, 0x7a, + 0x54, 0xb7, 0x35, 0xca, 0xd7, 0xf2, 0x82, 0x68, 0xcd, 0x59, 0x7c, 0x25, 0x47, 0x6f, 0xc2, 0x25, + 0x3a, 0x0a, 0x3c, 0x4a, 0x6d, 0x96, 0xdf, 0xf1, 0x8c, 0x46, 0x24, 0x20, 0x45, 0xac, 0x24, 0x05, + 0x22, 0xd3, 0x09, 0x59, 0xf4, 0x1e, 0x57, 0x66, 0xa6, 0xcb, 0x83, 0x48, 0x11, 0x2f, 0x27, 0x5c, + 0x66, 0xda, 0x6c, 0xf1, 0xf4, 0x45, 0xb6, 0xc0, 0x63, 0x45, 0x0e, 0xc7, 0x24, 0xd2, 0x60, 0xc5, + 0x21, 0x7a, 0x18, 0x05, 0xc4, 0xd4, 0x1e, 0x5b, 0xc4, 0x36, 0xc5, 0xc6, 0xb7, 0x9e, 0x39, 0xfd, + 0x8e, 0xa7, 0xa5, 0x75, 0x8f, 0xb7, 0xc6, 0xf5, 0x18, 0x4e, 0xd0, 0x2c, 0x73, 0x10, 0x7f, 0x68, + 0x05, 0x6a, 0xfd, 0x87, 0xfd, 0x41, 0x77, 0x57, 0xdb, 0xdd, 0xdb, 0xe8, 0xca, 0xeb, 0xc9, 0x7e, + 0x17, 0x0b, 0x32, 0xc7, 0xca, 0x07, 0x7b, 0x83, 0xf6, 0x8e, 0x36, 0xd8, 0xee, 0xdc, 0xef, 0x2b, + 0x79, 0x74, 0x15, 0x2e, 0x0d, 0xb6, 0xf0, 0xde, 0x60, 0xb0, 0xd3, 0xdd, 0xd0, 0xf6, 0xbb, 0x78, + 0x7b, 0x6f, 0xa3, 0xaf, 0x14, 0x10, 0x82, 0xfa, 0x98, 0x3d, 0xd8, 0xde, 0xed, 0x2a, 0x45, 0x54, + 0x83, 0xc5, 0xfd, 0x2e, 0xee, 0x74, 0x7b, 0x03, 0xa5, 0xa4, 0x7e, 0x53, 0x80, 0x5a, 0x4a, 0x8b, + 0xcc, 0x90, 0x83, 0x50, 0xe4, 0xf9, 0x45, 0xcc, 0x7e, 0xf9, 0x71, 0xaa, 0x6e, 0x8c, 0x84, 0x76, + 0x8a, 0x58, 0x10, 0x3c, 0xb7, 0xd7, 0x4f, 0x52, 0x7e, 0x5e, 0xc4, 0x15, 0x47, 0x3f, 0x11, 0x20, + 0xaf, 0xc1, 0xd2, 0x11, 0x09, 0x5c, 0x62, 0xcb, 0x72, 0xa1, 0x91, 0x9a, 0xe0, 0x89, 0x2a, 0xab, + 0xa0, 0xc8, 0x2a, 0x63, 0x18, 0xa1, 0x8e, 0xba, 0xe0, 0xef, 0xc6, 0x60, 0x57, 0xa0, 0x24, 0x8a, + 0x17, 0x45, 0xff, 0x9c, 0x60, 0xcb, 0x54, 0xf8, 0x54, 0xf7, 0x79, 0x7e, 0x57, 0xc4, 0xfc, 0x1f, + 0x1d, 0x4e, 0xeb, 0xa7, 0xcc, 0xf5, 0x73, 0x67, 0x76, 0x73, 0x7e, 0x96, 0x8a, 0x46, 0x89, 0x8a, + 0x16, 0xa1, 0x80, 0xe3, 0x3b, 0xbd, 0x4e, 0xbb, 0xb3, 0xc5, 0xd4, 0xb2, 0x0c, 0xd5, 0xdd, 0xf6, + 0xa7, 0xda, 0x41, 0x9f, 0x1f, 0x43, 0x21, 0x05, 0x96, 0xee, 0x77, 0x71, 0xaf, 0xbb, 0x23, 0x39, + 0x05, 0x74, 0x05, 0x14, 0xc9, 0x19, 0xd7, 0x2b, 0x32, 0x04, 0xf1, 0x5b, 0x42, 0x15, 0x28, 0xf6, + 0x1f, 0xb4, 0xf7, 0x95, 0xb2, 0xfa, 0x8f, 0x3c, 0xac, 0x88, 0x65, 0x21, 0xb9, 0x7d, 0x78, 0xf6, + 0xe9, 0x6b, 0xfa, 0x14, 0x21, 0x3f, 0x71, 0x8a, 0x90, 0x24, 0xa1, 0x7c, 0x55, 0x2f, 0x8c, 0x93, + 0x50, 0x7e, 0xfa, 0x30, 0x11, 0xf1, 0x8b, 0xb3, 0x44, 0xfc, 0x06, 0x2c, 0x3a, 0x24, 0x4c, 0xf4, + 0x56, 0xc5, 0x31, 0x89, 0x2c, 0xa8, 0xe9, 0xae, 0xeb, 0x51, 0x7e, 0x56, 0x17, 0x6f, 0x8b, 0x36, + 0x67, 0x3a, 0x15, 0x4c, 0x46, 0xdc, 0x6a, 0x8f, 0x91, 0x44, 0x60, 0x4e, 0x63, 0x37, 0x3f, 0x00, + 0xe5, 0x6c, 0x85, 0x59, 0x96, 0xc3, 0x37, 0xde, 0x19, 0xaf, 0x86, 0x84, 0xf9, 0xc5, 0x41, 0xef, + 0x7e, 0x6f, 0xef, 0x41, 0x4f, 0x59, 0x60, 0x04, 0x3e, 0xe8, 0xf5, 0xb6, 0x7b, 0x9b, 0x4a, 0x0e, + 0x01, 0x94, 0xbb, 0x9f, 0x6e, 0x0f, 0xba, 0x1b, 0x4a, 0x7e, 0xfd, 0x9f, 0xcb, 0x50, 0x16, 0x42, + 0xa2, 0xaf, 0x65, 0x26, 0x90, 0x7e, 0xd9, 0x82, 0x3e, 0x98, 0x39, 0xa3, 0x9e, 0x78, 0x2d, 0xd3, + 0xfc, 0x70, 0xee, 0xf6, 0xf2, 0xf6, 0x68, 0x01, 0xfd, 0x3a, 0x07, 0x4b, 0x13, 0xe7, 0xbb, 0x59, + 0x8f, 0x26, 0xcf, 0x79, 0x48, 0xd3, 0xfc, 0xfe, 0x5c, 0x6d, 0x13, 0x59, 0x7e, 0x95, 0x83, 0x5a, + 0xea, 0x09, 0x09, 0xba, 0x33, 0xcf, 0xb3, 0x13, 0x21, 0xc9, 0xdd, 0xf9, 0x5f, 0xac, 0xa8, 0x0b, + 0x6f, 0xe7, 0xd0, 0x2f, 0x73, 0x50, 0x4b, 0x3d, 0xa6, 0xc8, 0x2c, 0xca, 0xf4, 0xd3, 0x8f, 0xcc, + 0xa2, 0x9c, 0xf7, 0x76, 0x63, 0x01, 0xfd, 0x3c, 0x07, 0xd5, 0xe4, 0x61, 0x04, 0xba, 0x35, 0xfb, + 0x53, 0x0a, 0x21, 0xc4, 0xed, 0x79, 0xdf, 0x60, 0xa8, 0x0b, 0xe8, 0xa7, 0x50, 0x89, 0x5f, 0x11, + 0xa0, 0xac, 0xab, 0xd7, 0x99, 0x27, 0x0a, 0xcd, 0x5b, 0x33, 0xb7, 0x4b, 0x77, 0x1f, 0x5f, 0xed, + 0x67, 0xee, 0xfe, 0xcc, 0x23, 0x84, 0xe6, 0xad, 0x99, 0xdb, 0x25, 0xdd, 0x33, 0x4b, 0x48, 0xbd, + 0x00, 0xc8, 0x6c, 0x09, 0xd3, 0x4f, 0x0f, 0x32, 0x5b, 0xc2, 0x79, 0x0f, 0x0e, 0x84, 0x20, 0xa9, + 0x37, 0x04, 0x99, 0x05, 0x99, 0x7e, 0xa7, 0x90, 0x59, 0x90, 0x73, 0x9e, 0x2c, 0xa8, 0x0b, 0xe8, + 0xab, 0x5c, 0x7a, 0x5f, 0x70, 0x6b, 0xe6, 0xab, 0xf2, 0x19, 0x4d, 0x72, 0xea, 0xb2, 0x9e, 0x3b, + 0xe8, 0x57, 0xf2, 0x14, 0x43, 0xdc, 0xb4, 0xa3, 0x59, 0xc0, 0x26, 0x2e, 0xe7, 0x9b, 0x37, 0xe7, + 0x5b, 0x6c, 0xb8, 0x10, 0xbf, 0xc8, 0x01, 0x8c, 0xef, 0xe4, 0x33, 0x0b, 0x31, 0xf5, 0x18, 0xa0, + 0x79, 0x67, 0x8e, 0x96, 0x69, 0x07, 0x89, 0xef, 0x0c, 0x33, 0x3b, 0xc8, 0x99, 0x37, 0x03, 0x99, + 0x1d, 0xe4, 0xec, 0x7d, 0xbf, 0xba, 0xf0, 0xf1, 0xe2, 0x0f, 0x4b, 0x62, 0xf5, 0x2f, 0xf3, 0xcf, + 0xbb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x69, 0xd1, 0x3c, 0xf6, 0x2a, 0x00, 0x00, } diff --git a/plugins/drivers/proto/driver.proto b/plugins/drivers/proto/driver.proto index ed380aafe..974c5dc36 100644 --- a/plugins/drivers/proto/driver.proto +++ b/plugins/drivers/proto/driver.proto @@ -280,6 +280,36 @@ message ExecTaskResponse { ExitResult result = 3; } +message ExecTaskStreamingIOOperation { + bytes data = 1; + bool close = 2; +} + +message ExecTaskStreamingRequest { + message Setup { + string task_id = 1; + repeated string command = 2; + bool tty = 3; + } + + message TerminalSize { + int32 height = 1; + int32 width = 2; + } + + Setup setup = 1; + TerminalSize tty_size = 2; + ExecTaskStreamingIOOperation stdin = 3; +} + +message ExecTaskStreamingResponse { + ExecTaskStreamingIOOperation stdout = 1; + ExecTaskStreamingIOOperation stderr = 2; + + bool exited = 3; + ExitResult result = 4; +} + message DriverCapabilities { // SendSignals indicates that the driver can send process signals (ex. SIGUSR1) From 94ed649489a48f7e8c94cb7a615947dd877a2ae6 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:09:12 -0400 Subject: [PATCH 06/15] implemment streaming exec handling in driver grpc handlers Also add a helper that converts the adapts the high level interface to the low-level interface of nomad exec interfaces. --- plugins/drivers/client.go | 67 +++- plugins/drivers/execstreaming.go | 185 +++++++++ plugins/drivers/proto/driver.pb.go | 593 ++++++++++++++++------------- plugins/drivers/proto/driver.proto | 4 + plugins/drivers/server.go | 54 +++ 5 files changed, 641 insertions(+), 262 deletions(-) create mode 100644 plugins/drivers/execstreaming.go diff --git a/plugins/drivers/client.go b/plugins/drivers/client.go index 11babf249..ff284de6c 100644 --- a/plugins/drivers/client.go +++ b/plugins/drivers/client.go @@ -392,5 +392,70 @@ func (d *driverPluginClient) ExecTask(taskID string, cmd []string, timeout time. } return result, nil - +} + +var _ ExecTaskStreamingRawDriver = (*driverPluginClient)(nil) + +func (d *driverPluginClient) ExecTaskStreamingRaw(ctx context.Context, + taskID string, + command []string, + tty bool, + execStream ExecTaskStream) error { + + stream, err := d.client.ExecTaskStreaming(ctx) + if err != nil { + return grpcutils.HandleGrpcErr(err, d.doneCtx) + } + + err = stream.Send(&proto.ExecTaskStreamingRequest{ + Setup: &proto.ExecTaskStreamingRequest_Setup{ + TaskId: taskID, + Command: command, + Tty: tty, + }, + }) + if err != nil { + return grpcutils.HandleGrpcErr(err, d.doneCtx) + } + + errCh := make(chan error, 1) + + go func() { + for { + m, err := execStream.Recv() + if err == io.EOF { + return + } else if err != nil { + errCh <- err + return + } + + if err := stream.Send(m); err != nil { + errCh <- err + return + } + + } + }() + + for { + select { + case err := <-errCh: + return err + default: + } + + m, err := stream.Recv() + if err == io.EOF { + // Once we get to the end of stream successfully, we can ignore errCh: + // e.g. input write failures after process terminates shouldn't cause method to fail + return nil + } else if err != nil { + return err + } + + if err := execStream.Send(m); err != nil { + return err + } + } } diff --git a/plugins/drivers/execstreaming.go b/plugins/drivers/execstreaming.go new file mode 100644 index 000000000..ac0268327 --- /dev/null +++ b/plugins/drivers/execstreaming.go @@ -0,0 +1,185 @@ +package drivers + +import ( + "context" + "fmt" + "io" + "sync" + + "github.com/hashicorp/nomad/plugins/drivers/proto" +) + +// StreamToExecOptions is a convenience method to convert exec stream into +// ExecOptions object. +func StreamToExecOptions( + ctx context.Context, + command []string, + tty bool, + stream ExecTaskStream) (*ExecOptions, <-chan error) { + + inReader, inWriter := io.Pipe() + outReader, outWriter := io.Pipe() + errReader, errWriter := io.Pipe() + resize := make(chan TerminalSize, 2) + + errCh := make(chan error, 3) + + // handle input + go func() { + for { + msg, err := stream.Recv() + if err == io.EOF { + return + } else if err != nil { + errCh <- err + return + } + + if msg.Stdin != nil && !msg.Stdin.Close { + _, err := inWriter.Write(msg.Stdin.Data) + if err != nil { + errCh <- err + return + } + } else if msg.Stdin != nil && msg.Stdin.Close { + inWriter.Close() + } else if msg.TtySize != nil { + select { + case resize <- TerminalSize{ + Height: int(msg.TtySize.Height), + Width: int(msg.TtySize.Width), + }: + case <-ctx.Done(): + // process terminated before resize is processed + return + } + } else if isHeartbeat(msg) { + // do nothing + } else { + errCh <- fmt.Errorf("unexpected message type: %#v", msg) + } + } + }() + + var sendLock sync.Mutex + send := func(v *ExecTaskStreamingResponseMsg) error { + sendLock.Lock() + defer sendLock.Unlock() + + return stream.Send(v) + } + + var outWg sync.WaitGroup + outWg.Add(2) + // handle Stdout + go func() { + defer outWg.Done() + + reader := outReader + bytes := make([]byte, 1024) + msg := &ExecTaskStreamingResponseMsg{Stdout: &proto.ExecTaskStreamingIOOperation{}} + + for { + n, err := reader.Read(bytes) + // always send data if we read some + if n != 0 { + msg.Stdout.Data = bytes[:n] + if err := send(msg); err != nil { + errCh <- err + break + } + } + + // then handle error + if err == io.EOF || err == io.ErrClosedPipe { + msg.Stdout.Data = nil + msg.Stdout.Close = true + + if err := send(msg); err != nil { + errCh <- err + } + break + } + + if err != nil { + errCh <- err + break + } + } + + }() + // handle Stderr + go func() { + defer outWg.Done() + + reader := errReader + bytes := make([]byte, 1024) + msg := &ExecTaskStreamingResponseMsg{Stderr: &proto.ExecTaskStreamingIOOperation{}} + + for { + n, err := reader.Read(bytes) + // always send data if we read some + if n != 0 { + msg.Stderr.Data = bytes[:n] + if err := send(msg); err != nil { + errCh <- err + break + } + } + + // then handle error + if err == io.EOF || err == io.ErrClosedPipe { + msg.Stderr.Data = nil + msg.Stderr.Close = true + + if err := send(msg); err != nil { + errCh <- err + } + break + } + + if err != nil { + errCh <- err + break + } + } + + }() + + doneCh := make(chan error, 1) + go func() { + outWg.Wait() + + select { + case err := <-errCh: + doneCh <- err + default: + } + close(doneCh) + }() + + return &ExecOptions{ + Command: command, + Tty: tty, + + Stdin: inReader, + Stdout: outWriter, + Stderr: errWriter, + + ResizeCh: resize, + }, doneCh +} + +func NewExecStreamingResponseExit(exitCode int) *ExecTaskStreamingResponseMsg { + return &ExecTaskStreamingResponseMsg{ + Exited: true, + Result: &proto.ExitResult{ + ExitCode: int32(exitCode), + }, + } + +} + +func isHeartbeat(r *ExecTaskStreamingRequestMsg) bool { + return r.Stdin == nil && r.Setup == nil && r.TtySize == nil +} diff --git a/plugins/drivers/proto/driver.pb.go b/plugins/drivers/proto/driver.pb.go index 5473f7eab..845f606c0 100644 --- a/plugins/drivers/proto/driver.pb.go +++ b/plugins/drivers/proto/driver.pb.go @@ -50,7 +50,7 @@ func (x TaskState) String() string { return proto.EnumName(TaskState_name, int32(x)) } func (TaskState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{0} } type FingerprintResponse_HealthState int32 @@ -76,7 +76,7 @@ func (x FingerprintResponse_HealthState) String() string { return proto.EnumName(FingerprintResponse_HealthState_name, int32(x)) } func (FingerprintResponse_HealthState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{5, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{5, 0} } type StartTaskResponse_Result int32 @@ -102,7 +102,7 @@ func (x StartTaskResponse_Result) String() string { return proto.EnumName(StartTaskResponse_Result_name, int32(x)) } func (StartTaskResponse_Result) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{9, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{9, 0} } type DriverCapabilities_FSIsolation int32 @@ -128,7 +128,7 @@ func (x DriverCapabilities_FSIsolation) String() string { return proto.EnumName(DriverCapabilities_FSIsolation_name, int32(x)) } func (DriverCapabilities_FSIsolation) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{28, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{28, 0} } type CPUUsage_Fields int32 @@ -163,7 +163,7 @@ func (x CPUUsage_Fields) String() string { return proto.EnumName(CPUUsage_Fields_name, int32(x)) } func (CPUUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{46, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{46, 0} } type MemoryUsage_Fields int32 @@ -201,7 +201,7 @@ func (x MemoryUsage_Fields) String() string { return proto.EnumName(MemoryUsage_Fields_name, int32(x)) } func (MemoryUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{47, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{47, 0} } type TaskConfigSchemaRequest struct { @@ -214,7 +214,7 @@ func (m *TaskConfigSchemaRequest) Reset() { *m = TaskConfigSchemaRequest func (m *TaskConfigSchemaRequest) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaRequest) ProtoMessage() {} func (*TaskConfigSchemaRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{0} } func (m *TaskConfigSchemaRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaRequest.Unmarshal(m, b) @@ -246,7 +246,7 @@ func (m *TaskConfigSchemaResponse) Reset() { *m = TaskConfigSchemaRespon func (m *TaskConfigSchemaResponse) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaResponse) ProtoMessage() {} func (*TaskConfigSchemaResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{1} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{1} } func (m *TaskConfigSchemaResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaResponse.Unmarshal(m, b) @@ -283,7 +283,7 @@ func (m *CapabilitiesRequest) Reset() { *m = CapabilitiesRequest{} } func (m *CapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*CapabilitiesRequest) ProtoMessage() {} func (*CapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{2} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{2} } func (m *CapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesRequest.Unmarshal(m, b) @@ -318,7 +318,7 @@ func (m *CapabilitiesResponse) Reset() { *m = CapabilitiesResponse{} } func (m *CapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*CapabilitiesResponse) ProtoMessage() {} func (*CapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{3} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{3} } func (m *CapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesResponse.Unmarshal(m, b) @@ -355,7 +355,7 @@ func (m *FingerprintRequest) Reset() { *m = FingerprintRequest{} } func (m *FingerprintRequest) String() string { return proto.CompactTextString(m) } func (*FingerprintRequest) ProtoMessage() {} func (*FingerprintRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{4} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{4} } func (m *FingerprintRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintRequest.Unmarshal(m, b) @@ -398,7 +398,7 @@ func (m *FingerprintResponse) Reset() { *m = FingerprintResponse{} } func (m *FingerprintResponse) String() string { return proto.CompactTextString(m) } func (*FingerprintResponse) ProtoMessage() {} func (*FingerprintResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{5} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{5} } func (m *FingerprintResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintResponse.Unmarshal(m, b) @@ -453,7 +453,7 @@ func (m *RecoverTaskRequest) Reset() { *m = RecoverTaskRequest{} } func (m *RecoverTaskRequest) String() string { return proto.CompactTextString(m) } func (*RecoverTaskRequest) ProtoMessage() {} func (*RecoverTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{6} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{6} } func (m *RecoverTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskRequest.Unmarshal(m, b) @@ -497,7 +497,7 @@ func (m *RecoverTaskResponse) Reset() { *m = RecoverTaskResponse{} } func (m *RecoverTaskResponse) String() string { return proto.CompactTextString(m) } func (*RecoverTaskResponse) ProtoMessage() {} func (*RecoverTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{7} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{7} } func (m *RecoverTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskResponse.Unmarshal(m, b) @@ -529,7 +529,7 @@ func (m *StartTaskRequest) Reset() { *m = StartTaskRequest{} } func (m *StartTaskRequest) String() string { return proto.CompactTextString(m) } func (*StartTaskRequest) ProtoMessage() {} func (*StartTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{8} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{8} } func (m *StartTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskRequest.Unmarshal(m, b) @@ -583,7 +583,7 @@ func (m *StartTaskResponse) Reset() { *m = StartTaskResponse{} } func (m *StartTaskResponse) String() string { return proto.CompactTextString(m) } func (*StartTaskResponse) ProtoMessage() {} func (*StartTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{9} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{9} } func (m *StartTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskResponse.Unmarshal(m, b) @@ -643,7 +643,7 @@ func (m *WaitTaskRequest) Reset() { *m = WaitTaskRequest{} } func (m *WaitTaskRequest) String() string { return proto.CompactTextString(m) } func (*WaitTaskRequest) ProtoMessage() {} func (*WaitTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{10} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{10} } func (m *WaitTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskRequest.Unmarshal(m, b) @@ -684,7 +684,7 @@ func (m *WaitTaskResponse) Reset() { *m = WaitTaskResponse{} } func (m *WaitTaskResponse) String() string { return proto.CompactTextString(m) } func (*WaitTaskResponse) ProtoMessage() {} func (*WaitTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{11} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{11} } func (m *WaitTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskResponse.Unmarshal(m, b) @@ -736,7 +736,7 @@ func (m *StopTaskRequest) Reset() { *m = StopTaskRequest{} } func (m *StopTaskRequest) String() string { return proto.CompactTextString(m) } func (*StopTaskRequest) ProtoMessage() {} func (*StopTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{12} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{12} } func (m *StopTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskRequest.Unmarshal(m, b) @@ -787,7 +787,7 @@ func (m *StopTaskResponse) Reset() { *m = StopTaskResponse{} } func (m *StopTaskResponse) String() string { return proto.CompactTextString(m) } func (*StopTaskResponse) ProtoMessage() {} func (*StopTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{13} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{13} } func (m *StopTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskResponse.Unmarshal(m, b) @@ -821,7 +821,7 @@ func (m *DestroyTaskRequest) Reset() { *m = DestroyTaskRequest{} } func (m *DestroyTaskRequest) String() string { return proto.CompactTextString(m) } func (*DestroyTaskRequest) ProtoMessage() {} func (*DestroyTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{14} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{14} } func (m *DestroyTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskRequest.Unmarshal(m, b) @@ -865,7 +865,7 @@ func (m *DestroyTaskResponse) Reset() { *m = DestroyTaskResponse{} } func (m *DestroyTaskResponse) String() string { return proto.CompactTextString(m) } func (*DestroyTaskResponse) ProtoMessage() {} func (*DestroyTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{15} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{15} } func (m *DestroyTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskResponse.Unmarshal(m, b) @@ -897,7 +897,7 @@ func (m *InspectTaskRequest) Reset() { *m = InspectTaskRequest{} } func (m *InspectTaskRequest) String() string { return proto.CompactTextString(m) } func (*InspectTaskRequest) ProtoMessage() {} func (*InspectTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{16} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{16} } func (m *InspectTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskRequest.Unmarshal(m, b) @@ -940,7 +940,7 @@ func (m *InspectTaskResponse) Reset() { *m = InspectTaskResponse{} } func (m *InspectTaskResponse) String() string { return proto.CompactTextString(m) } func (*InspectTaskResponse) ProtoMessage() {} func (*InspectTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{17} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{17} } func (m *InspectTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskResponse.Unmarshal(m, b) @@ -995,7 +995,7 @@ func (m *TaskStatsRequest) Reset() { *m = TaskStatsRequest{} } func (m *TaskStatsRequest) String() string { return proto.CompactTextString(m) } func (*TaskStatsRequest) ProtoMessage() {} func (*TaskStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{18} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{18} } func (m *TaskStatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsRequest.Unmarshal(m, b) @@ -1041,7 +1041,7 @@ func (m *TaskStatsResponse) Reset() { *m = TaskStatsResponse{} } func (m *TaskStatsResponse) String() string { return proto.CompactTextString(m) } func (*TaskStatsResponse) ProtoMessage() {} func (*TaskStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{19} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{19} } func (m *TaskStatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsResponse.Unmarshal(m, b) @@ -1078,7 +1078,7 @@ func (m *TaskEventsRequest) Reset() { *m = TaskEventsRequest{} } func (m *TaskEventsRequest) String() string { return proto.CompactTextString(m) } func (*TaskEventsRequest) ProtoMessage() {} func (*TaskEventsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{20} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{20} } func (m *TaskEventsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskEventsRequest.Unmarshal(m, b) @@ -1112,7 +1112,7 @@ func (m *SignalTaskRequest) Reset() { *m = SignalTaskRequest{} } func (m *SignalTaskRequest) String() string { return proto.CompactTextString(m) } func (*SignalTaskRequest) ProtoMessage() {} func (*SignalTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{21} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{21} } func (m *SignalTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskRequest.Unmarshal(m, b) @@ -1156,7 +1156,7 @@ func (m *SignalTaskResponse) Reset() { *m = SignalTaskResponse{} } func (m *SignalTaskResponse) String() string { return proto.CompactTextString(m) } func (*SignalTaskResponse) ProtoMessage() {} func (*SignalTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{22} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{22} } func (m *SignalTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskResponse.Unmarshal(m, b) @@ -1193,7 +1193,7 @@ func (m *ExecTaskRequest) Reset() { *m = ExecTaskRequest{} } func (m *ExecTaskRequest) String() string { return proto.CompactTextString(m) } func (*ExecTaskRequest) ProtoMessage() {} func (*ExecTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{23} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{23} } func (m *ExecTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskRequest.Unmarshal(m, b) @@ -1250,7 +1250,7 @@ func (m *ExecTaskResponse) Reset() { *m = ExecTaskResponse{} } func (m *ExecTaskResponse) String() string { return proto.CompactTextString(m) } func (*ExecTaskResponse) ProtoMessage() {} func (*ExecTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{24} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{24} } func (m *ExecTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskResponse.Unmarshal(m, b) @@ -1303,7 +1303,7 @@ func (m *ExecTaskStreamingIOOperation) Reset() { *m = ExecTaskStreamingI func (m *ExecTaskStreamingIOOperation) String() string { return proto.CompactTextString(m) } func (*ExecTaskStreamingIOOperation) ProtoMessage() {} func (*ExecTaskStreamingIOOperation) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{25} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{25} } func (m *ExecTaskStreamingIOOperation) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskStreamingIOOperation.Unmarshal(m, b) @@ -1350,7 +1350,7 @@ func (m *ExecTaskStreamingRequest) Reset() { *m = ExecTaskStreamingReque func (m *ExecTaskStreamingRequest) String() string { return proto.CompactTextString(m) } func (*ExecTaskStreamingRequest) ProtoMessage() {} func (*ExecTaskStreamingRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{26} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{26} } func (m *ExecTaskStreamingRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskStreamingRequest.Unmarshal(m, b) @@ -1404,7 +1404,7 @@ func (m *ExecTaskStreamingRequest_Setup) Reset() { *m = ExecTaskStreamin func (m *ExecTaskStreamingRequest_Setup) String() string { return proto.CompactTextString(m) } func (*ExecTaskStreamingRequest_Setup) ProtoMessage() {} func (*ExecTaskStreamingRequest_Setup) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{26, 0} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{26, 0} } func (m *ExecTaskStreamingRequest_Setup) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskStreamingRequest_Setup.Unmarshal(m, b) @@ -1457,7 +1457,7 @@ func (m *ExecTaskStreamingRequest_TerminalSize) Reset() { *m = ExecTaskS func (m *ExecTaskStreamingRequest_TerminalSize) String() string { return proto.CompactTextString(m) } func (*ExecTaskStreamingRequest_TerminalSize) ProtoMessage() {} func (*ExecTaskStreamingRequest_TerminalSize) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{26, 1} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{26, 1} } func (m *ExecTaskStreamingRequest_TerminalSize) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskStreamingRequest_TerminalSize.Unmarshal(m, b) @@ -1505,7 +1505,7 @@ func (m *ExecTaskStreamingResponse) Reset() { *m = ExecTaskStreamingResp func (m *ExecTaskStreamingResponse) String() string { return proto.CompactTextString(m) } func (*ExecTaskStreamingResponse) ProtoMessage() {} func (*ExecTaskStreamingResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{27} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{27} } func (m *ExecTaskStreamingResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskStreamingResponse.Unmarshal(m, b) @@ -1571,7 +1571,7 @@ func (m *DriverCapabilities) Reset() { *m = DriverCapabilities{} } func (m *DriverCapabilities) String() string { return proto.CompactTextString(m) } func (*DriverCapabilities) ProtoMessage() {} func (*DriverCapabilities) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{28} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{28} } func (m *DriverCapabilities) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverCapabilities.Unmarshal(m, b) @@ -1657,7 +1657,7 @@ func (m *TaskConfig) Reset() { *m = TaskConfig{} } func (m *TaskConfig) String() string { return proto.CompactTextString(m) } func (*TaskConfig) ProtoMessage() {} func (*TaskConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{29} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{29} } func (m *TaskConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfig.Unmarshal(m, b) @@ -1796,7 +1796,7 @@ func (m *Resources) Reset() { *m = Resources{} } func (m *Resources) String() string { return proto.CompactTextString(m) } func (*Resources) ProtoMessage() {} func (*Resources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{30} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{30} } func (m *Resources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Resources.Unmarshal(m, b) @@ -1843,7 +1843,7 @@ func (m *AllocatedTaskResources) Reset() { *m = AllocatedTaskResources{} func (m *AllocatedTaskResources) String() string { return proto.CompactTextString(m) } func (*AllocatedTaskResources) ProtoMessage() {} func (*AllocatedTaskResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{31} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{31} } func (m *AllocatedTaskResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedTaskResources.Unmarshal(m, b) @@ -1895,7 +1895,7 @@ func (m *AllocatedCpuResources) Reset() { *m = AllocatedCpuResources{} } func (m *AllocatedCpuResources) String() string { return proto.CompactTextString(m) } func (*AllocatedCpuResources) ProtoMessage() {} func (*AllocatedCpuResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{32} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{32} } func (m *AllocatedCpuResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedCpuResources.Unmarshal(m, b) @@ -1933,7 +1933,7 @@ func (m *AllocatedMemoryResources) Reset() { *m = AllocatedMemoryResourc func (m *AllocatedMemoryResources) String() string { return proto.CompactTextString(m) } func (*AllocatedMemoryResources) ProtoMessage() {} func (*AllocatedMemoryResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{33} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{33} } func (m *AllocatedMemoryResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_AllocatedMemoryResources.Unmarshal(m, b) @@ -1976,7 +1976,7 @@ func (m *NetworkResource) Reset() { *m = NetworkResource{} } func (m *NetworkResource) String() string { return proto.CompactTextString(m) } func (*NetworkResource) ProtoMessage() {} func (*NetworkResource) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{34} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{34} } func (m *NetworkResource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkResource.Unmarshal(m, b) @@ -2050,7 +2050,7 @@ func (m *NetworkPort) Reset() { *m = NetworkPort{} } func (m *NetworkPort) String() string { return proto.CompactTextString(m) } func (*NetworkPort) ProtoMessage() {} func (*NetworkPort) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{35} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{35} } func (m *NetworkPort) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkPort.Unmarshal(m, b) @@ -2110,7 +2110,7 @@ func (m *LinuxResources) Reset() { *m = LinuxResources{} } func (m *LinuxResources) String() string { return proto.CompactTextString(m) } func (*LinuxResources) ProtoMessage() {} func (*LinuxResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{36} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{36} } func (m *LinuxResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_LinuxResources.Unmarshal(m, b) @@ -2202,7 +2202,7 @@ func (m *Mount) Reset() { *m = Mount{} } func (m *Mount) String() string { return proto.CompactTextString(m) } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{37} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{37} } func (m *Mount) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Mount.Unmarshal(m, b) @@ -2265,7 +2265,7 @@ func (m *Device) Reset() { *m = Device{} } func (m *Device) String() string { return proto.CompactTextString(m) } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{38} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{38} } func (m *Device) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Device.Unmarshal(m, b) @@ -2326,7 +2326,7 @@ func (m *TaskHandle) Reset() { *m = TaskHandle{} } func (m *TaskHandle) String() string { return proto.CompactTextString(m) } func (*TaskHandle) ProtoMessage() {} func (*TaskHandle) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{39} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{39} } func (m *TaskHandle) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskHandle.Unmarshal(m, b) @@ -2393,7 +2393,7 @@ func (m *NetworkOverride) Reset() { *m = NetworkOverride{} } func (m *NetworkOverride) String() string { return proto.CompactTextString(m) } func (*NetworkOverride) ProtoMessage() {} func (*NetworkOverride) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{40} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{40} } func (m *NetworkOverride) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkOverride.Unmarshal(m, b) @@ -2451,7 +2451,7 @@ func (m *ExitResult) Reset() { *m = ExitResult{} } func (m *ExitResult) String() string { return proto.CompactTextString(m) } func (*ExitResult) ProtoMessage() {} func (*ExitResult) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{41} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{41} } func (m *ExitResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExitResult.Unmarshal(m, b) @@ -2514,7 +2514,7 @@ func (m *TaskStatus) Reset() { *m = TaskStatus{} } func (m *TaskStatus) String() string { return proto.CompactTextString(m) } func (*TaskStatus) ProtoMessage() {} func (*TaskStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{42} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{42} } func (m *TaskStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatus.Unmarshal(m, b) @@ -2589,7 +2589,7 @@ func (m *TaskDriverStatus) Reset() { *m = TaskDriverStatus{} } func (m *TaskDriverStatus) String() string { return proto.CompactTextString(m) } func (*TaskDriverStatus) ProtoMessage() {} func (*TaskDriverStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{43} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{43} } func (m *TaskDriverStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskDriverStatus.Unmarshal(m, b) @@ -2634,7 +2634,7 @@ func (m *TaskStats) Reset() { *m = TaskStats{} } func (m *TaskStats) String() string { return proto.CompactTextString(m) } func (*TaskStats) ProtoMessage() {} func (*TaskStats) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{44} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{44} } func (m *TaskStats) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStats.Unmarshal(m, b) @@ -2696,7 +2696,7 @@ func (m *TaskResourceUsage) Reset() { *m = TaskResourceUsage{} } func (m *TaskResourceUsage) String() string { return proto.CompactTextString(m) } func (*TaskResourceUsage) ProtoMessage() {} func (*TaskResourceUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{45} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{45} } func (m *TaskResourceUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskResourceUsage.Unmarshal(m, b) @@ -2748,7 +2748,7 @@ func (m *CPUUsage) Reset() { *m = CPUUsage{} } func (m *CPUUsage) String() string { return proto.CompactTextString(m) } func (*CPUUsage) ProtoMessage() {} func (*CPUUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{46} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{46} } func (m *CPUUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CPUUsage.Unmarshal(m, b) @@ -2836,7 +2836,7 @@ func (m *MemoryUsage) Reset() { *m = MemoryUsage{} } func (m *MemoryUsage) String() string { return proto.CompactTextString(m) } func (*MemoryUsage) ProtoMessage() {} func (*MemoryUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{47} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{47} } func (m *MemoryUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MemoryUsage.Unmarshal(m, b) @@ -2934,7 +2934,7 @@ func (m *DriverTaskEvent) Reset() { *m = DriverTaskEvent{} } func (m *DriverTaskEvent) String() string { return proto.CompactTextString(m) } func (*DriverTaskEvent) ProtoMessage() {} func (*DriverTaskEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_8793c97f57ec8d84, []int{48} + return fileDescriptor_driver_26c1fb94e7ec6ab0, []int{48} } func (m *DriverTaskEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverTaskEvent.Unmarshal(m, b) @@ -3118,6 +3118,9 @@ type DriverClient interface { SignalTask(ctx context.Context, in *SignalTaskRequest, opts ...grpc.CallOption) (*SignalTaskResponse, error) // ExecTask executes a command inside the tasks execution context ExecTask(ctx context.Context, in *ExecTaskRequest, opts ...grpc.CallOption) (*ExecTaskResponse, error) + // ExecTaskStreaming executes a command inside the tasks execution context + // and streams back results + ExecTaskStreaming(ctx context.Context, opts ...grpc.CallOption) (Driver_ExecTaskStreamingClient, error) } type driverClient struct { @@ -3314,6 +3317,37 @@ func (c *driverClient) ExecTask(ctx context.Context, in *ExecTaskRequest, opts . return out, nil } +func (c *driverClient) ExecTaskStreaming(ctx context.Context, opts ...grpc.CallOption) (Driver_ExecTaskStreamingClient, error) { + stream, err := c.cc.NewStream(ctx, &_Driver_serviceDesc.Streams[3], "/hashicorp.nomad.plugins.drivers.proto.Driver/ExecTaskStreaming", opts...) + if err != nil { + return nil, err + } + x := &driverExecTaskStreamingClient{stream} + return x, nil +} + +type Driver_ExecTaskStreamingClient interface { + Send(*ExecTaskStreamingRequest) error + Recv() (*ExecTaskStreamingResponse, error) + grpc.ClientStream +} + +type driverExecTaskStreamingClient struct { + grpc.ClientStream +} + +func (x *driverExecTaskStreamingClient) Send(m *ExecTaskStreamingRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *driverExecTaskStreamingClient) Recv() (*ExecTaskStreamingResponse, error) { + m := new(ExecTaskStreamingResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // DriverServer is the server API for Driver service. type DriverServer interface { // TaskConfigSchema returns the schema for parsing the driver @@ -3359,6 +3393,9 @@ type DriverServer interface { SignalTask(context.Context, *SignalTaskRequest) (*SignalTaskResponse, error) // ExecTask executes a command inside the tasks execution context ExecTask(context.Context, *ExecTaskRequest) (*ExecTaskResponse, error) + // ExecTaskStreaming executes a command inside the tasks execution context + // and streams back results + ExecTaskStreaming(Driver_ExecTaskStreamingServer) error } func RegisterDriverServer(s *grpc.Server, srv DriverServer) { @@ -3608,6 +3645,32 @@ func _Driver_ExecTask_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _Driver_ExecTaskStreaming_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DriverServer).ExecTaskStreaming(&driverExecTaskStreamingServer{stream}) +} + +type Driver_ExecTaskStreamingServer interface { + Send(*ExecTaskStreamingResponse) error + Recv() (*ExecTaskStreamingRequest, error) + grpc.ServerStream +} + +type driverExecTaskStreamingServer struct { + grpc.ServerStream +} + +func (x *driverExecTaskStreamingServer) Send(m *ExecTaskStreamingResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *driverExecTaskStreamingServer) Recv() (*ExecTaskStreamingRequest, error) { + m := new(ExecTaskStreamingRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + var _Driver_serviceDesc = grpc.ServiceDesc{ ServiceName: "hashicorp.nomad.plugins.drivers.proto.Driver", HandlerType: (*DriverServer)(nil), @@ -3669,215 +3732,223 @@ var _Driver_serviceDesc = grpc.ServiceDesc{ Handler: _Driver_TaskEvents_Handler, ServerStreams: true, }, + { + StreamName: "ExecTaskStreaming", + Handler: _Driver_ExecTaskStreaming_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "plugins/drivers/proto/driver.proto", } func init() { - proto.RegisterFile("plugins/drivers/proto/driver.proto", fileDescriptor_driver_8793c97f57ec8d84) + proto.RegisterFile("plugins/drivers/proto/driver.proto", fileDescriptor_driver_26c1fb94e7ec6ab0) } -var fileDescriptor_driver_8793c97f57ec8d84 = []byte{ - // 3215 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0xdb, 0x6f, 0x1b, 0xc7, - 0xd5, 0x17, 0xaf, 0x22, 0x0f, 0x25, 0x6a, 0x3d, 0xb6, 0x13, 0x9a, 0xf9, 0xbe, 0x2f, 0xce, 0x02, - 0xf9, 0x20, 0x24, 0x31, 0x95, 0x28, 0xa8, 0x6f, 0xcd, 0x8d, 0xa1, 0x68, 0x49, 0xb1, 0x44, 0xa9, - 0x43, 0x0a, 0x8e, 0xeb, 0x26, 0xdb, 0xd5, 0xee, 0x98, 0x5c, 0x6b, 0x6f, 0xd9, 0x9d, 0x95, 0xa5, - 0x14, 0x45, 0x8b, 0x14, 0x28, 0xda, 0x87, 0x02, 0x7d, 0x09, 0xf2, 0xde, 0x3e, 0xf6, 0x1f, 0xe8, - 0x05, 0x79, 0xee, 0x1f, 0xd1, 0xbe, 0xb4, 0x40, 0x81, 0xbe, 0xf6, 0x3f, 0x28, 0xe6, 0xb2, 0xcb, - 0xa5, 0x28, 0xc7, 0x4b, 0xca, 0x4f, 0xbb, 0xe7, 0xcc, 0xcc, 0x6f, 0xce, 0xcc, 0xb9, 0xcc, 0x99, - 0x0b, 0xa8, 0xbe, 0x1d, 0x0d, 0x2d, 0x37, 0x5c, 0x33, 0x03, 0xeb, 0x98, 0x04, 0xe1, 0x9a, 0x1f, - 0x78, 0xd4, 0x93, 0x54, 0x8b, 0x13, 0xe8, 0xf5, 0x91, 0x1e, 0x8e, 0x2c, 0xc3, 0x0b, 0xfc, 0x96, - 0xeb, 0x39, 0xba, 0xd9, 0x92, 0x6d, 0x5a, 0xb2, 0x8d, 0xa8, 0xd6, 0xfc, 0xbf, 0xa1, 0xe7, 0x0d, - 0x6d, 0x22, 0x10, 0x0e, 0xa3, 0xc7, 0x6b, 0x66, 0x14, 0xe8, 0xd4, 0xf2, 0x5c, 0x59, 0xfe, 0xea, - 0xd9, 0x72, 0x6a, 0x39, 0x24, 0xa4, 0xba, 0xe3, 0xcb, 0x0a, 0x1f, 0x0d, 0x2d, 0x3a, 0x8a, 0x0e, - 0x5b, 0x86, 0xe7, 0xac, 0x25, 0x5d, 0xae, 0xf1, 0x2e, 0xd7, 0x62, 0x31, 0xc3, 0x91, 0x1e, 0x10, - 0x73, 0x6d, 0x64, 0xd8, 0xa1, 0x4f, 0x0c, 0xf6, 0xd5, 0xd8, 0x8f, 0x44, 0xd8, 0xcc, 0x8e, 0x10, - 0xd2, 0x20, 0x32, 0x68, 0x3c, 0x5e, 0x9d, 0xd2, 0xc0, 0x3a, 0x8c, 0x28, 0x11, 0x40, 0xea, 0x35, - 0x78, 0x79, 0xa0, 0x87, 0x47, 0x1d, 0xcf, 0x7d, 0x6c, 0x0d, 0xfb, 0xc6, 0x88, 0x38, 0x3a, 0x26, - 0x5f, 0x44, 0x24, 0xa4, 0xea, 0x8f, 0xa0, 0x31, 0x5d, 0x14, 0xfa, 0x9e, 0x1b, 0x12, 0xf4, 0x11, - 0x14, 0x99, 0x34, 0x8d, 0xdc, 0xf5, 0xdc, 0x6a, 0x6d, 0xfd, 0xad, 0xd6, 0xb3, 0x26, 0x4e, 0xc8, - 0xd0, 0x92, 0xa3, 0x68, 0xf5, 0x7d, 0x62, 0x60, 0xde, 0x52, 0xbd, 0x0a, 0x97, 0x3b, 0xba, 0xaf, - 0x1f, 0x5a, 0xb6, 0x45, 0x2d, 0x12, 0xc6, 0x9d, 0x46, 0x70, 0x65, 0x92, 0x2d, 0x3b, 0xfc, 0x0c, - 0x96, 0x8c, 0x14, 0x5f, 0x76, 0x7c, 0xa7, 0x95, 0x49, 0x63, 0xad, 0x0d, 0x4e, 0x4d, 0x00, 0x4f, - 0xc0, 0xa9, 0x57, 0x00, 0xdd, 0xb3, 0xdc, 0x21, 0x09, 0xfc, 0xc0, 0x72, 0x69, 0x2c, 0xcc, 0xb7, - 0x05, 0xb8, 0x3c, 0xc1, 0x96, 0xc2, 0x3c, 0x01, 0x48, 0xe6, 0x91, 0x89, 0x52, 0x58, 0xad, 0xad, - 0x7f, 0x92, 0x51, 0x94, 0x73, 0xf0, 0x5a, 0xed, 0x04, 0xac, 0xeb, 0xd2, 0xe0, 0x14, 0xa7, 0xd0, - 0xd1, 0xe7, 0x50, 0x1e, 0x11, 0xdd, 0xa6, 0xa3, 0x46, 0xfe, 0x7a, 0x6e, 0xb5, 0xbe, 0x7e, 0xef, - 0x02, 0xfd, 0x6c, 0x71, 0xa0, 0x3e, 0xd5, 0x29, 0xc1, 0x12, 0x15, 0xdd, 0x00, 0x24, 0xfe, 0x34, - 0x93, 0x84, 0x46, 0x60, 0xf9, 0xcc, 0x90, 0x1b, 0x85, 0xeb, 0xb9, 0xd5, 0x2a, 0xbe, 0x24, 0x4a, - 0x36, 0xc6, 0x05, 0x4d, 0x1f, 0x56, 0xce, 0x48, 0x8b, 0x14, 0x28, 0x1c, 0x91, 0x53, 0xae, 0x91, - 0x2a, 0x66, 0xbf, 0x68, 0x13, 0x4a, 0xc7, 0xba, 0x1d, 0x11, 0x2e, 0x72, 0x6d, 0xfd, 0x9d, 0xe7, - 0x99, 0x87, 0x34, 0xd1, 0xf1, 0x3c, 0x60, 0xd1, 0xfe, 0x6e, 0xfe, 0x76, 0x4e, 0xbd, 0x03, 0xb5, - 0x94, 0xdc, 0xa8, 0x0e, 0x70, 0xd0, 0xdb, 0xe8, 0x0e, 0xba, 0x9d, 0x41, 0x77, 0x43, 0x59, 0x40, - 0xcb, 0x50, 0x3d, 0xe8, 0x6d, 0x75, 0xdb, 0x3b, 0x83, 0xad, 0x87, 0x4a, 0x0e, 0xd5, 0x60, 0x31, - 0x26, 0xf2, 0xea, 0x09, 0x20, 0x4c, 0x0c, 0xef, 0x98, 0x04, 0xcc, 0x90, 0xa5, 0x56, 0xd1, 0xcb, - 0xb0, 0x48, 0xf5, 0xf0, 0x48, 0xb3, 0x4c, 0x29, 0x73, 0x99, 0x91, 0xdb, 0x26, 0xda, 0x86, 0xf2, - 0x48, 0x77, 0x4d, 0xfb, 0xf9, 0x72, 0x4f, 0x4e, 0x35, 0x03, 0xdf, 0xe2, 0x0d, 0xb1, 0x04, 0x60, - 0xd6, 0x3d, 0xd1, 0xb3, 0x50, 0x80, 0xfa, 0x10, 0x94, 0x3e, 0xd5, 0x03, 0x9a, 0x16, 0xa7, 0x0b, - 0x45, 0xd6, 0xbf, 0xb4, 0xe8, 0x59, 0xfa, 0x14, 0x9e, 0x89, 0x79, 0x73, 0xf5, 0x3f, 0x79, 0xb8, - 0x94, 0xc2, 0x96, 0x96, 0xfa, 0x00, 0xca, 0x01, 0x09, 0x23, 0x9b, 0x72, 0xf8, 0xfa, 0xfa, 0x87, - 0x19, 0xe1, 0xa7, 0x90, 0x5a, 0x98, 0xc3, 0x60, 0x09, 0x87, 0x56, 0x41, 0x11, 0x2d, 0x34, 0x12, - 0x04, 0x5e, 0xa0, 0x39, 0xe1, 0x90, 0xcf, 0x5a, 0x15, 0xd7, 0x05, 0xbf, 0xcb, 0xd8, 0xbb, 0xe1, - 0x30, 0x35, 0xab, 0x85, 0x0b, 0xce, 0x2a, 0xd2, 0x41, 0x71, 0x09, 0x7d, 0xea, 0x05, 0x47, 0x1a, - 0x9b, 0xda, 0xc0, 0x32, 0x49, 0xa3, 0xc8, 0x41, 0x6f, 0x66, 0x04, 0xed, 0x89, 0xe6, 0x7b, 0xb2, - 0x35, 0x5e, 0x71, 0x27, 0x19, 0xea, 0x9b, 0x50, 0x16, 0x23, 0x65, 0x96, 0xd4, 0x3f, 0xe8, 0x74, - 0xba, 0xfd, 0xbe, 0xb2, 0x80, 0xaa, 0x50, 0xc2, 0xdd, 0x01, 0x66, 0x16, 0x56, 0x85, 0xd2, 0xbd, - 0xf6, 0xa0, 0xbd, 0xa3, 0xe4, 0xd5, 0x37, 0x60, 0xe5, 0x81, 0x6e, 0xd1, 0x2c, 0xc6, 0xa5, 0x7a, - 0xa0, 0x8c, 0xeb, 0x4a, 0xed, 0x6c, 0x4f, 0x68, 0x27, 0xfb, 0xd4, 0x74, 0x4f, 0x2c, 0x7a, 0x46, - 0x1f, 0x0a, 0x14, 0x48, 0x10, 0x48, 0x15, 0xb0, 0x5f, 0xf5, 0x29, 0xac, 0xf4, 0xa9, 0xe7, 0x67, - 0xb2, 0xfc, 0x77, 0x61, 0x91, 0xad, 0x51, 0x5e, 0x44, 0xa5, 0xe9, 0x5f, 0x6b, 0x89, 0x35, 0xac, - 0x15, 0xaf, 0x61, 0xad, 0x0d, 0xb9, 0xc6, 0xe1, 0xb8, 0x26, 0x7a, 0x09, 0xca, 0xa1, 0x35, 0x74, - 0x75, 0x5b, 0x46, 0x0b, 0x49, 0xa9, 0x88, 0x19, 0x79, 0xdc, 0xb1, 0x34, 0xfc, 0x0e, 0xa0, 0x0d, - 0x12, 0xd2, 0xc0, 0x3b, 0xcd, 0x24, 0xcf, 0x15, 0x28, 0x3d, 0xf6, 0x02, 0x43, 0x38, 0x62, 0x05, - 0x0b, 0x82, 0x39, 0xd5, 0x04, 0x88, 0xc4, 0xbe, 0x01, 0x68, 0xdb, 0x65, 0x6b, 0x4a, 0x36, 0x45, - 0xfc, 0x36, 0x0f, 0x97, 0x27, 0xea, 0x4b, 0x65, 0xcc, 0xef, 0x87, 0x2c, 0x30, 0x45, 0xa1, 0xf0, - 0x43, 0xb4, 0x07, 0x65, 0x51, 0x43, 0xce, 0xe4, 0xad, 0x19, 0x80, 0xc4, 0x32, 0x25, 0xe1, 0x24, - 0xcc, 0xb9, 0x46, 0x5f, 0x78, 0xb1, 0x46, 0xff, 0x14, 0x94, 0x78, 0x1c, 0xe1, 0x73, 0x75, 0xf3, - 0x09, 0x5c, 0x36, 0x3c, 0xdb, 0x26, 0x06, 0xb3, 0x06, 0xcd, 0x72, 0x29, 0x09, 0x8e, 0x75, 0xfb, - 0xf9, 0x76, 0x83, 0xc6, 0xad, 0xb6, 0x65, 0x23, 0xf5, 0x11, 0x5c, 0x4a, 0x75, 0x2c, 0x15, 0x71, - 0x0f, 0x4a, 0x21, 0x63, 0x48, 0x4d, 0xbc, 0x3d, 0xa3, 0x26, 0x42, 0x2c, 0x9a, 0xab, 0x97, 0x05, - 0x78, 0xf7, 0x98, 0xb8, 0xc9, 0xb0, 0xd4, 0x0d, 0xb8, 0xd4, 0xe7, 0x66, 0x9a, 0xc9, 0x0e, 0xc7, - 0x26, 0x9e, 0x9f, 0x30, 0xf1, 0x2b, 0x80, 0xd2, 0x28, 0xd2, 0x10, 0x4f, 0x61, 0xa5, 0x7b, 0x42, - 0x8c, 0x4c, 0xc8, 0x0d, 0x58, 0x34, 0x3c, 0xc7, 0xd1, 0x5d, 0xb3, 0x91, 0xbf, 0x5e, 0x58, 0xad, - 0xe2, 0x98, 0x4c, 0xfb, 0x62, 0x21, 0xab, 0x2f, 0xaa, 0xbf, 0xc9, 0x81, 0x32, 0xee, 0x5b, 0x4e, - 0x24, 0x93, 0x9e, 0x9a, 0x0c, 0x88, 0xf5, 0xbd, 0x84, 0x25, 0x25, 0xf9, 0x71, 0xb8, 0x10, 0x7c, - 0x12, 0x04, 0xa9, 0x70, 0x54, 0xb8, 0x60, 0x38, 0x52, 0xb7, 0xe0, 0x7f, 0x62, 0x71, 0xfa, 0x34, - 0x20, 0xba, 0x63, 0xb9, 0xc3, 0xed, 0xbd, 0x3d, 0x9f, 0x08, 0xc1, 0x11, 0x82, 0xa2, 0xa9, 0x53, - 0x5d, 0x0a, 0xc6, 0xff, 0x99, 0xd3, 0x1b, 0xb6, 0x17, 0x26, 0x4e, 0xcf, 0x09, 0xf5, 0xaf, 0x05, - 0x68, 0x4c, 0x41, 0xc5, 0xd3, 0xfb, 0x08, 0x4a, 0x21, 0xa1, 0x91, 0x2f, 0x4d, 0xa5, 0x9b, 0x59, - 0xe0, 0xf3, 0xf1, 0x5a, 0x7d, 0x06, 0x86, 0x05, 0x26, 0x1a, 0x42, 0x85, 0xd2, 0x53, 0x2d, 0xb4, - 0xbe, 0x8c, 0x13, 0x82, 0x9d, 0x8b, 0xe2, 0x0f, 0x48, 0xe0, 0x58, 0xae, 0x6e, 0xf7, 0xad, 0x2f, - 0x09, 0x5e, 0xa4, 0xf4, 0x94, 0xfd, 0xa0, 0x87, 0xcc, 0xe0, 0x4d, 0xcb, 0x95, 0xd3, 0xde, 0x99, - 0xb7, 0x97, 0xd4, 0x04, 0x63, 0x81, 0xd8, 0xdc, 0x81, 0x12, 0x1f, 0xd3, 0x3c, 0x86, 0xa8, 0x40, - 0x81, 0xd2, 0x53, 0x2e, 0x54, 0x05, 0xb3, 0xdf, 0xe6, 0x7b, 0xb0, 0x94, 0x1e, 0x01, 0x33, 0xa4, - 0x11, 0xb1, 0x86, 0x23, 0x61, 0x60, 0x25, 0x2c, 0x29, 0xa6, 0xc9, 0xa7, 0x96, 0x29, 0x53, 0xd6, - 0x12, 0x16, 0x84, 0xfa, 0xc7, 0x3c, 0x5c, 0x3b, 0x67, 0x66, 0xa4, 0xb1, 0x3e, 0x9a, 0x30, 0xd6, - 0x17, 0x34, 0x0b, 0xb1, 0xc5, 0x3f, 0x9a, 0xb0, 0xf8, 0x17, 0x08, 0xce, 0xdc, 0xe6, 0x25, 0x28, - 0x93, 0x13, 0x8b, 0x12, 0x53, 0x4e, 0x95, 0xa4, 0x52, 0xee, 0x54, 0xbc, 0xa8, 0x3b, 0xfd, 0x2b, - 0x07, 0x68, 0x7a, 0x0f, 0x83, 0x5e, 0x83, 0xa5, 0x90, 0xb8, 0xa6, 0x26, 0xa2, 0x92, 0x08, 0x98, - 0x15, 0x5c, 0x63, 0x3c, 0x11, 0x9e, 0x42, 0xe6, 0x68, 0xe4, 0x84, 0x18, 0xd2, 0xa7, 0xf8, 0x3f, - 0x1a, 0xc1, 0xd2, 0xe3, 0x50, 0xb3, 0x42, 0xcf, 0xd6, 0x93, 0x64, 0xbf, 0x9e, 0xd9, 0x79, 0xa6, - 0xe5, 0x68, 0xdd, 0xeb, 0x6f, 0xc7, 0x60, 0xb8, 0xf6, 0x38, 0x4c, 0x08, 0xb5, 0x05, 0xb5, 0x54, - 0x19, 0xaa, 0x40, 0xb1, 0xb7, 0xd7, 0xeb, 0x2a, 0x0b, 0x08, 0xa0, 0xdc, 0xd9, 0xc2, 0x7b, 0x7b, - 0x03, 0x91, 0x50, 0x6d, 0xef, 0xb6, 0x37, 0xbb, 0x4a, 0x5e, 0xfd, 0x53, 0x19, 0x60, 0x9c, 0xd9, - 0xa2, 0x3a, 0xe4, 0x13, 0x7b, 0xcd, 0x5b, 0x26, 0x1b, 0x8c, 0xab, 0x3b, 0x44, 0x06, 0x63, 0xfe, - 0x8f, 0xd6, 0xe1, 0xaa, 0x13, 0x0e, 0x7d, 0xdd, 0x38, 0xd2, 0x64, 0x42, 0x6a, 0xf0, 0xc6, 0x7c, - 0x54, 0x4b, 0xf8, 0xb2, 0x2c, 0x94, 0x52, 0x0b, 0xdc, 0x1d, 0x28, 0x10, 0xf7, 0xb8, 0x51, 0xe4, - 0x1b, 0xb7, 0xbb, 0x33, 0x67, 0xdc, 0xad, 0xae, 0x7b, 0x2c, 0x36, 0x6a, 0x0c, 0x06, 0x69, 0x00, - 0x26, 0x39, 0xb6, 0x0c, 0xa2, 0x31, 0xd0, 0x12, 0x07, 0xfd, 0x68, 0x76, 0xd0, 0x0d, 0x8e, 0x91, - 0x40, 0x57, 0xcd, 0x98, 0x46, 0x3d, 0xa8, 0x06, 0x24, 0xf4, 0xa2, 0xc0, 0x20, 0x61, 0xa3, 0x3c, - 0xd3, 0xa2, 0x88, 0xe3, 0x76, 0x78, 0x0c, 0x81, 0x36, 0xa0, 0xec, 0x78, 0x91, 0x4b, 0xc3, 0xc6, - 0x22, 0x17, 0xf6, 0xad, 0x8c, 0x60, 0xbb, 0xac, 0x11, 0x96, 0x6d, 0xd1, 0x26, 0x2c, 0x0a, 0x11, - 0xc3, 0x46, 0x85, 0xc3, 0xdc, 0xc8, 0x6a, 0x40, 0xbc, 0x15, 0x8e, 0x5b, 0x33, 0xad, 0x46, 0x21, - 0x09, 0x1a, 0x55, 0xa1, 0x55, 0xf6, 0x8f, 0x5e, 0x81, 0xaa, 0x6e, 0xdb, 0x9e, 0xa1, 0x99, 0x56, - 0xd0, 0x00, 0x5e, 0x50, 0xe1, 0x8c, 0x0d, 0x2b, 0x40, 0xaf, 0x42, 0x4d, 0xf8, 0xb5, 0xe6, 0xeb, - 0x74, 0xd4, 0xa8, 0xf1, 0x62, 0x10, 0xac, 0x7d, 0x9d, 0x8e, 0x64, 0x05, 0x12, 0x04, 0xa2, 0xc2, - 0x52, 0x52, 0x81, 0x04, 0x01, 0xaf, 0xf0, 0xff, 0xb0, 0xc2, 0xa3, 0xe1, 0x30, 0xf0, 0x22, 0x5f, - 0xe3, 0x36, 0xb5, 0xcc, 0x2b, 0x2d, 0x33, 0xf6, 0x26, 0xe3, 0xf6, 0x98, 0x71, 0x5d, 0x83, 0xca, - 0x13, 0xef, 0x50, 0x54, 0xa8, 0xf3, 0x0a, 0x8b, 0x4f, 0xbc, 0xc3, 0xb8, 0x48, 0x48, 0x68, 0x99, - 0x8d, 0x15, 0x51, 0xc4, 0xe9, 0x6d, 0xb3, 0x79, 0x13, 0x2a, 0xb1, 0x1a, 0xcf, 0xd9, 0x1c, 0x5f, - 0x49, 0x6f, 0x8e, 0xab, 0xa9, 0x9d, 0x6e, 0xf3, 0x3d, 0xa8, 0x4f, 0x1a, 0xc1, 0x2c, 0xad, 0xd5, - 0xbf, 0xe5, 0xa0, 0x9a, 0xa8, 0x1b, 0xb9, 0x70, 0x99, 0x8b, 0xa3, 0x53, 0x62, 0x6a, 0x63, 0xeb, - 0x11, 0xb1, 0xf5, 0xfd, 0x8c, 0x9a, 0x6a, 0xc7, 0x08, 0x32, 0xad, 0x90, 0xa6, 0x84, 0x12, 0xe4, - 0x71, 0x7f, 0x9f, 0xc3, 0x8a, 0x6d, 0xb9, 0xd1, 0x49, 0xaa, 0x2f, 0x11, 0x6a, 0xbf, 0x97, 0xb1, - 0xaf, 0x1d, 0xd6, 0x7a, 0xdc, 0x47, 0xdd, 0x9e, 0xa0, 0xd5, 0xaf, 0xf3, 0xf0, 0xd2, 0xf9, 0xe2, - 0xa0, 0x1e, 0x14, 0x0c, 0x3f, 0x92, 0x43, 0x7b, 0x6f, 0xd6, 0xa1, 0x75, 0xfc, 0x68, 0xdc, 0x2b, - 0x03, 0x62, 0x7b, 0x66, 0x87, 0x38, 0x5e, 0x70, 0x2a, 0x47, 0xf0, 0xe1, 0xac, 0x90, 0xbb, 0xbc, - 0xf5, 0x18, 0x55, 0xc2, 0x21, 0x0c, 0x15, 0x99, 0x79, 0x87, 0x32, 0x4c, 0xcc, 0x98, 0xc1, 0xc7, - 0x90, 0x38, 0xc1, 0x51, 0x6f, 0xc2, 0xd5, 0x73, 0x87, 0x82, 0xfe, 0x17, 0xc0, 0xf0, 0x23, 0x8d, - 0x9f, 0xb0, 0x08, 0xbd, 0x17, 0x70, 0xd5, 0xf0, 0xa3, 0x3e, 0x67, 0xa8, 0xb7, 0xa0, 0xf1, 0x2c, - 0x79, 0x99, 0xf3, 0x09, 0x89, 0x35, 0xe7, 0x90, 0xcf, 0x41, 0x01, 0x57, 0x04, 0x63, 0xf7, 0x50, - 0xfd, 0x26, 0x0f, 0x2b, 0x67, 0xc4, 0x61, 0x2b, 0xa0, 0x70, 0xe6, 0x38, 0xb7, 0x10, 0x14, 0xf3, - 0x6c, 0xc3, 0x32, 0xe3, 0x5d, 0x29, 0xff, 0xe7, 0x31, 0xdd, 0x97, 0x3b, 0xc6, 0xbc, 0xe5, 0x33, - 0x83, 0x76, 0x0e, 0x2d, 0x1a, 0xf2, 0x45, 0xb2, 0x84, 0x05, 0x81, 0x1e, 0x42, 0x3d, 0x20, 0x21, - 0x09, 0x8e, 0x89, 0xa9, 0xf9, 0x5e, 0x40, 0xe3, 0x09, 0x5b, 0x9f, 0x6d, 0xc2, 0xf6, 0xbd, 0x80, - 0xe2, 0xe5, 0x18, 0x89, 0x51, 0x21, 0x7a, 0x00, 0xcb, 0xe6, 0xa9, 0xab, 0x3b, 0x96, 0x21, 0x91, - 0xcb, 0x73, 0x23, 0x2f, 0x49, 0x20, 0x0e, 0xac, 0xde, 0x81, 0x5a, 0xaa, 0x90, 0x0d, 0xcc, 0xd6, - 0x0f, 0x89, 0x2d, 0xe7, 0x44, 0x10, 0x93, 0xfe, 0x5b, 0x92, 0xfe, 0xab, 0xfe, 0x3e, 0x0f, 0xf5, - 0x49, 0x07, 0x88, 0xf5, 0xe7, 0x93, 0xc0, 0xf2, 0xcc, 0x94, 0xfe, 0xf6, 0x39, 0x83, 0xe9, 0x88, - 0x15, 0x7f, 0x11, 0x79, 0x54, 0x8f, 0x75, 0x64, 0xf8, 0xd1, 0x0f, 0x18, 0x7d, 0x46, 0xf7, 0x85, - 0x33, 0xba, 0x47, 0x6f, 0x01, 0x92, 0xfa, 0xb5, 0x2d, 0xc7, 0xa2, 0xda, 0xe1, 0x29, 0x25, 0x62, - 0xfe, 0x0b, 0x58, 0x11, 0x25, 0x3b, 0xac, 0xe0, 0x63, 0xc6, 0x47, 0x2a, 0x2c, 0x7b, 0x9e, 0xa3, - 0x85, 0x86, 0x17, 0x10, 0x4d, 0x37, 0x9f, 0x34, 0x4a, 0xbc, 0x62, 0xcd, 0xf3, 0x9c, 0x3e, 0xe3, - 0xb5, 0xcd, 0x27, 0x2c, 0xe0, 0x1a, 0x7e, 0x14, 0x12, 0xaa, 0xb1, 0x0f, 0x5f, 0xa3, 0xaa, 0x18, - 0x04, 0xab, 0xe3, 0x47, 0x61, 0xaa, 0x82, 0x43, 0x1c, 0xb6, 0xee, 0xa4, 0x2a, 0xec, 0x12, 0x87, - 0xf5, 0xb2, 0xb4, 0x4f, 0x02, 0x83, 0xb8, 0x74, 0x60, 0x19, 0x47, 0x6c, 0x49, 0xc9, 0xad, 0xe6, - 0xf0, 0x04, 0x4f, 0xfd, 0x0c, 0x4a, 0x7c, 0x09, 0x62, 0x83, 0xe7, 0xe1, 0x9b, 0x47, 0x77, 0x31, - 0xbd, 0x15, 0xc6, 0xe0, 0xb1, 0xfd, 0x15, 0xa8, 0x8e, 0xbc, 0x50, 0xae, 0x0d, 0xc2, 0xf2, 0x2a, - 0x8c, 0xc1, 0x0b, 0x9b, 0x50, 0x09, 0x88, 0x6e, 0x7a, 0xae, 0x1d, 0x27, 0xb6, 0x09, 0xad, 0x7e, - 0x01, 0x65, 0x11, 0x7e, 0x2f, 0x80, 0x7f, 0x03, 0x90, 0x21, 0x16, 0x15, 0x9f, 0x25, 0xca, 0x61, - 0x68, 0x79, 0x6e, 0x18, 0x9f, 0xa6, 0x8a, 0x92, 0xfd, 0x71, 0x81, 0xfa, 0xf7, 0x9c, 0xc8, 0x77, - 0xc4, 0x39, 0x17, 0xcb, 0xc5, 0x99, 0xa5, 0xb1, 0x9c, 0x4c, 0x24, 0xd4, 0x31, 0xc9, 0x72, 0x49, - 0x99, 0xd6, 0xe4, 0xe7, 0x3d, 0x26, 0x94, 0x00, 0xf1, 0xf6, 0x9a, 0xc8, 0xb4, 0x6f, 0xd6, 0xed, - 0x35, 0x11, 0xdb, 0x6b, 0xc2, 0x92, 0x4f, 0x99, 0x70, 0x09, 0xb8, 0x22, 0xcf, 0xb7, 0x6a, 0x66, - 0x72, 0x86, 0x41, 0xd4, 0x7f, 0xe7, 0x92, 0x58, 0x11, 0x9f, 0x35, 0xa0, 0xcf, 0xa1, 0xc2, 0xdc, - 0x4e, 0x73, 0x74, 0x5f, 0x9e, 0x9c, 0x77, 0xe6, 0x3b, 0xc6, 0x68, 0x31, 0x2f, 0xdb, 0xd5, 0x7d, - 0x91, 0x2e, 0x2d, 0xfa, 0x82, 0x62, 0x31, 0x47, 0x37, 0xc7, 0x31, 0x87, 0xfd, 0xa3, 0xd7, 0xa1, - 0xae, 0x47, 0xd4, 0xd3, 0x74, 0xf3, 0x98, 0x04, 0xd4, 0x0a, 0x89, 0xd4, 0xfd, 0x32, 0xe3, 0xb6, - 0x63, 0x66, 0xf3, 0x2e, 0x2c, 0xa5, 0x31, 0x9f, 0xb7, 0xfa, 0x96, 0xd2, 0xab, 0xef, 0x8f, 0x01, - 0xc6, 0x79, 0x3b, 0xb3, 0x11, 0xb6, 0x09, 0xd0, 0x0c, 0xcf, 0x24, 0x52, 0x95, 0x15, 0xc6, 0xe8, - 0x78, 0x26, 0x39, 0x73, 0xa8, 0x50, 0x8a, 0x0f, 0x15, 0x98, 0xd7, 0x32, 0x47, 0x3b, 0xb2, 0x6c, - 0x3b, 0xd9, 0x4b, 0x54, 0x3d, 0xcf, 0xb9, 0xcf, 0x19, 0xea, 0xb7, 0x79, 0x61, 0x2b, 0xe2, 0x78, - 0x28, 0x53, 0x6e, 0xfc, 0xa2, 0x54, 0x7d, 0x07, 0x20, 0xa4, 0x7a, 0xc0, 0x52, 0x09, 0x3d, 0xde, - 0xcd, 0x34, 0xa7, 0x4e, 0x25, 0x06, 0xf1, 0x2d, 0x17, 0xae, 0xca, 0xda, 0x6d, 0x8a, 0xde, 0x87, - 0x25, 0xc3, 0x73, 0x7c, 0x9b, 0xc8, 0xc6, 0xa5, 0xe7, 0x36, 0xae, 0x25, 0xf5, 0xdb, 0x34, 0xb5, - 0x87, 0x2a, 0x5f, 0x74, 0x0f, 0xf5, 0xe7, 0x9c, 0x38, 0xe5, 0x4a, 0x1f, 0xb2, 0xa1, 0xe1, 0x39, - 0x37, 0x39, 0x9b, 0x73, 0x9e, 0xd8, 0x7d, 0xd7, 0x35, 0x4e, 0xf3, 0xfd, 0x2c, 0xf7, 0x26, 0xcf, - 0x4e, 0xee, 0xfe, 0x52, 0x80, 0x6a, 0x72, 0xc0, 0x35, 0xa5, 0xfb, 0xdb, 0x50, 0x4d, 0xae, 0x18, - 0x65, 0x80, 0xf8, 0x4e, 0xf5, 0x24, 0x95, 0xd1, 0x63, 0x40, 0xfa, 0x70, 0x98, 0x24, 0x6d, 0x5a, - 0x14, 0xea, 0xc3, 0xf8, 0x78, 0xf1, 0xf6, 0x0c, 0xf3, 0x10, 0xaf, 0x5b, 0x07, 0xac, 0x3d, 0x56, - 0xf4, 0xe1, 0x70, 0x82, 0x83, 0x7e, 0x02, 0x57, 0x27, 0xfb, 0xd0, 0x0e, 0x4f, 0x35, 0xdf, 0x32, - 0xe5, 0x1e, 0x6c, 0x6b, 0xd6, 0x33, 0xbe, 0xd6, 0x04, 0xfc, 0xc7, 0xa7, 0xfb, 0x96, 0x29, 0xe6, - 0x1c, 0x05, 0x53, 0x05, 0xcd, 0x9f, 0xc1, 0xcb, 0xcf, 0xa8, 0x7e, 0x8e, 0x0e, 0x7a, 0x93, 0x77, - 0x57, 0xf3, 0x4f, 0x42, 0x4a, 0x7b, 0xbf, 0xcb, 0x89, 0xa3, 0xc8, 0xc9, 0x39, 0x69, 0xa7, 0xf3, - 0xd6, 0xb5, 0x8c, 0xfd, 0x74, 0xf6, 0x0f, 0x04, 0x3c, 0x4f, 0x55, 0x3f, 0x39, 0x93, 0xaa, 0x66, - 0x4d, 0x62, 0x44, 0xc6, 0x27, 0x80, 0x24, 0x82, 0xfa, 0x87, 0x02, 0x54, 0x62, 0x74, 0xbe, 0x83, - 0x3a, 0x0d, 0x29, 0x71, 0x34, 0x27, 0x0e, 0x61, 0x39, 0x0c, 0x82, 0xb5, 0xcb, 0x82, 0xd8, 0x2b, - 0x50, 0x65, 0x1b, 0x35, 0x51, 0x9c, 0xe7, 0xc5, 0x15, 0xc6, 0xe0, 0x85, 0xaf, 0x42, 0x8d, 0x7a, - 0x54, 0xb7, 0x35, 0xca, 0xd7, 0xf2, 0x82, 0x68, 0xcd, 0x59, 0x7c, 0x25, 0x47, 0x6f, 0xc2, 0x25, - 0x3a, 0x0a, 0x3c, 0x4a, 0x6d, 0x96, 0xdf, 0xf1, 0x8c, 0x46, 0x24, 0x20, 0x45, 0xac, 0x24, 0x05, - 0x22, 0xd3, 0x09, 0x59, 0xf4, 0x1e, 0x57, 0x66, 0xa6, 0xcb, 0x83, 0x48, 0x11, 0x2f, 0x27, 0x5c, - 0x66, 0xda, 0x6c, 0xf1, 0xf4, 0x45, 0xb6, 0xc0, 0x63, 0x45, 0x0e, 0xc7, 0x24, 0xd2, 0x60, 0xc5, - 0x21, 0x7a, 0x18, 0x05, 0xc4, 0xd4, 0x1e, 0x5b, 0xc4, 0x36, 0xc5, 0xc6, 0xb7, 0x9e, 0x39, 0xfd, - 0x8e, 0xa7, 0xa5, 0x75, 0x8f, 0xb7, 0xc6, 0xf5, 0x18, 0x4e, 0xd0, 0x2c, 0x73, 0x10, 0x7f, 0x68, - 0x05, 0x6a, 0xfd, 0x87, 0xfd, 0x41, 0x77, 0x57, 0xdb, 0xdd, 0xdb, 0xe8, 0xca, 0xeb, 0xc9, 0x7e, - 0x17, 0x0b, 0x32, 0xc7, 0xca, 0x07, 0x7b, 0x83, 0xf6, 0x8e, 0x36, 0xd8, 0xee, 0xdc, 0xef, 0x2b, - 0x79, 0x74, 0x15, 0x2e, 0x0d, 0xb6, 0xf0, 0xde, 0x60, 0xb0, 0xd3, 0xdd, 0xd0, 0xf6, 0xbb, 0x78, - 0x7b, 0x6f, 0xa3, 0xaf, 0x14, 0x10, 0x82, 0xfa, 0x98, 0x3d, 0xd8, 0xde, 0xed, 0x2a, 0x45, 0x54, - 0x83, 0xc5, 0xfd, 0x2e, 0xee, 0x74, 0x7b, 0x03, 0xa5, 0xa4, 0x7e, 0x53, 0x80, 0x5a, 0x4a, 0x8b, - 0xcc, 0x90, 0x83, 0x50, 0xe4, 0xf9, 0x45, 0xcc, 0x7e, 0xf9, 0x71, 0xaa, 0x6e, 0x8c, 0x84, 0x76, - 0x8a, 0x58, 0x10, 0x3c, 0xb7, 0xd7, 0x4f, 0x52, 0x7e, 0x5e, 0xc4, 0x15, 0x47, 0x3f, 0x11, 0x20, - 0xaf, 0xc1, 0xd2, 0x11, 0x09, 0x5c, 0x62, 0xcb, 0x72, 0xa1, 0x91, 0x9a, 0xe0, 0x89, 0x2a, 0xab, - 0xa0, 0xc8, 0x2a, 0x63, 0x18, 0xa1, 0x8e, 0xba, 0xe0, 0xef, 0xc6, 0x60, 0x57, 0xa0, 0x24, 0x8a, - 0x17, 0x45, 0xff, 0x9c, 0x60, 0xcb, 0x54, 0xf8, 0x54, 0xf7, 0x79, 0x7e, 0x57, 0xc4, 0xfc, 0x1f, - 0x1d, 0x4e, 0xeb, 0xa7, 0xcc, 0xf5, 0x73, 0x67, 0x76, 0x73, 0x7e, 0x96, 0x8a, 0x46, 0x89, 0x8a, - 0x16, 0xa1, 0x80, 0xe3, 0x3b, 0xbd, 0x4e, 0xbb, 0xb3, 0xc5, 0xd4, 0xb2, 0x0c, 0xd5, 0xdd, 0xf6, - 0xa7, 0xda, 0x41, 0x9f, 0x1f, 0x43, 0x21, 0x05, 0x96, 0xee, 0x77, 0x71, 0xaf, 0xbb, 0x23, 0x39, - 0x05, 0x74, 0x05, 0x14, 0xc9, 0x19, 0xd7, 0x2b, 0x32, 0x04, 0xf1, 0x5b, 0x42, 0x15, 0x28, 0xf6, - 0x1f, 0xb4, 0xf7, 0x95, 0xb2, 0xfa, 0x8f, 0x3c, 0xac, 0x88, 0x65, 0x21, 0xb9, 0x7d, 0x78, 0xf6, - 0xe9, 0x6b, 0xfa, 0x14, 0x21, 0x3f, 0x71, 0x8a, 0x90, 0x24, 0xa1, 0x7c, 0x55, 0x2f, 0x8c, 0x93, - 0x50, 0x7e, 0xfa, 0x30, 0x11, 0xf1, 0x8b, 0xb3, 0x44, 0xfc, 0x06, 0x2c, 0x3a, 0x24, 0x4c, 0xf4, - 0x56, 0xc5, 0x31, 0x89, 0x2c, 0xa8, 0xe9, 0xae, 0xeb, 0x51, 0x7e, 0x56, 0x17, 0x6f, 0x8b, 0x36, - 0x67, 0x3a, 0x15, 0x4c, 0x46, 0xdc, 0x6a, 0x8f, 0x91, 0x44, 0x60, 0x4e, 0x63, 0x37, 0x3f, 0x00, - 0xe5, 0x6c, 0x85, 0x59, 0x96, 0xc3, 0x37, 0xde, 0x19, 0xaf, 0x86, 0x84, 0xf9, 0xc5, 0x41, 0xef, - 0x7e, 0x6f, 0xef, 0x41, 0x4f, 0x59, 0x60, 0x04, 0x3e, 0xe8, 0xf5, 0xb6, 0x7b, 0x9b, 0x4a, 0x0e, - 0x01, 0x94, 0xbb, 0x9f, 0x6e, 0x0f, 0xba, 0x1b, 0x4a, 0x7e, 0xfd, 0x9f, 0xcb, 0x50, 0x16, 0x42, - 0xa2, 0xaf, 0x65, 0x26, 0x90, 0x7e, 0xd9, 0x82, 0x3e, 0x98, 0x39, 0xa3, 0x9e, 0x78, 0x2d, 0xd3, - 0xfc, 0x70, 0xee, 0xf6, 0xf2, 0xf6, 0x68, 0x01, 0xfd, 0x3a, 0x07, 0x4b, 0x13, 0xe7, 0xbb, 0x59, - 0x8f, 0x26, 0xcf, 0x79, 0x48, 0xd3, 0xfc, 0xfe, 0x5c, 0x6d, 0x13, 0x59, 0x7e, 0x95, 0x83, 0x5a, - 0xea, 0x09, 0x09, 0xba, 0x33, 0xcf, 0xb3, 0x13, 0x21, 0xc9, 0xdd, 0xf9, 0x5f, 0xac, 0xa8, 0x0b, - 0x6f, 0xe7, 0xd0, 0x2f, 0x73, 0x50, 0x4b, 0x3d, 0xa6, 0xc8, 0x2c, 0xca, 0xf4, 0xd3, 0x8f, 0xcc, - 0xa2, 0x9c, 0xf7, 0x76, 0x63, 0x01, 0xfd, 0x3c, 0x07, 0xd5, 0xe4, 0x61, 0x04, 0xba, 0x35, 0xfb, - 0x53, 0x0a, 0x21, 0xc4, 0xed, 0x79, 0xdf, 0x60, 0xa8, 0x0b, 0xe8, 0xa7, 0x50, 0x89, 0x5f, 0x11, - 0xa0, 0xac, 0xab, 0xd7, 0x99, 0x27, 0x0a, 0xcd, 0x5b, 0x33, 0xb7, 0x4b, 0x77, 0x1f, 0x5f, 0xed, - 0x67, 0xee, 0xfe, 0xcc, 0x23, 0x84, 0xe6, 0xad, 0x99, 0xdb, 0x25, 0xdd, 0x33, 0x4b, 0x48, 0xbd, - 0x00, 0xc8, 0x6c, 0x09, 0xd3, 0x4f, 0x0f, 0x32, 0x5b, 0xc2, 0x79, 0x0f, 0x0e, 0x84, 0x20, 0xa9, - 0x37, 0x04, 0x99, 0x05, 0x99, 0x7e, 0xa7, 0x90, 0x59, 0x90, 0x73, 0x9e, 0x2c, 0xa8, 0x0b, 0xe8, - 0xab, 0x5c, 0x7a, 0x5f, 0x70, 0x6b, 0xe6, 0xab, 0xf2, 0x19, 0x4d, 0x72, 0xea, 0xb2, 0x9e, 0x3b, - 0xe8, 0x57, 0xf2, 0x14, 0x43, 0xdc, 0xb4, 0xa3, 0x59, 0xc0, 0x26, 0x2e, 0xe7, 0x9b, 0x37, 0xe7, - 0x5b, 0x6c, 0xb8, 0x10, 0xbf, 0xc8, 0x01, 0x8c, 0xef, 0xe4, 0x33, 0x0b, 0x31, 0xf5, 0x18, 0xa0, - 0x79, 0x67, 0x8e, 0x96, 0x69, 0x07, 0x89, 0xef, 0x0c, 0x33, 0x3b, 0xc8, 0x99, 0x37, 0x03, 0x99, - 0x1d, 0xe4, 0xec, 0x7d, 0xbf, 0xba, 0xf0, 0xf1, 0xe2, 0x0f, 0x4b, 0x62, 0xf5, 0x2f, 0xf3, 0xcf, - 0xbb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x69, 0xd1, 0x3c, 0xf6, 0x2a, 0x00, 0x00, +var fileDescriptor_driver_26c1fb94e7ec6ab0 = []byte{ + // 3242 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0xcb, 0x73, 0x1b, 0xc7, + 0x99, 0x27, 0x9e, 0x04, 0x3e, 0x90, 0xe0, 0xa8, 0x25, 0xd9, 0x10, 0xbc, 0xbb, 0x96, 0xa7, 0xca, + 0x5b, 0x2c, 0xdb, 0x02, 0x6d, 0xba, 0x56, 0xaf, 0xb5, 0x2d, 0xc1, 0x20, 0x44, 0xd2, 0x22, 0x41, + 0x6e, 0x03, 0x2c, 0x59, 0xab, 0xb5, 0x67, 0x87, 0x33, 0x2d, 0x60, 0xc4, 0x79, 0x79, 0xa6, 0x87, + 0x22, 0xbd, 0xb5, 0xb5, 0x1b, 0xa7, 0x2a, 0x95, 0x1c, 0x52, 0x95, 0x8b, 0xcb, 0x97, 0x9c, 0x92, + 0x63, 0xfe, 0x81, 0x3c, 0xca, 0xe7, 0xfc, 0x11, 0xc9, 0x25, 0x87, 0x54, 0xe5, 0x9a, 0xff, 0x20, + 0xd5, 0x8f, 0x19, 0x0c, 0x08, 0xca, 0x1a, 0x80, 0x3a, 0xcd, 0x7c, 0x5f, 0x77, 0xff, 0xfa, 0xeb, + 0xfe, 0x1e, 0xfd, 0xf5, 0x03, 0x54, 0xdf, 0x8e, 0x86, 0x96, 0x1b, 0xae, 0x99, 0x81, 0x75, 0x4c, + 0x82, 0x70, 0xcd, 0x0f, 0x3c, 0xea, 0x49, 0xaa, 0xc5, 0x09, 0xf4, 0xf6, 0x48, 0x0f, 0x47, 0x96, + 0xe1, 0x05, 0x7e, 0xcb, 0xf5, 0x1c, 0xdd, 0x6c, 0xc9, 0x36, 0x2d, 0xd9, 0x46, 0x54, 0x6b, 0xfe, + 0xcb, 0xd0, 0xf3, 0x86, 0x36, 0x11, 0x08, 0x87, 0xd1, 0xd3, 0x35, 0x33, 0x0a, 0x74, 0x6a, 0x79, + 0xae, 0x2c, 0x7f, 0xf3, 0x6c, 0x39, 0xb5, 0x1c, 0x12, 0x52, 0xdd, 0xf1, 0x65, 0x85, 0xfb, 0x43, + 0x8b, 0x8e, 0xa2, 0xc3, 0x96, 0xe1, 0x39, 0x6b, 0x49, 0x97, 0x6b, 0xbc, 0xcb, 0xb5, 0x58, 0xcc, + 0x70, 0xa4, 0x07, 0xc4, 0x5c, 0x1b, 0x19, 0x76, 0xe8, 0x13, 0x83, 0x7d, 0x35, 0xf6, 0x23, 0x11, + 0x36, 0xb3, 0x23, 0x84, 0x34, 0x88, 0x0c, 0x1a, 0x8f, 0x57, 0xa7, 0x34, 0xb0, 0x0e, 0x23, 0x4a, + 0x04, 0x90, 0x7a, 0x0d, 0x5e, 0x1f, 0xe8, 0xe1, 0x51, 0xc7, 0x73, 0x9f, 0x5a, 0xc3, 0xbe, 0x31, + 0x22, 0x8e, 0x8e, 0xc9, 0x57, 0x11, 0x09, 0xa9, 0xfa, 0x5f, 0xd0, 0x98, 0x2e, 0x0a, 0x7d, 0xcf, + 0x0d, 0x09, 0xba, 0x0f, 0x45, 0x26, 0x4d, 0x23, 0x77, 0x3d, 0xb7, 0x5a, 0x5b, 0x7f, 0xaf, 0xf5, + 0xa2, 0x89, 0x13, 0x32, 0xb4, 0xe4, 0x28, 0x5a, 0x7d, 0x9f, 0x18, 0x98, 0xb7, 0x54, 0xaf, 0xc2, + 0xe5, 0x8e, 0xee, 0xeb, 0x87, 0x96, 0x6d, 0x51, 0x8b, 0x84, 0x71, 0xa7, 0x11, 0x5c, 0x99, 0x64, + 0xcb, 0x0e, 0xbf, 0x80, 0x25, 0x23, 0xc5, 0x97, 0x1d, 0xdf, 0x69, 0x65, 0xd2, 0x58, 0x6b, 0x83, + 0x53, 0x13, 0xc0, 0x13, 0x70, 0xea, 0x15, 0x40, 0x0f, 0x2c, 0x77, 0x48, 0x02, 0x3f, 0xb0, 0x5c, + 0x1a, 0x0b, 0xf3, 0x7d, 0x01, 0x2e, 0x4f, 0xb0, 0xa5, 0x30, 0xcf, 0x00, 0x92, 0x79, 0x64, 0xa2, + 0x14, 0x56, 0x6b, 0xeb, 0x9f, 0x65, 0x14, 0xe5, 0x1c, 0xbc, 0x56, 0x3b, 0x01, 0xeb, 0xba, 0x34, + 0x38, 0xc5, 0x29, 0x74, 0xf4, 0x25, 0x94, 0x47, 0x44, 0xb7, 0xe9, 0xa8, 0x91, 0xbf, 0x9e, 0x5b, + 0xad, 0xaf, 0x3f, 0xb8, 0x40, 0x3f, 0x5b, 0x1c, 0xa8, 0x4f, 0x75, 0x4a, 0xb0, 0x44, 0x45, 0x37, + 0x00, 0x89, 0x3f, 0xcd, 0x24, 0xa1, 0x11, 0x58, 0x3e, 0x33, 0xe4, 0x46, 0xe1, 0x7a, 0x6e, 0xb5, + 0x8a, 0x2f, 0x89, 0x92, 0x8d, 0x71, 0x41, 0xd3, 0x87, 0x95, 0x33, 0xd2, 0x22, 0x05, 0x0a, 0x47, + 0xe4, 0x94, 0x6b, 0xa4, 0x8a, 0xd9, 0x2f, 0xda, 0x84, 0xd2, 0xb1, 0x6e, 0x47, 0x84, 0x8b, 0x5c, + 0x5b, 0xff, 0xe0, 0x65, 0xe6, 0x21, 0x4d, 0x74, 0x3c, 0x0f, 0x58, 0xb4, 0xbf, 0x9b, 0xbf, 0x9d, + 0x53, 0xef, 0x40, 0x2d, 0x25, 0x37, 0xaa, 0x03, 0x1c, 0xf4, 0x36, 0xba, 0x83, 0x6e, 0x67, 0xd0, + 0xdd, 0x50, 0x16, 0xd0, 0x32, 0x54, 0x0f, 0x7a, 0x5b, 0xdd, 0xf6, 0xce, 0x60, 0xeb, 0xb1, 0x92, + 0x43, 0x35, 0x58, 0x8c, 0x89, 0xbc, 0x7a, 0x02, 0x08, 0x13, 0xc3, 0x3b, 0x26, 0x01, 0x33, 0x64, + 0xa9, 0x55, 0xf4, 0x3a, 0x2c, 0x52, 0x3d, 0x3c, 0xd2, 0x2c, 0x53, 0xca, 0x5c, 0x66, 0xe4, 0xb6, + 0x89, 0xb6, 0xa1, 0x3c, 0xd2, 0x5d, 0xd3, 0x7e, 0xb9, 0xdc, 0x93, 0x53, 0xcd, 0xc0, 0xb7, 0x78, + 0x43, 0x2c, 0x01, 0x98, 0x75, 0x4f, 0xf4, 0x2c, 0x14, 0xa0, 0x3e, 0x06, 0xa5, 0x4f, 0xf5, 0x80, + 0xa6, 0xc5, 0xe9, 0x42, 0x91, 0xf5, 0x2f, 0x2d, 0x7a, 0x96, 0x3e, 0x85, 0x67, 0x62, 0xde, 0x5c, + 0xfd, 0x7b, 0x1e, 0x2e, 0xa5, 0xb0, 0xa5, 0xa5, 0x3e, 0x82, 0x72, 0x40, 0xc2, 0xc8, 0xa6, 0x1c, + 0xbe, 0xbe, 0x7e, 0x2f, 0x23, 0xfc, 0x14, 0x52, 0x0b, 0x73, 0x18, 0x2c, 0xe1, 0xd0, 0x2a, 0x28, + 0xa2, 0x85, 0x46, 0x82, 0xc0, 0x0b, 0x34, 0x27, 0x1c, 0xf2, 0x59, 0xab, 0xe2, 0xba, 0xe0, 0x77, + 0x19, 0x7b, 0x37, 0x1c, 0xa6, 0x66, 0xb5, 0x70, 0xc1, 0x59, 0x45, 0x3a, 0x28, 0x2e, 0xa1, 0xcf, + 0xbd, 0xe0, 0x48, 0x63, 0x53, 0x1b, 0x58, 0x26, 0x69, 0x14, 0x39, 0xe8, 0xcd, 0x8c, 0xa0, 0x3d, + 0xd1, 0x7c, 0x4f, 0xb6, 0xc6, 0x2b, 0xee, 0x24, 0x43, 0x7d, 0x17, 0xca, 0x62, 0xa4, 0xcc, 0x92, + 0xfa, 0x07, 0x9d, 0x4e, 0xb7, 0xdf, 0x57, 0x16, 0x50, 0x15, 0x4a, 0xb8, 0x3b, 0xc0, 0xcc, 0xc2, + 0xaa, 0x50, 0x7a, 0xd0, 0x1e, 0xb4, 0x77, 0x94, 0xbc, 0xfa, 0x0e, 0xac, 0x3c, 0xd2, 0x2d, 0x9a, + 0xc5, 0xb8, 0x54, 0x0f, 0x94, 0x71, 0x5d, 0xa9, 0x9d, 0xed, 0x09, 0xed, 0x64, 0x9f, 0x9a, 0xee, + 0x89, 0x45, 0xcf, 0xe8, 0x43, 0x81, 0x02, 0x09, 0x02, 0xa9, 0x02, 0xf6, 0xab, 0x3e, 0x87, 0x95, + 0x3e, 0xf5, 0xfc, 0x4c, 0x96, 0xff, 0x21, 0x2c, 0xb2, 0x35, 0xca, 0x8b, 0xa8, 0x34, 0xfd, 0x6b, + 0x2d, 0xb1, 0x86, 0xb5, 0xe2, 0x35, 0xac, 0xb5, 0x21, 0xd7, 0x38, 0x1c, 0xd7, 0x44, 0xaf, 0x41, + 0x39, 0xb4, 0x86, 0xae, 0x6e, 0xcb, 0x68, 0x21, 0x29, 0x15, 0x31, 0x23, 0x8f, 0x3b, 0x96, 0x86, + 0xdf, 0x01, 0xb4, 0x41, 0x42, 0x1a, 0x78, 0xa7, 0x99, 0xe4, 0xb9, 0x02, 0xa5, 0xa7, 0x5e, 0x60, + 0x08, 0x47, 0xac, 0x60, 0x41, 0x30, 0xa7, 0x9a, 0x00, 0x91, 0xd8, 0x37, 0x00, 0x6d, 0xbb, 0x6c, + 0x4d, 0xc9, 0xa6, 0x88, 0x5f, 0xe4, 0xe1, 0xf2, 0x44, 0x7d, 0xa9, 0x8c, 0xf9, 0xfd, 0x90, 0x05, + 0xa6, 0x28, 0x14, 0x7e, 0x88, 0xf6, 0xa0, 0x2c, 0x6a, 0xc8, 0x99, 0xbc, 0x35, 0x03, 0x90, 0x58, + 0xa6, 0x24, 0x9c, 0x84, 0x39, 0xd7, 0xe8, 0x0b, 0xaf, 0xd6, 0xe8, 0x9f, 0x83, 0x12, 0x8f, 0x23, + 0x7c, 0xa9, 0x6e, 0x3e, 0x83, 0xcb, 0x86, 0x67, 0xdb, 0xc4, 0x60, 0xd6, 0xa0, 0x59, 0x2e, 0x25, + 0xc1, 0xb1, 0x6e, 0xbf, 0xdc, 0x6e, 0xd0, 0xb8, 0xd5, 0xb6, 0x6c, 0xa4, 0x3e, 0x81, 0x4b, 0xa9, + 0x8e, 0xa5, 0x22, 0x1e, 0x40, 0x29, 0x64, 0x0c, 0xa9, 0x89, 0xf7, 0x67, 0xd4, 0x44, 0x88, 0x45, + 0x73, 0xf5, 0xb2, 0x00, 0xef, 0x1e, 0x13, 0x37, 0x19, 0x96, 0xba, 0x01, 0x97, 0xfa, 0xdc, 0x4c, + 0x33, 0xd9, 0xe1, 0xd8, 0xc4, 0xf3, 0x13, 0x26, 0x7e, 0x05, 0x50, 0x1a, 0x45, 0x1a, 0xe2, 0x29, + 0xac, 0x74, 0x4f, 0x88, 0x91, 0x09, 0xb9, 0x01, 0x8b, 0x86, 0xe7, 0x38, 0xba, 0x6b, 0x36, 0xf2, + 0xd7, 0x0b, 0xab, 0x55, 0x1c, 0x93, 0x69, 0x5f, 0x2c, 0x64, 0xf5, 0x45, 0xf5, 0xe7, 0x39, 0x50, + 0xc6, 0x7d, 0xcb, 0x89, 0x64, 0xd2, 0x53, 0x93, 0x01, 0xb1, 0xbe, 0x97, 0xb0, 0xa4, 0x24, 0x3f, + 0x0e, 0x17, 0x82, 0x4f, 0x82, 0x20, 0x15, 0x8e, 0x0a, 0x17, 0x0c, 0x47, 0xea, 0x16, 0xfc, 0x53, + 0x2c, 0x4e, 0x9f, 0x06, 0x44, 0x77, 0x2c, 0x77, 0xb8, 0xbd, 0xb7, 0xe7, 0x13, 0x21, 0x38, 0x42, + 0x50, 0x34, 0x75, 0xaa, 0x4b, 0xc1, 0xf8, 0x3f, 0x73, 0x7a, 0xc3, 0xf6, 0xc2, 0xc4, 0xe9, 0x39, + 0xa1, 0xfe, 0xb1, 0x00, 0x8d, 0x29, 0xa8, 0x78, 0x7a, 0x9f, 0x40, 0x29, 0x24, 0x34, 0xf2, 0xa5, + 0xa9, 0x74, 0x33, 0x0b, 0x7c, 0x3e, 0x5e, 0xab, 0xcf, 0xc0, 0xb0, 0xc0, 0x44, 0x43, 0xa8, 0x50, + 0x7a, 0xaa, 0x85, 0xd6, 0xd7, 0x71, 0x42, 0xb0, 0x73, 0x51, 0xfc, 0x01, 0x09, 0x1c, 0xcb, 0xd5, + 0xed, 0xbe, 0xf5, 0x35, 0xc1, 0x8b, 0x94, 0x9e, 0xb2, 0x1f, 0xf4, 0x98, 0x19, 0xbc, 0x69, 0xb9, + 0x72, 0xda, 0x3b, 0xf3, 0xf6, 0x92, 0x9a, 0x60, 0x2c, 0x10, 0x9b, 0x3b, 0x50, 0xe2, 0x63, 0x9a, + 0xc7, 0x10, 0x15, 0x28, 0x50, 0x7a, 0xca, 0x85, 0xaa, 0x60, 0xf6, 0xdb, 0xfc, 0x08, 0x96, 0xd2, + 0x23, 0x60, 0x86, 0x34, 0x22, 0xd6, 0x70, 0x24, 0x0c, 0xac, 0x84, 0x25, 0xc5, 0x34, 0xf9, 0xdc, + 0x32, 0x65, 0xca, 0x5a, 0xc2, 0x82, 0x50, 0x7f, 0x9b, 0x87, 0x6b, 0xe7, 0xcc, 0x8c, 0x34, 0xd6, + 0x27, 0x13, 0xc6, 0xfa, 0x8a, 0x66, 0x21, 0xb6, 0xf8, 0x27, 0x13, 0x16, 0xff, 0x0a, 0xc1, 0x99, + 0xdb, 0xbc, 0x06, 0x65, 0x72, 0x62, 0x51, 0x62, 0xca, 0xa9, 0x92, 0x54, 0xca, 0x9d, 0x8a, 0x17, + 0x75, 0xa7, 0xbf, 0xe6, 0x00, 0x4d, 0xef, 0x61, 0xd0, 0x5b, 0xb0, 0x14, 0x12, 0xd7, 0xd4, 0x44, + 0x54, 0x12, 0x01, 0xb3, 0x82, 0x6b, 0x8c, 0x27, 0xc2, 0x53, 0xc8, 0x1c, 0x8d, 0x9c, 0x10, 0x43, + 0xfa, 0x14, 0xff, 0x47, 0x23, 0x58, 0x7a, 0x1a, 0x6a, 0x56, 0xe8, 0xd9, 0x7a, 0x92, 0xec, 0xd7, + 0x33, 0x3b, 0xcf, 0xb4, 0x1c, 0xad, 0x07, 0xfd, 0xed, 0x18, 0x0c, 0xd7, 0x9e, 0x86, 0x09, 0xa1, + 0xb6, 0xa0, 0x96, 0x2a, 0x43, 0x15, 0x28, 0xf6, 0xf6, 0x7a, 0x5d, 0x65, 0x01, 0x01, 0x94, 0x3b, + 0x5b, 0x78, 0x6f, 0x6f, 0x20, 0x12, 0xaa, 0xed, 0xdd, 0xf6, 0x66, 0x57, 0xc9, 0xab, 0xbf, 0x2b, + 0x03, 0x8c, 0x33, 0x5b, 0x54, 0x87, 0x7c, 0x62, 0xaf, 0x79, 0xcb, 0x64, 0x83, 0x71, 0x75, 0x87, + 0xc8, 0x60, 0xcc, 0xff, 0xd1, 0x3a, 0x5c, 0x75, 0xc2, 0xa1, 0xaf, 0x1b, 0x47, 0x9a, 0x4c, 0x48, + 0x0d, 0xde, 0x98, 0x8f, 0x6a, 0x09, 0x5f, 0x96, 0x85, 0x52, 0x6a, 0x81, 0xbb, 0x03, 0x05, 0xe2, + 0x1e, 0x37, 0x8a, 0x7c, 0xe3, 0x76, 0x77, 0xe6, 0x8c, 0xbb, 0xd5, 0x75, 0x8f, 0xc5, 0x46, 0x8d, + 0xc1, 0x20, 0x0d, 0xc0, 0x24, 0xc7, 0x96, 0x41, 0x34, 0x06, 0x5a, 0xe2, 0xa0, 0xf7, 0x67, 0x07, + 0xdd, 0xe0, 0x18, 0x09, 0x74, 0xd5, 0x8c, 0x69, 0xd4, 0x83, 0x6a, 0x40, 0x42, 0x2f, 0x0a, 0x0c, + 0x12, 0x36, 0xca, 0x33, 0x2d, 0x8a, 0x38, 0x6e, 0x87, 0xc7, 0x10, 0x68, 0x03, 0xca, 0x8e, 0x17, + 0xb9, 0x34, 0x6c, 0x2c, 0x72, 0x61, 0xdf, 0xcb, 0x08, 0xb6, 0xcb, 0x1a, 0x61, 0xd9, 0x16, 0x6d, + 0xc2, 0xa2, 0x10, 0x31, 0x6c, 0x54, 0x38, 0xcc, 0x8d, 0xac, 0x06, 0xc4, 0x5b, 0xe1, 0xb8, 0x35, + 0xd3, 0x6a, 0x14, 0x92, 0xa0, 0x51, 0x15, 0x5a, 0x65, 0xff, 0xe8, 0x0d, 0xa8, 0xea, 0xb6, 0xed, + 0x19, 0x9a, 0x69, 0x05, 0x0d, 0xe0, 0x05, 0x15, 0xce, 0xd8, 0xb0, 0x02, 0xf4, 0x26, 0xd4, 0x84, + 0x5f, 0x6b, 0xbe, 0x4e, 0x47, 0x8d, 0x1a, 0x2f, 0x06, 0xc1, 0xda, 0xd7, 0xe9, 0x48, 0x56, 0x20, + 0x41, 0x20, 0x2a, 0x2c, 0x25, 0x15, 0x48, 0x10, 0xf0, 0x0a, 0xff, 0x0a, 0x2b, 0x3c, 0x1a, 0x0e, + 0x03, 0x2f, 0xf2, 0x35, 0x6e, 0x53, 0xcb, 0xbc, 0xd2, 0x32, 0x63, 0x6f, 0x32, 0x6e, 0x8f, 0x19, + 0xd7, 0x35, 0xa8, 0x3c, 0xf3, 0x0e, 0x45, 0x85, 0x3a, 0xaf, 0xb0, 0xf8, 0xcc, 0x3b, 0x8c, 0x8b, + 0x84, 0x84, 0x96, 0xd9, 0x58, 0x11, 0x45, 0x9c, 0xde, 0x36, 0x9b, 0x37, 0xa1, 0x12, 0xab, 0xf1, + 0x9c, 0xcd, 0xf1, 0x95, 0xf4, 0xe6, 0xb8, 0x9a, 0xda, 0xe9, 0x36, 0x3f, 0x82, 0xfa, 0xa4, 0x11, + 0xcc, 0xd2, 0x5a, 0xfd, 0x53, 0x0e, 0xaa, 0x89, 0xba, 0x91, 0x0b, 0x97, 0xb9, 0x38, 0x3a, 0x25, + 0xa6, 0x36, 0xb6, 0x1e, 0x11, 0x5b, 0x3f, 0xce, 0xa8, 0xa9, 0x76, 0x8c, 0x20, 0xd3, 0x0a, 0x69, + 0x4a, 0x28, 0x41, 0x1e, 0xf7, 0xf7, 0x25, 0xac, 0xd8, 0x96, 0x1b, 0x9d, 0xa4, 0xfa, 0x12, 0xa1, + 0xf6, 0xdf, 0x32, 0xf6, 0xb5, 0xc3, 0x5a, 0x8f, 0xfb, 0xa8, 0xdb, 0x13, 0xb4, 0xfa, 0x6d, 0x1e, + 0x5e, 0x3b, 0x5f, 0x1c, 0xd4, 0x83, 0x82, 0xe1, 0x47, 0x72, 0x68, 0x1f, 0xcd, 0x3a, 0xb4, 0x8e, + 0x1f, 0x8d, 0x7b, 0x65, 0x40, 0x6c, 0xcf, 0xec, 0x10, 0xc7, 0x0b, 0x4e, 0xe5, 0x08, 0xee, 0xcd, + 0x0a, 0xb9, 0xcb, 0x5b, 0x8f, 0x51, 0x25, 0x1c, 0xc2, 0x50, 0x91, 0x99, 0x77, 0x28, 0xc3, 0xc4, + 0x8c, 0x19, 0x7c, 0x0c, 0x89, 0x13, 0x1c, 0xf5, 0x26, 0x5c, 0x3d, 0x77, 0x28, 0xe8, 0x9f, 0x01, + 0x0c, 0x3f, 0xd2, 0xf8, 0x09, 0x8b, 0xd0, 0x7b, 0x01, 0x57, 0x0d, 0x3f, 0xea, 0x73, 0x86, 0x7a, + 0x0b, 0x1a, 0x2f, 0x92, 0x97, 0x39, 0x9f, 0x90, 0x58, 0x73, 0x0e, 0xf9, 0x1c, 0x14, 0x70, 0x45, + 0x30, 0x76, 0x0f, 0xd5, 0xef, 0xf2, 0xb0, 0x72, 0x46, 0x1c, 0xb6, 0x02, 0x0a, 0x67, 0x8e, 0x73, + 0x0b, 0x41, 0x31, 0xcf, 0x36, 0x2c, 0x33, 0xde, 0x95, 0xf2, 0x7f, 0x1e, 0xd3, 0x7d, 0xb9, 0x63, + 0xcc, 0x5b, 0x3e, 0x33, 0x68, 0xe7, 0xd0, 0xa2, 0x21, 0x5f, 0x24, 0x4b, 0x58, 0x10, 0xe8, 0x31, + 0xd4, 0x03, 0x12, 0x92, 0xe0, 0x98, 0x98, 0x9a, 0xef, 0x05, 0x34, 0x9e, 0xb0, 0xf5, 0xd9, 0x26, + 0x6c, 0xdf, 0x0b, 0x28, 0x5e, 0x8e, 0x91, 0x18, 0x15, 0xa2, 0x47, 0xb0, 0x6c, 0x9e, 0xba, 0xba, + 0x63, 0x19, 0x12, 0xb9, 0x3c, 0x37, 0xf2, 0x92, 0x04, 0xe2, 0xc0, 0xea, 0x1d, 0xa8, 0xa5, 0x0a, + 0xd9, 0xc0, 0x6c, 0xfd, 0x90, 0xd8, 0x72, 0x4e, 0x04, 0x31, 0xe9, 0xbf, 0x25, 0xe9, 0xbf, 0xea, + 0xaf, 0xf3, 0x50, 0x9f, 0x74, 0x80, 0x58, 0x7f, 0x3e, 0x09, 0x2c, 0xcf, 0x4c, 0xe9, 0x6f, 0x9f, + 0x33, 0x98, 0x8e, 0x58, 0xf1, 0x57, 0x91, 0x47, 0xf5, 0x58, 0x47, 0x86, 0x1f, 0xfd, 0x07, 0xa3, + 0xcf, 0xe8, 0xbe, 0x70, 0x46, 0xf7, 0xe8, 0x3d, 0x40, 0x52, 0xbf, 0xb6, 0xe5, 0x58, 0x54, 0x3b, + 0x3c, 0xa5, 0x44, 0xcc, 0x7f, 0x01, 0x2b, 0xa2, 0x64, 0x87, 0x15, 0x7c, 0xca, 0xf8, 0x48, 0x85, + 0x65, 0xcf, 0x73, 0xb4, 0xd0, 0xf0, 0x02, 0xa2, 0xe9, 0xe6, 0xb3, 0x46, 0x89, 0x57, 0xac, 0x79, + 0x9e, 0xd3, 0x67, 0xbc, 0xb6, 0xf9, 0x8c, 0x05, 0x5c, 0xc3, 0x8f, 0x42, 0x42, 0x35, 0xf6, 0xe1, + 0x6b, 0x54, 0x15, 0x83, 0x60, 0x75, 0xfc, 0x28, 0x4c, 0x55, 0x70, 0x88, 0xc3, 0xd6, 0x9d, 0x54, + 0x85, 0x5d, 0xe2, 0xb0, 0x5e, 0x96, 0xf6, 0x49, 0x60, 0x10, 0x97, 0x0e, 0x2c, 0xe3, 0x88, 0x2d, + 0x29, 0xb9, 0xd5, 0x1c, 0x9e, 0xe0, 0xa9, 0x5f, 0x40, 0x89, 0x2f, 0x41, 0x6c, 0xf0, 0x3c, 0x7c, + 0xf3, 0xe8, 0x2e, 0xa6, 0xb7, 0xc2, 0x18, 0x3c, 0xb6, 0xbf, 0x01, 0xd5, 0x91, 0x17, 0xca, 0xb5, + 0x41, 0x58, 0x5e, 0x85, 0x31, 0x78, 0x61, 0x13, 0x2a, 0x01, 0xd1, 0x4d, 0xcf, 0xb5, 0xe3, 0xc4, + 0x36, 0xa1, 0xd5, 0xaf, 0xa0, 0x2c, 0xc2, 0xef, 0x05, 0xf0, 0x6f, 0x00, 0x32, 0xc4, 0xa2, 0xe2, + 0xb3, 0x44, 0x39, 0x0c, 0x2d, 0xcf, 0x0d, 0xe3, 0xd3, 0x54, 0x51, 0xb2, 0x3f, 0x2e, 0x50, 0xff, + 0x9c, 0x13, 0xf9, 0x8e, 0x38, 0xe7, 0x62, 0xb9, 0x38, 0xb3, 0x34, 0x96, 0x93, 0x89, 0x84, 0x3a, + 0x26, 0x59, 0x2e, 0x29, 0xd3, 0x9a, 0xfc, 0xbc, 0xc7, 0x84, 0x12, 0x20, 0xde, 0x5e, 0x13, 0x99, + 0xf6, 0xcd, 0xba, 0xbd, 0x26, 0x62, 0x7b, 0x4d, 0x58, 0xf2, 0x29, 0x13, 0x2e, 0x01, 0x57, 0xe4, + 0xf9, 0x56, 0xcd, 0x4c, 0xce, 0x30, 0x88, 0xfa, 0xb7, 0x5c, 0x12, 0x2b, 0xe2, 0xb3, 0x06, 0xf4, + 0x25, 0x54, 0x98, 0xdb, 0x69, 0x8e, 0xee, 0xcb, 0x93, 0xf3, 0xce, 0x7c, 0xc7, 0x18, 0x2d, 0xe6, + 0x65, 0xbb, 0xba, 0x2f, 0xd2, 0xa5, 0x45, 0x5f, 0x50, 0x2c, 0xe6, 0xe8, 0xe6, 0x38, 0xe6, 0xb0, + 0x7f, 0xf4, 0x36, 0xd4, 0xf5, 0x88, 0x7a, 0x9a, 0x6e, 0x1e, 0x93, 0x80, 0x5a, 0x21, 0x91, 0xba, + 0x5f, 0x66, 0xdc, 0x76, 0xcc, 0x6c, 0xde, 0x85, 0xa5, 0x34, 0xe6, 0xcb, 0x56, 0xdf, 0x52, 0x7a, + 0xf5, 0xfd, 0x6f, 0x80, 0x71, 0xde, 0xce, 0x6c, 0x84, 0x6d, 0x02, 0x34, 0xc3, 0x33, 0x89, 0x54, + 0x65, 0x85, 0x31, 0x3a, 0x9e, 0x49, 0xce, 0x1c, 0x2a, 0x94, 0xe2, 0x43, 0x05, 0xe6, 0xb5, 0xcc, + 0xd1, 0x8e, 0x2c, 0xdb, 0x4e, 0xf6, 0x12, 0x55, 0xcf, 0x73, 0x1e, 0x72, 0x86, 0xfa, 0x7d, 0x5e, + 0xd8, 0x8a, 0x38, 0x1e, 0xca, 0x94, 0x1b, 0xbf, 0x2a, 0x55, 0xdf, 0x01, 0x08, 0xa9, 0x1e, 0xb0, + 0x54, 0x42, 0x8f, 0x77, 0x33, 0xcd, 0xa9, 0x53, 0x89, 0x41, 0x7c, 0xcb, 0x85, 0xab, 0xb2, 0x76, + 0x9b, 0xa2, 0x8f, 0x61, 0xc9, 0xf0, 0x1c, 0xdf, 0x26, 0xb2, 0x71, 0xe9, 0xa5, 0x8d, 0x6b, 0x49, + 0xfd, 0x36, 0x4d, 0xed, 0xa1, 0xca, 0x17, 0xdd, 0x43, 0xfd, 0x3e, 0x27, 0x4e, 0xb9, 0xd2, 0x87, + 0x6c, 0x68, 0x78, 0xce, 0x4d, 0xce, 0xe6, 0x9c, 0x27, 0x76, 0x3f, 0x74, 0x8d, 0xd3, 0xfc, 0x38, + 0xcb, 0xbd, 0xc9, 0x8b, 0x93, 0xbb, 0x3f, 0x14, 0xa0, 0x9a, 0x1c, 0x70, 0x4d, 0xe9, 0xfe, 0x36, + 0x54, 0x93, 0x2b, 0x46, 0x19, 0x20, 0x7e, 0x50, 0x3d, 0x49, 0x65, 0xf4, 0x14, 0x90, 0x3e, 0x1c, + 0x26, 0x49, 0x9b, 0x16, 0x85, 0xfa, 0x30, 0x3e, 0x5e, 0xbc, 0x3d, 0xc3, 0x3c, 0xc4, 0xeb, 0xd6, + 0x01, 0x6b, 0x8f, 0x15, 0x7d, 0x38, 0x9c, 0xe0, 0xa0, 0xff, 0x81, 0xab, 0x93, 0x7d, 0x68, 0x87, + 0xa7, 0x9a, 0x6f, 0x99, 0x72, 0x0f, 0xb6, 0x35, 0xeb, 0x19, 0x5f, 0x6b, 0x02, 0xfe, 0xd3, 0xd3, + 0x7d, 0xcb, 0x14, 0x73, 0x8e, 0x82, 0xa9, 0x82, 0xe6, 0xff, 0xc1, 0xeb, 0x2f, 0xa8, 0x7e, 0x8e, + 0x0e, 0x7a, 0x93, 0x77, 0x57, 0xf3, 0x4f, 0x42, 0x4a, 0x7b, 0xbf, 0xca, 0x89, 0xa3, 0xc8, 0xc9, + 0x39, 0x69, 0xa7, 0xf3, 0xd6, 0xb5, 0x8c, 0xfd, 0x74, 0xf6, 0x0f, 0x04, 0x3c, 0x4f, 0x55, 0x3f, + 0x3b, 0x93, 0xaa, 0x66, 0x4d, 0x62, 0x44, 0xc6, 0x27, 0x80, 0x24, 0x82, 0xfa, 0x9b, 0x02, 0x54, + 0x62, 0x74, 0xbe, 0x83, 0x3a, 0x0d, 0x29, 0x71, 0x34, 0x27, 0x0e, 0x61, 0x39, 0x0c, 0x82, 0xb5, + 0xcb, 0x82, 0xd8, 0x1b, 0x50, 0x65, 0x1b, 0x35, 0x51, 0x9c, 0xe7, 0xc5, 0x15, 0xc6, 0xe0, 0x85, + 0x6f, 0x42, 0x8d, 0x7a, 0x54, 0xb7, 0x35, 0xca, 0xd7, 0xf2, 0x82, 0x68, 0xcd, 0x59, 0x7c, 0x25, + 0x47, 0xef, 0xc2, 0x25, 0x3a, 0x0a, 0x3c, 0x4a, 0x6d, 0x96, 0xdf, 0xf1, 0x8c, 0x46, 0x24, 0x20, + 0x45, 0xac, 0x24, 0x05, 0x22, 0xd3, 0x09, 0x59, 0xf4, 0x1e, 0x57, 0x66, 0xa6, 0xcb, 0x83, 0x48, + 0x11, 0x2f, 0x27, 0x5c, 0x66, 0xda, 0x6c, 0xf1, 0xf4, 0x45, 0xb6, 0xc0, 0x63, 0x45, 0x0e, 0xc7, + 0x24, 0xd2, 0x60, 0xc5, 0x21, 0x7a, 0x18, 0x05, 0xc4, 0xd4, 0x9e, 0x5a, 0xc4, 0x36, 0xc5, 0xc6, + 0xb7, 0x9e, 0x39, 0xfd, 0x8e, 0xa7, 0xa5, 0xf5, 0x80, 0xb7, 0xc6, 0xf5, 0x18, 0x4e, 0xd0, 0x2c, + 0x73, 0x10, 0x7f, 0x68, 0x05, 0x6a, 0xfd, 0xc7, 0xfd, 0x41, 0x77, 0x57, 0xdb, 0xdd, 0xdb, 0xe8, + 0xca, 0xeb, 0xc9, 0x7e, 0x17, 0x0b, 0x32, 0xc7, 0xca, 0x07, 0x7b, 0x83, 0xf6, 0x8e, 0x36, 0xd8, + 0xee, 0x3c, 0xec, 0x2b, 0x79, 0x74, 0x15, 0x2e, 0x0d, 0xb6, 0xf0, 0xde, 0x60, 0xb0, 0xd3, 0xdd, + 0xd0, 0xf6, 0xbb, 0x78, 0x7b, 0x6f, 0xa3, 0xaf, 0x14, 0x10, 0x82, 0xfa, 0x98, 0x3d, 0xd8, 0xde, + 0xed, 0x2a, 0x45, 0x54, 0x83, 0xc5, 0xfd, 0x2e, 0xee, 0x74, 0x7b, 0x03, 0xa5, 0xa4, 0x7e, 0x57, + 0x80, 0x5a, 0x4a, 0x8b, 0xcc, 0x90, 0x83, 0x50, 0xe4, 0xf9, 0x45, 0xcc, 0x7e, 0xf9, 0x71, 0xaa, + 0x6e, 0x8c, 0x84, 0x76, 0x8a, 0x58, 0x10, 0x3c, 0xb7, 0xd7, 0x4f, 0x52, 0x7e, 0x5e, 0xc4, 0x15, + 0x47, 0x3f, 0x11, 0x20, 0x6f, 0xc1, 0xd2, 0x11, 0x09, 0x5c, 0x62, 0xcb, 0x72, 0xa1, 0x91, 0x9a, + 0xe0, 0x89, 0x2a, 0xab, 0xa0, 0xc8, 0x2a, 0x63, 0x18, 0xa1, 0x8e, 0xba, 0xe0, 0xef, 0xc6, 0x60, + 0x57, 0xa0, 0x24, 0x8a, 0x17, 0x45, 0xff, 0x9c, 0x60, 0xcb, 0x54, 0xf8, 0x5c, 0xf7, 0x79, 0x7e, + 0x57, 0xc4, 0xfc, 0x1f, 0x1d, 0x4e, 0xeb, 0xa7, 0xcc, 0xf5, 0x73, 0x67, 0x76, 0x73, 0x7e, 0x91, + 0x8a, 0x46, 0x89, 0x8a, 0x16, 0xa1, 0x80, 0xe3, 0x3b, 0xbd, 0x4e, 0xbb, 0xb3, 0xc5, 0xd4, 0xb2, + 0x0c, 0xd5, 0xdd, 0xf6, 0xe7, 0xda, 0x41, 0x9f, 0x1f, 0x43, 0x21, 0x05, 0x96, 0x1e, 0x76, 0x71, + 0xaf, 0xbb, 0x23, 0x39, 0x05, 0x74, 0x05, 0x14, 0xc9, 0x19, 0xd7, 0x2b, 0x32, 0x04, 0xf1, 0x5b, + 0x42, 0x15, 0x28, 0xf6, 0x1f, 0xb5, 0xf7, 0x95, 0xb2, 0xfa, 0x97, 0x3c, 0xac, 0x88, 0x65, 0x21, + 0xb9, 0x7d, 0x78, 0xf1, 0xe9, 0x6b, 0xfa, 0x14, 0x21, 0x3f, 0x71, 0x8a, 0x90, 0x24, 0xa1, 0x7c, + 0x55, 0x2f, 0x8c, 0x93, 0x50, 0x7e, 0xfa, 0x30, 0x11, 0xf1, 0x8b, 0xb3, 0x44, 0xfc, 0x06, 0x2c, + 0x3a, 0x24, 0x4c, 0xf4, 0x56, 0xc5, 0x31, 0x89, 0x2c, 0xa8, 0xe9, 0xae, 0xeb, 0x51, 0x7e, 0x56, + 0x17, 0x6f, 0x8b, 0x36, 0x67, 0x3a, 0x15, 0x4c, 0x46, 0xdc, 0x6a, 0x8f, 0x91, 0x44, 0x60, 0x4e, + 0x63, 0x37, 0x3f, 0x01, 0xe5, 0x6c, 0x85, 0x59, 0x96, 0xc3, 0x77, 0x3e, 0x18, 0xaf, 0x86, 0x84, + 0xf9, 0xc5, 0x41, 0xef, 0x61, 0x6f, 0xef, 0x51, 0x4f, 0x59, 0x60, 0x04, 0x3e, 0xe8, 0xf5, 0xb6, + 0x7b, 0x9b, 0x4a, 0x0e, 0x01, 0x94, 0xbb, 0x9f, 0x6f, 0x0f, 0xba, 0x1b, 0x4a, 0x7e, 0xfd, 0x47, + 0x2b, 0x50, 0x16, 0x42, 0xa2, 0x6f, 0x65, 0x26, 0x90, 0x7e, 0xd9, 0x82, 0x3e, 0x99, 0x39, 0xa3, + 0x9e, 0x78, 0x2d, 0xd3, 0xbc, 0x37, 0x77, 0x7b, 0x79, 0x7b, 0xb4, 0x80, 0x7e, 0x96, 0x83, 0xa5, + 0x89, 0xf3, 0xdd, 0xac, 0x47, 0x93, 0xe7, 0x3c, 0xa4, 0x69, 0xfe, 0xfb, 0x5c, 0x6d, 0x13, 0x59, + 0x7e, 0x9a, 0x83, 0x5a, 0xea, 0x09, 0x09, 0xba, 0x33, 0xcf, 0xb3, 0x13, 0x21, 0xc9, 0xdd, 0xf9, + 0x5f, 0xac, 0xa8, 0x0b, 0xef, 0xe7, 0xd0, 0x4f, 0x72, 0x50, 0x4b, 0x3d, 0xa6, 0xc8, 0x2c, 0xca, + 0xf4, 0xd3, 0x8f, 0xcc, 0xa2, 0x9c, 0xf7, 0x76, 0x63, 0x01, 0xfd, 0x7f, 0x0e, 0xaa, 0xc9, 0xc3, + 0x08, 0x74, 0x6b, 0xf6, 0xa7, 0x14, 0x42, 0x88, 0xdb, 0xf3, 0xbe, 0xc1, 0x50, 0x17, 0xd0, 0xff, + 0x42, 0x25, 0x7e, 0x45, 0x80, 0xb2, 0xae, 0x5e, 0x67, 0x9e, 0x28, 0x34, 0x6f, 0xcd, 0xdc, 0x2e, + 0xdd, 0x7d, 0x7c, 0xb5, 0x9f, 0xb9, 0xfb, 0x33, 0x8f, 0x10, 0x9a, 0xb7, 0x66, 0x6e, 0x97, 0x74, + 0xcf, 0x2c, 0x21, 0xf5, 0x02, 0x20, 0xb3, 0x25, 0x4c, 0x3f, 0x3d, 0xc8, 0x6c, 0x09, 0xe7, 0x3d, + 0x38, 0x10, 0x82, 0xa4, 0xde, 0x10, 0x64, 0x16, 0x64, 0xfa, 0x9d, 0x42, 0x66, 0x41, 0xce, 0x79, + 0xb2, 0xa0, 0x2e, 0xa0, 0x6f, 0x72, 0xe9, 0x7d, 0xc1, 0xad, 0x99, 0xaf, 0xca, 0x67, 0x34, 0xc9, + 0xa9, 0xcb, 0x7a, 0xee, 0xa0, 0xdf, 0xc8, 0x53, 0x0c, 0x71, 0xd3, 0x8e, 0x66, 0x01, 0x9b, 0xb8, + 0x9c, 0x6f, 0xde, 0x9c, 0x6f, 0xb1, 0xe1, 0x42, 0xfc, 0x38, 0x07, 0x30, 0xbe, 0x93, 0xcf, 0x2c, + 0xc4, 0xd4, 0x63, 0x80, 0xe6, 0x9d, 0x39, 0x5a, 0xa6, 0x1d, 0x24, 0xbe, 0x33, 0xcc, 0xec, 0x20, + 0x67, 0xde, 0x0c, 0x64, 0x76, 0x90, 0xb3, 0xf7, 0xfd, 0xea, 0x02, 0xfa, 0x65, 0x0e, 0x2e, 0x4d, + 0xdd, 0x59, 0xa2, 0x7b, 0x17, 0xbc, 0xb6, 0x6e, 0xde, 0x9f, 0x1f, 0x20, 0x16, 0x6d, 0x35, 0xf7, + 0x7e, 0xee, 0xd3, 0xc5, 0xff, 0x2c, 0x89, 0xe4, 0xa4, 0xcc, 0x3f, 0x1f, 0xfe, 0x23, 0x00, 0x00, + 0xff, 0xff, 0x8b, 0x62, 0x9f, 0x34, 0x95, 0x2b, 0x00, 0x00, } diff --git a/plugins/drivers/proto/driver.proto b/plugins/drivers/proto/driver.proto index 974c5dc36..74628ee15 100644 --- a/plugins/drivers/proto/driver.proto +++ b/plugins/drivers/proto/driver.proto @@ -70,6 +70,10 @@ service Driver { // ExecTask executes a command inside the tasks execution context rpc ExecTask(ExecTaskRequest) returns (ExecTaskResponse) {} + + // ExecTaskStreaming executes a command inside the tasks execution context + // and streams back results + rpc ExecTaskStreaming(stream ExecTaskStreamingRequest) returns (stream ExecTaskStreamingResponse) {} } message TaskConfigSchemaRequest {} diff --git a/plugins/drivers/server.go b/plugins/drivers/server.go index 6de80d9e7..5b7723422 100644 --- a/plugins/drivers/server.go +++ b/plugins/drivers/server.go @@ -277,6 +277,60 @@ func (b *driverPluginServer) ExecTask(ctx context.Context, req *proto.ExecTaskRe return resp, nil } +func (b *driverPluginServer) ExecTaskStreaming(server proto.Driver_ExecTaskStreamingServer) error { + msg, err := server.Recv() + if err != nil { + return fmt.Errorf("failed to receive initial message: %v", err) + } + + if msg.Setup == nil { + return fmt.Errorf("first message should always be setup") + } + + if impl, ok := b.impl.(ExecTaskStreamingRawDriver); ok { + return impl.ExecTaskStreamingRaw(server.Context(), + msg.Setup.TaskId, msg.Setup.Command, msg.Setup.Tty, + server) + } + + d, ok := b.impl.(ExecTaskStreamingDriver) + if !ok { + return fmt.Errorf("driver does not support exec") + } + + execOpts, errCh := StreamToExecOptions(server.Context(), + msg.Setup.Command, msg.Setup.Tty, + server) + + result, err := d.ExecTaskStreaming(server.Context(), + msg.Setup.TaskId, execOpts) + + execOpts.Stdout.Close() + execOpts.Stderr.Close() + + if err != nil { + return err + } + + // wait for copy to be done + select { + case err = <-errCh: + case <-server.Context().Done(): + err = fmt.Errorf("exec timed out: %v", server.Context().Err()) + } + + if err != nil { + return err + } + + server.Send(&ExecTaskStreamingResponseMsg{ + Exited: true, + Result: exitResultToProto(result), + }) + + return err +} + func (b *driverPluginServer) SignalTask(ctx context.Context, req *proto.SignalTaskRequest) (*proto.SignalTaskResponse, error) { err := b.impl.SignalTask(req.TaskId, req.Signal) if err != nil { From cb4ad3fb4567e865ee6e0c52381cb89f7772c09d Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:21:00 -0400 Subject: [PATCH 07/15] drivers/mock: implement nomad exec interface --- drivers/mock/driver.go | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/drivers/mock/driver.go b/drivers/mock/driver.go index 5dced6c85..1c99a9a9a 100644 --- a/drivers/mock/driver.go +++ b/drivers/mock/driver.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io/ioutil" "math/rand" "strconv" "strings" @@ -91,6 +92,20 @@ var ( "stderr_string": hclspec.NewAttr("stderr_string", "string", false), "stderr_repeat": hclspec.NewAttr("stderr_repeat", "number", false), "stderr_repeat_duration": hclspec.NewAttr("stderr_repeat_duration", "string", false), + + "exec_command": hclspec.NewBlock("exec_command", false, hclspec.NewObject(map[string]*hclspec.Spec{ + "run_for": hclspec.NewAttr("run_for", "string", false), + "exit_code": hclspec.NewAttr("exit_code", "number", false), + "exit_signal": hclspec.NewAttr("exit_signal", "number", false), + "exit_err_msg": hclspec.NewAttr("exit_err_msg", "string", false), + "signal_error": hclspec.NewAttr("signal_error", "string", false), + "stdout_string": hclspec.NewAttr("stdout_string", "string", false), + "stdout_repeat": hclspec.NewAttr("stdout_repeat", "number", false), + "stdout_repeat_duration": hclspec.NewAttr("stdout_repeat_duration", "string", false), + "stderr_string": hclspec.NewAttr("stderr_string", "string", false), + "stderr_repeat": hclspec.NewAttr("stderr_repeat", "number", false), + "stderr_repeat_duration": hclspec.NewAttr("stderr_repeat_duration", "string", false), + })), }) ) @@ -216,6 +231,8 @@ type Command struct { type TaskConfig struct { Command + ExecCommand *Command `codec:"exec_command"` + // PluginExitAfter is the duration after which the mock driver indicates the // plugin has exited via the WaitTask call. PluginExitAfter string `codec:"plugin_exit_after"` @@ -413,6 +430,7 @@ func newTaskHandle(cfg *drivers.TaskConfig, driverConfig *TaskConfig, logger hcl h := &taskHandle{ taskConfig: cfg, command: driverConfig.Command, + execCommand: driverConfig.ExecCommand, pluginExitAfter: driverConfig.pluginExitAfterDuration, killAfter: driverConfig.killAfterDuration, logger: logger.With("task_name", cfg.Name), @@ -603,6 +621,38 @@ func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (* return &res, nil } +var _ drivers.ExecTaskStreamingDriver = (*Driver)(nil) + +func (d *Driver) ExecTaskStreaming(ctx context.Context, taskID string, execOpts *drivers.ExecOptions) (*drivers.ExitResult, error) { + h, ok := d.tasks.Get(taskID) + if !ok { + return nil, drivers.ErrTaskNotFound + } + + d.logger.Info("executing task", "command", h.execCommand, "task_id", taskID) + + if h.execCommand == nil { + return nil, errors.New("no exec command is configured") + } + + cancelCh := make(chan struct{}) + exitTimer := make(chan time.Time) + + cmd := *h.execCommand + if len(execOpts.Command) == 1 && execOpts.Command[0] == "showinput" { + stdin, _ := ioutil.ReadAll(execOpts.Stdin) + cmd = Command{ + RunFor: "1ms", + StdoutString: fmt.Sprintf("TTY: %v\nStdin:\n%s\n", + execOpts.Tty, + stdin, + ), + } + } + + return runCommand(cmd, execOpts.Stdout, execOpts.Stderr, cancelCh, exitTimer, d.logger), nil +} + // GetTaskConfig is unique to the mock driver and for testing purposes only. It // returns the *drivers.TaskConfig passed to StartTask and the decoded // *mock.TaskConfig created by the last StartTask call. From 979a6a17784feaa00c0af34d6692470029f0bcab Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:22:53 -0400 Subject: [PATCH 08/15] implement client endpoint of nomad exec Add a client streaming RPC endpoint for processing nomad exec tasks, by invoking the relevant task handler for execution. --- client/alloc_endpoint.go | 191 +++++ client/alloc_endpoint_test.go | 733 ++++++++++++++++++ client/allocrunner/alloc_runner.go | 18 + .../allocrunner/taskrunner/driver_handle.go | 42 + client/allocrunner/taskrunner/task_runner.go | 8 + client/client.go | 4 + client/fs_endpoint.go | 46 +- client/pluginmanager/drivermanager/manager.go | 7 + client/rpc.go | 2 +- client/structs/structs.go | 17 + 10 files changed, 1044 insertions(+), 24 deletions(-) diff --git a/client/alloc_endpoint.go b/client/alloc_endpoint.go index c141f0e36..febad3fbf 100644 --- a/client/alloc_endpoint.go +++ b/client/alloc_endpoint.go @@ -1,12 +1,22 @@ package client import ( + "bytes" + "context" + "errors" + "fmt" + "io" "time" metrics "github.com/armon/go-metrics" "github.com/hashicorp/nomad/acl" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" nstructs "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/ugorji/go/codec" ) // Allocations endpoint is used for interacting with client allocations @@ -14,6 +24,12 @@ type Allocations struct { c *Client } +func NewAllocationsEndpoint(c *Client) *Allocations { + a := &Allocations{c: c} + a.c.streamingRpcs.Register("Allocations.Exec", a.exec) + return a +} + // GarbageCollectAll is used to garbage collect all allocations on a client. func (a *Allocations) GarbageCollectAll(args *nstructs.NodeSpecificRequest, reply *nstructs.GenericResponse) error { defer metrics.MeasureSince([]string{"client", "allocations", "garbage_collect_all"}, time.Now()) @@ -100,3 +116,178 @@ func (a *Allocations) Stats(args *cstructs.AllocStatsRequest, reply *cstructs.Al reply.Stats = stats return nil } + +// exec is used to execute command in a running task +func (a *Allocations) exec(conn io.ReadWriteCloser) { + defer metrics.MeasureSince([]string{"client", "allocations", "exec"}, time.Now()) + defer conn.Close() + + execID := uuid.Generate() + decoder := codec.NewDecoder(conn, structs.MsgpackHandle) + encoder := codec.NewEncoder(conn, structs.MsgpackHandle) + + code, err := a.execImpl(encoder, decoder, execID) + if err != nil { + a.c.logger.Info("task exec session ended with an error", "error", err, "code", code) + handleStreamResultError(err, code, encoder) + return + } + + a.c.logger.Info("task exec session ended", "exec_id", execID) +} + +func (a *Allocations) execImpl(encoder *codec.Encoder, decoder *codec.Decoder, execID string) (code *int64, err error) { + + // Decode the arguments + var req cstructs.AllocExecRequest + if err := decoder.Decode(&req); err != nil { + return helper.Int64ToPtr(500), err + } + + aclObj, token, err := a.c.resolveTokenAndACL(req.QueryOptions.AuthToken) + { + // log access + tokenName, tokenID := "", "" + if token != nil { + tokenName, tokenID = token.Name, token.AccessorID + } + + a.c.logger.Info("task exec session starting", + "exec_id", execID, + "alloc_id", req.AllocID, + "task", req.Task, + "command", req.Cmd, + "tty", req.Tty, + "access_token_name", tokenName, + "access_token_id", tokenID, + ) + } + + // Check read permissions + if err != nil { + return nil, err + } else if aclObj != nil { + exec := aclObj.AllowNsOp(req.QueryOptions.Namespace, acl.NamespaceCapabilityAllocExec) + if !exec { + return nil, structs.ErrPermissionDenied + } + } + + // Validate the arguments + if req.AllocID == "" { + return helper.Int64ToPtr(400), allocIDNotPresentErr + } + if req.Task == "" { + return helper.Int64ToPtr(400), taskNotPresentErr + } + if len(req.Cmd) == 0 { + return helper.Int64ToPtr(400), errors.New("command is not present") + } + + ar, err := a.c.getAllocRunner(req.AllocID) + if err != nil { + code := helper.Int64ToPtr(500) + if structs.IsErrUnknownAllocation(err) { + code = helper.Int64ToPtr(404) + } + + return code, err + } + + capabilities, err := ar.GetTaskDriverCapabilities(req.Task) + if err != nil { + code := helper.Int64ToPtr(500) + if structs.IsErrUnknownAllocation(err) { + code = helper.Int64ToPtr(404) + } + + return code, err + } + + // check node access + if aclObj != nil && capabilities.FSIsolation == drivers.FSIsolationNone { + exec := aclObj.AllowNsOp(req.QueryOptions.Namespace, acl.NamespaceCapabilityAllocNodeExec) + if !exec { + return nil, structs.ErrPermissionDenied + } + } + + allocState, err := a.c.GetAllocState(req.AllocID) + if err != nil { + code := helper.Int64ToPtr(500) + if structs.IsErrUnknownAllocation(err) { + code = helper.Int64ToPtr(404) + } + + return code, err + } + + // Check that the task is there + taskState := allocState.TaskStates[req.Task] + if taskState == nil { + return helper.Int64ToPtr(400), fmt.Errorf("unknown task name %q", req.Task) + } + + if taskState.StartedAt.IsZero() { + return helper.Int64ToPtr(404), fmt.Errorf("task %q not started yet.", req.Task) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h := ar.GetTaskExecHandler(req.Task) + if h == nil { + return helper.Int64ToPtr(404), fmt.Errorf("task %q is not running.", req.Task) + } + + err = h(ctx, req.Cmd, req.Tty, newExecStream(cancel, decoder, encoder)) + if err != nil { + code := helper.Int64ToPtr(500) + return code, err + } + + return nil, nil +} + +// newExecStream returns a new exec stream as expected by drivers that interpolate with RPC streaming format +func newExecStream(cancelFn func(), decoder *codec.Decoder, encoder *codec.Encoder) drivers.ExecTaskStream { + buf := new(bytes.Buffer) + return &execStream{ + cancelFn: cancelFn, + decoder: decoder, + + buf: buf, + encoder: encoder, + frameCodec: codec.NewEncoder(buf, structs.JsonHandle), + } +} + +type execStream struct { + cancelFn func() + decoder *codec.Decoder + + encoder *codec.Encoder + buf *bytes.Buffer + frameCodec *codec.Encoder +} + +// Send sends driver output response across RPC mechanism using cstructs.StreamErrWrapper +func (s *execStream) Send(m *drivers.ExecTaskStreamingResponseMsg) error { + s.buf.Reset() + s.frameCodec.Reset(s.buf) + + s.frameCodec.MustEncode(m) + return s.encoder.Encode(cstructs.StreamErrWrapper{ + Payload: s.buf.Bytes(), + }) +} + +// Recv returns next exec user input from the RPC to be passed to driver exec handler +func (s *execStream) Recv() (*drivers.ExecTaskStreamingRequestMsg, error) { + req := drivers.ExecTaskStreamingRequestMsg{} + err := s.decoder.Decode(&req) + if err == io.EOF || err == io.ErrClosedPipe { + s.cancelFn() + } + return &req, err +} diff --git a/client/alloc_endpoint_test.go b/client/alloc_endpoint_test.go index a29d60eee..56b8cdeca 100644 --- a/client/alloc_endpoint_test.go +++ b/client/alloc_endpoint_test.go @@ -1,17 +1,30 @@ package client import ( + "encoding/json" "fmt" + "io" + "net" + "runtime" "strings" "testing" + "time" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/client/config" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper/pluginutils/catalog" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" nstructs "github.com/hashicorp/nomad/nomad/structs" + nconfig "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/require" + "github.com/ugorji/go/codec" + "golang.org/x/sys/unix" ) func TestAllocations_Restart(t *testing.T) { @@ -446,3 +459,723 @@ func TestAllocations_Stats_ACL(t *testing.T) { require.True(nstructs.IsErrUnknownAllocation(err)) } } + +func TestAlloc_ExecStreaming(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Start a server and client + s := nomad.TestServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + c, cleanup := TestClient(t, func(c *config.Config) { + c.Servers = []string{s.GetConfig().RPCAddr.String()} + }) + defer cleanup() + + expectedStdout := "Hello from the other side\n" + expectedStderr := "Hello from the other side\n" + job := mock.BatchJob() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "20s", + "exec_command": map[string]interface{}{ + "run_for": "1ms", + "stdout_string": expectedStdout, + "stderr_string": expectedStderr, + "exit_code": 3, + }, + } + + // Wait for client to be running job + testutil.WaitForRunning(t, s.RPC, job) + + // Get the allocation ID + args := nstructs.AllocListRequest{} + args.Region = "global" + resp := nstructs.AllocListResponse{} + require.NoError(s.RPC("Alloc.List", &args, &resp)) + require.Len(resp.Allocations, 1) + allocID := resp.Allocations[0].ID + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: allocID, + Task: job.TaskGroups[0].Tasks[0].Name, + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{Region: "global"}, + } + + // Get the handler + handler, err := c.StreamingRpcHandler("Allocations.Exec") + require.Nil(err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(encoder.Encode(req)) + + timeout := time.After(3 * time.Second) + + exitCode := -1 + receivedStdout := "" + receivedStderr := "" + +OUTER: + for { + select { + case <-timeout: + // time out report + require.Equal(expectedStdout, receivedStderr, "didn't receive expected stdout") + require.Equal(expectedStderr, receivedStderr, "didn't receive expected stderr") + require.Equal(3, exitCode, "failed to get exit code") + require.FailNow("timed out") + case err := <-errCh: + require.NoError(err) + case f := <-frames: + switch { + case f.Stdout != nil && len(f.Stdout.Data) != 0: + receivedStdout += string(f.Stdout.Data) + case f.Stderr != nil && len(f.Stderr.Data) != 0: + receivedStderr += string(f.Stderr.Data) + case f.Exited && f.Result != nil: + exitCode = int(f.Result.ExitCode) + default: + t.Logf("received unrelevant frame: %v", f) + } + + if expectedStdout == receivedStdout && expectedStderr == receivedStderr && exitCode == 3 { + break OUTER + } + } + } +} + +func TestAlloc_ExecStreaming_NoAllocation(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Start a server and client + s := nomad.TestServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + c, cleanup := TestClient(t, func(c *config.Config) { + c.Servers = []string{s.GetConfig().RPCAddr.String()} + }) + defer cleanup() + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: uuid.Generate(), + Task: "testtask", + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{Region: "global"}, + } + + // Get the handler + handler, err := c.StreamingRpcHandler("Allocations.Exec") + require.Nil(err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(encoder.Encode(req)) + + timeout := time.After(3 * time.Second) + + select { + case <-timeout: + require.FailNow("timed out") + case err := <-errCh: + require.True(nstructs.IsErrUnknownAllocation(err), "expected no allocation error but found: %v", err) + case f := <-frames: + require.Fail("received unexpected frame", "frame: %#v", f) + } +} + +func TestAlloc_ExecStreaming_ACL_Basic(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Start a server and client + s, root := nomad.TestACLServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + client, cleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{s.GetConfig().RPCAddr.String()} + }) + defer cleanup() + + // Create a bad token + policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) + tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) + + policyGood := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityReadFS}) + tokenGood := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyGood) + + cases := []struct { + Name string + Token string + ExpectedError string + }{ + { + Name: "bad token", + Token: tokenBad.SecretID, + ExpectedError: structs.ErrPermissionDenied.Error(), + }, + { + Name: "good token", + Token: tokenGood.SecretID, + ExpectedError: structs.ErrUnknownAllocationPrefix, + }, + { + Name: "root token", + Token: root.SecretID, + ExpectedError: structs.ErrUnknownAllocationPrefix, + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: uuid.Generate(), + Task: "testtask", + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{ + Region: "global", + AuthToken: c.Token, + Namespace: nstructs.DefaultNamespace, + }, + } + + // Get the handler + handler, err := client.StreamingRpcHandler("Allocations.Exec") + require.Nil(err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(encoder.Encode(req)) + + select { + case <-time.After(3 * time.Second): + require.FailNow("timed out") + case err := <-errCh: + require.Contains(err.Error(), c.ExpectedError) + case f := <-frames: + require.Fail("received unexpected frame", "frame: %#v", f) + } + }) + } +} + +// TestAlloc_ExecStreaming_ACL_WithIsolation_Image asserts that token only needs +// alloc-exec acl policy when image isolation is used +func TestAlloc_ExecStreaming_ACL_WithIsolation_Image(t *testing.T) { + t.Parallel() + isolation := drivers.FSIsolationImage + + // Start a server and client + s, root := nomad.TestACLServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + client, cleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{s.GetConfig().RPCAddr.String()} + + pluginConfig := []*nconfig.PluginConfig{ + { + Name: "mock_driver", + Config: map[string]interface{}{ + "fs_isolation": string(isolation), + }, + }, + } + + c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) + }) + defer cleanup() + + // Create a bad token + policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) + tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) + + policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec}) + tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyAllocExec) + + policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) + tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "valid2", policyAllocNodeExec) + + job := mock.BatchJob() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "20s", + "exec_command": map[string]interface{}{ + "run_for": "1ms", + "stdout_string": "some output", + }, + } + + // Wait for client to be running job + testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) + + // Get the allocation ID + args := nstructs.AllocListRequest{} + args.Region = "global" + args.AuthToken = root.SecretID + args.Namespace = nstructs.DefaultNamespace + resp := nstructs.AllocListResponse{} + require.NoError(t, s.RPC("Alloc.List", &args, &resp)) + require.Len(t, resp.Allocations, 1) + allocID := resp.Allocations[0].ID + + cases := []struct { + Name string + Token string + ExpectedError string + }{ + { + Name: "bad token", + Token: tokenBad.SecretID, + ExpectedError: structs.ErrPermissionDenied.Error(), + }, + { + Name: "alloc-exec token", + Token: tokenAllocExec.SecretID, + ExpectedError: "", + }, + { + Name: "alloc-node-exec token", + Token: tokenAllocNodeExec.SecretID, + ExpectedError: "", + }, + { + Name: "root token", + Token: root.SecretID, + ExpectedError: "", + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: allocID, + Task: job.TaskGroups[0].Tasks[0].Name, + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{ + Region: "global", + AuthToken: c.Token, + Namespace: nstructs.DefaultNamespace, + }, + } + + // Get the handler + handler, err := client.StreamingRpcHandler("Allocations.Exec") + require.Nil(t, err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(t, encoder.Encode(req)) + + select { + case <-time.After(3 * time.Second): + case err := <-errCh: + if c.ExpectedError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), c.ExpectedError) + } + case f := <-frames: + // we are good if we don't expect an error + if c.ExpectedError != "" { + require.Fail(t, "unexpected frame", "frame: %#v", f) + } + } + }) + } +} + +// TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot asserts that token only needs +// alloc-exec acl policy when chroot isolation is used +func TestAlloc_ExecStreaming_ACL_WithIsolation_Chroot(t *testing.T) { + t.Parallel() + + if runtime.GOOS != "linux" || unix.Geteuid() != 0 { + t.Skip("chroot isolation requires linux root") + } + + isolation := drivers.FSIsolationChroot + + // Start a server and client + s, root := nomad.TestACLServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + client, cleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{s.GetConfig().RPCAddr.String()} + + pluginConfig := []*nconfig.PluginConfig{ + { + Name: "mock_driver", + Config: map[string]interface{}{ + "fs_isolation": string(isolation), + }, + }, + } + + c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) + }) + defer cleanup() + + // Create a bad token + policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) + tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) + + policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec}) + tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-exec", policyAllocExec) + + policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) + tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-node-exec", policyAllocNodeExec) + + job := mock.BatchJob() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "20s", + "exec_command": map[string]interface{}{ + "run_for": "1ms", + "stdout_string": "some output", + }, + } + + // Wait for client to be running job + testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) + + // Get the allocation ID + args := nstructs.AllocListRequest{} + args.Region = "global" + args.AuthToken = root.SecretID + args.Namespace = nstructs.DefaultNamespace + resp := nstructs.AllocListResponse{} + require.NoError(t, s.RPC("Alloc.List", &args, &resp)) + require.Len(t, resp.Allocations, 1) + allocID := resp.Allocations[0].ID + + cases := []struct { + Name string + Token string + ExpectedError string + }{ + { + Name: "bad token", + Token: tokenBad.SecretID, + ExpectedError: structs.ErrPermissionDenied.Error(), + }, + { + Name: "alloc-exec token", + Token: tokenAllocExec.SecretID, + ExpectedError: "", + }, + { + Name: "alloc-node-exec token", + Token: tokenAllocNodeExec.SecretID, + ExpectedError: "", + }, + { + Name: "root token", + Token: root.SecretID, + ExpectedError: "", + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: allocID, + Task: job.TaskGroups[0].Tasks[0].Name, + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{ + Region: "global", + AuthToken: c.Token, + Namespace: nstructs.DefaultNamespace, + }, + } + + // Get the handler + handler, err := client.StreamingRpcHandler("Allocations.Exec") + require.Nil(t, err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(t, encoder.Encode(req)) + + select { + case <-time.After(3 * time.Second): + case err := <-errCh: + if c.ExpectedError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), c.ExpectedError) + } + case f := <-frames: + // we are good if we don't expect an error + if c.ExpectedError != "" { + require.Fail(t, "unexpected frame", "frame: %#v", f) + } + } + }) + } +} + +// TestAlloc_ExecStreaming_ACL_WithIsolation_None asserts that token needs +// alloc-node-exec acl policy as well when no isolation is used +func TestAlloc_ExecStreaming_ACL_WithIsolation_None(t *testing.T) { + t.Parallel() + isolation := drivers.FSIsolationNone + + // Start a server and client + s, root := nomad.TestACLServer(t, nil) + defer s.Shutdown() + testutil.WaitForLeader(t, s.RPC) + + client, cleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{s.GetConfig().RPCAddr.String()} + + pluginConfig := []*nconfig.PluginConfig{ + { + Name: "mock_driver", + Config: map[string]interface{}{ + "fs_isolation": string(isolation), + }, + }, + } + + c.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", map[string]string{}, pluginConfig) + }) + defer cleanup() + + // Create a bad token + policyBad := mock.NamespacePolicy("other", "", []string{acl.NamespaceCapabilityDeny}) + tokenBad := mock.CreatePolicyAndToken(t, s.State(), 1005, "invalid", policyBad) + + policyAllocExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec}) + tokenAllocExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-exec", policyAllocExec) + + policyAllocNodeExec := mock.NamespacePolicy(structs.DefaultNamespace, "", + []string{acl.NamespaceCapabilityAllocExec, acl.NamespaceCapabilityAllocNodeExec}) + tokenAllocNodeExec := mock.CreatePolicyAndToken(t, s.State(), 1009, "alloc-node-exec", policyAllocNodeExec) + + job := mock.BatchJob() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "20s", + "exec_command": map[string]interface{}{ + "run_for": "1ms", + "stdout_string": "some output", + }, + } + + // Wait for client to be running job + testutil.WaitForRunningWithToken(t, s.RPC, job, root.SecretID) + + // Get the allocation ID + args := nstructs.AllocListRequest{} + args.Region = "global" + args.AuthToken = root.SecretID + args.Namespace = nstructs.DefaultNamespace + resp := nstructs.AllocListResponse{} + require.NoError(t, s.RPC("Alloc.List", &args, &resp)) + require.Len(t, resp.Allocations, 1) + allocID := resp.Allocations[0].ID + + cases := []struct { + Name string + Token string + ExpectedError string + }{ + { + Name: "bad token", + Token: tokenBad.SecretID, + ExpectedError: structs.ErrPermissionDenied.Error(), + }, + { + Name: "alloc-exec token", + Token: tokenAllocExec.SecretID, + ExpectedError: structs.ErrPermissionDenied.Error(), + }, + { + Name: "alloc-node-exec token", + Token: tokenAllocNodeExec.SecretID, + ExpectedError: "", + }, + { + Name: "root token", + Token: root.SecretID, + ExpectedError: "", + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: allocID, + Task: job.TaskGroups[0].Tasks[0].Name, + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{ + Region: "global", + AuthToken: c.Token, + Namespace: nstructs.DefaultNamespace, + }, + } + + // Get the handler + handler, err := client.StreamingRpcHandler("Allocations.Exec") + require.Nil(t, err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(t, encoder.Encode(req)) + + select { + case <-time.After(3 * time.Second): + case err := <-errCh: + if c.ExpectedError == "" { + require.NoError(t, err) + } else { + require.Contains(t, err.Error(), c.ExpectedError) + } + case f := <-frames: + // we are good if we don't expect an error + if c.ExpectedError != "" { + require.Fail(t, "unexpected frame", "frame: %#v", f) + } + } + }) + } +} + +func decodeFrames(t *testing.T, p1 net.Conn, frames chan<- *drivers.ExecTaskStreamingResponseMsg, errCh chan<- error) { + // Start the decoder + decoder := codec.NewDecoder(p1, nstructs.MsgpackHandle) + + for { + var msg cstructs.StreamErrWrapper + if err := decoder.Decode(&msg); err != nil { + if err == io.EOF || strings.Contains(err.Error(), "closed") { + return + } + t.Logf("received error decoding: %#v", err) + + errCh <- fmt.Errorf("error decoding: %v", err) + return + } + + if msg.Error != nil { + errCh <- msg.Error + continue + } + + var frame drivers.ExecTaskStreamingResponseMsg + if err := json.Unmarshal(msg.Payload, &frame); err != nil { + errCh <- err + return + } + t.Logf("received message: %#v", msg) + frames <- &frame + } +} diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index 2b5041abe..e5232703a 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -989,3 +989,21 @@ func (ar *allocRunner) Signal(taskName, signal string) error { return err.ErrorOrNil() } + +func (ar *allocRunner) GetTaskExecHandler(taskName string) drivermanager.TaskExecHandler { + tr, ok := ar.tasks[taskName] + if !ok { + return nil + } + + return tr.TaskExecHandler() +} + +func (ar *allocRunner) GetTaskDriverCapabilities(taskName string) (*drivers.Capabilities, error) { + tr, ok := ar.tasks[taskName] + if !ok { + return nil, fmt.Errorf("task not found") + } + + return tr.DriverCapabilities() +} diff --git a/client/allocrunner/taskrunner/driver_handle.go b/client/allocrunner/taskrunner/driver_handle.go index f9908ec55..4f04a6605 100644 --- a/client/allocrunner/taskrunner/driver_handle.go +++ b/client/allocrunner/taskrunner/driver_handle.go @@ -2,6 +2,7 @@ package taskrunner import ( "context" + "fmt" "time" cstructs "github.com/hashicorp/nomad/client/structs" @@ -52,6 +53,7 @@ func (h *DriverHandle) Signal(s string) error { return h.driver.SignalTask(h.taskID, s) } +// Exec is the handled used by client endpoint handler to invoke the appropriate task driver exec. func (h *DriverHandle) Exec(timeout time.Duration, cmd string, args []string) ([]byte, int, error) { command := append([]string{cmd}, args...) res, err := h.driver.ExecTask(h.taskID, command, timeout) @@ -61,6 +63,46 @@ func (h *DriverHandle) Exec(timeout time.Duration, cmd string, args []string) ([ return res.Stdout, res.ExitResult.ExitCode, res.ExitResult.Err } +// ExecStreaming is the handled used by client endpoint handler to invoke the appropriate task driver exec. +// while allowing to stream input and output +func (h *DriverHandle) ExecStreaming(ctx context.Context, + command []string, + tty bool, + stream drivers.ExecTaskStream) error { + + if impl, ok := h.driver.(drivers.ExecTaskStreamingRawDriver); ok { + return impl.ExecTaskStreamingRaw(ctx, h.taskID, command, tty, stream) + } + + d, ok := h.driver.(drivers.ExecTaskStreamingDriver) + if !ok { + return fmt.Errorf("task driver does not support exec") + } + + execOpts, doneCh := drivers.StreamToExecOptions( + ctx, command, tty, stream) + + result, err := d.ExecTaskStreaming(ctx, h.taskID, execOpts) + if err != nil { + return err + } + + execOpts.Stdout.Close() + execOpts.Stderr.Close() + + select { + case err = <-doneCh: + case <-ctx.Done(): + err = fmt.Errorf("exec task timed out: %v", ctx.Err()) + } + + if err != nil { + return err + } + + return stream.Send(drivers.NewExecStreamingResponseExit(result.ExitCode)) +} + func (h *DriverHandle) Network() *drivers.DriverNetwork { return h.net } diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 9888c1421..ef9a09ef4 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -1220,3 +1220,11 @@ func appendTaskEvent(state *structs.TaskState, event *structs.TaskEvent, capacit state.Events = append(state.Events, event) } + +func (tr *TaskRunner) TaskExecHandler() drivermanager.TaskExecHandler { + return tr.getDriverHandle().ExecStreaming +} + +func (tr *TaskRunner) DriverCapabilities() (*drivers.Capabilities, error) { + return tr.driver.Capabilities() +} diff --git a/client/client.go b/client/client.go index c8bed1e95..773c356a7 100644 --- a/client/client.go +++ b/client/client.go @@ -44,6 +44,7 @@ import ( "github.com/hashicorp/nomad/nomad/structs" nconfig "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/drivers" vaultapi "github.com/hashicorp/vault/api" "github.com/shirou/gopsutil/host" ) @@ -133,6 +134,9 @@ type AllocRunner interface { RestartTask(taskName string, taskEvent *structs.TaskEvent) error RestartAll(taskEvent *structs.TaskEvent) error + + GetTaskExecHandler(taskName string) drivermanager.TaskExecHandler + GetTaskDriverCapabilities(taskName string) (*drivers.Capabilities, error) } // Client is used to implement the client interaction with Nomad. Clients diff --git a/client/fs_endpoint.go b/client/fs_endpoint.go index ded3ae0aa..2a6a25cf5 100644 --- a/client/fs_endpoint.go +++ b/client/fs_endpoint.go @@ -84,7 +84,7 @@ func NewFileSystemEndpoint(c *Client) *FileSystem { // handleStreamResultError is a helper for sending an error with a potential // error code. The transmission of the error is ignored if the error has been // generated by the closing of the underlying transport. -func (f *FileSystem) handleStreamResultError(err error, code *int64, encoder *codec.Encoder) { +func handleStreamResultError(err error, code *int64, encoder *codec.Encoder) { // Nothing to do as the conn is closed if err == io.EOF || strings.Contains(err.Error(), "closed") { return @@ -155,26 +155,26 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { encoder := codec.NewEncoder(conn, structs.MsgpackHandle) if err := decoder.Decode(&req); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } // Check read permissions if aclObj, err := f.c.ResolveToken(req.QueryOptions.AuthToken); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, helper.Int64ToPtr(403), encoder) return } else if aclObj != nil && !aclObj.AllowNsOp(req.Namespace, acl.NamespaceCapabilityReadFS) { - f.handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) + handleStreamResultError(structs.ErrPermissionDenied, helper.Int64ToPtr(403), encoder) return } // Validate the arguments if req.AllocID == "" { - f.handleStreamResultError(allocIDNotPresentErr, helper.Int64ToPtr(400), encoder) + handleStreamResultError(allocIDNotPresentErr, helper.Int64ToPtr(400), encoder) return } if req.Path == "" { - f.handleStreamResultError(pathNotPresentErr, helper.Int64ToPtr(400), encoder) + handleStreamResultError(pathNotPresentErr, helper.Int64ToPtr(400), encoder) return } switch req.Origin { @@ -182,7 +182,7 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { case "": req.Origin = "start" default: - f.handleStreamResultError(invalidOrigin, helper.Int64ToPtr(400), encoder) + handleStreamResultError(invalidOrigin, helper.Int64ToPtr(400), encoder) return } @@ -193,18 +193,18 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } // Calculate the offset fileInfo, err := fs.Stat(req.Path) if err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } if fileInfo.IsDir { - f.handleStreamResultError( + handleStreamResultError( fmt.Errorf("file %q is a directory", req.Path), helper.Int64ToPtr(400), encoder) return @@ -312,7 +312,7 @@ OUTER: } if streamErr != nil { - f.handleStreamResultError(streamErr, helper.Int64ToPtr(500), encoder) + handleStreamResultError(streamErr, helper.Int64ToPtr(500), encoder) return } } @@ -328,36 +328,36 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { encoder := codec.NewEncoder(conn, structs.MsgpackHandle) if err := decoder.Decode(&req); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } // Check read permissions if aclObj, err := f.c.ResolveToken(req.QueryOptions.AuthToken); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } else if aclObj != nil { readfs := aclObj.AllowNsOp(req.QueryOptions.Namespace, acl.NamespaceCapabilityReadFS) logs := aclObj.AllowNsOp(req.QueryOptions.Namespace, acl.NamespaceCapabilityReadLogs) if !readfs && !logs { - f.handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) + handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) return } } // Validate the arguments if req.AllocID == "" { - f.handleStreamResultError(allocIDNotPresentErr, helper.Int64ToPtr(400), encoder) + handleStreamResultError(allocIDNotPresentErr, helper.Int64ToPtr(400), encoder) return } if req.Task == "" { - f.handleStreamResultError(taskNotPresentErr, helper.Int64ToPtr(400), encoder) + handleStreamResultError(taskNotPresentErr, helper.Int64ToPtr(400), encoder) return } switch req.LogType { case "stdout", "stderr": default: - f.handleStreamResultError(logTypeNotPresentErr, helper.Int64ToPtr(400), encoder) + handleStreamResultError(logTypeNotPresentErr, helper.Int64ToPtr(400), encoder) return } switch req.Origin { @@ -365,7 +365,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { case "": req.Origin = "start" default: - f.handleStreamResultError(invalidOrigin, helper.Int64ToPtr(400), encoder) + handleStreamResultError(invalidOrigin, helper.Int64ToPtr(400), encoder) return } @@ -376,7 +376,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } @@ -387,14 +387,14 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } // Check that the task is there taskState := allocState.TaskStates[req.Task] if taskState == nil { - f.handleStreamResultError( + handleStreamResultError( fmt.Errorf("unknown task name %q", req.Task), helper.Int64ToPtr(400), encoder) @@ -402,7 +402,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { } if taskState.StartedAt.IsZero() { - f.handleStreamResultError( + handleStreamResultError( fmt.Errorf("task %q not started yet. No logs available", req.Task), helper.Int64ToPtr(404), encoder) @@ -494,7 +494,7 @@ OUTER: if codedErr, ok := streamErr.(interface{ Code() int }); ok { code = int64(codedErr.Code()) } - f.handleStreamResultError(streamErr, &code, encoder) + handleStreamResultError(streamErr, &code, encoder) return } } diff --git a/client/pluginmanager/drivermanager/manager.go b/client/pluginmanager/drivermanager/manager.go index 2c0474eea..5f5a19bc7 100644 --- a/client/pluginmanager/drivermanager/manager.go +++ b/client/pluginmanager/drivermanager/manager.go @@ -29,6 +29,13 @@ type Manager interface { Dispense(driver string) (drivers.DriverPlugin, error) } +// TaskExecHandler is function to be called for executing commands in a task +type TaskExecHandler func( + ctx context.Context, + command []string, + tty bool, + stream drivers.ExecTaskStream) error + // EventHandler is a callback to be called for a task. // The handler should not block execution. type EventHandler func(*drivers.TaskEvent) diff --git a/client/rpc.go b/client/rpc.go index 2caa5de2c..beaec6f2c 100644 --- a/client/rpc.go +++ b/client/rpc.go @@ -217,7 +217,7 @@ func (c *Client) setupClientRpc() { // Initialize the RPC handlers c.endpoints.ClientStats = &ClientStats{c} c.endpoints.FileSystem = NewFileSystemEndpoint(c) - c.endpoints.Allocations = &Allocations{c} + c.endpoints.Allocations = NewAllocationsEndpoint(c) // Create the RPC Server c.rpcServer = rpc.NewServer() diff --git a/client/structs/structs.go b/client/structs/structs.go index 35ac59805..45439a08d 100644 --- a/client/structs/structs.go +++ b/client/structs/structs.go @@ -144,6 +144,23 @@ type StreamErrWrapper struct { Payload []byte } +// AllocExecRequest is the initial request for execing into an Alloc task +type AllocExecRequest struct { + // AllocID is the allocation to stream logs from + AllocID string + + // Task is the task to stream logs from + Task string + + // Tty indicates whether to allocate a pseudo-TTY + Tty bool + + // Cmd is the command to be executed + Cmd []string + + structs.QueryOptions +} + // AllocStatsRequest is used to request the resource usage of a given // allocation, potentially filtering by task type AllocStatsRequest struct { From a77a3ba9b02f1035cafeee8403004312ae857afa Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:25:27 -0400 Subject: [PATCH 09/15] server: server forwarding logic for nomad exec endpoint --- nomad/client_alloc_endpoint.go | 131 ++++++++++++++++++++ nomad/client_alloc_endpoint_test.go | 184 ++++++++++++++++++++++++++++ nomad/client_fs_endpoint.go | 80 ++++++------ nomad/server.go | 1 + 4 files changed, 356 insertions(+), 40 deletions(-) diff --git a/nomad/client_alloc_endpoint.go b/nomad/client_alloc_endpoint.go index 44713a9b7..3764463ff 100644 --- a/nomad/client_alloc_endpoint.go +++ b/nomad/client_alloc_endpoint.go @@ -2,11 +2,16 @@ package nomad import ( "errors" + "fmt" + "io" + "net" "time" metrics "github.com/armon/go-metrics" log "github.com/hashicorp/go-hclog" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper" + "github.com/ugorji/go/codec" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/structs" @@ -19,6 +24,10 @@ type ClientAllocations struct { logger log.Logger } +func (a *ClientAllocations) register() { + a.srv.streamingRpcs.Register("Allocations.Exec", a.exec) +} + // GarbageCollectAll is used to garbage collect all allocations on a client. func (a *ClientAllocations) GarbageCollectAll(args *structs.NodeSpecificRequest, reply *structs.GenericResponse) error { // We only allow stale reads since the only potentially stale information is @@ -287,3 +296,125 @@ func (a *ClientAllocations) Stats(args *cstructs.AllocStatsRequest, reply *cstru // Make the RPC return NodeRpc(state.Session, "Allocations.Stats", args, reply) } + +// exec is used to execute command in a running task +func (a *ClientAllocations) exec(conn io.ReadWriteCloser) { + defer conn.Close() + defer metrics.MeasureSince([]string{"nomad", "alloc", "exec"}, time.Now()) + + // Decode the arguments + var args cstructs.AllocExecRequest + decoder := codec.NewDecoder(conn, structs.MsgpackHandle) + encoder := codec.NewEncoder(conn, structs.MsgpackHandle) + + if err := decoder.Decode(&args); err != nil { + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + return + } + + // Check if we need to forward to a different region + if r := args.RequestRegion(); r != a.srv.Region() { + forwardRegionStreamingRpc(a.srv, conn, encoder, &args, "Allocations.Exec", + args.AllocID, &args.QueryOptions) + return + } + + // Check node read permissions + if aclObj, err := a.srv.ResolveToken(args.AuthToken); err != nil { + handleStreamResultError(err, nil, encoder) + return + } else if aclObj != nil { + // client ultimately checks if AllocNodeExec is required + exec := aclObj.AllowNsOp(args.QueryOptions.Namespace, acl.NamespaceCapabilityAllocExec) + if !exec { + handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) + return + } + } + + // Verify the arguments. + if args.AllocID == "" { + handleStreamResultError(errors.New("missing AllocID"), helper.Int64ToPtr(400), encoder) + return + } + + // Retrieve the allocation + snap, err := a.srv.State().Snapshot() + if err != nil { + handleStreamResultError(err, nil, encoder) + return + } + + alloc, err := snap.AllocByID(nil, args.AllocID) + if err != nil { + handleStreamResultError(err, nil, encoder) + return + } + if alloc == nil { + handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), helper.Int64ToPtr(404), encoder) + return + } + nodeID := alloc.NodeID + + // Make sure Node is valid and new enough to support RPC + node, err := snap.NodeByID(nil, nodeID) + if err != nil { + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + return + } + + if node == nil { + err := fmt.Errorf("Unknown node %q", nodeID) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + return + } + + if err := nodeSupportsRpc(node); err != nil { + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + return + } + + // Get the connection to the client either by forwarding to another server + // or creating a direct stream + var clientConn net.Conn + state, ok := a.srv.getNodeConn(nodeID) + if !ok { + // Determine the Server that has a connection to the node. + srv, err := a.srv.serverWithNodeConn(nodeID, a.srv.Region()) + if err != nil { + var code *int64 + if structs.IsErrNoNodeConn(err) { + code = helper.Int64ToPtr(404) + } + handleStreamResultError(err, code, encoder) + return + } + + // Get a connection to the server + conn, err := a.srv.streamingRpc(srv, "Allocations.Exec") + if err != nil { + handleStreamResultError(err, nil, encoder) + return + } + + clientConn = conn + } else { + stream, err := NodeStreamingRpc(state.Session, "Allocations.Exec") + if err != nil { + handleStreamResultError(err, nil, encoder) + return + } + clientConn = stream + } + defer clientConn.Close() + + // Send the request. + outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle) + if err := outEncoder.Encode(args); err != nil { + handleStreamResultError(err, nil, encoder) + return + } + + structs.Bridge(conn, clientConn) + return +} diff --git a/nomad/client_alloc_endpoint_test.go b/nomad/client_alloc_endpoint_test.go index 3e97740cf..6ed7575a7 100644 --- a/nomad/client_alloc_endpoint_test.go +++ b/nomad/client_alloc_endpoint_test.go @@ -1,8 +1,13 @@ package nomad import ( + "encoding/json" "fmt" + "io" + "net" + "strings" "testing" + "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" @@ -12,9 +17,12 @@ import ( "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + nstructs "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/testutil" "github.com/kr/pretty" "github.com/stretchr/testify/require" + "github.com/ugorji/go/codec" ) func TestClientAllocations_GarbageCollectAll_Local(t *testing.T) { @@ -1040,3 +1048,179 @@ func TestClientAllocations_Restart_ACL(t *testing.T) { }) } } + +// TestAlloc_ExecStreaming asserts that exec task requests are forwarded +// to appropriate server or remote regions +func TestAlloc_ExecStreaming(t *testing.T) { + t.Skip("try skipping") + t.Parallel() + + ////// Nomad clusters topology - not specific to test + localServer := TestServer(t, nil) + defer localServer.Shutdown() + + remoteServer := TestServer(t, func(c *Config) { + c.DevDisableBootstrap = true + }) + defer remoteServer.Shutdown() + + remoteRegionServer := TestServer(t, func(c *Config) { + c.Region = "two" + }) + defer remoteRegionServer.Shutdown() + + TestJoin(t, localServer, remoteServer) + TestJoin(t, localServer, remoteRegionServer) + testutil.WaitForLeader(t, localServer.RPC) + testutil.WaitForLeader(t, remoteServer.RPC) + testutil.WaitForLeader(t, remoteRegionServer.RPC) + + c, cleanup := client.TestClient(t, func(c *config.Config) { + c.Servers = []string{localServer.config.RPCAddr.String()} + }) + defer cleanup() + + // Wait for the client to connect + testutil.WaitForResult(func() (bool, error) { + nodes := remoteServer.connectedNodes() + return len(nodes) == 1, nil + }, func(err error) { + require.NoError(t, err, "failed to have a client") + }) + + // Force remove the connection locally in case it exists + remoteServer.nodeConnsLock.Lock() + delete(remoteServer.nodeConns, c.NodeID()) + remoteServer.nodeConnsLock.Unlock() + + ///// Start task + a := mock.BatchAlloc() + a.NodeID = c.NodeID() + a.Job.Type = structs.JobTypeBatch + a.Job.TaskGroups[0].Count = 1 + a.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "20s", + "exec_command": map[string]interface{}{ + "run_for": "1ms", + "stdout_string": "expected output", + "exit_code": 3, + }, + } + + // Upsert the allocation + localState := localServer.State() + require.Nil(t, localState.UpsertJob(999, a.Job)) + require.Nil(t, localState.UpsertAllocs(1003, []*structs.Allocation{a})) + remoteState := remoteServer.State() + require.Nil(t, remoteState.UpsertJob(999, a.Job)) + require.Nil(t, remoteState.UpsertAllocs(1003, []*structs.Allocation{a})) + + // Wait for the client to run the allocation + testutil.WaitForResult(func() (bool, error) { + alloc, err := localState.AllocByID(nil, a.ID) + if err != nil { + return false, err + } + if alloc == nil { + return false, fmt.Errorf("unknown alloc") + } + if alloc.ClientStatus != structs.AllocClientStatusRunning { + return false, fmt.Errorf("alloc client status: %v", alloc.ClientStatus) + } + + return true, nil + }, func(err error) { + require.NoError(t, err, "task didn't start yet") + }) + + ///////// Actually run query now + cases := []struct { + name string + rpc func(string) (structs.StreamingRpcHandler, error) + }{ + {"client", c.StreamingRpcHandler}, + {"local_server", localServer.StreamingRpcHandler}, + {"remote_server", remoteServer.StreamingRpcHandler}, + {"remote_region", remoteRegionServer.StreamingRpcHandler}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + + // Make the request + req := &cstructs.AllocExecRequest{ + AllocID: a.ID, + Task: a.Job.TaskGroups[0].Tasks[0].Name, + Tty: true, + Cmd: []string{"placeholder command"}, + QueryOptions: nstructs.QueryOptions{Region: "global"}, + } + + // Get the handler + handler, err := tc.rpc("Allocations.Exec") + require.Nil(t, err) + + // Create a pipe + p1, p2 := net.Pipe() + defer p1.Close() + defer p2.Close() + + errCh := make(chan error) + frames := make(chan *drivers.ExecTaskStreamingResponseMsg) + + // Start the handler + go handler(p2) + go decodeFrames(t, p1, frames, errCh) + + // Send the request + encoder := codec.NewEncoder(p1, nstructs.MsgpackHandle) + require.Nil(t, encoder.Encode(req)) + + timeout := time.After(3 * time.Second) + + OUTER: + for { + select { + case <-timeout: + require.FailNow(t, "timed out before getting exit code") + case err := <-errCh: + require.NoError(t, err) + case f := <-frames: + if f.Exited && f.Result != nil { + code := int(f.Result.ExitCode) + require.Equal(t, 3, code) + break OUTER + } + } + } + }) + } +} + +func decodeFrames(t *testing.T, p1 net.Conn, frames chan<- *drivers.ExecTaskStreamingResponseMsg, errCh chan<- error) { + // Start the decoder + decoder := codec.NewDecoder(p1, nstructs.MsgpackHandle) + + for { + var msg cstructs.StreamErrWrapper + if err := decoder.Decode(&msg); err != nil { + if err == io.EOF || strings.Contains(err.Error(), "closed") { + return + } + t.Logf("received error decoding: %#v", err) + + errCh <- fmt.Errorf("error decoding: %v", err) + return + } + + if msg.Error != nil { + errCh <- msg.Error + continue + } + + var frame drivers.ExecTaskStreamingResponseMsg + json.Unmarshal(msg.Payload, &frame) + t.Logf("received message: %#v", msg) + frames <- &frame + } +} diff --git a/nomad/client_fs_endpoint.go b/nomad/client_fs_endpoint.go index 120927a54..ef152e44a 100644 --- a/nomad/client_fs_endpoint.go +++ b/nomad/client_fs_endpoint.go @@ -33,7 +33,7 @@ func (f *FileSystem) register() { // handleStreamResultError is a helper for sending an error with a potential // error code. The transmission of the error is ignored if the error has been // generated by the closing of the underlying transport. -func (f *FileSystem) handleStreamResultError(err error, code *int64, encoder *codec.Encoder) { +func handleStreamResultError(err error, code *int64, encoder *codec.Encoder) { // Nothing to do as the conn is closed if err == io.EOF || strings.Contains(err.Error(), "closed") { return @@ -48,7 +48,7 @@ func (f *FileSystem) handleStreamResultError(err error, code *int64, encoder *co // forwardRegionStreamingRpc is used to make a streaming RPC to a different // region. It looks up the allocation in the remote region to determine what // remote server can route the request. -func (f *FileSystem) forwardRegionStreamingRpc(conn io.ReadWriteCloser, +func forwardRegionStreamingRpc(fsrv *Server, conn io.ReadWriteCloser, encoder *codec.Encoder, args interface{}, method, allocID string, qo *structs.QueryOptions) { // Request the allocation from the target region allocReq := &structs.AllocSpecificRequest{ @@ -56,31 +56,31 @@ func (f *FileSystem) forwardRegionStreamingRpc(conn io.ReadWriteCloser, QueryOptions: *qo, } var allocResp structs.SingleAllocResponse - if err := f.srv.forwardRegion(qo.RequestRegion(), "Alloc.GetAlloc", allocReq, &allocResp); err != nil { - f.handleStreamResultError(err, nil, encoder) + if err := fsrv.forwardRegion(qo.RequestRegion(), "Alloc.GetAlloc", allocReq, &allocResp); err != nil { + handleStreamResultError(err, nil, encoder) return } if allocResp.Alloc == nil { - f.handleStreamResultError(structs.NewErrUnknownAllocation(allocID), helper.Int64ToPtr(404), encoder) + handleStreamResultError(structs.NewErrUnknownAllocation(allocID), helper.Int64ToPtr(404), encoder) return } // Determine the Server that has a connection to the node. - srv, err := f.srv.serverWithNodeConn(allocResp.Alloc.NodeID, qo.RequestRegion()) + srv, err := fsrv.serverWithNodeConn(allocResp.Alloc.NodeID, qo.RequestRegion()) if err != nil { var code *int64 if structs.IsErrNoNodeConn(err) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } // Get a connection to the server - srvConn, err := f.srv.streamingRpc(srv, method) + srvConn, err := fsrv.streamingRpc(srv, method) if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } defer srvConn.Close() @@ -88,7 +88,7 @@ func (f *FileSystem) forwardRegionStreamingRpc(conn io.ReadWriteCloser, // Send the request. outEncoder := codec.NewEncoder(srvConn, structs.MsgpackHandle) if err := outEncoder.Encode(args); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } @@ -217,46 +217,46 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { encoder := codec.NewEncoder(conn, structs.MsgpackHandle) if err := decoder.Decode(&args); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } // Check if we need to forward to a different region if r := args.RequestRegion(); r != f.srv.Region() { - f.forwardRegionStreamingRpc(conn, encoder, &args, "FileSystem.Stream", + forwardRegionStreamingRpc(f.srv, conn, encoder, &args, "FileSystem.Stream", args.AllocID, &args.QueryOptions) return } // Check node read permissions if aclObj, err := f.srv.ResolveToken(args.AuthToken); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } else if aclObj != nil && !aclObj.AllowNsOp(args.Namespace, acl.NamespaceCapabilityReadFS) { - f.handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) + handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) return } // Verify the arguments. if args.AllocID == "" { - f.handleStreamResultError(errors.New("missing AllocID"), helper.Int64ToPtr(400), encoder) + handleStreamResultError(errors.New("missing AllocID"), helper.Int64ToPtr(400), encoder) return } // Retrieve the allocation snap, err := f.srv.State().Snapshot() if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } alloc, err := snap.AllocByID(nil, args.AllocID) if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } if alloc == nil { - f.handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), helper.Int64ToPtr(404), encoder) + handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), helper.Int64ToPtr(404), encoder) return } nodeID := alloc.NodeID @@ -264,18 +264,18 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { // Make sure Node is valid and new enough to support RPC node, err := snap.NodeByID(nil, nodeID) if err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } if node == nil { err := fmt.Errorf("Unknown node %q", nodeID) - f.handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } if err := nodeSupportsRpc(node); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } @@ -291,14 +291,14 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { if structs.IsErrNoNodeConn(err) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } // Get a connection to the server conn, err := f.srv.streamingRpc(srv, "FileSystem.Stream") if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } @@ -306,7 +306,7 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { } else { stream, err := NodeStreamingRpc(state.Session, "FileSystem.Stream") if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } clientConn = stream @@ -316,7 +316,7 @@ func (f *FileSystem) stream(conn io.ReadWriteCloser) { // Send the request. outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle) if err := outEncoder.Encode(args); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } @@ -335,50 +335,50 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { encoder := codec.NewEncoder(conn, structs.MsgpackHandle) if err := decoder.Decode(&args); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } // Check if we need to forward to a different region if r := args.RequestRegion(); r != f.srv.Region() { - f.forwardRegionStreamingRpc(conn, encoder, &args, "FileSystem.Logs", + forwardRegionStreamingRpc(f.srv, conn, encoder, &args, "FileSystem.Logs", args.AllocID, &args.QueryOptions) return } // Check node read permissions if aclObj, err := f.srv.ResolveToken(args.AuthToken); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } else if aclObj != nil { readfs := aclObj.AllowNsOp(args.QueryOptions.Namespace, acl.NamespaceCapabilityReadFS) logs := aclObj.AllowNsOp(args.QueryOptions.Namespace, acl.NamespaceCapabilityReadLogs) if !readfs && !logs { - f.handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) + handleStreamResultError(structs.ErrPermissionDenied, nil, encoder) return } } // Verify the arguments. if args.AllocID == "" { - f.handleStreamResultError(errors.New("missing AllocID"), helper.Int64ToPtr(400), encoder) + handleStreamResultError(errors.New("missing AllocID"), helper.Int64ToPtr(400), encoder) return } // Retrieve the allocation snap, err := f.srv.State().Snapshot() if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } alloc, err := snap.AllocByID(nil, args.AllocID) if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } if alloc == nil { - f.handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), helper.Int64ToPtr(404), encoder) + handleStreamResultError(structs.NewErrUnknownAllocation(args.AllocID), helper.Int64ToPtr(404), encoder) return } nodeID := alloc.NodeID @@ -386,18 +386,18 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { // Make sure Node is valid and new enough to support RPC node, err := snap.NodeByID(nil, nodeID) if err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(500), encoder) + handleStreamResultError(err, helper.Int64ToPtr(500), encoder) return } if node == nil { err := fmt.Errorf("Unknown node %q", nodeID) - f.handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } if err := nodeSupportsRpc(node); err != nil { - f.handleStreamResultError(err, helper.Int64ToPtr(400), encoder) + handleStreamResultError(err, helper.Int64ToPtr(400), encoder) return } @@ -413,14 +413,14 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { if structs.IsErrNoNodeConn(err) { code = helper.Int64ToPtr(404) } - f.handleStreamResultError(err, code, encoder) + handleStreamResultError(err, code, encoder) return } // Get a connection to the server conn, err := f.srv.streamingRpc(srv, "FileSystem.Logs") if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } @@ -428,7 +428,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { } else { stream, err := NodeStreamingRpc(state.Session, "FileSystem.Logs") if err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } clientConn = stream @@ -438,7 +438,7 @@ func (f *FileSystem) logs(conn io.ReadWriteCloser) { // Send the request. outEncoder := codec.NewEncoder(clientConn, structs.MsgpackHandle) if err := outEncoder.Encode(args); err != nil { - f.handleStreamResultError(err, nil, encoder) + handleStreamResultError(err, nil, encoder) return } diff --git a/nomad/server.go b/nomad/server.go index c35124c51..21dd5178b 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1027,6 +1027,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // Client endpoints s.staticEndpoints.ClientStats = &ClientStats{srv: s, logger: s.logger.Named("client_stats")} s.staticEndpoints.ClientAllocations = &ClientAllocations{srv: s, logger: s.logger.Named("client_allocs")} + s.staticEndpoints.ClientAllocations.register() // Streaming endpoints s.staticEndpoints.FileSystem = &FileSystem{srv: s, logger: s.logger.Named("client_fs")} From bfb4f0ca2dfe5524efd458e269fd1905d4eeec1d Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:33:25 -0400 Subject: [PATCH 10/15] agent: add websocket handler for nomad exec This adds a websocket endpoint for handling `nomad exec`. The endpoint is a websocket interface, as we require a bi-directional streaming (to handle both input and output), which is not very appropriate for plain HTTP 1.0. Using websocket makes implementing the web ui a bit simpler. I considered using golang http hijack capability to treat http request as a plain connection, but the web interface would be too complicated potentially. Furthermore, the API endpoint operates against the raw core nomad exec streaming datastructures, defined in protobuf, with json serializer. Our APIs use json interfaces in general, and protobuf generates json friendly golang structs. Reusing the structs here simplify interface and reduce conversion overhead. --- command/agent/alloc_endpoint.go | 193 ++++++++++++++++++++++++++++++++ command/agent/http.go | 9 ++ 2 files changed, 202 insertions(+) diff --git a/command/agent/alloc_endpoint.go b/command/agent/alloc_endpoint.go index 72e6108c2..dd06f283d 100644 --- a/command/agent/alloc_endpoint.go +++ b/command/agent/alloc_endpoint.go @@ -1,14 +1,21 @@ package agent import ( + "context" "encoding/json" "fmt" + "io" + "net" "net/http" + "strconv" "strings" "github.com/golang/snappy" + "github.com/gorilla/websocket" cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/ugorji/go/codec" ) const ( @@ -129,6 +136,8 @@ func (s *HTTPServer) ClientAllocRequest(resp http.ResponseWriter, req *http.Requ switch tokens[1] { case "stats": return s.allocStats(allocID, resp, req) + case "exec": + return s.allocExec(allocID, resp, req) case "snapshot": if s.agent.client == nil { return nil, clientNotRunning @@ -347,3 +356,187 @@ func (s *HTTPServer) allocStats(allocID string, resp http.ResponseWriter, req *h return reply.Stats, rpcErr } + +func (s *HTTPServer) allocExec(allocID string, resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Build the request and parse the ACL token + task := req.URL.Query().Get("task") + cmdJsonStr := req.URL.Query().Get("command") + var command []string + err := json.Unmarshal([]byte(cmdJsonStr), &command) + if err != nil { + // this shouldn't happen, []string is always be serializable to json + return nil, fmt.Errorf("failed to marshal command into json: %v", err) + } + + ttyB := false + if tty := req.URL.Query().Get("tty"); tty != "" { + ttyB, err = strconv.ParseBool(tty) + if err != nil { + return nil, fmt.Errorf("tty value is not a boolean: %v", err) + } + } + + args := cstructs.AllocExecRequest{ + AllocID: allocID, + Task: task, + Cmd: command, + Tty: ttyB, + } + s.parse(resp, req, &args.QueryOptions.Region, &args.QueryOptions) + + conn, err := s.wsUpgrader.Upgrade(resp, req, nil) + if err != nil { + return nil, fmt.Errorf("failed to upgrade connection: %v", err) + } + + return s.execStreamImpl(conn, &args) +} + +func (s *HTTPServer) execStreamImpl(ws *websocket.Conn, args *cstructs.AllocExecRequest) (interface{}, error) { + allocID := args.AllocID + method := "Allocations.Exec" + + // Get the correct handler + localClient, remoteClient, localServer := s.rpcHandlerForAlloc(allocID) + var handler structs.StreamingRpcHandler + var handlerErr error + if localClient { + handler, handlerErr = s.agent.Client().StreamingRpcHandler(method) + } else if remoteClient { + handler, handlerErr = s.agent.Client().RemoteStreamingRpcHandler(method) + } else if localServer { + handler, handlerErr = s.agent.Server().StreamingRpcHandler(method) + } + + if handlerErr != nil { + return nil, CodedError(500, handlerErr.Error()) + } + + // Create a pipe connecting the (possibly remote) handler to the http response + httpPipe, handlerPipe := net.Pipe() + decoder := codec.NewDecoder(httpPipe, structs.MsgpackHandle) + encoder := codec.NewEncoder(httpPipe, structs.MsgpackHandle) + + // Create a goroutine that closes the pipe if the connection closes. + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-ctx.Done() + httpPipe.Close() + + // don't close ws - wait to drain messages + }() + + // Create a channel that decodes the results + errCh := make(chan HTTPCodedError, 2) + + // stream response + go func() { + defer cancel() + + // Send the request + if err := encoder.Encode(args); err != nil { + errCh <- CodedError(500, err.Error()) + return + } + + go forwardExecInput(encoder, ws, errCh) + + for { + select { + case <-ctx.Done(): + errCh <- nil + return + default: + } + + var res cstructs.StreamErrWrapper + err := decoder.Decode(&res) + if isClosedError(err) { + ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + errCh <- nil + return + } + + if err != nil { + errCh <- CodedError(500, err.Error()) + return + } + decoder.Reset(httpPipe) + + if err := res.Error; err != nil { + code := 500 + if err.Code != nil { + code = int(*err.Code) + } + errCh <- CodedError(code, err.Error()) + return + } + + if err := ws.WriteMessage(websocket.TextMessage, res.Payload); err != nil { + errCh <- CodedError(500, err.Error()) + return + } + } + }() + + // start streaming request to streaming RPC - returns when streaming completes or errors + handler(handlerPipe) + // stop streaming background goroutines for streaming - but not websocket activity + cancel() + // retreieve any error and/or wait until goroutine stop and close errCh connection before + // closing websocket connection + codedErr := <-errCh + + if isClosedError(codedErr) { + codedErr = nil + } else if codedErr != nil { + ws.WriteMessage(websocket.CloseMessage, + websocket.FormatCloseMessage(toWsCode(codedErr.Code()), codedErr.Error())) + } + ws.Close() + + return nil, codedErr +} + +func toWsCode(httpCode int) int { + switch httpCode { + case 500: + return websocket.CloseInternalServerErr + default: + // placeholder error code + return websocket.ClosePolicyViolation + } +} + +func isClosedError(err error) bool { + if err == nil { + return false + } + + return err == io.EOF || + err == io.ErrClosedPipe || + strings.Contains(err.Error(), "closed") || + strings.Contains(err.Error(), "EOF") +} + +// forwardExecInput forwards exec input (e.g. stdin) from websocket connection +// to the streaming RPC connection to client +func forwardExecInput(encoder *codec.Encoder, ws *websocket.Conn, errCh chan<- HTTPCodedError) { + for { + sf := &drivers.ExecTaskStreamingRequestMsg{} + err := ws.ReadJSON(sf) + if err == io.EOF { + return + } + + if err != nil { + errCh <- CodedError(500, err.Error()) + return + } + + err = encoder.Encode(sf) + if err != nil { + errCh <- CodedError(500, err.Error()) + } + } +} diff --git a/command/agent/http.go b/command/agent/http.go index e1e33fa34..1bb673a2a 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -15,6 +15,7 @@ import ( "github.com/NYTimes/gziphandler" assetfs "github.com/elazarl/go-bindata-assetfs" + "github.com/gorilla/websocket" log "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/helper/tlsutil" "github.com/hashicorp/nomad/nomad/structs" @@ -54,6 +55,8 @@ type HTTPServer struct { listenerCh chan struct{} logger log.Logger Addr string + + wsUpgrader *websocket.Upgrader } // NewHTTPServer starts new HTTP server over the agent @@ -85,6 +88,11 @@ func NewHTTPServer(agent *Agent, config *Config) (*HTTPServer, error) { // Create the mux mux := http.NewServeMux() + wsUpgrader := &websocket.Upgrader{ + ReadBufferSize: 2048, + WriteBufferSize: 2048, + } + // Create the server srv := &HTTPServer{ agent: agent, @@ -93,6 +101,7 @@ func NewHTTPServer(agent *Agent, config *Config) (*HTTPServer, error) { listenerCh: make(chan struct{}), logger: agent.httpLogger, Addr: ln.Addr().String(), + wsUpgrader: wsUpgrader, } srv.registerHandlers(config.EnableDebug) From f908d6bcf559c56083953e8b2e0a42829a0fdfad Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Mon, 29 Apr 2019 11:14:18 -0400 Subject: [PATCH 11/15] vendor github.com/gorilla/websocket --- vendor/github.com/gorilla/websocket/AUTHORS | 9 + vendor/github.com/gorilla/websocket/LICENSE | 22 + vendor/github.com/gorilla/websocket/README.md | 64 + vendor/github.com/gorilla/websocket/client.go | 395 ++++++ .../gorilla/websocket/client_clone.go | 16 + .../gorilla/websocket/client_clone_legacy.go | 38 + .../gorilla/websocket/compression.go | 148 +++ vendor/github.com/gorilla/websocket/conn.go | 1166 +++++++++++++++++ .../gorilla/websocket/conn_write.go | 15 + .../gorilla/websocket/conn_write_legacy.go | 18 + vendor/github.com/gorilla/websocket/doc.go | 227 ++++ vendor/github.com/gorilla/websocket/go.mod | 1 + vendor/github.com/gorilla/websocket/go.sum | 2 + vendor/github.com/gorilla/websocket/join.go | 42 + vendor/github.com/gorilla/websocket/json.go | 60 + vendor/github.com/gorilla/websocket/mask.go | 54 + .../github.com/gorilla/websocket/mask_safe.go | 15 + .../github.com/gorilla/websocket/prepared.go | 102 ++ vendor/github.com/gorilla/websocket/proxy.go | 77 ++ vendor/github.com/gorilla/websocket/server.go | 363 +++++ vendor/github.com/gorilla/websocket/trace.go | 19 + .../github.com/gorilla/websocket/trace_17.go | 12 + vendor/github.com/gorilla/websocket/util.go | 283 ++++ .../gorilla/websocket/x_net_proxy.go | 473 +++++++ vendor/vendor.json | 1 + 25 files changed, 3622 insertions(+) create mode 100644 vendor/github.com/gorilla/websocket/AUTHORS create mode 100644 vendor/github.com/gorilla/websocket/LICENSE create mode 100644 vendor/github.com/gorilla/websocket/README.md create mode 100644 vendor/github.com/gorilla/websocket/client.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone.go create mode 100644 vendor/github.com/gorilla/websocket/client_clone_legacy.go create mode 100644 vendor/github.com/gorilla/websocket/compression.go create mode 100644 vendor/github.com/gorilla/websocket/conn.go create mode 100644 vendor/github.com/gorilla/websocket/conn_write.go create mode 100644 vendor/github.com/gorilla/websocket/conn_write_legacy.go create mode 100644 vendor/github.com/gorilla/websocket/doc.go create mode 100644 vendor/github.com/gorilla/websocket/go.mod create mode 100644 vendor/github.com/gorilla/websocket/go.sum create mode 100644 vendor/github.com/gorilla/websocket/join.go create mode 100644 vendor/github.com/gorilla/websocket/json.go create mode 100644 vendor/github.com/gorilla/websocket/mask.go create mode 100644 vendor/github.com/gorilla/websocket/mask_safe.go create mode 100644 vendor/github.com/gorilla/websocket/prepared.go create mode 100644 vendor/github.com/gorilla/websocket/proxy.go create mode 100644 vendor/github.com/gorilla/websocket/server.go create mode 100644 vendor/github.com/gorilla/websocket/trace.go create mode 100644 vendor/github.com/gorilla/websocket/trace_17.go create mode 100644 vendor/github.com/gorilla/websocket/util.go create mode 100644 vendor/github.com/gorilla/websocket/x_net_proxy.go diff --git a/vendor/github.com/gorilla/websocket/AUTHORS b/vendor/github.com/gorilla/websocket/AUTHORS new file mode 100644 index 000000000..1931f4006 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/AUTHORS @@ -0,0 +1,9 @@ +# This is the official list of Gorilla WebSocket authors for copyright +# purposes. +# +# Please keep the list sorted. + +Gary Burd +Google LLC (https://opensource.google.com/) +Joachim Bauch + diff --git a/vendor/github.com/gorilla/websocket/LICENSE b/vendor/github.com/gorilla/websocket/LICENSE new file mode 100644 index 000000000..9171c9722 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/gorilla/websocket/README.md b/vendor/github.com/gorilla/websocket/README.md new file mode 100644 index 000000000..24696947e --- /dev/null +++ b/vendor/github.com/gorilla/websocket/README.md @@ -0,0 +1,64 @@ +# Gorilla WebSocket + +Gorilla WebSocket is a [Go](http://golang.org/) implementation of the +[WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. + +[![Build Status](https://travis-ci.org/gorilla/websocket.svg?branch=master)](https://travis-ci.org/gorilla/websocket) +[![GoDoc](https://godoc.org/github.com/gorilla/websocket?status.svg)](https://godoc.org/github.com/gorilla/websocket) + +### Documentation + +* [API Reference](http://godoc.org/github.com/gorilla/websocket) +* [Chat example](https://github.com/gorilla/websocket/tree/master/examples/chat) +* [Command example](https://github.com/gorilla/websocket/tree/master/examples/command) +* [Client and server example](https://github.com/gorilla/websocket/tree/master/examples/echo) +* [File watch example](https://github.com/gorilla/websocket/tree/master/examples/filewatch) + +### Status + +The Gorilla WebSocket package provides a complete and tested implementation of +the [WebSocket](http://www.rfc-editor.org/rfc/rfc6455.txt) protocol. The +package API is stable. + +### Installation + + go get github.com/gorilla/websocket + +### Protocol Compliance + +The Gorilla WebSocket package passes the server tests in the [Autobahn Test +Suite](https://github.com/crossbario/autobahn-testsuite) using the application in the [examples/autobahn +subdirectory](https://github.com/gorilla/websocket/tree/master/examples/autobahn). + +### Gorilla WebSocket compared with other packages + + + + + + + + + + + + + + + + + + +
github.com/gorillagolang.org/x/net
RFC 6455 Features
Passes Autobahn Test SuiteYesNo
Receive fragmented messageYesNo, see note 1
Send close messageYesNo
Send pings and receive pongsYesNo
Get the type of a received data messageYesYes, see note 2
Other Features
Compression ExtensionsExperimentalNo
Read message using io.ReaderYesNo, see note 3
Write message using io.WriteCloserYesNo, see note 3
+ +Notes: + +1. Large messages are fragmented in [Chrome's new WebSocket implementation](http://www.ietf.org/mail-archive/web/hybi/current/msg10503.html). +2. The application can get the type of a received data message by implementing + a [Codec marshal](http://godoc.org/golang.org/x/net/websocket#Codec.Marshal) + function. +3. The go.net io.Reader and io.Writer operate across WebSocket frame boundaries. + Read returns when the input buffer is full or a frame boundary is + encountered. Each call to Write sends a single frame message. The Gorilla + io.Reader and io.WriteCloser operate on a single WebSocket message. + diff --git a/vendor/github.com/gorilla/websocket/client.go b/vendor/github.com/gorilla/websocket/client.go new file mode 100644 index 000000000..962c06a39 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client.go @@ -0,0 +1,395 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptrace" + "net/url" + "strings" + "time" +) + +// ErrBadHandshake is returned when the server response to opening handshake is +// invalid. +var ErrBadHandshake = errors.New("websocket: bad handshake") + +var errInvalidCompression = errors.New("websocket: invalid compression negotiation") + +// NewClient creates a new client connection using the given net connection. +// The URL u specifies the host and request URI. Use requestHeader to specify +// the origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies +// (Cookie). Use the response.Header to get the selected subprotocol +// (Sec-WebSocket-Protocol) and cookies (Set-Cookie). +// +// If the WebSocket handshake fails, ErrBadHandshake is returned along with a +// non-nil *http.Response so that callers can handle redirects, authentication, +// etc. +// +// Deprecated: Use Dialer instead. +func NewClient(netConn net.Conn, u *url.URL, requestHeader http.Header, readBufSize, writeBufSize int) (c *Conn, response *http.Response, err error) { + d := Dialer{ + ReadBufferSize: readBufSize, + WriteBufferSize: writeBufSize, + NetDial: func(net, addr string) (net.Conn, error) { + return netConn, nil + }, + } + return d.Dial(u.String(), requestHeader) +} + +// A Dialer contains options for connecting to WebSocket server. +type Dialer struct { + // NetDial specifies the dial function for creating TCP connections. If + // NetDial is nil, net.Dial is used. + NetDial func(network, addr string) (net.Conn, error) + + // NetDialContext specifies the dial function for creating TCP connections. If + // NetDialContext is nil, net.DialContext is used. + NetDialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*http.Request) (*url.URL, error) + + // TLSClientConfig specifies the TLS configuration to use with tls.Client. + // If nil, the default configuration is used. + TLSClientConfig *tls.Config + + // HandshakeTimeout specifies the duration for the handshake to complete. + HandshakeTimeout time.Duration + + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer + // size is zero, then a useful default size is used. The I/O buffer sizes + // do not limit the size of the messages that can be sent or received. + ReadBufferSize, WriteBufferSize int + + // WriteBufferPool is a pool of buffers for write operations. If the value + // is not set, then write buffers are allocated to the connection for the + // lifetime of the connection. + // + // A pool is most useful when the application has a modest volume of writes + // across a large number of connections. + // + // Applications should use a single pool for each unique value of + // WriteBufferSize. + WriteBufferPool BufferPool + + // Subprotocols specifies the client's requested subprotocols. + Subprotocols []string + + // EnableCompression specifies if the client should attempt to negotiate + // per message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool + + // Jar specifies the cookie jar. + // If Jar is nil, cookies are not sent in requests and ignored + // in responses. + Jar http.CookieJar +} + +// Dial creates a new client connection by calling DialContext with a background context. +func (d *Dialer) Dial(urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { + return d.DialContext(context.Background(), urlStr, requestHeader) +} + +var errMalformedURL = errors.New("malformed ws or wss URL") + +func hostPortNoPort(u *url.URL) (hostPort, hostNoPort string) { + hostPort = u.Host + hostNoPort = u.Host + if i := strings.LastIndex(u.Host, ":"); i > strings.LastIndex(u.Host, "]") { + hostNoPort = hostNoPort[:i] + } else { + switch u.Scheme { + case "wss": + hostPort += ":443" + case "https": + hostPort += ":443" + default: + hostPort += ":80" + } + } + return hostPort, hostNoPort +} + +// DefaultDialer is a dialer with all fields set to the default values. +var DefaultDialer = &Dialer{ + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: 45 * time.Second, +} + +// nilDialer is dialer to use when receiver is nil. +var nilDialer = *DefaultDialer + +// DialContext creates a new client connection. Use requestHeader to specify the +// origin (Origin), subprotocols (Sec-WebSocket-Protocol) and cookies (Cookie). +// Use the response.Header to get the selected subprotocol +// (Sec-WebSocket-Protocol) and cookies (Set-Cookie). +// +// The context will be used in the request and in the Dialer. +// +// If the WebSocket handshake fails, ErrBadHandshake is returned along with a +// non-nil *http.Response so that callers can handle redirects, authentication, +// etcetera. The response body may not contain the entire response and does not +// need to be closed by the application. +func (d *Dialer) DialContext(ctx context.Context, urlStr string, requestHeader http.Header) (*Conn, *http.Response, error) { + if d == nil { + d = &nilDialer + } + + challengeKey, err := generateChallengeKey() + if err != nil { + return nil, nil, err + } + + u, err := url.Parse(urlStr) + if err != nil { + return nil, nil, err + } + + switch u.Scheme { + case "ws": + u.Scheme = "http" + case "wss": + u.Scheme = "https" + default: + return nil, nil, errMalformedURL + } + + if u.User != nil { + // User name and password are not allowed in websocket URIs. + return nil, nil, errMalformedURL + } + + req := &http.Request{ + Method: "GET", + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Host: u.Host, + } + req = req.WithContext(ctx) + + // Set the cookies present in the cookie jar of the dialer + if d.Jar != nil { + for _, cookie := range d.Jar.Cookies(u) { + req.AddCookie(cookie) + } + } + + // Set the request headers using the capitalization for names and values in + // RFC examples. Although the capitalization shouldn't matter, there are + // servers that depend on it. The Header.Set method is not used because the + // method canonicalizes the header names. + req.Header["Upgrade"] = []string{"websocket"} + req.Header["Connection"] = []string{"Upgrade"} + req.Header["Sec-WebSocket-Key"] = []string{challengeKey} + req.Header["Sec-WebSocket-Version"] = []string{"13"} + if len(d.Subprotocols) > 0 { + req.Header["Sec-WebSocket-Protocol"] = []string{strings.Join(d.Subprotocols, ", ")} + } + for k, vs := range requestHeader { + switch { + case k == "Host": + if len(vs) > 0 { + req.Host = vs[0] + } + case k == "Upgrade" || + k == "Connection" || + k == "Sec-Websocket-Key" || + k == "Sec-Websocket-Version" || + k == "Sec-Websocket-Extensions" || + (k == "Sec-Websocket-Protocol" && len(d.Subprotocols) > 0): + return nil, nil, errors.New("websocket: duplicate header not allowed: " + k) + case k == "Sec-Websocket-Protocol": + req.Header["Sec-WebSocket-Protocol"] = vs + default: + req.Header[k] = vs + } + } + + if d.EnableCompression { + req.Header["Sec-WebSocket-Extensions"] = []string{"permessage-deflate; server_no_context_takeover; client_no_context_takeover"} + } + + if d.HandshakeTimeout != 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, d.HandshakeTimeout) + defer cancel() + } + + // Get network dial function. + var netDial func(network, add string) (net.Conn, error) + + if d.NetDialContext != nil { + netDial = func(network, addr string) (net.Conn, error) { + return d.NetDialContext(ctx, network, addr) + } + } else if d.NetDial != nil { + netDial = d.NetDial + } else { + netDialer := &net.Dialer{} + netDial = func(network, addr string) (net.Conn, error) { + return netDialer.DialContext(ctx, network, addr) + } + } + + // If needed, wrap the dial function to set the connection deadline. + if deadline, ok := ctx.Deadline(); ok { + forwardDial := netDial + netDial = func(network, addr string) (net.Conn, error) { + c, err := forwardDial(network, addr) + if err != nil { + return nil, err + } + err = c.SetDeadline(deadline) + if err != nil { + c.Close() + return nil, err + } + return c, nil + } + } + + // If needed, wrap the dial function to connect through a proxy. + if d.Proxy != nil { + proxyURL, err := d.Proxy(req) + if err != nil { + return nil, nil, err + } + if proxyURL != nil { + dialer, err := proxy_FromURL(proxyURL, netDialerFunc(netDial)) + if err != nil { + return nil, nil, err + } + netDial = dialer.Dial + } + } + + hostPort, hostNoPort := hostPortNoPort(u) + trace := httptrace.ContextClientTrace(ctx) + if trace != nil && trace.GetConn != nil { + trace.GetConn(hostPort) + } + + netConn, err := netDial("tcp", hostPort) + if trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{ + Conn: netConn, + }) + } + if err != nil { + return nil, nil, err + } + + defer func() { + if netConn != nil { + netConn.Close() + } + }() + + if u.Scheme == "https" { + cfg := cloneTLSConfig(d.TLSClientConfig) + if cfg.ServerName == "" { + cfg.ServerName = hostNoPort + } + tlsConn := tls.Client(netConn, cfg) + netConn = tlsConn + + var err error + if trace != nil { + err = doHandshakeWithTrace(trace, tlsConn, cfg) + } else { + err = doHandshake(tlsConn, cfg) + } + + if err != nil { + return nil, nil, err + } + } + + conn := newConn(netConn, false, d.ReadBufferSize, d.WriteBufferSize, d.WriteBufferPool, nil, nil) + + if err := req.Write(netConn); err != nil { + return nil, nil, err + } + + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := conn.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } + + resp, err := http.ReadResponse(conn.br, req) + if err != nil { + return nil, nil, err + } + + if d.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + d.Jar.SetCookies(u, rc) + } + } + + if resp.StatusCode != 101 || + !strings.EqualFold(resp.Header.Get("Upgrade"), "websocket") || + !strings.EqualFold(resp.Header.Get("Connection"), "upgrade") || + resp.Header.Get("Sec-Websocket-Accept") != computeAcceptKey(challengeKey) { + // Before closing the network connection on return from this + // function, slurp up some of the response to aid application + // debugging. + buf := make([]byte, 1024) + n, _ := io.ReadFull(resp.Body, buf) + resp.Body = ioutil.NopCloser(bytes.NewReader(buf[:n])) + return nil, resp, ErrBadHandshake + } + + for _, ext := range parseExtensions(resp.Header) { + if ext[""] != "permessage-deflate" { + continue + } + _, snct := ext["server_no_context_takeover"] + _, cnct := ext["client_no_context_takeover"] + if !snct || !cnct { + return nil, resp, errInvalidCompression + } + conn.newCompressionWriter = compressNoContextTakeover + conn.newDecompressionReader = decompressNoContextTakeover + break + } + + resp.Body = ioutil.NopCloser(bytes.NewReader([]byte{})) + conn.subprotocol = resp.Header.Get("Sec-Websocket-Protocol") + + netConn.SetDeadline(time.Time{}) + netConn = nil // to avoid close in defer. + return conn, resp, nil +} + +func doHandshake(tlsConn *tls.Conn, cfg *tls.Config) error { + if err := tlsConn.Handshake(); err != nil { + return err + } + if !cfg.InsecureSkipVerify { + if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/gorilla/websocket/client_clone.go b/vendor/github.com/gorilla/websocket/client_clone.go new file mode 100644 index 000000000..4f0d94372 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone.go @@ -0,0 +1,16 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "crypto/tls" + +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return cfg.Clone() +} diff --git a/vendor/github.com/gorilla/websocket/client_clone_legacy.go b/vendor/github.com/gorilla/websocket/client_clone_legacy.go new file mode 100644 index 000000000..babb007fb --- /dev/null +++ b/vendor/github.com/gorilla/websocket/client_clone_legacy.go @@ -0,0 +1,38 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +import "crypto/tls" + +// cloneTLSConfig clones all public fields except the fields +// SessionTicketsDisabled and SessionTicketKey. This avoids copying the +// sync.Mutex in the sync.Once and makes it safe to call cloneTLSConfig on a +// config in active use. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + return &tls.Config{ + Rand: cfg.Rand, + Time: cfg.Time, + Certificates: cfg.Certificates, + NameToCertificate: cfg.NameToCertificate, + GetCertificate: cfg.GetCertificate, + RootCAs: cfg.RootCAs, + NextProtos: cfg.NextProtos, + ServerName: cfg.ServerName, + ClientAuth: cfg.ClientAuth, + ClientCAs: cfg.ClientCAs, + InsecureSkipVerify: cfg.InsecureSkipVerify, + CipherSuites: cfg.CipherSuites, + PreferServerCipherSuites: cfg.PreferServerCipherSuites, + ClientSessionCache: cfg.ClientSessionCache, + MinVersion: cfg.MinVersion, + MaxVersion: cfg.MaxVersion, + CurvePreferences: cfg.CurvePreferences, + } +} diff --git a/vendor/github.com/gorilla/websocket/compression.go b/vendor/github.com/gorilla/websocket/compression.go new file mode 100644 index 000000000..813ffb1e8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/compression.go @@ -0,0 +1,148 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "compress/flate" + "errors" + "io" + "strings" + "sync" +) + +const ( + minCompressionLevel = -2 // flate.HuffmanOnly not defined in Go < 1.6 + maxCompressionLevel = flate.BestCompression + defaultCompressionLevel = 1 +) + +var ( + flateWriterPools [maxCompressionLevel - minCompressionLevel + 1]sync.Pool + flateReaderPool = sync.Pool{New: func() interface{} { + return flate.NewReader(nil) + }} +) + +func decompressNoContextTakeover(r io.Reader) io.ReadCloser { + const tail = + // Add four bytes as specified in RFC + "\x00\x00\xff\xff" + + // Add final block to squelch unexpected EOF error from flate reader. + "\x01\x00\x00\xff\xff" + + fr, _ := flateReaderPool.Get().(io.ReadCloser) + fr.(flate.Resetter).Reset(io.MultiReader(r, strings.NewReader(tail)), nil) + return &flateReadWrapper{fr} +} + +func isValidCompressionLevel(level int) bool { + return minCompressionLevel <= level && level <= maxCompressionLevel +} + +func compressNoContextTakeover(w io.WriteCloser, level int) io.WriteCloser { + p := &flateWriterPools[level-minCompressionLevel] + tw := &truncWriter{w: w} + fw, _ := p.Get().(*flate.Writer) + if fw == nil { + fw, _ = flate.NewWriter(tw, level) + } else { + fw.Reset(tw) + } + return &flateWriteWrapper{fw: fw, tw: tw, p: p} +} + +// truncWriter is an io.Writer that writes all but the last four bytes of the +// stream to another io.Writer. +type truncWriter struct { + w io.WriteCloser + n int + p [4]byte +} + +func (w *truncWriter) Write(p []byte) (int, error) { + n := 0 + + // fill buffer first for simplicity. + if w.n < len(w.p) { + n = copy(w.p[w.n:], p) + p = p[n:] + w.n += n + if len(p) == 0 { + return n, nil + } + } + + m := len(p) + if m > len(w.p) { + m = len(w.p) + } + + if nn, err := w.w.Write(w.p[:m]); err != nil { + return n + nn, err + } + + copy(w.p[:], w.p[m:]) + copy(w.p[len(w.p)-m:], p[len(p)-m:]) + nn, err := w.w.Write(p[:len(p)-m]) + return n + nn, err +} + +type flateWriteWrapper struct { + fw *flate.Writer + tw *truncWriter + p *sync.Pool +} + +func (w *flateWriteWrapper) Write(p []byte) (int, error) { + if w.fw == nil { + return 0, errWriteClosed + } + return w.fw.Write(p) +} + +func (w *flateWriteWrapper) Close() error { + if w.fw == nil { + return errWriteClosed + } + err1 := w.fw.Flush() + w.p.Put(w.fw) + w.fw = nil + if w.tw.p != [4]byte{0, 0, 0xff, 0xff} { + return errors.New("websocket: internal error, unexpected bytes at end of flate stream") + } + err2 := w.tw.w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +type flateReadWrapper struct { + fr io.ReadCloser +} + +func (r *flateReadWrapper) Read(p []byte) (int, error) { + if r.fr == nil { + return 0, io.ErrClosedPipe + } + n, err := r.fr.Read(p) + if err == io.EOF { + // Preemptively place the reader back in the pool. This helps with + // scenarios where the application does not call NextReader() soon after + // this final read. + r.Close() + } + return n, err +} + +func (r *flateReadWrapper) Close() error { + if r.fr == nil { + return io.ErrClosedPipe + } + err := r.fr.Close() + flateReaderPool.Put(r.fr) + r.fr = nil + return err +} diff --git a/vendor/github.com/gorilla/websocket/conn.go b/vendor/github.com/gorilla/websocket/conn.go new file mode 100644 index 000000000..3848ab4a9 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn.go @@ -0,0 +1,1166 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "encoding/binary" + "errors" + "io" + "io/ioutil" + "math/rand" + "net" + "strconv" + "sync" + "time" + "unicode/utf8" +) + +const ( + // Frame header byte 0 bits from Section 5.2 of RFC 6455 + finalBit = 1 << 7 + rsv1Bit = 1 << 6 + rsv2Bit = 1 << 5 + rsv3Bit = 1 << 4 + + // Frame header byte 1 bits from Section 5.2 of RFC 6455 + maskBit = 1 << 7 + + maxFrameHeaderSize = 2 + 8 + 4 // Fixed header + length + mask + maxControlFramePayloadSize = 125 + + writeWait = time.Second + + defaultReadBufferSize = 4096 + defaultWriteBufferSize = 4096 + + continuationFrame = 0 + noFrame = -1 +) + +// Close codes defined in RFC 6455, section 11.7. +const ( + CloseNormalClosure = 1000 + CloseGoingAway = 1001 + CloseProtocolError = 1002 + CloseUnsupportedData = 1003 + CloseNoStatusReceived = 1005 + CloseAbnormalClosure = 1006 + CloseInvalidFramePayloadData = 1007 + ClosePolicyViolation = 1008 + CloseMessageTooBig = 1009 + CloseMandatoryExtension = 1010 + CloseInternalServerErr = 1011 + CloseServiceRestart = 1012 + CloseTryAgainLater = 1013 + CloseTLSHandshake = 1015 +) + +// The message types are defined in RFC 6455, section 11.8. +const ( + // TextMessage denotes a text data message. The text message payload is + // interpreted as UTF-8 encoded text data. + TextMessage = 1 + + // BinaryMessage denotes a binary data message. + BinaryMessage = 2 + + // CloseMessage denotes a close control message. The optional message + // payload contains a numeric code and text. Use the FormatCloseMessage + // function to format a close message payload. + CloseMessage = 8 + + // PingMessage denotes a ping control message. The optional message payload + // is UTF-8 encoded text. + PingMessage = 9 + + // PongMessage denotes a pong control message. The optional message payload + // is UTF-8 encoded text. + PongMessage = 10 +) + +// ErrCloseSent is returned when the application writes a message to the +// connection after sending a close message. +var ErrCloseSent = errors.New("websocket: close sent") + +// ErrReadLimit is returned when reading a message that is larger than the +// read limit set for the connection. +var ErrReadLimit = errors.New("websocket: read limit exceeded") + +// netError satisfies the net Error interface. +type netError struct { + msg string + temporary bool + timeout bool +} + +func (e *netError) Error() string { return e.msg } +func (e *netError) Temporary() bool { return e.temporary } +func (e *netError) Timeout() bool { return e.timeout } + +// CloseError represents a close message. +type CloseError struct { + // Code is defined in RFC 6455, section 11.7. + Code int + + // Text is the optional text payload. + Text string +} + +func (e *CloseError) Error() string { + s := []byte("websocket: close ") + s = strconv.AppendInt(s, int64(e.Code), 10) + switch e.Code { + case CloseNormalClosure: + s = append(s, " (normal)"...) + case CloseGoingAway: + s = append(s, " (going away)"...) + case CloseProtocolError: + s = append(s, " (protocol error)"...) + case CloseUnsupportedData: + s = append(s, " (unsupported data)"...) + case CloseNoStatusReceived: + s = append(s, " (no status)"...) + case CloseAbnormalClosure: + s = append(s, " (abnormal closure)"...) + case CloseInvalidFramePayloadData: + s = append(s, " (invalid payload data)"...) + case ClosePolicyViolation: + s = append(s, " (policy violation)"...) + case CloseMessageTooBig: + s = append(s, " (message too big)"...) + case CloseMandatoryExtension: + s = append(s, " (mandatory extension missing)"...) + case CloseInternalServerErr: + s = append(s, " (internal server error)"...) + case CloseTLSHandshake: + s = append(s, " (TLS handshake error)"...) + } + if e.Text != "" { + s = append(s, ": "...) + s = append(s, e.Text...) + } + return string(s) +} + +// IsCloseError returns boolean indicating whether the error is a *CloseError +// with one of the specified codes. +func IsCloseError(err error, codes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range codes { + if e.Code == code { + return true + } + } + } + return false +} + +// IsUnexpectedCloseError returns boolean indicating whether the error is a +// *CloseError with a code not in the list of expected codes. +func IsUnexpectedCloseError(err error, expectedCodes ...int) bool { + if e, ok := err.(*CloseError); ok { + for _, code := range expectedCodes { + if e.Code == code { + return false + } + } + return true + } + return false +} + +var ( + errWriteTimeout = &netError{msg: "websocket: write timeout", timeout: true, temporary: true} + errUnexpectedEOF = &CloseError{Code: CloseAbnormalClosure, Text: io.ErrUnexpectedEOF.Error()} + errBadWriteOpCode = errors.New("websocket: bad write message type") + errWriteClosed = errors.New("websocket: write closed") + errInvalidControlFrame = errors.New("websocket: invalid control frame") +) + +func newMaskKey() [4]byte { + n := rand.Uint32() + return [4]byte{byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24)} +} + +func hideTempErr(err error) error { + if e, ok := err.(net.Error); ok && e.Temporary() { + err = &netError{msg: e.Error(), timeout: e.Timeout()} + } + return err +} + +func isControl(frameType int) bool { + return frameType == CloseMessage || frameType == PingMessage || frameType == PongMessage +} + +func isData(frameType int) bool { + return frameType == TextMessage || frameType == BinaryMessage +} + +var validReceivedCloseCodes = map[int]bool{ + // see http://www.iana.org/assignments/websocket/websocket.xhtml#close-code-number + + CloseNormalClosure: true, + CloseGoingAway: true, + CloseProtocolError: true, + CloseUnsupportedData: true, + CloseNoStatusReceived: false, + CloseAbnormalClosure: false, + CloseInvalidFramePayloadData: true, + ClosePolicyViolation: true, + CloseMessageTooBig: true, + CloseMandatoryExtension: true, + CloseInternalServerErr: true, + CloseServiceRestart: true, + CloseTryAgainLater: true, + CloseTLSHandshake: false, +} + +func isValidReceivedCloseCode(code int) bool { + return validReceivedCloseCodes[code] || (code >= 3000 && code <= 4999) +} + +// BufferPool represents a pool of buffers. The *sync.Pool type satisfies this +// interface. The type of the value stored in a pool is not specified. +type BufferPool interface { + // Get gets a value from the pool or returns nil if the pool is empty. + Get() interface{} + // Put adds a value to the pool. + Put(interface{}) +} + +// writePoolData is the type added to the write buffer pool. This wrapper is +// used to prevent applications from peeking at and depending on the values +// added to the pool. +type writePoolData struct{ buf []byte } + +// The Conn type represents a WebSocket connection. +type Conn struct { + conn net.Conn + isServer bool + subprotocol string + + // Write fields + mu chan bool // used as mutex to protect write to conn + writeBuf []byte // frame is constructed in this buffer. + writePool BufferPool + writeBufSize int + writeDeadline time.Time + writer io.WriteCloser // the current writer returned to the application + isWriting bool // for best-effort concurrent write detection + + writeErrMu sync.Mutex + writeErr error + + enableWriteCompression bool + compressionLevel int + newCompressionWriter func(io.WriteCloser, int) io.WriteCloser + + // Read fields + reader io.ReadCloser // the current reader returned to the application + readErr error + br *bufio.Reader + readRemaining int64 // bytes remaining in current frame. + readFinal bool // true the current message has more frames. + readLength int64 // Message size. + readLimit int64 // Maximum message size. + readMaskPos int + readMaskKey [4]byte + handlePong func(string) error + handlePing func(string) error + handleClose func(int, string) error + readErrCount int + messageReader *messageReader // the current low-level reader + + readDecompress bool // whether last read frame had RSV1 set + newDecompressionReader func(io.Reader) io.ReadCloser +} + +func newConn(conn net.Conn, isServer bool, readBufferSize, writeBufferSize int, writeBufferPool BufferPool, br *bufio.Reader, writeBuf []byte) *Conn { + + if br == nil { + if readBufferSize == 0 { + readBufferSize = defaultReadBufferSize + } else if readBufferSize < maxControlFramePayloadSize { + // must be large enough for control frame + readBufferSize = maxControlFramePayloadSize + } + br = bufio.NewReaderSize(conn, readBufferSize) + } + + if writeBufferSize <= 0 { + writeBufferSize = defaultWriteBufferSize + } + writeBufferSize += maxFrameHeaderSize + + if writeBuf == nil && writeBufferPool == nil { + writeBuf = make([]byte, writeBufferSize) + } + + mu := make(chan bool, 1) + mu <- true + c := &Conn{ + isServer: isServer, + br: br, + conn: conn, + mu: mu, + readFinal: true, + writeBuf: writeBuf, + writePool: writeBufferPool, + writeBufSize: writeBufferSize, + enableWriteCompression: true, + compressionLevel: defaultCompressionLevel, + } + c.SetCloseHandler(nil) + c.SetPingHandler(nil) + c.SetPongHandler(nil) + return c +} + +// Subprotocol returns the negotiated protocol for the connection. +func (c *Conn) Subprotocol() string { + return c.subprotocol +} + +// Close closes the underlying network connection without sending or waiting +// for a close message. +func (c *Conn) Close() error { + return c.conn.Close() +} + +// LocalAddr returns the local network address. +func (c *Conn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +// RemoteAddr returns the remote network address. +func (c *Conn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +// Write methods + +func (c *Conn) writeFatal(err error) error { + err = hideTempErr(err) + c.writeErrMu.Lock() + if c.writeErr == nil { + c.writeErr = err + } + c.writeErrMu.Unlock() + return err +} + +func (c *Conn) read(n int) ([]byte, error) { + p, err := c.br.Peek(n) + if err == io.EOF { + err = errUnexpectedEOF + } + c.br.Discard(len(p)) + return p, err +} + +func (c *Conn) write(frameType int, deadline time.Time, buf0, buf1 []byte) error { + <-c.mu + defer func() { c.mu <- true }() + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + c.conn.SetWriteDeadline(deadline) + if len(buf1) == 0 { + _, err = c.conn.Write(buf0) + } else { + err = c.writeBufs(buf0, buf1) + } + if err != nil { + return c.writeFatal(err) + } + if frameType == CloseMessage { + c.writeFatal(ErrCloseSent) + } + return nil +} + +// WriteControl writes a control message with the given deadline. The allowed +// message types are CloseMessage, PingMessage and PongMessage. +func (c *Conn) WriteControl(messageType int, data []byte, deadline time.Time) error { + if !isControl(messageType) { + return errBadWriteOpCode + } + if len(data) > maxControlFramePayloadSize { + return errInvalidControlFrame + } + + b0 := byte(messageType) | finalBit + b1 := byte(len(data)) + if !c.isServer { + b1 |= maskBit + } + + buf := make([]byte, 0, maxFrameHeaderSize+maxControlFramePayloadSize) + buf = append(buf, b0, b1) + + if c.isServer { + buf = append(buf, data...) + } else { + key := newMaskKey() + buf = append(buf, key[:]...) + buf = append(buf, data...) + maskBytes(key, 0, buf[6:]) + } + + d := time.Hour * 1000 + if !deadline.IsZero() { + d = deadline.Sub(time.Now()) + if d < 0 { + return errWriteTimeout + } + } + + timer := time.NewTimer(d) + select { + case <-c.mu: + timer.Stop() + case <-timer.C: + return errWriteTimeout + } + defer func() { c.mu <- true }() + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + c.conn.SetWriteDeadline(deadline) + _, err = c.conn.Write(buf) + if err != nil { + return c.writeFatal(err) + } + if messageType == CloseMessage { + c.writeFatal(ErrCloseSent) + } + return err +} + +// beginMessage prepares a connection and message writer for a new message. +func (c *Conn) beginMessage(mw *messageWriter, messageType int) error { + // Close previous writer if not already closed by the application. It's + // probably better to return an error in this situation, but we cannot + // change this without breaking existing applications. + if c.writer != nil { + c.writer.Close() + c.writer = nil + } + + if !isControl(messageType) && !isData(messageType) { + return errBadWriteOpCode + } + + c.writeErrMu.Lock() + err := c.writeErr + c.writeErrMu.Unlock() + if err != nil { + return err + } + + mw.c = c + mw.frameType = messageType + mw.pos = maxFrameHeaderSize + + if c.writeBuf == nil { + wpd, ok := c.writePool.Get().(writePoolData) + if ok { + c.writeBuf = wpd.buf + } else { + c.writeBuf = make([]byte, c.writeBufSize) + } + } + return nil +} + +// NextWriter returns a writer for the next message to send. The writer's Close +// method flushes the complete message to the network. +// +// There can be at most one open writer on a connection. NextWriter closes the +// previous writer if the application has not already done so. +// +// All message types (TextMessage, BinaryMessage, CloseMessage, PingMessage and +// PongMessage) are supported. +func (c *Conn) NextWriter(messageType int) (io.WriteCloser, error) { + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { + return nil, err + } + c.writer = &mw + if c.newCompressionWriter != nil && c.enableWriteCompression && isData(messageType) { + w := c.newCompressionWriter(c.writer, c.compressionLevel) + mw.compress = true + c.writer = w + } + return c.writer, nil +} + +type messageWriter struct { + c *Conn + compress bool // whether next call to flushFrame should set RSV1 + pos int // end of data in writeBuf. + frameType int // type of the current frame. + err error +} + +func (w *messageWriter) endMessage(err error) error { + if w.err != nil { + return err + } + c := w.c + w.err = err + c.writer = nil + if c.writePool != nil { + c.writePool.Put(writePoolData{buf: c.writeBuf}) + c.writeBuf = nil + } + return err +} + +// flushFrame writes buffered data and extra as a frame to the network. The +// final argument indicates that this is the last frame in the message. +func (w *messageWriter) flushFrame(final bool, extra []byte) error { + c := w.c + length := w.pos - maxFrameHeaderSize + len(extra) + + // Check for invalid control frames. + if isControl(w.frameType) && + (!final || length > maxControlFramePayloadSize) { + return w.endMessage(errInvalidControlFrame) + } + + b0 := byte(w.frameType) + if final { + b0 |= finalBit + } + if w.compress { + b0 |= rsv1Bit + } + w.compress = false + + b1 := byte(0) + if !c.isServer { + b1 |= maskBit + } + + // Assume that the frame starts at beginning of c.writeBuf. + framePos := 0 + if c.isServer { + // Adjust up if mask not included in the header. + framePos = 4 + } + + switch { + case length >= 65536: + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | 127 + binary.BigEndian.PutUint64(c.writeBuf[framePos+2:], uint64(length)) + case length > 125: + framePos += 6 + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | 126 + binary.BigEndian.PutUint16(c.writeBuf[framePos+2:], uint16(length)) + default: + framePos += 8 + c.writeBuf[framePos] = b0 + c.writeBuf[framePos+1] = b1 | byte(length) + } + + if !c.isServer { + key := newMaskKey() + copy(c.writeBuf[maxFrameHeaderSize-4:], key[:]) + maskBytes(key, 0, c.writeBuf[maxFrameHeaderSize:w.pos]) + if len(extra) > 0 { + return w.endMessage(c.writeFatal(errors.New("websocket: internal error, extra used in client mode"))) + } + } + + // Write the buffers to the connection with best-effort detection of + // concurrent writes. See the concurrency section in the package + // documentation for more info. + + if c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = true + + err := c.write(w.frameType, c.writeDeadline, c.writeBuf[framePos:w.pos], extra) + + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + + if err != nil { + return w.endMessage(err) + } + + if final { + w.endMessage(errWriteClosed) + return nil + } + + // Setup for next frame. + w.pos = maxFrameHeaderSize + w.frameType = continuationFrame + return nil +} + +func (w *messageWriter) ncopy(max int) (int, error) { + n := len(w.c.writeBuf) - w.pos + if n <= 0 { + if err := w.flushFrame(false, nil); err != nil { + return 0, err + } + n = len(w.c.writeBuf) - w.pos + } + if n > max { + n = max + } + return n, nil +} + +func (w *messageWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + + if len(p) > 2*len(w.c.writeBuf) && w.c.isServer { + // Don't buffer large messages. + err := w.flushFrame(false, p) + if err != nil { + return 0, err + } + return len(p), nil + } + + nn := len(p) + for len(p) > 0 { + n, err := w.ncopy(len(p)) + if err != nil { + return 0, err + } + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n + p = p[n:] + } + return nn, nil +} + +func (w *messageWriter) WriteString(p string) (int, error) { + if w.err != nil { + return 0, w.err + } + + nn := len(p) + for len(p) > 0 { + n, err := w.ncopy(len(p)) + if err != nil { + return 0, err + } + copy(w.c.writeBuf[w.pos:], p[:n]) + w.pos += n + p = p[n:] + } + return nn, nil +} + +func (w *messageWriter) ReadFrom(r io.Reader) (nn int64, err error) { + if w.err != nil { + return 0, w.err + } + for { + if w.pos == len(w.c.writeBuf) { + err = w.flushFrame(false, nil) + if err != nil { + break + } + } + var n int + n, err = r.Read(w.c.writeBuf[w.pos:]) + w.pos += n + nn += int64(n) + if err != nil { + if err == io.EOF { + err = nil + } + break + } + } + return nn, err +} + +func (w *messageWriter) Close() error { + if w.err != nil { + return w.err + } + if err := w.flushFrame(true, nil); err != nil { + return err + } + return nil +} + +// WritePreparedMessage writes prepared message into connection. +func (c *Conn) WritePreparedMessage(pm *PreparedMessage) error { + frameType, frameData, err := pm.frame(prepareKey{ + isServer: c.isServer, + compress: c.newCompressionWriter != nil && c.enableWriteCompression && isData(pm.messageType), + compressionLevel: c.compressionLevel, + }) + if err != nil { + return err + } + if c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = true + err = c.write(frameType, c.writeDeadline, frameData, nil) + if !c.isWriting { + panic("concurrent write to websocket connection") + } + c.isWriting = false + return err +} + +// WriteMessage is a helper method for getting a writer using NextWriter, +// writing the message and closing the writer. +func (c *Conn) WriteMessage(messageType int, data []byte) error { + + if c.isServer && (c.newCompressionWriter == nil || !c.enableWriteCompression) { + // Fast path with no allocations and single frame. + + var mw messageWriter + if err := c.beginMessage(&mw, messageType); err != nil { + return err + } + n := copy(c.writeBuf[mw.pos:], data) + mw.pos += n + data = data[n:] + return mw.flushFrame(true, data) + } + + w, err := c.NextWriter(messageType) + if err != nil { + return err + } + if _, err = w.Write(data); err != nil { + return err + } + return w.Close() +} + +// SetWriteDeadline sets the write deadline on the underlying network +// connection. After a write has timed out, the websocket state is corrupt and +// all future writes will return an error. A zero value for t means writes will +// not time out. +func (c *Conn) SetWriteDeadline(t time.Time) error { + c.writeDeadline = t + return nil +} + +// Read methods + +func (c *Conn) advanceFrame() (int, error) { + // 1. Skip remainder of previous frame. + + if c.readRemaining > 0 { + if _, err := io.CopyN(ioutil.Discard, c.br, c.readRemaining); err != nil { + return noFrame, err + } + } + + // 2. Read and parse first two bytes of frame header. + + p, err := c.read(2) + if err != nil { + return noFrame, err + } + + final := p[0]&finalBit != 0 + frameType := int(p[0] & 0xf) + mask := p[1]&maskBit != 0 + c.readRemaining = int64(p[1] & 0x7f) + + c.readDecompress = false + if c.newDecompressionReader != nil && (p[0]&rsv1Bit) != 0 { + c.readDecompress = true + p[0] &^= rsv1Bit + } + + if rsv := p[0] & (rsv1Bit | rsv2Bit | rsv3Bit); rsv != 0 { + return noFrame, c.handleProtocolError("unexpected reserved bits 0x" + strconv.FormatInt(int64(rsv), 16)) + } + + switch frameType { + case CloseMessage, PingMessage, PongMessage: + if c.readRemaining > maxControlFramePayloadSize { + return noFrame, c.handleProtocolError("control frame length > 125") + } + if !final { + return noFrame, c.handleProtocolError("control frame not final") + } + case TextMessage, BinaryMessage: + if !c.readFinal { + return noFrame, c.handleProtocolError("message start before final message frame") + } + c.readFinal = final + case continuationFrame: + if c.readFinal { + return noFrame, c.handleProtocolError("continuation after final message frame") + } + c.readFinal = final + default: + return noFrame, c.handleProtocolError("unknown opcode " + strconv.Itoa(frameType)) + } + + // 3. Read and parse frame length. + + switch c.readRemaining { + case 126: + p, err := c.read(2) + if err != nil { + return noFrame, err + } + c.readRemaining = int64(binary.BigEndian.Uint16(p)) + case 127: + p, err := c.read(8) + if err != nil { + return noFrame, err + } + c.readRemaining = int64(binary.BigEndian.Uint64(p)) + } + + // 4. Handle frame masking. + + if mask != c.isServer { + return noFrame, c.handleProtocolError("incorrect mask flag") + } + + if mask { + c.readMaskPos = 0 + p, err := c.read(len(c.readMaskKey)) + if err != nil { + return noFrame, err + } + copy(c.readMaskKey[:], p) + } + + // 5. For text and binary messages, enforce read limit and return. + + if frameType == continuationFrame || frameType == TextMessage || frameType == BinaryMessage { + + c.readLength += c.readRemaining + if c.readLimit > 0 && c.readLength > c.readLimit { + c.WriteControl(CloseMessage, FormatCloseMessage(CloseMessageTooBig, ""), time.Now().Add(writeWait)) + return noFrame, ErrReadLimit + } + + return frameType, nil + } + + // 6. Read control frame payload. + + var payload []byte + if c.readRemaining > 0 { + payload, err = c.read(int(c.readRemaining)) + c.readRemaining = 0 + if err != nil { + return noFrame, err + } + if c.isServer { + maskBytes(c.readMaskKey, 0, payload) + } + } + + // 7. Process control frame payload. + + switch frameType { + case PongMessage: + if err := c.handlePong(string(payload)); err != nil { + return noFrame, err + } + case PingMessage: + if err := c.handlePing(string(payload)); err != nil { + return noFrame, err + } + case CloseMessage: + closeCode := CloseNoStatusReceived + closeText := "" + if len(payload) >= 2 { + closeCode = int(binary.BigEndian.Uint16(payload)) + if !isValidReceivedCloseCode(closeCode) { + return noFrame, c.handleProtocolError("invalid close code") + } + closeText = string(payload[2:]) + if !utf8.ValidString(closeText) { + return noFrame, c.handleProtocolError("invalid utf8 payload in close frame") + } + } + if err := c.handleClose(closeCode, closeText); err != nil { + return noFrame, err + } + return noFrame, &CloseError{Code: closeCode, Text: closeText} + } + + return frameType, nil +} + +func (c *Conn) handleProtocolError(message string) error { + c.WriteControl(CloseMessage, FormatCloseMessage(CloseProtocolError, message), time.Now().Add(writeWait)) + return errors.New("websocket: " + message) +} + +// NextReader returns the next data message received from the peer. The +// returned messageType is either TextMessage or BinaryMessage. +// +// There can be at most one open reader on a connection. NextReader discards +// the previous message if the application has not already consumed it. +// +// Applications must break out of the application's read loop when this method +// returns a non-nil error value. Errors returned from this method are +// permanent. Once this method returns a non-nil error, all subsequent calls to +// this method return the same error. +func (c *Conn) NextReader() (messageType int, r io.Reader, err error) { + // Close previous reader, only relevant for decompression. + if c.reader != nil { + c.reader.Close() + c.reader = nil + } + + c.messageReader = nil + c.readLength = 0 + + for c.readErr == nil { + frameType, err := c.advanceFrame() + if err != nil { + c.readErr = hideTempErr(err) + break + } + if frameType == TextMessage || frameType == BinaryMessage { + c.messageReader = &messageReader{c} + c.reader = c.messageReader + if c.readDecompress { + c.reader = c.newDecompressionReader(c.reader) + } + return frameType, c.reader, nil + } + } + + // Applications that do handle the error returned from this method spin in + // tight loop on connection failure. To help application developers detect + // this error, panic on repeated reads to the failed connection. + c.readErrCount++ + if c.readErrCount >= 1000 { + panic("repeated read on failed websocket connection") + } + + return noFrame, nil, c.readErr +} + +type messageReader struct{ c *Conn } + +func (r *messageReader) Read(b []byte) (int, error) { + c := r.c + if c.messageReader != r { + return 0, io.EOF + } + + for c.readErr == nil { + + if c.readRemaining > 0 { + if int64(len(b)) > c.readRemaining { + b = b[:c.readRemaining] + } + n, err := c.br.Read(b) + c.readErr = hideTempErr(err) + if c.isServer { + c.readMaskPos = maskBytes(c.readMaskKey, c.readMaskPos, b[:n]) + } + c.readRemaining -= int64(n) + if c.readRemaining > 0 && c.readErr == io.EOF { + c.readErr = errUnexpectedEOF + } + return n, c.readErr + } + + if c.readFinal { + c.messageReader = nil + return 0, io.EOF + } + + frameType, err := c.advanceFrame() + switch { + case err != nil: + c.readErr = hideTempErr(err) + case frameType == TextMessage || frameType == BinaryMessage: + c.readErr = errors.New("websocket: internal error, unexpected text or binary in Reader") + } + } + + err := c.readErr + if err == io.EOF && c.messageReader == r { + err = errUnexpectedEOF + } + return 0, err +} + +func (r *messageReader) Close() error { + return nil +} + +// ReadMessage is a helper method for getting a reader using NextReader and +// reading from that reader to a buffer. +func (c *Conn) ReadMessage() (messageType int, p []byte, err error) { + var r io.Reader + messageType, r, err = c.NextReader() + if err != nil { + return messageType, nil, err + } + p, err = ioutil.ReadAll(r) + return messageType, p, err +} + +// SetReadDeadline sets the read deadline on the underlying network connection. +// After a read has timed out, the websocket connection state is corrupt and +// all future reads will return an error. A zero value for t means reads will +// not time out. +func (c *Conn) SetReadDeadline(t time.Time) error { + return c.conn.SetReadDeadline(t) +} + +// SetReadLimit sets the maximum size in bytes for a message read from the peer. If a +// message exceeds the limit, the connection sends a close message to the peer +// and returns ErrReadLimit to the application. +func (c *Conn) SetReadLimit(limit int64) { + c.readLimit = limit +} + +// CloseHandler returns the current close handler +func (c *Conn) CloseHandler() func(code int, text string) error { + return c.handleClose +} + +// SetCloseHandler sets the handler for close messages received from the peer. +// The code argument to h is the received close code or CloseNoStatusReceived +// if the close message is empty. The default close handler sends a close +// message back to the peer. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// close messages as described in the section on Control Messages above. +// +// The connection read methods return a CloseError when a close message is +// received. Most applications should handle close messages as part of their +// normal error handling. Applications should only set a close handler when the +// application must perform some action before sending a close message back to +// the peer. +func (c *Conn) SetCloseHandler(h func(code int, text string) error) { + if h == nil { + h = func(code int, text string) error { + message := FormatCloseMessage(code, "") + c.WriteControl(CloseMessage, message, time.Now().Add(writeWait)) + return nil + } + } + c.handleClose = h +} + +// PingHandler returns the current ping handler +func (c *Conn) PingHandler() func(appData string) error { + return c.handlePing +} + +// SetPingHandler sets the handler for ping messages received from the peer. +// The appData argument to h is the PING message application data. The default +// ping handler sends a pong to the peer. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// ping messages as described in the section on Control Messages above. +func (c *Conn) SetPingHandler(h func(appData string) error) { + if h == nil { + h = func(message string) error { + err := c.WriteControl(PongMessage, []byte(message), time.Now().Add(writeWait)) + if err == ErrCloseSent { + return nil + } else if e, ok := err.(net.Error); ok && e.Temporary() { + return nil + } + return err + } + } + c.handlePing = h +} + +// PongHandler returns the current pong handler +func (c *Conn) PongHandler() func(appData string) error { + return c.handlePong +} + +// SetPongHandler sets the handler for pong messages received from the peer. +// The appData argument to h is the PONG message application data. The default +// pong handler does nothing. +// +// The handler function is called from the NextReader, ReadMessage and message +// reader Read methods. The application must read the connection to process +// pong messages as described in the section on Control Messages above. +func (c *Conn) SetPongHandler(h func(appData string) error) { + if h == nil { + h = func(string) error { return nil } + } + c.handlePong = h +} + +// UnderlyingConn returns the internal net.Conn. This can be used to further +// modifications to connection specific flags. +func (c *Conn) UnderlyingConn() net.Conn { + return c.conn +} + +// EnableWriteCompression enables and disables write compression of +// subsequent text and binary messages. This function is a noop if +// compression was not negotiated with the peer. +func (c *Conn) EnableWriteCompression(enable bool) { + c.enableWriteCompression = enable +} + +// SetCompressionLevel sets the flate compression level for subsequent text and +// binary messages. This function is a noop if compression was not negotiated +// with the peer. See the compress/flate package for a description of +// compression levels. +func (c *Conn) SetCompressionLevel(level int) error { + if !isValidCompressionLevel(level) { + return errors.New("websocket: invalid compression level") + } + c.compressionLevel = level + return nil +} + +// FormatCloseMessage formats closeCode and text as a WebSocket close message. +// An empty message is returned for code CloseNoStatusReceived. +func FormatCloseMessage(closeCode int, text string) []byte { + if closeCode == CloseNoStatusReceived { + // Return empty message because it's illegal to send + // CloseNoStatusReceived. Return non-nil value in case application + // checks for nil. + return []byte{} + } + buf := make([]byte, 2+len(text)) + binary.BigEndian.PutUint16(buf, uint16(closeCode)) + copy(buf[2:], text) + return buf +} diff --git a/vendor/github.com/gorilla/websocket/conn_write.go b/vendor/github.com/gorilla/websocket/conn_write.go new file mode 100644 index 000000000..a509a21f8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.8 + +package websocket + +import "net" + +func (c *Conn) writeBufs(bufs ...[]byte) error { + b := net.Buffers(bufs) + _, err := b.WriteTo(c.conn) + return err +} diff --git a/vendor/github.com/gorilla/websocket/conn_write_legacy.go b/vendor/github.com/gorilla/websocket/conn_write_legacy.go new file mode 100644 index 000000000..37edaff5a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/conn_write_legacy.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.8 + +package websocket + +func (c *Conn) writeBufs(bufs ...[]byte) error { + for _, buf := range bufs { + if len(buf) > 0 { + if _, err := c.conn.Write(buf); err != nil { + return err + } + } + } + return nil +} diff --git a/vendor/github.com/gorilla/websocket/doc.go b/vendor/github.com/gorilla/websocket/doc.go new file mode 100644 index 000000000..c6f4df896 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/doc.go @@ -0,0 +1,227 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package websocket implements the WebSocket protocol defined in RFC 6455. +// +// Overview +// +// The Conn type represents a WebSocket connection. A server application calls +// the Upgrader.Upgrade method from an HTTP request handler to get a *Conn: +// +// var upgrader = websocket.Upgrader{ +// ReadBufferSize: 1024, +// WriteBufferSize: 1024, +// } +// +// func handler(w http.ResponseWriter, r *http.Request) { +// conn, err := upgrader.Upgrade(w, r, nil) +// if err != nil { +// log.Println(err) +// return +// } +// ... Use conn to send and receive messages. +// } +// +// Call the connection's WriteMessage and ReadMessage methods to send and +// receive messages as a slice of bytes. This snippet of code shows how to echo +// messages using these methods: +// +// for { +// messageType, p, err := conn.ReadMessage() +// if err != nil { +// log.Println(err) +// return +// } +// if err := conn.WriteMessage(messageType, p); err != nil { +// log.Println(err) +// return +// } +// } +// +// In above snippet of code, p is a []byte and messageType is an int with value +// websocket.BinaryMessage or websocket.TextMessage. +// +// An application can also send and receive messages using the io.WriteCloser +// and io.Reader interfaces. To send a message, call the connection NextWriter +// method to get an io.WriteCloser, write the message to the writer and close +// the writer when done. To receive a message, call the connection NextReader +// method to get an io.Reader and read until io.EOF is returned. This snippet +// shows how to echo messages using the NextWriter and NextReader methods: +// +// for { +// messageType, r, err := conn.NextReader() +// if err != nil { +// return +// } +// w, err := conn.NextWriter(messageType) +// if err != nil { +// return err +// } +// if _, err := io.Copy(w, r); err != nil { +// return err +// } +// if err := w.Close(); err != nil { +// return err +// } +// } +// +// Data Messages +// +// The WebSocket protocol distinguishes between text and binary data messages. +// Text messages are interpreted as UTF-8 encoded text. The interpretation of +// binary messages is left to the application. +// +// This package uses the TextMessage and BinaryMessage integer constants to +// identify the two data message types. The ReadMessage and NextReader methods +// return the type of the received message. The messageType argument to the +// WriteMessage and NextWriter methods specifies the type of a sent message. +// +// It is the application's responsibility to ensure that text messages are +// valid UTF-8 encoded text. +// +// Control Messages +// +// The WebSocket protocol defines three types of control messages: close, ping +// and pong. Call the connection WriteControl, WriteMessage or NextWriter +// methods to send a control message to the peer. +// +// Connections handle received close messages by calling the handler function +// set with the SetCloseHandler method and by returning a *CloseError from the +// NextReader, ReadMessage or the message Read method. The default close +// handler sends a close message to the peer. +// +// Connections handle received ping messages by calling the handler function +// set with the SetPingHandler method. The default ping handler sends a pong +// message to the peer. +// +// Connections handle received pong messages by calling the handler function +// set with the SetPongHandler method. The default pong handler does nothing. +// If an application sends ping messages, then the application should set a +// pong handler to receive the corresponding pong. +// +// The control message handler functions are called from the NextReader, +// ReadMessage and message reader Read methods. The default close and ping +// handlers can block these methods for a short time when the handler writes to +// the connection. +// +// The application must read the connection to process close, ping and pong +// messages sent from the peer. If the application is not otherwise interested +// in messages from the peer, then the application should start a goroutine to +// read and discard messages from the peer. A simple example is: +// +// func readLoop(c *websocket.Conn) { +// for { +// if _, _, err := c.NextReader(); err != nil { +// c.Close() +// break +// } +// } +// } +// +// Concurrency +// +// Connections support one concurrent reader and one concurrent writer. +// +// Applications are responsible for ensuring that no more than one goroutine +// calls the write methods (NextWriter, SetWriteDeadline, WriteMessage, +// WriteJSON, EnableWriteCompression, SetCompressionLevel) concurrently and +// that no more than one goroutine calls the read methods (NextReader, +// SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, SetPingHandler) +// concurrently. +// +// The Close and WriteControl methods can be called concurrently with all other +// methods. +// +// Origin Considerations +// +// Web browsers allow Javascript applications to open a WebSocket connection to +// any host. It's up to the server to enforce an origin policy using the Origin +// request header sent by the browser. +// +// The Upgrader calls the function specified in the CheckOrigin field to check +// the origin. If the CheckOrigin function returns false, then the Upgrade +// method fails the WebSocket handshake with HTTP status 403. +// +// If the CheckOrigin field is nil, then the Upgrader uses a safe default: fail +// the handshake if the Origin request header is present and the Origin host is +// not equal to the Host request header. +// +// The deprecated package-level Upgrade function does not perform origin +// checking. The application is responsible for checking the Origin header +// before calling the Upgrade function. +// +// Buffers +// +// Connections buffer network input and output to reduce the number +// of system calls when reading or writing messages. +// +// Write buffers are also used for constructing WebSocket frames. See RFC 6455, +// Section 5 for a discussion of message framing. A WebSocket frame header is +// written to the network each time a write buffer is flushed to the network. +// Decreasing the size of the write buffer can increase the amount of framing +// overhead on the connection. +// +// The buffer sizes in bytes are specified by the ReadBufferSize and +// WriteBufferSize fields in the Dialer and Upgrader. The Dialer uses a default +// size of 4096 when a buffer size field is set to zero. The Upgrader reuses +// buffers created by the HTTP server when a buffer size field is set to zero. +// The HTTP server buffers have a size of 4096 at the time of this writing. +// +// The buffer sizes do not limit the size of a message that can be read or +// written by a connection. +// +// Buffers are held for the lifetime of the connection by default. If the +// Dialer or Upgrader WriteBufferPool field is set, then a connection holds the +// write buffer only when writing a message. +// +// Applications should tune the buffer sizes to balance memory use and +// performance. Increasing the buffer size uses more memory, but can reduce the +// number of system calls to read or write the network. In the case of writing, +// increasing the buffer size can reduce the number of frame headers written to +// the network. +// +// Some guidelines for setting buffer parameters are: +// +// Limit the buffer sizes to the maximum expected message size. Buffers larger +// than the largest message do not provide any benefit. +// +// Depending on the distribution of message sizes, setting the buffer size to +// to a value less than the maximum expected message size can greatly reduce +// memory use with a small impact on performance. Here's an example: If 99% of +// the messages are smaller than 256 bytes and the maximum message size is 512 +// bytes, then a buffer size of 256 bytes will result in 1.01 more system calls +// than a buffer size of 512 bytes. The memory savings is 50%. +// +// A write buffer pool is useful when the application has a modest number +// writes over a large number of connections. when buffers are pooled, a larger +// buffer size has a reduced impact on total memory use and has the benefit of +// reducing system calls and frame overhead. +// +// Compression EXPERIMENTAL +// +// Per message compression extensions (RFC 7692) are experimentally supported +// by this package in a limited capacity. Setting the EnableCompression option +// to true in Dialer or Upgrader will attempt to negotiate per message deflate +// support. +// +// var upgrader = websocket.Upgrader{ +// EnableCompression: true, +// } +// +// If compression was successfully negotiated with the connection's peer, any +// message received in compressed form will be automatically decompressed. +// All Read methods will return uncompressed bytes. +// +// Per message compression of messages written to a connection can be enabled +// or disabled by calling the corresponding Conn method: +// +// conn.EnableWriteCompression(false) +// +// Currently this package does not support compression with "context takeover". +// This means that messages must be compressed and decompressed in isolation, +// without retaining sliding window or dictionary state across messages. For +// more details refer to RFC 7692. +// +// Use of compression is experimental and may result in decreased performance. +package websocket diff --git a/vendor/github.com/gorilla/websocket/go.mod b/vendor/github.com/gorilla/websocket/go.mod new file mode 100644 index 000000000..93a9e924a --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.mod @@ -0,0 +1 @@ +module github.com/gorilla/websocket diff --git a/vendor/github.com/gorilla/websocket/go.sum b/vendor/github.com/gorilla/websocket/go.sum new file mode 100644 index 000000000..cf4fbbaa0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= diff --git a/vendor/github.com/gorilla/websocket/join.go b/vendor/github.com/gorilla/websocket/join.go new file mode 100644 index 000000000..c64f8c829 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/join.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "io" + "strings" +) + +// JoinMessages concatenates received messages to create a single io.Reader. +// The string term is appended to each message. The returned reader does not +// support concurrent calls to the Read method. +func JoinMessages(c *Conn, term string) io.Reader { + return &joinReader{c: c, term: term} +} + +type joinReader struct { + c *Conn + term string + r io.Reader +} + +func (r *joinReader) Read(p []byte) (int, error) { + if r.r == nil { + var err error + _, r.r, err = r.c.NextReader() + if err != nil { + return 0, err + } + if r.term != "" { + r.r = io.MultiReader(r.r, strings.NewReader(r.term)) + } + } + n, err := r.r.Read(p) + if err == io.EOF { + err = nil + r.r = nil + } + return n, err +} diff --git a/vendor/github.com/gorilla/websocket/json.go b/vendor/github.com/gorilla/websocket/json.go new file mode 100644 index 000000000..dc2c1f641 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/json.go @@ -0,0 +1,60 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "encoding/json" + "io" +) + +// WriteJSON writes the JSON encoding of v as a message. +// +// Deprecated: Use c.WriteJSON instead. +func WriteJSON(c *Conn, v interface{}) error { + return c.WriteJSON(v) +} + +// WriteJSON writes the JSON encoding of v as a message. +// +// See the documentation for encoding/json Marshal for details about the +// conversion of Go values to JSON. +func (c *Conn) WriteJSON(v interface{}) error { + w, err := c.NextWriter(TextMessage) + if err != nil { + return err + } + err1 := json.NewEncoder(w).Encode(v) + err2 := w.Close() + if err1 != nil { + return err1 + } + return err2 +} + +// ReadJSON reads the next JSON-encoded message from the connection and stores +// it in the value pointed to by v. +// +// Deprecated: Use c.ReadJSON instead. +func ReadJSON(c *Conn, v interface{}) error { + return c.ReadJSON(v) +} + +// ReadJSON reads the next JSON-encoded message from the connection and stores +// it in the value pointed to by v. +// +// See the documentation for the encoding/json Unmarshal function for details +// about the conversion of JSON to a Go value. +func (c *Conn) ReadJSON(v interface{}) error { + _, r, err := c.NextReader() + if err != nil { + return err + } + err = json.NewDecoder(r).Decode(v) + if err == io.EOF { + // One value is expected in the message. + err = io.ErrUnexpectedEOF + } + return err +} diff --git a/vendor/github.com/gorilla/websocket/mask.go b/vendor/github.com/gorilla/websocket/mask.go new file mode 100644 index 000000000..577fce9ef --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask.go @@ -0,0 +1,54 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build !appengine + +package websocket + +import "unsafe" + +const wordSize = int(unsafe.Sizeof(uintptr(0))) + +func maskBytes(key [4]byte, pos int, b []byte) int { + // Mask one byte at a time for small buffers. + if len(b) < 2*wordSize { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 + } + + // Mask one byte at a time to word boundary. + if n := int(uintptr(unsafe.Pointer(&b[0]))) % wordSize; n != 0 { + n = wordSize - n + for i := range b[:n] { + b[i] ^= key[pos&3] + pos++ + } + b = b[n:] + } + + // Create aligned word size key. + var k [wordSize]byte + for i := range k { + k[i] = key[(pos+i)&3] + } + kw := *(*uintptr)(unsafe.Pointer(&k)) + + // Mask one word at a time. + n := (len(b) / wordSize) * wordSize + for i := 0; i < n; i += wordSize { + *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&b[0])) + uintptr(i))) ^= kw + } + + // Mask one byte at a time for remaining bytes. + b = b[n:] + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/mask_safe.go b/vendor/github.com/gorilla/websocket/mask_safe.go new file mode 100644 index 000000000..2aac060e5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/mask_safe.go @@ -0,0 +1,15 @@ +// Copyright 2016 The Gorilla WebSocket Authors. All rights reserved. Use of +// this source code is governed by a BSD-style license that can be found in the +// LICENSE file. + +// +build appengine + +package websocket + +func maskBytes(key [4]byte, pos int, b []byte) int { + for i := range b { + b[i] ^= key[pos&3] + pos++ + } + return pos & 3 +} diff --git a/vendor/github.com/gorilla/websocket/prepared.go b/vendor/github.com/gorilla/websocket/prepared.go new file mode 100644 index 000000000..74ec565d2 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/prepared.go @@ -0,0 +1,102 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bytes" + "net" + "sync" + "time" +) + +// PreparedMessage caches on the wire representations of a message payload. +// Use PreparedMessage to efficiently send a message payload to multiple +// connections. PreparedMessage is especially useful when compression is used +// because the CPU and memory expensive compression operation can be executed +// once for a given set of compression options. +type PreparedMessage struct { + messageType int + data []byte + mu sync.Mutex + frames map[prepareKey]*preparedFrame +} + +// prepareKey defines a unique set of options to cache prepared frames in PreparedMessage. +type prepareKey struct { + isServer bool + compress bool + compressionLevel int +} + +// preparedFrame contains data in wire representation. +type preparedFrame struct { + once sync.Once + data []byte +} + +// NewPreparedMessage returns an initialized PreparedMessage. You can then send +// it to connection using WritePreparedMessage method. Valid wire +// representation will be calculated lazily only once for a set of current +// connection options. +func NewPreparedMessage(messageType int, data []byte) (*PreparedMessage, error) { + pm := &PreparedMessage{ + messageType: messageType, + frames: make(map[prepareKey]*preparedFrame), + data: data, + } + + // Prepare a plain server frame. + _, frameData, err := pm.frame(prepareKey{isServer: true, compress: false}) + if err != nil { + return nil, err + } + + // To protect against caller modifying the data argument, remember the data + // copied to the plain server frame. + pm.data = frameData[len(frameData)-len(data):] + return pm, nil +} + +func (pm *PreparedMessage) frame(key prepareKey) (int, []byte, error) { + pm.mu.Lock() + frame, ok := pm.frames[key] + if !ok { + frame = &preparedFrame{} + pm.frames[key] = frame + } + pm.mu.Unlock() + + var err error + frame.once.Do(func() { + // Prepare a frame using a 'fake' connection. + // TODO: Refactor code in conn.go to allow more direct construction of + // the frame. + mu := make(chan bool, 1) + mu <- true + var nc prepareConn + c := &Conn{ + conn: &nc, + mu: mu, + isServer: key.isServer, + compressionLevel: key.compressionLevel, + enableWriteCompression: true, + writeBuf: make([]byte, defaultWriteBufferSize+maxFrameHeaderSize), + } + if key.compress { + c.newCompressionWriter = compressNoContextTakeover + } + err = c.WriteMessage(pm.messageType, pm.data) + frame.data = nc.buf.Bytes() + }) + return pm.messageType, frame.data, err +} + +type prepareConn struct { + buf bytes.Buffer + net.Conn +} + +func (pc *prepareConn) Write(p []byte) (int, error) { return pc.buf.Write(p) } +func (pc *prepareConn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/vendor/github.com/gorilla/websocket/proxy.go b/vendor/github.com/gorilla/websocket/proxy.go new file mode 100644 index 000000000..e87a8c9f0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/proxy.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "encoding/base64" + "errors" + "net" + "net/http" + "net/url" + "strings" +) + +type netDialerFunc func(network, addr string) (net.Conn, error) + +func (fn netDialerFunc) Dial(network, addr string) (net.Conn, error) { + return fn(network, addr) +} + +func init() { + proxy_RegisterDialerType("http", func(proxyURL *url.URL, forwardDialer proxy_Dialer) (proxy_Dialer, error) { + return &httpProxyDialer{proxyURL: proxyURL, forwardDial: forwardDialer.Dial}, nil + }) +} + +type httpProxyDialer struct { + proxyURL *url.URL + forwardDial func(network, addr string) (net.Conn, error) +} + +func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) { + hostPort, _ := hostPortNoPort(hpd.proxyURL) + conn, err := hpd.forwardDial(network, hostPort) + if err != nil { + return nil, err + } + + connectHeader := make(http.Header) + if user := hpd.proxyURL.User; user != nil { + proxyUser := user.Username() + if proxyPassword, passwordSet := user.Password(); passwordSet { + credential := base64.StdEncoding.EncodeToString([]byte(proxyUser + ":" + proxyPassword)) + connectHeader.Set("Proxy-Authorization", "Basic "+credential) + } + } + + connectReq := &http.Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: addr}, + Host: addr, + Header: connectHeader, + } + + if err := connectReq.Write(conn); err != nil { + conn.Close() + return nil, err + } + + // Read response. It's OK to use and discard buffered reader here becaue + // the remote server does not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, connectReq) + if err != nil { + conn.Close() + return nil, err + } + + if resp.StatusCode != 200 { + conn.Close() + f := strings.SplitN(resp.Status, " ", 2) + return nil, errors.New(f[1]) + } + return conn, nil +} diff --git a/vendor/github.com/gorilla/websocket/server.go b/vendor/github.com/gorilla/websocket/server.go new file mode 100644 index 000000000..3d4480a47 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/server.go @@ -0,0 +1,363 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "bufio" + "errors" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +// HandshakeError describes an error with the handshake from the peer. +type HandshakeError struct { + message string +} + +func (e HandshakeError) Error() string { return e.message } + +// Upgrader specifies parameters for upgrading an HTTP connection to a +// WebSocket connection. +type Upgrader struct { + // HandshakeTimeout specifies the duration for the handshake to complete. + HandshakeTimeout time.Duration + + // ReadBufferSize and WriteBufferSize specify I/O buffer sizes in bytes. If a buffer + // size is zero, then buffers allocated by the HTTP server are used. The + // I/O buffer sizes do not limit the size of the messages that can be sent + // or received. + ReadBufferSize, WriteBufferSize int + + // WriteBufferPool is a pool of buffers for write operations. If the value + // is not set, then write buffers are allocated to the connection for the + // lifetime of the connection. + // + // A pool is most useful when the application has a modest volume of writes + // across a large number of connections. + // + // Applications should use a single pool for each unique value of + // WriteBufferSize. + WriteBufferPool BufferPool + + // Subprotocols specifies the server's supported protocols in order of + // preference. If this field is not nil, then the Upgrade method negotiates a + // subprotocol by selecting the first match in this list with a protocol + // requested by the client. If there's no match, then no protocol is + // negotiated (the Sec-Websocket-Protocol header is not included in the + // handshake response). + Subprotocols []string + + // Error specifies the function for generating HTTP error responses. If Error + // is nil, then http.Error is used to generate the HTTP response. + Error func(w http.ResponseWriter, r *http.Request, status int, reason error) + + // CheckOrigin returns true if the request Origin header is acceptable. If + // CheckOrigin is nil, then a safe default is used: return false if the + // Origin request header is present and the origin host is not equal to + // request Host header. + // + // A CheckOrigin function should carefully validate the request origin to + // prevent cross-site request forgery. + CheckOrigin func(r *http.Request) bool + + // EnableCompression specify if the server should attempt to negotiate per + // message compression (RFC 7692). Setting this value to true does not + // guarantee that compression will be supported. Currently only "no context + // takeover" modes are supported. + EnableCompression bool +} + +func (u *Upgrader) returnError(w http.ResponseWriter, r *http.Request, status int, reason string) (*Conn, error) { + err := HandshakeError{reason} + if u.Error != nil { + u.Error(w, r, status, err) + } else { + w.Header().Set("Sec-Websocket-Version", "13") + http.Error(w, http.StatusText(status), status) + } + return nil, err +} + +// checkSameOrigin returns true if the origin is not set or is equal to the request host. +func checkSameOrigin(r *http.Request) bool { + origin := r.Header["Origin"] + if len(origin) == 0 { + return true + } + u, err := url.Parse(origin[0]) + if err != nil { + return false + } + return equalASCIIFold(u.Host, r.Host) +} + +func (u *Upgrader) selectSubprotocol(r *http.Request, responseHeader http.Header) string { + if u.Subprotocols != nil { + clientProtocols := Subprotocols(r) + for _, serverProtocol := range u.Subprotocols { + for _, clientProtocol := range clientProtocols { + if clientProtocol == serverProtocol { + return clientProtocol + } + } + } + } else if responseHeader != nil { + return responseHeader.Get("Sec-Websocket-Protocol") + } + return "" +} + +// Upgrade upgrades the HTTP server connection to the WebSocket protocol. +// +// The responseHeader is included in the response to the client's upgrade +// request. Use the responseHeader to specify cookies (Set-Cookie) and the +// application negotiated subprotocol (Sec-WebSocket-Protocol). +// +// If the upgrade fails, then Upgrade replies to the client with an HTTP error +// response. +func (u *Upgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*Conn, error) { + const badHandshake = "websocket: the client is not using the websocket protocol: " + + if !tokenListContainsValue(r.Header, "Connection", "upgrade") { + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'upgrade' token not found in 'Connection' header") + } + + if !tokenListContainsValue(r.Header, "Upgrade", "websocket") { + return u.returnError(w, r, http.StatusBadRequest, badHandshake+"'websocket' token not found in 'Upgrade' header") + } + + if r.Method != "GET" { + return u.returnError(w, r, http.StatusMethodNotAllowed, badHandshake+"request method is not GET") + } + + if !tokenListContainsValue(r.Header, "Sec-Websocket-Version", "13") { + return u.returnError(w, r, http.StatusBadRequest, "websocket: unsupported version: 13 not found in 'Sec-Websocket-Version' header") + } + + if _, ok := responseHeader["Sec-Websocket-Extensions"]; ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: application specific 'Sec-WebSocket-Extensions' headers are unsupported") + } + + checkOrigin := u.CheckOrigin + if checkOrigin == nil { + checkOrigin = checkSameOrigin + } + if !checkOrigin(r) { + return u.returnError(w, r, http.StatusForbidden, "websocket: request origin not allowed by Upgrader.CheckOrigin") + } + + challengeKey := r.Header.Get("Sec-Websocket-Key") + if challengeKey == "" { + return u.returnError(w, r, http.StatusBadRequest, "websocket: not a websocket handshake: `Sec-WebSocket-Key' header is missing or blank") + } + + subprotocol := u.selectSubprotocol(r, responseHeader) + + // Negotiate PMCE + var compress bool + if u.EnableCompression { + for _, ext := range parseExtensions(r.Header) { + if ext[""] != "permessage-deflate" { + continue + } + compress = true + break + } + } + + h, ok := w.(http.Hijacker) + if !ok { + return u.returnError(w, r, http.StatusInternalServerError, "websocket: response does not implement http.Hijacker") + } + var brw *bufio.ReadWriter + netConn, brw, err := h.Hijack() + if err != nil { + return u.returnError(w, r, http.StatusInternalServerError, err.Error()) + } + + if brw.Reader.Buffered() > 0 { + netConn.Close() + return nil, errors.New("websocket: client sent data before handshake is complete") + } + + var br *bufio.Reader + if u.ReadBufferSize == 0 && bufioReaderSize(netConn, brw.Reader) > 256 { + // Reuse hijacked buffered reader as connection reader. + br = brw.Reader + } + + buf := bufioWriterBuffer(netConn, brw.Writer) + + var writeBuf []byte + if u.WriteBufferPool == nil && u.WriteBufferSize == 0 && len(buf) >= maxFrameHeaderSize+256 { + // Reuse hijacked write buffer as connection buffer. + writeBuf = buf + } + + c := newConn(netConn, true, u.ReadBufferSize, u.WriteBufferSize, u.WriteBufferPool, br, writeBuf) + c.subprotocol = subprotocol + + if compress { + c.newCompressionWriter = compressNoContextTakeover + c.newDecompressionReader = decompressNoContextTakeover + } + + // Use larger of hijacked buffer and connection write buffer for header. + p := buf + if len(c.writeBuf) > len(p) { + p = c.writeBuf + } + p = p[:0] + + p = append(p, "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: "...) + p = append(p, computeAcceptKey(challengeKey)...) + p = append(p, "\r\n"...) + if c.subprotocol != "" { + p = append(p, "Sec-WebSocket-Protocol: "...) + p = append(p, c.subprotocol...) + p = append(p, "\r\n"...) + } + if compress { + p = append(p, "Sec-WebSocket-Extensions: permessage-deflate; server_no_context_takeover; client_no_context_takeover\r\n"...) + } + for k, vs := range responseHeader { + if k == "Sec-Websocket-Protocol" { + continue + } + for _, v := range vs { + p = append(p, k...) + p = append(p, ": "...) + for i := 0; i < len(v); i++ { + b := v[i] + if b <= 31 { + // prevent response splitting. + b = ' ' + } + p = append(p, b) + } + p = append(p, "\r\n"...) + } + } + p = append(p, "\r\n"...) + + // Clear deadlines set by HTTP server. + netConn.SetDeadline(time.Time{}) + + if u.HandshakeTimeout > 0 { + netConn.SetWriteDeadline(time.Now().Add(u.HandshakeTimeout)) + } + if _, err = netConn.Write(p); err != nil { + netConn.Close() + return nil, err + } + if u.HandshakeTimeout > 0 { + netConn.SetWriteDeadline(time.Time{}) + } + + return c, nil +} + +// Upgrade upgrades the HTTP server connection to the WebSocket protocol. +// +// Deprecated: Use websocket.Upgrader instead. +// +// Upgrade does not perform origin checking. The application is responsible for +// checking the Origin header before calling Upgrade. An example implementation +// of the same origin policy check is: +// +// if req.Header.Get("Origin") != "http://"+req.Host { +// http.Error(w, "Origin not allowed", http.StatusForbidden) +// return +// } +// +// If the endpoint supports subprotocols, then the application is responsible +// for negotiating the protocol used on the connection. Use the Subprotocols() +// function to get the subprotocols requested by the client. Use the +// Sec-Websocket-Protocol response header to specify the subprotocol selected +// by the application. +// +// The responseHeader is included in the response to the client's upgrade +// request. Use the responseHeader to specify cookies (Set-Cookie) and the +// negotiated subprotocol (Sec-Websocket-Protocol). +// +// The connection buffers IO to the underlying network connection. The +// readBufSize and writeBufSize parameters specify the size of the buffers to +// use. Messages can be larger than the buffers. +// +// If the request is not a valid WebSocket handshake, then Upgrade returns an +// error of type HandshakeError. Applications should handle this error by +// replying to the client with an HTTP error response. +func Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header, readBufSize, writeBufSize int) (*Conn, error) { + u := Upgrader{ReadBufferSize: readBufSize, WriteBufferSize: writeBufSize} + u.Error = func(w http.ResponseWriter, r *http.Request, status int, reason error) { + // don't return errors to maintain backwards compatibility + } + u.CheckOrigin = func(r *http.Request) bool { + // allow all connections by default + return true + } + return u.Upgrade(w, r, responseHeader) +} + +// Subprotocols returns the subprotocols requested by the client in the +// Sec-Websocket-Protocol header. +func Subprotocols(r *http.Request) []string { + h := strings.TrimSpace(r.Header.Get("Sec-Websocket-Protocol")) + if h == "" { + return nil + } + protocols := strings.Split(h, ",") + for i := range protocols { + protocols[i] = strings.TrimSpace(protocols[i]) + } + return protocols +} + +// IsWebSocketUpgrade returns true if the client requested upgrade to the +// WebSocket protocol. +func IsWebSocketUpgrade(r *http.Request) bool { + return tokenListContainsValue(r.Header, "Connection", "upgrade") && + tokenListContainsValue(r.Header, "Upgrade", "websocket") +} + +// bufioReaderSize size returns the size of a bufio.Reader. +func bufioReaderSize(originalReader io.Reader, br *bufio.Reader) int { + // This code assumes that peek on a reset reader returns + // bufio.Reader.buf[:0]. + // TODO: Use bufio.Reader.Size() after Go 1.10 + br.Reset(originalReader) + if p, err := br.Peek(0); err == nil { + return cap(p) + } + return 0 +} + +// writeHook is an io.Writer that records the last slice passed to it vio +// io.Writer.Write. +type writeHook struct { + p []byte +} + +func (wh *writeHook) Write(p []byte) (int, error) { + wh.p = p + return len(p), nil +} + +// bufioWriterBuffer grabs the buffer from a bufio.Writer. +func bufioWriterBuffer(originalWriter io.Writer, bw *bufio.Writer) []byte { + // This code assumes that bufio.Writer.buf[:1] is passed to the + // bufio.Writer's underlying writer. + var wh writeHook + bw.Reset(&wh) + bw.WriteByte(0) + bw.Flush() + + bw.Reset(originalWriter) + + return wh.p[:cap(wh.p)] +} diff --git a/vendor/github.com/gorilla/websocket/trace.go b/vendor/github.com/gorilla/websocket/trace.go new file mode 100644 index 000000000..834f122a0 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/trace.go @@ -0,0 +1,19 @@ +// +build go1.8 + +package websocket + +import ( + "crypto/tls" + "net/http/httptrace" +) + +func doHandshakeWithTrace(trace *httptrace.ClientTrace, tlsConn *tls.Conn, cfg *tls.Config) error { + if trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } + err := doHandshake(tlsConn, cfg) + if trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tlsConn.ConnectionState(), err) + } + return err +} diff --git a/vendor/github.com/gorilla/websocket/trace_17.go b/vendor/github.com/gorilla/websocket/trace_17.go new file mode 100644 index 000000000..77d05a0b5 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/trace_17.go @@ -0,0 +1,12 @@ +// +build !go1.8 + +package websocket + +import ( + "crypto/tls" + "net/http/httptrace" +) + +func doHandshakeWithTrace(trace *httptrace.ClientTrace, tlsConn *tls.Conn, cfg *tls.Config) error { + return doHandshake(tlsConn, cfg) +} diff --git a/vendor/github.com/gorilla/websocket/util.go b/vendor/github.com/gorilla/websocket/util.go new file mode 100644 index 000000000..7bf2f66c6 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/util.go @@ -0,0 +1,283 @@ +// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package websocket + +import ( + "crypto/rand" + "crypto/sha1" + "encoding/base64" + "io" + "net/http" + "strings" + "unicode/utf8" +) + +var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11") + +func computeAcceptKey(challengeKey string) string { + h := sha1.New() + h.Write([]byte(challengeKey)) + h.Write(keyGUID) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func generateChallengeKey() (string, error) { + p := make([]byte, 16) + if _, err := io.ReadFull(rand.Reader, p); err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(p), nil +} + +// Token octets per RFC 2616. +var isTokenOctet = [256]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +// skipSpace returns a slice of the string s with all leading RFC 2616 linear +// whitespace removed. +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if b := s[i]; b != ' ' && b != '\t' { + break + } + } + return s[i:] +} + +// nextToken returns the leading RFC 2616 token of s and the string following +// the token. +func nextToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if !isTokenOctet[s[i]] { + break + } + } + return s[:i], s[i:] +} + +// nextTokenOrQuoted returns the leading token or quoted string per RFC 2616 +// and the string following the token or quoted string. +func nextTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return nextToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + 1; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} + +// equalASCIIFold returns true if s is equal to t with ASCII case folding as +// defined in RFC 4790. +func equalASCIIFold(s, t string) bool { + for s != "" && t != "" { + sr, size := utf8.DecodeRuneInString(s) + s = s[size:] + tr, size := utf8.DecodeRuneInString(t) + t = t[size:] + if sr == tr { + continue + } + if 'A' <= sr && sr <= 'Z' { + sr = sr + 'a' - 'A' + } + if 'A' <= tr && tr <= 'Z' { + tr = tr + 'a' - 'A' + } + if sr != tr { + return false + } + } + return s == t +} + +// tokenListContainsValue returns true if the 1#token header with the given +// name contains a token equal to value with ASCII case folding. +func tokenListContainsValue(header http.Header, name string, value string) bool { +headers: + for _, s := range header[name] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + s = skipSpace(s) + if s != "" && s[0] != ',' { + continue headers + } + if equalASCIIFold(t, value) { + return true + } + if s == "" { + continue headers + } + s = s[1:] + } + } + return false +} + +// parseExtensions parses WebSocket extensions from a header. +func parseExtensions(header http.Header) []map[string]string { + // From RFC 6455: + // + // Sec-WebSocket-Extensions = extension-list + // extension-list = 1#extension + // extension = extension-token *( ";" extension-param ) + // extension-token = registered-token + // registered-token = token + // extension-param = token [ "=" (token | quoted-string) ] + // ;When using the quoted-string syntax variant, the value + // ;after quoted-string unescaping MUST conform to the + // ;'token' ABNF. + + var result []map[string]string +headers: + for _, s := range header["Sec-Websocket-Extensions"] { + for { + var t string + t, s = nextToken(skipSpace(s)) + if t == "" { + continue headers + } + ext := map[string]string{"": t} + for { + s = skipSpace(s) + if !strings.HasPrefix(s, ";") { + break + } + var k string + k, s = nextToken(skipSpace(s[1:])) + if k == "" { + continue headers + } + s = skipSpace(s) + var v string + if strings.HasPrefix(s, "=") { + v, s = nextTokenOrQuoted(skipSpace(s[1:])) + s = skipSpace(s) + } + if s != "" && s[0] != ',' && s[0] != ';' { + continue headers + } + ext[k] = v + } + if s != "" && s[0] != ',' { + continue headers + } + result = append(result, ext) + if s == "" { + continue headers + } + s = s[1:] + } + } + return result +} diff --git a/vendor/github.com/gorilla/websocket/x_net_proxy.go b/vendor/github.com/gorilla/websocket/x_net_proxy.go new file mode 100644 index 000000000..2e668f6b8 --- /dev/null +++ b/vendor/github.com/gorilla/websocket/x_net_proxy.go @@ -0,0 +1,473 @@ +// Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. +//go:generate bundle -o x_net_proxy.go golang.org/x/net/proxy + +// Package proxy provides support for a variety of protocols to proxy network +// data. +// + +package websocket + +import ( + "errors" + "io" + "net" + "net/url" + "os" + "strconv" + "strings" + "sync" +) + +type proxy_direct struct{} + +// Direct is a direct proxy: one that makes network connections directly. +var proxy_Direct = proxy_direct{} + +func (proxy_direct) Dial(network, addr string) (net.Conn, error) { + return net.Dial(network, addr) +} + +// A PerHost directs connections to a default Dialer unless the host name +// requested matches one of a number of exceptions. +type proxy_PerHost struct { + def, bypass proxy_Dialer + + bypassNetworks []*net.IPNet + bypassIPs []net.IP + bypassZones []string + bypassHosts []string +} + +// NewPerHost returns a PerHost Dialer that directs connections to either +// defaultDialer or bypass, depending on whether the connection matches one of +// the configured rules. +func proxy_NewPerHost(defaultDialer, bypass proxy_Dialer) *proxy_PerHost { + return &proxy_PerHost{ + def: defaultDialer, + bypass: bypass, + } +} + +// Dial connects to the address addr on the given network through either +// defaultDialer or bypass. +func (p *proxy_PerHost) Dial(network, addr string) (c net.Conn, err error) { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + return p.dialerForRequest(host).Dial(network, addr) +} + +func (p *proxy_PerHost) dialerForRequest(host string) proxy_Dialer { + if ip := net.ParseIP(host); ip != nil { + for _, net := range p.bypassNetworks { + if net.Contains(ip) { + return p.bypass + } + } + for _, bypassIP := range p.bypassIPs { + if bypassIP.Equal(ip) { + return p.bypass + } + } + return p.def + } + + for _, zone := range p.bypassZones { + if strings.HasSuffix(host, zone) { + return p.bypass + } + if host == zone[1:] { + // For a zone ".example.com", we match "example.com" + // too. + return p.bypass + } + } + for _, bypassHost := range p.bypassHosts { + if bypassHost == host { + return p.bypass + } + } + return p.def +} + +// AddFromString parses a string that contains comma-separated values +// specifying hosts that should use the bypass proxy. Each value is either an +// IP address, a CIDR range, a zone (*.example.com) or a host name +// (localhost). A best effort is made to parse the string and errors are +// ignored. +func (p *proxy_PerHost) AddFromString(s string) { + hosts := strings.Split(s, ",") + for _, host := range hosts { + host = strings.TrimSpace(host) + if len(host) == 0 { + continue + } + if strings.Contains(host, "/") { + // We assume that it's a CIDR address like 127.0.0.0/8 + if _, net, err := net.ParseCIDR(host); err == nil { + p.AddNetwork(net) + } + continue + } + if ip := net.ParseIP(host); ip != nil { + p.AddIP(ip) + continue + } + if strings.HasPrefix(host, "*.") { + p.AddZone(host[1:]) + continue + } + p.AddHost(host) + } +} + +// AddIP specifies an IP address that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match an IP. +func (p *proxy_PerHost) AddIP(ip net.IP) { + p.bypassIPs = append(p.bypassIPs, ip) +} + +// AddNetwork specifies an IP range that will use the bypass proxy. Note that +// this will only take effect if a literal IP address is dialed. A connection +// to a named host will never match. +func (p *proxy_PerHost) AddNetwork(net *net.IPNet) { + p.bypassNetworks = append(p.bypassNetworks, net) +} + +// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of +// "example.com" matches "example.com" and all of its subdomains. +func (p *proxy_PerHost) AddZone(zone string) { + if strings.HasSuffix(zone, ".") { + zone = zone[:len(zone)-1] + } + if !strings.HasPrefix(zone, ".") { + zone = "." + zone + } + p.bypassZones = append(p.bypassZones, zone) +} + +// AddHost specifies a host name that will use the bypass proxy. +func (p *proxy_PerHost) AddHost(host string) { + if strings.HasSuffix(host, ".") { + host = host[:len(host)-1] + } + p.bypassHosts = append(p.bypassHosts, host) +} + +// A Dialer is a means to establish a connection. +type proxy_Dialer interface { + // Dial connects to the given address via the proxy. + Dial(network, addr string) (c net.Conn, err error) +} + +// Auth contains authentication parameters that specific Dialers may require. +type proxy_Auth struct { + User, Password string +} + +// FromEnvironment returns the dialer specified by the proxy related variables in +// the environment. +func proxy_FromEnvironment() proxy_Dialer { + allProxy := proxy_allProxyEnv.Get() + if len(allProxy) == 0 { + return proxy_Direct + } + + proxyURL, err := url.Parse(allProxy) + if err != nil { + return proxy_Direct + } + proxy, err := proxy_FromURL(proxyURL, proxy_Direct) + if err != nil { + return proxy_Direct + } + + noProxy := proxy_noProxyEnv.Get() + if len(noProxy) == 0 { + return proxy + } + + perHost := proxy_NewPerHost(proxy, proxy_Direct) + perHost.AddFromString(noProxy) + return perHost +} + +// proxySchemes is a map from URL schemes to a function that creates a Dialer +// from a URL with such a scheme. +var proxy_proxySchemes map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error) + +// RegisterDialerType takes a URL scheme and a function to generate Dialers from +// a URL with that scheme and a forwarding Dialer. Registered schemes are used +// by FromURL. +func proxy_RegisterDialerType(scheme string, f func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) { + if proxy_proxySchemes == nil { + proxy_proxySchemes = make(map[string]func(*url.URL, proxy_Dialer) (proxy_Dialer, error)) + } + proxy_proxySchemes[scheme] = f +} + +// FromURL returns a Dialer given a URL specification and an underlying +// Dialer for it to make network requests. +func proxy_FromURL(u *url.URL, forward proxy_Dialer) (proxy_Dialer, error) { + var auth *proxy_Auth + if u.User != nil { + auth = new(proxy_Auth) + auth.User = u.User.Username() + if p, ok := u.User.Password(); ok { + auth.Password = p + } + } + + switch u.Scheme { + case "socks5": + return proxy_SOCKS5("tcp", u.Host, auth, forward) + } + + // If the scheme doesn't match any of the built-in schemes, see if it + // was registered by another package. + if proxy_proxySchemes != nil { + if f, ok := proxy_proxySchemes[u.Scheme]; ok { + return f(u, forward) + } + } + + return nil, errors.New("proxy: unknown scheme: " + u.Scheme) +} + +var ( + proxy_allProxyEnv = &proxy_envOnce{ + names: []string{"ALL_PROXY", "all_proxy"}, + } + proxy_noProxyEnv = &proxy_envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +// envOnce looks up an environment variable (optionally by multiple +// names) once. It mitigates expensive lookups on some platforms +// (e.g. Windows). +// (Borrowed from net/http/transport.go) +type proxy_envOnce struct { + names []string + once sync.Once + val string +} + +func (e *proxy_envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *proxy_envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given address +// with an optional username and password. See RFC 1928 and RFC 1929. +func proxy_SOCKS5(network, addr string, auth *proxy_Auth, forward proxy_Dialer) (proxy_Dialer, error) { + s := &proxy_socks5{ + network: network, + addr: addr, + forward: forward, + } + if auth != nil { + s.user = auth.User + s.password = auth.Password + } + + return s, nil +} + +type proxy_socks5 struct { + user, password string + network, addr string + forward proxy_Dialer +} + +const proxy_socks5Version = 5 + +const ( + proxy_socks5AuthNone = 0 + proxy_socks5AuthPassword = 2 +) + +const proxy_socks5Connect = 1 + +const ( + proxy_socks5IP4 = 1 + proxy_socks5Domain = 3 + proxy_socks5IP6 = 4 +) + +var proxy_socks5Errors = []string{ + "", + "general failure", + "connection forbidden", + "network unreachable", + "host unreachable", + "connection refused", + "TTL expired", + "command not supported", + "address type not supported", +} + +// Dial connects to the address addr on the given network via the SOCKS5 proxy. +func (s *proxy_socks5) Dial(network, addr string) (net.Conn, error) { + switch network { + case "tcp", "tcp6", "tcp4": + default: + return nil, errors.New("proxy: no support for SOCKS5 proxy connections of type " + network) + } + + conn, err := s.forward.Dial(s.network, s.addr) + if err != nil { + return nil, err + } + if err := s.connect(conn, addr); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +// connect takes an existing connection to a socks5 proxy server, +// and commands the server to extend that connection to target, +// which must be a canonical address with a host and port. +func (s *proxy_socks5) connect(conn net.Conn, target string) error { + host, portStr, err := net.SplitHostPort(target) + if err != nil { + return err + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return errors.New("proxy: failed to parse port number: " + portStr) + } + if port < 1 || port > 0xffff { + return errors.New("proxy: port number out of range: " + portStr) + } + + // the size here is just an estimate + buf := make([]byte, 0, 6+len(host)) + + buf = append(buf, proxy_socks5Version) + if len(s.user) > 0 && len(s.user) < 256 && len(s.password) < 256 { + buf = append(buf, 2 /* num auth methods */, proxy_socks5AuthNone, proxy_socks5AuthPassword) + } else { + buf = append(buf, 1 /* num auth methods */, proxy_socks5AuthNone) + } + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write greeting to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read greeting from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + if buf[0] != 5 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " has unexpected version " + strconv.Itoa(int(buf[0]))) + } + if buf[1] == 0xff { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " requires authentication") + } + + // See RFC 1929 + if buf[1] == proxy_socks5AuthPassword { + buf = buf[:0] + buf = append(buf, 1 /* password protocol version */) + buf = append(buf, uint8(len(s.user))) + buf = append(buf, s.user...) + buf = append(buf, uint8(len(s.password))) + buf = append(buf, s.password...) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write authentication request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read authentication reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if buf[1] != 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " rejected username/password") + } + } + + buf = buf[:0] + buf = append(buf, proxy_socks5Version, proxy_socks5Connect, 0 /* reserved */) + + if ip := net.ParseIP(host); ip != nil { + if ip4 := ip.To4(); ip4 != nil { + buf = append(buf, proxy_socks5IP4) + ip = ip4 + } else { + buf = append(buf, proxy_socks5IP6) + } + buf = append(buf, ip...) + } else { + if len(host) > 255 { + return errors.New("proxy: destination host name too long: " + host) + } + buf = append(buf, proxy_socks5Domain) + buf = append(buf, byte(len(host))) + buf = append(buf, host...) + } + buf = append(buf, byte(port>>8), byte(port)) + + if _, err := conn.Write(buf); err != nil { + return errors.New("proxy: failed to write connect request to SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + if _, err := io.ReadFull(conn, buf[:4]); err != nil { + return errors.New("proxy: failed to read connect reply from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + failure := "unknown error" + if int(buf[1]) < len(proxy_socks5Errors) { + failure = proxy_socks5Errors[buf[1]] + } + + if len(failure) > 0 { + return errors.New("proxy: SOCKS5 proxy at " + s.addr + " failed to connect: " + failure) + } + + bytesToDiscard := 0 + switch buf[3] { + case proxy_socks5IP4: + bytesToDiscard = net.IPv4len + case proxy_socks5IP6: + bytesToDiscard = net.IPv6len + case proxy_socks5Domain: + _, err := io.ReadFull(conn, buf[:1]) + if err != nil { + return errors.New("proxy: failed to read domain length from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + bytesToDiscard = int(buf[0]) + default: + return errors.New("proxy: got unknown address type " + strconv.Itoa(int(buf[3])) + " from SOCKS5 proxy at " + s.addr) + } + + if cap(buf) < bytesToDiscard { + buf = make([]byte, bytesToDiscard) + } else { + buf = buf[:bytesToDiscard] + } + if _, err := io.ReadFull(conn, buf); err != nil { + return errors.New("proxy: failed to read address from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + // Also need to discard the port number + if _, err := io.ReadFull(conn, buf[:2]); err != nil { + return errors.New("proxy: failed to read port from SOCKS5 proxy at " + s.addr + ": " + err.Error()) + } + + return nil +} diff --git a/vendor/vendor.json b/vendor/vendor.json index ebd689d5d..33577bae2 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -156,6 +156,7 @@ {"path":"github.com/gorhill/cronexpr/cronexpr","checksumSHA1":"Nd/7mZb0T6Gj6+AymyOPsNCQSJs=","comment":"1.0.0","revision":"a557574d6c024ed6e36acc8b610f5f211c91568a"}, {"path":"github.com/gorilla/context","checksumSHA1":"g/V4qrXjUGG9B+e3hB+4NAYJ5Gs=","revision":"08b5f424b9271eedf6f9f0ce86cb9396ed337a42","revisionTime":"2016-08-17T18:46:32Z"}, {"path":"github.com/gorilla/mux","checksumSHA1":"STQSdSj2FcpCf0NLfdsKhNutQT0=","revision":"e48e440e4c92e3251d812f8ce7858944dfa3331c","revisionTime":"2018-08-07T07:52:56Z"}, + {"path":"github.com/gorilla/websocket","checksumSHA1":"gr0edNJuVv4+olNNZl5ZmwLgscA=","revision":"0ec3d1bd7fe50c503d6df98ee649d81f4857c564","revisionTime":"2019-03-06T00:42:57Z"}, {"path":"github.com/hashicorp/consul-template","checksumSHA1":"+AGSqY+9kpGX5rrQDBWpgzaDKSA=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, {"path":"github.com/hashicorp/consul-template/child","checksumSHA1":"AhDPiKa7wzh3SE6Gx0WrsDYwBHg=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, {"path":"github.com/hashicorp/consul-template/config","checksumSHA1":"0vr6paBMXD7ZYSmtfJpjfjZJKic=","revision":"9a0f301b69d841c32f36b78008afb2dee8a9c40b","revisionTime":"2019-02-20T00:40:33Z"}, From 3a77502cae7dbb982a6349d83a69d960fd97f06b Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:12:53 -0400 Subject: [PATCH 12/15] Add basic drivers conformance tests Add consolidated testing package to serve as conformance tests for all drivers. --- plugins/drivers/testutils/exec_testing.go | 354 ++++++++++++++++++++++ plugins/drivers/testutils/testing.go | 31 +- 2 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 plugins/drivers/testutils/exec_testing.go diff --git a/plugins/drivers/testutils/exec_testing.go b/plugins/drivers/testutils/exec_testing.go new file mode 100644 index 000000000..7a7902570 --- /dev/null +++ b/plugins/drivers/testutils/exec_testing.go @@ -0,0 +1,354 @@ +package testutils + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "regexp" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/hashicorp/nomad/plugins/drivers" + dproto "github.com/hashicorp/nomad/plugins/drivers/proto" + "github.com/stretchr/testify/require" +) + +func ExecTaskStreamingConformanceTests(t *testing.T, driver *DriverHarness, taskID string) { + t.Helper() + + if runtime.GOOS == "windows" { + // tests assume unix-ism now + t.Skip("test assume unix tasks") + } + + TestExecTaskStreamingBasicResponses(t, driver, taskID) + TestExecFSIsolation(t, driver, taskID) +} + +var ExecTaskStreamingBasicCases = []struct { + Name string + Command string + Tty bool + Stdin string + Stdout interface{} + Stderr interface{} + ExitCode int +}{ + { + Name: "notty: basic", + Command: "echo hello stdout; echo hello stderr >&2; exit 43", + Tty: false, + Stdout: "hello stdout\n", + Stderr: "hello stderr\n", + ExitCode: 43, + }, + { + Name: "notty: streaming", + Command: "for n in 1 2 3; do echo $n; sleep 1; done", + Tty: false, + Stdout: "1\n2\n3\n", + ExitCode: 0, + }, + { + Name: "notty: stty check", + Command: "stty size", + Tty: false, + Stderr: regexp.MustCompile("stty: .?standard input.?: Inappropriate ioctl for device\n"), + ExitCode: 1, + }, + { + Name: "notty: stdin passing", + Command: "echo hello from command; head -n1", + Tty: false, + Stdin: "hello from stdin\n", + Stdout: "hello from command\nhello from stdin\n", + ExitCode: 0, + }, + { + Name: "notty: children processes", + Command: "(( sleep 3; echo from background ) & ); echo from main; exec sleep 1", + Tty: false, + // when not using tty; wait for all processes to exit matching behavior of `docker exec` + Stdout: "from main\nfrom background\n", + ExitCode: 0, + }, + + // TTY cases - difference is new lines add `\r` and child process waiting is different + { + Name: "tty: basic", + Command: "echo hello stdout; echo hello stderr >&2; exit 43", + Tty: true, + Stdout: "hello stdout\r\nhello stderr\r\n", + ExitCode: 43, + }, + { + Name: "tty: streaming", + Command: "for n in 1 2 3; do echo $n; sleep 1; done", + Tty: true, + Stdout: "1\r\n2\r\n3\r\n", + ExitCode: 0, + }, + { + Name: "tty: stty check", + Command: "sleep 1; stty size", + Tty: true, + Stdout: "100 100\r\n", + ExitCode: 0, + }, + { + Name: "tty: stdin passing", + Command: "head -n1", + Tty: true, + Stdin: "hello from stdin\n", + // in tty mode, we emit line twice: once for tty echoing and one for the actual head output + Stdout: "hello from stdin\r\nhello from stdin\r\n", + ExitCode: 0, + }, + { + Name: "tty: children processes", + Command: "(( sleep 3; echo from background ) & ); echo from main; exec sleep 1", + Tty: true, + // when using tty; wait for lead process only, like `docker exec -it` + Stdout: "from main\r\n", + ExitCode: 0, + }, +} + +func TestExecTaskStreamingBasicResponses(t *testing.T, driver *DriverHarness, taskID string) { + for _, c := range ExecTaskStreamingBasicCases { + t.Run("basic: "+c.Name, func(t *testing.T) { + + result := execTask(t, driver, taskID, c.Command, c.Tty, c.Stdin) + + require.Equal(t, c.ExitCode, result.exitCode) + + switch s := c.Stdout.(type) { + case string: + require.Equal(t, s, result.stdout) + case *regexp.Regexp: + require.Regexp(t, s, result.stdout) + case nil: + require.Empty(t, result.stdout) + default: + require.Fail(t, "unexpected stdout type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) + } + + switch s := c.Stderr.(type) { + case string: + require.Equal(t, s, result.stderr) + case *regexp.Regexp: + require.Regexp(t, s, result.stderr) + case nil: + require.Empty(t, result.stderr) + default: + require.Fail(t, "unexpected stderr type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) + } + + }) + } +} + +// TestExecFSIsolation asserts that exec occurs inside chroot/isolation environment rather than +// on host +func TestExecFSIsolation(t *testing.T, driver *DriverHarness, taskID string) { + t.Run("isolation", func(t *testing.T) { + caps, err := driver.Capabilities() + require.NoError(t, err) + + isolated := (caps.FSIsolation != drivers.FSIsolationNone) + + text := "hello from the other side" + + // write to a file and check it presence in host + w := execTask(t, driver, taskID, + fmt.Sprintf(`FILE=$(mktemp); echo "$FILE"; echo %q >> "${FILE}"`, text), + false, "") + require.Zero(t, w.exitCode) + + tempfile := strings.TrimSpace(w.stdout) + if !isolated { + defer os.Remove(tempfile) + } + + t.Logf("created file in task: %v", tempfile) + + // read from host + b, err := ioutil.ReadFile(tempfile) + if !isolated { + require.NoError(t, err) + require.Equal(t, text, strings.TrimSpace(string(b))) + } else { + require.Error(t, err) + require.True(t, os.IsNotExist(err)) + } + + // read should succeed from task again + r := execTask(t, driver, taskID, + fmt.Sprintf("cat %q", tempfile), + false, "") + require.Zero(t, r.exitCode) + require.Equal(t, text, strings.TrimSpace(r.stdout)) + + // we always run in a cgroup - testing freezer cgroup + r = execTask(t, driver, taskID, + fmt.Sprintf("cat /proc/self/cgroup"), + false, "") + require.Zero(t, r.exitCode) + + if !strings.Contains(r.stdout, ":freezer:/nomad") && !strings.Contains(r.stdout, "freezer:/docker") { + require.Fail(t, "unexpected freezer cgroup", "expected freezer to be /nomad/ or /docker/, but found:\n%s", r.stdout) + + } + }) +} + +func execTask(t *testing.T, driver *DriverHarness, taskID string, cmd string, tty bool, stdin string) execResult { + stream := newTestExecStream(t, tty, stdin) + + ctx, cancelFn := context.WithTimeout(context.Background(), 30*time.Second) + defer cancelFn() + + command := []string{"/bin/sh", "-c", cmd} + + isRaw := false + exitCode := -2 + if raw, ok := driver.impl.(drivers.ExecTaskStreamingRawDriver); ok { + isRaw = true + err := raw.ExecTaskStreamingRaw(ctx, taskID, + command, tty, stream) + require.NoError(t, err) + } else if d, ok := driver.impl.(drivers.ExecTaskStreamingDriver); ok { + execOpts, errCh := drivers.StreamToExecOptions(ctx, command, tty, stream) + + r, err := d.ExecTaskStreaming(ctx, taskID, execOpts) + require.NoError(t, err) + + select { + case err := <-errCh: + require.NoError(t, err) + default: + // all good + } + + exitCode = r.ExitCode + } else { + require.Fail(t, "driver does not support exec") + } + + result := stream.currentResult() + require.NoError(t, result.err) + + if !isRaw { + result.exitCode = exitCode + } + + return result +} + +type execResult struct { + exitCode int + stdout string + stderr string + + err error +} + +func newTestExecStream(t *testing.T, tty bool, stdin string) *testExecStream { + + return &testExecStream{ + t: t, + input: newInputStream(tty, stdin), + result: &execResult{exitCode: -2}, + } +} + +func newInputStream(tty bool, stdin string) []*drivers.ExecTaskStreamingRequestMsg { + input := []*drivers.ExecTaskStreamingRequestMsg{} + if tty { + // emit two resize to ensure we honor latest + input = append(input, &drivers.ExecTaskStreamingRequestMsg{ + TtySize: &dproto.ExecTaskStreamingRequest_TerminalSize{ + Height: 50, + Width: 40, + }}) + input = append(input, &drivers.ExecTaskStreamingRequestMsg{ + TtySize: &dproto.ExecTaskStreamingRequest_TerminalSize{ + Height: 100, + Width: 100, + }}) + + } + + input = append(input, &drivers.ExecTaskStreamingRequestMsg{ + Stdin: &dproto.ExecTaskStreamingIOOperation{ + Data: []byte(stdin), + }, + }) + + if !tty { + // don't close stream in interactive session and risk closing tty prematurely + input = append(input, &drivers.ExecTaskStreamingRequestMsg{ + Stdin: &dproto.ExecTaskStreamingIOOperation{ + Close: true, + }, + }) + } + + return input +} + +var _ drivers.ExecTaskStream = (*testExecStream)(nil) + +type testExecStream struct { + t *testing.T + + // input + input []*drivers.ExecTaskStreamingRequestMsg + recvCalled int + + // result so far + resultLock sync.Mutex + result *execResult +} + +func (s *testExecStream) currentResult() execResult { + s.resultLock.Lock() + defer s.resultLock.Unlock() + + // make a copy + return *s.result +} + +func (s *testExecStream) Recv() (*drivers.ExecTaskStreamingRequestMsg, error) { + if s.recvCalled >= len(s.input) { + return nil, io.EOF + } + + i := s.input[s.recvCalled] + s.recvCalled++ + return i, nil +} + +func (s *testExecStream) Send(m *drivers.ExecTaskStreamingResponseMsg) error { + s.resultLock.Lock() + defer s.resultLock.Unlock() + + switch { + case m.Stdout != nil && m.Stdout.Data != nil: + s.t.Logf("received stdout: %s", string(m.Stdout.Data)) + s.result.stdout += string(m.Stdout.Data) + case m.Stderr != nil && m.Stderr.Data != nil: + s.t.Logf("received stderr: %s", string(m.Stderr.Data)) + s.result.stderr += string(m.Stderr.Data) + case m.Exited && m.Result != nil: + s.result.exitCode = int(m.Result.ExitCode) + } + + return nil +} diff --git a/plugins/drivers/testutils/testing.go b/plugins/drivers/testutils/testing.go index a77b4ede4..f28c3d4fa 100644 --- a/plugins/drivers/testutils/testing.go +++ b/plugins/drivers/testutils/testing.go @@ -181,19 +181,20 @@ func (h *DriverHarness) WaitUntilStarted(taskID string, timeout time.Duration) e // is passed through the base plugin layer. type MockDriver struct { base.MockPlugin - TaskConfigSchemaF func() (*hclspec.Spec, error) - FingerprintF func(context.Context) (<-chan *drivers.Fingerprint, error) - CapabilitiesF func() (*drivers.Capabilities, error) - RecoverTaskF func(*drivers.TaskHandle) error - StartTaskF func(*drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) - WaitTaskF func(context.Context, string) (<-chan *drivers.ExitResult, error) - StopTaskF func(string, time.Duration, string) error - DestroyTaskF func(string, bool) error - InspectTaskF func(string) (*drivers.TaskStatus, error) - TaskStatsF func(context.Context, string, time.Duration) (<-chan *drivers.TaskResourceUsage, error) - TaskEventsF func(context.Context) (<-chan *drivers.TaskEvent, error) - SignalTaskF func(string, string) error - ExecTaskF func(string, []string, time.Duration) (*drivers.ExecTaskResult, error) + TaskConfigSchemaF func() (*hclspec.Spec, error) + FingerprintF func(context.Context) (<-chan *drivers.Fingerprint, error) + CapabilitiesF func() (*drivers.Capabilities, error) + RecoverTaskF func(*drivers.TaskHandle) error + StartTaskF func(*drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) + WaitTaskF func(context.Context, string) (<-chan *drivers.ExitResult, error) + StopTaskF func(string, time.Duration, string) error + DestroyTaskF func(string, bool) error + InspectTaskF func(string) (*drivers.TaskStatus, error) + TaskStatsF func(context.Context, string, time.Duration) (<-chan *drivers.TaskResourceUsage, error) + TaskEventsF func(context.Context) (<-chan *drivers.TaskEvent, error) + SignalTaskF func(string, string) error + ExecTaskF func(string, []string, time.Duration) (*drivers.ExecTaskResult, error) + ExecTaskStreamingF func(context.Context, string, *drivers.ExecOptions) (*drivers.ExitResult, error) } func (d *MockDriver) TaskConfigSchema() (*hclspec.Spec, error) { return d.TaskConfigSchemaF() } @@ -230,6 +231,10 @@ func (d *MockDriver) ExecTask(taskID string, cmd []string, timeout time.Duration return d.ExecTaskF(taskID, cmd, timeout) } +func (d *MockDriver) ExecTaskStreaming(ctx context.Context, taskID string, execOpts *drivers.ExecOptions) (*drivers.ExitResult, error) { + return d.ExecTaskStreamingF(ctx, taskID, execOpts) +} + // SetEnvvars sets path and host env vars depending on the FS isolation used. func SetEnvvars(envBuilder *taskenv.Builder, fsi drivers.FSIsolation, taskDir *allocdir.TaskDir, conf *config.Config) { // Set driver-specific environment variables From 67160a6302f6c48f0946b722000b6db58aa393b8 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:26:15 -0400 Subject: [PATCH 13/15] drivers/docker: implement streaming exec --- drivers/docker/docklog/docker_logger.go | 6 +- drivers/docker/driver.go | 89 +++++++++++++++++++++++++ drivers/docker/driver_unix_test.go | 30 +++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) diff --git a/drivers/docker/docklog/docker_logger.go b/drivers/docker/docklog/docker_logger.go index e09c371fc..730d3e288 100644 --- a/drivers/docker/docklog/docker_logger.go +++ b/drivers/docker/docklog/docker_logger.go @@ -98,7 +98,11 @@ func (d *dockerLogger) Start(opts *StartOpts) error { Follow: true, Stdout: true, Stderr: true, - RawTerminal: opts.TTY, + + // When running in TTY, we must use a raw terminal. + // If not, we set RawTerminal to false to allow docker client + // to interpret special stdout/stderr messages + RawTerminal: opts.TTY, } err := client.Logs(logOpts) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 93232315f..3754d032b 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -1215,6 +1215,95 @@ func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (* return h.Exec(ctx, cmd[0], cmd[1:]) } +var _ drivers.ExecTaskStreamingDriver = (*Driver)(nil) + +func (d *Driver) ExecTaskStreaming(ctx context.Context, taskID string, opts *drivers.ExecOptions) (*drivers.ExitResult, error) { + defer opts.Stdout.Close() + defer opts.Stderr.Close() + + done := make(chan interface{}) + defer close(done) + + h, ok := d.tasks.Get(taskID) + if !ok { + return nil, drivers.ErrTaskNotFound + } + + if len(opts.Command) == 0 { + return nil, fmt.Errorf("command is required but was empty") + } + + createExecOpts := docker.CreateExecOptions{ + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + Tty: opts.Tty, + Cmd: opts.Command, + Container: h.containerID, + Context: ctx, + } + exec, err := h.client.CreateExec(createExecOpts) + if err != nil { + return nil, fmt.Errorf("failed to create exec object: %v", err) + } + + go func() { + for { + select { + case <-ctx.Done(): + return + case <-done: + return + case s, ok := <-opts.ResizeCh: + if !ok { + return + } + client.ResizeExecTTY(exec.ID, s.Height, s.Width) + } + } + }() + + startOpts := docker.StartExecOptions{ + Detach: false, + + // When running in TTY, we must use a raw terminal. + // If not, we set RawTerminal to false to allow docker client + // to interpret special stdout/stderr messages + Tty: opts.Tty, + RawTerminal: opts.Tty, + + InputStream: opts.Stdin, + OutputStream: opts.Stdout, + ErrorStream: opts.Stderr, + Context: ctx, + } + if err := client.StartExec(exec.ID, startOpts); err != nil { + return nil, fmt.Errorf("failed to start exec: %v", err) + } + + // StartExec returns after process completes, but InspectExec seems to have a delay + // get in getting status code + + const execTerminatingTimeout = 3 * time.Second + start := time.Now() + var res *docker.ExecInspect + for res == nil || res.Running || time.Since(start) > execTerminatingTimeout { + res, err = client.InspectExec(exec.ID) + if err != nil { + return nil, fmt.Errorf("failed to inspect exec result: %v", err) + } + time.Sleep(50 * time.Millisecond) + } + + if res == nil || res.Running { + return nil, fmt.Errorf("failed to retrieve exec result") + } + + return &drivers.ExitResult{ + ExitCode: res.ExitCode, + }, nil +} + // dockerClients creates two *docker.Client, one for long running operations and // the other for shorter operations. In test / dev mode we can use ENV vars to // connect to the docker daemon. In production mode we will read docker.endpoint diff --git a/drivers/docker/driver_unix_test.go b/drivers/docker/driver_unix_test.go index bac2a0e92..9faa95326 100644 --- a/drivers/docker/driver_unix_test.go +++ b/drivers/docker/driver_unix_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/plugins/drivers" + dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" tu "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -754,3 +755,32 @@ func copyFile(src, dst string, t *testing.T) { t.Fatalf("copying %v -> %v failed: %v", src, dst, err) } } + +func TestDocker_ExecTaskStreaming(t *testing.T) { + if !tu.IsCI() { + t.Parallel() + } + testutil.DockerCompatible(t) + + taskCfg := newTaskConfig("", []string{"/bin/sleep", "1000"}) + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "nc-demo", + AllocID: uuid.Generate(), + Resources: basicResources, + } + require.NoError(t, task.EncodeConcreteDriverConfig(&taskCfg)) + + d := dockerDriverHarness(t, nil) + cleanup := d.MkAllocDir(task, true) + defer cleanup() + copyImage(t, task.TaskDir(), "busybox.tar") + + _, _, err := d.StartTask(task) + require.NoError(t, err) + + defer d.DestroyTask(task.ID, true) + + dtestutil.ExecTaskStreamingConformanceTests(t, d, task.ID) + +} From 6168cc59be65d1e0d6bc2fe0486902960c4894a6 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:34:01 -0400 Subject: [PATCH 14/15] add api support for nomad exec Adds nomad exec support in our API, by hitting the websocket endpoint. We introduce API structs that correspond to the drivers streaming exec structs. For creating the websocket connection, we reuse the transport setting from api http client. --- api/allocations.go | 264 ++++++++++++++++++ api/api.go | 58 ++++ .../testing/apitests/streamingsync_test.go | 100 +++++++ 3 files changed, 422 insertions(+) create mode 100644 internal/testing/apitests/streamingsync_test.go diff --git a/api/allocations.go b/api/allocations.go index d9ea7063d..fde558e11 100644 --- a/api/allocations.go +++ b/api/allocations.go @@ -1,9 +1,18 @@ package api import ( + "context" + "encoding/json" + "errors" "fmt" + "io" + "net" "sort" + "strconv" + "sync" "time" + + "github.com/gorilla/websocket" ) var ( @@ -61,6 +70,222 @@ func (a *Allocations) Info(allocID string, q *QueryOptions) (*Allocation, *Query return &resp, qm, nil } +// Exec is used to execute a command inside a running task. The command is to run inside +// the task environment. +// +// The parameters are: +// * ctx: context to set deadlines or timeout +// * allocation: the allocation to execute command inside +// * task: the task's name to execute command in +// * tty: indicates whether to start a pseudo-tty for the command +// * stdin, stdout, stderr: the std io to pass to command. +// If tty is true, then streams need to point to a tty that's alive for the whole process +// * terminalSizeCh: A channel to send new tty terminal sizes +// +// The call blocks until command terminates (or an error occurs), and returns the exit code. +func (a *Allocations) Exec(ctx context.Context, + alloc *Allocation, task string, tty bool, command []string, + stdin io.Reader, stdout, stderr io.Writer, + terminalSizeCh <-chan TerminalSize, q *QueryOptions) (exitCode int, err error) { + + ctx, cancelFn := context.WithCancel(ctx) + defer cancelFn() + + errCh := make(chan error, 4) + + sender, output := a.execFrames(ctx, alloc, task, tty, command, errCh, q) + + select { + case err := <-errCh: + return -2, err + default: + } + + // Errors resulting from sending input (in goroutines) are silently dropped. + // To mitigate this, extra care is needed to distinguish between actual send errors + // and from send errors due to command terminating and our race to detect failures. + // If we have an actual network failure or send a bad input, we'd get an + // error in the reading side of websocket. + + go func() { + + bytes := make([]byte, 2048) + for { + if ctx.Err() != nil { + return + } + + input := ExecStreamingInput{Stdin: &ExecStreamingIOOperation{}} + + n, err := stdin.Read(bytes) + + // always send data if we read some + if n != 0 { + input.Stdin.Data = bytes[:n] + sender(&input) + } + + // then handle error + if err == io.EOF { + // if n != 0, send data and we'll get n = 0 on next read + if n == 0 { + input.Stdin.Close = true + sender(&input) + return + } + } else if err != nil { + errCh <- err + return + } + } + }() + + // forwarding terminal size + go func() { + for { + resizeInput := ExecStreamingInput{} + + select { + case <-ctx.Done(): + return + case size, ok := <-terminalSizeCh: + if !ok { + return + } + resizeInput.TTYSize = &size + sender(&resizeInput) + } + + } + }() + + // send a heartbeat every 10 seconds + go func() { + for { + select { + case <-ctx.Done(): + return + // heartbeat message + case <-time.After(10 * time.Second): + sender(&execStreamingInputHeartbeat) + } + + } + }() + + for { + select { + case err := <-errCh: + // drop websocket code, not relevant to user + if wsErr, ok := err.(*websocket.CloseError); ok && wsErr.Text != "" { + return -2, errors.New(wsErr.Text) + } + return -2, err + case <-ctx.Done(): + return -2, ctx.Err() + case frame, ok := <-output: + if !ok { + return -2, errors.New("disconnected without receiving the exit code") + } + + switch { + case frame.Stdout != nil: + if len(frame.Stdout.Data) != 0 { + stdout.Write(frame.Stdout.Data) + } + // don't really do anything if stdout is closing + case frame.Stderr != nil: + if len(frame.Stderr.Data) != 0 { + stderr.Write(frame.Stderr.Data) + } + // don't really do anything if stderr is closing + case frame.Exited && frame.Result != nil: + return frame.Result.ExitCode, nil + default: + // noop - heartbeat + } + } + } +} + +func (a *Allocations) execFrames(ctx context.Context, alloc *Allocation, task string, tty bool, command []string, + errCh chan<- error, q *QueryOptions) (sendFn func(*ExecStreamingInput) error, output <-chan *ExecStreamingOutput) { + + nodeClient, err := a.client.GetNodeClientWithTimeout(alloc.NodeID, ClientConnTimeout, q) + if err != nil { + errCh <- err + return nil, nil + } + + if q == nil { + q = &QueryOptions{} + } + if q.Params == nil { + q.Params = make(map[string]string) + } + + commandBytes, err := json.Marshal(command) + if err != nil { + errCh <- fmt.Errorf("failed to marshal command: %s", err) + return nil, nil + } + + q.Params["tty"] = strconv.FormatBool(tty) + q.Params["task"] = task + q.Params["command"] = string(commandBytes) + + reqPath := fmt.Sprintf("/v1/client/allocation/%s/exec", alloc.ID) + + conn, _, err := nodeClient.websocket(reqPath, q) + if err != nil { + // There was an error talking directly to the client. Non-network + // errors are fatal, but network errors can attempt to route via RPC. + if _, ok := err.(net.Error); !ok { + errCh <- err + return nil, nil + } + + conn, _, err = a.client.websocket(reqPath, q) + if err != nil { + errCh <- err + return nil, nil + } + } + + // Create the output channel + frames := make(chan *ExecStreamingOutput, 10) + + go func() { + defer conn.Close() + for ctx.Err() == nil { + + // Decode the next frame + var frame ExecStreamingOutput + err := conn.ReadJSON(&frame) + if websocket.IsCloseError(err, websocket.CloseNormalClosure) { + close(frames) + return + } else if err != nil { + errCh <- err + return + } + + frames <- &frame + } + }() + + var sendLock sync.Mutex + send := func(v *ExecStreamingInput) error { + sendLock.Lock() + defer sendLock.Unlock() + + return conn.WriteJSON(v) + } + + return send, frames + +} + func (a *Allocations) Stats(alloc *Allocation, q *QueryOptions) (*AllocResourceUsage, error) { var resp AllocResourceUsage path := fmt.Sprintf("/v1/client/allocation/%s/stats", alloc.ID) @@ -339,3 +564,42 @@ type DesiredTransition struct { func (d DesiredTransition) ShouldMigrate() bool { return d.Migrate != nil && *d.Migrate } + +// ExecStreamingIOOperation represents a stream write operation: either appending data or close (exclusively) +type ExecStreamingIOOperation struct { + Data []byte `json:"data,omitempty"` + Close bool `json:"close,omitempty"` +} + +// TerminalSize represents the size of the terminal +type TerminalSize struct { + Height int `json:"height,omitempty"` + Width int `json:"width,omitempty"` +} + +var execStreamingInputHeartbeat = ExecStreamingInput{} + +// ExecStreamingInput represents user input to be sent to nomad exec handler. +// +// At most one field should be set. +type ExecStreamingInput struct { + Stdin *ExecStreamingIOOperation `json:"stdin,omitempty"` + TTYSize *TerminalSize `json:"tty_size,omitempty"` +} + +// ExecStreamingExitResults captures the exit code of just completed nomad exec command +type ExecStreamingExitResult struct { + ExitCode int `json:"exit_code"` +} + +// ExecStreamingInput represents an output streaming entity, e.g. stdout/stderr update or termination +// +// At most one of these fields should be set: `Stdout`, `Stderr`, or `Result`. +// If `Exited` is true, then `Result` is non-nil, and other fields are nil. +type ExecStreamingOutput struct { + Stdout *ExecStreamingIOOperation `json:"stdout,omitempty"` + Stderr *ExecStreamingIOOperation `json:"stderr,omitempty"` + + Exited bool `json:"exited,omitempty"` + Result *ExecStreamingExitResult `json:"result,omitempty"` +} diff --git a/api/api.go b/api/api.go index 03ede6e5f..43f505097 100644 --- a/api/api.go +++ b/api/api.go @@ -15,6 +15,7 @@ import ( "strings" "time" + "github.com/gorilla/websocket" cleanhttp "github.com/hashicorp/go-cleanhttp" rootcerts "github.com/hashicorp/go-rootcerts" ) @@ -655,6 +656,63 @@ func (c *Client) rawQuery(endpoint string, q *QueryOptions) (io.ReadCloser, erro return resp.Body, nil } +// websocket makes a websocket request to the specific endpoint +func (c *Client) websocket(endpoint string, q *QueryOptions) (*websocket.Conn, *http.Response, error) { + + transport, ok := c.config.httpClient.Transport.(*http.Transport) + if !ok { + return nil, nil, fmt.Errorf("unsupported transport") + } + dialer := websocket.Dialer{ + ReadBufferSize: 4096, + WriteBufferSize: 4096, + HandshakeTimeout: c.config.httpClient.Timeout, + + // values to inherit from http client configuration + NetDial: transport.Dial, + NetDialContext: transport.DialContext, + Proxy: transport.Proxy, + TLSClientConfig: transport.TLSClientConfig, + } + + // build request object for header and parameters + r, err := c.newRequest("GET", endpoint) + if err != nil { + return nil, nil, err + } + r.setQueryOptions(q) + + rhttp, err := r.toHTTP() + if err != nil { + return nil, nil, err + } + + // convert scheme + wsScheme := "" + switch rhttp.URL.Scheme { + case "http": + wsScheme = "ws" + case "https": + wsScheme = "wss" + default: + return nil, nil, fmt.Errorf("unsupported scheme: %v", rhttp.URL.Scheme) + } + rhttp.URL.Scheme = wsScheme + + conn, resp, err := dialer.Dial(rhttp.URL.String(), rhttp.Header) + + // check resp status code, as it's more informative than handshake error we get from ws library + if resp != nil && resp.StatusCode != 101 { + var buf bytes.Buffer + io.Copy(&buf, resp.Body) + resp.Body.Close() + + return nil, nil, fmt.Errorf("Unexpected response code: %d (%s)", resp.StatusCode, buf.Bytes()) + } + + return conn, resp, err +} + // query is used to do a GET request against an endpoint // and deserialize the response into an interface using // standard Nomad conventions. diff --git a/internal/testing/apitests/streamingsync_test.go b/internal/testing/apitests/streamingsync_test.go new file mode 100644 index 000000000..e17853331 --- /dev/null +++ b/internal/testing/apitests/streamingsync_test.go @@ -0,0 +1,100 @@ +package apitests + +import ( + "encoding/json" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/stretchr/testify/require" +) + +// TestExecStreamingInputIsInSync asserts that a rountrip of exec streaming input doesn't lose any data +func TestExecStreamingInputIsInSync(t *testing.T) { + cases := []struct { + name string + input api.ExecStreamingInput + }{ + { + "stdin_data", + api.ExecStreamingInput{Stdin: &api.ExecStreamingIOOperation{Data: []byte("hello there")}}, + }, + { + "stdin_close", + api.ExecStreamingInput{Stdin: &api.ExecStreamingIOOperation{Close: true}}, + }, + { + "tty_size", + api.ExecStreamingInput{TTYSize: &api.TerminalSize{Height: 10, Width: 20}}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b, err := json.Marshal(c.input) + require.NoError(t, err) + + var proto drivers.ExecTaskStreamingRequestMsg + err = json.Unmarshal(b, &proto) + require.NoError(t, err) + + protoB, err := json.Marshal(proto) + require.NoError(t, err) + + var roundtrip api.ExecStreamingInput + err = json.Unmarshal(protoB, &roundtrip) + require.NoError(t, err) + + require.EqualValues(t, c.input, roundtrip) + }) + } +} + +// TestExecStreamingOutputIsInSync asserts that a rountrip of exec streaming input doesn't lose any data +func TestExecStreamingOutputIsInSync(t *testing.T) { + cases := []struct { + name string + input api.ExecStreamingOutput + }{ + { + "stdout_data", + api.ExecStreamingOutput{Stdout: &api.ExecStreamingIOOperation{Data: []byte("hello there")}}, + }, + { + "stdout_close", + api.ExecStreamingOutput{Stdout: &api.ExecStreamingIOOperation{Close: true}}, + }, + { + "stderr_data", + api.ExecStreamingOutput{Stderr: &api.ExecStreamingIOOperation{Data: []byte("hello there")}}, + }, + { + "stderr_close", + api.ExecStreamingOutput{Stderr: &api.ExecStreamingIOOperation{Close: true}}, + }, + { + "exited", + api.ExecStreamingOutput{Exited: true, Result: &api.ExecStreamingExitResult{ExitCode: 21}}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b, err := json.Marshal(c.input) + require.NoError(t, err) + + var proto drivers.ExecTaskStreamingResponseMsg + err = json.Unmarshal(b, &proto) + require.NoError(t, err) + + protoB, err := json.Marshal(proto) + require.NoError(t, err) + + var roundtrip api.ExecStreamingOutput + err = json.Unmarshal(protoB, &roundtrip) + require.NoError(t, err) + + require.EqualValues(t, c.input, roundtrip) + }) + } +} From 980e4f586618a4895988fc18d481406af2ffc62d Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Sun, 28 Apr 2019 17:35:42 -0400 Subject: [PATCH 15/15] add e2e tests for nomad exec --- e2e/e2e_test.go | 1 + e2e/nomadexec/exec.go | 144 ++++++++++++++++++++++++++++ e2e/nomadexec/testdata/docker.nomad | 20 ++++ 3 files changed, 165 insertions(+) create mode 100644 e2e/nomadexec/exec.go create mode 100644 e2e/nomadexec/testdata/docker.nomad diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 8e1b58ee5..264832340 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -10,6 +10,7 @@ import ( _ "github.com/hashicorp/nomad/e2e/consultemplate" _ "github.com/hashicorp/nomad/e2e/example" _ "github.com/hashicorp/nomad/e2e/nomad09upgrade" + _ "github.com/hashicorp/nomad/e2e/nomadexec" _ "github.com/hashicorp/nomad/e2e/spread" _ "github.com/hashicorp/nomad/e2e/taskevents" ) diff --git a/e2e/nomadexec/exec.go b/e2e/nomadexec/exec.go new file mode 100644 index 000000000..6877f1b3a --- /dev/null +++ b/e2e/nomadexec/exec.go @@ -0,0 +1,144 @@ +package nomadexec + +import ( + "bytes" + "context" + "fmt" + "io" + "reflect" + "regexp" + "testing" + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper/uuid" + dtestutils "github.com/hashicorp/nomad/plugins/drivers/testutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type NomadExecE2ETest struct { + framework.TC + + name string + jobFilePath string + + jobID string + alloc api.Allocation +} + +func init() { + framework.AddSuites(&framework.TestSuite{ + Component: "Nomad exec", + CanRunLocal: true, + Cases: []framework.TestCase{ + newNomadExecE2eTest("docker", "./nomadexec/testdata/docker.nomad"), + }, + }) +} + +func newNomadExecE2eTest(name, jobFilePath string) *NomadExecE2ETest { + return &NomadExecE2ETest{ + name: name, + jobFilePath: jobFilePath, + } +} + +func (tc *NomadExecE2ETest) Name() string { + return fmt.Sprintf("%v (%v)", tc.TC.Name(), tc.name) +} + +func (tc *NomadExecE2ETest) BeforeAll(f *framework.F) { + // Ensure cluster has leader before running tests + e2eutil.WaitForLeader(f.T(), tc.Nomad()) + e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) + + // register a job for execing into + tc.jobID = "nomad-exec" + uuid.Generate()[:8] + allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), tc.Nomad(), tc.jobFilePath, tc.jobID) + f.Len(allocs, 1) + + e2eutil.WaitForAllocRunning(f.T(), tc.Nomad(), allocs[0].ID) + + tc.alloc = api.Allocation{ + ID: allocs[0].ID, + Namespace: allocs[0].Namespace, + NodeID: allocs[0].NodeID, + } +} + +func (tc *NomadExecE2ETest) TestExecBasicResponses(f *framework.F) { + for _, c := range dtestutils.ExecTaskStreamingBasicCases { + f.T().Run(c.Name, func(t *testing.T) { + + stdin := newTestStdin(c.Tty, c.Stdin) + var stdout, stderr bytes.Buffer + + resizeCh := make(chan api.TerminalSize) + go func() { + resizeCh <- api.TerminalSize{Height: 100, Width: 100} + }() + + ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second) + defer cancelFn() + + exitCode, err := tc.Nomad().Allocations().Exec(ctx, + &tc.alloc, "task", c.Tty, + []string{"/bin/sh", "-c", c.Command}, + stdin, &stdout, &stderr, + resizeCh, nil) + + require.NoError(t, err) + + assert.Equal(t, c.ExitCode, exitCode) + + switch s := c.Stdout.(type) { + case string: + require.Equal(t, s, stdout.String()) + case *regexp.Regexp: + require.Regexp(t, s, stdout.String()) + case nil: + require.Empty(t, stdout.String()) + default: + require.Fail(t, "unexpected stdout type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) + } + + switch s := c.Stderr.(type) { + case string: + require.Equal(t, s, stderr.String()) + case *regexp.Regexp: + require.Regexp(t, s, stderr.String()) + case nil: + require.Empty(t, stderr.String()) + default: + require.Fail(t, "unexpected stderr type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) + } + }) + } +} + +func (tc *NomadExecE2ETest) AfterAll(f *framework.F) { + jobs := tc.Nomad().Jobs() + if tc.jobID != "" { + jobs.Deregister(tc.jobID, true, nil) + } + tc.Nomad().System().GarbageCollect() +} + +func newTestStdin(tty bool, d string) io.Reader { + pr, pw := io.Pipe() + go func() { + pw.Write([]byte(d)) + + // when testing TTY, leave connection open for the entire duration of command + // closing stdin may cause TTY session prematurely before command completes + if !tty { + pw.Close() + } + + }() + + return pr +} diff --git a/e2e/nomadexec/testdata/docker.nomad b/e2e/nomadexec/testdata/docker.nomad new file mode 100644 index 000000000..acf3dc9c9 --- /dev/null +++ b/e2e/nomadexec/testdata/docker.nomad @@ -0,0 +1,20 @@ +job "nomadexec-docker" { + datacenters = ["dc1"] + + group "group" { + task "task" { + driver = "docker" + + config { + image = "busybox:1.29.2" + command = "/bin/sleep" + args = ["1000"] + } + + resources { + cpu = 500 + memory = 256 + } + } + } +} \ No newline at end of file