From 57f40c7e3e545cb592762172515d6923c27bae74 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Fri, 28 Sep 2018 10:09:01 -0700 Subject: [PATCH 1/7] Device manager Introduce a device manager that manages the lifecycle of device plugins on the client. It fingerprints, collects stats, and forwards Reserve requests to the correct plugin. The manager, also handles device plugins failing and validates their output. --- GNUmakefile | 10 +- client/client.go | 50 +- client/devicemanager/instance.go | 518 ++++++++++++++++++ client/devicemanager/manager.go | 302 ++++++++++ client/devicemanager/utils.go | 102 ++++ client/logmon/proto/logmon.pb.go | 62 ++- client/state/db_test.go | 26 + client/state/interface.go | 9 + client/state/memdb.go | 19 + client/state/noopdb.go | 9 + client/state/state_database.go | 61 ++- command/agent/command.go | 56 +- devices/gpu/nvidia/README.md | 2 - devices/gpu/nvidia/device.go | 19 +- devices/gpu/nvidia/fingerprint.go | 6 +- devices/gpu/nvidia/stats.go | 4 +- .../docker/docklog/proto/docker_logger.pb.go | 63 ++- drivers/exec/driver.go | 7 +- drivers/qemu/driver.go | 10 +- drivers/rawexec/driver.go | 7 +- drivers/rkt/driver.go | 7 +- plugins/base/plugin.go | 4 + plugins/base/proto/base.pb.go | 93 ++-- plugins/base/server.go | 3 +- plugins/device/client.go | 74 ++- plugins/device/cmd/example/README.md | 2 - plugins/device/cmd/example/device.go | 31 +- plugins/device/device.go | 51 +- plugins/device/mock.go | 7 +- plugins/device/plugin.go | 3 +- plugins/device/plugin_test.go | 15 +- plugins/device/proto/device.pb.go | 202 +++---- plugins/device/proto/device.proto | 6 +- plugins/device/server.go | 19 +- plugins/drivers/proto/driver.pb.go | 454 +++++++-------- plugins/shared/cmd/launcher/command/device.go | 3 +- plugins/shared/hclspec/hcl_spec.pb.go | 112 ++-- plugins/shared/loader/init.go | 5 +- plugins/shared/loader/loader.go | 7 + plugins/shared/loader/plugin_test.go | 3 +- .../plugin_reattach_config.go | 10 +- plugins/shared/singleton/singleton.go | 9 +- plugins/shared/structs/proto/attribute.pb.go | 39 +- plugins/shared/util.go | 5 +- vendor/github.com/LK4D4/joincontext/LICENSE | 21 + vendor/github.com/LK4D4/joincontext/README.md | 39 ++ .../github.com/LK4D4/joincontext/context.go | 102 ++++ vendor/vendor.json | 1 + 48 files changed, 2048 insertions(+), 621 deletions(-) create mode 100644 client/devicemanager/instance.go create mode 100644 client/devicemanager/manager.go create mode 100644 client/devicemanager/utils.go rename plugins/{drivers/utils => shared}/plugin_reattach_config.go (91%) create mode 100644 vendor/github.com/LK4D4/joincontext/LICENSE create mode 100644 vendor/github.com/LK4D4/joincontext/README.md create mode 100644 vendor/github.com/LK4D4/joincontext/context.go diff --git a/GNUmakefile b/GNUmakefile index d337b46da..3ff8a5664 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -191,10 +191,18 @@ checkscripts: ## Lint shell scripts @echo "==> Linting scripts..." @shellcheck ./scripts/* +.PHONY: generate generate: LOCAL_PACKAGES = $(shell go list ./... | grep -v '/vendor/') -generate: ## Update generated code +generate: proto ## Update generated code @go generate $(LOCAL_PACKAGES) +.PHONY: proto +proto: + @for file in $$(git ls-files "*.proto" | grep -v "vendor\/.*.proto"); do \ + protoc -I . -I ../../.. --go_out=plugins=grpc:. $$file; \ + done + + vendorfmt: @echo "--> Formatting vendor/vendor.json" test -x $(GOPATH)/bin/vendorfmt || go get -u github.com/magiconair/vendorfmt/cmd/vendorfmt diff --git a/client/client.go b/client/client.go index f4a3810ac..c7898718f 100644 --- a/client/client.go +++ b/client/client.go @@ -1,6 +1,7 @@ package client import ( + "context" "errors" "fmt" "io/ioutil" @@ -16,31 +17,32 @@ import ( metrics "github.com/armon/go-metrics" consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/lib" hclog "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" - "github.com/hashicorp/nomad/client/allocrunner/interfaces" - arstate "github.com/hashicorp/nomad/client/allocrunner/state" - consulApi "github.com/hashicorp/nomad/client/consul" - cstructs "github.com/hashicorp/nomad/client/structs" - hstats "github.com/hashicorp/nomad/helper/stats" - nconfig "github.com/hashicorp/nomad/nomad/structs/config" - vaultapi "github.com/hashicorp/vault/api" - - "github.com/hashicorp/consul/lib" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/allocrunner" + "github.com/hashicorp/nomad/client/allocrunner/interfaces" + arstate "github.com/hashicorp/nomad/client/allocrunner/state" "github.com/hashicorp/nomad/client/allocwatcher" "github.com/hashicorp/nomad/client/config" + consulApi "github.com/hashicorp/nomad/client/consul" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/client/servers" "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/stats" + cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/pool" + hstats "github.com/hashicorp/nomad/helper/stats" "github.com/hashicorp/nomad/helper/tlsutil" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" + nconfig "github.com/hashicorp/nomad/nomad/structs/config" + vaultapi "github.com/hashicorp/vault/api" + "github.com/kr/pretty" "github.com/shirou/gopsutil/host" ) @@ -190,6 +192,10 @@ type Client struct { shutdownCh chan struct{} shutdownLock sync.Mutex + // ctx is cancelled at the same time as the shutdownCh + ctx context.Context + ctxCancel context.CancelFunc + // vaultClient is used to interact with Vault for token and secret renewals vaultClient vaultclient.VaultClient @@ -205,6 +211,9 @@ type Client struct { endpoints rpcEndpoints streamingRpcs *structs.StreamingRpcRegistry + // devicemanger is responsible for managing device plugins. + devicemanager devicemanager.Manager + // baseLabels are used when emitting tagged metrics. All client metrics will // have these tags, and optionally more. baseLabels []metrics.Label @@ -235,6 +244,9 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic // Create the logger logger := cfg.Logger.ResetNamed("client") + // Create a context + ctx, cancel := context.WithCancel(context.Background()) + // Create the client c := &Client{ config: cfg, @@ -249,6 +261,8 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic allocs: make(map[string]AllocRunner), allocUpdates: make(chan *structs.Allocation, 64), shutdownCh: make(chan struct{}), + ctx: ctx, + ctxCancel: cancel, triggerDiscoveryCh: make(chan struct{}), triggerNodeUpdate: make(chan struct{}, 8), triggerEmitNodeEvent: make(chan *structs.NodeEvent, 8), @@ -306,6 +320,20 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic return nil, fmt.Errorf("fingerprinting failed: %v", err) } + // Setup the device manager + devConfig := &devicemanager.Config{ + Logger: c.logger, + Loader: c.configCopy.PluginSingletonLoader, + PluginConfig: c.configCopy.NomadPluginConfig(), + Updater: func(devs []*structs.NodeDeviceResource) { + c.logger.Debug("Device Updater called", "count", len(devs), "devices", hclog.Fmt("% #v", pretty.Formatter(devs))) + }, + StatsInterval: c.configCopy.StatsCollectionInterval, + State: c.stateDB, + } + c.devicemanager = devicemanager.New(devConfig) + go c.devicemanager.Run() + // Set the preconfigured list of static servers c.configLock.RLock() if len(c.configCopy.Servers) > 0 { @@ -533,6 +561,9 @@ func (c *Client) Shutdown() error { } }() + // Shutdown the device manager + c.devicemanager.Shutdown() + // Stop renewing tokens and secrets if c.vaultClient != nil { c.vaultClient.Stop() @@ -553,6 +584,7 @@ func (c *Client) Shutdown() error { c.shutdown = true close(c.shutdownCh) + c.ctxCancel() c.connPool.Shutdown() return nil } diff --git a/client/devicemanager/instance.go b/client/devicemanager/instance.go new file mode 100644 index 000000000..7b255fc7e --- /dev/null +++ b/client/devicemanager/instance.go @@ -0,0 +1,518 @@ +package devicemanager + +import ( + "context" + "fmt" + "sync" + "time" + + log "github.com/hashicorp/go-hclog" + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared/loader" + "github.com/hashicorp/nomad/plugins/shared/singleton" +) + +const ( + // statsBackoffBaseline is the baseline time for exponential backoff while + // collecting device stats. + statsBackoffBaseline = 5 * time.Second + + // statsBackoffLimit is the limit of the exponential backoff for collecting + // device statistics. + statsBackoffLimit = 30 * time.Minute +) + +// instanceManagerConfig configures a device instance manager +type instanceManagerConfig struct { + // Logger is the logger used by the device instance manager + Logger log.Logger + + // Ctx is used to shutdown the device instance manager + Ctx context.Context + + // Loader is the plugin loader + Loader loader.PluginCatalog + + // StoreReattach is used to store a plugins reattach config + StoreReattach StorePluginReattachFn + + // PluginConfig is the config passed to the launched plugins + PluginConfig *base.ClientAgentConfig + + // Id is the ID of the plugin being managed + Id *loader.PluginID + + // FingerprintOutCh is used to emit new fingerprinted devices + FingerprintOutCh chan<- struct{} + + // StatsInterval is the interval at which we collect statistics. + StatsInterval time.Duration +} + +// instanceManager is used to manage a single device plugin +type instanceManager struct { + // logger is the logger used by the device instance manager + logger log.Logger + + // ctx is used to shutdown the device manager + ctx context.Context + + // cancel is used to shutdown management of this device plugin + cancel context.CancelFunc + + // loader is the plugin loader + loader loader.PluginCatalog + + // storeReattach is used to store a plugins reattach config + storeReattach StorePluginReattachFn + + // pluginConfig is the config passed to the launched plugins + pluginConfig *base.ClientAgentConfig + + // id is the ID of the plugin being managed + id *loader.PluginID + + // fingerprintOutCh is used to emit new fingerprinted devices + fingerprintOutCh chan<- struct{} + + // plugin is the plugin instance being managed + plugin loader.PluginInstance + + // device is the device plugin being managed + device device.DevicePlugin + + // pluginLock locks access to the device and plugin + pluginLock sync.Mutex + + // shutdownLock is used to serialize attempts to shutdown + shutdownLock sync.Mutex + + // devices is the set of fingerprinted devices + devices []*device.DeviceGroup + deviceLock sync.RWMutex + + // statsInterval is the interval at which we collect statistics. + statsInterval time.Duration + + // deviceStats is the set of statistics objects per devices + deviceStats []*device.DeviceGroupStats + deviceStatsLock sync.RWMutex + + // firstFingerprintCh is used to trigger that we have successfully + // fingerprinted once. It is used to gate launching the stats collection. + firstFingerprintCh chan struct{} + hasFingerprinted bool + + // wg is used to track the launched goroutines + wg sync.WaitGroup +} + +// newInstanceManager returns a new device instance manager. It is expected that +// the context passed in the configuration is cancelled in order to shutdown +// launched goroutines. +func newInstanceManager(c *instanceManagerConfig) *instanceManager { + + ctx, cancel := context.WithCancel(c.Ctx) + i := &instanceManager{ + logger: c.Logger.With("plugin", c.Id.Name), + ctx: ctx, + cancel: cancel, + loader: c.Loader, + storeReattach: c.StoreReattach, + pluginConfig: c.PluginConfig, + id: c.Id, + fingerprintOutCh: c.FingerprintOutCh, + statsInterval: c.StatsInterval, + firstFingerprintCh: make(chan struct{}), + } + + go i.run() + return i +} + +// HasDevices returns if the instance is managing the passed devices +func (i *instanceManager) HasDevices(d *structs.AllocatedDeviceResource) bool { + i.deviceLock.Lock() + defer i.deviceLock.Unlock() + +OUTER: + for _, dev := range i.devices { + if dev.Name != d.Name || dev.Type != d.Type || dev.Vendor != d.Vendor { + continue + } + + // Check that we have all the requested devices + ids := make(map[string]struct{}, len(dev.Devices)) + for _, inst := range dev.Devices { + ids[inst.ID] = struct{}{} + } + + for _, reqID := range d.DeviceIDs { + if _, ok := ids[reqID]; !ok { + continue OUTER + } + } + + return true + } + + return false +} + +// AllStats returns all the device statistics returned by the device plugin. +func (i *instanceManager) AllStats() []*device.DeviceGroupStats { + i.deviceStatsLock.Lock() + defer i.deviceStatsLock.Unlock() + return i.deviceStats +} + +// DeviceStats returns the device statistics for the request devices. +func (i *instanceManager) DeviceStats(d *structs.AllocatedDeviceResource) *device.DeviceGroupStats { + i.deviceStatsLock.Lock() + defer i.deviceStatsLock.Unlock() + + // Find the device in question and then gather the instance statistics we + // are interested in + for _, group := range i.deviceStats { + if group.Vendor != d.Vendor || group.Type != d.Type || group.Name != d.Name { + continue + } + + // We found the group we want so now grab the instance stats + out := &device.DeviceGroupStats{ + Vendor: d.Vendor, + Type: d.Type, + Name: d.Name, + InstanceStats: make(map[string]*device.DeviceStats, len(d.DeviceIDs)), + } + + for _, id := range d.DeviceIDs { + out.InstanceStats[id] = group.InstanceStats[id] + } + + return out + } + + return nil +} + +// Reserve reserves the given devices +func (i *instanceManager) Reserve(d *structs.AllocatedDeviceResource) (*device.ContainerReservation, error) { + // Get a device plugin + devicePlugin, err := i.dispense() + if err != nil { + i.logger.Error("dispensing plugin failed", "error", err) + return nil, err + } + + // Send the reserve request + return devicePlugin.Reserve(d.DeviceIDs) +} + +// Devices returns the detected devices. +func (i *instanceManager) Devices() []*device.DeviceGroup { + i.deviceLock.Lock() + defer i.deviceLock.Unlock() + return i.devices +} + +// WaitForFirstFingerprint waits until either the plugin fingerprints, the +// passed context is done, or the plugin instance manager is shutdown. +func (i *instanceManager) WaitForFirstFingerprint(ctx context.Context) { + select { + case <-i.ctx.Done(): + case <-ctx.Done(): + case <-i.firstFingerprintCh: + } +} + +// run is a long lived goroutine that starts the fingerprinting and stats +// collection goroutine and then shutsdown the plugin on exit. +func (i *instanceManager) run() { + // Dispense once to ensure we are given a valid plugin + if _, err := i.dispense(); err != nil { + i.logger.Error("dispensing initial plugin failed", "error", err) + return + } + + // Start the fingerprinter + i.wg.Add(1) + go i.fingerprint() + + // Wait for a valid result before starting stats collection + select { + case <-i.ctx.Done(): + goto DONE + case <-i.firstFingerprintCh: + } + + // Start stats + i.wg.Add(1) + go i.collectStats() + + // Do a final cleanup +DONE: + i.wg.Wait() + i.cleanup() +} + +// dispense is used to dispense a plugin. +func (i *instanceManager) dispense() (plugin device.DevicePlugin, err error) { + i.pluginLock.Lock() + defer i.pluginLock.Unlock() + + // See if we already have a running instance + if i.plugin != nil && !i.plugin.Exited() { + return i.device, nil + } + + // Get an instance of the plugin + pluginInstance, err := i.loader.Dispense(i.id.Name, i.id.PluginType, i.pluginConfig, i.logger) + if err != nil { + // Retry as the error just indicates the singleton has exited + if err == singleton.SingletonPluginExited { + pluginInstance, err = i.loader.Dispense(i.id.Name, i.id.PluginType, i.pluginConfig, i.logger) + } + + // If we still have an error there is a real problem + if err != nil { + return nil, fmt.Errorf("failed to start plugin: %v", err) + } + } + + // Convert to a fingerprint plugin + device, ok := pluginInstance.Plugin().(device.DevicePlugin) + if !ok { + pluginInstance.Kill() + return nil, fmt.Errorf("plugin loaded does not implement the driver interface") + } + + // Store the plugin and device + i.plugin = pluginInstance + i.device = device + + // Store the reattach config + if c, ok := pluginInstance.ReattachConfig(); ok { + i.storeReattach(c) + } + + return device, nil +} + +// cleanup shutsdown the plugin +func (i *instanceManager) cleanup() { + i.shutdownLock.Lock() + defer i.shutdownLock.Unlock() + + if i.plugin != nil && !i.plugin.Exited() { + i.plugin.Kill() + i.storeReattach(nil) + } +} + +// fingerprint is a long lived routine used to fingerprint the device +func (i *instanceManager) fingerprint() { + defer i.wg.Done() + +START: + // Get a device plugin + devicePlugin, err := i.dispense() + if err != nil { + i.logger.Error("dispensing plugin failed", "error", err) + i.cancel() + return + } + + // Start fingerprinting + fingerprintCh, err := devicePlugin.Fingerprint(i.ctx) + if err != nil { + i.logger.Error("fingerprinting failed", "error", err) + i.handleFingerprintError() + return + } + + var fresp *device.FingerprintResponse + var ok bool + for { + select { + case <-i.ctx.Done(): + return + case fresp, ok = <-fingerprintCh: + } + + if !ok { + i.logger.Debug("exiting since fingerprinting gracefully shutdown") + i.handleFingerprintError() + return + } + + // Guard against error by the plugin + if fresp == nil { + continue + } + + // Handle any errors + if fresp.Error != nil { + if fresp.Error == base.ErrPluginShutdown { + i.logger.Error("plugin exited unexpectedly") + goto START + } + + i.logger.Error("fingerprinting returned an error", "error", err) + i.handleFingerprintError() + return + } + + if err := i.handleFingerprint(fresp); err != nil { + // Cancel the context so we cleanup all goroutines + i.logger.Error("returned devices failed fingerprinting", "error", err) + i.handleFingerprintError() + } + } +} + +// handleFingerprintError exits the manager and shutsdown the plugin. +func (i *instanceManager) handleFingerprintError() { + // Clear out the devices and trigger a node update + i.deviceLock.Lock() + defer i.deviceLock.Unlock() + + // If we have fingerprinted before clear it out + if i.hasFingerprinted { + // Store the new devices + i.devices = nil + + // Trigger that the we have new devices + select { + case i.fingerprintOutCh <- struct{}{}: + default: + } + } + + // Cancel the context so we cleanup all goroutines + i.cancel() + + return +} + +// handleFingerprint stores the new devices and triggers the fingerprint output +// channel. An error is returned if the passed devices don't pass validation. +func (i *instanceManager) handleFingerprint(f *device.FingerprintResponse) error { + // Safety check + if f.Devices == nil { + return nil + } + + // Validate the received devices + var validationErr multierror.Error + for i, d := range f.Devices { + if err := d.Validate(); err != nil { + multierror.Append(&validationErr, multierror.Prefix(err, fmt.Sprintf("device group %d: ", i))) + } + } + + if err := validationErr.ErrorOrNil(); err != nil { + return err + } + + i.deviceLock.Lock() + defer i.deviceLock.Unlock() + + // Store the new devices + i.devices = f.Devices + + // Mark that we have received data + if !i.hasFingerprinted { + close(i.firstFingerprintCh) + i.hasFingerprinted = true + } + + // Trigger that we have data to pull + select { + case i.fingerprintOutCh <- struct{}{}: + default: + } + + return nil +} + +// collectStats is a long lived goroutine for collecting device statistics. It +// handles errors by backing off exponentially and retrying. +func (i *instanceManager) collectStats() { + defer i.wg.Done() + + attempt := 0 + +START: + // Get a device plugin + devicePlugin, err := i.dispense() + if err != nil { + i.logger.Error("dispensing plugin failed", "error", err) + i.cancel() + return + } + + // Start stats collection + statsCh, err := devicePlugin.Stats(i.ctx, i.statsInterval) + if err != nil { + i.logger.Error("stats collection failed", "error", err) + return + } + + var sresp *device.StatsResponse + var ok bool + for { + select { + case <-i.ctx.Done(): + return + case sresp, ok = <-statsCh: + } + + if !ok { + i.logger.Debug("exiting since stats gracefully shutdown") + return + } + + // Guard against error by the plugin + if sresp == nil { + continue + } + + // Handle any errors + if sresp.Error != nil { + if sresp.Error == base.ErrPluginShutdown { + i.logger.Error("plugin exited unexpectedly") + goto START + } + + // Retry with an exponential backoff + backoff := (1 << (2 * uint64(attempt))) * statsBackoffBaseline + if backoff > statsBackoffLimit { + backoff = statsBackoffLimit + } + attempt++ + + i.logger.Error("stats returned an error", "error", err, "retry", backoff) + + select { + case <-i.ctx.Done(): + return + case <-time.After(backoff): + goto START + } + } + + // Reset the attempt since we got statistics + attempt = 0 + + // Store the new stats + if sresp.Groups != nil { + i.deviceStatsLock.Lock() + i.deviceStats = sresp.Groups + i.deviceStatsLock.Unlock() + } + } +} diff --git a/client/devicemanager/manager.go b/client/devicemanager/manager.go new file mode 100644 index 000000000..c1dcb7977 --- /dev/null +++ b/client/devicemanager/manager.go @@ -0,0 +1,302 @@ +// Package devicemanager is used to manage device plugins +package devicemanager + +import ( + "context" + "fmt" + "sync" + "time" + + log "github.com/hashicorp/go-hclog" + multierror "github.com/hashicorp/go-multierror" + plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared" + "github.com/hashicorp/nomad/plugins/shared/loader" +) + +// Manaager is the interface used to manage device plugins +type Manager interface { + // Run starts the device manager + Run() + + // Shutdown shutsdown the manager and all launched plugins + Shutdown() + + // Reserve is used to reserve a set of devices + Reserve(d *structs.AllocatedDeviceResource) (*device.ContainerReservation, error) + + // AllStats is used to retrieve all the latest statistics for all devices. + AllStats() []*device.DeviceGroupStats + + // DeviceStats returns the device statistics for the given device. + DeviceStats(d *structs.AllocatedDeviceResource) (*device.DeviceGroupStats, error) +} + +// StateStorage is used to persist the device managers state across +// agent restarts. +type StateStorage interface { + // GetDevicePluginState is used to retrieve the device manager's plugin + // state. + GetDevicePluginState() (*PluginState, error) + + // PutDevicePluginState is used to store the device manager's plugin + // state. + PutDevicePluginState(state *PluginState) error +} + +// UpdateNodeDevices is a callback for updating the set of devices on a node. +type UpdateNodeDevices func(devices []*structs.NodeDeviceResource) + +// StorePluginReattachFn is used to store plugin reattachment configurations. +type StorePluginReattachFn func(*plugin.ReattachConfig) error + +// Config is used to configure a device manager +type Config struct { + // Logger is the logger used by the device manager + Logger log.Logger + + // Loader is the plugin loader + Loader loader.PluginCatalog + + // PluginConfig is the config passed to the launched plugins + PluginConfig *base.ClientAgentConfig + + // Updater is used to update the node when device information changes + Updater UpdateNodeDevices + + // StatsInterval is the interval at which to collect statistics + StatsInterval time.Duration + + // State is used to manage the device managers state + State StateStorage +} + +// manager is used to manage a set of device plugins +type manager struct { + // logger is the logger used by the device manager + logger log.Logger + + // state is used to manage the device managers state + state StateStorage + + // ctx is used to shutdown the device manager + ctx context.Context + cancel context.CancelFunc + + // loader is the plugin loader + loader loader.PluginCatalog + + // pluginConfig is the config passed to the launched plugins + pluginConfig *base.ClientAgentConfig + + // updater is used to update the node when device information changes + updater UpdateNodeDevices + + // statsInterval is the duration at which to collect statistics + statsInterval time.Duration + + // fingerprintResCh is used to be triggered that there are new devices + fingerprintResCh chan struct{} + + // instances is the list of managed devices + instances map[loader.PluginID]*instanceManager + + // reattachConfigs stores the plugin reattach configs + reattachConfigs map[loader.PluginID]*shared.ReattachConfig + reattachConfigLock sync.Mutex +} + +// New returns a new device manager +func New(c *Config) *manager { + ctx, cancel := context.WithCancel(context.Background()) + return &manager{ + logger: c.Logger.Named("device_mgr"), + state: c.State, + ctx: ctx, + cancel: cancel, + loader: c.Loader, + pluginConfig: c.PluginConfig, + updater: c.Updater, + instances: make(map[loader.PluginID]*instanceManager), + reattachConfigs: make(map[loader.PluginID]*shared.ReattachConfig), + fingerprintResCh: make(chan struct{}, 1), + } +} + +// Run starts thed device manager. The manager will shutdown any previously +// launched plugin and then begin fingerprinting and stats collection on all new +// device plugins. +func (m *manager) Run() { + // Check if there are any plugins that didn't get cleanly shutdown before + // and if there are shut them down. + m.cleanupStalePlugins() + + // Get device plugins + devices := m.loader.Catalog()[base.PluginTypeDevice] + for _, d := range devices { + id := loader.PluginInfoID(d) + storeFn := func(c *plugin.ReattachConfig) error { + id := id + return m.storePluginReattachConfig(id, c) + } + m.instances[id] = newInstanceManager(&instanceManagerConfig{ + Logger: m.logger, + Ctx: m.ctx, + Loader: m.loader, + StoreReattach: storeFn, + PluginConfig: m.pluginConfig, + Id: &id, + FingerprintOutCh: m.fingerprintResCh, + StatsInterval: m.statsInterval, + }) + } + + // XXX we should eventually remove this and have it be done in the client + // Give all the fingerprinters a chance to run at least once before we + // update the node. This prevents initial fingerprinting from causing too + // many server side updates. + ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) + for _, i := range m.instances { + i.WaitForFirstFingerprint(ctx) + } + cancel() + + // Now start the fingerprint handler + for { + select { + case <-m.ctx.Done(): + return + case <-m.fingerprintResCh: + } + + // Collect the data + var fingerprinted []*device.DeviceGroup + for _, i := range m.instances { + fingerprinted = append(fingerprinted, i.Devices()...) + } + + // Convert and update + out := make([]*structs.NodeDeviceResource, len(fingerprinted)) + for i, f := range fingerprinted { + out[i] = convertDeviceGroup(f) + } + + // Call the updater + m.updater(out) + } +} + +// Shutdown cleans up all the plugins +func (m *manager) Shutdown() { + // Cancel the context to stop any requests + m.cancel() + + // Go through and shut everything down + for _, i := range m.instances { + i.cleanup() + } +} + +// Reserve reserves the given allocated device. If the device is unknown, an +// UnknownDeviceErr is returned. +func (m *manager) Reserve(d *structs.AllocatedDeviceResource) (*device.ContainerReservation, error) { + // Go through each plugin and see if it can reserve the resources + for _, i := range m.instances { + if !i.HasDevices(d) { + continue + } + + // We found a match so reserve + return i.Reserve(d) + } + + return nil, UnknownDeviceErrFromAllocated("failed to reserve devices", d) +} + +// AllStats returns statistics for all the devices +func (m *manager) AllStats() []*device.DeviceGroupStats { + // Go through each plugin and collect stats + var stats []*device.DeviceGroupStats + for _, i := range m.instances { + stats = append(stats, i.AllStats()...) + } + + return stats +} + +// DeviceStats returns the statistics for the passed devices. If the device is unknown, an +// UnknownDeviceErr is returned. +func (m *manager) DeviceStats(d *structs.AllocatedDeviceResource) (*device.DeviceGroupStats, error) { + // Go through each plugin and see if it has the requested devices + for _, i := range m.instances { + if !i.HasDevices(d) { + continue + } + + // We found a match so reserve + return i.DeviceStats(d), nil + } + + return nil, UnknownDeviceErrFromAllocated("failed to collect statistics", d) +} + +// cleanupStalePlugins reads the device managers state and shuts down any +// previously launched plugin. +func (m *manager) cleanupStalePlugins() error { + + // Read the old plugin state + s, err := m.state.GetDevicePluginState() + if err != nil { + return fmt.Errorf("failed to read plugin state: %v", err) + } + + // No state was stored so there is nothing to do. + if s == nil { + return nil + } + + // For each plugin go through and try to shut it down + var mErr multierror.Error + for name, c := range s.ReattachConfigs { + rc, err := shared.ReattachConfigToGoPlugin(c) + if err != nil { + multierror.Append(&mErr, fmt.Errorf("failed to convert reattach config: %v", err)) + continue + } + + instance, err := m.loader.Reattach(name, base.PluginTypeDevice, rc) + if err != nil { + multierror.Append(&mErr, fmt.Errorf("failed to reattach to plugin %q: %v", name, err)) + continue + } + + // Kill the instance + instance.Kill() + } + + return mErr.ErrorOrNil() +} + +// storePluginReattachConfig is used as a callback to the instance managers and +// persists thhe plugin reattach configurations. +func (m *manager) storePluginReattachConfig(id loader.PluginID, c *plugin.ReattachConfig) error { + m.reattachConfigLock.Lock() + defer m.reattachConfigLock.Unlock() + + // Store the new reattach config + m.reattachConfigs[id] = shared.ReattachConfigFromGoPlugin(c) + + // Persist the state + s := &PluginState{ + ReattachConfigs: make(map[string]*shared.ReattachConfig, len(m.reattachConfigs)), + } + + for id, c := range m.reattachConfigs { + s.ReattachConfigs[id.Name] = c + } + + return m.state.PutDevicePluginState(s) +} diff --git a/client/devicemanager/utils.go b/client/devicemanager/utils.go new file mode 100644 index 000000000..4bed98c98 --- /dev/null +++ b/client/devicemanager/utils.go @@ -0,0 +1,102 @@ +package devicemanager + +import ( + "errors" + "fmt" + + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared" + psstructs "github.com/hashicorp/nomad/plugins/shared/structs" +) + +// PluginState is used to store the device managers state across restarts of the +// agent +type PluginState struct { + // ReattachConfigs are the set of reattach configs for plugin's launched by + // the device manager + ReattachConfigs map[string]*shared.ReattachConfig +} + +// UnknownDeviceError is returned when an operation is attempted on an unknown +// device. +type UnknownDeviceError struct { + Err error + Name string + Vendor string + Type string + IDs []string +} + +// NewUnknownDeviceError returns a new UnknownDeviceError for the given device. +func NewUnknownDeviceError(err error, name, vendor, devType string, ids []string) *UnknownDeviceError { + return &UnknownDeviceError{ + Err: err, + Name: name, + Vendor: vendor, Type: devType, + IDs: ids, + } +} + +// Error returns an error formatting that reveals which unknown devices were +// requested +func (u *UnknownDeviceError) Error() string { + return fmt.Sprintf("operation on unknown device(s) \"%s/%s/%s\" (%v): %v", + u.Vendor, u.Type, u.Name, u.IDs, u.Err) +} + +// UnknownDeviceErrFromAllocated is a helper that returns an UnknownDeviceError +// populating it via the AllocatedDeviceResource struct. +func UnknownDeviceErrFromAllocated(err string, d *structs.AllocatedDeviceResource) *UnknownDeviceError { + return NewUnknownDeviceError(errors.New(err), d.Name, d.Vendor, d.Type, d.DeviceIDs) +} + +// convertDeviceGroup converts a device group to a structs NodeDeviceResource +func convertDeviceGroup(d *device.DeviceGroup) *structs.NodeDeviceResource { + if d == nil { + return nil + } + + return &structs.NodeDeviceResource{ + Vendor: d.Vendor, + Type: d.Type, + Name: d.Name, + Instances: convertDevices(d.Devices), + Attributes: psstructs.CopyMapStringAttribute(d.Attributes), + } +} + +func convertDevices(devs []*device.Device) []*structs.NodeDevice { + if devs == nil { + return nil + } + + out := make([]*structs.NodeDevice, len(devs)) + for i, dev := range devs { + out[i] = convertDevice(dev) + } + return out +} + +func convertDevice(dev *device.Device) *structs.NodeDevice { + if dev == nil { + return nil + } + + return &structs.NodeDevice{ + ID: dev.ID, + Healthy: dev.Healthy, + HealthDescription: dev.HealthDesc, + Locality: convertHwLocality(dev.HwLocality), + } +} + +func convertHwLocality(l *device.DeviceLocality) *structs.NodeDeviceLocality { + if l == nil { + return nil + } + + return &structs.NodeDeviceLocality{ + PciBusID: l.PciBusID, + } +} diff --git a/client/logmon/proto/logmon.pb.go b/client/logmon/proto/logmon.pb.go index 9103ac270..ac1539b55 100644 --- a/client/logmon/proto/logmon.pb.go +++ b/client/logmon/proto/logmon.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: logmon.proto +// source: client/logmon/proto/logmon.proto package proto @@ -40,7 +40,7 @@ func (m *StartRequest) Reset() { *m = StartRequest{} } func (m *StartRequest) String() string { return proto.CompactTextString(m) } func (*StartRequest) ProtoMessage() {} func (*StartRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_logmon_0690d412b596ec9e, []int{0} + return fileDescriptor_logmon_6dbff459851a9ae9, []int{0} } func (m *StartRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartRequest.Unmarshal(m, b) @@ -119,7 +119,7 @@ func (m *StartResponse) Reset() { *m = StartResponse{} } func (m *StartResponse) String() string { return proto.CompactTextString(m) } func (*StartResponse) ProtoMessage() {} func (*StartResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_logmon_0690d412b596ec9e, []int{1} + return fileDescriptor_logmon_6dbff459851a9ae9, []int{1} } func (m *StartResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartResponse.Unmarshal(m, b) @@ -149,7 +149,7 @@ func (m *StopRequest) Reset() { *m = StopRequest{} } func (m *StopRequest) String() string { return proto.CompactTextString(m) } func (*StopRequest) ProtoMessage() {} func (*StopRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_logmon_0690d412b596ec9e, []int{2} + return fileDescriptor_logmon_6dbff459851a9ae9, []int{2} } func (m *StopRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopRequest.Unmarshal(m, b) @@ -179,7 +179,7 @@ func (m *StopResponse) Reset() { *m = StopResponse{} } func (m *StopResponse) String() string { return proto.CompactTextString(m) } func (*StopResponse) ProtoMessage() {} func (*StopResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_logmon_0690d412b596ec9e, []int{3} + return fileDescriptor_logmon_6dbff459851a9ae9, []int{3} } func (m *StopResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopResponse.Unmarshal(m, b) @@ -308,31 +308,33 @@ var _LogMon_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "logmon.proto", + Metadata: "client/logmon/proto/logmon.proto", } -func init() { proto.RegisterFile("logmon.proto", fileDescriptor_logmon_0690d412b596ec9e) } - -var fileDescriptor_logmon_0690d412b596ec9e = []byte{ - // 314 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x31, 0x6f, 0xf2, 0x30, - 0x10, 0x86, 0xbf, 0xf0, 0x41, 0x28, 0x07, 0xa1, 0xc8, 0x4b, 0x23, 0x3a, 0x14, 0xa5, 0x43, 0x99, - 0xa2, 0x42, 0xff, 0x41, 0x55, 0x75, 0x2a, 0x1d, 0x60, 0xeb, 0x12, 0x19, 0xb8, 0x04, 0x4b, 0x71, - 0x2e, 0xb5, 0x8d, 0x84, 0x58, 0xfb, 0x6b, 0xfb, 0x2f, 0xaa, 0x18, 0x13, 0x65, 0x84, 0x29, 0xca, - 0xdd, 0xf3, 0xea, 0x1e, 0xbf, 0x30, 0xc8, 0x29, 0x93, 0x54, 0xc4, 0xa5, 0x22, 0x43, 0xec, 0x71, - 0xc7, 0xf5, 0x4e, 0x6c, 0x48, 0x95, 0x71, 0x41, 0x92, 0x6f, 0xe3, 0x4d, 0x2e, 0xb0, 0x30, 0x71, - 0x13, 0x8a, 0x7e, 0x5a, 0x30, 0x58, 0x19, 0xae, 0xcc, 0x12, 0xbf, 0xf7, 0xa8, 0x0d, 0xbb, 0x83, - 0x6e, 0x4e, 0x59, 0xb2, 0x15, 0x2a, 0xf4, 0x26, 0xde, 0xb4, 0xb7, 0xf4, 0x73, 0xca, 0xde, 0x84, - 0x62, 0x53, 0x18, 0x69, 0xb3, 0xa5, 0xbd, 0x49, 0x52, 0x91, 0x63, 0x52, 0x70, 0x89, 0x61, 0xcb, - 0x12, 0xc3, 0xd3, 0xfc, 0x5d, 0xe4, 0xf8, 0xc9, 0x25, 0x3a, 0x12, 0x95, 0x6a, 0x90, 0xff, 0x6b, - 0x12, 0x95, 0xaa, 0xc9, 0x7b, 0xe8, 0x49, 0x7e, 0xb0, 0x98, 0x0e, 0xdb, 0x13, 0x6f, 0x1a, 0x2c, - 0x6f, 0x24, 0x3f, 0x54, 0x7b, 0xcd, 0x9e, 0x60, 0x74, 0x5e, 0x26, 0x5a, 0x1c, 0x31, 0x91, 0xeb, - 0xb0, 0x63, 0x99, 0xc0, 0x31, 0x2b, 0x71, 0xc4, 0xc5, 0x9a, 0x3d, 0x40, 0xbf, 0x36, 0x4b, 0x29, - 0xf4, 0xed, 0x29, 0x38, 0x4b, 0xa5, 0xe4, 0x80, 0x93, 0x50, 0x4a, 0x61, 0xb7, 0x06, 0xac, 0x4b, - 0x4a, 0xd1, 0x2d, 0x04, 0xae, 0x04, 0x5d, 0x52, 0xa1, 0x31, 0x0a, 0xa0, 0xbf, 0x32, 0x54, 0xba, - 0x52, 0xa2, 0x61, 0x55, 0x52, 0xf5, 0x7b, 0x5a, 0xcf, 0x7f, 0x3d, 0xf0, 0x3f, 0x28, 0x5b, 0x50, - 0xc1, 0x4a, 0xe8, 0xd8, 0x28, 0x9b, 0xc5, 0x17, 0xf4, 0x1d, 0x37, 0xbb, 0x1e, 0xcf, 0xaf, 0x89, - 0x38, 0xb3, 0x7f, 0x4c, 0x42, 0xbb, 0x92, 0x61, 0xcf, 0x17, 0xa6, 0xeb, 0x67, 0x8c, 0x67, 0x57, - 0x24, 0xce, 0xe7, 0x5e, 0xbb, 0x5f, 0x1d, 0x3b, 0x5f, 0xfb, 0xf6, 0xf3, 0xf2, 0x17, 0x00, 0x00, - 0xff, 0xff, 0x1b, 0x8f, 0x8b, 0x51, 0x66, 0x02, 0x00, 0x00, +func init() { + proto.RegisterFile("client/logmon/proto/logmon.proto", fileDescriptor_logmon_6dbff459851a9ae9) +} + +var fileDescriptor_logmon_6dbff459851a9ae9 = []byte{ + // 320 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x31, 0x6f, 0xc2, 0x30, + 0x10, 0x85, 0x1b, 0x0a, 0xa1, 0x1c, 0x0d, 0x45, 0x5e, 0x1a, 0xd1, 0xa1, 0x28, 0x1d, 0xca, 0x14, + 0x0a, 0xfd, 0x07, 0x55, 0xd5, 0xa9, 0x74, 0x80, 0xad, 0x4b, 0x64, 0xc0, 0x09, 0x96, 0x62, 0x5f, + 0x6a, 0x1b, 0x09, 0xb1, 0xf6, 0xd7, 0xf6, 0x5f, 0x54, 0x71, 0x4c, 0x94, 0x11, 0x26, 0xeb, 0xee, + 0x7d, 0x4f, 0xf7, 0xfc, 0x60, 0xbc, 0xc9, 0x39, 0x93, 0x66, 0x9a, 0x63, 0x26, 0x50, 0x4e, 0x0b, + 0x85, 0x06, 0xdd, 0x10, 0xdb, 0x81, 0x3c, 0xed, 0xa8, 0xde, 0xf1, 0x0d, 0xaa, 0x22, 0x96, 0x28, + 0xe8, 0x36, 0xae, 0x1c, 0x71, 0x13, 0x8a, 0x7e, 0x5b, 0x70, 0xbb, 0x32, 0x54, 0x99, 0x25, 0xfb, + 0xd9, 0x33, 0x6d, 0xc8, 0x3d, 0x74, 0x73, 0xcc, 0x92, 0x2d, 0x57, 0xa1, 0x37, 0xf6, 0x26, 0xbd, + 0xa5, 0x9f, 0x63, 0xf6, 0xce, 0x15, 0x99, 0xc0, 0x50, 0x9b, 0x2d, 0xee, 0x4d, 0x92, 0xf2, 0x9c, + 0x25, 0x92, 0x0a, 0x16, 0xb6, 0x2c, 0x31, 0xa8, 0xf6, 0x1f, 0x3c, 0x67, 0x5f, 0x54, 0x30, 0x47, + 0x32, 0xa5, 0x1a, 0xe4, 0x75, 0x4d, 0x32, 0xa5, 0x6a, 0xf2, 0x01, 0x7a, 0x82, 0x1e, 0x2c, 0xa6, + 0xc3, 0xf6, 0xd8, 0x9b, 0x04, 0xcb, 0x1b, 0x41, 0x0f, 0xa5, 0xae, 0xc9, 0x33, 0x0c, 0x4f, 0x62, + 0xa2, 0xf9, 0x91, 0x25, 0x62, 0x1d, 0x76, 0x2c, 0x13, 0x38, 0x66, 0xc5, 0x8f, 0x6c, 0xb1, 0x26, + 0x8f, 0xd0, 0xaf, 0x93, 0xa5, 0x18, 0xfa, 0xf6, 0x14, 0x9c, 0x42, 0xa5, 0xe8, 0x80, 0x2a, 0x50, + 0x8a, 0x61, 0xb7, 0x06, 0x6c, 0x96, 0x14, 0xa3, 0x3b, 0x08, 0x5c, 0x09, 0xba, 0x40, 0xa9, 0x59, + 0x14, 0x40, 0x7f, 0x65, 0xb0, 0x70, 0xa5, 0x44, 0x83, 0xb2, 0xa4, 0x72, 0xac, 0xe4, 0xf9, 0x9f, + 0x07, 0xfe, 0x27, 0x66, 0x0b, 0x94, 0xa4, 0x80, 0x8e, 0xb5, 0x92, 0x59, 0x7c, 0x46, 0xdf, 0x71, + 0xb3, 0xeb, 0xd1, 0xfc, 0x12, 0x8b, 0x4b, 0x76, 0x45, 0x04, 0xb4, 0xcb, 0x30, 0xe4, 0xe5, 0x4c, + 0x77, 0xfd, 0x8d, 0xd1, 0xec, 0x02, 0xc7, 0xe9, 0xdc, 0x5b, 0xf7, 0xbb, 0x63, 0xf7, 0x6b, 0xdf, + 0x3e, 0xaf, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd8, 0xb4, 0x87, 0xfc, 0x7a, 0x02, 0x00, 0x00, } diff --git a/client/state/db_test.go b/client/state/db_test.go index ab50025e5..1024436e1 100644 --- a/client/state/db_test.go +++ b/client/state/db_test.go @@ -7,6 +7,7 @@ import ( "testing" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/kr/pretty" @@ -190,3 +191,28 @@ func TestStateDB_TaskState(t *testing.T) { require.Nil(ts) }) } + +// TestStateDB_DeviceManager asserts the behavior of device manager state related StateDB +// methods. +func TestStateDB_DeviceManager(t *testing.T) { + t.Parallel() + + testDB(t, func(t *testing.T, db StateDB) { + require := require.New(t) + + // Getting nonexistent state should return nils + ps, err := db.GetDevicePluginState() + require.NoError(err) + require.Nil(ps) + + // Putting PluginState should work + state := &devicemanager.PluginState{} + require.NoError(db.PutDevicePluginState(state)) + + // Getting should return the available state + ps, err = db.GetDevicePluginState() + require.NoError(err) + require.NotNil(ps) + require.Equal(state, ps) + }) +} diff --git a/client/state/interface.go b/client/state/interface.go index 0bea1de19..528ad154b 100644 --- a/client/state/interface.go +++ b/client/state/interface.go @@ -2,6 +2,7 @@ package state import ( "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/nomad/structs" ) @@ -41,6 +42,14 @@ type StateDB interface { // exists. No error is returned if it does not exist. DeleteAllocationBucket(allocID string) error + // GetDevicePluginState is used to retrieve the device manager's plugin + // state. + GetDevicePluginState() (*devicemanager.PluginState, error) + + // PutDevicePluginState is used to store the device manager's plugin + // state. + PutDevicePluginState(state *devicemanager.PluginState) error + // Close the database. Unsafe for further use after calling regardless // of return value. Close() error diff --git a/client/state/memdb.go b/client/state/memdb.go index f4e67ff05..27fd8a294 100644 --- a/client/state/memdb.go +++ b/client/state/memdb.go @@ -4,6 +4,7 @@ import ( "sync" "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/nomad/structs" ) @@ -17,6 +18,9 @@ type MemDB struct { localTaskState map[string]map[string]*state.LocalState taskState map[string]map[string]*structs.TaskState + // devicemanager -> plugin-state + devManagerPs *devicemanager.PluginState + mu sync.RWMutex } @@ -131,6 +135,21 @@ func (m *MemDB) DeleteAllocationBucket(allocID string) error { return nil } +func (m *MemDB) PutDevicePluginState(ps *devicemanager.PluginState) error { + m.mu.Lock() + defer m.mu.Unlock() + m.devManagerPs = ps + return nil +} + +// GetDevicePluginState stores the device manager's plugin state or returns an +// error. +func (m *MemDB) GetDevicePluginState() (*devicemanager.PluginState, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.devManagerPs, nil +} + func (m *MemDB) Close() error { m.mu.Lock() defer m.mu.Unlock() diff --git a/client/state/noopdb.go b/client/state/noopdb.go index c20eae88d..e0d1697ab 100644 --- a/client/state/noopdb.go +++ b/client/state/noopdb.go @@ -2,6 +2,7 @@ package state import ( "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/nomad/structs" ) @@ -40,6 +41,14 @@ func (n NoopDB) DeleteAllocationBucket(allocID string) error { return nil } +func (n NoopDB) PutDevicePluginState(ps *devicemanager.PluginState) error { + return nil +} + +func (n NoopDB) GetDevicePluginState() (*devicemanager.PluginState, error) { + return nil, nil +} + func (n NoopDB) Close() error { return nil } diff --git a/client/state/state_database.go b/client/state/state_database.go index 524848e13..8eb8bc43a 100644 --- a/client/state/state_database.go +++ b/client/state/state_database.go @@ -5,12 +5,13 @@ import ( "path/filepath" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" + "github.com/hashicorp/nomad/client/devicemanager" "github.com/hashicorp/nomad/helper/boltdd" "github.com/hashicorp/nomad/nomad/structs" ) /* -The client has a boltDB backed state store. The schema as of 0.6 looks as follows: +The client has a boltDB backed state store. The schema as of 0.9 looks as follows: allocations/ (bucket) |--> / (bucket) @@ -18,6 +19,9 @@ allocations/ (bucket) |--> alloc_runner persisted objects (k/v) |--> / (bucket) |--> task_runner persisted objects (k/v) + +devicemanager/ +|--> plugin-state -> *devicemanager.PluginState */ var ( @@ -36,6 +40,14 @@ var ( // allocations -> $allocid -> $taskname -> the keys below taskLocalStateKey = []byte("local_state") taskStateKey = []byte("task_state") + + // devManagerBucket is the bucket name containing all device manager related + // data + devManagerBucket = []byte("devicemanager") + + // devManagerPluginStateKey is the key serialized device manager + // plugin state is stored at + devManagerPluginStateKey = []byte("plugin-state") ) // NewStateDBFunc creates a StateDB given a state directory. @@ -354,3 +366,50 @@ func getTaskBucket(tx *boltdd.Tx, allocID, taskName string) (*boltdd.Bucket, err return task, nil } + +// PutDevicePluginState stores the device manager's plugin state or returns an +// error. +func (s *BoltStateDB) PutDevicePluginState(ps *devicemanager.PluginState) error { + return s.db.Update(func(tx *boltdd.Tx) error { + // Retrieve the root device manager bucket + devBkt, err := tx.CreateBucketIfNotExists(devManagerBucket) + if err != nil { + return err + } + + return devBkt.Put(devManagerPluginStateKey, ps) + }) +} + +// GetDevicePluginState stores the device manager's plugin state or returns an +// error. +func (s *BoltStateDB) GetDevicePluginState() (*devicemanager.PluginState, error) { + var ps *devicemanager.PluginState + + err := s.db.View(func(tx *boltdd.Tx) error { + devBkt := tx.Bucket(devManagerBucket) + if devBkt == nil { + // No state, return + return nil + } + + // Restore Plugin State if it exists + ps = &devicemanager.PluginState{} + if err := devBkt.Get(devManagerPluginStateKey, ps); err != nil { + if !boltdd.IsErrNotFound(err) { + return fmt.Errorf("failed to read device manager plugin state: %v", err) + } + + // Key not found, reset ps to nil + ps = nil + } + + return nil + }) + + if err != nil { + return nil, err + } + + return ps, nil +} diff --git a/command/agent/command.go b/command/agent/command.go index 5c049be50..4ff44e4d4 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -118,6 +118,7 @@ func (c *Command) readConfig() *Config { }), "consul-auto-advertise", "") flags.StringVar(&cmdConfig.Consul.CAFile, "consul-ca-file", "", "") flags.StringVar(&cmdConfig.Consul.CertFile, "consul-cert-file", "", "") + flags.StringVar(&cmdConfig.Consul.KeyFile, "consul-key-file", "", "") flags.Var((flaghelper.FuncBoolVar)(func(b bool) error { cmdConfig.Consul.ChecksUseAdvertise = &b return nil @@ -128,7 +129,6 @@ func (c *Command) readConfig() *Config { }), "consul-client-auto-join", "") flags.StringVar(&cmdConfig.Consul.ClientServiceName, "consul-client-service-name", "", "") flags.StringVar(&cmdConfig.Consul.ClientHTTPCheckName, "consul-client-http-check-name", "", "") - flags.StringVar(&cmdConfig.Consul.KeyFile, "consul-key-file", "", "") flags.StringVar(&cmdConfig.Consul.ServerServiceName, "consul-server-service-name", "", "") flags.StringVar(&cmdConfig.Consul.ServerHTTPCheckName, "consul-server-http-check-name", "", "") flags.StringVar(&cmdConfig.Consul.ServerSerfCheckName, "consul-server-serf-check-name", "", "") @@ -454,7 +454,59 @@ func (c *Command) AutocompleteFlags() complete.Flags { complete.PredictFiles("*.hcl")) return map[string]complete.Predictor{ - "-config": configFilePredictor, + "-dev": complete.PredictNothing, + "-server": complete.PredictNothing, + "-client": complete.PredictNothing, + "-bootstrap-expect": complete.PredictAnything, + "-encrypt": complete.PredictAnything, + "-raft-protocol": complete.PredictAnything, + "-rejoin": complete.PredictNothing, + "-join": complete.PredictAnything, + "-retry-join": complete.PredictAnything, + "-retry-max": complete.PredictAnything, + "-state-dir": complete.PredictDirs("*"), + "-alloc-dir": complete.PredictDirs("*"), + "-node-class": complete.PredictAnything, + "-servers": complete.PredictAnything, + "-meta": complete.PredictAnything, + "-config": configFilePredictor, + "-bind": complete.PredictAnything, + "-region": complete.PredictAnything, + "-data-dir": complete.PredictDirs("*"), + "-plugin-dir": complete.PredictDirs("*"), + "-dc": complete.PredictAnything, + "-log-level": complete.PredictAnything, + "-node": complete.PredictAnything, + "-consul-auth": complete.PredictAnything, + "-consul-auto-advertise": complete.PredictNothing, + "-consul-ca-file": complete.PredictAnything, + "-consul-cert-file": complete.PredictAnything, + "-consul-key-file": complete.PredictAnything, + "-consul-checks-use-advertise": complete.PredictNothing, + "-consul-client-auto-join": complete.PredictNothing, + "-consul-client-service-name": complete.PredictAnything, + "-consul-client-http-check-name": complete.PredictAnything, + "-consul-server-service-name": complete.PredictAnything, + "-consul-server-http-check-name": complete.PredictAnything, + "-consul-server-serf-check-name": complete.PredictAnything, + "-consul-server-rpc-check-name": complete.PredictAnything, + "-consul-server-auto-join": complete.PredictNothing, + "-consul-ssl": complete.PredictNothing, + "-consul-verify-ssl": complete.PredictNothing, + "-consul-address": complete.PredictAnything, + "-vault-enabled": complete.PredictNothing, + "-vault-allow-unauthenticated": complete.PredictNothing, + "-vault-token": complete.PredictAnything, + "-vault-address": complete.PredictAnything, + "-vault-create-from-role": complete.PredictAnything, + "-vault-ca-file": complete.PredictAnything, + "-vault-ca-path": complete.PredictAnything, + "-vault-cert-file": complete.PredictAnything, + "-vault-key-file": complete.PredictAnything, + "-vault-tls-skip-verify": complete.PredictNothing, + "-vault-tls-server-name": complete.PredictAnything, + "-acl-enabled": complete.PredictNothing, + "-acl-replication-token": complete.PredictAnything, } } diff --git a/devices/gpu/nvidia/README.md b/devices/gpu/nvidia/README.md index 55d1e7f13..d057d11f3 100644 --- a/devices/gpu/nvidia/README.md +++ b/devices/gpu/nvidia/README.md @@ -12,7 +12,6 @@ The configuration should be passed via an HCL file that begins with a top level config { ignored_gpu_ids = ["uuid1", "uuid2"] fingerprint_period = "5s" - stats_period = "5s" } ``` @@ -20,4 +19,3 @@ The valid configuration options are: * `ignored_gpu_ids` (`list(string)`: `[]`): list of GPU UUIDs strings that should not be exposed to nomad * `fingerprint_period` (`string`: `"5s"`): The interval to repeat fingerprint process to identify possible changes. -* `stats_period` (`string`: `"5s"`): The interval at which to emit statistics about the devices. diff --git a/devices/gpu/nvidia/device.go b/devices/gpu/nvidia/device.go index c68971373..8058589c2 100644 --- a/devices/gpu/nvidia/device.go +++ b/devices/gpu/nvidia/device.go @@ -8,7 +8,6 @@ import ( "time" log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/devices/gpu/nvidia/nvml" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/device" @@ -52,11 +51,7 @@ var ( ), "fingerprint_period": hclspec.NewDefault( hclspec.NewAttr("fingerprint_period", "string", false), - hclspec.NewLiteral("\"5s\""), - ), - "stats_period": hclspec.NewDefault( - hclspec.NewAttr("stats_period", "string", false), - hclspec.NewLiteral("\"5s\""), + hclspec.NewLiteral("\"1m\""), ), }) ) @@ -65,7 +60,6 @@ var ( type Config struct { IgnoredGPUIDs []string `codec:"ignored_gpu_ids"` FingerprintPeriod string `codec:"fingerprint_period"` - StatsPeriod string `codec:"stats_period"` } // NvidiaDevice contains all plugin specific data @@ -137,13 +131,6 @@ func (d *NvidiaDevice) SetConfig(data []byte, cfg *base.ClientAgentConfig) error } d.fingerprintPeriod = period - // Convert the stats period - speriod, err := time.ParseDuration(config.StatsPeriod) - if err != nil { - return fmt.Errorf("failed to parse stats period %q: %v", config.StatsPeriod, err) - } - d.statsPeriod = speriod - return nil } @@ -202,8 +189,8 @@ func (d *NvidiaDevice) Reserve(deviceIDs []string) (*device.ContainerReservation } // Stats streams statistics for the detected devices. -func (d *NvidiaDevice) Stats(ctx context.Context) (<-chan *device.StatsResponse, error) { +func (d *NvidiaDevice) Stats(ctx context.Context, interval time.Duration) (<-chan *device.StatsResponse, error) { outCh := make(chan *device.StatsResponse) - go d.stats(ctx, outCh) + go d.stats(ctx, outCh, interval) return outCh, nil } diff --git a/devices/gpu/nvidia/fingerprint.go b/devices/gpu/nvidia/fingerprint.go index f8adcab9e..45bb34fa3 100644 --- a/devices/gpu/nvidia/fingerprint.go +++ b/devices/gpu/nvidia/fingerprint.go @@ -31,12 +31,10 @@ func (d *NvidiaDevice) fingerprint(ctx context.Context, devices chan<- *device.F if d.initErr.Error() != nvml.UnavailableLib.Error() { d.logger.Error("exiting fingerprinting due to problems with NVML loading", "error", d.initErr) devices <- device.NewFingerprintError(d.initErr) - } else { - // write empty fingerprint response to let server know that there are - // no working Nvidia GPU units - devices <- device.NewFingerprint() } + // Just close the channel to let server know that there are no working + // Nvidia GPU units return } diff --git a/devices/gpu/nvidia/stats.go b/devices/gpu/nvidia/stats.go index f433e7175..dfc9df368 100644 --- a/devices/gpu/nvidia/stats.go +++ b/devices/gpu/nvidia/stats.go @@ -50,7 +50,7 @@ const ( ) // stats is the long running goroutine that streams device statistics -func (d *NvidiaDevice) stats(ctx context.Context, stats chan<- *device.StatsResponse) { +func (d *NvidiaDevice) stats(ctx context.Context, stats chan<- *device.StatsResponse, interval time.Duration) { defer close(stats) if d.initErr != nil { @@ -70,7 +70,7 @@ func (d *NvidiaDevice) stats(ctx context.Context, stats chan<- *device.StatsResp case <-ctx.Done(): return case <-ticker.C: - ticker.Reset(d.statsPeriod) + ticker.Reset(interval) } d.writeStatsToChannel(stats, time.Now()) diff --git a/drivers/docker/docklog/proto/docker_logger.pb.go b/drivers/docker/docklog/proto/docker_logger.pb.go index 9de0a1de7..d6d2e0da6 100644 --- a/drivers/docker/docklog/proto/docker_logger.pb.go +++ b/drivers/docker/docklog/proto/docker_logger.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: docker_logger.proto +// source: drivers/docker/docklog/proto/docker_logger.proto package proto @@ -40,7 +40,7 @@ func (m *StartRequest) Reset() { *m = StartRequest{} } func (m *StartRequest) String() string { return proto.CompactTextString(m) } func (*StartRequest) ProtoMessage() {} func (*StartRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_docker_logger_54dce3369d4ecf54, []int{0} + return fileDescriptor_docker_logger_0aa5a411831bd10e, []int{0} } func (m *StartRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartRequest.Unmarshal(m, b) @@ -119,7 +119,7 @@ func (m *StartResponse) Reset() { *m = StartResponse{} } func (m *StartResponse) String() string { return proto.CompactTextString(m) } func (*StartResponse) ProtoMessage() {} func (*StartResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_docker_logger_54dce3369d4ecf54, []int{1} + return fileDescriptor_docker_logger_0aa5a411831bd10e, []int{1} } func (m *StartResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartResponse.Unmarshal(m, b) @@ -149,7 +149,7 @@ func (m *StopRequest) Reset() { *m = StopRequest{} } func (m *StopRequest) String() string { return proto.CompactTextString(m) } func (*StopRequest) ProtoMessage() {} func (*StopRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_docker_logger_54dce3369d4ecf54, []int{2} + return fileDescriptor_docker_logger_0aa5a411831bd10e, []int{2} } func (m *StopRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopRequest.Unmarshal(m, b) @@ -179,7 +179,7 @@ func (m *StopResponse) Reset() { *m = StopResponse{} } func (m *StopResponse) String() string { return proto.CompactTextString(m) } func (*StopResponse) ProtoMessage() {} func (*StopResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_docker_logger_54dce3369d4ecf54, []int{3} + return fileDescriptor_docker_logger_0aa5a411831bd10e, []int{3} } func (m *StopResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopResponse.Unmarshal(m, b) @@ -308,31 +308,34 @@ var _DockerLogger_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "docker_logger.proto", + Metadata: "drivers/docker/docklog/proto/docker_logger.proto", } -func init() { proto.RegisterFile("docker_logger.proto", fileDescriptor_docker_logger_54dce3369d4ecf54) } - -var fileDescriptor_docker_logger_54dce3369d4ecf54 = []byte{ - // 319 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x50, 0xbd, 0x4e, 0xe3, 0x40, - 0x10, 0x3e, 0xe7, 0x62, 0x27, 0x37, 0x49, 0xee, 0xa4, 0x3d, 0x21, 0x4c, 0x1a, 0xc0, 0x15, 0x05, - 0x72, 0x01, 0x15, 0xa1, 0x03, 0x84, 0x84, 0xa0, 0x4a, 0x3a, 0x1a, 0xcb, 0x78, 0x27, 0xc9, 0x2a, - 0xc6, 0x63, 0x66, 0x27, 0x48, 0xa9, 0x90, 0x78, 0x06, 0x9e, 0x8e, 0xa7, 0x41, 0x5e, 0x1b, 0x8b, - 0x36, 0xa9, 0x76, 0xe7, 0xfb, 0x19, 0xcd, 0xf7, 0xc1, 0x7f, 0x4d, 0xd9, 0x0a, 0x39, 0xc9, 0x69, - 0xb1, 0x40, 0x8e, 0x4b, 0x26, 0x21, 0x75, 0xba, 0x4c, 0xed, 0xd2, 0x64, 0xc4, 0x65, 0x5c, 0xd0, - 0x73, 0xaa, 0x63, 0xcd, 0xe6, 0x15, 0xd9, 0xc6, 0xb5, 0xd8, 0x3d, 0x39, 0x2d, 0x6a, 0x75, 0xf4, - 0xe9, 0xc1, 0x70, 0x26, 0x29, 0xcb, 0x14, 0x5f, 0xd6, 0x68, 0x45, 0x8d, 0xa1, 0x8f, 0x85, 0x2e, - 0xc9, 0x14, 0x12, 0x7a, 0x47, 0xde, 0xc9, 0x9f, 0x69, 0x3b, 0xab, 0x63, 0x18, 0x66, 0x54, 0x48, - 0x6a, 0x0a, 0xe4, 0xc4, 0xe8, 0xb0, 0xe3, 0xf8, 0x41, 0x8b, 0xdd, 0x69, 0x75, 0x08, 0x03, 0x2b, - 0x9a, 0xd6, 0x92, 0xcc, 0xcd, 0x9c, 0xc2, 0xdf, 0x4e, 0x01, 0x35, 0x74, 0x6b, 0xe6, 0xd4, 0x08, - 0x90, 0xb9, 0x16, 0x74, 0x5b, 0x01, 0x32, 0x3b, 0xc1, 0x01, 0xf4, 0x25, 0xb7, 0x49, 0x86, 0x2c, - 0xa1, 0xef, 0xd8, 0x9e, 0xe4, 0xf6, 0x1a, 0x59, 0xd4, 0x3e, 0x54, 0xdf, 0x64, 0x85, 0x9b, 0x30, - 0x70, 0x4c, 0x20, 0xb9, 0xbd, 0xc7, 0x8d, 0xda, 0x83, 0xc0, 0x79, 0xd2, 0xb0, 0xe7, 0x70, 0xbf, - 0x72, 0xa4, 0xd1, 0x3f, 0x18, 0x35, 0xd9, 0x6c, 0x49, 0x85, 0xc5, 0x68, 0x04, 0x83, 0x99, 0x50, - 0xd9, 0x64, 0x8d, 0xfe, 0x56, 0xd9, 0xab, 0xb1, 0xa6, 0xcf, 0x3e, 0x3a, 0x30, 0xbc, 0x71, 0x2d, - 0x3d, 0xb8, 0x46, 0xd5, 0xbb, 0x07, 0xbe, 0xdb, 0xa0, 0x26, 0xf1, 0x36, 0xb5, 0xc6, 0x3f, 0x2b, - 0x1d, 0x5f, 0xee, 0xe4, 0x6d, 0x4e, 0xfe, 0xa5, 0xde, 0xa0, 0x5b, 0x5d, 0xa9, 0x2e, 0xb6, 0x5d, - 0xd3, 0x06, 0x1d, 0x4f, 0x76, 0xb1, 0x7e, 0x1f, 0x70, 0xd5, 0x7b, 0xf4, 0x1d, 0xfe, 0x14, 0xb8, - 0xe7, 0xfc, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x2e, 0xe1, 0xdf, 0xc9, 0x78, 0x02, 0x00, 0x00, +func init() { + proto.RegisterFile("drivers/docker/docklog/proto/docker_logger.proto", fileDescriptor_docker_logger_0aa5a411831bd10e) +} + +var fileDescriptor_docker_logger_0aa5a411831bd10e = []byte{ + // 328 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x50, 0xb1, 0x4e, 0xeb, 0x40, + 0x10, 0x7c, 0xce, 0x8b, 0x9d, 0x70, 0x49, 0x40, 0x3a, 0x09, 0x61, 0xd2, 0x00, 0xae, 0x28, 0x90, + 0x83, 0xa0, 0x22, 0x74, 0x80, 0x90, 0x10, 0x54, 0x49, 0x47, 0x63, 0x19, 0x7b, 0x93, 0x58, 0x31, + 0x5e, 0xb3, 0xb7, 0x41, 0x4a, 0x85, 0xc4, 0x37, 0xf0, 0x75, 0x7c, 0x0d, 0xf2, 0xfa, 0xb0, 0x68, + 0x93, 0xea, 0x6e, 0x67, 0x67, 0x46, 0x3b, 0xa3, 0xce, 0x53, 0xca, 0xde, 0x81, 0xcc, 0x28, 0xc5, + 0x64, 0x09, 0x24, 0x4f, 0x8e, 0xf3, 0x51, 0x49, 0xc8, 0x68, 0xc1, 0x28, 0xc7, 0xf9, 0x1c, 0x28, + 0x14, 0x4c, 0x9f, 0x2d, 0x62, 0xb3, 0xc8, 0x12, 0xa4, 0x32, 0x2c, 0xf0, 0x35, 0x4e, 0x43, 0xeb, + 0x10, 0xd6, 0xe4, 0xd0, 0x3a, 0xd4, 0xec, 0xe0, 0xdb, 0x51, 0xfd, 0x29, 0xc7, 0xc4, 0x13, 0x78, + 0x5b, 0x81, 0x61, 0x3d, 0x54, 0x5d, 0x28, 0xd2, 0x12, 0xb3, 0x82, 0x7d, 0xe7, 0xd8, 0x39, 0xdd, + 0x99, 0x34, 0xb3, 0x3e, 0x51, 0xfd, 0x04, 0x0b, 0x8e, 0xb3, 0x02, 0x28, 0xca, 0x52, 0xbf, 0x25, + 0xfb, 0x5e, 0x83, 0x3d, 0xa4, 0xfa, 0x48, 0xf5, 0x0c, 0xa7, 0xb8, 0xe2, 0x68, 0x96, 0xcd, 0xd0, + 0xff, 0x2f, 0x0c, 0x55, 0x43, 0xf7, 0xd9, 0x0c, 0x2d, 0x01, 0x88, 0x6a, 0x42, 0xbb, 0x21, 0x00, + 0x91, 0x10, 0x0e, 0x55, 0x97, 0x73, 0x13, 0x25, 0x40, 0xec, 0xbb, 0xb2, 0xed, 0x70, 0x6e, 0x6e, + 0x81, 0x58, 0x1f, 0xa8, 0xea, 0x1b, 0x2d, 0x61, 0xed, 0x7b, 0xb2, 0xf1, 0x38, 0x37, 0x8f, 0xb0, + 0xd6, 0xfb, 0xca, 0x13, 0x4d, 0xec, 0x77, 0x04, 0x77, 0x2b, 0x45, 0x1c, 0xec, 0xa9, 0x81, 0xcd, + 0x66, 0x4a, 0x2c, 0x0c, 0x04, 0x03, 0xd5, 0x9b, 0x32, 0x96, 0x36, 0x6b, 0xb0, 0x5b, 0x65, 0xaf, + 0xc6, 0x7a, 0x7d, 0xf1, 0xd5, 0x52, 0xfd, 0x3b, 0x69, 0xe9, 0x49, 0x1a, 0xd5, 0x9f, 0x8e, 0x72, + 0xc5, 0x41, 0x8f, 0xc3, 0x4d, 0x6a, 0x0d, 0xff, 0x56, 0x3a, 0xbc, 0xde, 0x4a, 0x6b, 0x4f, 0xfe, + 0xa7, 0x3f, 0x54, 0xbb, 0xba, 0x52, 0x5f, 0x6d, 0x6a, 0xd3, 0x04, 0x1d, 0x8e, 0xb7, 0x91, 0xfe, + 0x1e, 0x70, 0xd3, 0x79, 0x76, 0x05, 0x7f, 0xf1, 0xe4, 0xb9, 0xfc, 0x09, 0x00, 0x00, 0xff, 0xff, + 0x43, 0xfe, 0x38, 0xd2, 0x95, 0x02, 0x00, 0x00, } diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 28867f43d..d0dde400b 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" + "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" "golang.org/x/net/context" @@ -108,7 +109,7 @@ type TaskConfig struct { // StartTask. This information is needed to rebuild the task state and handler // during recovery. type TaskState struct { - ReattachConfig *utils.ReattachConfig + ReattachConfig *shared.ReattachConfig TaskConfig *drivers.TaskConfig Pid int StartedAt time.Time @@ -187,7 +188,7 @@ func (d *ExecDriver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("failed to decode state: %v", err) } - plugRC, err := utils.ReattachConfigToGoPlugin(taskState.ReattachConfig) + plugRC, err := shared.ReattachConfigToGoPlugin(taskState.ReattachConfig) if err != nil { logger.Error("failed to build reattach config during task recovery", "error", err) return fmt.Errorf("failed to build reattach config: %v", err) @@ -279,7 +280,7 @@ func (d *ExecDriver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *c } driverState := &TaskState{ - ReattachConfig: utils.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), + ReattachConfig: shared.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, TaskConfig: cfg, StartedAt: h.startedAt, diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 9c9a3a3fe..363771515 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -3,6 +3,7 @@ package qemu import ( "errors" "fmt" + "net" "os" "os/exec" "path/filepath" @@ -11,8 +12,6 @@ import ( "strings" "time" - "net" - "github.com/coreos/go-semver/semver" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-plugin" @@ -23,6 +22,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" + "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" "golang.org/x/net/context" @@ -118,7 +118,7 @@ type TaskConfig struct { // This information is needed to rebuild the taskConfig state and handler // during recovery. type TaskState struct { - ReattachConfig *utils.ReattachConfig + ReattachConfig *shared.ReattachConfig TaskConfig *drivers.TaskConfig Pid int StartedAt time.Time @@ -259,7 +259,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("failed to decode taskConfig state from handle: %v", err) } - plugRC, err := utils.ReattachConfigToGoPlugin(taskState.ReattachConfig) + plugRC, err := shared.ReattachConfigToGoPlugin(taskState.ReattachConfig) if err != nil { d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err) @@ -451,7 +451,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru } qemuDriverState := TaskState{ - ReattachConfig: utils.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), + ReattachConfig: shared.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, TaskConfig: cfg, StartedAt: h.startedAt, diff --git a/drivers/rawexec/driver.go b/drivers/rawexec/driver.go index 4363e9a9a..411095475 100644 --- a/drivers/rawexec/driver.go +++ b/drivers/rawexec/driver.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" + "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" "golang.org/x/net/context" @@ -144,7 +145,7 @@ type TaskConfig struct { // StartTask. This information is needed to rebuild the task state and handler // during recovery. type TaskState struct { - ReattachConfig *utils.ReattachConfig + ReattachConfig *shared.ReattachConfig TaskConfig *drivers.TaskConfig Pid int StartedAt time.Time @@ -261,7 +262,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("failed to decode task state from handle: %v", err) } - plugRC, err := utils.ReattachConfigToGoPlugin(taskState.ReattachConfig) + plugRC, err := shared.ReattachConfigToGoPlugin(taskState.ReattachConfig) if err != nil { d.logger.Error("failed to build ReattachConfig from task state", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to build ReattachConfig from task state: %v", err) @@ -347,7 +348,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru } driverState := TaskState{ - ReattachConfig: utils.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), + ReattachConfig: shared.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, TaskConfig: cfg, StartedAt: h.startedAt, diff --git a/drivers/rkt/driver.go b/drivers/rkt/driver.go index e7f89c758..4ffa69f8d 100644 --- a/drivers/rkt/driver.go +++ b/drivers/rkt/driver.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" + "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" rktv1 "github.com/rkt/rkt/api/v1" @@ -160,7 +161,7 @@ type TaskConfig struct { // StartTask. This information is needed to rebuild the taskConfig state and handler // during recovery. type TaskState struct { - ReattachConfig *utils.ReattachConfig + ReattachConfig *shared.ReattachConfig TaskConfig *drivers.TaskConfig Pid int StartedAt time.Time @@ -332,7 +333,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("failed to decode taskConfig state from handle: %v", err) } - plugRC, err := utils.ReattachConfigToGoPlugin(taskState.ReattachConfig) + plugRC, err := shared.ReattachConfigToGoPlugin(taskState.ReattachConfig) if err != nil { d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err) @@ -675,7 +676,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru } rktDriverState := TaskState{ - ReattachConfig: utils.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), + ReattachConfig: shared.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, TaskConfig: cfg, StartedAt: h.startedAt, diff --git a/plugins/base/plugin.go b/plugins/base/plugin.go index 739c877d5..a386d2c45 100644 --- a/plugins/base/plugin.go +++ b/plugins/base/plugin.go @@ -3,6 +3,7 @@ package base import ( "bytes" "context" + "errors" "reflect" plugin "github.com/hashicorp/go-plugin" @@ -29,6 +30,9 @@ var ( MagicCookieKey: "NOMAD_PLUGIN_MAGIC_COOKIE", MagicCookieValue: "e4327c2e01eabfd75a8a67adb114fb34a757d57eee7728d857a8cec6e91a7255", } + + // ErrPluginShutdown is returned when the plugin has shutdown. + ErrPluginShutdown = errors.New("plugin is shut down") ) // PluginBase is wraps a BasePlugin and implements go-plugins GRPCPlugin diff --git a/plugins/base/proto/base.pb.go b/plugins/base/proto/base.pb.go index 2c3616895..dff5d046f 100644 --- a/plugins/base/proto/base.pb.go +++ b/plugins/base/proto/base.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: base.proto +// source: plugins/base/proto/base.proto package proto @@ -48,7 +48,7 @@ func (x PluginType) String() string { return proto.EnumName(PluginType_name, int32(x)) } func (PluginType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{0} + return fileDescriptor_base_6813a9f13eda45d4, []int{0} } // PluginInfoRequest is used to request the plugins basic information. @@ -62,7 +62,7 @@ func (m *PluginInfoRequest) Reset() { *m = PluginInfoRequest{} } func (m *PluginInfoRequest) String() string { return proto.CompactTextString(m) } func (*PluginInfoRequest) ProtoMessage() {} func (*PluginInfoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{0} + return fileDescriptor_base_6813a9f13eda45d4, []int{0} } func (m *PluginInfoRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginInfoRequest.Unmarshal(m, b) @@ -104,7 +104,7 @@ func (m *PluginInfoResponse) Reset() { *m = PluginInfoResponse{} } func (m *PluginInfoResponse) String() string { return proto.CompactTextString(m) } func (*PluginInfoResponse) ProtoMessage() {} func (*PluginInfoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{1} + return fileDescriptor_base_6813a9f13eda45d4, []int{1} } func (m *PluginInfoResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PluginInfoResponse.Unmarshal(m, b) @@ -163,7 +163,7 @@ func (m *ConfigSchemaRequest) Reset() { *m = ConfigSchemaRequest{} } func (m *ConfigSchemaRequest) String() string { return proto.CompactTextString(m) } func (*ConfigSchemaRequest) ProtoMessage() {} func (*ConfigSchemaRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{2} + return fileDescriptor_base_6813a9f13eda45d4, []int{2} } func (m *ConfigSchemaRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ConfigSchemaRequest.Unmarshal(m, b) @@ -196,7 +196,7 @@ func (m *ConfigSchemaResponse) Reset() { *m = ConfigSchemaResponse{} } func (m *ConfigSchemaResponse) String() string { return proto.CompactTextString(m) } func (*ConfigSchemaResponse) ProtoMessage() {} func (*ConfigSchemaResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{3} + return fileDescriptor_base_6813a9f13eda45d4, []int{3} } func (m *ConfigSchemaResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ConfigSchemaResponse.Unmarshal(m, b) @@ -238,7 +238,7 @@ func (m *SetConfigRequest) Reset() { *m = SetConfigRequest{} } func (m *SetConfigRequest) String() string { return proto.CompactTextString(m) } func (*SetConfigRequest) ProtoMessage() {} func (*SetConfigRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{4} + return fileDescriptor_base_6813a9f13eda45d4, []int{4} } func (m *SetConfigRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SetConfigRequest.Unmarshal(m, b) @@ -285,7 +285,7 @@ func (m *NomadConfig) Reset() { *m = NomadConfig{} } func (m *NomadConfig) String() string { return proto.CompactTextString(m) } func (*NomadConfig) ProtoMessage() {} func (*NomadConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{5} + return fileDescriptor_base_6813a9f13eda45d4, []int{5} } func (m *NomadConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NomadConfig.Unmarshal(m, b) @@ -330,7 +330,7 @@ func (m *NomadDriverConfig) Reset() { *m = NomadDriverConfig{} } func (m *NomadDriverConfig) String() string { return proto.CompactTextString(m) } func (*NomadDriverConfig) ProtoMessage() {} func (*NomadDriverConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{6} + return fileDescriptor_base_6813a9f13eda45d4, []int{6} } func (m *NomadDriverConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NomadDriverConfig.Unmarshal(m, b) @@ -375,7 +375,7 @@ func (m *SetConfigResponse) Reset() { *m = SetConfigResponse{} } func (m *SetConfigResponse) String() string { return proto.CompactTextString(m) } func (*SetConfigResponse) ProtoMessage() {} func (*SetConfigResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_base_9f9181869eb492df, []int{7} + return fileDescriptor_base_6813a9f13eda45d4, []int{7} } func (m *SetConfigResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SetConfigResponse.Unmarshal(m, b) @@ -548,43 +548,44 @@ var _BasePlugin_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{}, - Metadata: "base.proto", + Metadata: "plugins/base/proto/base.proto", } -func init() { proto.RegisterFile("base.proto", fileDescriptor_base_9f9181869eb492df) } +func init() { proto.RegisterFile("plugins/base/proto/base.proto", fileDescriptor_base_6813a9f13eda45d4) } -var fileDescriptor_base_9f9181869eb492df = []byte{ - // 512 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0xdf, 0x6f, 0x12, 0x41, - 0x10, 0xc7, 0x7b, 0x80, 0x34, 0x1d, 0xa0, 0xa1, 0x5b, 0x4d, 0x08, 0x4f, 0xcd, 0x45, 0x93, 0xc6, - 0x34, 0x7b, 0x11, 0x45, 0x7d, 0xac, 0x50, 0x1e, 0x88, 0x29, 0x36, 0x87, 0xa2, 0x31, 0x26, 0x64, - 0x39, 0xb6, 0xdc, 0x46, 0x6e, 0x77, 0xbd, 0x3d, 0x1a, 0x6b, 0xe2, 0x93, 0xcf, 0xfd, 0xa3, 0xfc, - 0xcf, 0xcc, 0xed, 0xee, 0xc1, 0xe1, 0x8f, 0x78, 0x3c, 0xed, 0x32, 0xf3, 0x99, 0xef, 0xec, 0x0c, - 0xdf, 0x03, 0x98, 0x11, 0x45, 0xb1, 0x8c, 0x45, 0x22, 0x90, 0x1b, 0x12, 0x15, 0xb2, 0x40, 0xc4, - 0x12, 0x73, 0x11, 0x91, 0x39, 0x96, 0xcb, 0xd5, 0x82, 0x71, 0x85, 0x37, 0x4c, 0xfb, 0x7c, 0xc1, - 0x92, 0x70, 0x35, 0xc3, 0x81, 0x88, 0xbc, 0x35, 0xee, 0x69, 0xdc, 0xb3, 0xb8, 0xa7, 0x42, 0x12, - 0xd3, 0xb9, 0x17, 0x06, 0x4b, 0x25, 0x69, 0x90, 0x9e, 0xd3, 0xf4, 0x62, 0x14, 0xdc, 0x63, 0x38, - 0xba, 0xd2, 0xe0, 0x90, 0x5f, 0x0b, 0x9f, 0x7e, 0x59, 0x51, 0x95, 0xb8, 0x3f, 0x1d, 0x40, 0xf9, - 0xa8, 0x92, 0x82, 0x2b, 0x8a, 0x7a, 0x50, 0x49, 0x6e, 0x25, 0x6d, 0x39, 0x27, 0xce, 0xe9, 0x61, - 0x07, 0xe3, 0xff, 0x3f, 0x10, 0x1b, 0x95, 0xb7, 0xb7, 0x92, 0xfa, 0xba, 0x16, 0x9d, 0x01, 0x32, - 0xd8, 0x94, 0x48, 0x36, 0xbd, 0xa1, 0xb1, 0x62, 0x82, 0xb7, 0x4a, 0x27, 0xce, 0xe9, 0x81, 0xdf, - 0x34, 0x99, 0x57, 0x92, 0x4d, 0x4c, 0x1c, 0x3d, 0x82, 0x43, 0x4b, 0x67, 0x64, 0x59, 0x93, 0x0d, - 0x13, 0xcd, 0x30, 0x04, 0x15, 0x4e, 0x22, 0xda, 0xaa, 0xe8, 0xa4, 0xbe, 0xbb, 0x0f, 0xe0, 0xb8, - 0x2f, 0xf8, 0x35, 0x5b, 0x8c, 0x83, 0x90, 0x46, 0x24, 0x1b, 0xed, 0x03, 0xdc, 0xdf, 0x0e, 0xdb, - 0xd9, 0xce, 0xa1, 0x92, 0x6e, 0x45, 0xcf, 0x56, 0xeb, 0x9c, 0xfd, 0x73, 0x36, 0xb3, 0x4d, 0x6c, - 0xb7, 0x89, 0xc7, 0x92, 0x06, 0xbe, 0xae, 0x74, 0xef, 0x1c, 0x68, 0x8e, 0x69, 0x62, 0xd4, 0x6d, - 0xbb, 0x74, 0x80, 0x48, 0x2d, 0x24, 0x09, 0x3e, 0x4f, 0x03, 0x9d, 0xd0, 0x0d, 0xea, 0x7e, 0xc3, - 0x46, 0x0d, 0x8d, 0x7c, 0xa8, 0xeb, 0x36, 0x19, 0x54, 0xd2, 0xaf, 0xf0, 0x8a, 0x6c, 0x78, 0x94, - 0x26, 0x6c, 0xd3, 0x1a, 0xdf, 0xfc, 0x70, 0x3f, 0x41, 0x2d, 0x97, 0x43, 0x97, 0x50, 0x9d, 0xc7, - 0xec, 0x86, 0xc6, 0x76, 0xc4, 0x6e, 0x61, 0xf1, 0x0b, 0x5d, 0x66, 0x5b, 0x58, 0x11, 0x77, 0x0a, - 0x47, 0x7f, 0x24, 0xd1, 0x43, 0x68, 0xf4, 0x97, 0x8c, 0xf2, 0xe4, 0x92, 0x7c, 0xbd, 0x12, 0x71, - 0xa2, 0x5b, 0x35, 0xfc, 0xed, 0x60, 0x8e, 0x62, 0x5c, 0x53, 0xa5, 0x2d, 0xca, 0x04, 0x53, 0x63, - 0xe6, 0xb6, 0x69, 0xfe, 0xa5, 0xc7, 0x4f, 0x00, 0x36, 0x8e, 0x42, 0x35, 0xd8, 0x7f, 0x37, 0x7a, - 0x3d, 0x7a, 0xf3, 0x7e, 0xd4, 0xdc, 0x43, 0x00, 0xd5, 0x0b, 0x7f, 0x38, 0x19, 0xf8, 0xcd, 0x92, - 0xbe, 0x0f, 0x26, 0xc3, 0xfe, 0xa0, 0x59, 0xee, 0xdc, 0x95, 0x01, 0x7a, 0x44, 0x51, 0x53, 0x87, - 0xbe, 0x67, 0x0a, 0xa9, 0xb3, 0x51, 0xb7, 0xb8, 0x87, 0x73, 0xdf, 0x47, 0xfb, 0xf9, 0xae, 0x65, - 0xe6, 0xf9, 0xee, 0x1e, 0xfa, 0xe1, 0x40, 0x3d, 0xef, 0x3f, 0xf4, 0xa2, 0x88, 0xd4, 0x5f, 0x8c, - 0xdc, 0x7e, 0xb9, 0x7b, 0xe1, 0xfa, 0x15, 0xdf, 0xe0, 0x60, 0xbd, 0x5b, 0xf4, 0xac, 0x88, 0xd0, - 0xef, 0xc6, 0x6e, 0x77, 0x77, 0xac, 0xca, 0x7a, 0xf7, 0xf6, 0x3f, 0xde, 0xd3, 0xc9, 0x59, 0x55, - 0x1f, 0x4f, 0x7f, 0x05, 0x00, 0x00, 0xff, 0xff, 0x20, 0xf9, 0x80, 0xf0, 0xf4, 0x04, 0x00, 0x00, +var fileDescriptor_base_6813a9f13eda45d4 = []byte{ + // 519 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x93, 0x5d, 0x6b, 0x1a, 0x4d, + 0x14, 0xc7, 0xb3, 0xea, 0x63, 0xc8, 0x51, 0x83, 0x99, 0x3c, 0x05, 0x11, 0x0a, 0x61, 0x69, 0x21, + 0x94, 0xb0, 0x4b, 0x6d, 0x6d, 0x7b, 0x99, 0x6a, 0xbc, 0x90, 0x12, 0x1b, 0xd6, 0xd6, 0x96, 0x52, + 0x90, 0x71, 0x9d, 0xb8, 0x43, 0x75, 0x66, 0xba, 0xb3, 0x86, 0xa6, 0xd0, 0xab, 0x5e, 0xe7, 0x43, + 0xf5, 0x9b, 0x95, 0x3d, 0x33, 0xab, 0x6b, 0x5f, 0xa8, 0x5e, 0xcd, 0xf1, 0x9c, 0xdf, 0xf9, 0x9f, + 0x17, 0xcf, 0xc2, 0x7d, 0x35, 0x5f, 0xce, 0xb8, 0xd0, 0xfe, 0x84, 0x6a, 0xe6, 0xab, 0x58, 0x26, + 0x12, 0x4d, 0x0f, 0x4d, 0xe2, 0x46, 0x54, 0x47, 0x3c, 0x94, 0xb1, 0xf2, 0x84, 0x5c, 0xd0, 0xa9, + 0x67, 0x71, 0x6f, 0xcd, 0x34, 0xcf, 0x67, 0x3c, 0x89, 0x96, 0x13, 0x2f, 0x94, 0x0b, 0x7f, 0x85, + 0xfb, 0x88, 0xfb, 0x99, 0xba, 0x8e, 0x68, 0xcc, 0xa6, 0x7e, 0x14, 0xce, 0xb5, 0x62, 0x61, 0xfa, + 0x8e, 0x53, 0xc3, 0x28, 0xb8, 0xc7, 0x70, 0x74, 0x85, 0x60, 0x5f, 0x5c, 0xcb, 0x80, 0x7d, 0x5e, + 0x32, 0x9d, 0xb8, 0x3f, 0x1c, 0x20, 0x79, 0xaf, 0x56, 0x52, 0x68, 0x46, 0x3a, 0x50, 0x4a, 0x6e, + 0x15, 0x6b, 0x38, 0x27, 0xce, 0xe9, 0x61, 0xcb, 0xf3, 0xfe, 0xdd, 0xa0, 0x67, 0x54, 0xde, 0xdc, + 0x2a, 0x16, 0x60, 0x2e, 0x39, 0x03, 0x62, 0xb0, 0x31, 0x55, 0x7c, 0x7c, 0xc3, 0x62, 0xcd, 0xa5, + 0x68, 0x14, 0x4e, 0x9c, 0xd3, 0x83, 0xa0, 0x6e, 0x22, 0x2f, 0x15, 0x1f, 0x19, 0x3f, 0x79, 0x08, + 0x87, 0x96, 0xce, 0xc8, 0x22, 0x92, 0x35, 0xe3, 0xcd, 0x30, 0x02, 0x25, 0x41, 0x17, 0xac, 0x51, + 0xc2, 0x20, 0xda, 0xee, 0x3d, 0x38, 0xee, 0x4a, 0x71, 0xcd, 0x67, 0xc3, 0x30, 0x62, 0x0b, 0x9a, + 0x8d, 0xf6, 0x1e, 0xfe, 0xdf, 0x74, 0xdb, 0xd9, 0xce, 0xa1, 0x94, 0x6e, 0x05, 0x67, 0xab, 0xb4, + 0xce, 0xfe, 0x3a, 0x9b, 0xd9, 0xa6, 0x67, 0xb7, 0xe9, 0x0d, 0x15, 0x0b, 0x03, 0xcc, 0x74, 0xef, + 0x1c, 0xa8, 0x0f, 0x59, 0x62, 0xd4, 0x6d, 0xb9, 0x74, 0x80, 0x85, 0x9e, 0x29, 0x1a, 0x7e, 0x1a, + 0x87, 0x18, 0xc0, 0x02, 0xd5, 0xa0, 0x66, 0xbd, 0x86, 0x26, 0x01, 0x54, 0xb1, 0x4c, 0x06, 0x15, + 0xb0, 0x0b, 0x7f, 0x9b, 0x0d, 0x0f, 0xd2, 0x80, 0x2d, 0x5a, 0x11, 0xeb, 0x1f, 0xee, 0x47, 0xa8, + 0xe4, 0x62, 0xe4, 0x12, 0xca, 0xd3, 0x98, 0xdf, 0xb0, 0xd8, 0x8e, 0xd8, 0xde, 0x5a, 0xfc, 0x02, + 0xd3, 0x6c, 0x09, 0x2b, 0xe2, 0x8e, 0xe1, 0xe8, 0xb7, 0x20, 0x79, 0x00, 0xb5, 0xee, 0x9c, 0x33, + 0x91, 0x5c, 0xd2, 0x2f, 0x57, 0x32, 0x4e, 0xb0, 0x54, 0x2d, 0xd8, 0x74, 0xe6, 0x28, 0x2e, 0x90, + 0x2a, 0x6c, 0x50, 0xc6, 0x99, 0x1e, 0x66, 0x6e, 0x9b, 0xe6, 0x5f, 0x7a, 0xf4, 0x18, 0x60, 0x7d, + 0x51, 0xa4, 0x02, 0xfb, 0x6f, 0x07, 0xaf, 0x06, 0xaf, 0xdf, 0x0d, 0xea, 0x7b, 0x04, 0xa0, 0x7c, + 0x11, 0xf4, 0x47, 0xbd, 0xa0, 0x5e, 0x40, 0xbb, 0x37, 0xea, 0x77, 0x7b, 0xf5, 0x62, 0xeb, 0xae, + 0x08, 0xd0, 0xa1, 0x9a, 0x99, 0x3c, 0xf2, 0x2d, 0x53, 0x48, 0x2f, 0x9b, 0xb4, 0xb7, 0xbf, 0xe1, + 0xdc, 0xf7, 0xd1, 0x7c, 0xb6, 0x6b, 0x9a, 0x69, 0xdf, 0xdd, 0x23, 0xdf, 0x1d, 0xa8, 0xe6, 0xef, + 0x8f, 0x3c, 0xdf, 0x46, 0xea, 0x0f, 0x87, 0xdc, 0x7c, 0xb1, 0x7b, 0xe2, 0xaa, 0x8b, 0xaf, 0x70, + 0xb0, 0xda, 0x2d, 0x79, 0xba, 0x8d, 0xd0, 0xaf, 0x87, 0xdd, 0x6c, 0xef, 0x98, 0x95, 0xd5, 0xee, + 0xec, 0x7f, 0xf8, 0x0f, 0x83, 0x93, 0x32, 0x3e, 0x4f, 0x7e, 0x06, 0x00, 0x00, 0xff, 0xff, 0x53, + 0xfe, 0xaa, 0x3a, 0x07, 0x05, 0x00, 0x00, } diff --git a/plugins/base/server.go b/plugins/base/server.go index 82fd1c210..f7ccc3fb7 100644 --- a/plugins/base/server.go +++ b/plugins/base/server.go @@ -3,10 +3,9 @@ package base import ( "fmt" - "golang.org/x/net/context" - plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/plugins/base/proto" + "golang.org/x/net/context" ) // basePluginServer wraps a base plugin and exposes it via gRPC. diff --git a/plugins/device/client.go b/plugins/device/client.go index 3868642b7..4548689cb 100644 --- a/plugins/device/client.go +++ b/plugins/device/client.go @@ -3,7 +3,10 @@ package device import ( "context" "io" + "time" + "github.com/LK4D4/joincontext" + "github.com/golang/protobuf/ptypes" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/device/proto" netctx "golang.org/x/net/context" @@ -18,6 +21,9 @@ type devicePluginClient struct { *base.BasePluginClient client proto.DevicePluginClient + + // doneCtx is closed when the plugin exits + doneCtx context.Context } // Fingerprint is used to retrieve the set of devices and their health from the @@ -25,6 +31,9 @@ type devicePluginClient struct { // could not be made or as part of the streaming response. If the context is // cancelled, the error will be propogated. func (d *devicePluginClient) Fingerprint(ctx context.Context) (<-chan *FingerprintResponse, error) { + // Join the passed context and the shutdown context + ctx, _ = joincontext.Join(ctx, d.doneCtx) + var req proto.FingerprintRequest stream, err := d.client.Fingerprint(ctx, &req) if err != nil { @@ -47,14 +56,9 @@ func (d *devicePluginClient) handleFingerprint( for { resp, err := stream.Recv() if err != nil { - // Handle a non-graceful stream error if err != io.EOF { - if errStatus := status.FromContextError(ctx.Err()); errStatus.Code() == codes.Canceled { - err = context.Canceled - } - out <- &FingerprintResponse{ - Error: err, + Error: d.handleStreamErr(err, ctx), } } @@ -91,8 +95,13 @@ func (d *devicePluginClient) Reserve(deviceIDs []string) (*ContainerReservation, // may be immediately returned if the stats call could not be made or as part of // the streaming response. If the context is cancelled, the error will be // propogated. -func (d *devicePluginClient) Stats(ctx context.Context) (<-chan *StatsResponse, error) { - var req proto.StatsRequest +func (d *devicePluginClient) Stats(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { + // Join the passed context and the shutdown context + ctx, _ = joincontext.Join(ctx, d.doneCtx) + + req := proto.StatsRequest{ + CollectionInterval: ptypes.DurationProto(interval), + } stream, err := d.client.Stats(ctx, &req) if err != nil { return nil, err @@ -114,14 +123,9 @@ func (d *devicePluginClient) handleStats( for { resp, err := stream.Recv() if err != nil { - // Handle a non-graceful stream error if err != io.EOF { - if errStatus := status.FromContextError(ctx.Err()); errStatus.Code() == codes.Canceled { - err = context.Canceled - } - out <- &StatsResponse{ - Error: err, + Error: d.handleStreamErr(err, ctx), } } @@ -136,3 +140,45 @@ func (d *devicePluginClient) handleStats( } } } + +// handleStreamErr is used to handle a non io.EOF error in a stream. It handles +// detecting if the plugin has shutdown +func (d *devicePluginClient) handleStreamErr(err error, ctx context.Context) error { + if err == nil { + return nil + } + + // Determine if the error is because the plugin shutdown + if errStatus, ok := status.FromError(err); ok && errStatus.Code() == codes.Unavailable { + // Potentially wait a little before returning an error so we can detect + // the exit + select { + case <-d.doneCtx.Done(): + err = base.ErrPluginShutdown + case <-ctx.Done(): + err = ctx.Err() + + // There is no guarantee that the select will choose the + // doneCtx first so we have to double check + select { + case <-d.doneCtx.Done(): + err = base.ErrPluginShutdown + default: + } + case <-time.After(3 * time.Second): + // Its okay to wait a while since the connection isn't available and + // on local host it is likely shutting down. It is not expected for + // this to ever reach even close to 3 seconds. + } + + // It is an error we don't know how to handle, so return it + return err + } + + // Context was cancelled + if errStatus := status.FromContextError(ctx.Err()); errStatus.Code() == codes.Canceled { + return context.Canceled + } + + return err +} diff --git a/plugins/device/cmd/example/README.md b/plugins/device/cmd/example/README.md index 09dfe9b41..04fbe833d 100644 --- a/plugins/device/cmd/example/README.md +++ b/plugins/device/cmd/example/README.md @@ -13,7 +13,6 @@ The configuration should be passed via an HCL file that begins with a top level config { dir = "/my/path/to/scan" list_period = "1s" - stats_period = "5s" unhealthy_perm = "-rw-rw-rw-" } ``` @@ -22,5 +21,4 @@ The valid configuration options are: * `dir` (`string`: `"."`): The directory to scan for files that will represent fake devices. * `list_period` (`string`: `"5s"`): The interval to scan the directory for changes. -* `stats_period` (`string`: `"5s"`): The interval at which to emit statistics about the devices. * `unhealthy_perm` (`string`: `"-rwxrwxrwx"`): The file mode permission that if set on a detected file will casue the device to be considered unhealthy. diff --git a/plugins/device/cmd/example/device.go b/plugins/device/cmd/example/device.go index f486d72cf..943d3b981 100644 --- a/plugins/device/cmd/example/device.go +++ b/plugins/device/cmd/example/device.go @@ -10,13 +10,12 @@ import ( "time" log "github.com/hashicorp/go-hclog" - "github.com/kr/pretty" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/device" "github.com/hashicorp/nomad/plugins/shared/hclspec" + "github.com/kr/pretty" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -56,10 +55,6 @@ var ( hclspec.NewAttr("unhealthy_perm", "string", false), hclspec.NewLiteral("\"-rwxrwxrwx\""), ), - "stats_period": hclspec.NewDefault( - hclspec.NewAttr("stats_period", "string", false), - hclspec.NewLiteral("\"5s\""), - ), }) ) @@ -67,7 +62,6 @@ var ( type Config struct { Dir string `codec:"dir"` ListPeriod string `codec:"list_period"` - StatsPeriod string `codec:"stats_period"` UnhealthyPerm string `codec:"unhealthy_perm"` } @@ -88,10 +82,6 @@ type FsDevice struct { // devices listPeriod time.Duration - // statsPeriod is how often we should collect statistics for fingerprinted - // devices. - statsPeriod time.Duration - // devices is the set of detected devices and maps whether they are healthy devices map[string]bool deviceLock sync.RWMutex @@ -133,13 +123,6 @@ func (d *FsDevice) SetConfig(data []byte, cfg *base.ClientAgentConfig) error { } d.listPeriod = period - // Convert the stats period - speriod, err := time.ParseDuration(config.StatsPeriod) - if err != nil { - return fmt.Errorf("failed to parse list period %q: %v", config.StatsPeriod, err) - } - d.statsPeriod = speriod - d.logger.Debug("test debug") d.logger.Info("config set", "config", log.Fmt("% #v", pretty.Formatter(config))) return nil @@ -290,14 +273,14 @@ func (d *FsDevice) Reserve(deviceIDs []string) (*device.ContainerReservation, er } // Stats streams statistics for the detected devices. -func (d *FsDevice) Stats(ctx context.Context) (<-chan *device.StatsResponse, error) { +func (d *FsDevice) Stats(ctx context.Context, interval time.Duration) (<-chan *device.StatsResponse, error) { outCh := make(chan *device.StatsResponse) - go d.stats(ctx, outCh) + go d.stats(ctx, outCh, interval) return outCh, nil } // stats is the long running goroutine that streams device statistics -func (d *FsDevice) stats(ctx context.Context, stats chan *device.StatsResponse) { +func (d *FsDevice) stats(ctx context.Context, stats chan *device.StatsResponse, interval time.Duration) { defer close(stats) // Create a timer that will fire immediately for the first detection @@ -308,7 +291,7 @@ func (d *FsDevice) stats(ctx context.Context, stats chan *device.StatsResponse) case <-ctx.Done(): return case <-ticker.C: - ticker.Reset(d.listPeriod) + ticker.Reset(interval) } deviceStats, err := d.collectStats() diff --git a/plugins/device/device.go b/plugins/device/device.go index 11ceb9344..cdaf8c2b1 100644 --- a/plugins/device/device.go +++ b/plugins/device/device.go @@ -2,8 +2,10 @@ package device import ( "context" + "fmt" "time" + multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/shared/structs" ) @@ -25,8 +27,9 @@ type DevicePlugin interface { // instructions. Reserve(deviceIDs []string) (*ContainerReservation, error) - // Stats returns a stream of statistics per device. - Stats(ctx context.Context) (<-chan *StatsResponse, error) + // Stats returns a stream of statistics per device collected at the passed + // interval. + Stats(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) } // FingerprintResponse includes a set of detected devices or an error in the @@ -73,6 +76,41 @@ type DeviceGroup struct { Attributes map[string]*structs.Attribute } +// Validate validates that the device group is valid +func (d *DeviceGroup) Validate() error { + var mErr multierror.Error + + if d.Vendor == "" { + multierror.Append(&mErr, fmt.Errorf("device vendor must be specified")) + } + if d.Type == "" { + multierror.Append(&mErr, fmt.Errorf("device type must be specified")) + } + if d.Name == "" { + multierror.Append(&mErr, fmt.Errorf("device name must be specified")) + } + + for i, dev := range d.Devices { + if dev == nil { + multierror.Append(&mErr, fmt.Errorf("device %d is nil", i)) + continue + } + + if err := dev.Validate(); err != nil { + multierror.Append(&mErr, multierror.Prefix(err, fmt.Sprintf("device %d: ", i))) + } + } + + for k, v := range d.Attributes { + if err := v.Validate(); err != nil { + multierror.Append(&mErr, fmt.Errorf("device attribute %q invalid: %v", k, err)) + } + } + + return mErr.ErrorOrNil() + +} + // Device is an instance of a particular device. type Device struct { // ID is the identifier for the device. @@ -89,6 +127,15 @@ type Device struct { HwLocality *DeviceLocality } +// Validate validates that the device is valid +func (d *Device) Validate() error { + if d.ID == "" { + return fmt.Errorf("device ID must be specified") + } + + return nil +} + // DeviceLocality captures hardware locality information for a device. type DeviceLocality struct { // PciBusID is the PCI bus ID of the device. diff --git a/plugins/device/mock.go b/plugins/device/mock.go index 6f93ef6bb..4fe6c8dd2 100644 --- a/plugins/device/mock.go +++ b/plugins/device/mock.go @@ -2,6 +2,7 @@ package device import ( "context" + "time" "github.com/hashicorp/nomad/plugins/base" ) @@ -13,7 +14,7 @@ type MockDevicePlugin struct { *base.MockPlugin FingerprintF func(context.Context) (<-chan *FingerprintResponse, error) ReserveF func([]string) (*ContainerReservation, error) - StatsF func(context.Context) (<-chan *StatsResponse, error) + StatsF func(context.Context, time.Duration) (<-chan *StatsResponse, error) } func (p *MockDevicePlugin) Fingerprint(ctx context.Context) (<-chan *FingerprintResponse, error) { @@ -24,6 +25,6 @@ func (p *MockDevicePlugin) Reserve(devices []string) (*ContainerReservation, err return p.ReserveF(devices) } -func (p *MockDevicePlugin) Stats(ctx context.Context) (<-chan *StatsResponse, error) { - return p.StatsF(ctx) +func (p *MockDevicePlugin) Stats(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { + return p.StatsF(ctx, interval) } diff --git a/plugins/device/plugin.go b/plugins/device/plugin.go index 1f6d52c3f..f03733857 100644 --- a/plugins/device/plugin.go +++ b/plugins/device/plugin.go @@ -28,7 +28,8 @@ func (p *PluginDevice) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) err func (p *PluginDevice) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { return &devicePluginClient{ - client: proto.NewDevicePluginClient(c), + doneCtx: ctx, + client: proto.NewDevicePluginClient(c), BasePluginClient: &base.BasePluginClient{ Client: bproto.NewBasePluginClient(c), }, diff --git a/plugins/device/plugin_test.go b/plugins/device/plugin_test.go index ab410ce24..67cd25552 100644 --- a/plugins/device/plugin_test.go +++ b/plugins/device/plugin_test.go @@ -9,11 +9,10 @@ import ( pb "github.com/golang/protobuf/proto" plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/helper" - psstructs "github.com/hashicorp/nomad/plugins/shared/structs" - "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/shared/hclspec" + psstructs "github.com/hashicorp/nomad/plugins/shared/structs" "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/require" "github.com/zclconf/go-cty/cty" @@ -521,7 +520,7 @@ func TestDevicePlugin_Stats(t *testing.T) { } mock := &MockDevicePlugin{ - StatsF: func(ctx context.Context) (<-chan *StatsResponse, error) { + StatsF: func(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { outCh := make(chan *StatsResponse, 1) go func() { // Send two messages @@ -561,7 +560,7 @@ func TestDevicePlugin_Stats(t *testing.T) { defer cancel() // Get the stream - stream, err := impl.Stats(ctx) + stream, err := impl.Stats(ctx, time.Millisecond) require.NoError(err) // Get the first message @@ -600,7 +599,7 @@ func TestDevicePlugin_Stats_StreamErr(t *testing.T) { ferr := fmt.Errorf("mock stats failed") mock := &MockDevicePlugin{ - StatsF: func(ctx context.Context) (<-chan *StatsResponse, error) { + StatsF: func(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { outCh := make(chan *StatsResponse, 1) go func() { // Send the error @@ -639,7 +638,7 @@ func TestDevicePlugin_Stats_StreamErr(t *testing.T) { defer cancel() // Get the stream - stream, err := impl.Stats(ctx) + stream, err := impl.Stats(ctx, time.Millisecond) require.NoError(err) // Get the first message @@ -659,7 +658,7 @@ func TestDevicePlugin_Stats_CancelCtx(t *testing.T) { require := require.New(t) mock := &MockDevicePlugin{ - StatsF: func(ctx context.Context) (<-chan *StatsResponse, error) { + StatsF: func(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { outCh := make(chan *StatsResponse, 1) go func() { <-ctx.Done() @@ -691,7 +690,7 @@ func TestDevicePlugin_Stats_CancelCtx(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) // Get the stream - stream, err := impl.Stats(ctx) + stream, err := impl.Stats(ctx, time.Millisecond) require.NoError(err) // Get the first message diff --git a/plugins/device/proto/device.pb.go b/plugins/device/proto/device.pb.go index a5b6c28c8..d09f1583f 100644 --- a/plugins/device/proto/device.pb.go +++ b/plugins/device/proto/device.pb.go @@ -1,11 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: github.com/hashicorp/nomad/plugins/device/proto/device.proto +// source: plugins/device/proto/device.proto package proto import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import duration "github.com/golang/protobuf/ptypes/duration" import timestamp "github.com/golang/protobuf/ptypes/timestamp" import proto1 "github.com/hashicorp/nomad/plugins/shared/structs/proto" @@ -36,7 +37,7 @@ func (m *FingerprintRequest) Reset() { *m = FingerprintRequest{} } func (m *FingerprintRequest) String() string { return proto.CompactTextString(m) } func (*FingerprintRequest) ProtoMessage() {} func (*FingerprintRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{0} + return fileDescriptor_device_4daf5409089620fd, []int{0} } func (m *FingerprintRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintRequest.Unmarshal(m, b) @@ -71,7 +72,7 @@ func (m *FingerprintResponse) Reset() { *m = FingerprintResponse{} } func (m *FingerprintResponse) String() string { return proto.CompactTextString(m) } func (*FingerprintResponse) ProtoMessage() {} func (*FingerprintResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{1} + return fileDescriptor_device_4daf5409089620fd, []int{1} } func (m *FingerprintResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintResponse.Unmarshal(m, b) @@ -120,7 +121,7 @@ func (m *DeviceGroup) Reset() { *m = DeviceGroup{} } func (m *DeviceGroup) String() string { return proto.CompactTextString(m) } func (*DeviceGroup) ProtoMessage() {} func (*DeviceGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{2} + return fileDescriptor_device_4daf5409089620fd, []int{2} } func (m *DeviceGroup) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeviceGroup.Unmarshal(m, b) @@ -197,7 +198,7 @@ func (m *DetectedDevice) Reset() { *m = DetectedDevice{} } func (m *DetectedDevice) String() string { return proto.CompactTextString(m) } func (*DetectedDevice) ProtoMessage() {} func (*DetectedDevice) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{3} + return fileDescriptor_device_4daf5409089620fd, []int{3} } func (m *DetectedDevice) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DetectedDevice.Unmarshal(m, b) @@ -259,7 +260,7 @@ func (m *DeviceLocality) Reset() { *m = DeviceLocality{} } func (m *DeviceLocality) String() string { return proto.CompactTextString(m) } func (*DeviceLocality) ProtoMessage() {} func (*DeviceLocality) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{4} + return fileDescriptor_device_4daf5409089620fd, []int{4} } func (m *DeviceLocality) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeviceLocality.Unmarshal(m, b) @@ -300,7 +301,7 @@ func (m *ReserveRequest) Reset() { *m = ReserveRequest{} } func (m *ReserveRequest) String() string { return proto.CompactTextString(m) } func (*ReserveRequest) ProtoMessage() {} func (*ReserveRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{5} + return fileDescriptor_device_4daf5409089620fd, []int{5} } func (m *ReserveRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ReserveRequest.Unmarshal(m, b) @@ -343,7 +344,7 @@ func (m *ReserveResponse) Reset() { *m = ReserveResponse{} } func (m *ReserveResponse) String() string { return proto.CompactTextString(m) } func (*ReserveResponse) ProtoMessage() {} func (*ReserveResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{6} + return fileDescriptor_device_4daf5409089620fd, []int{6} } func (m *ReserveResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ReserveResponse.Unmarshal(m, b) @@ -388,7 +389,7 @@ func (m *ContainerReservation) Reset() { *m = ContainerReservation{} } func (m *ContainerReservation) String() string { return proto.CompactTextString(m) } func (*ContainerReservation) ProtoMessage() {} func (*ContainerReservation) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{7} + return fileDescriptor_device_4daf5409089620fd, []int{7} } func (m *ContainerReservation) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ContainerReservation.Unmarshal(m, b) @@ -447,7 +448,7 @@ func (m *Mount) Reset() { *m = Mount{} } func (m *Mount) String() string { return proto.CompactTextString(m) } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{8} + return fileDescriptor_device_4daf5409089620fd, []int{8} } func (m *Mount) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Mount.Unmarshal(m, b) @@ -508,7 +509,7 @@ func (m *DeviceSpec) Reset() { *m = DeviceSpec{} } func (m *DeviceSpec) String() string { return proto.CompactTextString(m) } func (*DeviceSpec) ProtoMessage() {} func (*DeviceSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{9} + return fileDescriptor_device_4daf5409089620fd, []int{9} } func (m *DeviceSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeviceSpec.Unmarshal(m, b) @@ -551,16 +552,18 @@ func (m *DeviceSpec) GetPermissions() string { // StatsRequest is used to parameterize the retrieval of statistics. type StatsRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // collection_interval is the duration in which to collect statistics. + CollectionInterval *duration.Duration `protobuf:"bytes,1,opt,name=collection_interval,json=collectionInterval,proto3" json:"collection_interval,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *StatsRequest) Reset() { *m = StatsRequest{} } func (m *StatsRequest) String() string { return proto.CompactTextString(m) } func (*StatsRequest) ProtoMessage() {} func (*StatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{10} + return fileDescriptor_device_4daf5409089620fd, []int{10} } func (m *StatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsRequest.Unmarshal(m, b) @@ -580,6 +583,13 @@ func (m *StatsRequest) XXX_DiscardUnknown() { var xxx_messageInfo_StatsRequest proto.InternalMessageInfo +func (m *StatsRequest) GetCollectionInterval() *duration.Duration { + if m != nil { + return m.CollectionInterval + } + return nil +} + // StatsResponse returns the statistics for each device group. type StatsResponse struct { // groups contains statistics for each device group. @@ -593,7 +603,7 @@ func (m *StatsResponse) Reset() { *m = StatsResponse{} } func (m *StatsResponse) String() string { return proto.CompactTextString(m) } func (*StatsResponse) ProtoMessage() {} func (*StatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{11} + return fileDescriptor_device_4daf5409089620fd, []int{11} } func (m *StatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatsResponse.Unmarshal(m, b) @@ -637,7 +647,7 @@ func (m *DeviceGroupStats) Reset() { *m = DeviceGroupStats{} } func (m *DeviceGroupStats) String() string { return proto.CompactTextString(m) } func (*DeviceGroupStats) ProtoMessage() {} func (*DeviceGroupStats) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{12} + return fileDescriptor_device_4daf5409089620fd, []int{12} } func (m *DeviceGroupStats) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeviceGroupStats.Unmarshal(m, b) @@ -703,7 +713,7 @@ func (m *DeviceStats) Reset() { *m = DeviceStats{} } func (m *DeviceStats) String() string { return proto.CompactTextString(m) } func (*DeviceStats) ProtoMessage() {} func (*DeviceStats) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{13} + return fileDescriptor_device_4daf5409089620fd, []int{13} } func (m *DeviceStats) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DeviceStats.Unmarshal(m, b) @@ -760,7 +770,7 @@ func (m *StatObject) Reset() { *m = StatObject{} } func (m *StatObject) String() string { return proto.CompactTextString(m) } func (*StatObject) ProtoMessage() {} func (*StatObject) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{14} + return fileDescriptor_device_4daf5409089620fd, []int{14} } func (m *StatObject) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatObject.Unmarshal(m, b) @@ -825,7 +835,7 @@ func (m *StatValue) Reset() { *m = StatValue{} } func (m *StatValue) String() string { return proto.CompactTextString(m) } func (*StatValue) ProtoMessage() {} func (*StatValue) Descriptor() ([]byte, []int) { - return fileDescriptor_device_c5ad1463d0f8d225, []int{15} + return fileDescriptor_device_4daf5409089620fd, []int{15} } func (m *StatValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StatValue.Unmarshal(m, b) @@ -1131,84 +1141,86 @@ var _DevicePlugin_serviceDesc = grpc.ServiceDesc{ ServerStreams: true, }, }, - Metadata: "github.com/hashicorp/nomad/plugins/device/proto/device.proto", + Metadata: "plugins/device/proto/device.proto", } func init() { - proto.RegisterFile("github.com/hashicorp/nomad/plugins/device/proto/device.proto", fileDescriptor_device_c5ad1463d0f8d225) + proto.RegisterFile("plugins/device/proto/device.proto", fileDescriptor_device_4daf5409089620fd) } -var fileDescriptor_device_c5ad1463d0f8d225 = []byte{ - // 1129 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xef, 0x8e, 0xdb, 0x44, - 0x10, 0x27, 0xff, 0x93, 0xc9, 0xf5, 0xda, 0xdb, 0x3b, 0x50, 0x08, 0x94, 0x9e, 0x2c, 0x21, 0x95, - 0x83, 0x3a, 0x25, 0x45, 0x50, 0x95, 0x7f, 0x6d, 0x2f, 0xe5, 0x7a, 0x12, 0xdc, 0x55, 0x6e, 0x55, - 0xa9, 0x45, 0xc2, 0xda, 0xd8, 0xdb, 0x78, 0x5b, 0x7b, 0xd7, 0x78, 0xd7, 0xa9, 0xc2, 0x27, 0xbe, - 0xf2, 0x26, 0x7c, 0xe9, 0x0b, 0xf0, 0x10, 0x3c, 0x04, 0x4f, 0x82, 0xf6, 0x8f, 0x13, 0xe7, 0x7a, - 0x6d, 0x12, 0xf8, 0xe4, 0xdd, 0x99, 0xf9, 0xfd, 0x66, 0x3c, 0x3b, 0x3b, 0x3b, 0xf0, 0xcd, 0x84, - 0xca, 0x28, 0x1f, 0xbb, 0x01, 0x4f, 0x06, 0x11, 0x16, 0x11, 0x0d, 0x78, 0x96, 0x0e, 0x18, 0x4f, - 0x70, 0x38, 0x48, 0xe3, 0x7c, 0x42, 0x99, 0x18, 0x84, 0x64, 0x4a, 0x03, 0x32, 0x48, 0x33, 0x2e, - 0xb9, 0xdd, 0xb8, 0x7a, 0x83, 0x3e, 0x9a, 0x43, 0x5c, 0x0d, 0x71, 0x2d, 0xc4, 0x35, 0x56, 0xfd, - 0x2b, 0x13, 0xce, 0x27, 0xb1, 0x85, 0x8e, 0xf3, 0x67, 0x03, 0x49, 0x13, 0x22, 0x24, 0x4e, 0x52, - 0x43, 0xd0, 0x3f, 0x5a, 0xc3, 0xbd, 0x88, 0x70, 0x46, 0xc2, 0x81, 0x90, 0x59, 0x1e, 0x48, 0x61, - 0xc3, 0xc0, 0x52, 0x66, 0x74, 0x9c, 0x4b, 0x1b, 0x89, 0xb3, 0x07, 0xe8, 0x07, 0xca, 0x26, 0x24, - 0x4b, 0x33, 0xca, 0xa4, 0x47, 0x7e, 0xcd, 0x89, 0x90, 0x0e, 0x81, 0xdd, 0x25, 0xa9, 0x48, 0x39, - 0x13, 0x04, 0x9d, 0xc0, 0x96, 0x09, 0xd0, 0x9f, 0x64, 0x3c, 0x4f, 0x7b, 0x95, 0xfd, 0xda, 0xd5, - 0xee, 0xf0, 0x53, 0xf7, 0xed, 0x7f, 0xe3, 0x8e, 0xf4, 0xe7, 0x48, 0x41, 0xbc, 0x6e, 0xb8, 0xd8, - 0x38, 0xbf, 0xd7, 0xa0, 0x5b, 0x52, 0xa2, 0xf7, 0xa0, 0x39, 0x25, 0x2c, 0xe4, 0x59, 0xaf, 0xb2, - 0x5f, 0xb9, 0xda, 0xf1, 0xec, 0x0e, 0x5d, 0x01, 0x0b, 0xf3, 0xe5, 0x2c, 0x25, 0xbd, 0xaa, 0x56, - 0x82, 0x11, 0x3d, 0x9a, 0xa5, 0xa4, 0x64, 0xc0, 0x70, 0x42, 0x7a, 0xb5, 0xb2, 0xc1, 0x09, 0x4e, - 0x08, 0xba, 0x0f, 0x2d, 0xb3, 0x13, 0xbd, 0xba, 0x0e, 0xda, 0x5d, 0x1d, 0xb4, 0x24, 0x81, 0x24, - 0xa1, 0x89, 0xcf, 0x2b, 0xe0, 0xe8, 0x67, 0x80, 0x79, 0x0e, 0x45, 0xaf, 0xa1, 0xc9, 0xbe, 0xde, - 0x20, 0x03, 0xee, 0x9d, 0x39, 0xfa, 0x1e, 0x93, 0xd9, 0xcc, 0x2b, 0xd1, 0xf5, 0x53, 0xb8, 0x78, - 0x46, 0x8d, 0x2e, 0x41, 0xed, 0x05, 0x99, 0xd9, 0x84, 0xa8, 0x25, 0x3a, 0x82, 0xc6, 0x14, 0xc7, - 0xb9, 0xc9, 0x43, 0x77, 0xf8, 0xf9, 0x1b, 0x9d, 0x9b, 0x02, 0x70, 0x6d, 0x01, 0x2c, 0x1c, 0x7b, - 0x06, 0x7f, 0xab, 0x7a, 0xb3, 0xe2, 0xfc, 0x55, 0x81, 0xed, 0xe5, 0x5f, 0x45, 0xdb, 0x50, 0x3d, - 0x1e, 0x59, 0x87, 0xd5, 0xe3, 0x11, 0xea, 0x41, 0x2b, 0x22, 0x38, 0x96, 0xd1, 0x4c, 0x7b, 0x6c, - 0x7b, 0xc5, 0x16, 0x5d, 0x03, 0x64, 0x96, 0x7e, 0x48, 0x44, 0x90, 0xd1, 0x54, 0x52, 0xce, 0x6c, - 0xf6, 0x77, 0x8c, 0x66, 0xb4, 0x50, 0xa0, 0x53, 0xe8, 0x46, 0x2f, 0xfd, 0x98, 0x07, 0x38, 0xa6, - 0x72, 0xd6, 0xab, 0xeb, 0xf0, 0xdd, 0xf5, 0x72, 0xf7, 0xa3, 0x45, 0x79, 0x10, 0xbd, 0x2c, 0xd6, - 0x8e, 0xab, 0x62, 0x2f, 0x6b, 0xd1, 0x87, 0x00, 0x69, 0x40, 0xfd, 0x71, 0x2e, 0x7c, 0x1a, 0xda, - 0x7f, 0x68, 0xa7, 0x01, 0xbd, 0x9b, 0x8b, 0xe3, 0xd0, 0x19, 0xc0, 0xb6, 0x47, 0x04, 0xc9, 0xa6, - 0xc4, 0x16, 0x3a, 0xba, 0x0c, 0xb6, 0x4a, 0x7c, 0x1a, 0x0a, 0x5d, 0xcf, 0x1d, 0xaf, 0x63, 0x24, - 0xc7, 0xa1, 0x70, 0x62, 0xb8, 0x38, 0x07, 0xd8, 0x3b, 0xf0, 0x04, 0x2e, 0x04, 0x9c, 0x49, 0x4c, - 0x19, 0xc9, 0xfc, 0x8c, 0x08, 0xed, 0xa4, 0x3b, 0xfc, 0x62, 0xd5, 0x6f, 0x1c, 0x16, 0x20, 0x43, - 0x88, 0x55, 0x46, 0xbc, 0xad, 0xa0, 0x24, 0x75, 0xfe, 0xac, 0xc2, 0xde, 0x79, 0x66, 0xc8, 0x83, - 0x3a, 0x61, 0x53, 0x61, 0xef, 0xdb, 0x77, 0xff, 0xc5, 0x95, 0x7b, 0x8f, 0x4d, 0x6d, 0xc1, 0x69, - 0x2e, 0xf4, 0x2d, 0x34, 0x13, 0x9e, 0x33, 0x29, 0x7a, 0x55, 0xcd, 0xfa, 0xf1, 0x2a, 0xd6, 0x9f, - 0x94, 0xb5, 0x67, 0x41, 0x68, 0xb4, 0xb8, 0x50, 0x35, 0x8d, 0x3f, 0x58, 0xef, 0x1c, 0x1f, 0xa6, - 0x24, 0x98, 0x5f, 0xa6, 0xfe, 0x57, 0xd0, 0x99, 0xc7, 0x75, 0x4e, 0xa5, 0xef, 0x95, 0x2b, 0xbd, - 0x53, 0x2e, 0xdb, 0x5f, 0xa0, 0xa1, 0xe3, 0x41, 0x1f, 0x40, 0x47, 0x62, 0xf1, 0xc2, 0x4f, 0xb1, - 0x8c, 0x8a, 0xf3, 0x56, 0x82, 0x07, 0x58, 0x46, 0x4a, 0x19, 0x71, 0x21, 0x8d, 0xd2, 0x70, 0xb4, - 0x95, 0xa0, 0x50, 0x66, 0x04, 0x87, 0x3e, 0x67, 0xf1, 0x4c, 0xd7, 0x6c, 0xdb, 0x6b, 0x2b, 0xc1, - 0x29, 0x8b, 0x67, 0x4e, 0x04, 0xb0, 0x88, 0xf7, 0x7f, 0x38, 0xd9, 0x87, 0x6e, 0x4a, 0xb2, 0x84, - 0x0a, 0x41, 0x39, 0x13, 0xf6, 0x6a, 0x94, 0x45, 0xce, 0x36, 0x6c, 0x3d, 0x94, 0x58, 0x8a, 0xa2, - 0xf5, 0x3e, 0x81, 0x0b, 0x76, 0x6f, 0x0b, 0xee, 0x3e, 0x34, 0x75, 0xb7, 0x2d, 0x8e, 0xff, 0xfa, - 0x06, 0xcd, 0xc6, 0x30, 0x59, 0xbc, 0xf3, 0xaa, 0x0a, 0x97, 0xce, 0x2a, 0xdf, 0xd8, 0x73, 0x11, - 0xd4, 0x4b, 0xcd, 0x56, 0xaf, 0x95, 0xac, 0xd4, 0x5f, 0xf5, 0x1a, 0x3d, 0x87, 0x6d, 0xca, 0x84, - 0xc4, 0x2c, 0x20, 0xbe, 0x50, 0x8c, 0xb6, 0xc1, 0x1e, 0x6e, 0x1a, 0xa6, 0x7b, 0x6c, 0x69, 0xf4, - 0xce, 0x94, 0xea, 0x05, 0x5a, 0x96, 0xf5, 0x13, 0x40, 0xaf, 0x1b, 0x9d, 0x53, 0x37, 0x77, 0x96, - 0x3b, 0xe4, 0x9a, 0x0f, 0x94, 0x49, 0x56, 0xa9, 0xc8, 0xfe, 0xae, 0x14, 0xcf, 0x93, 0x49, 0xd5, - 0x21, 0xb4, 0x44, 0x9e, 0x24, 0x38, 0x9b, 0xd9, 0x4b, 0xff, 0xc9, 0x2a, 0x62, 0x85, 0x7b, 0xac, - 0xf8, 0xbc, 0x02, 0x89, 0x6e, 0x43, 0xc3, 0xa4, 0xc9, 0xc4, 0x76, 0xb0, 0x0e, 0xc5, 0xe9, 0xf8, - 0x39, 0x09, 0xa4, 0x67, 0x80, 0xe8, 0x26, 0x74, 0xe6, 0xe3, 0x80, 0x3e, 0x8a, 0xee, 0xb0, 0xef, - 0x9a, 0x81, 0xc1, 0x2d, 0x06, 0x06, 0xf7, 0x51, 0x61, 0xe1, 0x2d, 0x8c, 0x9d, 0x3f, 0x6a, 0x00, - 0x0b, 0x3e, 0x74, 0x02, 0x4d, 0x46, 0x84, 0x24, 0xa1, 0xad, 0xac, 0x2f, 0xd7, 0x8f, 0xc5, 0x3d, - 0xd1, 0x40, 0x73, 0x4a, 0x96, 0x05, 0x3d, 0x5d, 0x7a, 0x1a, 0x4d, 0x5b, 0xb9, 0xb5, 0x01, 0xe7, - 0xdb, 0x5e, 0x46, 0x02, 0xdd, 0x92, 0xcb, 0x73, 0xce, 0xfc, 0xf6, 0xf2, 0x99, 0x6f, 0x94, 0xd7, - 0xf9, 0x91, 0xf7, 0xa3, 0x75, 0x1e, 0xe0, 0xef, 0x97, 0x5d, 0x6d, 0x50, 0x05, 0xa5, 0xe2, 0x7a, - 0x55, 0x85, 0xce, 0x5c, 0x81, 0x5c, 0xd8, 0x7d, 0x16, 0x73, 0x2c, 0x7d, 0x96, 0x27, 0x24, 0xc3, - 0x92, 0x67, 0xfe, 0x14, 0xc7, 0xda, 0x69, 0xc5, 0xdb, 0xd1, 0xaa, 0x93, 0x42, 0xf3, 0x18, 0xc7, - 0x68, 0x08, 0xef, 0x1a, 0xfb, 0x90, 0x30, 0x9e, 0x50, 0x36, 0x47, 0x54, 0x35, 0xc2, 0x90, 0x8d, - 0x16, 0x3a, 0x85, 0x39, 0x80, 0x1d, 0xca, 0xce, 0x7a, 0x50, 0xf5, 0x53, 0xf3, 0x2e, 0x52, 0xb6, - 0xcc, 0xef, 0xc2, 0xae, 0xb2, 0x3d, 0xcb, 0x5e, 0xd7, 0xd6, 0x8a, 0xe6, 0x0c, 0xf7, 0x65, 0x00, - 0x21, 0x33, 0xca, 0x26, 0xda, 0xac, 0xa1, 0x73, 0xd5, 0x31, 0x12, 0xa5, 0x7e, 0x1f, 0xda, 0x63, - 0xce, 0x63, 0xad, 0x6c, 0x9a, 0x19, 0x42, 0xed, 0x95, 0x0a, 0x41, 0x3d, 0x67, 0x54, 0xf6, 0x5a, - 0xa6, 0xa7, 0xa8, 0xb5, 0x92, 0xa9, 0x81, 0xa2, 0xd7, 0x36, 0x32, 0xb5, 0x1e, 0xfe, 0x53, 0x85, - 0x2d, 0x73, 0x19, 0x1f, 0xe8, 0xec, 0xa2, 0xdf, 0xa0, 0x5b, 0x9a, 0x51, 0xd1, 0x70, 0xd5, 0x29, - 0xbc, 0x3e, 0xe6, 0xf6, 0x6f, 0x6c, 0x84, 0x31, 0xfd, 0xd8, 0x79, 0xe7, 0x7a, 0x05, 0xc5, 0xd0, - 0xb2, 0x73, 0x01, 0x5a, 0x39, 0xbf, 0x2c, 0x4f, 0x1c, 0xfd, 0xc1, 0xda, 0xf6, 0x85, 0x3f, 0x14, - 0x41, 0xc3, 0x34, 0xa0, 0xcf, 0xd6, 0xa9, 0xb4, 0xe2, 0x25, 0xe9, 0x5f, 0x5b, 0xd3, 0x7a, 0xf1, - 0x5f, 0x77, 0x5b, 0x4f, 0x1b, 0xa6, 0x83, 0x34, 0xf5, 0xe7, 0xc6, 0xbf, 0x01, 0x00, 0x00, 0xff, - 0xff, 0xe1, 0x29, 0x35, 0x33, 0xe7, 0x0c, 0x00, 0x00, +var fileDescriptor_device_4daf5409089620fd = []byte{ + // 1168 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdf, 0x8e, 0xdb, 0xc4, + 0x17, 0xfe, 0xe5, 0x7f, 0x72, 0xb2, 0xdd, 0xed, 0xce, 0xee, 0x0f, 0xa5, 0x81, 0xb6, 0x8b, 0x25, + 0xa4, 0xb2, 0x50, 0xa7, 0xa4, 0x08, 0xaa, 0x22, 0xa0, 0xed, 0xa6, 0x6c, 0x83, 0x60, 0xb7, 0x72, + 0xab, 0x4a, 0x5d, 0x24, 0x2c, 0xc7, 0x9e, 0xc6, 0xd3, 0xda, 0x33, 0xc6, 0x33, 0x4e, 0x15, 0xae, + 0xb8, 0xe5, 0x4d, 0xb8, 0xe9, 0x0b, 0xf0, 0x10, 0x3c, 0x04, 0x4f, 0x82, 0xe6, 0x8f, 0x13, 0x27, + 0xbb, 0x6d, 0x12, 0xb8, 0xf2, 0xcc, 0x39, 0xe7, 0xfb, 0xce, 0xf1, 0xcc, 0x37, 0x67, 0x06, 0x3e, + 0x4c, 0xa2, 0x6c, 0x4c, 0x28, 0xef, 0x05, 0x78, 0x42, 0x7c, 0xdc, 0x4b, 0x52, 0x26, 0x98, 0x99, + 0xd8, 0x6a, 0x82, 0xae, 0x85, 0x1e, 0x0f, 0x89, 0xcf, 0xd2, 0xc4, 0xa6, 0x2c, 0xf6, 0x02, 0xdb, + 0x40, 0x6c, 0x1d, 0xd5, 0xbd, 0x3e, 0x66, 0x6c, 0x1c, 0x19, 0xe8, 0x28, 0x7b, 0xd1, 0x13, 0x24, + 0xc6, 0x5c, 0x78, 0x71, 0xa2, 0x09, 0xba, 0xd7, 0x96, 0x03, 0x82, 0x2c, 0xf5, 0x04, 0x61, 0xd4, + 0xf8, 0x8f, 0xc7, 0x44, 0x84, 0xd9, 0xc8, 0xf6, 0x59, 0xdc, 0x9b, 0xe5, 0xea, 0xa9, 0x5c, 0xbd, + 0xbc, 0x3c, 0x1e, 0x7a, 0x29, 0x0e, 0x7a, 0x5c, 0xa4, 0x99, 0x2f, 0xb8, 0x29, 0xd3, 0x13, 0x22, + 0x25, 0xa3, 0x4c, 0x98, 0x4a, 0xad, 0x7d, 0x40, 0xdf, 0x11, 0x3a, 0xc6, 0x69, 0x92, 0x12, 0x2a, + 0x1c, 0xfc, 0x4b, 0x86, 0xb9, 0xb0, 0x30, 0xec, 0x2d, 0x58, 0x79, 0xc2, 0x28, 0xc7, 0xe8, 0x04, + 0xb6, 0xf4, 0x0f, 0xb8, 0xe3, 0x94, 0x65, 0x49, 0xa7, 0x74, 0x50, 0xb9, 0xd1, 0xee, 0x7f, 0x62, + 0xbf, 0xfb, 0x6f, 0xed, 0x81, 0xfa, 0x1c, 0x4b, 0x88, 0xd3, 0x0e, 0xe6, 0x13, 0xeb, 0xb7, 0x0a, + 0xb4, 0x0b, 0x4e, 0xf4, 0x1e, 0xd4, 0x27, 0x98, 0x06, 0x2c, 0xed, 0x94, 0x0e, 0x4a, 0x37, 0x5a, + 0x8e, 0x99, 0xa1, 0xeb, 0x60, 0x60, 0xae, 0x98, 0x26, 0xb8, 0x53, 0x56, 0x4e, 0xd0, 0xa6, 0xa7, + 0xd3, 0x04, 0x17, 0x02, 0xa8, 0x17, 0xe3, 0x4e, 0xa5, 0x18, 0x70, 0xe2, 0xc5, 0x18, 0x3d, 0x82, + 0x86, 0x9e, 0xf1, 0x4e, 0x55, 0x15, 0x6d, 0xaf, 0x2e, 0x5a, 0x60, 0x5f, 0xe0, 0x40, 0xd7, 0xe7, + 0xe4, 0x70, 0xf4, 0x13, 0xc0, 0x6c, 0x0d, 0x79, 0xa7, 0xa6, 0xc8, 0xbe, 0xda, 0x60, 0x05, 0xec, + 0xfb, 0x33, 0xf4, 0x43, 0x2a, 0xd2, 0xa9, 0x53, 0xa0, 0xeb, 0x26, 0xb0, 0xb3, 0xe4, 0x46, 0x97, + 0xa1, 0xf2, 0x0a, 0x4f, 0xcd, 0x82, 0xc8, 0x21, 0x3a, 0x86, 0xda, 0xc4, 0x8b, 0x32, 0xbd, 0x0e, + 0xed, 0xfe, 0x67, 0x6f, 0x4d, 0xae, 0x05, 0x60, 0x1b, 0x01, 0xcc, 0x13, 0x3b, 0x1a, 0x7f, 0xb7, + 0x7c, 0xa7, 0x64, 0xfd, 0x59, 0x82, 0xed, 0xc5, 0x5f, 0x45, 0xdb, 0x50, 0x1e, 0x0e, 0x4c, 0xc2, + 0xf2, 0x70, 0x80, 0x3a, 0xd0, 0x08, 0xb1, 0x17, 0x89, 0x70, 0xaa, 0x32, 0x36, 0x9d, 0x7c, 0x8a, + 0x6e, 0x02, 0xd2, 0x43, 0x37, 0xc0, 0xdc, 0x4f, 0x49, 0x22, 0x15, 0x6a, 0x56, 0x7f, 0x57, 0x7b, + 0x06, 0x73, 0x07, 0x3a, 0x85, 0x76, 0xf8, 0xda, 0x8d, 0x98, 0xef, 0x45, 0x44, 0x4c, 0x3b, 0x55, + 0x55, 0xbe, 0xbd, 0xde, 0xda, 0xfd, 0x60, 0x50, 0x0e, 0x84, 0xaf, 0xf3, 0xb1, 0x65, 0xcb, 0xda, + 0x8b, 0x5e, 0xf4, 0x01, 0x40, 0xe2, 0x13, 0x77, 0x94, 0x71, 0x97, 0x04, 0xe6, 0x1f, 0x9a, 0x89, + 0x4f, 0x1e, 0x64, 0x7c, 0x18, 0x58, 0x3d, 0xd8, 0x76, 0x30, 0xc7, 0xe9, 0x04, 0x1b, 0xa1, 0xa3, + 0xab, 0x60, 0x54, 0xe2, 0x92, 0x80, 0x2b, 0x3d, 0xb7, 0x9c, 0x96, 0xb6, 0x0c, 0x03, 0x6e, 0x45, + 0xb0, 0x33, 0x03, 0x98, 0x33, 0xf0, 0x1c, 0x2e, 0xf9, 0x8c, 0x0a, 0x8f, 0x50, 0x9c, 0xba, 0x29, + 0xe6, 0x2a, 0x49, 0xbb, 0xff, 0xf9, 0xaa, 0xdf, 0x38, 0xca, 0x41, 0x9a, 0x50, 0x1d, 0x66, 0x67, + 0xcb, 0x2f, 0x58, 0xad, 0x3f, 0xca, 0xb0, 0x7f, 0x51, 0x18, 0x72, 0xa0, 0x8a, 0xe9, 0x84, 0x9b, + 0xf3, 0xf6, 0xcd, 0xbf, 0x49, 0x65, 0x3f, 0xa4, 0x13, 0x23, 0x38, 0xc5, 0x85, 0xbe, 0x86, 0x7a, + 0xcc, 0x32, 0x2a, 0x78, 0xa7, 0xac, 0x58, 0x3f, 0x5a, 0xc5, 0xfa, 0xa3, 0x8c, 0x76, 0x0c, 0x08, + 0x0d, 0xe6, 0x07, 0xaa, 0xa2, 0xf0, 0x87, 0xeb, 0xed, 0xe3, 0x93, 0x04, 0xfb, 0xb3, 0xc3, 0xd4, + 0xfd, 0x12, 0x5a, 0xb3, 0xba, 0x2e, 0x50, 0xfa, 0x7e, 0x51, 0xe9, 0xad, 0xa2, 0x6c, 0x7f, 0x86, + 0x9a, 0xaa, 0x07, 0xbd, 0x0f, 0x2d, 0xe1, 0xf1, 0x57, 0x6e, 0xe2, 0x89, 0x30, 0xdf, 0x6f, 0x69, + 0x78, 0xec, 0x89, 0x50, 0x3a, 0x43, 0xc6, 0x85, 0x76, 0x6a, 0x8e, 0xa6, 0x34, 0xe4, 0xce, 0x14, + 0x7b, 0x81, 0xcb, 0x68, 0x34, 0x55, 0x9a, 0x6d, 0x3a, 0x4d, 0x69, 0x38, 0xa5, 0xd1, 0xd4, 0x0a, + 0x01, 0xe6, 0xf5, 0xfe, 0x87, 0x24, 0x07, 0xd0, 0x4e, 0x70, 0x1a, 0x13, 0xce, 0x09, 0xa3, 0xdc, + 0x1c, 0x8d, 0xa2, 0xc9, 0x3a, 0x83, 0xad, 0x27, 0xc2, 0x13, 0x3c, 0x57, 0xe4, 0xf7, 0xb0, 0xe7, + 0xb3, 0x28, 0xc2, 0xbe, 0xdc, 0x35, 0x97, 0x50, 0x21, 0x77, 0x30, 0x32, 0x2a, 0xbb, 0x62, 0xeb, + 0x7b, 0xc1, 0xce, 0xef, 0x05, 0x7b, 0x60, 0xee, 0x05, 0x07, 0xcd, 0x51, 0x43, 0x03, 0xb2, 0x9e, + 0xc3, 0x25, 0xc3, 0x6d, 0xc4, 0xfb, 0x08, 0xea, 0xaa, 0x73, 0xe7, 0x52, 0xba, 0xb5, 0x41, 0xe3, + 0xd2, 0x4c, 0x06, 0x6f, 0xbd, 0x29, 0xc3, 0xe5, 0x65, 0xe7, 0x5b, 0xfb, 0x37, 0x82, 0x6a, 0xa1, + 0x71, 0xab, 0xb1, 0xb4, 0x15, 0x7a, 0xb5, 0x1a, 0xa3, 0x97, 0xb0, 0x4d, 0x28, 0x17, 0x1e, 0xf5, + 0xb1, 0xcb, 0x25, 0xa3, 0x69, 0xd6, 0x47, 0x9b, 0x96, 0x69, 0x0f, 0x0d, 0x8d, 0x9a, 0x69, 0xd9, + 0x5f, 0x22, 0x45, 0x5b, 0x37, 0x06, 0x74, 0x3e, 0xe8, 0x02, 0x0d, 0xde, 0x5f, 0xec, 0xb6, 0x6b, + 0x5e, 0x76, 0x7a, 0xb1, 0x0a, 0x82, 0xfd, 0xab, 0x94, 0x5f, 0x75, 0x7a, 0xa9, 0x8e, 0xa0, 0xc1, + 0xb3, 0x38, 0xf6, 0xd2, 0xa9, 0xd9, 0xda, 0x8f, 0x57, 0x11, 0x4b, 0xdc, 0x33, 0xc9, 0xe7, 0xe4, + 0x48, 0x74, 0x0f, 0x6a, 0x7a, 0x99, 0x74, 0x6d, 0x87, 0xeb, 0x50, 0x9c, 0x8e, 0x5e, 0x62, 0x5f, + 0x38, 0x1a, 0x88, 0xee, 0x40, 0x6b, 0xf6, 0xf4, 0x50, 0x5b, 0xd1, 0xee, 0x77, 0xcf, 0x69, 0xec, + 0x69, 0x1e, 0xe1, 0xcc, 0x83, 0xad, 0xdf, 0x2b, 0x00, 0x73, 0x3e, 0x74, 0x02, 0x75, 0x8a, 0xb9, + 0xc0, 0x81, 0x51, 0xd6, 0x17, 0xeb, 0xd7, 0x62, 0x9f, 0x28, 0xa0, 0xde, 0x25, 0xc3, 0x82, 0xce, + 0x16, 0xae, 0x59, 0xdd, 0xa2, 0xee, 0x6e, 0xc0, 0xf9, 0xae, 0x5b, 0x16, 0x43, 0xbb, 0x90, 0xf2, + 0x82, 0x3d, 0xbf, 0xb7, 0xb8, 0xe7, 0x1b, 0xad, 0xeb, 0x6c, 0xcb, 0xbb, 0xe1, 0x3a, 0x97, 0xf9, + 0xb7, 0x8b, 0xa9, 0x36, 0x50, 0x41, 0x41, 0x5c, 0x6f, 0xca, 0xd0, 0x9a, 0x39, 0x90, 0x0d, 0x7b, + 0x2f, 0x22, 0xe6, 0x09, 0x97, 0x66, 0x31, 0x4e, 0x3d, 0xc1, 0x52, 0x37, 0xef, 0x20, 0x25, 0x67, + 0x57, 0xb9, 0x4e, 0x72, 0xcf, 0x33, 0x2f, 0x42, 0x7d, 0xf8, 0xbf, 0x8e, 0x0f, 0x30, 0x65, 0x31, + 0xa1, 0x33, 0x44, 0x59, 0x21, 0x34, 0xd9, 0x60, 0xee, 0x93, 0x98, 0x43, 0xd8, 0x25, 0x74, 0x39, + 0x83, 0xd4, 0x4f, 0xc5, 0xd9, 0x21, 0x74, 0x91, 0xdf, 0x86, 0x3d, 0x19, 0xbb, 0xcc, 0x5e, 0x55, + 0xd1, 0x92, 0x66, 0x89, 0xfb, 0x2a, 0x00, 0x17, 0x29, 0xa1, 0x63, 0x15, 0x56, 0x53, 0x6b, 0xd5, + 0xd2, 0x16, 0xe9, 0xbe, 0x02, 0xcd, 0x11, 0x63, 0x91, 0x72, 0xd6, 0xf5, 0x7b, 0x44, 0xce, 0xa5, + 0x0b, 0x41, 0x35, 0xa3, 0x44, 0x74, 0x1a, 0xba, 0xa7, 0xc8, 0xb1, 0xb4, 0xc9, 0xc7, 0x49, 0xa7, + 0xa9, 0x6d, 0x72, 0xdc, 0xff, 0xbb, 0x0c, 0x5b, 0xfa, 0x30, 0x3e, 0x56, 0xab, 0x8b, 0x7e, 0x85, + 0x76, 0xe1, 0xbd, 0x8b, 0xfa, 0xab, 0x76, 0xe1, 0xfc, 0x93, 0xb9, 0x7b, 0x7b, 0x23, 0x8c, 0xee, + 0xc7, 0xd6, 0xff, 0x6e, 0x95, 0x50, 0x04, 0x0d, 0xf3, 0xc6, 0x40, 0x2b, 0xdf, 0x42, 0x8b, 0xaf, + 0x97, 0x6e, 0x6f, 0xed, 0xf8, 0x3c, 0x1f, 0x0a, 0xa1, 0xa6, 0x1b, 0xd0, 0xa7, 0xeb, 0x28, 0x2d, + 0xbf, 0x95, 0xba, 0x37, 0xd7, 0x8c, 0x9e, 0xff, 0xd7, 0x83, 0xc6, 0x59, 0x4d, 0x77, 0x90, 0xba, + 0xfa, 0xdc, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0x78, 0x61, 0x84, 0x9b, 0x38, 0x0d, 0x00, 0x00, } diff --git a/plugins/device/proto/device.proto b/plugins/device/proto/device.proto index 09b2212bc..fdf9aa8c7 100644 --- a/plugins/device/proto/device.proto +++ b/plugins/device/proto/device.proto @@ -3,6 +3,7 @@ package hashicorp.nomad.plugins.device; option go_package = "proto"; import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; import "github.com/hashicorp/nomad/plugins/shared/structs/proto/attribute.proto"; // DevicePlugin is the API exposed by device plugins @@ -137,7 +138,10 @@ message DeviceSpec { // StatsRequest is used to parameterize the retrieval of statistics. -message StatsRequest {} +message StatsRequest { + // collection_interval is the duration in which to collect statistics. + google.protobuf.Duration collection_interval = 1; +} // StatsResponse returns the statistics for each device group. message StatsResponse { diff --git a/plugins/device/server.go b/plugins/device/server.go index 8a8c28953..ead4922bf 100644 --- a/plugins/device/server.go +++ b/plugins/device/server.go @@ -1,10 +1,13 @@ package device import ( - context "golang.org/x/net/context" + "fmt" + "time" + "github.com/golang/protobuf/ptypes" plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/plugins/device/proto" + context "golang.org/x/net/context" ) // devicePluginServer wraps a device plugin and exposes it via gRPC. @@ -67,7 +70,19 @@ func (d *devicePluginServer) Reserve(ctx context.Context, req *proto.ReserveRequ func (d *devicePluginServer) Stats(req *proto.StatsRequest, stream proto.DevicePlugin_StatsServer) error { ctx := stream.Context() - outCh, err := d.impl.Stats(ctx) + + // Retrieve the collection interval + interval, err := ptypes.Duration(req.CollectionInterval) + if err != nil { + return fmt.Errorf("failed to parse collection interval: %v", err) + } + + // Default the duration if we get an invalid duration + if interval.Nanoseconds() == 0 { + interval = time.Second + } + + outCh, err := d.impl.Stats(ctx, interval) if err != nil { return err } diff --git a/plugins/drivers/proto/driver.pb.go b/plugins/drivers/proto/driver.pb.go index 7f60d3e07..46feba143 100644 --- a/plugins/drivers/proto/driver.pb.go +++ b/plugins/drivers/proto/driver.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: driver.proto +// source: plugins/drivers/proto/driver.proto package proto @@ -49,7 +49,7 @@ func (x TaskState) String() string { return proto.EnumName(TaskState_name, int32(x)) } func (TaskState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{0} } type FingerprintResponse_HealthState int32 @@ -75,7 +75,7 @@ func (x FingerprintResponse_HealthState) String() string { return proto.EnumName(FingerprintResponse_HealthState_name, int32(x)) } func (FingerprintResponse_HealthState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{5, 0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{5, 0} } type StartTaskResponse_Result int32 @@ -101,7 +101,7 @@ func (x StartTaskResponse_Result) String() string { return proto.EnumName(StartTaskResponse_Result_name, int32(x)) } func (StartTaskResponse_Result) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{9, 0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{9, 0} } type DriverCapabilities_FSIsolation int32 @@ -127,7 +127,7 @@ func (x DriverCapabilities_FSIsolation) String() string { return proto.EnumName(DriverCapabilities_FSIsolation_name, int32(x)) } func (DriverCapabilities_FSIsolation) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{25, 0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{25, 0} } type CPUUsage_Fields int32 @@ -162,7 +162,7 @@ func (x CPUUsage_Fields) String() string { return proto.EnumName(CPUUsage_Fields_name, int32(x)) } func (CPUUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{41, 0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{41, 0} } type MemoryUsage_Fields int32 @@ -194,7 +194,7 @@ func (x MemoryUsage_Fields) String() string { return proto.EnumName(MemoryUsage_Fields_name, int32(x)) } func (MemoryUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{42, 0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{42, 0} } type TaskConfigSchemaRequest struct { @@ -207,7 +207,7 @@ func (m *TaskConfigSchemaRequest) Reset() { *m = TaskConfigSchemaRequest func (m *TaskConfigSchemaRequest) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaRequest) ProtoMessage() {} func (*TaskConfigSchemaRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{0} + return fileDescriptor_driver_5bef65efb3792c9f, []int{0} } func (m *TaskConfigSchemaRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaRequest.Unmarshal(m, b) @@ -239,7 +239,7 @@ func (m *TaskConfigSchemaResponse) Reset() { *m = TaskConfigSchemaRespon func (m *TaskConfigSchemaResponse) String() string { return proto.CompactTextString(m) } func (*TaskConfigSchemaResponse) ProtoMessage() {} func (*TaskConfigSchemaResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{1} + return fileDescriptor_driver_5bef65efb3792c9f, []int{1} } func (m *TaskConfigSchemaResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfigSchemaResponse.Unmarshal(m, b) @@ -276,7 +276,7 @@ func (m *CapabilitiesRequest) Reset() { *m = CapabilitiesRequest{} } func (m *CapabilitiesRequest) String() string { return proto.CompactTextString(m) } func (*CapabilitiesRequest) ProtoMessage() {} func (*CapabilitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{2} + return fileDescriptor_driver_5bef65efb3792c9f, []int{2} } func (m *CapabilitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesRequest.Unmarshal(m, b) @@ -311,7 +311,7 @@ func (m *CapabilitiesResponse) Reset() { *m = CapabilitiesResponse{} } func (m *CapabilitiesResponse) String() string { return proto.CompactTextString(m) } func (*CapabilitiesResponse) ProtoMessage() {} func (*CapabilitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{3} + return fileDescriptor_driver_5bef65efb3792c9f, []int{3} } func (m *CapabilitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CapabilitiesResponse.Unmarshal(m, b) @@ -348,7 +348,7 @@ func (m *FingerprintRequest) Reset() { *m = FingerprintRequest{} } func (m *FingerprintRequest) String() string { return proto.CompactTextString(m) } func (*FingerprintRequest) ProtoMessage() {} func (*FingerprintRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{4} + return fileDescriptor_driver_5bef65efb3792c9f, []int{4} } func (m *FingerprintRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintRequest.Unmarshal(m, b) @@ -391,7 +391,7 @@ func (m *FingerprintResponse) Reset() { *m = FingerprintResponse{} } func (m *FingerprintResponse) String() string { return proto.CompactTextString(m) } func (*FingerprintResponse) ProtoMessage() {} func (*FingerprintResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{5} + return fileDescriptor_driver_5bef65efb3792c9f, []int{5} } func (m *FingerprintResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FingerprintResponse.Unmarshal(m, b) @@ -446,7 +446,7 @@ func (m *RecoverTaskRequest) Reset() { *m = RecoverTaskRequest{} } func (m *RecoverTaskRequest) String() string { return proto.CompactTextString(m) } func (*RecoverTaskRequest) ProtoMessage() {} func (*RecoverTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{6} + return fileDescriptor_driver_5bef65efb3792c9f, []int{6} } func (m *RecoverTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskRequest.Unmarshal(m, b) @@ -490,7 +490,7 @@ func (m *RecoverTaskResponse) Reset() { *m = RecoverTaskResponse{} } func (m *RecoverTaskResponse) String() string { return proto.CompactTextString(m) } func (*RecoverTaskResponse) ProtoMessage() {} func (*RecoverTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{7} + return fileDescriptor_driver_5bef65efb3792c9f, []int{7} } func (m *RecoverTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RecoverTaskResponse.Unmarshal(m, b) @@ -522,7 +522,7 @@ func (m *StartTaskRequest) Reset() { *m = StartTaskRequest{} } func (m *StartTaskRequest) String() string { return proto.CompactTextString(m) } func (*StartTaskRequest) ProtoMessage() {} func (*StartTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{8} + return fileDescriptor_driver_5bef65efb3792c9f, []int{8} } func (m *StartTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskRequest.Unmarshal(m, b) @@ -576,7 +576,7 @@ func (m *StartTaskResponse) Reset() { *m = StartTaskResponse{} } func (m *StartTaskResponse) String() string { return proto.CompactTextString(m) } func (*StartTaskResponse) ProtoMessage() {} func (*StartTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{9} + return fileDescriptor_driver_5bef65efb3792c9f, []int{9} } func (m *StartTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StartTaskResponse.Unmarshal(m, b) @@ -636,7 +636,7 @@ func (m *WaitTaskRequest) Reset() { *m = WaitTaskRequest{} } func (m *WaitTaskRequest) String() string { return proto.CompactTextString(m) } func (*WaitTaskRequest) ProtoMessage() {} func (*WaitTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{10} + return fileDescriptor_driver_5bef65efb3792c9f, []int{10} } func (m *WaitTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskRequest.Unmarshal(m, b) @@ -677,7 +677,7 @@ func (m *WaitTaskResponse) Reset() { *m = WaitTaskResponse{} } func (m *WaitTaskResponse) String() string { return proto.CompactTextString(m) } func (*WaitTaskResponse) ProtoMessage() {} func (*WaitTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{11} + return fileDescriptor_driver_5bef65efb3792c9f, []int{11} } func (m *WaitTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_WaitTaskResponse.Unmarshal(m, b) @@ -729,7 +729,7 @@ func (m *StopTaskRequest) Reset() { *m = StopTaskRequest{} } func (m *StopTaskRequest) String() string { return proto.CompactTextString(m) } func (*StopTaskRequest) ProtoMessage() {} func (*StopTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{12} + return fileDescriptor_driver_5bef65efb3792c9f, []int{12} } func (m *StopTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskRequest.Unmarshal(m, b) @@ -780,7 +780,7 @@ func (m *StopTaskResponse) Reset() { *m = StopTaskResponse{} } func (m *StopTaskResponse) String() string { return proto.CompactTextString(m) } func (*StopTaskResponse) ProtoMessage() {} func (*StopTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{13} + return fileDescriptor_driver_5bef65efb3792c9f, []int{13} } func (m *StopTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StopTaskResponse.Unmarshal(m, b) @@ -814,7 +814,7 @@ func (m *DestroyTaskRequest) Reset() { *m = DestroyTaskRequest{} } func (m *DestroyTaskRequest) String() string { return proto.CompactTextString(m) } func (*DestroyTaskRequest) ProtoMessage() {} func (*DestroyTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{14} + return fileDescriptor_driver_5bef65efb3792c9f, []int{14} } func (m *DestroyTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskRequest.Unmarshal(m, b) @@ -858,7 +858,7 @@ func (m *DestroyTaskResponse) Reset() { *m = DestroyTaskResponse{} } func (m *DestroyTaskResponse) String() string { return proto.CompactTextString(m) } func (*DestroyTaskResponse) ProtoMessage() {} func (*DestroyTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{15} + return fileDescriptor_driver_5bef65efb3792c9f, []int{15} } func (m *DestroyTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DestroyTaskResponse.Unmarshal(m, b) @@ -890,7 +890,7 @@ func (m *InspectTaskRequest) Reset() { *m = InspectTaskRequest{} } func (m *InspectTaskRequest) String() string { return proto.CompactTextString(m) } func (*InspectTaskRequest) ProtoMessage() {} func (*InspectTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{16} + return fileDescriptor_driver_5bef65efb3792c9f, []int{16} } func (m *InspectTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskRequest.Unmarshal(m, b) @@ -933,7 +933,7 @@ func (m *InspectTaskResponse) Reset() { *m = InspectTaskResponse{} } func (m *InspectTaskResponse) String() string { return proto.CompactTextString(m) } func (*InspectTaskResponse) ProtoMessage() {} func (*InspectTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{17} + return fileDescriptor_driver_5bef65efb3792c9f, []int{17} } func (m *InspectTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_InspectTaskResponse.Unmarshal(m, b) @@ -986,7 +986,7 @@ func (m *TaskStatsRequest) Reset() { *m = TaskStatsRequest{} } func (m *TaskStatsRequest) String() string { return proto.CompactTextString(m) } func (*TaskStatsRequest) ProtoMessage() {} func (*TaskStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{18} + return fileDescriptor_driver_5bef65efb3792c9f, []int{18} } func (m *TaskStatsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsRequest.Unmarshal(m, b) @@ -1025,7 +1025,7 @@ func (m *TaskStatsResponse) Reset() { *m = TaskStatsResponse{} } func (m *TaskStatsResponse) String() string { return proto.CompactTextString(m) } func (*TaskStatsResponse) ProtoMessage() {} func (*TaskStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{19} + return fileDescriptor_driver_5bef65efb3792c9f, []int{19} } func (m *TaskStatsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatsResponse.Unmarshal(m, b) @@ -1062,7 +1062,7 @@ func (m *TaskEventsRequest) Reset() { *m = TaskEventsRequest{} } func (m *TaskEventsRequest) String() string { return proto.CompactTextString(m) } func (*TaskEventsRequest) ProtoMessage() {} func (*TaskEventsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{20} + return fileDescriptor_driver_5bef65efb3792c9f, []int{20} } func (m *TaskEventsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskEventsRequest.Unmarshal(m, b) @@ -1096,7 +1096,7 @@ func (m *SignalTaskRequest) Reset() { *m = SignalTaskRequest{} } func (m *SignalTaskRequest) String() string { return proto.CompactTextString(m) } func (*SignalTaskRequest) ProtoMessage() {} func (*SignalTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{21} + return fileDescriptor_driver_5bef65efb3792c9f, []int{21} } func (m *SignalTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskRequest.Unmarshal(m, b) @@ -1140,7 +1140,7 @@ func (m *SignalTaskResponse) Reset() { *m = SignalTaskResponse{} } func (m *SignalTaskResponse) String() string { return proto.CompactTextString(m) } func (*SignalTaskResponse) ProtoMessage() {} func (*SignalTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{22} + return fileDescriptor_driver_5bef65efb3792c9f, []int{22} } func (m *SignalTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SignalTaskResponse.Unmarshal(m, b) @@ -1177,7 +1177,7 @@ func (m *ExecTaskRequest) Reset() { *m = ExecTaskRequest{} } func (m *ExecTaskRequest) String() string { return proto.CompactTextString(m) } func (*ExecTaskRequest) ProtoMessage() {} func (*ExecTaskRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{23} + return fileDescriptor_driver_5bef65efb3792c9f, []int{23} } func (m *ExecTaskRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskRequest.Unmarshal(m, b) @@ -1234,7 +1234,7 @@ func (m *ExecTaskResponse) Reset() { *m = ExecTaskResponse{} } func (m *ExecTaskResponse) String() string { return proto.CompactTextString(m) } func (*ExecTaskResponse) ProtoMessage() {} func (*ExecTaskResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{24} + return fileDescriptor_driver_5bef65efb3792c9f, []int{24} } func (m *ExecTaskResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExecTaskResponse.Unmarshal(m, b) @@ -1293,7 +1293,7 @@ func (m *DriverCapabilities) Reset() { *m = DriverCapabilities{} } func (m *DriverCapabilities) String() string { return proto.CompactTextString(m) } func (*DriverCapabilities) ProtoMessage() {} func (*DriverCapabilities) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{25} + return fileDescriptor_driver_5bef65efb3792c9f, []int{25} } func (m *DriverCapabilities) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverCapabilities.Unmarshal(m, b) @@ -1368,7 +1368,7 @@ func (m *TaskConfig) Reset() { *m = TaskConfig{} } func (m *TaskConfig) String() string { return proto.CompactTextString(m) } func (*TaskConfig) ProtoMessage() {} func (*TaskConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{26} + return fileDescriptor_driver_5bef65efb3792c9f, []int{26} } func (m *TaskConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskConfig.Unmarshal(m, b) @@ -1479,7 +1479,7 @@ func (m *Resources) Reset() { *m = Resources{} } func (m *Resources) String() string { return proto.CompactTextString(m) } func (*Resources) ProtoMessage() {} func (*Resources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{27} + return fileDescriptor_driver_5bef65efb3792c9f, []int{27} } func (m *Resources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Resources.Unmarshal(m, b) @@ -1528,7 +1528,7 @@ func (m *RawResources) Reset() { *m = RawResources{} } func (m *RawResources) String() string { return proto.CompactTextString(m) } func (*RawResources) ProtoMessage() {} func (*RawResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{28} + return fileDescriptor_driver_5bef65efb3792c9f, []int{28} } func (m *RawResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RawResources.Unmarshal(m, b) @@ -1599,7 +1599,7 @@ func (m *NetworkResource) Reset() { *m = NetworkResource{} } func (m *NetworkResource) String() string { return proto.CompactTextString(m) } func (*NetworkResource) ProtoMessage() {} func (*NetworkResource) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{29} + return fileDescriptor_driver_5bef65efb3792c9f, []int{29} } func (m *NetworkResource) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkResource.Unmarshal(m, b) @@ -1673,7 +1673,7 @@ func (m *NetworkPort) Reset() { *m = NetworkPort{} } func (m *NetworkPort) String() string { return proto.CompactTextString(m) } func (*NetworkPort) ProtoMessage() {} func (*NetworkPort) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{30} + return fileDescriptor_driver_5bef65efb3792c9f, []int{30} } func (m *NetworkPort) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkPort.Unmarshal(m, b) @@ -1731,7 +1731,7 @@ func (m *LinuxResources) Reset() { *m = LinuxResources{} } func (m *LinuxResources) String() string { return proto.CompactTextString(m) } func (*LinuxResources) ProtoMessage() {} func (*LinuxResources) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{31} + return fileDescriptor_driver_5bef65efb3792c9f, []int{31} } func (m *LinuxResources) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_LinuxResources.Unmarshal(m, b) @@ -1816,7 +1816,7 @@ func (m *Mount) Reset() { *m = Mount{} } func (m *Mount) String() string { return proto.CompactTextString(m) } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{32} + return fileDescriptor_driver_5bef65efb3792c9f, []int{32} } func (m *Mount) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Mount.Unmarshal(m, b) @@ -1879,7 +1879,7 @@ func (m *Device) Reset() { *m = Device{} } func (m *Device) String() string { return proto.CompactTextString(m) } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{33} + return fileDescriptor_driver_5bef65efb3792c9f, []int{33} } func (m *Device) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Device.Unmarshal(m, b) @@ -1937,7 +1937,7 @@ func (m *TaskHandle) Reset() { *m = TaskHandle{} } func (m *TaskHandle) String() string { return proto.CompactTextString(m) } func (*TaskHandle) ProtoMessage() {} func (*TaskHandle) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{34} + return fileDescriptor_driver_5bef65efb3792c9f, []int{34} } func (m *TaskHandle) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskHandle.Unmarshal(m, b) @@ -1997,7 +1997,7 @@ func (m *NetworkOverride) Reset() { *m = NetworkOverride{} } func (m *NetworkOverride) String() string { return proto.CompactTextString(m) } func (*NetworkOverride) ProtoMessage() {} func (*NetworkOverride) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{35} + return fileDescriptor_driver_5bef65efb3792c9f, []int{35} } func (m *NetworkOverride) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_NetworkOverride.Unmarshal(m, b) @@ -2055,7 +2055,7 @@ func (m *ExitResult) Reset() { *m = ExitResult{} } func (m *ExitResult) String() string { return proto.CompactTextString(m) } func (*ExitResult) ProtoMessage() {} func (*ExitResult) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{36} + return fileDescriptor_driver_5bef65efb3792c9f, []int{36} } func (m *ExitResult) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ExitResult.Unmarshal(m, b) @@ -2118,7 +2118,7 @@ func (m *TaskStatus) Reset() { *m = TaskStatus{} } func (m *TaskStatus) String() string { return proto.CompactTextString(m) } func (*TaskStatus) ProtoMessage() {} func (*TaskStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{37} + return fileDescriptor_driver_5bef65efb3792c9f, []int{37} } func (m *TaskStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStatus.Unmarshal(m, b) @@ -2193,7 +2193,7 @@ func (m *TaskDriverStatus) Reset() { *m = TaskDriverStatus{} } func (m *TaskDriverStatus) String() string { return proto.CompactTextString(m) } func (*TaskDriverStatus) ProtoMessage() {} func (*TaskDriverStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{38} + return fileDescriptor_driver_5bef65efb3792c9f, []int{38} } func (m *TaskDriverStatus) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskDriverStatus.Unmarshal(m, b) @@ -2238,7 +2238,7 @@ func (m *TaskStats) Reset() { *m = TaskStats{} } func (m *TaskStats) String() string { return proto.CompactTextString(m) } func (*TaskStats) ProtoMessage() {} func (*TaskStats) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{39} + return fileDescriptor_driver_5bef65efb3792c9f, []int{39} } func (m *TaskStats) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskStats.Unmarshal(m, b) @@ -2300,7 +2300,7 @@ func (m *TaskResourceUsage) Reset() { *m = TaskResourceUsage{} } func (m *TaskResourceUsage) String() string { return proto.CompactTextString(m) } func (*TaskResourceUsage) ProtoMessage() {} func (*TaskResourceUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{40} + return fileDescriptor_driver_5bef65efb3792c9f, []int{40} } func (m *TaskResourceUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TaskResourceUsage.Unmarshal(m, b) @@ -2352,7 +2352,7 @@ func (m *CPUUsage) Reset() { *m = CPUUsage{} } func (m *CPUUsage) String() string { return proto.CompactTextString(m) } func (*CPUUsage) ProtoMessage() {} func (*CPUUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{41} + return fileDescriptor_driver_5bef65efb3792c9f, []int{41} } func (m *CPUUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CPUUsage.Unmarshal(m, b) @@ -2438,7 +2438,7 @@ func (m *MemoryUsage) Reset() { *m = MemoryUsage{} } func (m *MemoryUsage) String() string { return proto.CompactTextString(m) } func (*MemoryUsage) ProtoMessage() {} func (*MemoryUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{42} + return fileDescriptor_driver_5bef65efb3792c9f, []int{42} } func (m *MemoryUsage) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_MemoryUsage.Unmarshal(m, b) @@ -2518,7 +2518,7 @@ func (m *DriverTaskEvent) Reset() { *m = DriverTaskEvent{} } func (m *DriverTaskEvent) String() string { return proto.CompactTextString(m) } func (*DriverTaskEvent) ProtoMessage() {} func (*DriverTaskEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_driver_60d7e28aa66468bf, []int{43} + return fileDescriptor_driver_5bef65efb3792c9f, []int{43} } func (m *DriverTaskEvent) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DriverTaskEvent.Unmarshal(m, b) @@ -3205,184 +3205,186 @@ var _Driver_serviceDesc = grpc.ServiceDesc{ ServerStreams: true, }, }, - Metadata: "driver.proto", + Metadata: "plugins/drivers/proto/driver.proto", } -func init() { proto.RegisterFile("driver.proto", fileDescriptor_driver_60d7e28aa66468bf) } +func init() { + proto.RegisterFile("plugins/drivers/proto/driver.proto", fileDescriptor_driver_5bef65efb3792c9f) +} -var fileDescriptor_driver_60d7e28aa66468bf = []byte{ - // 2753 bytes of a gzipped FileDescriptorProto +var fileDescriptor_driver_5bef65efb3792c9f = []byte{ + // 2758 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x59, 0xcb, 0x6f, 0x23, 0xc7, - 0xd1, 0x17, 0x9f, 0x22, 0x8b, 0x14, 0x35, 0xdb, 0xbb, 0xfb, 0x99, 0xa6, 0xf1, 0x7d, 0x5e, 0x0f, - 0x60, 0x40, 0xb0, 0xbd, 0x94, 0x2d, 0xe3, 0xf3, 0x3e, 0x02, 0x3f, 0x68, 0x72, 0x76, 0x25, 0xaf, + 0xd1, 0x17, 0x9f, 0x22, 0x8b, 0x12, 0x35, 0xdb, 0xbb, 0xfb, 0x99, 0xa6, 0xf1, 0x7d, 0x5e, 0x0f, + 0x60, 0x40, 0xb0, 0xbd, 0x94, 0x2d, 0xe3, 0xf3, 0x3e, 0x02, 0x3f, 0x68, 0x6a, 0x76, 0x25, 0xaf, 0x44, 0x29, 0x4d, 0x0a, 0xeb, 0x4d, 0x62, 0x4f, 0x46, 0x33, 0x2d, 0x72, 0x2c, 0xce, 0xc3, 0xdd, - 0x43, 0xad, 0x84, 0x20, 0x48, 0x90, 0x00, 0x41, 0x12, 0x20, 0x40, 0x2e, 0x41, 0xee, 0xb9, 0x05, - 0xb9, 0xe6, 0x94, 0xc7, 0x25, 0x40, 0xfe, 0x87, 0x1c, 0x73, 0x09, 0x90, 0x6b, 0xfe, 0x82, 0x04, - 0xfd, 0x98, 0xe1, 0x50, 0xd2, 0xda, 0x43, 0x2a, 0x27, 0x4e, 0x55, 0x77, 0xff, 0xba, 0x58, 0x55, - 0x5d, 0x55, 0xdd, 0x05, 0x75, 0x87, 0xba, 0xa7, 0x84, 0xb6, 0x43, 0x1a, 0x44, 0x01, 0x7a, 0x7d, - 0x6c, 0xb1, 0xb1, 0x6b, 0x07, 0x34, 0x6c, 0xfb, 0x81, 0x67, 0x39, 0xed, 0x70, 0x32, 0x1d, 0xb9, - 0x3e, 0x6b, 0xcb, 0x59, 0x4c, 0x4e, 0x6b, 0xfd, 0xdf, 0x28, 0x08, 0x46, 0x13, 0xb2, 0x29, 0xa8, - 0xa3, 0xe9, 0xf1, 0xa6, 0x33, 0xa5, 0x56, 0xe4, 0x06, 0xbe, 0x1a, 0x7f, 0xf5, 0xe2, 0x78, 0xe4, - 0x7a, 0x84, 0x45, 0x96, 0x17, 0xaa, 0x09, 0x1f, 0x8d, 0xdc, 0x68, 0x3c, 0x3d, 0x6a, 0xdb, 0x81, - 0xb7, 0x99, 0x6c, 0xb9, 0x29, 0xb6, 0xdc, 0x54, 0x5b, 0x6e, 0xb2, 0xb1, 0x45, 0x89, 0xb3, 0x39, - 0xb6, 0x27, 0x2c, 0x24, 0x36, 0xff, 0x35, 0xf9, 0x87, 0x44, 0xd0, 0x5f, 0x86, 0x97, 0x86, 0x16, - 0x3b, 0xe9, 0x06, 0xfe, 0xb1, 0x3b, 0x1a, 0xd8, 0x63, 0xe2, 0x59, 0x98, 0x7c, 0x39, 0x25, 0x2c, - 0xd2, 0xbf, 0x03, 0xcd, 0xcb, 0x43, 0x2c, 0x0c, 0x7c, 0x46, 0xd0, 0x47, 0x50, 0xe4, 0x20, 0xcd, - 0xdc, 0x9d, 0xdc, 0x46, 0x6d, 0xeb, 0xad, 0xf6, 0x8b, 0xfe, 0xaf, 0xdc, 0xbc, 0xad, 0x36, 0x6f, - 0x0f, 0x42, 0x62, 0x63, 0xb1, 0x52, 0xbf, 0x0d, 0x37, 0xbb, 0x56, 0x68, 0x1d, 0xb9, 0x13, 0x37, - 0x72, 0x09, 0x8b, 0x37, 0x9d, 0xc2, 0xad, 0x79, 0xb6, 0xda, 0xf0, 0x33, 0xa8, 0xdb, 0x29, 0xbe, - 0xda, 0xf8, 0x41, 0x3b, 0x93, 0xa2, 0xdb, 0x3d, 0x41, 0xcd, 0x01, 0xcf, 0xc1, 0xe9, 0xb7, 0x00, - 0x3d, 0x72, 0xfd, 0x11, 0xa1, 0x21, 0x75, 0xfd, 0x28, 0x16, 0xe6, 0xe7, 0x05, 0xb8, 0x39, 0xc7, - 0x56, 0xc2, 0x7c, 0x01, 0x60, 0x45, 0x11, 0x75, 0x8f, 0xa6, 0x91, 0x10, 0xa5, 0xb0, 0x51, 0xdb, - 0xfa, 0x24, 0xa3, 0x28, 0x57, 0xe0, 0xb5, 0x3b, 0x09, 0x98, 0xe1, 0x47, 0xf4, 0x1c, 0xa7, 0xd0, - 0xd1, 0xe7, 0x50, 0x1e, 0x13, 0x6b, 0x12, 0x8d, 0x9b, 0xf9, 0x3b, 0xb9, 0x8d, 0xc6, 0xd6, 0xa3, - 0x6b, 0xec, 0xb3, 0x2d, 0x80, 0x06, 0x91, 0x15, 0x11, 0xac, 0x50, 0xd1, 0x5d, 0x40, 0xf2, 0xcb, - 0x74, 0x08, 0xb3, 0xa9, 0x1b, 0x72, 0xff, 0x6b, 0x16, 0xee, 0xe4, 0x36, 0xaa, 0xf8, 0x86, 0x1c, - 0xe9, 0xcd, 0x06, 0x5a, 0xef, 0xc3, 0xfa, 0x05, 0x69, 0x91, 0x06, 0x85, 0x13, 0x72, 0x2e, 0x2c, - 0x52, 0xc5, 0xfc, 0x13, 0xdd, 0x82, 0xd2, 0xa9, 0x35, 0x99, 0x12, 0x21, 0x72, 0x15, 0x4b, 0xe2, - 0x61, 0xfe, 0x7e, 0x4e, 0x7f, 0x00, 0xb5, 0x94, 0x10, 0xa8, 0x01, 0x70, 0xd8, 0xef, 0x19, 0x43, - 0xa3, 0x3b, 0x34, 0x7a, 0xda, 0x0a, 0x5a, 0x83, 0xea, 0x61, 0x7f, 0xdb, 0xe8, 0xec, 0x0e, 0xb7, - 0x9f, 0x69, 0x39, 0x54, 0x83, 0xd5, 0x98, 0xc8, 0xeb, 0x67, 0x80, 0x30, 0xb1, 0x83, 0x53, 0x42, - 0xb9, 0x57, 0x2a, 0x13, 0xa1, 0x97, 0x60, 0x35, 0xb2, 0xd8, 0x89, 0xe9, 0x3a, 0x4a, 0x80, 0x32, - 0x27, 0x77, 0x1c, 0xb4, 0x03, 0xe5, 0xb1, 0xe5, 0x3b, 0x13, 0x29, 0x44, 0x6d, 0xeb, 0x9d, 0x8c, - 0x7a, 0xe3, 0xe0, 0xdb, 0x62, 0x21, 0x56, 0x00, 0xdc, 0x55, 0xe7, 0x76, 0x96, 0xda, 0xd4, 0x9f, - 0x81, 0x36, 0x88, 0x2c, 0x1a, 0xa5, 0xc5, 0x31, 0xa0, 0xc8, 0xf7, 0x57, 0xee, 0xb9, 0xc8, 0x9e, - 0xf2, 0x98, 0x61, 0xb1, 0x5c, 0xff, 0x57, 0x1e, 0x6e, 0xa4, 0xb0, 0x95, 0xdb, 0x3d, 0x85, 0x32, - 0x25, 0x6c, 0x3a, 0x89, 0x04, 0x7c, 0x63, 0xeb, 0xc3, 0x8c, 0xf0, 0x97, 0x90, 0xda, 0x58, 0xc0, - 0x60, 0x05, 0x87, 0x36, 0x40, 0x93, 0x2b, 0x4c, 0x42, 0x69, 0x40, 0x4d, 0x8f, 0x8d, 0x94, 0xe9, - 0x1a, 0x92, 0x6f, 0x70, 0xf6, 0x1e, 0x1b, 0xa5, 0xb4, 0x5a, 0xb8, 0xa6, 0x56, 0x91, 0x05, 0x9a, - 0x4f, 0xa2, 0xe7, 0x01, 0x3d, 0x31, 0xb9, 0x6a, 0xa9, 0xeb, 0x90, 0x66, 0x51, 0x80, 0xbe, 0x97, - 0x11, 0xb4, 0x2f, 0x97, 0xef, 0xab, 0xd5, 0x78, 0xdd, 0x9f, 0x67, 0xe8, 0x6f, 0x42, 0x59, 0xfe, - 0x53, 0xee, 0x49, 0x83, 0xc3, 0x6e, 0xd7, 0x18, 0x0c, 0xb4, 0x15, 0x54, 0x85, 0x12, 0x36, 0x86, - 0x98, 0x7b, 0x58, 0x15, 0x4a, 0x8f, 0x3a, 0xc3, 0xce, 0xae, 0x96, 0xd7, 0xdf, 0x80, 0xf5, 0xa7, - 0x96, 0x1b, 0x65, 0x71, 0x2e, 0x3d, 0x00, 0x6d, 0x36, 0x57, 0x59, 0x67, 0x67, 0xce, 0x3a, 0xd9, - 0x55, 0x63, 0x9c, 0xb9, 0xd1, 0x05, 0x7b, 0x68, 0x50, 0x20, 0x94, 0x2a, 0x13, 0xf0, 0x4f, 0xfd, - 0x39, 0xac, 0x0f, 0xa2, 0x20, 0xcc, 0xe4, 0xf9, 0xef, 0xc2, 0x2a, 0xcf, 0x13, 0xc1, 0x34, 0x52, - 0xae, 0xff, 0x72, 0x5b, 0xe6, 0x91, 0x76, 0x9c, 0x47, 0xda, 0x3d, 0x95, 0x67, 0x70, 0x3c, 0x13, - 0xfd, 0x0f, 0x94, 0x99, 0x3b, 0xf2, 0xad, 0x89, 0x3a, 0xfa, 0x8a, 0xd2, 0x11, 0x77, 0xf2, 0x78, - 0x63, 0xe5, 0xf8, 0x5d, 0x40, 0x3d, 0xc2, 0x22, 0x1a, 0x9c, 0x67, 0x92, 0xe7, 0x16, 0x94, 0x8e, - 0x03, 0x6a, 0xcb, 0x83, 0x58, 0xc1, 0x92, 0xe0, 0x87, 0x6a, 0x0e, 0x44, 0x61, 0xdf, 0x05, 0xb4, - 0xe3, 0xf3, 0x04, 0x91, 0xcd, 0x10, 0xbf, 0xcc, 0xc3, 0xcd, 0xb9, 0xf9, 0xca, 0x18, 0xcb, 0x9f, - 0x43, 0x1e, 0x98, 0xa6, 0x4c, 0x9e, 0x43, 0xb4, 0x0f, 0x65, 0x39, 0x43, 0x69, 0xf2, 0xde, 0x02, - 0x40, 0x32, 0xe7, 0x28, 0x38, 0x05, 0x73, 0xa5, 0xd3, 0x17, 0xfe, 0xdb, 0x4e, 0xaf, 0xc5, 0xff, - 0x83, 0x7d, 0xad, 0xfe, 0xbe, 0x0d, 0x37, 0x52, 0x93, 0x95, 0xf2, 0x1e, 0x41, 0x89, 0x71, 0x86, - 0xd2, 0xde, 0xdb, 0x0b, 0x6a, 0x8f, 0x61, 0xb9, 0x5c, 0xbf, 0x29, 0xc1, 0x8d, 0x53, 0xe2, 0x27, - 0xa2, 0xe8, 0x3d, 0xb8, 0x31, 0x10, 0xae, 0x95, 0xc9, 0x77, 0x66, 0x6e, 0x99, 0x9f, 0x73, 0xcb, - 0x5b, 0x80, 0xd2, 0x28, 0xca, 0x79, 0xce, 0x61, 0xdd, 0x38, 0x23, 0x76, 0x26, 0xe4, 0x26, 0xac, - 0xda, 0x81, 0xe7, 0x59, 0xbe, 0xd3, 0xcc, 0xdf, 0x29, 0x6c, 0x54, 0x71, 0x4c, 0xa6, 0xcf, 0x4f, - 0x21, 0xeb, 0xf9, 0xd1, 0x7f, 0x91, 0x03, 0x6d, 0xb6, 0xb7, 0x52, 0x24, 0x97, 0x3e, 0x72, 0x38, - 0x10, 0xdf, 0xbb, 0x8e, 0x15, 0xa5, 0xf8, 0xf1, 0x11, 0x97, 0x7c, 0x42, 0x69, 0x2a, 0x84, 0x14, - 0xae, 0x19, 0x42, 0xf4, 0x7f, 0xe4, 0x00, 0x5d, 0xae, 0x7a, 0xd0, 0x6b, 0x50, 0x67, 0xc4, 0x77, - 0x4c, 0xa9, 0x46, 0x69, 0xe1, 0x0a, 0xae, 0x71, 0x9e, 0xd4, 0x27, 0x43, 0x08, 0x8a, 0xe4, 0x8c, - 0xd8, 0xea, 0xb4, 0x8a, 0x6f, 0x34, 0x86, 0xfa, 0x31, 0x33, 0x5d, 0x16, 0x4c, 0xac, 0xa4, 0x3c, - 0x68, 0x6c, 0x19, 0x4b, 0x57, 0x5f, 0xed, 0x47, 0x83, 0x9d, 0x18, 0x0c, 0xd7, 0x8e, 0x59, 0x42, - 0xe8, 0x6d, 0xa8, 0xa5, 0xc6, 0x50, 0x05, 0x8a, 0xfd, 0xfd, 0xbe, 0xa1, 0xad, 0x20, 0x80, 0x72, - 0x77, 0x1b, 0xef, 0xef, 0x0f, 0x65, 0xd4, 0xde, 0xd9, 0xeb, 0x3c, 0x36, 0xb4, 0xbc, 0xfe, 0xa7, - 0x22, 0xc0, 0x2c, 0x7d, 0xa2, 0x06, 0xe4, 0x13, 0x4b, 0xe7, 0x5d, 0x87, 0xff, 0x19, 0xdf, 0xf2, - 0xe2, 0x42, 0x44, 0x7c, 0xa3, 0x2d, 0xb8, 0xed, 0xb1, 0x51, 0x68, 0xd9, 0x27, 0xa6, 0xca, 0x7a, - 0xb6, 0x58, 0x2c, 0xfe, 0x55, 0x1d, 0xdf, 0x54, 0x83, 0x4a, 0x6a, 0x89, 0xbb, 0x0b, 0x05, 0xe2, - 0x9f, 0x36, 0x8b, 0xa2, 0xd4, 0x7b, 0xb8, 0x70, 0x5a, 0x6f, 0x1b, 0xfe, 0xa9, 0x2c, 0xed, 0x38, - 0x0c, 0xea, 0x43, 0x95, 0x12, 0x16, 0x4c, 0xa9, 0x4d, 0x58, 0xb3, 0xb4, 0xd0, 0x21, 0xc3, 0xf1, - 0x3a, 0x3c, 0x83, 0x40, 0x3d, 0x28, 0x7b, 0xc1, 0xd4, 0x8f, 0x58, 0xb3, 0x2c, 0x04, 0x7c, 0x2b, - 0x23, 0xd8, 0x1e, 0x5f, 0x84, 0xd5, 0x5a, 0xf4, 0x18, 0x56, 0x1d, 0x72, 0xea, 0x72, 0x99, 0x56, - 0x05, 0xcc, 0xdd, 0xac, 0xf6, 0x15, 0xab, 0x70, 0xbc, 0x9a, 0x2b, 0x7d, 0xca, 0x08, 0x6d, 0x56, - 0xa4, 0xd2, 0xf9, 0x37, 0x7a, 0x05, 0xaa, 0xd6, 0x64, 0x12, 0xd8, 0xa6, 0xe3, 0xd2, 0x66, 0x55, - 0x0c, 0x54, 0x04, 0xa3, 0xe7, 0x52, 0xf4, 0x2a, 0xd4, 0xe4, 0xc9, 0x30, 0x43, 0x2b, 0x1a, 0x37, - 0x41, 0x0c, 0x83, 0x64, 0x1d, 0x58, 0xd1, 0x58, 0x4d, 0x20, 0x94, 0xca, 0x09, 0xb5, 0x64, 0x02, - 0xa1, 0x94, 0x4f, 0x68, 0xbd, 0x07, 0x95, 0x58, 0xc5, 0x0b, 0xd5, 0xa3, 0x7f, 0xcd, 0x41, 0x35, - 0x51, 0x29, 0xfa, 0x14, 0xd6, 0xa8, 0xf5, 0xdc, 0x9c, 0xd9, 0x46, 0x06, 0xc0, 0x77, 0xb3, 0xda, - 0xc6, 0x7a, 0x3e, 0x33, 0x4f, 0x9d, 0xa6, 0x28, 0xf4, 0x39, 0xac, 0x4f, 0x5c, 0x7f, 0x7a, 0x96, - 0xc2, 0x96, 0x19, 0xe5, 0xff, 0x33, 0x62, 0xef, 0xf2, 0xd5, 0x33, 0xf4, 0xc6, 0x64, 0x8e, 0xd6, - 0x7f, 0x9f, 0x83, 0x7a, 0x7a, 0x7b, 0xae, 0x04, 0x3b, 0x9c, 0x8a, 0x3f, 0x50, 0xc0, 0xfc, 0x93, - 0x07, 0x1d, 0x8f, 0x78, 0x01, 0x3d, 0x17, 0x3b, 0x17, 0xb0, 0xa2, 0xb8, 0xb5, 0x1c, 0x97, 0x9d, - 0x08, 0xef, 0x2f, 0x60, 0xf1, 0xcd, 0x79, 0x6e, 0x10, 0x32, 0x51, 0x8f, 0x15, 0xb0, 0xf8, 0x46, - 0x18, 0x2a, 0x2a, 0xd5, 0x70, 0x9f, 0x2d, 0x2c, 0x9e, 0xb2, 0x62, 0xe1, 0x70, 0x82, 0xa3, 0xff, - 0x3a, 0x0f, 0xeb, 0x17, 0x46, 0xb9, 0x9c, 0xd2, 0x91, 0xe2, 0x80, 0x2d, 0x29, 0x2e, 0x93, 0xed, - 0x3a, 0x71, 0x55, 0x24, 0xbe, 0xc5, 0x71, 0x0f, 0x55, 0xc5, 0x92, 0x77, 0x43, 0x6e, 0x68, 0xef, - 0xc8, 0x8d, 0xa4, 0xe0, 0x25, 0x2c, 0x09, 0xf4, 0x0c, 0x1a, 0x94, 0x30, 0x42, 0x4f, 0x89, 0x63, - 0x86, 0x01, 0x8d, 0x62, 0xf9, 0xb7, 0x16, 0x93, 0xff, 0x20, 0xa0, 0x11, 0x5e, 0x8b, 0x91, 0x38, - 0xc5, 0xd0, 0x53, 0x58, 0x73, 0xce, 0x7d, 0xcb, 0x73, 0x6d, 0x85, 0x5c, 0x5e, 0x1a, 0xb9, 0xae, - 0x80, 0x04, 0x30, 0xbf, 0x28, 0xa5, 0x06, 0xf9, 0x1f, 0x9b, 0x58, 0x47, 0x64, 0xa2, 0x74, 0x22, - 0x89, 0x79, 0xbf, 0x2e, 0x29, 0xbf, 0xd6, 0xff, 0x9d, 0x83, 0xc6, 0xbc, 0xbb, 0xa0, 0xff, 0x05, - 0xb0, 0xc3, 0xa9, 0x19, 0x12, 0xea, 0x06, 0x8e, 0x72, 0x8a, 0xaa, 0x1d, 0x4e, 0x0f, 0x04, 0x83, - 0x1f, 0x4e, 0x3e, 0xfc, 0xe5, 0x34, 0x88, 0x2c, 0xe5, 0x1d, 0x15, 0x3b, 0x9c, 0x7e, 0x93, 0xd3, - 0xf1, 0x5a, 0x71, 0x93, 0x67, 0xca, 0x4b, 0xf8, 0xf4, 0x81, 0x60, 0xa0, 0xb7, 0x00, 0x49, 0x47, - 0x32, 0x27, 0xae, 0xe7, 0x46, 0xe6, 0xd1, 0x39, 0xbf, 0x13, 0x4b, 0xc7, 0xd1, 0xe4, 0xc8, 0x2e, - 0x1f, 0xf8, 0x98, 0xf3, 0x91, 0x0e, 0x6b, 0x41, 0xe0, 0x99, 0xcc, 0x0e, 0x28, 0x31, 0x2d, 0xe7, - 0x0b, 0x11, 0xfd, 0x0a, 0xb8, 0x16, 0x04, 0xde, 0x80, 0xf3, 0x3a, 0xce, 0x17, 0xfc, 0xb0, 0xdb, - 0xe1, 0x94, 0x91, 0xc8, 0xe4, 0x3f, 0xcd, 0xb2, 0x3c, 0xec, 0x92, 0xd5, 0x0d, 0xa7, 0x2c, 0x35, - 0xc1, 0x23, 0x1e, 0x0f, 0x56, 0xa9, 0x09, 0x7b, 0xc4, 0x63, 0xfa, 0x67, 0x50, 0x12, 0xa1, 0x8d, - 0xff, 0x31, 0x91, 0xfd, 0x45, 0xd4, 0x90, 0xaa, 0xab, 0x70, 0x86, 0x08, 0x2a, 0xaf, 0x40, 0x75, - 0x1c, 0x30, 0x15, 0x73, 0xa4, 0x57, 0x55, 0x38, 0x43, 0x0c, 0xb6, 0xa0, 0x42, 0x89, 0xe5, 0x04, - 0xfe, 0xe4, 0x5c, 0xfc, 0xe7, 0x0a, 0x4e, 0x68, 0xfd, 0x4b, 0x28, 0xcb, 0x90, 0x77, 0x0d, 0xfc, - 0xbb, 0x80, 0xec, 0x11, 0x0d, 0xa6, 0x21, 0x37, 0x8a, 0xe7, 0x32, 0xe6, 0x06, 0x3e, 0x8b, 0xaf, - 0xdd, 0x72, 0xe4, 0x60, 0x36, 0xa0, 0xff, 0x25, 0x27, 0xd3, 0x9c, 0xbc, 0x43, 0xf1, 0x42, 0x41, - 0xe5, 0xac, 0xa5, 0x2f, 0x9a, 0x0a, 0x20, 0x2e, 0xf6, 0x88, 0x7a, 0x5e, 0x58, 0xb4, 0xd8, 0x23, - 0xb2, 0xd8, 0x23, 0xbc, 0xb2, 0x50, 0xd9, 0x54, 0xc2, 0xc9, 0x64, 0x5a, 0x73, 0x92, 0x2a, 0x98, - 0xe8, 0xff, 0xcc, 0x25, 0xa7, 0x3d, 0xae, 0x56, 0xd1, 0xe7, 0x50, 0xe1, 0x07, 0xc7, 0xf4, 0xac, - 0x50, 0x3d, 0xa4, 0x74, 0x97, 0x2b, 0x84, 0xdb, 0xfc, 0x9c, 0xec, 0x59, 0xa1, 0x4c, 0xb3, 0xab, - 0xa1, 0xa4, 0x78, 0xd4, 0xb0, 0x9c, 0x59, 0xd4, 0xe0, 0xdf, 0xe8, 0x75, 0x68, 0x58, 0xd3, 0x28, - 0x30, 0x2d, 0xe7, 0x94, 0xd0, 0xc8, 0x65, 0x44, 0x59, 0x78, 0x8d, 0x73, 0x3b, 0x31, 0xb3, 0xf5, - 0x10, 0xea, 0x69, 0xcc, 0xaf, 0xcb, 0x2b, 0xa5, 0x74, 0x5e, 0xf9, 0x2e, 0xc0, 0xac, 0x28, 0xe3, - 0x9e, 0x40, 0xce, 0xdc, 0xc8, 0xb4, 0x03, 0x47, 0x46, 0xb5, 0x12, 0xae, 0x70, 0x46, 0x37, 0x70, - 0xc8, 0x85, 0x12, 0xb7, 0x14, 0x97, 0xb8, 0xfc, 0xdc, 0xf1, 0xa3, 0x72, 0xe2, 0x4e, 0x26, 0xc4, - 0x51, 0x12, 0x56, 0x83, 0xc0, 0x7b, 0x22, 0x18, 0xfa, 0x9f, 0xf3, 0xd2, 0x23, 0xe4, 0x05, 0x23, - 0x53, 0xe1, 0x93, 0x98, 0xba, 0x70, 0x3d, 0x53, 0x3f, 0x00, 0x60, 0x91, 0x45, 0x23, 0xe2, 0x98, - 0x56, 0xa4, 0xee, 0xec, 0xad, 0x4b, 0x35, 0xf2, 0x30, 0x7e, 0xab, 0xc4, 0x55, 0x35, 0xbb, 0x13, - 0xa1, 0xf7, 0xa1, 0x6e, 0x07, 0x5e, 0x38, 0x21, 0x6a, 0x71, 0xe9, 0x6b, 0x17, 0xd7, 0x92, 0xf9, - 0x9d, 0x28, 0x55, 0x20, 0x97, 0xaf, 0x5b, 0x20, 0xff, 0x21, 0x27, 0xef, 0x49, 0xe9, 0x6b, 0x1a, - 0x1a, 0x5d, 0xf1, 0xb0, 0xf7, 0x78, 0xc9, 0x3b, 0xdf, 0x57, 0xbd, 0xea, 0x5d, 0xf7, 0x19, 0xed, - 0x8f, 0x05, 0xa8, 0x26, 0xd7, 0xad, 0x4b, 0xb6, 0xbf, 0x0f, 0xd5, 0xe4, 0xa1, 0x58, 0x95, 0x19, - 0x5f, 0x69, 0x9e, 0x64, 0x32, 0x3a, 0x06, 0x64, 0x8d, 0x46, 0x49, 0x91, 0x62, 0x4e, 0x99, 0x35, - 0x8a, 0x2f, 0xa8, 0xf7, 0x17, 0xd0, 0x43, 0x9c, 0x79, 0x0e, 0xf9, 0x7a, 0xac, 0x59, 0xa3, 0xd1, - 0x1c, 0x07, 0x7d, 0x0f, 0x6e, 0xcf, 0xef, 0x61, 0x1e, 0x9d, 0x9b, 0xa1, 0xeb, 0xa8, 0x02, 0x7b, - 0x7b, 0xd1, 0x1b, 0x67, 0x7b, 0x0e, 0xfe, 0xe3, 0xf3, 0x03, 0xd7, 0x91, 0x3a, 0x47, 0xf4, 0xd2, - 0x40, 0xeb, 0x07, 0xf0, 0xd2, 0x0b, 0xa6, 0x5f, 0x61, 0x83, 0x7e, 0xda, 0x06, 0xd7, 0x51, 0x42, - 0xca, 0x7a, 0xbf, 0xc9, 0xc9, 0x8b, 0xf1, 0xbc, 0x4e, 0x3a, 0xb3, 0x8a, 0xad, 0xb6, 0xb5, 0x99, - 0x71, 0x9f, 0xee, 0xc1, 0xa1, 0x84, 0x17, 0x25, 0xde, 0x27, 0x73, 0x25, 0x5e, 0xf6, 0x32, 0x64, - 0x4f, 0x2c, 0x92, 0x40, 0x0a, 0x41, 0xff, 0x5d, 0x01, 0x2a, 0x31, 0xba, 0xa8, 0xbf, 0xcf, 0x59, - 0x44, 0x3c, 0xd3, 0x8b, 0x43, 0x58, 0x0e, 0x83, 0x64, 0xed, 0xf1, 0x20, 0xf6, 0x0a, 0x54, 0x79, - 0x99, 0x2f, 0x87, 0xf3, 0x62, 0xb8, 0xc2, 0x19, 0x62, 0xf0, 0x55, 0xa8, 0x45, 0x41, 0x64, 0x4d, - 0xcc, 0xc8, 0xb5, 0x4f, 0x64, 0x92, 0xcb, 0x61, 0x10, 0xac, 0x21, 0xe7, 0xa0, 0x37, 0xe1, 0x46, - 0x34, 0xa6, 0x41, 0x14, 0x4d, 0x78, 0x85, 0x26, 0x6a, 0x12, 0x59, 0x42, 0x14, 0xb1, 0x96, 0x0c, - 0xc8, 0x5a, 0x85, 0xf1, 0xe8, 0x3d, 0x9b, 0xcc, 0x5d, 0x57, 0x04, 0x91, 0x22, 0x5e, 0x4b, 0xb8, - 0xdc, 0xb5, 0xf9, 0xfd, 0x3e, 0x24, 0xd4, 0x26, 0xbe, 0x8c, 0x15, 0x39, 0x1c, 0x93, 0xc8, 0x84, - 0x75, 0x8f, 0x58, 0x6c, 0x4a, 0x89, 0x63, 0x1e, 0xbb, 0x64, 0xe2, 0xc8, 0xfb, 0x4e, 0x23, 0x73, - 0x3d, 0x1b, 0xab, 0xa5, 0xfd, 0x48, 0xac, 0xc6, 0x8d, 0x18, 0x4e, 0xd2, 0xbc, 0x3e, 0x90, 0x5f, - 0x68, 0x1d, 0x6a, 0x83, 0x67, 0x83, 0xa1, 0xb1, 0x67, 0xee, 0xed, 0xf7, 0x0c, 0xf5, 0xc0, 0x3d, - 0x30, 0xb0, 0x24, 0x73, 0x7c, 0x7c, 0xb8, 0x3f, 0xec, 0xec, 0x9a, 0xc3, 0x9d, 0xee, 0x93, 0x81, - 0x96, 0x47, 0xb7, 0xe1, 0xc6, 0x70, 0x1b, 0xef, 0x0f, 0x87, 0xbb, 0x46, 0xcf, 0x3c, 0x30, 0xf0, - 0xce, 0x7e, 0x6f, 0xa0, 0x15, 0x10, 0x82, 0xc6, 0x8c, 0x3d, 0xdc, 0xd9, 0x33, 0xb4, 0x22, 0xaa, - 0xc1, 0xea, 0x81, 0x81, 0xbb, 0x46, 0x7f, 0xa8, 0x95, 0xf4, 0xbf, 0xe5, 0xa1, 0x96, 0xb2, 0x22, - 0x77, 0x64, 0xca, 0xe4, 0xfd, 0xa5, 0x88, 0xf9, 0x27, 0x0f, 0x26, 0xb6, 0x65, 0x8f, 0xa5, 0x75, - 0x8a, 0x58, 0x12, 0xdc, 0x6e, 0x9e, 0x75, 0x96, 0x3a, 0xe7, 0x45, 0x5c, 0xf1, 0xac, 0x33, 0x09, - 0xf2, 0x1a, 0xd4, 0x4f, 0x08, 0xf5, 0xc9, 0x44, 0x8d, 0x4b, 0x8b, 0xd4, 0x24, 0x4f, 0x4e, 0xd9, - 0x00, 0x4d, 0x4d, 0x99, 0xc1, 0x48, 0x73, 0x34, 0x24, 0x7f, 0x2f, 0x06, 0x3b, 0xba, 0xac, 0xf5, - 0xb2, 0xd0, 0xfa, 0x83, 0xc5, 0x9d, 0xf4, 0x45, 0x8a, 0x1f, 0x24, 0x8a, 0x5f, 0x85, 0x02, 0x8e, - 0xdf, 0x7a, 0xbb, 0x9d, 0xee, 0x36, 0x57, 0xf6, 0x1a, 0x54, 0xf7, 0x3a, 0x9f, 0x9a, 0x87, 0x03, - 0xf1, 0x72, 0x80, 0x34, 0xa8, 0x3f, 0x31, 0x70, 0xdf, 0xd8, 0x55, 0x9c, 0x02, 0xba, 0x05, 0x9a, - 0xe2, 0xcc, 0xe6, 0x15, 0xf5, 0xdf, 0xe6, 0x61, 0x5d, 0xc6, 0xf5, 0xe4, 0x31, 0xeb, 0xc5, 0xaf, - 0x4a, 0xcb, 0x87, 0xde, 0x26, 0xac, 0x7a, 0x84, 0x25, 0x76, 0xa8, 0xe2, 0x98, 0x44, 0x2e, 0xd4, - 0x2c, 0xdf, 0x0f, 0x22, 0xf1, 0x22, 0xc2, 0x54, 0x88, 0x7c, 0xbc, 0xd0, 0xdb, 0x4b, 0x22, 0x79, - 0xbb, 0x33, 0x43, 0x92, 0x11, 0x32, 0x8d, 0xdd, 0xfa, 0x00, 0xb4, 0x8b, 0x13, 0x16, 0xc9, 0x4b, - 0x6f, 0xbc, 0x33, 0x4b, 0x4b, 0x84, 0x3b, 0xe8, 0x61, 0xff, 0x49, 0x7f, 0xff, 0x69, 0x5f, 0x5b, - 0xe1, 0x04, 0x3e, 0xec, 0xf7, 0x77, 0xfa, 0x8f, 0xb5, 0x1c, 0x02, 0x28, 0x1b, 0x9f, 0xee, 0x0c, - 0x8d, 0x9e, 0x96, 0xdf, 0xfa, 0xfb, 0x1a, 0x94, 0xa5, 0x90, 0xe8, 0x57, 0x2a, 0x25, 0xa7, 0x3b, - 0x8e, 0xe8, 0x83, 0x85, 0x4b, 0xdb, 0xb9, 0x2e, 0x66, 0xeb, 0xc3, 0xa5, 0xd7, 0xab, 0x47, 0xc5, - 0x15, 0xf4, 0xb3, 0x1c, 0xd4, 0xe7, 0x5e, 0xd1, 0xb2, 0x3e, 0x00, 0x5d, 0xd1, 0xe0, 0x6c, 0x7d, - 0x63, 0xa9, 0xb5, 0x89, 0x2c, 0x3f, 0xcd, 0x41, 0x2d, 0xd5, 0xda, 0x43, 0x0f, 0x96, 0x69, 0x07, - 0x4a, 0x49, 0x1e, 0x2e, 0xdf, 0x49, 0xd4, 0x57, 0xde, 0xce, 0xa1, 0x9f, 0xe4, 0xa0, 0x96, 0xea, - 0x8b, 0x65, 0x16, 0xe5, 0x72, 0x17, 0x2f, 0xb3, 0x28, 0x57, 0xb5, 0xe1, 0x56, 0xd0, 0x0f, 0x73, - 0x50, 0x4d, 0x7a, 0x5c, 0xe8, 0xde, 0xe2, 0x5d, 0x31, 0x29, 0xc4, 0xfd, 0x65, 0xdb, 0x69, 0xfa, - 0x0a, 0xfa, 0x3e, 0x54, 0xe2, 0x86, 0x10, 0xca, 0x9a, 0x46, 0x2e, 0x74, 0x9b, 0x5a, 0xf7, 0x16, - 0x5e, 0x97, 0xde, 0x3e, 0xee, 0xd2, 0x64, 0xde, 0xfe, 0x42, 0x3f, 0xa9, 0x75, 0x6f, 0xe1, 0x75, - 0xc9, 0xf6, 0xdc, 0x13, 0x52, 0xcd, 0x9c, 0xcc, 0x9e, 0x70, 0xb9, 0x8b, 0x94, 0xd9, 0x13, 0xae, - 0xea, 0x1d, 0x49, 0x41, 0x52, 0xed, 0xa0, 0xcc, 0x82, 0x5c, 0x6e, 0x39, 0x65, 0x16, 0xe4, 0x8a, - 0xee, 0x93, 0x72, 0xc9, 0x59, 0x81, 0x7e, 0x6f, 0xe1, 0x0e, 0xca, 0x82, 0x2e, 0x79, 0xa9, 0x87, - 0xa3, 0xaf, 0xa0, 0x1f, 0xa9, 0x27, 0x03, 0xd9, 0x7e, 0x41, 0x8b, 0x40, 0xcd, 0x75, 0x6c, 0x5a, - 0xef, 0x2d, 0x97, 0x6a, 0x44, 0x8c, 0xf8, 0x71, 0x0e, 0x60, 0xd6, 0xa8, 0xc9, 0x2c, 0xc4, 0xa5, - 0x0e, 0x51, 0xeb, 0xc1, 0x12, 0x2b, 0xd3, 0xc7, 0x23, 0xee, 0xcd, 0x64, 0x3e, 0x1e, 0x17, 0x1a, - 0x49, 0x99, 0x8f, 0xc7, 0xc5, 0x26, 0x90, 0xbe, 0xf2, 0xf1, 0xea, 0xb7, 0x4a, 0x32, 0xf7, 0x97, - 0xc5, 0xcf, 0xbb, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x34, 0x96, 0x6b, 0x66, 0x2d, 0x24, 0x00, - 0x00, + 0x33, 0x5a, 0x09, 0x41, 0x90, 0x20, 0x01, 0x82, 0x24, 0x40, 0x80, 0x5c, 0x82, 0xdc, 0x73, 0x0b, + 0x72, 0xcd, 0x29, 0x8f, 0x4b, 0x80, 0xfc, 0x0f, 0x39, 0xe6, 0x12, 0x20, 0xd7, 0xfc, 0x05, 0x09, + 0xfa, 0x31, 0xc3, 0xa1, 0xa4, 0xb5, 0x87, 0x54, 0x4e, 0x9c, 0xae, 0xee, 0xfa, 0x75, 0xb1, 0xaa, + 0xba, 0xaa, 0xba, 0x0b, 0xf4, 0x70, 0x12, 0x8f, 0x5c, 0x9f, 0x6d, 0x38, 0xd4, 0x3d, 0x25, 0x94, + 0x6d, 0x84, 0x34, 0x88, 0x02, 0x35, 0xea, 0x88, 0x01, 0x7a, 0x7d, 0x6c, 0xb1, 0xb1, 0x6b, 0x07, + 0x34, 0xec, 0xf8, 0x81, 0x67, 0x39, 0x1d, 0xc5, 0xd3, 0x51, 0x3c, 0x72, 0x59, 0xfb, 0xff, 0x46, + 0x41, 0x30, 0x9a, 0x10, 0x89, 0x70, 0x14, 0x1f, 0x6f, 0x38, 0x31, 0xb5, 0x22, 0x37, 0xf0, 0xd5, + 0xfc, 0xab, 0x17, 0xe7, 0x23, 0xd7, 0x23, 0x2c, 0xb2, 0xbc, 0x50, 0x2d, 0xf8, 0x68, 0xe4, 0x46, + 0xe3, 0xf8, 0xa8, 0x63, 0x07, 0xde, 0x46, 0xba, 0xe5, 0x86, 0xd8, 0x72, 0x23, 0x11, 0x93, 0x8d, + 0x2d, 0x4a, 0x9c, 0x8d, 0xb1, 0x3d, 0x61, 0x21, 0xb1, 0xf9, 0xaf, 0xc9, 0x3f, 0x24, 0x82, 0xfe, + 0x32, 0xbc, 0x34, 0xb4, 0xd8, 0x49, 0x2f, 0xf0, 0x8f, 0xdd, 0xd1, 0xc0, 0x1e, 0x13, 0xcf, 0xc2, + 0xe4, 0xcb, 0x98, 0xb0, 0x48, 0xff, 0x0e, 0xb4, 0x2e, 0x4f, 0xb1, 0x30, 0xf0, 0x19, 0x41, 0x1f, + 0x41, 0x99, 0x83, 0xb4, 0x0a, 0x77, 0x0a, 0xeb, 0x8d, 0xcd, 0xb7, 0x3a, 0x2f, 0xfa, 0xbf, 0x72, + 0xf3, 0x8e, 0xda, 0xbc, 0x33, 0x08, 0x89, 0x8d, 0x05, 0xa7, 0x7e, 0x1b, 0x6e, 0xf6, 0xac, 0xd0, + 0x3a, 0x72, 0x27, 0x6e, 0xe4, 0x12, 0x96, 0x6c, 0x1a, 0xc3, 0xad, 0x59, 0xb2, 0xda, 0xf0, 0x33, + 0x58, 0xb1, 0x33, 0x74, 0xb5, 0xf1, 0x83, 0x4e, 0x2e, 0x45, 0x77, 0xb6, 0xc4, 0x68, 0x06, 0x78, + 0x06, 0x4e, 0xbf, 0x05, 0xe8, 0x91, 0xeb, 0x8f, 0x08, 0x0d, 0xa9, 0xeb, 0x47, 0x89, 0x30, 0x3f, + 0x2f, 0xc1, 0xcd, 0x19, 0xb2, 0x12, 0xe6, 0x0b, 0x00, 0x2b, 0x8a, 0xa8, 0x7b, 0x14, 0x47, 0x42, + 0x94, 0xd2, 0x7a, 0x63, 0xf3, 0x93, 0x9c, 0xa2, 0x5c, 0x81, 0xd7, 0xe9, 0xa6, 0x60, 0x86, 0x1f, + 0xd1, 0x73, 0x9c, 0x41, 0x47, 0x9f, 0x43, 0x75, 0x4c, 0xac, 0x49, 0x34, 0x6e, 0x15, 0xef, 0x14, + 0xd6, 0x9b, 0x9b, 0x8f, 0xae, 0xb1, 0xcf, 0xb6, 0x00, 0x1a, 0x44, 0x56, 0x44, 0xb0, 0x42, 0x45, + 0x77, 0x01, 0xc9, 0x2f, 0xd3, 0x21, 0xcc, 0xa6, 0x6e, 0xc8, 0xfd, 0xaf, 0x55, 0xba, 0x53, 0x58, + 0xaf, 0xe3, 0x1b, 0x72, 0x66, 0x6b, 0x3a, 0xd1, 0x7e, 0x1f, 0xd6, 0x2e, 0x48, 0x8b, 0x34, 0x28, + 0x9d, 0x90, 0x73, 0x61, 0x91, 0x3a, 0xe6, 0x9f, 0xe8, 0x16, 0x54, 0x4e, 0xad, 0x49, 0x4c, 0x84, + 0xc8, 0x75, 0x2c, 0x07, 0x0f, 0x8b, 0xf7, 0x0b, 0xfa, 0x03, 0x68, 0x64, 0x84, 0x40, 0x4d, 0x80, + 0xc3, 0xfe, 0x96, 0x31, 0x34, 0x7a, 0x43, 0x63, 0x4b, 0x5b, 0x42, 0xab, 0x50, 0x3f, 0xec, 0x6f, + 0x1b, 0xdd, 0xdd, 0xe1, 0xf6, 0x33, 0xad, 0x80, 0x1a, 0xb0, 0x9c, 0x0c, 0x8a, 0xfa, 0x19, 0x20, + 0x4c, 0xec, 0xe0, 0x94, 0x50, 0xee, 0x95, 0xca, 0x44, 0xe8, 0x25, 0x58, 0x8e, 0x2c, 0x76, 0x62, + 0xba, 0x8e, 0x12, 0xa0, 0xca, 0x87, 0x3b, 0x0e, 0xda, 0x81, 0xea, 0xd8, 0xf2, 0x9d, 0x89, 0x14, + 0xa2, 0xb1, 0xf9, 0x4e, 0x4e, 0xbd, 0x71, 0xf0, 0x6d, 0xc1, 0x88, 0x15, 0x00, 0x77, 0xd5, 0x99, + 0x9d, 0xa5, 0x36, 0xf5, 0x67, 0xa0, 0x0d, 0x22, 0x8b, 0x46, 0x59, 0x71, 0x0c, 0x28, 0xf3, 0xfd, + 0x95, 0x7b, 0xce, 0xb3, 0xa7, 0x3c, 0x66, 0x58, 0xb0, 0xeb, 0xff, 0x2a, 0xc2, 0x8d, 0x0c, 0xb6, + 0x72, 0xbb, 0xa7, 0x50, 0xa5, 0x84, 0xc5, 0x93, 0x48, 0xc0, 0x37, 0x37, 0x3f, 0xcc, 0x09, 0x7f, + 0x09, 0xa9, 0x83, 0x05, 0x0c, 0x56, 0x70, 0x68, 0x1d, 0x34, 0xc9, 0x61, 0x12, 0x4a, 0x03, 0x6a, + 0x7a, 0x6c, 0xa4, 0x4c, 0xd7, 0x94, 0x74, 0x83, 0x93, 0xf7, 0xd8, 0x28, 0xa3, 0xd5, 0xd2, 0x35, + 0xb5, 0x8a, 0x2c, 0xd0, 0x7c, 0x12, 0x3d, 0x0f, 0xe8, 0x89, 0xc9, 0x55, 0x4b, 0x5d, 0x87, 0xb4, + 0xca, 0x02, 0xf4, 0xbd, 0x9c, 0xa0, 0x7d, 0xc9, 0xbe, 0xaf, 0xb8, 0xf1, 0x9a, 0x3f, 0x4b, 0xd0, + 0xdf, 0x84, 0xaa, 0xfc, 0xa7, 0xdc, 0x93, 0x06, 0x87, 0xbd, 0x9e, 0x31, 0x18, 0x68, 0x4b, 0xa8, + 0x0e, 0x15, 0x6c, 0x0c, 0x31, 0xf7, 0xb0, 0x3a, 0x54, 0x1e, 0x75, 0x87, 0xdd, 0x5d, 0xad, 0xa8, + 0xbf, 0x01, 0x6b, 0x4f, 0x2d, 0x37, 0xca, 0xe3, 0x5c, 0x7a, 0x00, 0xda, 0x74, 0xad, 0xb2, 0xce, + 0xce, 0x8c, 0x75, 0xf2, 0xab, 0xc6, 0x38, 0x73, 0xa3, 0x0b, 0xf6, 0xd0, 0xa0, 0x44, 0x28, 0x55, + 0x26, 0xe0, 0x9f, 0xfa, 0x73, 0x58, 0x1b, 0x44, 0x41, 0x98, 0xcb, 0xf3, 0xdf, 0x85, 0x65, 0x9e, + 0x27, 0x82, 0x38, 0x52, 0xae, 0xff, 0x72, 0x47, 0xe6, 0x91, 0x4e, 0x92, 0x47, 0x3a, 0x5b, 0x2a, + 0xcf, 0xe0, 0x64, 0x25, 0xfa, 0x1f, 0xa8, 0x32, 0x77, 0xe4, 0x5b, 0x13, 0x75, 0xf4, 0xd5, 0x48, + 0x47, 0xdc, 0xc9, 0x93, 0x8d, 0x95, 0xe3, 0xf7, 0x00, 0x6d, 0x11, 0x16, 0xd1, 0xe0, 0x3c, 0x97, + 0x3c, 0xb7, 0xa0, 0x72, 0x1c, 0x50, 0x5b, 0x1e, 0xc4, 0x1a, 0x96, 0x03, 0x7e, 0xa8, 0x66, 0x40, + 0x14, 0xf6, 0x5d, 0x40, 0x3b, 0x3e, 0x4f, 0x10, 0xf9, 0x0c, 0xf1, 0xcb, 0x22, 0xdc, 0x9c, 0x59, + 0xaf, 0x8c, 0xb1, 0xf8, 0x39, 0xe4, 0x81, 0x29, 0x66, 0xf2, 0x1c, 0xa2, 0x7d, 0xa8, 0xca, 0x15, + 0x4a, 0x93, 0xf7, 0xe6, 0x00, 0x92, 0x39, 0x47, 0xc1, 0x29, 0x98, 0x2b, 0x9d, 0xbe, 0xf4, 0xdf, + 0x76, 0x7a, 0x2d, 0xf9, 0x1f, 0xec, 0x6b, 0xf5, 0xf7, 0x6d, 0xb8, 0x91, 0x59, 0xac, 0x94, 0xf7, + 0x08, 0x2a, 0x8c, 0x13, 0x94, 0xf6, 0xde, 0x9e, 0x53, 0x7b, 0x0c, 0x4b, 0x76, 0xfd, 0xa6, 0x04, + 0x37, 0x4e, 0x89, 0x9f, 0x8a, 0xa2, 0x6f, 0xc1, 0x8d, 0x81, 0x70, 0xad, 0x5c, 0xbe, 0x33, 0x75, + 0xcb, 0xe2, 0x8c, 0x5b, 0xde, 0x02, 0x94, 0x45, 0x51, 0xce, 0x73, 0x0e, 0x6b, 0xc6, 0x19, 0xb1, + 0x73, 0x21, 0xb7, 0x60, 0xd9, 0x0e, 0x3c, 0xcf, 0xf2, 0x9d, 0x56, 0xf1, 0x4e, 0x69, 0xbd, 0x8e, + 0x93, 0x61, 0xf6, 0xfc, 0x94, 0xf2, 0x9e, 0x1f, 0xfd, 0x17, 0x05, 0xd0, 0xa6, 0x7b, 0x2b, 0x45, + 0x72, 0xe9, 0x23, 0x87, 0x03, 0xf1, 0xbd, 0x57, 0xb0, 0x1a, 0x29, 0x7a, 0x72, 0xc4, 0x25, 0x9d, + 0x50, 0x9a, 0x09, 0x21, 0xa5, 0x6b, 0x86, 0x10, 0xfd, 0x1f, 0x05, 0x40, 0x97, 0xab, 0x1e, 0xf4, + 0x1a, 0xac, 0x30, 0xe2, 0x3b, 0xa6, 0x54, 0xa3, 0xb4, 0x70, 0x0d, 0x37, 0x38, 0x4d, 0xea, 0x93, + 0x21, 0x04, 0x65, 0x72, 0x46, 0x6c, 0x75, 0x5a, 0xc5, 0x37, 0x1a, 0xc3, 0xca, 0x31, 0x33, 0x5d, + 0x16, 0x4c, 0xac, 0xb4, 0x3c, 0x68, 0x6e, 0x1a, 0x0b, 0x57, 0x5f, 0x9d, 0x47, 0x83, 0x9d, 0x04, + 0x0c, 0x37, 0x8e, 0x59, 0x3a, 0xd0, 0x3b, 0xd0, 0xc8, 0xcc, 0xa1, 0x1a, 0x94, 0xfb, 0xfb, 0x7d, + 0x43, 0x5b, 0x42, 0x00, 0xd5, 0xde, 0x36, 0xde, 0xdf, 0x1f, 0xca, 0xa8, 0xbd, 0xb3, 0xd7, 0x7d, + 0x6c, 0x68, 0x45, 0xfd, 0x4f, 0x65, 0x80, 0x69, 0xfa, 0x44, 0x4d, 0x28, 0xa6, 0x96, 0x2e, 0xba, + 0x0e, 0xff, 0x33, 0xbe, 0xe5, 0x25, 0x85, 0x88, 0xf8, 0x46, 0x9b, 0x70, 0xdb, 0x63, 0xa3, 0xd0, + 0xb2, 0x4f, 0x4c, 0x95, 0xf5, 0x6c, 0xc1, 0x2c, 0xfe, 0xd5, 0x0a, 0xbe, 0xa9, 0x26, 0x95, 0xd4, + 0x12, 0x77, 0x17, 0x4a, 0xc4, 0x3f, 0x6d, 0x95, 0x45, 0xa9, 0xf7, 0x70, 0xee, 0xb4, 0xde, 0x31, + 0xfc, 0x53, 0x59, 0xda, 0x71, 0x18, 0xd4, 0x87, 0x3a, 0x25, 0x2c, 0x88, 0xa9, 0x4d, 0x58, 0xab, + 0x32, 0xd7, 0x21, 0xc3, 0x09, 0x1f, 0x9e, 0x42, 0xa0, 0x2d, 0xa8, 0x7a, 0x41, 0xec, 0x47, 0xac, + 0x55, 0x15, 0x02, 0xbe, 0x95, 0x13, 0x6c, 0x8f, 0x33, 0x61, 0xc5, 0x8b, 0x1e, 0xc3, 0xb2, 0x43, + 0x4e, 0x5d, 0x2e, 0xd3, 0xb2, 0x80, 0xb9, 0x9b, 0xd7, 0xbe, 0x82, 0x0b, 0x27, 0xdc, 0x5c, 0xe9, + 0x31, 0x23, 0xb4, 0x55, 0x93, 0x4a, 0xe7, 0xdf, 0xe8, 0x15, 0xa8, 0x5b, 0x93, 0x49, 0x60, 0x9b, + 0x8e, 0x4b, 0x5b, 0x75, 0x31, 0x51, 0x13, 0x84, 0x2d, 0x97, 0xa2, 0x57, 0xa1, 0x21, 0x4f, 0x86, + 0x19, 0x5a, 0xd1, 0xb8, 0x05, 0x62, 0x1a, 0x24, 0xe9, 0xc0, 0x8a, 0xc6, 0x6a, 0x01, 0xa1, 0x54, + 0x2e, 0x68, 0xa4, 0x0b, 0x08, 0xa5, 0x7c, 0x41, 0xfb, 0x3d, 0xa8, 0x25, 0x2a, 0x9e, 0xab, 0x1e, + 0xfd, 0x6b, 0x01, 0xea, 0xa9, 0x4a, 0xd1, 0xa7, 0xb0, 0x4a, 0xad, 0xe7, 0xe6, 0xd4, 0x36, 0x32, + 0x00, 0xbe, 0x9b, 0xd7, 0x36, 0xd6, 0xf3, 0xa9, 0x79, 0x56, 0x68, 0x66, 0x84, 0x3e, 0x87, 0xb5, + 0x89, 0xeb, 0xc7, 0x67, 0x19, 0x6c, 0x99, 0x51, 0xfe, 0x3f, 0x27, 0xf6, 0x2e, 0xe7, 0x9e, 0xa2, + 0x37, 0x27, 0x33, 0x63, 0xfd, 0xf7, 0x05, 0x58, 0xc9, 0x6e, 0xcf, 0x95, 0x60, 0x87, 0xb1, 0xf8, + 0x03, 0x25, 0xcc, 0x3f, 0x79, 0xd0, 0xf1, 0x88, 0x17, 0xd0, 0x73, 0xb1, 0x73, 0x09, 0xab, 0x11, + 0xb7, 0x96, 0xe3, 0xb2, 0x13, 0xe1, 0xfd, 0x25, 0x2c, 0xbe, 0x39, 0xcd, 0x0d, 0x42, 0x26, 0xea, + 0xb1, 0x12, 0x16, 0xdf, 0x08, 0x43, 0x4d, 0xa5, 0x1a, 0xee, 0xb3, 0xa5, 0xf9, 0x53, 0x56, 0x22, + 0x1c, 0x4e, 0x71, 0xf4, 0x5f, 0x17, 0x61, 0xed, 0xc2, 0x2c, 0x97, 0x53, 0x3a, 0x52, 0x12, 0xb0, + 0xe5, 0x88, 0xcb, 0x64, 0xbb, 0x4e, 0x52, 0x15, 0x89, 0x6f, 0x71, 0xdc, 0x43, 0x55, 0xb1, 0x14, + 0xdd, 0x90, 0x1b, 0xda, 0x3b, 0x72, 0x23, 0x29, 0x78, 0x05, 0xcb, 0x01, 0x7a, 0x06, 0x4d, 0x4a, + 0x18, 0xa1, 0xa7, 0xc4, 0x31, 0xc3, 0x80, 0x46, 0x89, 0xfc, 0x9b, 0xf3, 0xc9, 0x7f, 0x10, 0xd0, + 0x08, 0xaf, 0x26, 0x48, 0x7c, 0xc4, 0xd0, 0x53, 0x58, 0x75, 0xce, 0x7d, 0xcb, 0x73, 0x6d, 0x85, + 0x5c, 0x5d, 0x18, 0x79, 0x45, 0x01, 0x09, 0x60, 0x7e, 0x51, 0xca, 0x4c, 0xf2, 0x3f, 0x36, 0xb1, + 0x8e, 0xc8, 0x44, 0xe9, 0x44, 0x0e, 0x66, 0xfd, 0xba, 0xa2, 0xfc, 0x5a, 0xff, 0x77, 0x01, 0x9a, + 0xb3, 0xee, 0x82, 0xfe, 0x17, 0xc0, 0x0e, 0x63, 0x33, 0x24, 0xd4, 0x0d, 0x1c, 0xe5, 0x14, 0x75, + 0x3b, 0x8c, 0x0f, 0x04, 0x81, 0x1f, 0x4e, 0x3e, 0xfd, 0x65, 0x1c, 0x44, 0x96, 0xf2, 0x8e, 0x9a, + 0x1d, 0xc6, 0xdf, 0xe4, 0xe3, 0x84, 0x57, 0xdc, 0xe4, 0x99, 0xf2, 0x12, 0xbe, 0x7c, 0x20, 0x08, + 0xe8, 0x2d, 0x40, 0xd2, 0x91, 0xcc, 0x89, 0xeb, 0xb9, 0x91, 0x79, 0x74, 0xce, 0xef, 0xc4, 0xd2, + 0x71, 0x34, 0x39, 0xb3, 0xcb, 0x27, 0x3e, 0xe6, 0x74, 0xa4, 0xc3, 0x6a, 0x10, 0x78, 0x26, 0xb3, + 0x03, 0x4a, 0x4c, 0xcb, 0xf9, 0x42, 0x44, 0xbf, 0x12, 0x6e, 0x04, 0x81, 0x37, 0xe0, 0xb4, 0xae, + 0xf3, 0x05, 0x3f, 0xec, 0x76, 0x18, 0x33, 0x12, 0x99, 0xfc, 0xa7, 0x55, 0x95, 0x87, 0x5d, 0x92, + 0x7a, 0x61, 0xcc, 0x32, 0x0b, 0x3c, 0xe2, 0xf1, 0x60, 0x95, 0x59, 0xb0, 0x47, 0x3c, 0xa6, 0x7f, + 0x06, 0x15, 0x11, 0xda, 0xf8, 0x1f, 0x13, 0xd9, 0x5f, 0x44, 0x0d, 0xa9, 0xba, 0x1a, 0x27, 0x88, + 0xa0, 0xf2, 0x0a, 0xd4, 0xc7, 0x01, 0x53, 0x31, 0x47, 0x7a, 0x55, 0x8d, 0x13, 0xc4, 0x64, 0x1b, + 0x6a, 0x94, 0x58, 0x4e, 0xe0, 0x4f, 0xce, 0xc5, 0x7f, 0xae, 0xe1, 0x74, 0xac, 0x7f, 0x09, 0x55, + 0x19, 0xf2, 0xae, 0x81, 0x7f, 0x17, 0x90, 0x3d, 0xa2, 0x41, 0x1c, 0x72, 0xa3, 0x78, 0x2e, 0x63, + 0x6e, 0xe0, 0xb3, 0xe4, 0xda, 0x2d, 0x67, 0x0e, 0xa6, 0x13, 0xfa, 0x5f, 0x0a, 0x32, 0xcd, 0xc9, + 0x3b, 0x14, 0x2f, 0x14, 0x54, 0xce, 0x5a, 0xf8, 0xa2, 0xa9, 0x00, 0x92, 0x62, 0x8f, 0xa8, 0xe7, + 0x85, 0x79, 0x8b, 0x3d, 0x22, 0x8b, 0x3d, 0xc2, 0x2b, 0x0b, 0x95, 0x4d, 0x25, 0x9c, 0x4c, 0xa6, + 0x0d, 0x27, 0xad, 0x82, 0x89, 0xfe, 0xcf, 0x42, 0x7a, 0xda, 0x93, 0x6a, 0x15, 0x7d, 0x0e, 0x35, + 0x7e, 0x70, 0x4c, 0xcf, 0x0a, 0xd5, 0x43, 0x4a, 0x6f, 0xb1, 0x42, 0xb8, 0xc3, 0xcf, 0xc9, 0x9e, + 0x15, 0xca, 0x34, 0xbb, 0x1c, 0xca, 0x11, 0x8f, 0x1a, 0x96, 0x33, 0x8d, 0x1a, 0xfc, 0x1b, 0xbd, + 0x0e, 0x4d, 0x2b, 0x8e, 0x02, 0xd3, 0x72, 0x4e, 0x09, 0x8d, 0x5c, 0x46, 0x94, 0x85, 0x57, 0x39, + 0xb5, 0x9b, 0x10, 0xdb, 0x0f, 0x61, 0x25, 0x8b, 0xf9, 0x75, 0x79, 0xa5, 0x92, 0xcd, 0x2b, 0xdf, + 0x05, 0x98, 0x16, 0x65, 0xdc, 0x13, 0xc8, 0x99, 0x1b, 0x99, 0x76, 0xe0, 0xc8, 0xa8, 0x56, 0xc1, + 0x35, 0x4e, 0xe8, 0x05, 0x0e, 0xb9, 0x50, 0xe2, 0x56, 0x92, 0x12, 0x97, 0x9f, 0x3b, 0x7e, 0x54, + 0x4e, 0xdc, 0xc9, 0x84, 0x38, 0x4a, 0xc2, 0x7a, 0x10, 0x78, 0x4f, 0x04, 0x41, 0xff, 0x73, 0x51, + 0x7a, 0x84, 0xbc, 0x60, 0xe4, 0x2a, 0x7c, 0x52, 0x53, 0x97, 0xae, 0x67, 0xea, 0x07, 0x00, 0x2c, + 0xb2, 0x68, 0x44, 0x1c, 0xd3, 0x8a, 0xd4, 0x9d, 0xbd, 0x7d, 0xa9, 0x46, 0x1e, 0x26, 0x6f, 0x95, + 0xb8, 0xae, 0x56, 0x77, 0x23, 0xf4, 0x3e, 0xac, 0xd8, 0x81, 0x17, 0x4e, 0x88, 0x62, 0xae, 0x7c, + 0x2d, 0x73, 0x23, 0x5d, 0xdf, 0x8d, 0x32, 0x05, 0x72, 0xf5, 0xba, 0x05, 0xf2, 0x1f, 0x0a, 0xf2, + 0x9e, 0x94, 0xbd, 0xa6, 0xa1, 0xd1, 0x15, 0x0f, 0x7b, 0x8f, 0x17, 0xbc, 0xf3, 0x7d, 0xd5, 0xab, + 0xde, 0x75, 0x9f, 0xd1, 0xfe, 0x58, 0x82, 0x7a, 0x7a, 0xdd, 0xba, 0x64, 0xfb, 0xfb, 0x50, 0x4f, + 0x1f, 0x8a, 0x55, 0x99, 0xf1, 0x95, 0xe6, 0x49, 0x17, 0xa3, 0x63, 0x40, 0xd6, 0x68, 0x94, 0x16, + 0x29, 0x66, 0xcc, 0xac, 0x51, 0x72, 0x41, 0xbd, 0x3f, 0x87, 0x1e, 0x92, 0xcc, 0x73, 0xc8, 0xf9, + 0xb1, 0x66, 0x8d, 0x46, 0x33, 0x14, 0xf4, 0x3d, 0xb8, 0x3d, 0xbb, 0x87, 0x79, 0x74, 0x6e, 0x86, + 0xae, 0xa3, 0x0a, 0xec, 0xed, 0x79, 0x6f, 0x9c, 0x9d, 0x19, 0xf8, 0x8f, 0xcf, 0x0f, 0x5c, 0x47, + 0xea, 0x1c, 0xd1, 0x4b, 0x13, 0xed, 0x1f, 0xc0, 0x4b, 0x2f, 0x58, 0x7e, 0x85, 0x0d, 0xfa, 0x59, + 0x1b, 0x5c, 0x47, 0x09, 0x19, 0xeb, 0xfd, 0xa6, 0x20, 0x2f, 0xc6, 0xb3, 0x3a, 0xe9, 0x4e, 0x2b, + 0xb6, 0xc6, 0xe6, 0x46, 0xce, 0x7d, 0x7a, 0x07, 0x87, 0x12, 0x5e, 0x94, 0x78, 0x9f, 0xcc, 0x94, + 0x78, 0xf9, 0xcb, 0x90, 0x3d, 0xc1, 0x24, 0x81, 0x14, 0x82, 0xfe, 0xbb, 0x12, 0xd4, 0x12, 0x74, + 0x51, 0x7f, 0x9f, 0xb3, 0x88, 0x78, 0xa6, 0x97, 0x84, 0xb0, 0x02, 0x06, 0x49, 0xda, 0xe3, 0x41, + 0xec, 0x15, 0xa8, 0xf3, 0x32, 0x5f, 0x4e, 0x17, 0xc5, 0x74, 0x8d, 0x13, 0xc4, 0xe4, 0xab, 0xd0, + 0x88, 0x82, 0xc8, 0x9a, 0x98, 0x91, 0x6b, 0x9f, 0xc8, 0x24, 0x57, 0xc0, 0x20, 0x48, 0x43, 0x4e, + 0x41, 0x6f, 0xc2, 0x8d, 0x68, 0x4c, 0x83, 0x28, 0x9a, 0xf0, 0x0a, 0x4d, 0xd4, 0x24, 0xb2, 0x84, + 0x28, 0x63, 0x2d, 0x9d, 0x90, 0xb5, 0x0a, 0xe3, 0xd1, 0x7b, 0xba, 0x98, 0xbb, 0xae, 0x08, 0x22, + 0x65, 0xbc, 0x9a, 0x52, 0xb9, 0x6b, 0xf3, 0xfb, 0x7d, 0x48, 0xa8, 0x4d, 0x7c, 0x19, 0x2b, 0x0a, + 0x38, 0x19, 0x22, 0x13, 0xd6, 0x3c, 0x62, 0xb1, 0x98, 0x12, 0xc7, 0x3c, 0x76, 0xc9, 0xc4, 0x91, + 0xf7, 0x9d, 0x66, 0xee, 0x7a, 0x36, 0x51, 0x4b, 0xe7, 0x91, 0xe0, 0xc6, 0xcd, 0x04, 0x4e, 0x8e, + 0x79, 0x7d, 0x20, 0xbf, 0xd0, 0x1a, 0x34, 0x06, 0xcf, 0x06, 0x43, 0x63, 0xcf, 0xdc, 0xdb, 0xdf, + 0x32, 0xd4, 0x03, 0xf7, 0xc0, 0xc0, 0x72, 0x58, 0xe0, 0xf3, 0xc3, 0xfd, 0x61, 0x77, 0xd7, 0x1c, + 0xee, 0xf4, 0x9e, 0x0c, 0xb4, 0x22, 0xba, 0x0d, 0x37, 0x86, 0xdb, 0x78, 0x7f, 0x38, 0xdc, 0x35, + 0xb6, 0xcc, 0x03, 0x03, 0xef, 0xec, 0x6f, 0x0d, 0xb4, 0x12, 0x42, 0xd0, 0x9c, 0x92, 0x87, 0x3b, + 0x7b, 0x86, 0x56, 0x46, 0x0d, 0x58, 0x3e, 0x30, 0x70, 0xcf, 0xe8, 0x0f, 0xb5, 0x8a, 0xfe, 0xb7, + 0x22, 0x34, 0x32, 0x56, 0xe4, 0x8e, 0x4c, 0x99, 0xbc, 0xbf, 0x94, 0x31, 0xff, 0xe4, 0xc1, 0xc4, + 0xb6, 0xec, 0xb1, 0xb4, 0x4e, 0x19, 0xcb, 0x01, 0xb7, 0x9b, 0x67, 0x9d, 0x65, 0xce, 0x79, 0x19, + 0xd7, 0x3c, 0xeb, 0x4c, 0x82, 0xbc, 0x06, 0x2b, 0x27, 0x84, 0xfa, 0x64, 0xa2, 0xe6, 0xa5, 0x45, + 0x1a, 0x92, 0x26, 0x97, 0xac, 0x83, 0xa6, 0x96, 0x4c, 0x61, 0xa4, 0x39, 0x9a, 0x92, 0xbe, 0x97, + 0x80, 0x1d, 0x5d, 0xd6, 0x7a, 0x55, 0x68, 0xfd, 0xc1, 0xfc, 0x4e, 0xfa, 0x22, 0xc5, 0x0f, 0x52, + 0xc5, 0x2f, 0x43, 0x09, 0x27, 0x6f, 0xbd, 0xbd, 0x6e, 0x6f, 0x9b, 0x2b, 0x7b, 0x15, 0xea, 0x7b, + 0xdd, 0x4f, 0xcd, 0xc3, 0x81, 0x78, 0x39, 0x40, 0x1a, 0xac, 0x3c, 0x31, 0x70, 0xdf, 0xd8, 0x55, + 0x94, 0x12, 0xba, 0x05, 0x9a, 0xa2, 0x4c, 0xd7, 0x95, 0xf5, 0xdf, 0x16, 0x61, 0x4d, 0xc6, 0xf5, + 0xf4, 0x31, 0xeb, 0xc5, 0xaf, 0x4a, 0x8b, 0x87, 0xde, 0x16, 0x2c, 0x7b, 0x84, 0xa5, 0x76, 0xa8, + 0xe3, 0x64, 0x88, 0x5c, 0x68, 0x58, 0xbe, 0x1f, 0x44, 0xe2, 0x45, 0x84, 0xa9, 0x10, 0xf9, 0x78, + 0xae, 0xb7, 0x97, 0x54, 0xf2, 0x4e, 0x77, 0x8a, 0x24, 0x23, 0x64, 0x16, 0xbb, 0xfd, 0x01, 0x68, + 0x17, 0x17, 0xcc, 0x93, 0x97, 0xde, 0x78, 0x67, 0x9a, 0x96, 0x08, 0x77, 0xd0, 0xc3, 0xfe, 0x93, + 0xfe, 0xfe, 0xd3, 0xbe, 0xb6, 0xc4, 0x07, 0xf8, 0xb0, 0xdf, 0xdf, 0xe9, 0x3f, 0xd6, 0x0a, 0x08, + 0xa0, 0x6a, 0x7c, 0xba, 0x33, 0x34, 0xb6, 0xb4, 0xe2, 0xe6, 0xdf, 0x57, 0xa1, 0x2a, 0x85, 0x44, + 0xbf, 0x52, 0x29, 0x39, 0xdb, 0x71, 0x44, 0x1f, 0xcc, 0x5d, 0xda, 0xce, 0x74, 0x31, 0xdb, 0x1f, + 0x2e, 0xcc, 0xaf, 0x1e, 0x15, 0x97, 0xd0, 0xcf, 0x0a, 0xb0, 0x32, 0xf3, 0x8a, 0x96, 0xf7, 0x01, + 0xe8, 0x8a, 0x06, 0x67, 0xfb, 0x1b, 0x0b, 0xf1, 0xa6, 0xb2, 0xfc, 0xb4, 0x00, 0x8d, 0x4c, 0x6b, + 0x0f, 0x3d, 0x58, 0xa4, 0x1d, 0x28, 0x25, 0x79, 0xb8, 0x78, 0x27, 0x51, 0x5f, 0x7a, 0xbb, 0x80, + 0x7e, 0x52, 0x80, 0x46, 0xa6, 0x2f, 0x96, 0x5b, 0x94, 0xcb, 0x5d, 0xbc, 0xdc, 0xa2, 0x5c, 0xd5, + 0x86, 0x5b, 0x42, 0x3f, 0x2c, 0x40, 0x3d, 0xed, 0x71, 0xa1, 0x7b, 0xf3, 0x77, 0xc5, 0xa4, 0x10, + 0xf7, 0x17, 0x6d, 0xa7, 0xe9, 0x4b, 0xe8, 0xfb, 0x50, 0x4b, 0x1a, 0x42, 0x28, 0x6f, 0x1a, 0xb9, + 0xd0, 0x6d, 0x6a, 0xdf, 0x9b, 0x9b, 0x2f, 0xbb, 0x7d, 0xd2, 0xa5, 0xc9, 0xbd, 0xfd, 0x85, 0x7e, + 0x52, 0xfb, 0xde, 0xdc, 0x7c, 0xe9, 0xf6, 0xdc, 0x13, 0x32, 0xcd, 0x9c, 0xdc, 0x9e, 0x70, 0xb9, + 0x8b, 0x94, 0xdb, 0x13, 0xae, 0xea, 0x1d, 0x49, 0x41, 0x32, 0xed, 0xa0, 0xdc, 0x82, 0x5c, 0x6e, + 0x39, 0xe5, 0x16, 0xe4, 0x8a, 0xee, 0x93, 0x72, 0xc9, 0x69, 0x81, 0x7e, 0x6f, 0xee, 0x0e, 0xca, + 0x9c, 0x2e, 0x79, 0xa9, 0x87, 0xa3, 0x2f, 0xa1, 0x1f, 0xa9, 0x27, 0x03, 0xd9, 0x7e, 0x41, 0xf3, + 0x40, 0xcd, 0x74, 0x6c, 0xda, 0xef, 0x2d, 0x96, 0x6a, 0x44, 0x8c, 0xf8, 0x71, 0x01, 0x60, 0xda, + 0xa8, 0xc9, 0x2d, 0xc4, 0xa5, 0x0e, 0x51, 0xfb, 0xc1, 0x02, 0x9c, 0xd9, 0xe3, 0x91, 0xf4, 0x66, + 0x72, 0x1f, 0x8f, 0x0b, 0x8d, 0xa4, 0xdc, 0xc7, 0xe3, 0x62, 0x13, 0x48, 0x5f, 0xfa, 0x78, 0xf9, + 0x5b, 0x15, 0x99, 0xfb, 0xab, 0xe2, 0xe7, 0xdd, 0xff, 0x04, 0x00, 0x00, 0xff, 0xff, 0xb2, 0x79, + 0xa9, 0xc4, 0x43, 0x24, 0x00, 0x00, } diff --git a/plugins/shared/cmd/launcher/command/device.go b/plugins/shared/cmd/launcher/command/device.go index fd99582fc..aadc215eb 100644 --- a/plugins/shared/cmd/launcher/command/device.go +++ b/plugins/shared/cmd/launcher/command/device.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "time" hclog "github.com/hashicorp/go-hclog" plugin "github.com/hashicorp/go-plugin" @@ -345,7 +346,7 @@ func (c *Device) replOutput(ctx context.Context, startFingerprint, startStats <- c.Ui.Output(fmt.Sprintf("> fingerprint: % #v", pretty.Formatter(resp))) case ctx := <-startStats: var err error - stats, err = c.dev.Stats(ctx) + stats, err = c.dev.Stats(ctx, 1*time.Second) if err != nil { c.Ui.Error(fmt.Sprintf("stats: %s", err)) os.Exit(1) diff --git a/plugins/shared/hclspec/hcl_spec.pb.go b/plugins/shared/hclspec/hcl_spec.pb.go index 0b455dbdf..e02b7ac7b 100644 --- a/plugins/shared/hclspec/hcl_spec.pb.go +++ b/plugins/shared/hclspec/hcl_spec.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: hcl_spec.proto +// source: plugins/shared/hclspec/hcl_spec.proto package hclspec @@ -94,7 +94,7 @@ func (m *Spec) Reset() { *m = Spec{} } func (m *Spec) String() string { return proto.CompactTextString(m) } func (*Spec) ProtoMessage() {} func (*Spec) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{0} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{0} } func (m *Spec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Spec.Unmarshal(m, b) @@ -522,7 +522,7 @@ func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{1} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{1} } func (m *Attr) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attr.Unmarshal(m, b) @@ -611,7 +611,7 @@ func (m *Block) Reset() { *m = Block{} } func (m *Block) String() string { return proto.CompactTextString(m) } func (*Block) ProtoMessage() {} func (*Block) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{2} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{2} } func (m *Block) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Block.Unmarshal(m, b) @@ -697,7 +697,7 @@ func (m *BlockAttrs) Reset() { *m = BlockAttrs{} } func (m *BlockAttrs) String() string { return proto.CompactTextString(m) } func (*BlockAttrs) ProtoMessage() {} func (*BlockAttrs) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{3} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{3} } func (m *BlockAttrs) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BlockAttrs.Unmarshal(m, b) @@ -792,7 +792,7 @@ func (m *BlockList) Reset() { *m = BlockList{} } func (m *BlockList) String() string { return proto.CompactTextString(m) } func (*BlockList) ProtoMessage() {} func (*BlockList) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{4} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{4} } func (m *BlockList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BlockList.Unmarshal(m, b) @@ -875,7 +875,7 @@ func (m *BlockSet) Reset() { *m = BlockSet{} } func (m *BlockSet) String() string { return proto.CompactTextString(m) } func (*BlockSet) ProtoMessage() {} func (*BlockSet) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{5} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{5} } func (m *BlockSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BlockSet.Unmarshal(m, b) @@ -974,7 +974,7 @@ func (m *BlockMap) Reset() { *m = BlockMap{} } func (m *BlockMap) String() string { return proto.CompactTextString(m) } func (*BlockMap) ProtoMessage() {} func (*BlockMap) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{6} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{6} } func (m *BlockMap) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BlockMap.Unmarshal(m, b) @@ -1044,7 +1044,7 @@ func (m *Literal) Reset() { *m = Literal{} } func (m *Literal) String() string { return proto.CompactTextString(m) } func (*Literal) ProtoMessage() {} func (*Literal) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{7} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{7} } func (m *Literal) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Literal.Unmarshal(m, b) @@ -1108,7 +1108,7 @@ func (m *Default) Reset() { *m = Default{} } func (m *Default) String() string { return proto.CompactTextString(m) } func (*Default) ProtoMessage() {} func (*Default) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{8} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{8} } func (m *Default) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Default.Unmarshal(m, b) @@ -1182,7 +1182,7 @@ func (m *Object) Reset() { *m = Object{} } func (m *Object) String() string { return proto.CompactTextString(m) } func (*Object) ProtoMessage() {} func (*Object) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{9} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{9} } func (m *Object) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Object.Unmarshal(m, b) @@ -1238,7 +1238,7 @@ func (m *Array) Reset() { *m = Array{} } func (m *Array) String() string { return proto.CompactTextString(m) } func (*Array) ProtoMessage() {} func (*Array) Descriptor() ([]byte, []int) { - return fileDescriptor_hcl_spec_b17a3e0d58741859, []int{10} + return fileDescriptor_hcl_spec_45ead239ae3df7c4, []int{10} } func (m *Array) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Array.Unmarshal(m, b) @@ -1280,47 +1280,49 @@ func init() { proto.RegisterType((*Array)(nil), "hashicorp.nomad.plugins.shared.hclspec.Array") } -func init() { proto.RegisterFile("hcl_spec.proto", fileDescriptor_hcl_spec_b17a3e0d58741859) } - -var fileDescriptor_hcl_spec_b17a3e0d58741859 = []byte{ - // 617 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x96, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xc7, 0xe3, 0xc4, 0x9f, 0x53, 0x09, 0xd0, 0x0a, 0x21, 0xab, 0x1c, 0xa8, 0x7c, 0x40, 0x3d, - 0x80, 0x81, 0x72, 0x41, 0x1c, 0x90, 0x1a, 0xb5, 0xc8, 0x40, 0xa3, 0x56, 0x5b, 0xc1, 0x81, 0x03, - 0xd1, 0xda, 0x59, 0x88, 0x89, 0xbf, 0xd8, 0xdd, 0xa0, 0x46, 0x82, 0x07, 0xe1, 0x00, 0x3c, 0x15, - 0xef, 0x83, 0xf6, 0xc3, 0x49, 0x41, 0x39, 0xc4, 0xa1, 0x07, 0x6e, 0x3b, 0x3b, 0xf9, 0xff, 0x3c, - 0xb3, 0x3b, 0xb3, 0x13, 0xb8, 0x36, 0xcd, 0x8a, 0x31, 0x6f, 0x68, 0x16, 0x37, 0xac, 0x16, 0x35, - 0xba, 0x3b, 0x25, 0x7c, 0x9a, 0x67, 0x35, 0x6b, 0xe2, 0xaa, 0x2e, 0xc9, 0x24, 0x6e, 0x8a, 0xf9, - 0x87, 0xbc, 0xe2, 0x31, 0x9f, 0x12, 0x46, 0x27, 0xf1, 0x34, 0x2b, 0xe4, 0xaf, 0xa3, 0xef, 0x2e, - 0xd8, 0xe7, 0x0d, 0xcd, 0x50, 0x02, 0x6e, 0x9d, 0x7e, 0xa4, 0x99, 0x08, 0xad, 0x3d, 0x6b, 0x7f, - 0xe7, 0x20, 0x8e, 0x37, 0x23, 0xc4, 0xa7, 0x4a, 0x95, 0xf4, 0xb0, 0xd1, 0xa3, 0x63, 0x70, 0x08, - 0x63, 0x64, 0x11, 0xf6, 0x15, 0xe8, 0xfe, 0xa6, 0xa0, 0x43, 0x29, 0x4a, 0x7a, 0x58, 0xab, 0xd1, - 0x10, 0xec, 0x43, 0x21, 0x58, 0x38, 0x50, 0x94, 0x7b, 0x1b, 0x53, 0x84, 0x60, 0x49, 0x0f, 0x2b, - 0x2d, 0x3a, 0x83, 0x9d, 0xb4, 0xa8, 0xb3, 0xd9, 0xf8, 0x33, 0x29, 0xe6, 0x34, 0xb4, 0xbb, 0x05, - 0x34, 0x94, 0xd2, 0xa4, 0x87, 0x41, 0x31, 0xde, 0x48, 0x04, 0x7a, 0xdd, 0x12, 0x89, 0x10, 0x8c, - 0x87, 0x8e, 0x22, 0x1e, 0x74, 0x22, 0xca, 0xc8, 0xf8, 0x12, 0xab, 0x2c, 0x84, 0x41, 0x5b, 0xe3, - 0x22, 0xe7, 0x22, 0x74, 0x15, 0xf5, 0x51, 0x27, 0xea, 0x49, 0xce, 0xe5, 0x25, 0x04, 0x69, 0x6b, - 0xa0, 0x53, 0xd0, 0xc6, 0x98, 0x53, 0x11, 0x7a, 0x0a, 0xf9, 0xb0, 0x13, 0xf2, 0x9c, 0x4a, 0xa2, - 0x9f, 0x9a, 0xf5, 0x0a, 0x58, 0x92, 0x26, 0xf4, 0xb7, 0x00, 0x8e, 0x48, 0xb3, 0x04, 0x8e, 0x48, - 0x83, 0x5e, 0x81, 0x37, 0xa1, 0xef, 0xc9, 0xbc, 0x10, 0x61, 0xa0, 0x70, 0x0f, 0x36, 0xc5, 0x1d, - 0x69, 0x59, 0xd2, 0xc3, 0x2d, 0x41, 0xc2, 0x8a, 0x5c, 0x50, 0x46, 0x8a, 0x10, 0xba, 0xc1, 0x4e, - 0xb4, 0x4c, 0xc2, 0x0c, 0x61, 0xe8, 0x81, 0xa3, 0xa2, 0x8c, 0x5e, 0xea, 0x2a, 0x44, 0x08, 0xec, - 0x8a, 0x94, 0x54, 0x35, 0x47, 0x80, 0xd5, 0x5a, 0xee, 0x89, 0x45, 0x43, 0x55, 0x9d, 0x07, 0x58, - 0xad, 0xd1, 0x2e, 0xf8, 0x8c, 0x7e, 0x9a, 0xe7, 0x8c, 0x4e, 0x54, 0xe5, 0xfa, 0x78, 0x69, 0x47, - 0x5f, 0xc1, 0x51, 0xc7, 0xb0, 0x16, 0x76, 0x59, 0xd8, 0xff, 0x53, 0x88, 0x8e, 0xc0, 0xad, 0x28, - 0x17, 0x06, 0xd9, 0xa1, 0x19, 0x64, 0x67, 0x63, 0xa3, 0x8d, 0xce, 0x00, 0x56, 0xf5, 0x77, 0x25, - 0x09, 0xfd, 0xb4, 0x20, 0x58, 0x16, 0xdf, 0x5a, 0xe2, 0x6d, 0x08, 0xca, 0xbc, 0x1a, 0xe7, 0x82, - 0x96, 0x5c, 0x61, 0x6d, 0xec, 0x97, 0x79, 0xf5, 0x42, 0xda, 0xca, 0x49, 0x2e, 0x8c, 0x73, 0x60, - 0x9c, 0xe4, 0x42, 0x3b, 0x57, 0x39, 0xdb, 0xff, 0x90, 0xf3, 0x0f, 0x0b, 0xfc, 0xb6, 0x96, 0xff, - 0xcb, 0x00, 0xbf, 0x98, 0xf8, 0x64, 0x3b, 0xac, 0x8b, 0xef, 0x16, 0xb8, 0x05, 0x49, 0x69, 0x21, - 0x83, 0x1b, 0xec, 0x07, 0xd8, 0x58, 0x57, 0x54, 0x12, 0x77, 0xc0, 0x33, 0xc5, 0x8f, 0x6e, 0x82, - 0xa3, 0x1f, 0x49, 0xfd, 0x75, 0x6d, 0x44, 0xdf, 0x2c, 0xf0, 0x4c, 0xaf, 0xa1, 0xe7, 0xe0, 0x35, - 0x2c, 0x2f, 0x09, 0x5b, 0x98, 0x11, 0xd1, 0xed, 0x9b, 0xad, 0x58, 0x72, 0xda, 0xae, 0xef, 0x6f, - 0xc3, 0x31, 0xe2, 0xe8, 0x97, 0x05, 0xae, 0x1e, 0x3e, 0xe8, 0x1d, 0x80, 0x7c, 0x8f, 0xf3, 0x74, - 0x2e, 0x28, 0x0f, 0xad, 0xbd, 0xc1, 0xfe, 0xce, 0xc1, 0xb3, 0x6e, 0x03, 0x4c, 0x0d, 0x0e, 0x0d, - 0x38, 0xae, 0x04, 0x5b, 0xe0, 0x4b, 0xc4, 0xdd, 0x19, 0x5c, 0xff, 0xcb, 0x8d, 0x6e, 0xc0, 0x60, - 0x46, 0x17, 0xe6, 0xb4, 0xe4, 0x12, 0x0d, 0xdb, 0x13, 0xdc, 0x26, 0x2b, 0x2d, 0x7d, 0xda, 0x7f, - 0x62, 0x45, 0x23, 0x70, 0xd4, 0x28, 0x94, 0x77, 0xac, 0x76, 0xdb, 0x8c, 0x3a, 0xde, 0xb1, 0xd6, - 0x0e, 0x83, 0xb7, 0x9e, 0xd9, 0x4f, 0x5d, 0xf5, 0xdf, 0xe0, 0xf1, 0xef, 0x00, 0x00, 0x00, 0xff, - 0xff, 0x18, 0x16, 0x39, 0xa4, 0x2d, 0x08, 0x00, 0x00, +func init() { + proto.RegisterFile("plugins/shared/hclspec/hcl_spec.proto", fileDescriptor_hcl_spec_45ead239ae3df7c4) +} + +var fileDescriptor_hcl_spec_45ead239ae3df7c4 = []byte{ + // 624 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x96, 0x4d, 0x6f, 0xd3, 0x4c, + 0x10, 0xc7, 0xe3, 0xc4, 0xaf, 0xd3, 0xc3, 0xf3, 0x68, 0x85, 0x90, 0x55, 0x0e, 0x54, 0x96, 0x40, + 0x3d, 0x80, 0x0b, 0xe5, 0x82, 0x38, 0x20, 0x35, 0x6a, 0x91, 0x81, 0x46, 0xad, 0xb6, 0x82, 0x03, + 0x07, 0xa2, 0xb5, 0xb3, 0x10, 0x13, 0xbf, 0xb1, 0xbb, 0x41, 0x8d, 0x04, 0x1f, 0x84, 0x03, 0xf0, + 0xa9, 0xf8, 0x3e, 0x68, 0x5f, 0x9c, 0x14, 0x94, 0x43, 0x1c, 0x7a, 0xe0, 0x94, 0x19, 0x8f, 0xfe, + 0x3f, 0xcf, 0xec, 0xce, 0x78, 0x02, 0x77, 0x9a, 0x62, 0xfe, 0x3e, 0xaf, 0xf8, 0x01, 0x9f, 0x12, + 0x46, 0x27, 0x07, 0xd3, 0xac, 0xe0, 0x0d, 0xcd, 0xe4, 0xef, 0x58, 0x1a, 0x71, 0xc3, 0x6a, 0x51, + 0xa3, 0xbb, 0x53, 0xc2, 0xa7, 0x79, 0x56, 0xb3, 0x26, 0xae, 0xea, 0x92, 0x4c, 0x62, 0x23, 0x8b, + 0xb5, 0x2c, 0x36, 0xb2, 0xe8, 0x9b, 0x0b, 0xf6, 0x45, 0x43, 0x33, 0x94, 0x80, 0x5b, 0xa7, 0x1f, + 0x68, 0x26, 0x42, 0x6b, 0xcf, 0xda, 0xdf, 0x39, 0x8c, 0xe3, 0xcd, 0x08, 0xf1, 0x99, 0x52, 0x25, + 0x3d, 0x6c, 0xf4, 0xe8, 0x04, 0x1c, 0xc2, 0x18, 0x59, 0x84, 0x7d, 0x05, 0xba, 0xbf, 0x29, 0xe8, + 0x48, 0x8a, 0x92, 0x1e, 0xd6, 0x6a, 0x34, 0x04, 0xfb, 0x48, 0x08, 0x16, 0x0e, 0x14, 0xe5, 0xde, + 0xc6, 0x14, 0x21, 0x58, 0xd2, 0xc3, 0x4a, 0x8b, 0xce, 0x61, 0x27, 0x2d, 0xea, 0x6c, 0x36, 0xfe, + 0x44, 0x8a, 0x39, 0x0d, 0xed, 0x6e, 0x09, 0x0d, 0xa5, 0x34, 0xe9, 0x61, 0x50, 0x8c, 0xd7, 0x12, + 0x81, 0x5e, 0xb5, 0x44, 0x22, 0x04, 0xe3, 0xa1, 0xa3, 0x88, 0x87, 0x9d, 0x88, 0x32, 0x33, 0xbe, + 0xc4, 0x2a, 0x0f, 0x61, 0xd0, 0xde, 0xb8, 0xc8, 0xb9, 0x08, 0x5d, 0x45, 0x7d, 0xd8, 0x89, 0x7a, + 0x9a, 0x73, 0x79, 0x09, 0x41, 0xda, 0x3a, 0xe8, 0x0c, 0xb4, 0x33, 0xe6, 0x54, 0x84, 0x9e, 0x42, + 0x3e, 0xe8, 0x84, 0xbc, 0xa0, 0x92, 0xe8, 0xa7, 0xc6, 0x5e, 0x01, 0x4b, 0xd2, 0x84, 0xfe, 0x16, + 0xc0, 0x11, 0x69, 0x96, 0xc0, 0x11, 0x69, 0xd0, 0x4b, 0xf0, 0x26, 0xf4, 0x1d, 0x99, 0x17, 0x22, + 0x0c, 0x14, 0xee, 0x60, 0x53, 0xdc, 0xb1, 0x96, 0x25, 0x3d, 0xdc, 0x12, 0x24, 0xac, 0xc8, 0x05, + 0x65, 0xa4, 0x08, 0xa1, 0x1b, 0xec, 0x54, 0xcb, 0x24, 0xcc, 0x10, 0x86, 0x1e, 0x38, 0x2a, 0xcb, + 0xe8, 0x85, 0xee, 0x42, 0x84, 0xc0, 0xae, 0x48, 0x49, 0xd5, 0x70, 0x04, 0x58, 0xd9, 0xf2, 0x99, + 0x58, 0x34, 0x54, 0xf5, 0x79, 0x80, 0x95, 0x8d, 0x76, 0xc1, 0x67, 0xf4, 0xe3, 0x3c, 0x67, 0x74, + 0xa2, 0x3a, 0xd7, 0xc7, 0x4b, 0x3f, 0xfa, 0x02, 0x8e, 0x3a, 0x86, 0xb5, 0xb0, 0xab, 0xc2, 0xfe, + 0xef, 0x42, 0x74, 0x0c, 0x6e, 0x45, 0xb9, 0x30, 0xc8, 0x0e, 0xc3, 0x20, 0x27, 0x1b, 0x1b, 0x6d, + 0x74, 0x0e, 0xb0, 0xea, 0xbf, 0x6b, 0x29, 0xe8, 0x87, 0x05, 0xc1, 0xb2, 0xf9, 0xd6, 0x12, 0x6f, + 0x41, 0x50, 0xe6, 0xd5, 0x38, 0x17, 0xb4, 0xe4, 0x0a, 0x6b, 0x63, 0xbf, 0xcc, 0xab, 0xe7, 0xd2, + 0x57, 0x41, 0x72, 0x69, 0x82, 0x03, 0x13, 0x24, 0x97, 0x3a, 0xb8, 0xaa, 0xd9, 0xfe, 0x8b, 0x9a, + 0xbf, 0x5b, 0xe0, 0xb7, 0xbd, 0xfc, 0x4f, 0x26, 0xf8, 0xd9, 0xe4, 0x27, 0xc7, 0x61, 0x5d, 0x7e, + 0x37, 0xc1, 0x2d, 0x48, 0x4a, 0x0b, 0x99, 0xdc, 0x60, 0x3f, 0xc0, 0xc6, 0xbb, 0xa6, 0x96, 0xb8, + 0x0d, 0x9e, 0x69, 0x7e, 0x74, 0x03, 0x1c, 0xfd, 0x91, 0xd4, 0x6f, 0xd7, 0x4e, 0xf4, 0xd5, 0x02, + 0xcf, 0xcc, 0x1a, 0x7a, 0x06, 0x5e, 0xc3, 0xf2, 0x92, 0xb0, 0x85, 0x59, 0x11, 0xdd, 0xde, 0xd9, + 0x8a, 0x25, 0xa7, 0x9d, 0xfa, 0xfe, 0x36, 0x1c, 0x23, 0x8e, 0x7e, 0x5a, 0xe0, 0xea, 0xe5, 0x83, + 0xde, 0x02, 0xc8, 0xef, 0x71, 0x9e, 0xce, 0x05, 0xe5, 0xa1, 0xb5, 0x37, 0xd8, 0xdf, 0x39, 0x7c, + 0xda, 0x6d, 0x81, 0xa9, 0xc5, 0xa1, 0x01, 0x27, 0x95, 0x60, 0x0b, 0x7c, 0x85, 0xb8, 0x3b, 0x83, + 0xff, 0xfe, 0x08, 0xa3, 0xff, 0x61, 0x30, 0xa3, 0x0b, 0x73, 0x5a, 0xd2, 0x44, 0xc3, 0xf6, 0x04, + 0xb7, 0xa9, 0x4a, 0x4b, 0x9f, 0xf4, 0x1f, 0x5b, 0xd1, 0x08, 0x1c, 0xb5, 0x0a, 0xe5, 0x1d, 0xab, + 0xa7, 0x6d, 0x45, 0x1d, 0xef, 0x58, 0x6b, 0x87, 0xc1, 0x1b, 0xcf, 0x3c, 0x4f, 0x5d, 0xf5, 0xdf, + 0xe0, 0xd1, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x32, 0x20, 0x9f, 0xf2, 0x44, 0x08, 0x00, 0x00, } diff --git a/plugins/shared/loader/init.go b/plugins/shared/loader/init.go index f69d3c2e1..9013fb13d 100644 --- a/plugins/shared/loader/init.go +++ b/plugins/shared/loader/init.go @@ -375,9 +375,10 @@ func (l *PluginLoader) validePluginConfig(id PluginID, info *pluginInfo) error { return multierror.Prefix(&mErr, "failed converting config schema:") } - // If there is no config there is nothing to do + // If there is no config, initialize it to an empty map so we can still + // handle defaults if info.config == nil { - return nil + info.config = map[string]interface{}{} } // Parse the config using the spec diff --git a/plugins/shared/loader/loader.go b/plugins/shared/loader/loader.go index f15d353f2..760ef5d6f 100644 --- a/plugins/shared/loader/loader.go +++ b/plugins/shared/loader/loader.go @@ -54,6 +54,13 @@ func (id PluginID) String() string { return fmt.Sprintf("%q (%v)", id.Name, id.PluginType) } +func PluginInfoID(resp *base.PluginInfoResponse) PluginID { + return PluginID{ + Name: resp.Name, + PluginType: resp.Type, + } +} + // PluginLoaderConfig configures a plugin loader. type PluginLoaderConfig struct { // Logger is the logger used by the plugin loader diff --git a/plugins/shared/loader/plugin_test.go b/plugins/shared/loader/plugin_test.go index 9c16c22f2..ea9b129fd 100644 --- a/plugins/shared/loader/plugin_test.go +++ b/plugins/shared/loader/plugin_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "testing" + "time" log "github.com/hashicorp/go-hclog" plugin "github.com/hashicorp/go-plugin" @@ -165,6 +166,6 @@ func (m *mockPlugin) Reserve(deviceIDs []string) (*device.ContainerReservation, }, nil } -func (m *mockPlugin) Stats(ctx context.Context) (<-chan *device.StatsResponse, error) { +func (m *mockPlugin) Stats(ctx context.Context, interval time.Duration) (<-chan *device.StatsResponse, error) { return make(chan *device.StatsResponse), nil } diff --git a/plugins/drivers/utils/plugin_reattach_config.go b/plugins/shared/plugin_reattach_config.go similarity index 91% rename from plugins/drivers/utils/plugin_reattach_config.go rename to plugins/shared/plugin_reattach_config.go index 02008279f..1c4fd1e55 100644 --- a/plugins/drivers/utils/plugin_reattach_config.go +++ b/plugins/shared/plugin_reattach_config.go @@ -1,4 +1,4 @@ -package utils +package shared import ( "fmt" @@ -19,6 +19,10 @@ type ReattachConfig struct { // ReattachConfigToGoPlugin converts a ReattachConfig wrapper struct into a go // plugin ReattachConfig struct func ReattachConfigToGoPlugin(rc *ReattachConfig) (*plugin.ReattachConfig, error) { + if rc == nil { + return nil, fmt.Errorf("nil ReattachConfig cannot be converted") + } + plug := &plugin.ReattachConfig{ Protocol: plugin.Protocol(rc.Protocol), Pid: rc.Pid, @@ -53,6 +57,10 @@ func ReattachConfigToGoPlugin(rc *ReattachConfig) (*plugin.ReattachConfig, error // ReattachConfigFromGoPlugin converts a go plugin ReattachConfig into a // ReattachConfig wrapper struct func ReattachConfigFromGoPlugin(plug *plugin.ReattachConfig) *ReattachConfig { + if plug == nil { + return nil + } + rc := &ReattachConfig{ Protocol: string(plug.Protocol), Network: plug.Addr.Network(), diff --git a/plugins/shared/singleton/singleton.go b/plugins/shared/singleton/singleton.go index baf7f6244..100e69070 100644 --- a/plugins/shared/singleton/singleton.go +++ b/plugins/shared/singleton/singleton.go @@ -10,6 +10,13 @@ import ( "github.com/hashicorp/nomad/plugins/shared/loader" ) +var ( + // SingletonPluginExited is returned when the dispense is called and the + // existing plugin has exited. The caller should retry, and this will issue + // a new plugin instance. + SingletonPluginExited = fmt.Errorf("singleton plugin exited") +) + // SingletonLoader is used to only load a single external plugin at a time. type SingletonLoader struct { // Loader is the underlying plugin loader that we wrap to give a singleton @@ -87,7 +94,7 @@ func (s *SingletonLoader) getPlugin(reattach bool, name, pluginType string, logg if i.Exited() { s.clearFuture(id, f) - return nil, fmt.Errorf("plugin %q exited", id) + return nil, SingletonPluginExited } return i, nil diff --git a/plugins/shared/structs/proto/attribute.pb.go b/plugins/shared/structs/proto/attribute.pb.go index b0abdf024..fa61b0cc1 100644 --- a/plugins/shared/structs/proto/attribute.pb.go +++ b/plugins/shared/structs/proto/attribute.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: github.com/hashicorp/nomad/plugins/shared/structs/proto/attribute.proto +// source: plugins/shared/structs/proto/attribute.proto package proto @@ -38,7 +38,7 @@ func (m *Attribute) Reset() { *m = Attribute{} } func (m *Attribute) String() string { return proto.CompactTextString(m) } func (*Attribute) ProtoMessage() {} func (*Attribute) Descriptor() ([]byte, []int) { - return fileDescriptor_attribute_5e512eadce7864ec, []int{0} + return fileDescriptor_attribute_47573829d12e3945, []int{0} } func (m *Attribute) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attribute.Unmarshal(m, b) @@ -230,24 +230,23 @@ func init() { } func init() { - proto.RegisterFile("github.com/hashicorp/nomad/plugins/shared/structs/proto/attribute.proto", fileDescriptor_attribute_5e512eadce7864ec) + proto.RegisterFile("plugins/shared/structs/proto/attribute.proto", fileDescriptor_attribute_47573829d12e3945) } -var fileDescriptor_attribute_5e512eadce7864ec = []byte{ - // 229 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x3c, 0x8f, 0xb1, 0x4e, 0xc3, 0x30, - 0x10, 0x40, 0x63, 0xda, 0x34, 0xc9, 0x8d, 0x99, 0x8a, 0x10, 0x22, 0x62, 0x40, 0x99, 0xec, 0x81, - 0x2f, 0xa0, 0x0b, 0x99, 0x3d, 0x30, 0xb0, 0x20, 0x27, 0x0d, 0x89, 0x25, 0xd7, 0x17, 0xd9, 0xe7, - 0x7e, 0x0f, 0x9f, 0x8a, 0x72, 0x29, 0x4c, 0xf6, 0xbd, 0x77, 0x6f, 0x38, 0x78, 0x9f, 0x2c, 0xcd, - 0xa9, 0x97, 0x03, 0x5e, 0xd4, 0x6c, 0xe2, 0x6c, 0x07, 0x0c, 0x8b, 0xf2, 0x78, 0x31, 0x67, 0xb5, - 0xb8, 0x34, 0x59, 0x1f, 0x55, 0x9c, 0x4d, 0x18, 0xcf, 0x2a, 0x52, 0x48, 0x03, 0x45, 0xb5, 0x04, - 0x24, 0x54, 0x86, 0x28, 0xd8, 0x3e, 0xd1, 0x28, 0x79, 0xae, 0x5f, 0xfe, 0x6b, 0xc9, 0xb5, 0xbc, - 0xd5, 0x72, 0xab, 0xe5, 0xad, 0x7e, 0xfe, 0x11, 0x50, 0xbd, 0xfd, 0xb5, 0xf5, 0x23, 0x54, 0xdf, - 0x0e, 0x0d, 0x7d, 0x5d, 0x8d, 0x3b, 0x8a, 0x46, 0xb4, 0xa2, 0xcb, 0x74, 0xc9, 0xe8, 0xc3, 0xb8, - 0xfa, 0x1e, 0x0a, 0xeb, 0x37, 0x79, 0xd7, 0x88, 0x76, 0xd7, 0x65, 0xfa, 0x60, 0x3d, 0xab, 0x27, - 0x80, 0x48, 0xc1, 0xfa, 0x89, 0xed, 0xae, 0x11, 0x6d, 0xd5, 0x65, 0xba, 0xda, 0xd8, 0xba, 0xf0, - 0x00, 0x65, 0x8f, 0xe8, 0x58, 0xef, 0x1b, 0xd1, 0x96, 0x5d, 0xa6, 0x8b, 0x95, 0xac, 0xb2, 0x86, - 0x7d, 0xf2, 0x96, 0x8e, 0xf9, 0xda, 0x69, 0xfe, 0x9f, 0x0a, 0xc8, 0xaf, 0xc6, 0xa5, 0xf1, 0x54, - 0x7c, 0xe6, 0x7c, 0x53, 0x7f, 0xe0, 0xe7, 0xf5, 0x37, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x2e, 0x1a, - 0x5b, 0x25, 0x01, 0x00, 0x00, +var fileDescriptor_attribute_47573829d12e3945 = []byte{ + // 218 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8f, 0xb1, 0x4e, 0xc3, 0x30, + 0x10, 0x40, 0x63, 0xda, 0x34, 0xc9, 0x8d, 0x99, 0x8a, 0x10, 0x22, 0x62, 0x40, 0x19, 0x90, 0x33, + 0xf0, 0x05, 0x74, 0xf2, 0xec, 0x81, 0x81, 0x05, 0x5d, 0xda, 0xd0, 0x58, 0x32, 0x76, 0x64, 0x9f, + 0xfb, 0x3d, 0x7c, 0x2a, 0xf2, 0x25, 0x4c, 0xf6, 0xbd, 0x77, 0x6f, 0x38, 0x78, 0x5d, 0x6c, 0xba, + 0x1a, 0x17, 0x87, 0x38, 0x63, 0x98, 0x2e, 0x43, 0xa4, 0x90, 0xce, 0x14, 0x87, 0x25, 0x78, 0xf2, + 0x03, 0x12, 0x05, 0x33, 0x26, 0x9a, 0x24, 0xcf, 0xed, 0xcb, 0x8c, 0x71, 0x36, 0x67, 0x1f, 0x16, + 0xe9, 0xfc, 0x0f, 0x5e, 0xe4, 0x56, 0xcb, 0xb5, 0x96, 0x5b, 0xfd, 0xfc, 0x2b, 0xa0, 0x79, 0xff, + 0x6f, 0xdb, 0x47, 0x68, 0xbe, 0xad, 0x47, 0xfa, 0xba, 0xa1, 0x3d, 0x8a, 0x4e, 0xf4, 0x42, 0x15, + 0xba, 0x66, 0xf4, 0x81, 0xb6, 0xbd, 0x87, 0xca, 0xb8, 0x55, 0xde, 0x75, 0xa2, 0xdf, 0xa9, 0x42, + 0x1f, 0x8c, 0x63, 0xf5, 0x04, 0x10, 0x29, 0x18, 0x77, 0x65, 0xbb, 0xeb, 0x44, 0xdf, 0xa8, 0x42, + 0x37, 0x2b, 0xcb, 0x0b, 0x0f, 0x50, 0x8f, 0xde, 0x5b, 0xd6, 0xfb, 0x4e, 0xf4, 0xb5, 0x2a, 0x74, + 0x95, 0x49, 0x96, 0x2d, 0xec, 0x93, 0x33, 0x74, 0x2c, 0x73, 0xa7, 0xf9, 0x7f, 0xaa, 0xa0, 0xbc, + 0xa1, 0x4d, 0xd3, 0xa9, 0xfa, 0x2c, 0xf9, 0xa6, 0xf1, 0xc0, 0xcf, 0xdb, 0x5f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x77, 0x2b, 0x7a, 0x7c, 0x0a, 0x01, 0x00, 0x00, } diff --git a/plugins/shared/util.go b/plugins/shared/util.go index 252180ddd..9152915b4 100644 --- a/plugins/shared/util.go +++ b/plugins/shared/util.go @@ -4,12 +4,11 @@ import ( "bytes" "fmt" + "github.com/hashicorp/hcl2/hcl" hjson "github.com/hashicorp/hcl2/hcl/json" + "github.com/hashicorp/hcl2/hcldec" "github.com/hashicorp/nomad/nomad/structs" "github.com/ugorji/go/codec" - - "github.com/hashicorp/hcl2/hcl" - "github.com/hashicorp/hcl2/hcldec" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" "github.com/zclconf/go-cty/cty/function/stdlib" diff --git a/vendor/github.com/LK4D4/joincontext/LICENSE b/vendor/github.com/LK4D4/joincontext/LICENSE new file mode 100644 index 000000000..890c0c74b --- /dev/null +++ b/vendor/github.com/LK4D4/joincontext/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Alexander Morozov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/LK4D4/joincontext/README.md b/vendor/github.com/LK4D4/joincontext/README.md new file mode 100644 index 000000000..65417ec7e --- /dev/null +++ b/vendor/github.com/LK4D4/joincontext/README.md @@ -0,0 +1,39 @@ +# joincontext + +[![Build Status](https://travis-ci.org/LK4D4/joincontext.svg?branch=master)](https://travis-ci.org/LK4D4/joincontext) +[![GoDoc](https://godoc.org/github.com/LK4D4/joincontext?status.svg)](https://godoc.org/github.com/LK4D4/joincontext) + +Package joincontext provides a way to combine two contexts. +For example it might be useful for grpc server to cancel all handlers in +addition to provided handler context. + +For additional info see [godoc page](https://godoc.org/github.com/LK4D4/joincontext) + +## Example +```go + ctx1, cancel1 := context.WithCancel(context.Background()) + defer cancel1() + ctx2 := context.Background() + + ctx, cancel := joincontext.Join(ctx1, ctx2) + defer cancel() + select { + case <-ctx.Done(): + default: + fmt.Println("context alive") + } + + cancel1() + + // give some time to propagate + time.Sleep(100 * time.Millisecond) + + select { + case <-ctx.Done(): + fmt.Println(ctx.Err()) + default: + } + + // Output: context alive + // context canceled +``` diff --git a/vendor/github.com/LK4D4/joincontext/context.go b/vendor/github.com/LK4D4/joincontext/context.go new file mode 100644 index 000000000..9d5783bd3 --- /dev/null +++ b/vendor/github.com/LK4D4/joincontext/context.go @@ -0,0 +1,102 @@ +// Package joincontext provides a way to combine two contexts. +// For example it might be useful for grpc server to cancel all handlers in +// addition to provided handler context. +package joincontext + +import ( + "sync" + "time" + + "golang.org/x/net/context" +) + +type joinContext struct { + mu sync.Mutex + ctx1 context.Context + ctx2 context.Context + done chan struct{} + err error +} + +// Join returns new context which is child for two passed contexts. +// It starts new goroutine which tracks both contexts. +// +// Done() channel is closed when one of parents contexts is done. +// +// Deadline() returns earliest deadline between parent contexts. +// +// Err() returns error from first done parent context. +// +// Value(key) looks for key in parent contexts. First found is returned. +func Join(ctx1, ctx2 context.Context) (context.Context, context.CancelFunc) { + c := &joinContext{ctx1: ctx1, ctx2: ctx2, done: make(chan struct{})} + go c.run() + return c, c.cancel +} + +func (c *joinContext) Deadline() (deadline time.Time, ok bool) { + d1, ok1 := c.ctx1.Deadline() + if !ok1 { + return c.ctx2.Deadline() + } + d2, ok2 := c.ctx2.Deadline() + if !ok2 { + return d1, true + } + + if d2.Before(d1) { + return d2, true + } + return d1, true +} + +func (c *joinContext) Done() <-chan struct{} { + return c.done +} + +func (c *joinContext) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *joinContext) Value(key interface{}) interface{} { + v := c.ctx1.Value(key) + if v == nil { + v = c.ctx2.Value(key) + } + return v +} + +func (c *joinContext) run() { + var doneCtx context.Context + select { + case <-c.ctx1.Done(): + doneCtx = c.ctx1 + case <-c.ctx2.Done(): + doneCtx = c.ctx2 + case <-c.done: + return + } + + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return + } + c.err = doneCtx.Err() + c.mu.Unlock() + close(c.done) +} + +func (c *joinContext) cancel() { + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return + } + c.err = context.Canceled + + c.mu.Unlock() + close(c.done) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 7af3c8d61..50aadc225 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -7,6 +7,7 @@ {"path":"github.com/Azure/go-ansiterm","checksumSHA1":"9NFR6RG8H2fNyKHscGmuGLQhRm4=","revision":"d6e3b3328b783f23731bc4d058875b0371ff8109","revisionTime":"2017-09-29T23:40:23Z","version":"master","versionExact":"master"}, {"path":"github.com/Azure/go-ansiterm/winterm","checksumSHA1":"3/UphB+6Hbx5otA4PjFjvObT+L4=","revision":"d6e3b3328b783f23731bc4d058875b0371ff8109","revisionTime":"2017-09-29T23:40:23Z","version":"master","versionExact":"master"}, {"path":"github.com/DataDog/datadog-go/statsd","checksumSHA1":"WvApwvvSe3i/3KO8300dyeFmkbI=","revision":"b10af4b12965a1ad08d164f57d14195b4140d8de","revisionTime":"2017-08-09T10:47:06Z"}, + {"path":"github.com/LK4D4/joincontext","checksumSHA1":"Jmf4AnrptgBdQ5TPBJ2M89nooIQ=","revision":"1724345da6d5bcc8b66fefb843b607ab918e175c","revisionTime":"2017-10-26T17:01:39Z"}, {"path":"github.com/Microsoft/go-winio","checksumSHA1":"PbR6ZKoLeSZl8aXxDQqXih0wSgE=","revision":"97e4973ce50b2ff5f09635a57e2b88a037aae829","revisionTime":"2018-08-23T22:24:21Z"}, {"path":"github.com/NVIDIA/gpu-monitoring-tools","checksumSHA1":"kF1vk+8Xvb3nGBiw9+qbUc0SZ4M=","revision":"86f2a9fac6c5b597dc494420005144b8ef7ec9fb","revisionTime":"2018-08-29T22:20:09Z"}, {"path":"github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml","checksumSHA1":"P8FATSSgpe5A17FyPrGpsX95Xw8=","revision":"86f2a9fac6c5b597dc494420005144b8ef7ec9fb","revisionTime":"2018-08-29T22:20:09Z"}, From a8e95502fe2ffc9eeec373947b3b483de6cd6b54 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 31 Oct 2018 18:00:30 -0700 Subject: [PATCH 2/7] tests --- client/devicemanager/manager.go | 13 +- client/devicemanager/manager_test.go | 501 +++++++++++++++++++++++++++ client/devicemanager/state/state.go | 11 + client/devicemanager/utils.go | 9 - client/state/db_test.go | 4 +- client/state/interface.go | 6 +- client/state/memdb.go | 8 +- client/state/noopdb.go | 6 +- client/state/state_database.go | 12 +- plugins/base/testing.go | 39 ++- plugins/device/mock.go | 91 ++++- plugins/shared/loader/testing.go | 36 ++ 12 files changed, 699 insertions(+), 37 deletions(-) create mode 100644 client/devicemanager/manager_test.go create mode 100644 client/devicemanager/state/state.go diff --git a/client/devicemanager/manager.go b/client/devicemanager/manager.go index c1dcb7977..91126f3e9 100644 --- a/client/devicemanager/manager.go +++ b/client/devicemanager/manager.go @@ -10,6 +10,7 @@ import ( log "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/device" @@ -40,11 +41,11 @@ type Manager interface { type StateStorage interface { // GetDevicePluginState is used to retrieve the device manager's plugin // state. - GetDevicePluginState() (*PluginState, error) + GetDevicePluginState() (*state.PluginState, error) // PutDevicePluginState is used to store the device manager's plugin // state. - PutDevicePluginState(state *PluginState) error + PutDevicePluginState(state *state.PluginState) error } // UpdateNodeDevices is a callback for updating the set of devices on a node. @@ -136,6 +137,12 @@ func (m *manager) Run() { // Get device plugins devices := m.loader.Catalog()[base.PluginTypeDevice] + if len(devices) == 0 { + m.logger.Debug("exiting since there are no device plugins") + m.cancel() + return + } + for _, d := range devices { id := loader.PluginInfoID(d) storeFn := func(c *plugin.ReattachConfig) error { @@ -290,7 +297,7 @@ func (m *manager) storePluginReattachConfig(id loader.PluginID, c *plugin.Reatta m.reattachConfigs[id] = shared.ReattachConfigFromGoPlugin(c) // Persist the state - s := &PluginState{ + s := &state.PluginState{ ReattachConfigs: make(map[string]*shared.ReattachConfig, len(m.reattachConfigs)), } diff --git a/client/devicemanager/manager_test.go b/client/devicemanager/manager_test.go new file mode 100644 index 000000000..7a97b848f --- /dev/null +++ b/client/devicemanager/manager_test.go @@ -0,0 +1,501 @@ +package devicemanager + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + log "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/client/state" + "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared/loader" + psstructs "github.com/hashicorp/nomad/plugins/shared/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/kr/pretty" + "github.com/stretchr/testify/require" +) + +var ( + nvidiaDevice0ID = uuid.Generate() + nvidiaDevice1ID = uuid.Generate() + nvidiaDeviceGroup = &device.DeviceGroup{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + Devices: []*device.Device{ + { + ID: nvidiaDevice0ID, + Healthy: true, + }, + { + ID: nvidiaDevice1ID, + Healthy: true, + }, + }, + Attributes: map[string]*psstructs.Attribute{ + "memory": { + Int: helper.Int64ToPtr(4), + Unit: "GB", + }, + }, + } + + intelDeviceID = uuid.Generate() + intelDeviceGroup = &device.DeviceGroup{ + Vendor: "intel", + Type: "gpu", + Name: "640GT", + Devices: []*device.Device{ + { + ID: intelDeviceID, + Healthy: true, + }, + }, + Attributes: map[string]*psstructs.Attribute{ + "memory": { + Int: helper.Int64ToPtr(2), + Unit: "GB", + }, + }, + } + + nvidiaDeviceGroupStats = &device.DeviceGroupStats{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + InstanceStats: map[string]*device.DeviceStats{ + nvidiaDevice0ID: &device.DeviceStats{ + Summary: &device.StatValue{ + IntNumeratorVal: 212, + Unit: "F", + Desc: "Temperature", + }, + }, + nvidiaDevice1ID: &device.DeviceStats{ + Summary: &device.StatValue{ + IntNumeratorVal: 218, + Unit: "F", + Desc: "Temperature", + }, + }, + }, + } + + intelDeviceGroupStats = &device.DeviceGroupStats{ + Vendor: "intel", + Type: "gpu", + Name: "640GT", + InstanceStats: map[string]*device.DeviceStats{ + intelDeviceID: &device.DeviceStats{ + Summary: &device.StatValue{ + IntNumeratorVal: 220, + Unit: "F", + Desc: "Temperature", + }, + }, + }, + } +) + +func baseTestConfig(t *testing.T) ( + config *Config, + deviceUpdateCh chan []*structs.NodeDeviceResource, + catalog *loader.MockCatalog) { + + // Create an update handler + deviceUpdates := make(chan []*structs.NodeDeviceResource, 1) + updateFn := func(devices []*structs.NodeDeviceResource) { + deviceUpdates <- devices + } + + // Create a mock plugin catalog + mc := &loader.MockCatalog{} + + // Create the config + config = &Config{ + Logger: testlog.HCLogger(t), + PluginConfig: &base.ClientAgentConfig{}, + StatsInterval: 100 * time.Millisecond, + State: state.NewMemDB(), + Updater: updateFn, + Loader: mc, + } + + return config, deviceUpdates, mc +} + +func configureCatalogWith(catalog *loader.MockCatalog, plugins map[*base.PluginInfoResponse]loader.PluginInstance) { + + catalog.DispenseF = func(name, _ string, _ *base.ClientAgentConfig, _ log.Logger) (loader.PluginInstance, error) { + for info, v := range plugins { + if info.Name == name { + return v, nil + } + } + + return nil, fmt.Errorf("no matching plugin") + } + + catalog.ReattachF = func(name, _ string, _ *plugin.ReattachConfig) (loader.PluginInstance, error) { + for info, v := range plugins { + if info.Name == name { + return v, nil + } + } + + return nil, fmt.Errorf("no matching plugin") + } + + catalog.CatalogF = func() map[string][]*base.PluginInfoResponse { + devices := make([]*base.PluginInfoResponse, 0, len(plugins)) + for k := range plugins { + devices = append(devices, k) + } + out := map[string][]*base.PluginInfoResponse{ + base.PluginTypeDevice: devices, + } + return out + } +} + +func pluginInfoResponse(name string) *base.PluginInfoResponse { + return &base.PluginInfoResponse{ + Type: base.PluginTypeDevice, + PluginApiVersion: "v0.0.1", + PluginVersion: "v0.0.1", + Name: name, + } +} + +// drainNodeDeviceUpdates drains all updates to the node device fingerprint channel +func drainNodeDeviceUpdates(ctx context.Context, in chan []*structs.NodeDeviceResource) { + go func() { + for { + select { + case <-ctx.Done(): + return + case <-in: + } + } + }() +} + +func deviceReserveFn(ids []string) (*device.ContainerReservation, error) { + return &device.ContainerReservation{ + Envs: map[string]string{ + "DEVICES": strings.Join(ids, ","), + }, + }, nil +} + +// nvidiaAndIntelDefaultPlugins adds an nvidia and intel mock plugin to the +// catalog +func nvidiaAndIntelDefaultPlugins(catalog *loader.MockCatalog) { + pluginInfoNvidia := pluginInfoResponse("nvidia") + deviceNvidia := &device.MockDevicePlugin{ + MockPlugin: &base.MockPlugin{ + PluginInfoF: base.StaticInfo(pluginInfoNvidia), + ConfigSchemaF: base.TestConfigSchema(), + SetConfigF: base.NoopSetConfig(), + }, + FingerprintF: device.StaticFingerprinter([]*device.DeviceGroup{nvidiaDeviceGroup}), + ReserveF: deviceReserveFn, + StatsF: device.StaticStats([]*device.DeviceGroupStats{nvidiaDeviceGroupStats}), + } + pluginNvidia := loader.MockBasicExternalPlugin(deviceNvidia) + + pluginInfoIntel := pluginInfoResponse("intel") + deviceIntel := &device.MockDevicePlugin{ + MockPlugin: &base.MockPlugin{ + PluginInfoF: base.StaticInfo(pluginInfoIntel), + ConfigSchemaF: base.TestConfigSchema(), + SetConfigF: base.NoopSetConfig(), + }, + FingerprintF: device.StaticFingerprinter([]*device.DeviceGroup{intelDeviceGroup}), + ReserveF: deviceReserveFn, + StatsF: device.StaticStats([]*device.DeviceGroupStats{intelDeviceGroupStats}), + } + pluginIntel := loader.MockBasicExternalPlugin(deviceIntel) + + // Configure the catalog with two plugins + configureCatalogWith(catalog, map[*base.PluginInfoResponse]loader.PluginInstance{ + pluginInfoNvidia: pluginNvidia, + pluginInfoIntel: pluginIntel, + }) +} + +// Test collecting statistics from all devices +func TestManager_AllStats(t *testing.T) { + t.Parallel() + require := require.New(t) + + config, updateCh, catalog := baseTestConfig(t) + nvidiaAndIntelDefaultPlugins(catalog) + + m := New(config) + go m.Run() + defer m.Shutdown() + + // Wait till we get a fingerprint result + select { + case <-time.After(5 * time.Second): + t.Fatal("timeout") + case devices := <-updateCh: + require.Len(devices, 2) + } + + // Now collect all the stats + var stats []*device.DeviceGroupStats + testutil.WaitForResult(func() (bool, error) { + stats = m.AllStats() + l := len(stats) + if l == 2 { + return true, nil + } + + return false, fmt.Errorf("expected count 2; got %d", l) + }, func(err error) { + t.Fatal(err) + }) + + // Check we got stats from both the devices + var nstats, istats bool + for _, stat := range stats { + switch stat.Vendor { + case "intel": + istats = true + case "nvidia": + nstats = true + default: + t.Fatalf("unexpected vendor %q", stat.Vendor) + } + } + require.True(nstats) + require.True(istats) +} + +// Test collecting statistics from a particular device +func TestManager_DeviceStats(t *testing.T) { + t.Parallel() + require := require.New(t) + + config, updateCh, catalog := baseTestConfig(t) + nvidiaAndIntelDefaultPlugins(catalog) + + m := New(config) + go m.Run() + defer m.Shutdown() + + // Wait till we get a fingerprint result + select { + case <-time.After(5 * time.Second): + t.Fatal("timeout") + case devices := <-updateCh: + require.Len(devices, 2) + } + + testutil.WaitForResult(func() (bool, error) { + stats := m.AllStats() + l := len(stats) + if l == 2 { + t.Logf("% #v", pretty.Formatter(stats)) + return true, nil + } + + return false, fmt.Errorf("expected count 2; got %d", l) + }, func(err error) { + t.Fatal(err) + }) + + // Now collect the stats for one nvidia device + stat, err := m.DeviceStats(&structs.AllocatedDeviceResource{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + DeviceIDs: []string{nvidiaDevice1ID}, + }) + require.NoError(err) + require.NotNil(stat) + + require.Len(stat.InstanceStats, 1) + require.Contains(stat.InstanceStats, nvidiaDevice1ID) + + istat := stat.InstanceStats[nvidiaDevice1ID] + require.EqualValues(218, istat.Summary.IntNumeratorVal) +} + +// Test reserving a particular device +func TestManager_Reserve(t *testing.T) { + t.Parallel() + r := require.New(t) + + config, updateCh, catalog := baseTestConfig(t) + nvidiaAndIntelDefaultPlugins(catalog) + + m := New(config) + go m.Run() + defer m.Shutdown() + + // Wait till we get a fingerprint result + select { + case <-time.After(5 * time.Second): + t.Fatal("timeout") + case devices := <-updateCh: + r.Len(devices, 2) + } + + cases := []struct { + in *structs.AllocatedDeviceResource + expected string + err bool + }{ + { + in: &structs.AllocatedDeviceResource{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + DeviceIDs: []string{nvidiaDevice1ID}, + }, + expected: nvidiaDevice1ID, + }, + { + in: &structs.AllocatedDeviceResource{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + DeviceIDs: []string{nvidiaDevice0ID}, + }, + expected: nvidiaDevice0ID, + }, + { + in: &structs.AllocatedDeviceResource{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + DeviceIDs: []string{nvidiaDevice0ID, nvidiaDevice1ID}, + }, + expected: fmt.Sprintf("%s,%s", nvidiaDevice0ID, nvidiaDevice1ID), + }, + { + in: &structs.AllocatedDeviceResource{ + Vendor: "nvidia", + Type: "gpu", + Name: "1080ti", + DeviceIDs: []string{nvidiaDevice0ID, nvidiaDevice1ID, "foo"}, + }, + err: true, + }, + { + in: &structs.AllocatedDeviceResource{ + Vendor: "intel", + Type: "gpu", + Name: "640GT", + DeviceIDs: []string{intelDeviceID}, + }, + expected: intelDeviceID, + }, + { + in: &structs.AllocatedDeviceResource{ + Vendor: "intel", + Type: "gpu", + Name: "foo", + DeviceIDs: []string{intelDeviceID}, + }, + err: true, + }, + } + + for i, c := range cases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + r = require.New(t) + + // Reserve a particular device + res, err := m.Reserve(c.in) + if !c.err { + r.NoError(err) + r.NotNil(res) + + r.Len(res.Envs, 1) + r.Equal(res.Envs["DEVICES"], c.expected) + } else { + r.Error(err) + } + }) + } +} + +// Test that shutdown shutsdown the plugins +func TestManager_Shutdown(t *testing.T) { + t.Parallel() + require := require.New(t) + + config, updateCh, catalog := baseTestConfig(t) + nvidiaAndIntelDefaultPlugins(catalog) + + m := New(config) + go m.Run() + defer m.Shutdown() + + // Wait till we get a fingerprint result + select { + case <-time.After(5 * time.Second): + t.Fatal("timeout") + case devices := <-updateCh: + require.Len(devices, 2) + } + + // Call shutdown and assert that we killed the plugins + m.Shutdown() + + for _, resp := range catalog.Catalog()[base.PluginTypeDevice] { + pinst, _ := catalog.Dispense(resp.Name, resp.Type, &base.ClientAgentConfig{}, config.Logger) + require.True(pinst.Exited()) + } +} + +// Test that startup shutsdown previously launched plugins +func TestManager_Run_ShutdownOld(t *testing.T) { + t.Parallel() + require := require.New(t) + + config, updateCh, catalog := baseTestConfig(t) + nvidiaAndIntelDefaultPlugins(catalog) + + m := New(config) + go m.Run() + defer m.Shutdown() + + // Wait till we get a fingerprint result + select { + case <-time.After(5 * time.Second): + t.Fatal("timeout") + case devices := <-updateCh: + require.Len(devices, 2) + } + + // Create a new manager with the same config so that it reads the old state + m2 := New(config) + go m2.Run() + defer m2.Shutdown() + + testutil.WaitForResult(func() (bool, error) { + for _, resp := range catalog.Catalog()[base.PluginTypeDevice] { + pinst, _ := catalog.Dispense(resp.Name, resp.Type, &base.ClientAgentConfig{}, config.Logger) + if !pinst.Exited() { + return false, fmt.Errorf("plugin %q not shutdown", resp.Name) + } + } + + return true, nil + }, func(err error) { + t.Fatal(err) + }) +} diff --git a/client/devicemanager/state/state.go b/client/devicemanager/state/state.go new file mode 100644 index 000000000..003c554c0 --- /dev/null +++ b/client/devicemanager/state/state.go @@ -0,0 +1,11 @@ +package state + +import "github.com/hashicorp/nomad/plugins/shared" + +// PluginState is used to store the device managers state across restarts of the +// agent +type PluginState struct { + // ReattachConfigs are the set of reattach configs for plugin's launched by + // the device manager + ReattachConfigs map[string]*shared.ReattachConfig +} diff --git a/client/devicemanager/utils.go b/client/devicemanager/utils.go index 4bed98c98..cb9083db0 100644 --- a/client/devicemanager/utils.go +++ b/client/devicemanager/utils.go @@ -6,18 +6,9 @@ import ( "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/device" - "github.com/hashicorp/nomad/plugins/shared" psstructs "github.com/hashicorp/nomad/plugins/shared/structs" ) -// PluginState is used to store the device managers state across restarts of the -// agent -type PluginState struct { - // ReattachConfigs are the set of reattach configs for plugin's launched by - // the device manager - ReattachConfigs map[string]*shared.ReattachConfig -} - // UnknownDeviceError is returned when an operation is attempted on an unknown // device. type UnknownDeviceError struct { diff --git a/client/state/db_test.go b/client/state/db_test.go index 1024436e1..9e6be8a14 100644 --- a/client/state/db_test.go +++ b/client/state/db_test.go @@ -7,7 +7,7 @@ import ( "testing" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" - "github.com/hashicorp/nomad/client/devicemanager" + dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/kr/pretty" @@ -206,7 +206,7 @@ func TestStateDB_DeviceManager(t *testing.T) { require.Nil(ps) // Putting PluginState should work - state := &devicemanager.PluginState{} + state := &dmstate.PluginState{} require.NoError(db.PutDevicePluginState(state)) // Getting should return the available state diff --git a/client/state/interface.go b/client/state/interface.go index 528ad154b..5ccc3f0c3 100644 --- a/client/state/interface.go +++ b/client/state/interface.go @@ -2,7 +2,7 @@ package state import ( "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" - "github.com/hashicorp/nomad/client/devicemanager" + dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/nomad/structs" ) @@ -44,11 +44,11 @@ type StateDB interface { // GetDevicePluginState is used to retrieve the device manager's plugin // state. - GetDevicePluginState() (*devicemanager.PluginState, error) + GetDevicePluginState() (*dmstate.PluginState, error) // PutDevicePluginState is used to store the device manager's plugin // state. - PutDevicePluginState(state *devicemanager.PluginState) error + PutDevicePluginState(state *dmstate.PluginState) error // Close the database. Unsafe for further use after calling regardless // of return value. diff --git a/client/state/memdb.go b/client/state/memdb.go index 27fd8a294..d16e6f232 100644 --- a/client/state/memdb.go +++ b/client/state/memdb.go @@ -4,7 +4,7 @@ import ( "sync" "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" - "github.com/hashicorp/nomad/client/devicemanager" + dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/nomad/structs" ) @@ -19,7 +19,7 @@ type MemDB struct { taskState map[string]map[string]*structs.TaskState // devicemanager -> plugin-state - devManagerPs *devicemanager.PluginState + devManagerPs *dmstate.PluginState mu sync.RWMutex } @@ -135,7 +135,7 @@ func (m *MemDB) DeleteAllocationBucket(allocID string) error { return nil } -func (m *MemDB) PutDevicePluginState(ps *devicemanager.PluginState) error { +func (m *MemDB) PutDevicePluginState(ps *dmstate.PluginState) error { m.mu.Lock() defer m.mu.Unlock() m.devManagerPs = ps @@ -144,7 +144,7 @@ func (m *MemDB) PutDevicePluginState(ps *devicemanager.PluginState) error { // GetDevicePluginState stores the device manager's plugin state or returns an // error. -func (m *MemDB) GetDevicePluginState() (*devicemanager.PluginState, error) { +func (m *MemDB) GetDevicePluginState() (*dmstate.PluginState, error) { m.mu.Lock() defer m.mu.Unlock() return m.devManagerPs, nil diff --git a/client/state/noopdb.go b/client/state/noopdb.go index e0d1697ab..1d4975de2 100644 --- a/client/state/noopdb.go +++ b/client/state/noopdb.go @@ -2,7 +2,7 @@ package state import ( "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" - "github.com/hashicorp/nomad/client/devicemanager" + dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/nomad/structs" ) @@ -41,11 +41,11 @@ func (n NoopDB) DeleteAllocationBucket(allocID string) error { return nil } -func (n NoopDB) PutDevicePluginState(ps *devicemanager.PluginState) error { +func (n NoopDB) PutDevicePluginState(ps *dmstate.PluginState) error { return nil } -func (n NoopDB) GetDevicePluginState() (*devicemanager.PluginState, error) { +func (n NoopDB) GetDevicePluginState() (*dmstate.PluginState, error) { return nil, nil } diff --git a/client/state/state_database.go b/client/state/state_database.go index 8eb8bc43a..46a1df3bf 100644 --- a/client/state/state_database.go +++ b/client/state/state_database.go @@ -5,7 +5,7 @@ import ( "path/filepath" trstate "github.com/hashicorp/nomad/client/allocrunner/taskrunner/state" - "github.com/hashicorp/nomad/client/devicemanager" + dmstate "github.com/hashicorp/nomad/client/devicemanager/state" "github.com/hashicorp/nomad/helper/boltdd" "github.com/hashicorp/nomad/nomad/structs" ) @@ -21,7 +21,7 @@ allocations/ (bucket) |--> task_runner persisted objects (k/v) devicemanager/ -|--> plugin-state -> *devicemanager.PluginState +|--> plugin-state -> *dmstate.PluginState */ var ( @@ -369,7 +369,7 @@ func getTaskBucket(tx *boltdd.Tx, allocID, taskName string) (*boltdd.Bucket, err // PutDevicePluginState stores the device manager's plugin state or returns an // error. -func (s *BoltStateDB) PutDevicePluginState(ps *devicemanager.PluginState) error { +func (s *BoltStateDB) PutDevicePluginState(ps *dmstate.PluginState) error { return s.db.Update(func(tx *boltdd.Tx) error { // Retrieve the root device manager bucket devBkt, err := tx.CreateBucketIfNotExists(devManagerBucket) @@ -383,8 +383,8 @@ func (s *BoltStateDB) PutDevicePluginState(ps *devicemanager.PluginState) error // GetDevicePluginState stores the device manager's plugin state or returns an // error. -func (s *BoltStateDB) GetDevicePluginState() (*devicemanager.PluginState, error) { - var ps *devicemanager.PluginState +func (s *BoltStateDB) GetDevicePluginState() (*dmstate.PluginState, error) { + var ps *dmstate.PluginState err := s.db.View(func(tx *boltdd.Tx) error { devBkt := tx.Bucket(devManagerBucket) @@ -394,7 +394,7 @@ func (s *BoltStateDB) GetDevicePluginState() (*devicemanager.PluginState, error) } // Restore Plugin State if it exists - ps = &devicemanager.PluginState{} + ps = &dmstate.PluginState{} if err := devBkt.Get(devManagerPluginStateKey, ps); err != nil { if !boltdd.IsErrNotFound(err) { return fmt.Errorf("failed to read device manager plugin state: %v", err) diff --git a/plugins/base/testing.go b/plugins/base/testing.go index 09cf7c1a3..bf72fe1f9 100644 --- a/plugins/base/testing.go +++ b/plugins/base/testing.go @@ -22,7 +22,7 @@ var ( Block: &hclspec.Spec_Attr{ Attr: &hclspec.Attr{ Type: "number", - Required: true, + Required: false, }, }, }, @@ -46,13 +46,17 @@ type TestConfig struct { Baz bool `cty:"baz" codec:"baz"` } +type PluginInfoFn func() (*PluginInfoResponse, error) +type ConfigSchemaFn func() (*hclspec.Spec, error) +type SetConfigFn func([]byte, *ClientAgentConfig) error + // MockPlugin is used for testing. // Each function can be set as a closure to make assertions about how data // is passed through the base plugin layer. type MockPlugin struct { - PluginInfoF func() (*PluginInfoResponse, error) - ConfigSchemaF func() (*hclspec.Spec, error) - SetConfigF func([]byte, *ClientAgentConfig) error + PluginInfoF PluginInfoFn + ConfigSchemaF ConfigSchemaFn + SetConfigF SetConfigFn } func (p *MockPlugin) PluginInfo() (*PluginInfoResponse, error) { return p.PluginInfoF() } @@ -60,3 +64,30 @@ func (p *MockPlugin) ConfigSchema() (*hclspec.Spec, error) { return p.Config func (p *MockPlugin) SetConfig(data []byte, cfg *ClientAgentConfig) error { return p.SetConfigF(data, cfg) } + +// Below are static implementations of the base plugin functions + +// StaticInfo returns the passed PluginInfoResponse with no error +func StaticInfo(out *PluginInfoResponse) PluginInfoFn { + return func() (*PluginInfoResponse, error) { + return out, nil + } +} + +// StaticConfigSchema returns the passed Spec with no error +func StaticConfigSchema(out *hclspec.Spec) ConfigSchemaFn { + return func() (*hclspec.Spec, error) { + return out, nil + } +} + +// TestConfigSchema returns a ConfigSchemaFn that statically returns the +// TestSpec +func TestConfigSchema() ConfigSchemaFn { + return StaticConfigSchema(TestSpec) +} + +// NoopSetConfig is a noop implementation of set config +func NoopSetConfig() SetConfigFn { + return func(_ []byte, _ *ClientAgentConfig) error { return nil } +} diff --git a/plugins/device/mock.go b/plugins/device/mock.go index 4fe6c8dd2..a8ab10b1b 100644 --- a/plugins/device/mock.go +++ b/plugins/device/mock.go @@ -7,14 +7,18 @@ import ( "github.com/hashicorp/nomad/plugins/base" ) +type FingerprintFn func(context.Context) (<-chan *FingerprintResponse, error) +type ReserveFn func([]string) (*ContainerReservation, error) +type StatsFn func(context.Context, time.Duration) (<-chan *StatsResponse, error) + // MockDevicePlugin is used for testing. // Each function can be set as a closure to make assertions about how data // is passed through the base plugin layer. type MockDevicePlugin struct { *base.MockPlugin - FingerprintF func(context.Context) (<-chan *FingerprintResponse, error) - ReserveF func([]string) (*ContainerReservation, error) - StatsF func(context.Context, time.Duration) (<-chan *StatsResponse, error) + FingerprintF FingerprintFn + ReserveF ReserveFn + StatsF StatsFn } func (p *MockDevicePlugin) Fingerprint(ctx context.Context) (<-chan *FingerprintResponse, error) { @@ -28,3 +32,84 @@ func (p *MockDevicePlugin) Reserve(devices []string) (*ContainerReservation, err func (p *MockDevicePlugin) Stats(ctx context.Context, interval time.Duration) (<-chan *StatsResponse, error) { return p.StatsF(ctx, interval) } + +// Below are static implementations of the device functions + +// StaticFingerprinter fingerprints the passed devices just once +func StaticFingerprinter(devices []*DeviceGroup) FingerprintFn { + return func(_ context.Context) (<-chan *FingerprintResponse, error) { + outCh := make(chan *FingerprintResponse, 1) + outCh <- &FingerprintResponse{ + Devices: devices, + } + return outCh, nil + } +} + +// ErrorChFingerprinter returns an error fingerprinting over the channel +func ErrorChFingerprinter(err error) FingerprintFn { + return func(_ context.Context) (<-chan *FingerprintResponse, error) { + outCh := make(chan *FingerprintResponse, 1) + outCh <- &FingerprintResponse{ + Error: err, + } + return outCh, nil + } +} + +// StaticReserve returns the passed container reservation +func StaticReserve(out *ContainerReservation) ReserveFn { + return func(_ []string) (*ContainerReservation, error) { + return out, nil + } +} + +// ErrorReserve returns the passed error +func ErrorReserve(err error) ReserveFn { + return func(_ []string) (*ContainerReservation, error) { + return nil, err + } +} + +// StaticStats returns the passed statistics only updating the timestamp +func StaticStats(out []*DeviceGroupStats) StatsFn { + return func(ctx context.Context, intv time.Duration) (<-chan *StatsResponse, error) { + outCh := make(chan *StatsResponse, 1) + + go func() { + ticker := time.NewTimer(0) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ticker.Reset(intv) + } + + now := time.Now() + for _, g := range out { + for _, i := range g.InstanceStats { + i.Timestamp = now + } + } + + outCh <- &StatsResponse{ + Groups: out, + } + } + }() + + return outCh, nil + } +} + +// ErrorChStats returns an error collecting stats over the channel +func ErrorChStats(err error) StatsFn { + return func(_ context.Context, _ time.Duration) (<-chan *StatsResponse, error) { + outCh := make(chan *StatsResponse, 1) + outCh <- &StatsResponse{ + Error: err, + } + return outCh, nil + } +} diff --git a/plugins/shared/loader/testing.go b/plugins/shared/loader/testing.go index 59041b4a3..8ca5bedf3 100644 --- a/plugins/shared/loader/testing.go +++ b/plugins/shared/loader/testing.go @@ -1,8 +1,11 @@ package loader import ( + "net" + log "github.com/hashicorp/go-hclog" plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/plugins/base" ) @@ -39,3 +42,36 @@ func (m *MockInstance) Kill() { m.KillF func (m *MockInstance) ReattachConfig() (*plugin.ReattachConfig, bool) { return m.ReattachConfigF() } func (m *MockInstance) Plugin() interface{} { return m.PluginF() } func (m *MockInstance) Exited() bool { return m.ExitedF() } + +// MockBasicExternalPlugin returns a MockInstance that simulates an external +// plugin returning it has been exited after kill is called. It returns the +// passed inst as the plugin +func MockBasicExternalPlugin(inst interface{}) *MockInstance { + killed := helper.BoolToPtr(false) + return &MockInstance{ + InternalPlugin: false, + KillF: func() { + *killed = true + }, + + ReattachConfigF: func() (*plugin.ReattachConfig, bool) { + return &plugin.ReattachConfig{ + Protocol: "tcp", + Addr: &net.TCPAddr{ + IP: net.IPv4(127, 0, 0, 1), + Port: 3200, + Zone: "", + }, + Pid: 1000, + }, true + }, + + PluginF: func() interface{} { + return inst + }, + + ExitedF: func() bool { + return *killed + }, + } +} From eae45992dbafd1981e6b961f33cd3b588d55a416 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 1 Nov 2018 16:23:04 -0700 Subject: [PATCH 3/7] reserve uses donectx --- plugins/device/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/device/client.go b/plugins/device/client.go index 4548689cb..d20146e75 100644 --- a/plugins/device/client.go +++ b/plugins/device/client.go @@ -81,7 +81,7 @@ func (d *devicePluginClient) Reserve(deviceIDs []string) (*ContainerReservation, } // Make the request - resp, err := d.client.Reserve(context.Background(), req) + resp, err := d.client.Reserve(d.doneCtx, req) if err != nil { return nil, err } From ad4c26a1e3f4dd8531d4688d19245f5a13ed7c49 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Nov 2018 11:09:30 -0800 Subject: [PATCH 4/7] review comments --- client/client.go | 11 ------ client/devicemanager/instance.go | 51 +++++++++++++++------------- client/devicemanager/manager.go | 6 ++-- client/devicemanager/manager_test.go | 6 ++-- client/state/state_database.go | 2 +- drivers/java/driver.go | 7 ++-- scheduler/feasible.go | 2 +- scheduler/rank_test.go | 12 +++---- 8 files changed, 46 insertions(+), 51 deletions(-) diff --git a/client/client.go b/client/client.go index c7898718f..923ab9fa9 100644 --- a/client/client.go +++ b/client/client.go @@ -1,7 +1,6 @@ package client import ( - "context" "errors" "fmt" "io/ioutil" @@ -192,10 +191,6 @@ type Client struct { shutdownCh chan struct{} shutdownLock sync.Mutex - // ctx is cancelled at the same time as the shutdownCh - ctx context.Context - ctxCancel context.CancelFunc - // vaultClient is used to interact with Vault for token and secret renewals vaultClient vaultclient.VaultClient @@ -244,9 +239,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic // Create the logger logger := cfg.Logger.ResetNamed("client") - // Create a context - ctx, cancel := context.WithCancel(context.Background()) - // Create the client c := &Client{ config: cfg, @@ -261,8 +253,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic allocs: make(map[string]AllocRunner), allocUpdates: make(chan *structs.Allocation, 64), shutdownCh: make(chan struct{}), - ctx: ctx, - ctxCancel: cancel, triggerDiscoveryCh: make(chan struct{}), triggerNodeUpdate: make(chan struct{}, 8), triggerEmitNodeEvent: make(chan *structs.NodeEvent, 8), @@ -584,7 +574,6 @@ func (c *Client) Shutdown() error { c.shutdown = true close(c.shutdownCh) - c.ctxCancel() c.connPool.Shutdown() return nil } diff --git a/client/devicemanager/instance.go b/client/devicemanager/instance.go index 7b255fc7e..ab67f496f 100644 --- a/client/devicemanager/instance.go +++ b/client/devicemanager/instance.go @@ -105,9 +105,6 @@ type instanceManager struct { // fingerprinted once. It is used to gate launching the stats collection. firstFingerprintCh chan struct{} hasFingerprinted bool - - // wg is used to track the launched goroutines - wg sync.WaitGroup } // newInstanceManager returns a new device instance manager. It is expected that @@ -135,8 +132,8 @@ func newInstanceManager(c *instanceManagerConfig) *instanceManager { // HasDevices returns if the instance is managing the passed devices func (i *instanceManager) HasDevices(d *structs.AllocatedDeviceResource) bool { - i.deviceLock.Lock() - defer i.deviceLock.Unlock() + i.deviceLock.RLock() + defer i.deviceLock.RUnlock() OUTER: for _, dev := range i.devices { @@ -164,15 +161,15 @@ OUTER: // AllStats returns all the device statistics returned by the device plugin. func (i *instanceManager) AllStats() []*device.DeviceGroupStats { - i.deviceStatsLock.Lock() - defer i.deviceStatsLock.Unlock() + i.deviceStatsLock.RLock() + defer i.deviceStatsLock.RUnlock() return i.deviceStats } // DeviceStats returns the device statistics for the request devices. func (i *instanceManager) DeviceStats(d *structs.AllocatedDeviceResource) *device.DeviceGroupStats { - i.deviceStatsLock.Lock() - defer i.deviceStatsLock.Unlock() + i.deviceStatsLock.RLock() + defer i.deviceStatsLock.RUnlock() // Find the device in question and then gather the instance statistics we // are interested in @@ -214,8 +211,8 @@ func (i *instanceManager) Reserve(d *structs.AllocatedDeviceResource) (*device.C // Devices returns the detected devices. func (i *instanceManager) Devices() []*device.DeviceGroup { - i.deviceLock.Lock() - defer i.deviceLock.Unlock() + i.deviceLock.RLock() + defer i.deviceLock.RUnlock() return i.devices } @@ -238,9 +235,16 @@ func (i *instanceManager) run() { return } + // Create a waitgroup to block on shutdown for all created goroutines to + // exit + var wg sync.WaitGroup + // Start the fingerprinter - i.wg.Add(1) - go i.fingerprint() + wg.Add(1) + go func() { + i.fingerprint() + wg.Done() + }() // Wait for a valid result before starting stats collection select { @@ -250,12 +254,15 @@ func (i *instanceManager) run() { } // Start stats - i.wg.Add(1) - go i.collectStats() + wg.Add(1) + go func() { + i.collectStats() + wg.Done() + }() // Do a final cleanup DONE: - i.wg.Wait() + wg.Wait() i.cleanup() } @@ -305,6 +312,8 @@ func (i *instanceManager) dispense() (plugin device.DevicePlugin, err error) { // cleanup shutsdown the plugin func (i *instanceManager) cleanup() { i.shutdownLock.Lock() + i.pluginLock.Lock() + defer i.pluginLock.Unlock() defer i.shutdownLock.Unlock() if i.plugin != nil && !i.plugin.Exited() { @@ -315,8 +324,6 @@ func (i *instanceManager) cleanup() { // fingerprint is a long lived routine used to fingerprint the device func (i *instanceManager) fingerprint() { - defer i.wg.Done() - START: // Get a device plugin devicePlugin, err := i.dispense() @@ -344,7 +351,7 @@ START: } if !ok { - i.logger.Debug("exiting since fingerprinting gracefully shutdown") + i.logger.Trace("exiting since fingerprinting gracefully shutdown") i.handleFingerprintError() return } @@ -401,7 +408,7 @@ func (i *instanceManager) handleFingerprintError() { // handleFingerprint stores the new devices and triggers the fingerprint output // channel. An error is returned if the passed devices don't pass validation. func (i *instanceManager) handleFingerprint(f *device.FingerprintResponse) error { - // Safety check + // If no devices are returned then there is nothing to do. if f.Devices == nil { return nil } @@ -442,8 +449,6 @@ func (i *instanceManager) handleFingerprint(f *device.FingerprintResponse) error // collectStats is a long lived goroutine for collecting device statistics. It // handles errors by backing off exponentially and retrying. func (i *instanceManager) collectStats() { - defer i.wg.Done() - attempt := 0 START: @@ -472,7 +477,7 @@ START: } if !ok { - i.logger.Debug("exiting since stats gracefully shutdown") + i.logger.Trace("exiting since stats gracefully shutdown") return } diff --git a/client/devicemanager/manager.go b/client/devicemanager/manager.go index 91126f3e9..bf101677b 100644 --- a/client/devicemanager/manager.go +++ b/client/devicemanager/manager.go @@ -49,7 +49,7 @@ type StateStorage interface { } // UpdateNodeDevices is a callback for updating the set of devices on a node. -type UpdateNodeDevices func(devices []*structs.NodeDeviceResource) +type UpdateNodeDevicesFn func(devices []*structs.NodeDeviceResource) // StorePluginReattachFn is used to store plugin reattachment configurations. type StorePluginReattachFn func(*plugin.ReattachConfig) error @@ -66,7 +66,7 @@ type Config struct { PluginConfig *base.ClientAgentConfig // Updater is used to update the node when device information changes - Updater UpdateNodeDevices + Updater UpdateNodeDevicesFn // StatsInterval is the interval at which to collect statistics StatsInterval time.Duration @@ -94,7 +94,7 @@ type manager struct { pluginConfig *base.ClientAgentConfig // updater is used to update the node when device information changes - updater UpdateNodeDevices + updater UpdateNodeDevicesFn // statsInterval is the duration at which to collect statistics statsInterval time.Duration diff --git a/client/devicemanager/manager_test.go b/client/devicemanager/manager_test.go index 7a97b848f..4138aeec7 100644 --- a/client/devicemanager/manager_test.go +++ b/client/devicemanager/manager_test.go @@ -72,14 +72,14 @@ var ( Type: "gpu", Name: "1080ti", InstanceStats: map[string]*device.DeviceStats{ - nvidiaDevice0ID: &device.DeviceStats{ + nvidiaDevice0ID: { Summary: &device.StatValue{ IntNumeratorVal: 212, Unit: "F", Desc: "Temperature", }, }, - nvidiaDevice1ID: &device.DeviceStats{ + nvidiaDevice1ID: { Summary: &device.StatValue{ IntNumeratorVal: 218, Unit: "F", @@ -94,7 +94,7 @@ var ( Type: "gpu", Name: "640GT", InstanceStats: map[string]*device.DeviceStats{ - intelDeviceID: &device.DeviceStats{ + intelDeviceID: { Summary: &device.StatValue{ IntNumeratorVal: 220, Unit: "F", diff --git a/client/state/state_database.go b/client/state/state_database.go index 46a1df3bf..ef499b502 100644 --- a/client/state/state_database.go +++ b/client/state/state_database.go @@ -47,7 +47,7 @@ var ( // devManagerPluginStateKey is the key serialized device manager // plugin state is stored at - devManagerPluginStateKey = []byte("plugin-state") + devManagerPluginStateKey = []byte("plugin_state") ) // NewStateDBFunc creates a StateDB given a state directory. diff --git a/drivers/java/driver.go b/drivers/java/driver.go index 51c0d7d29..dce0467f8 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" "github.com/hashicorp/nomad/plugins/drivers/utils" + "github.com/hashicorp/nomad/plugins/shared" "github.com/hashicorp/nomad/plugins/shared/hclspec" "github.com/hashicorp/nomad/plugins/shared/loader" "golang.org/x/net/context" @@ -106,7 +107,7 @@ type TaskConfig struct { // StartTask. This information is needed to rebuild the taskConfig state and handler // during recovery. type TaskState struct { - ReattachConfig *utils.ReattachConfig + ReattachConfig *shared.ReattachConfig TaskConfig *drivers.TaskConfig Pid int StartedAt time.Time @@ -259,7 +260,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return fmt.Errorf("failed to decode taskConfig state from handle: %v", err) } - plugRC, err := utils.ReattachConfigToGoPlugin(taskState.ReattachConfig) + plugRC, err := shared.ReattachConfigToGoPlugin(taskState.ReattachConfig) if err != nil { d.logger.Error("failed to build ReattachConfig from taskConfig state", "error", err, "task_id", handle.Config.ID) return fmt.Errorf("failed to build ReattachConfig from taskConfig state: %v", err) @@ -361,7 +362,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru } driverState := TaskState{ - ReattachConfig: utils.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), + ReattachConfig: shared.ReattachConfigFromGoPlugin(pluginClient.ReattachConfig()), Pid: ps.Pid, TaskConfig: cfg, StartedAt: h.startedAt, diff --git a/scheduler/feasible.go b/scheduler/feasible.go index 872d5a81c..4db5ae127 100644 --- a/scheduler/feasible.go +++ b/scheduler/feasible.go @@ -899,7 +899,7 @@ OUTER: continue } - // Check the constriants + // Check the constraints if nodeDeviceMatches(c.ctx, d, req) { // Consume the instances available[d] -= desiredCount diff --git a/scheduler/rank_test.go b/scheduler/rank_test.go index 61e3dee7a..03b6fdad5 100644 --- a/scheduler/rank_test.go +++ b/scheduler/rank_test.go @@ -523,7 +523,7 @@ func TestBinPackIterator_Devices(t *testing.T) { }, }, ExpectedPlacements: map[string]map[structs.DeviceIdTuple]devPlacementTuple{ - "web": map[structs.DeviceIdTuple]devPlacementTuple{ + "web": { { Vendor: "nvidia", Type: "gpu", @@ -556,7 +556,7 @@ func TestBinPackIterator_Devices(t *testing.T) { }, }, ExpectedPlacements: map[string]map[structs.DeviceIdTuple]devPlacementTuple{ - "web": map[structs.DeviceIdTuple]devPlacementTuple{ + "web": { { Vendor: "nvidia", Type: "gpu", @@ -597,7 +597,7 @@ func TestBinPackIterator_Devices(t *testing.T) { }, }, ExpectedPlacements: map[string]map[structs.DeviceIdTuple]devPlacementTuple{ - "web": map[structs.DeviceIdTuple]devPlacementTuple{ + "web": { { Vendor: "nvidia", Type: "gpu", @@ -677,7 +677,7 @@ func TestBinPackIterator_Devices(t *testing.T) { }, }, ExpectedPlacements: map[string]map[structs.DeviceIdTuple]devPlacementTuple{ - "web": map[structs.DeviceIdTuple]devPlacementTuple{ + "web": { { Vendor: "nvidia", Type: "gpu", @@ -712,7 +712,7 @@ func TestBinPackIterator_Devices(t *testing.T) { }, }, ExpectedPlacements: map[string]map[structs.DeviceIdTuple]devPlacementTuple{ - "web": map[structs.DeviceIdTuple]devPlacementTuple{ + "web": { { Vendor: "nvidia", Type: "gpu", @@ -751,7 +751,7 @@ func TestBinPackIterator_Devices(t *testing.T) { require.NoError(state.UpsertAllocs(1000, c.ExistingAllocs)) } - static := NewStaticRankIterator(ctx, []*RankedNode{&RankedNode{Node: c.Node}}) + static := NewStaticRankIterator(ctx, []*RankedNode{{Node: c.Node}}) binp := NewBinPackIterator(ctx, static, false, 0) binp.SetTaskGroup(c.TaskGroup) From 873cf30af14baaaa58ed857920d30c5eb14f4deb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Nov 2018 11:51:03 -0800 Subject: [PATCH 5/7] seperate struct and proto generation --- GNUmakefile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 3ff8a5664..36711883d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -191,9 +191,12 @@ checkscripts: ## Lint shell scripts @echo "==> Linting scripts..." @shellcheck ./scripts/* -.PHONY: generate -generate: LOCAL_PACKAGES = $(shell go list ./... | grep -v '/vendor/') -generate: proto ## Update generated code +.PHONY: generate-all +generate-all: generate-structs proto + +.PHONY: generate-structs +generate-structs: LOCAL_PACKAGES = $(shell go list ./... | grep -v '/vendor/') +generate-structs: ## Update generated code @go generate $(LOCAL_PACKAGES) .PHONY: proto @@ -232,7 +235,7 @@ dev: vendorfmt changelogfmt ## Build for the current development platform .PHONY: prerelease prerelease: GO_TAGS=ui release -prerelease: check generate ember-dist static-assets ## Generate all the static assets for a Nomad release +prerelease: check generate-all ember-dist static-assets ## Generate all the static assets for a Nomad release .PHONY: release release: GO_TAGS=ui release @@ -288,7 +291,7 @@ clean: ## Remove build artifacts .PHONY: travis travis: ## Run Nomad test suites with output to prevent timeouts under Travis CI @if [ ! $(SKIP_NOMAD_TESTS) ]; then \ - make generate; \ + make generate-structs; \ fi @sh -C "$(PROJECT_ROOT)/scripts/travis.sh" From 72755c20399914f975c63d6646311e9ce6060a0f Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Nov 2018 11:55:06 -0800 Subject: [PATCH 6/7] remove unused struct field --- devices/gpu/nvidia/device.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/devices/gpu/nvidia/device.go b/devices/gpu/nvidia/device.go index 8058589c2..e5995eb7c 100644 --- a/devices/gpu/nvidia/device.go +++ b/devices/gpu/nvidia/device.go @@ -77,10 +77,6 @@ type NvidiaDevice struct { // fingerprintPeriod is how often we should call nvml to get list of devices fingerprintPeriod time.Duration - // statsPeriod is how often we should collect statistics for fingerprinted - // devices. - statsPeriod time.Duration - // devices is the set of detected eligible devices devices map[string]struct{} deviceLock sync.RWMutex From 37f239ea74f1ec9aef8851e57920f053f7127ddc Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Nov 2018 12:22:07 -0800 Subject: [PATCH 7/7] fix race --- client/devicemanager/manager_test.go | 2 -- plugins/device/mock.go | 9 +-------- plugins/shared/loader/testing.go | 6 ++++++ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/client/devicemanager/manager_test.go b/client/devicemanager/manager_test.go index 4138aeec7..a9d0e9d12 100644 --- a/client/devicemanager/manager_test.go +++ b/client/devicemanager/manager_test.go @@ -19,7 +19,6 @@ import ( "github.com/hashicorp/nomad/plugins/shared/loader" psstructs "github.com/hashicorp/nomad/plugins/shared/structs" "github.com/hashicorp/nomad/testutil" - "github.com/kr/pretty" "github.com/stretchr/testify/require" ) @@ -306,7 +305,6 @@ func TestManager_DeviceStats(t *testing.T) { stats := m.AllStats() l := len(stats) if l == 2 { - t.Logf("% #v", pretty.Formatter(stats)) return true, nil } diff --git a/plugins/device/mock.go b/plugins/device/mock.go index a8ab10b1b..5ac1b4678 100644 --- a/plugins/device/mock.go +++ b/plugins/device/mock.go @@ -71,7 +71,7 @@ func ErrorReserve(err error) ReserveFn { } } -// StaticStats returns the passed statistics only updating the timestamp +// StaticStats returns the passed statistics func StaticStats(out []*DeviceGroupStats) StatsFn { return func(ctx context.Context, intv time.Duration) (<-chan *StatsResponse, error) { outCh := make(chan *StatsResponse, 1) @@ -86,13 +86,6 @@ func StaticStats(out []*DeviceGroupStats) StatsFn { ticker.Reset(intv) } - now := time.Now() - for _, g := range out { - for _, i := range g.InstanceStats { - i.Timestamp = now - } - } - outCh <- &StatsResponse{ Groups: out, } diff --git a/plugins/shared/loader/testing.go b/plugins/shared/loader/testing.go index 8ca5bedf3..501e33166 100644 --- a/plugins/shared/loader/testing.go +++ b/plugins/shared/loader/testing.go @@ -2,6 +2,7 @@ package loader import ( "net" + "sync" log "github.com/hashicorp/go-hclog" plugin "github.com/hashicorp/go-plugin" @@ -47,10 +48,13 @@ func (m *MockInstance) Exited() bool { return // plugin returning it has been exited after kill is called. It returns the // passed inst as the plugin func MockBasicExternalPlugin(inst interface{}) *MockInstance { + var killedLock sync.Mutex killed := helper.BoolToPtr(false) return &MockInstance{ InternalPlugin: false, KillF: func() { + killedLock.Lock() + defer killedLock.Unlock() *killed = true }, @@ -71,6 +75,8 @@ func MockBasicExternalPlugin(inst interface{}) *MockInstance { }, ExitedF: func() bool { + killedLock.Lock() + defer killedLock.Unlock() return *killed }, }