diff --git a/plugins/shared/loader/init.go b/plugins/shared/loader/init.go new file mode 100644 index 000000000..4c3dbad5a --- /dev/null +++ b/plugins/shared/loader/init.go @@ -0,0 +1,375 @@ +package loader + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + multierror "github.com/hashicorp/go-multierror" + plugin "github.com/hashicorp/go-plugin" + version "github.com/hashicorp/go-version" + hcl2 "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/shared" + "github.com/hashicorp/nomad/plugins/shared/hclspec" + "github.com/zclconf/go-cty/cty/msgpack" +) + +var ( + // configParseCtx is the context used to parse a plugin's configuration + // stanza + configParseCtx = &hcl2.EvalContext{ + Functions: shared.GetStdlibFuncs(), + } +) + +// validateConfig returns whether or not the configuration is valid +func validateConfig(config *PluginLoaderConfig) error { + var mErr multierror.Error + if config == nil { + return fmt.Errorf("nil config passed") + } else if config.Logger == nil { + multierror.Append(&mErr, fmt.Errorf("nil logger passed")) + } else if config.PluginDir == "" { + multierror.Append(&mErr, fmt.Errorf("invalid plugin dir %q passed", config.PluginDir)) + } + + // Validate that all plugins have a binary name + for _, c := range config.Configs { + if c.Name == "" { + multierror.Append(&mErr, fmt.Errorf("plugin config passed without binary name")) + } + } + + // Validate internal plugins + for k, config := range config.InternalPlugins { + // Validate config + if config == nil { + multierror.Append(&mErr, fmt.Errorf("nil config passed for internal plugin %s", k)) + continue + } else if config.Factory == nil { + multierror.Append(&mErr, fmt.Errorf("nil factory passed for internal plugin %s", k)) + continue + } + } + + return mErr.ErrorOrNil() +} + +// init initializes the plugin loader by compiling both internal and external +// plugins and selecting the highest versioned version of any given plugin. +func (l *PluginLoader) init(config *PluginLoaderConfig) error { + // Initialize the internal plugins + internal, err := l.initInternal(config.InternalPlugins) + if err != nil { + return fmt.Errorf("failed to fingerprint internal plugins: %v", err) + } + + // Scan for eligibile binaries + plugins, err := l.scan() + if err != nil { + return fmt.Errorf("failed to scan plugin directory %q: %v", l.pluginDir, err) + } + + // Fingerprint the passed plugins + configMap := configMap(config.Configs) + external, err := l.fingerprintPlugins(plugins, configMap) + if err != nil { + return fmt.Errorf("failed to fingerprint plugins: %v", err) + } + + // Merge external and internal plugins + l.plugins = l.mergePlugins(internal, external) + + // Validate that the configs are valid for the plugins + if err := l.validatePluginConfigs(); err != nil { + return fmt.Errorf("parsing plugin configurations failed: %v", err) + } + + return nil +} + +// initInternal initializes internal plugins. +func (l *PluginLoader) initInternal(plugins map[PluginID]*InternalPluginConfig) (map[PluginID]*pluginInfo, error) { + var mErr multierror.Error + fingerprinted := make(map[PluginID]*pluginInfo, len(plugins)) + for k, config := range plugins { + // Create an instance + raw := config.Factory(l.logger) + base, ok := raw.(base.BasePlugin) + if !ok { + multierror.Append(&mErr, fmt.Errorf("internal plugin %s doesn't meet base plugin interface", k)) + continue + } + + info := &pluginInfo{ + factory: config.Factory, + config: config.Config, + } + + // Fingerprint base info + i, err := base.PluginInfo() + if err != nil { + multierror.Append(&mErr, fmt.Errorf("PluginInfo info failed for internal plugin %s: %v", k, err)) + continue + } + info.baseInfo = i + + // Parse and set the plugin version + if v, err := version.NewVersion(i.PluginVersion); err != nil { + multierror.Append(&mErr, fmt.Errorf("failed to parse version %q for internal plugin %s: %v", i.PluginVersion, k, err)) + continue + } else { + info.version = v + } + + // Get the config schema + schema, err := base.ConfigSchema() + if err != nil { + multierror.Append(&mErr, fmt.Errorf("failed to retrieve config schema for internal plugin %s: %v", k, err)) + continue + } + info.configSchema = schema + + // Store the fingerprinted config + fingerprinted[k] = info + } + + return fingerprinted, nil +} + +// scan scans the plugin directory and retrieves potentially eligible binaries +func (l *PluginLoader) scan() ([]os.FileInfo, error) { + // Capture the list of binaries in the plugins folder + files, err := ioutil.ReadDir(l.pluginDir) + if err != nil { + return nil, fmt.Errorf("failed to read plugin directory %q: %v", l.pluginDir, err) + } + + var plugins []os.FileInfo + for _, f := range files { + if f.IsDir() { + l.logger.Debug("skipping sub-dir in plugin folder", "sub-dir", f.Name()) + continue + } + plugins = append(plugins, f) + } + + return plugins, nil +} + +// fingerprintPlugins fingerprints all external plugin binaries +func (l *PluginLoader) fingerprintPlugins(plugins []os.FileInfo, configs map[string]*config.PluginConfig) (map[PluginID]*pluginInfo, error) { + var mErr multierror.Error + fingerprinted := make(map[PluginID]*pluginInfo, len(plugins)) + for _, p := range plugins { + name := cleanPluginExecutable(p.Name()) + c := configs[name] + info, err := l.fingerprintPlugin(p, c) + if err != nil { + l.logger.Error("failed to fingerprint plugin", "plugin", name) + multierror.Append(&mErr, err) + continue + } + + id := PluginID{ + Name: info.baseInfo.Name, + PluginType: info.baseInfo.Type, + } + + // Detect if we already have seen a version of this plugin + if prev, ok := fingerprinted[id]; ok { + + // Determine if we should keep the previous version or override + if prev.version.GreaterThan(info.version) { + l.logger.Info("multiple versions of plugin detected", "plugin", info.baseInfo.Name) + continue + } + } + + // Add the plugin + fingerprinted[id] = info + } + + if err := mErr.ErrorOrNil(); err != nil { + return nil, err + } + + return fingerprinted, nil +} + +// fingerprintPlugin fingerprints the passed external plugin +func (l *PluginLoader) fingerprintPlugin(pluginExe os.FileInfo, config *config.PluginConfig) (*pluginInfo, error) { + info := &pluginInfo{ + exePath: filepath.Join(l.pluginDir, pluginExe.Name()), + } + + // Build the command + cmd := exec.Command(info.exePath) + if config != nil { + cmd.Args = append(cmd.Args, config.Args...) + info.args = config.Args + info.config = config.Config + } + + // Launch the plugin + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: base.Handshake, + Plugins: map[string]plugin.Plugin{ + base.PluginTypeBase: &base.PluginBase{}, + }, + Cmd: cmd, + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: l.logger, + }) + defer client.Kill() + + // Connect via RPC + rpcClient, err := client.Client() + if err != nil { + return nil, err + } + + // Request the plugin + raw, err := rpcClient.Dispense(base.PluginTypeBase) + if err != nil { + return nil, err + } + + // Cast the plugin to the base type + bplugin := raw.(base.BasePlugin) + + // Retrieve base plugin information + i, err := bplugin.PluginInfo() + if err != nil { + return nil, fmt.Errorf("failed to get plugin info for plugin %q: %v", info.exePath, err) + } + info.baseInfo = i + + // Parse and set the plugin version + if v, err := version.NewVersion(i.PluginVersion); err != nil { + return nil, fmt.Errorf("failed to parse plugin %q (%v) version %q: %v", + i.Name, info.exePath, i.PluginVersion, err) + } else { + info.version = v + } + + // Retrieve the schema + schema, err := bplugin.ConfigSchema() + if err != nil { + return nil, fmt.Errorf("failed to get plugin config schema for plugin %q: %v", info.exePath, err) + } + info.configSchema = schema + + return info, nil +} + +// mergePlugins merges internal and external plugins, preferring the highest +// version. +func (l *PluginLoader) mergePlugins(internal, external map[PluginID]*pluginInfo) map[PluginID]*pluginInfo { + finalized := make(map[PluginID]*pluginInfo, len(internal)) + + // Load the internal plugins + for k, v := range internal { + finalized[k] = v + } + + for k, extPlugin := range external { + internal, ok := finalized[k] + if ok { + // We have overlapping plugins, determine if we should keep the + // internal version or override + if extPlugin.version.LessThan(internal.version) { + l.logger.Info("preferring external version of plugin", + "plugin", extPlugin.baseInfo.Name, "internal_version", internal.version.String(), + "external_version", extPlugin.version.String()) + continue + } + } + + // Add external plugin + finalized[k] = extPlugin + } + + return finalized +} + +// validatePluginConfigs is used to validate each plugins' configuration. If the +// plugin has a config, it is parsed with the plugins config schema and +// SetConfig is called to ensure the config is valid. +func (l *PluginLoader) validatePluginConfigs() error { + var mErr multierror.Error + for id, info := range l.plugins { + if err := l.validePluginConfig(id, info); err != nil { + wrapped := multierror.Prefix(err, fmt.Sprintf("plugin %s:", id)) + multierror.Append(&mErr, wrapped) + } + } + + return mErr.ErrorOrNil() +} + +// validatePluginConfig is used to validate the plugin's configuration. If the +// plugin has a config, it is parsed with the plugins config schema and +// SetConfig is called to ensure the config is valid. +func (l *PluginLoader) validePluginConfig(id PluginID, info *pluginInfo) error { + var mErr multierror.Error + + // Check if a config is allowed + if info.configSchema == nil { + if info.config != nil { + return fmt.Errorf("configuration not allowed but config passed") + } + + // Nothing to do + return nil + } + + // Convert the schema to hcl + spec, diag := hclspec.Convert(info.configSchema) + if diag.HasErrors() { + multierror.Append(&mErr, diag.Errs()...) + return multierror.Prefix(&mErr, "failed converting config schema:") + } + + // If there is no config there is nothing to do + if info.config == nil { + return nil + } + + // Parse the config using the spec + val, diag := shared.ParseHclInterface(info.config, spec, configParseCtx) + if diag.HasErrors() { + multierror.Append(&mErr, diag.Errs()...) + return multierror.Prefix(&mErr, "failed parsing config:") + } + + // Marshal the value + cdata, err := msgpack.Marshal(val, val.Type()) + if err != nil { + return fmt.Errorf("failed to msgpack encode config: %v", err) + } + + // Store the marshalled config + info.msgpackConfig = cdata + + // Dispense the plugin and set its config and ensure it is error free + instance, err := l.Dispense(id.Name, id.PluginType, l.logger) + if err != nil { + return fmt.Errorf("failed to dispense plugin: %v", err) + } + defer instance.Kill() + + base, ok := instance.Plugin().(base.BasePlugin) + if !ok { + return fmt.Errorf("dispensed plugin %s doesn't meet base plugin interface", id) + } + + if err := base.SetConfig(cdata); err != nil { + return fmt.Errorf("setting config on plugin failed: %v", err) + } + return nil +} diff --git a/plugins/shared/loader/loader.go b/plugins/shared/loader/loader.go new file mode 100644 index 000000000..2a0bcf255 --- /dev/null +++ b/plugins/shared/loader/loader.go @@ -0,0 +1,261 @@ +package loader + +import ( + "fmt" + "os/exec" + + log "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/go-version" + "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared/hclspec" +) + +// PluginCatalog is used to retrieve plugins, either external or internal +type PluginCatalog interface { + // Dispense returns the plugin given its name and type. This will also + // configure the plugin + Dispense(name, pluginType string, logger log.Logger) (*PluginInstance, error) + + // Reattach is used to reattach to a previously launched external plugin. + Reattach(pluginType string, config *plugin.ReattachConfig) (*PluginInstance, error) + + // Catalog returns the catalog of all plugins keyed by plugin type + Catalog() map[string][]*base.PluginInfoResponse +} + +// PluginInstance wraps an instance of a plugin. If the plugin is external, it +// provides methods to retrieve the ReattachConfig and to kill the plugin. +type PluginInstance struct { + client *plugin.Client + instance interface{} +} + +// Internal returns if the plugin is internal +func (p *PluginInstance) Internal() bool { + return p.client == nil +} + +// Kill kills the plugin if it is external. It is safe to call on internal +// plugins. +func (p *PluginInstance) Kill() { + if p.client != nil { + p.client.Kill() + } +} + +// ReattachConfig returns the ReattachConfig and whether the plugin is internal +// or not. If the second return value is true, no ReattachConfig is possible to +// return. +func (p *PluginInstance) ReattachConfig() (*plugin.ReattachConfig, bool) { + if p.client == nil { + return nil, false + } + return p.client.ReattachConfig(), true +} + +// Plugin returns the wrapped plugin instance. +func (p *PluginInstance) Plugin() interface{} { + return p.instance +} + +// PluginLoader is used to retrieve plugins either externally or from internal +// factories. +type PluginLoader struct { + // logger is the plugin loaders logger + logger log.Logger + + // pluginDir is the directory containing plugin binaries + pluginDir string + + // plugins maps a plugin to information required to launch it + plugins map[PluginID]*pluginInfo +} + +// PluginID is a tuple identifying a plugin +type PluginID struct { + // Name is the name of the plugin + Name string + + // PluginType is the plugin's type + PluginType string +} + +// String returns a friendly representation of the plugin. +func (id PluginID) String() string { + return fmt.Sprintf("%q (%v)", id.Name, id.PluginType) +} + +// PluginLoaderConfig configures a plugin loader. +type PluginLoaderConfig struct { + // Logger is the logger used by the plugin loader + Logger log.Logger + + // PluginDir is the directory scanned for loading plugins + PluginDir string + + // Configs is an optional set of configs for plugins + Configs []*config.PluginConfig + + // InternalPlugins allows registering internal plugins. + InternalPlugins map[PluginID]*InternalPluginConfig +} + +// InternalPluginConfig is used to configure launching an internal plugin. +type InternalPluginConfig struct { + Config map[string]interface{} + Factory plugins.PluginFactory +} + +// pluginInfo captures the necessary information to launch and configure a +// plugin. +type pluginInfo struct { + factory plugins.PluginFactory + + exePath string + args []string + + baseInfo *base.PluginInfoResponse + version *version.Version + + configSchema *hclspec.Spec + config map[string]interface{} + msgpackConfig []byte +} + +// NewPluginLoader returns an instance of a plugin loader or an error if the +// plugins could not be loaded +func NewPluginLoader(config *PluginLoaderConfig) (*PluginLoader, error) { + if err := validateConfig(config); err != nil { + return nil, fmt.Errorf("invalid plugin loader configuration passed: %v", err) + } + + logger := config.Logger.Named("plugin-loader").With("plugin-dir", config.PluginDir) + l := &PluginLoader{ + logger: logger, + pluginDir: config.PluginDir, + plugins: make(map[PluginID]*pluginInfo), + } + + if err := l.init(config); err != nil { + return nil, fmt.Errorf("failed to initialize plugin loader: %v", err) + } + + return l, nil +} + +// Dispense returns a plugin instance, loading it either internally or by +// launching an external plugin. +func (l *PluginLoader) Dispense(name, pluginType string, logger log.Logger) (*PluginInstance, error) { + id := PluginID{ + Name: name, + PluginType: pluginType, + } + pinfo, ok := l.plugins[id] + if !ok { + return nil, fmt.Errorf("unknown plugin with name %q and type %q", name, pluginType) + } + + // If the plugin is internal, launch via the factory + var instance *PluginInstance + if pinfo.factory != nil { + instance = &PluginInstance{ + instance: pinfo.factory(logger), + } + } else { + var err error + instance, err = l.dispensePlugin(pinfo.baseInfo.Type, pinfo.exePath, pinfo.args, nil, logger) + if err != nil { + return nil, fmt.Errorf("failed to launch plugin: %v", err) + } + } + + // Cast to the base type and set the config + base, ok := instance.Plugin().(base.BasePlugin) + if !ok { + return nil, fmt.Errorf("plugin %s doesn't implement base plugin interface", id) + } + + if len(pinfo.msgpackConfig) != 0 { + if err := base.SetConfig(pinfo.msgpackConfig); err != nil { + return nil, fmt.Errorf("setting config for plugin %s failed: %v", id, err) + } + } + + return instance, nil +} + +// Reattach reattaches to a previously launched external plugin. +func (l *PluginLoader) Reattach(pluginType string, config *plugin.ReattachConfig) (*PluginInstance, error) { + return l.dispensePlugin(pluginType, "", nil, config, l.logger) +} + +// dispensePlugin is used to launch or reattach to an external plugin. +func (l *PluginLoader) dispensePlugin( + pluginType, cmd string, args []string, reattach *plugin.ReattachConfig, + logger log.Logger) (*PluginInstance, error) { + + var pluginCmd *exec.Cmd + if cmd != "" && reattach != nil { + return nil, fmt.Errorf("both launch command and reattach config specified") + } else if cmd == "" && reattach == nil { + return nil, fmt.Errorf("one of launch command or reattach config must be specified") + } else if cmd != "" { + pluginCmd = exec.Command(cmd, args...) + } + + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: base.Handshake, + Plugins: getPluginMap(pluginType), + Cmd: pluginCmd, + AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, + Logger: logger, + Reattach: reattach, + }) + + // Connect via RPC + rpcClient, err := client.Client() + if err != nil { + client.Kill() + return nil, err + } + + // Request the plugin + raw, err := rpcClient.Dispense(pluginType) + if err != nil { + client.Kill() + return nil, err + } + + instance := &PluginInstance{ + client: client, + instance: raw, + } + return instance, nil +} + +// getPluginMap returns a plugin map based on the type of plugin being launched. +func getPluginMap(pluginType string) map[string]plugin.Plugin { + pmap := map[string]plugin.Plugin{ + base.PluginTypeBase: &base.PluginBase{}, + } + + switch pluginType { + case base.PluginTypeDevice: + pmap[base.PluginTypeDevice] = &device.PluginDevice{} + } + + return pmap +} + +// Catalog returns the catalog of all plugins +func (l *PluginLoader) Catalog() map[string][]*base.PluginInfoResponse { + c := make(map[string][]*base.PluginInfoResponse, 3) + for id, info := range l.plugins { + c[id.PluginType] = append(c[id.PluginType], info.baseInfo) + } + return c +} diff --git a/plugins/shared/loader/loader_test.go b/plugins/shared/loader/loader_test.go new file mode 100644 index 000000000..fcd82533d --- /dev/null +++ b/plugins/shared/loader/loader_test.go @@ -0,0 +1,851 @@ +package loader + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "testing" + + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/stretchr/testify/require" +) + +// harness is used to build a temp directory and copy our own test executable +// into it, allowing the plugin loader to scan for plugins. +type harness struct { + t *testing.T + tmpDir string +} + +// newHarness returns a harness and copies our test binary to the temp directory +// with teh passed plugin names. +func newHarness(t *testing.T, plugins []string) *harness { + t.Helper() + + h := &harness{ + t: t, + } + + // Build a temp directory + path, err := ioutil.TempDir("", t.Name()) + if err != nil { + t.Fatalf("failed to build tmp directory") + } + h.tmpDir = path + + // Get our own executable path + selfExe, err := os.Executable() + if err != nil { + t.Fatalf("failed to get self executable path: %v", err) + } + + for _, p := range plugins { + dest := filepath.Join(h.tmpDir, p) + if err := copyFile(selfExe, dest); err != nil { + t.Fatalf("failed to copy file: %v", err) + } + } + + return h +} + +// copyFile copies the src file to dst. +func copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + if _, err = io.Copy(out, in); err != nil { + return err + } + if err := out.Close(); err != nil { + return err + } + + return os.Chmod(dst, 0777) +} + +// pluginDir returns the plugin directory. +func (h *harness) pluginDir() string { + return h.tmpDir +} + +// cleanup removes the temp directory +func (h *harness) cleanup() { + if err := os.RemoveAll(h.tmpDir); err != nil { + h.t.Fatalf("failed to remove tmp directory %q: %v", h.tmpDir, err) + } +} + +func TestPluginLoader_External(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device", "mock-device-2"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[0]}, + }, + { + Name: plugins[1], + Args: []string{"-plugin", "-name", plugins[1], + "-type", base.PluginTypeDevice, "-version", pluginVersions[1]}, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 2) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[0], + PluginApiVersion: "v0.1.0", + }, + { + Name: plugins[1], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +func TestPluginLoader_External_Config(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device", "mock-device-2"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[0]}, + Config: map[string]interface{}{ + "foo": "1", + "bar": "2", + }, + }, + { + Name: plugins[1], + Args: []string{"-plugin", "-name", plugins[1], + "-type", base.PluginTypeDevice, "-version", pluginVersions[1]}, + Config: map[string]interface{}{ + "foo": "3", + "bar": "4", + }, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 2) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[0], + PluginApiVersion: "v0.1.0", + }, + { + Name: plugins[1], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +// Pass a config but make sure it is fatal +func TestPluginLoader_External_Config_Bad(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device"} + pluginVersions := []string{"v0.0.1"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[0]}, + Config: map[string]interface{}{ + "foo": "1", + "bar": "2", + "non-existant": "3", + }, + }, + }, + } + + _, err := NewPluginLoader(lconfig) + require.Error(err) + require.Contains(err.Error(), "No argument or block type is named \"non-existant\"") +} + +func TestPluginLoader_External_VersionOverlap(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device", "mock-device"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[0]}, + }, + { + Name: plugins[1], + Args: []string{"-plugin", "-name", plugins[1], + "-type", base.PluginTypeDevice, "-version", pluginVersions[1]}, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 1) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[1], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +func TestPluginLoader_Internal(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create the harness + h := newHarness(t, nil) + defer h.cleanup() + + plugins := []string{"mock-device", "mock-device-2"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugins[0], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[0], base.PluginTypeDevice, pluginVersions[0], true), + }, + { + Name: plugins[1], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[1], base.PluginTypeDevice, pluginVersions[1], true), + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 2) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[0], + PluginApiVersion: "v0.1.0", + }, + { + Name: plugins[1], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +func TestPluginLoader_Internal_Config(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create the harness + h := newHarness(t, nil) + defer h.cleanup() + + plugins := []string{"mock-device", "mock-device-2"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugins[0], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[0], base.PluginTypeDevice, pluginVersions[0], true), + Config: map[string]interface{}{ + "foo": "1", + "bar": "2", + }, + }, + { + Name: plugins[1], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[1], base.PluginTypeDevice, pluginVersions[1], true), + Config: map[string]interface{}{ + "foo": "3", + "bar": "4", + }, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 2) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[0], + PluginApiVersion: "v0.1.0", + }, + { + Name: plugins[1], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +// Pass a config but make sure it is fatal +func TestPluginLoader_Internal_Config_Bad(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create the harness + h := newHarness(t, nil) + defer h.cleanup() + + plugins := []string{"mock-device"} + pluginVersions := []string{"v0.0.1"} + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugins[0], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[0], base.PluginTypeDevice, pluginVersions[0], true), + Config: map[string]interface{}{ + "foo": "1", + "bar": "2", + "non-existant": "3", + }, + }, + }, + } + + _, err := NewPluginLoader(lconfig) + require.Error(err) + require.Contains(err.Error(), "No argument or block type is named \"non-existant\"") +} + +func TestPluginLoader_InternalOverrideExternal(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[0]}, + }, + }, + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugins[0], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[0], base.PluginTypeDevice, pluginVersions[1], true), + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 1) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +func TestPluginLoader_ExternalOverrideInternal(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugins := []string{"mock-device"} + pluginVersions := []string{"v0.0.1", "v0.0.2"} + h := newHarness(t, plugins) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugins[0], + Args: []string{"-plugin", "-name", plugins[0], + "-type", base.PluginTypeDevice, "-version", pluginVersions[1]}, + }, + }, + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugins[0], + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugins[0], base.PluginTypeDevice, pluginVersions[0], true), + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Get the catalog and assert we have the two plugins + c := l.Catalog() + require.Len(c, 1) + require.Contains(c, base.PluginTypeDevice) + detected := c[base.PluginTypeDevice] + require.Len(detected, 1) + sort.Slice(detected, func(i, j int) bool { return detected[i].Name < detected[j].Name }) + + expected := []*base.PluginInfoResponse{ + { + Name: plugins[0], + Type: base.PluginTypeDevice, + PluginVersion: pluginVersions[1], + PluginApiVersion: "v0.1.0", + }, + } + require.EqualValues(expected, detected) +} + +func TestPluginLoader_Dispense_External(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugin := "mock-device" + pluginVersion := "v0.0.1" + h := newHarness(t, []string{plugin}) + defer h.cleanup() + + expKey := "set_config_worked" + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugin, + Args: []string{"-plugin", "-name", plugin, + "-type", base.PluginTypeDevice, "-version", pluginVersion}, + Config: map[string]interface{}{ + "res_key": expKey, + }, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Dispense a device plugin + p, err := l.Dispense(plugin, base.PluginTypeDevice, logger) + require.NoError(err) + defer p.Kill() + + instance, ok := p.Plugin().(device.DevicePlugin) + require.True(ok) + + res, err := instance.Reserve([]string{"fake"}) + require.NoError(err) + require.NotNil(res) + require.Contains(res.Envs, expKey) +} + +func TestPluginLoader_Dispense_Internal(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugin := "mock-device" + pluginVersion := "v0.0.1" + h := newHarness(t, nil) + defer h.cleanup() + + expKey := "set_config_worked" + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + InternalPlugins: map[PluginID]*InternalPluginConfig{ + { + Name: plugin, + PluginType: base.PluginTypeDevice, + }: { + Factory: mockFactory(plugin, base.PluginTypeDevice, pluginVersion, true), + Config: map[string]interface{}{ + "res_key": expKey, + }, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Dispense a device plugin + p, err := l.Dispense(plugin, base.PluginTypeDevice, logger) + require.NoError(err) + defer p.Kill() + + instance, ok := p.Plugin().(device.DevicePlugin) + require.True(ok) + + res, err := instance.Reserve([]string{"fake"}) + require.NoError(err) + require.NotNil(res) + require.Contains(res.Envs, expKey) +} + +func TestPluginLoader_Dispense_NoConfigSchema_External(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugin := "mock-device" + pluginVersion := "v0.0.1" + h := newHarness(t, []string{plugin}) + defer h.cleanup() + + expKey := "set_config_worked" + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugin, + Args: []string{"-plugin", "-config-schema=false", "-name", plugin, + "-type", base.PluginTypeDevice, "-version", pluginVersion}, + Config: map[string]interface{}{ + "res_key": expKey, + }, + }, + }, + } + + _, err := NewPluginLoader(lconfig) + require.Error(err) + require.Contains(err.Error(), "configuration not allowed") + + // Remove the config and try again + lconfig.Configs[0].Config = nil + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Dispense a device plugin + p, err := l.Dispense(plugin, base.PluginTypeDevice, logger) + require.NoError(err) + defer p.Kill() + + _, ok := p.Plugin().(device.DevicePlugin) + require.True(ok) +} + +func TestPluginLoader_Dispense_NoConfigSchema_Internal(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create two plugins + plugin := "mock-device" + pluginVersion := "v0.0.1" + h := newHarness(t, nil) + defer h.cleanup() + + expKey := "set_config_worked" + + logger := log.Default() + logger.SetLevel(log.Trace) + pid := PluginID{ + Name: plugin, + PluginType: base.PluginTypeDevice, + } + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + InternalPlugins: map[PluginID]*InternalPluginConfig{ + pid: { + Factory: mockFactory(plugin, base.PluginTypeDevice, pluginVersion, false), + Config: map[string]interface{}{ + "res_key": expKey, + }, + }, + }, + } + + _, err := NewPluginLoader(lconfig) + require.Error(err) + require.Contains(err.Error(), "configuration not allowed") + + // Remove the config and try again + lconfig.InternalPlugins[pid].Factory = mockFactory(plugin, base.PluginTypeDevice, pluginVersion, true) + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Dispense a device plugin + p, err := l.Dispense(plugin, base.PluginTypeDevice, logger) + require.NoError(err) + defer p.Kill() + + _, ok := p.Plugin().(device.DevicePlugin) + require.True(ok) +} + +func TestPluginLoader_Reattach_External(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create a plugin + plugin := "mock-device" + pluginVersion := "v0.0.1" + h := newHarness(t, []string{plugin}) + defer h.cleanup() + + expKey := "set_config_worked" + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugin, + Args: []string{"-plugin", "-name", plugin, + "-type", base.PluginTypeDevice, "-version", pluginVersion}, + Config: map[string]interface{}{ + "res_key": expKey, + }, + }, + }, + } + + l, err := NewPluginLoader(lconfig) + require.NoError(err) + + // Dispense a device plugin + p, err := l.Dispense(plugin, base.PluginTypeDevice, logger) + require.NoError(err) + defer p.Kill() + + instance, ok := p.Plugin().(device.DevicePlugin) + require.True(ok) + + res, err := instance.Reserve([]string{"fake"}) + require.NoError(err) + require.NotNil(res) + require.Contains(res.Envs, expKey) + + // Reattach to the plugin + reattach, ok := p.ReattachConfig() + require.True(ok) + + p2, err := l.Reattach(base.PluginTypeDevice, reattach) + require.NoError(err) + + // Get the reattached plugin and ensure its teh same + instance2, ok := p2.Plugin().(device.DevicePlugin) + require.True(ok) + + res2, err := instance2.Reserve([]string{"fake"}) + require.NoError(err) + require.NotNil(res2) + require.Contains(res2.Envs, expKey) +} + +// Test the loader trying to launch a non-plugin binary +func TestPluginLoader_Bad_Executable(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create a plugin + plugin := "mock-device" + h := newHarness(t, []string{plugin}) + defer h.cleanup() + + logger := log.Default() + logger.SetLevel(log.Trace) + lconfig := &PluginLoaderConfig{ + Logger: logger, // XXX Use testlog package + PluginDir: h.pluginDir(), + Configs: []*config.PluginConfig{ + { + Name: plugin, + Args: []string{"-bad-flag"}, + }, + }, + } + + _, err := NewPluginLoader(lconfig) + require.Error(err) + require.Contains(err.Error(), "failed to fingerprint plugin") +} diff --git a/plugins/shared/loader/plugin_test.go b/plugins/shared/loader/plugin_test.go new file mode 100644 index 000000000..4d799ffac --- /dev/null +++ b/plugins/shared/loader/plugin_test.go @@ -0,0 +1,167 @@ +package loader + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + + log "github.com/hashicorp/go-hclog" + plugin "github.com/hashicorp/go-plugin" + "github.com/hashicorp/nomad/plugins/base" + "github.com/hashicorp/nomad/plugins/device" + "github.com/hashicorp/nomad/plugins/shared/hclspec" +) + +// TestMain runs either the tests or runs a mock plugin based on the passed +// flags +func TestMain(m *testing.M) { + var plugin, configSchema bool + var name, pluginType, pluginVersion string + flag.BoolVar(&plugin, "plugin", false, "run binary as a plugin") + flag.BoolVar(&configSchema, "config-schema", true, "return a config schema") + flag.StringVar(&name, "name", "", "plugin name") + flag.StringVar(&pluginType, "type", "", "plugin type") + flag.StringVar(&pluginVersion, "version", "", "plugin version") + flag.Parse() + + if plugin { + if err := pluginMain(name, pluginType, pluginVersion, configSchema); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + } else { + os.Exit(m.Run()) + } +} + +// pluginMain starts a mock plugin using the passed parameters +func pluginMain(name, pluginType, version string, config bool) error { + // Validate passed parameters + if name == "" || pluginType == "" { + return fmt.Errorf("name and plugin type must be specified") + } + + switch pluginType { + case base.PluginTypeDevice: + default: + return fmt.Errorf("unsupported plugin type %q", pluginType) + } + + // Create the mock plugin + m := &mockPlugin{ + name: name, + ptype: pluginType, + version: version, + configSchema: config, + } + + // Build the plugin map + pmap := map[string]plugin.Plugin{ + base.PluginTypeBase: &base.PluginBase{Impl: m}, + } + switch pluginType { + case base.PluginTypeDevice: + pmap[base.PluginTypeDevice] = &device.PluginDevice{Impl: m} + } + + // Serve the plugin + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: base.Handshake, + Plugins: pmap, + GRPCServer: plugin.DefaultGRPCServer, + }) + + return nil +} + +// mockFactory returns a PluginFactory method which creates the mock plugin with +// the passed parameters +func mockFactory(name, ptype, version string, configSchema bool) func(log log.Logger) interface{} { + return func(log log.Logger) interface{} { + return &mockPlugin{ + name: name, + ptype: ptype, + version: version, + configSchema: configSchema, + } + } +} + +// mockPlugin is a plugin that meets various plugin interfaces but is only +// useful for testing. +type mockPlugin struct { + name string + ptype string + version string + configSchema bool + + // config is built on SetConfig + config *mockPluginConfig +} + +// mockPluginConfig is the configuration for the mock plugin +type mockPluginConfig struct { + Foo string `codec:"foo"` + Bar int `codec:"bar"` + + // ResKey is a key that is populated in the Env map when a device is + // reserved. + ResKey string `codec:"res_key"` +} + +// PluginInfo returns the plugin information based on the passed fields when +// building the mock plugin +func (m *mockPlugin) PluginInfo() (*base.PluginInfoResponse, error) { + return &base.PluginInfoResponse{ + Type: m.ptype, + PluginApiVersion: "v0.1.0", + PluginVersion: m.version, + Name: m.name, + }, nil +} + +func (m *mockPlugin) ConfigSchema() (*hclspec.Spec, error) { + if !m.configSchema { + return nil, nil + } + + // configSpec is the hclspec for parsing the mock's configuration + configSpec := hclspec.NewObject(map[string]*hclspec.Spec{ + "foo": hclspec.NewAttr("foo", "string", false), + "bar": hclspec.NewAttr("bar", "number", false), + "res_key": hclspec.NewAttr("res_key", "string", false), + }) + + return configSpec, nil +} + +// SetConfig decodes the configuration and stores it +func (m *mockPlugin) SetConfig(data []byte) error { + var config mockPluginConfig + if err := base.MsgPackDecode(data, &config); err != nil { + return err + } + + m.config = &config + return nil +} + +func (m *mockPlugin) Fingerprint(ctx context.Context) (<-chan *device.FingerprintResponse, error) { + return make(chan *device.FingerprintResponse), nil +} + +func (m *mockPlugin) Reserve(deviceIDs []string) (*device.ContainerReservation, error) { + if m.config == nil || m.config.ResKey == "" { + return nil, nil + } + + return &device.ContainerReservation{ + Envs: map[string]string{m.config.ResKey: "config-set"}, + }, nil +} + +func (m *mockPlugin) Stats(ctx context.Context) (<-chan *device.StatsResponse, error) { + return make(chan *device.StatsResponse), nil +} diff --git a/plugins/shared/loader/util.go b/plugins/shared/loader/util.go new file mode 100644 index 000000000..6409b95de --- /dev/null +++ b/plugins/shared/loader/util.go @@ -0,0 +1,26 @@ +package loader + +import ( + "strings" + + "github.com/hashicorp/nomad/nomad/structs/config" +) + +// configMap returns a mapping of plugin binary name to config. +func configMap(configs []*config.PluginConfig) map[string]*config.PluginConfig { + pluginMapping := make(map[string]*config.PluginConfig, len(configs)) + for _, c := range configs { + pluginMapping[c.Name] = c + } + return pluginMapping +} + +// cleanPluginExecutable strips the executable name of common suffixes +func cleanPluginExecutable(name string) string { + switch { + case strings.HasSuffix(name, ".exe"): + return strings.TrimSuffix(name, ".exe") + default: + return name + } +}