diff --git a/client/allocrunner/alloc_runner_hooks.go b/client/allocrunner/alloc_runner_hooks.go index 2517ee3ae..7f3c536fe 100644 --- a/client/allocrunner/alloc_runner_hooks.go +++ b/client/allocrunner/alloc_runner_hooks.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/davecgh/go-spew/spew" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/client/allocrunner/interfaces" clientconfig "github.com/hashicorp/nomad/client/config" @@ -112,6 +113,8 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { // create network configurator nc := newNetworkConfigurator(ar.Alloc(), config) + spew.Dump(config.BridgeNetworkName) + spew.Dump(config.BridgeNetworkAllocSubnet) // Create the alloc directory hook. This is run first to ensure the // directory path exists for other hooks. diff --git a/client/allocrunner/networking_bridge_linux.go b/client/allocrunner/networking_bridge_linux.go index ea3e69c1c..05126314b 100644 --- a/client/allocrunner/networking_bridge_linux.go +++ b/client/allocrunner/networking_bridge_linux.go @@ -7,6 +7,7 @@ import ( "path/filepath" "github.com/containernetworking/cni/libcni" + "github.com/davecgh/go-spew/spew" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" ) @@ -74,6 +75,8 @@ func (b *bridgeNetworkConfigurator) Setup(alloc *structs.Allocation, spec *drive return err } + spew.Dump(netconf) + result, err := b.cniConfig.AddNetworkList(b.ctx, netconf, b.runtimeConf(alloc, spec)) if result != nil { result.Print() diff --git a/client/client.go b/client/client.go index f99c2f330..49fe95644 100644 --- a/client/client.go +++ b/client/client.go @@ -354,6 +354,17 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic c.configCopy = c.config.Copy() c.configLock.Unlock() + if c.config.AutoFetchCNI { + cniGetter := NewCNIGetter(c.config.AutoFetchCNIURL, c.config.AutoFetchCNIDir) + c.logger.Info("downloading CNI plugins", "url", cniGetter.src) + if err := cniGetter.Get(); err != nil { + c.logger.Warn("failed to fetch CNI plugins", "url", cniGetter.src, "error", err) + } else { + c.config.CNIPath = cniGetter.CNIPath(c.config.CNIPath) + c.logger.Debug("using new CNI Path", "cni_path", c.config.CNIPath) + } + } + fingerprintManager := NewFingerprintManager( c.configCopy.PluginSingletonLoader, c.GetConfig, c.configCopy.Node, c.shutdownCh, c.updateNodeFromFingerprint, c.logger) @@ -556,6 +567,35 @@ func (c *Client) init() error { } c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir) + + // Ensure the cnibin dir exists if we have one + if c.config.AutoFetchCNIDir != "" { + if err := os.MkdirAll(c.config.AutoFetchCNIDir, 0755); err != nil { + return fmt.Errorf("failed creating cnibin dir: %s", err) + } + } else { + // Otherwise make a temp directory to use. + p, err := ioutil.TempDir("", "NomadClient") + if err != nil { + return fmt.Errorf("failed creating temporary directory for the AutoFetchCNIDir: %v", err) + } + + p, err = filepath.EvalSymlinks(p) + if err != nil { + return fmt.Errorf("failed to find temporary directory for the AutoFetchCNIDir: %v", err) + } + + // Change the permissions to have the execute bit + if err := os.Chmod(p, 0755); err != nil { + return fmt.Errorf("failed to change directory permissions for the AutoFetchCNIdir: %v", err) + } + + c.config.AutoFetchCNIDir = p + } + + if c.config.AutoFetchCNI { + c.logger.Info("using alloc directory", "alloc_dir", c.config.AllocDir) + } return nil } diff --git a/client/cni_autofetch.go b/client/cni_autofetch.go new file mode 100644 index 000000000..5a19ed5a8 --- /dev/null +++ b/client/cni_autofetch.go @@ -0,0 +1,51 @@ +package client + +import ( + "fmt" + "path/filepath" + "runtime" + "strings" + + getter "github.com/hashicorp/go-getter" +) + +const ( + nomadCNIBinDir = "cnibin" +) + +var ( + defaultCNIGetterChecksums = map[string]string{ + "linux-amd64": "sha256:e9bfc78acd3ae71be77eb8f3e890cc9078a33cc3797703b8ff2fc3077a232252", + "linux-arm": "sha256:ae6ddbd87c05a79aceb92e1c8c32d11e302f6fc55045f87f6a3ea7e0268b2fda", + "linux-arm64": "sha256:acde854e3def3c776c532ae521c19d8784534918cc56449ff16945a2909bff6d", + "windows-amd64": "sha256:a8a24e9cf93f4db92321afca3fe53bd3ccdf2b7117c403c55a5bac162d8d79cc", + } + defaultCNIGetterSrc = fmt.Sprintf("https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-%s-%s-v0.8.1.tgz?checksum=%s", + runtime.GOOS, runtime.GOARCH, defaultCNIGetterChecksums[runtime.GOOS+"-"+runtime.GOARCH]) +) + +type CNIGetter struct { + src string + dst string +} + +func NewCNIGetter(src, dataDir string) *CNIGetter { + if src == "" { + src = defaultCNIGetterSrc + } + return &CNIGetter{ + src: src, + dst: filepath.Join(dataDir, nomadCNIBinDir), + } +} + +func (g *CNIGetter) Get() error { + return getter.Get(g.dst, g.src) +} + +func (g *CNIGetter) CNIPath(path string) string { + if path == "" { + return g.dst + } + return strings.Join([]string{path, g.dst}, ":") +} diff --git a/client/config/config.go b/client/config/config.go index 9bac1803a..06bc96d4b 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -234,6 +234,18 @@ type Config struct { // for allocations in bridge networking mode. Subnet must be in CIDR // notation BridgeNetworkAllocSubnet string + + // AutoFetchCNI is a toggle to enable auto downloading of the CNI standard + // plugins managed by the CNI team. This defaults to false + AutoFetchCNI bool + + // AutoFetchCNIURL is the go-getter URL to use when auto downloading CNI + // plugins + AutoFetchCNIURL string + + // AutoFetchCNIDir is the destination dir to use when auto doanloading CNI plugins. + // This directory will be appended to the CNIPath so it is searched last + AutoFetchCNIDir string } func (c *Config) Copy() *Config { @@ -268,6 +280,7 @@ func DefaultConfig() *Config { DisableRemoteExec: false, BackwardsCompatibleMetrics: false, RPCHoldTimeout: 5 * time.Second, + AutoFetchCNI: false, } } diff --git a/command/agent/agent.go b/command/agent/agent.go index f0f2cebf5..0b8ce9269 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -437,6 +437,7 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { if agentConfig.DataDir != "" { conf.StateDir = filepath.Join(agentConfig.DataDir, "client") conf.AllocDir = filepath.Join(agentConfig.DataDir, "alloc") + conf.AutoFetchCNIDir = filepath.Join(agentConfig.DataDir, "cnibin") } if agentConfig.Client.StateDir != "" { conf.StateDir = agentConfig.Client.StateDir @@ -542,6 +543,8 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { conf.CNIPath = agentConfig.Client.CNIPath conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet + conf.AutoFetchCNI = agentConfig.Client.AutoFetchCNIPlugins + conf.AutoFetchCNIURL = agentConfig.Client.AutoFetchCNIPluginsURL return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index fe3f74c4b..96d0fa4fa 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -260,6 +260,12 @@ type ClientConfig struct { // creating allocations with bridge networking mode. This range is local to // the host BridgeNetworkSubnet string `hcl:"bridge_network_subnet"` + + // AutoFetchCNIPlugins + AutoFetchCNIPlugins bool `hcl:"auto_fetch_cni_plugins` + + // AutoFetchCNIPluginsURL + AutoFetchCNIPluginsURL string `hcl:"auto_fetch_cni_plugins_url` } // ACLConfig is configuration specific to the ACL system @@ -674,6 +680,7 @@ func DevConfig() *Config { conf.Telemetry.PrometheusMetrics = true conf.Telemetry.PublishAllocationMetrics = true conf.Telemetry.PublishNodeMetrics = true + conf.Client.AutoFetchCNIPlugins = true return conf }