diff --git a/GNUmakefile b/GNUmakefile index f09f0fc30..d1f64d27e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -16,14 +16,9 @@ ifeq (,$(findstring $(THIS_OS),Darwin Linux FreeBSD)) $(error Building Nomad is currently only supported on Darwin and Linux.) endif -# On Linux we build for Linux, Windows, and potentially Linux+LXC +# On Linux we build for Linux and Windows ifeq (Linux,$(THIS_OS)) -# Detect if we have LXC on the path -ifeq (0,$(shell pkg-config --exists lxc; echo $$?)) -HAS_LXC="true" -endif - ifeq ($(TRAVIS),true) $(info Running in Travis, verbose mode is disabled) else @@ -38,9 +33,6 @@ ALL_TARGETS += linux_386 \ windows_386 \ windows_amd64 -ifeq ("true",$(HAS_LXC)) -ALL_TARGETS += linux_amd64-lxc -endif endif # On MacOS, we only build for MacOS @@ -122,14 +114,6 @@ pkg/windows_amd64/nomad: $(SOURCE_FILES) ## Build Nomad for windows/amd64 -tags "$(GO_TAGS)" \ -o "$@.exe" -pkg/linux_amd64-lxc/nomad: $(SOURCE_FILES) ## Build Nomad+LXC for linux/amd64 - @echo "==> Building $@ with tags $(GO_TAGS)..." - @CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \ - go build \ - -ldflags $(GO_LDFLAGS) \ - -tags "$(GO_TAGS) lxc" \ - -o "$@" - # Define package targets for each of the build targets we actually have on this system define makePackageTarget @@ -222,7 +206,7 @@ changelogfmt: dev: GOOS=$(shell go env GOOS) dev: GOARCH=$(shell go env GOARCH) dev: GOPATH=$(shell go env GOPATH) -dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)$(if $(HAS_LXC),-lxc)/nomad +dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad dev: vendorfmt changelogfmt ## Build for the current development platform @echo "==> Removing old development build..." @rm -f $(PROJECT_ROOT)/$(DEV_TARGET) @@ -268,7 +252,7 @@ test-nomad: dev ## Run Nomad test suites $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ -cover \ -timeout=15m \ - -tags="$(if $(HAS_LXC),lxc)" ./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code) + ./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code) @if [ $(VERBOSE) ] ; then \ bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \ fi diff --git a/drivers/lxc/driver.go b/drivers/lxc/driver.go deleted file mode 100644 index d9d1ea97e..000000000 --- a/drivers/lxc/driver.go +++ /dev/null @@ -1,513 +0,0 @@ -//+build linux,lxc - -package lxc - -import ( - "context" - "fmt" - "strconv" - "time" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/client/stats" - "github.com/hashicorp/nomad/drivers/shared/eventer" - "github.com/hashicorp/nomad/plugins/base" - "github.com/hashicorp/nomad/plugins/drivers" - "github.com/hashicorp/nomad/plugins/shared/hclspec" - "github.com/hashicorp/nomad/plugins/shared/loader" - pstructs "github.com/hashicorp/nomad/plugins/shared/structs" - "gopkg.in/lxc/go-lxc.v2" -) - -const ( - // pluginName is the name of the plugin - pluginName = "lxc" - - // fingerprintPeriod is the interval at which the driver will send fingerprint responses - fingerprintPeriod = 30 * time.Second -) - -var ( - // PluginID is the rawexec plugin metadata registered in the plugin - // catalog. - PluginID = loader.PluginID{ - Name: pluginName, - PluginType: base.PluginTypeDriver, - } - - // PluginConfig is the rawexec factory function registered in the - // plugin catalog. - PluginConfig = &loader.InternalPluginConfig{ - Config: map[string]interface{}{}, - Factory: func(l hclog.Logger) interface{} { return NewLXCDriver(l) }, - } -) - -// PluginLoader maps pre-0.9 client driver options to post-0.9 plugin options. -func PluginLoader(opts map[string]string) (map[string]interface{}, error) { - conf := map[string]interface{}{} - if v, err := strconv.ParseBool(opts["driver.lxc.enable"]); err == nil { - conf["enabled"] = v - } - if v, err := strconv.ParseBool(opts["lxc.volumes.enabled"]); err == nil { - conf["volumes"] = v - } - if v, ok := opts["driver.lxc.path"]; ok { - conf["path"] = v - } - return conf, nil -} - -var ( - // pluginInfo is the response returned for the PluginInfo RPC - pluginInfo = &base.PluginInfoResponse{ - Type: base.PluginTypeDriver, - PluginApiVersions: []string{drivers.ApiVersion010}, - PluginVersion: "0.1.0", - Name: pluginName, - } - - // configSpec is the hcl specification returned by the ConfigSchema RPC - configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ - "enabled": hclspec.NewDefault( - hclspec.NewAttr("enabled", "bool", false), - hclspec.NewLiteral("true"), - ), - "volumes": hclspec.NewDefault( - hclspec.NewAttr("volumes", "bool", false), - hclspec.NewLiteral("true"), - ), - "path": hclspec.NewDefault( - hclspec.NewAttr("path", "string", false), - hclspec.NewLiteral("\"\""), - ), - }) - - // taskConfigSpec is the hcl specification for the driver config section of - // a task within a job. It is returned in the TaskConfigSchema RPC - taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{ - "template": hclspec.NewAttr("template", "string", true), - "distro": hclspec.NewAttr("distro", "string", false), - "release": hclspec.NewAttr("release", "string", false), - "arch": hclspec.NewAttr("arch", "string", false), - "image_variant": hclspec.NewAttr("image_variant", "string", false), - "image_server": hclspec.NewAttr("image_server", "string", false), - "gpg_key_id": hclspec.NewAttr("gpg_key_id", "string", false), - "gpg_key_server": hclspec.NewAttr("gpg_key_server", "string", false), - "disable_gpg": hclspec.NewAttr("disable_gpg", "string", false), - "flush_cache": hclspec.NewAttr("flush_cache", "string", false), - "force_cache": hclspec.NewAttr("force_cache", "string", false), - "template_args": hclspec.NewAttr("template_args", "list(string)", false), - "log_level": hclspec.NewAttr("log_level", "string", false), - "verbosity": hclspec.NewAttr("verbosity", "string", false), - "volumes": hclspec.NewAttr("volumes", "list(string)", false), - }) - - // capabilities is returned by the Capabilities RPC and indicates what - // optional features this driver supports - capabilities = &drivers.Capabilities{ - SendSignals: false, - Exec: false, - FSIsolation: drivers.FSIsolationImage, - } -) - -// Driver is a driver for running LXC containers -type Driver struct { - // eventer is used to handle multiplexing of TaskEvents calls such that an - // event can be broadcast to all callers - eventer *eventer.Eventer - - // config is the driver configuration set by the SetConfig RPC - config *Config - - // nomadConfig is the client config from nomad - nomadConfig *base.ClientDriverConfig - - // tasks is the in memory datastore mapping taskIDs to rawExecDriverHandles - tasks *taskStore - - // ctx is the context for the driver. It is passed to other subsystems to - // coordinate shutdown - ctx context.Context - - // signalShutdown is called when the driver is shutting down and cancels the - // ctx passed to any subsystems - signalShutdown context.CancelFunc - - // logger will log to the Nomad agent - logger hclog.Logger -} - -// Config is the driver configuration set by the SetConfig RPC call -type Config struct { - // Enabled is set to true to enable the lxc driver - Enabled bool `codec:"enabled"` - - AllowVolumes bool `codec:"volumes"` - - Path string `codec:"path"` -} - -// TaskConfig is the driver configuration of a task within a job -type TaskConfig struct { - Template string `codec:"template"` - Distro string `codec:"distro"` - Release string `codec:"release"` - Arch string `codec:"arch"` - ImageVariant string `codec:"image_variant"` - ImageServer string `codec:"image_server"` - GPGKeyID string `codec:"gpg_key_id"` - GPGKeyServer string `codec:"gpg_key_server"` - DisableGPGValidation bool `codec:"disable_gpg"` - FlushCache bool `codec:"flush_cache"` - ForceCache bool `codec:"force_cache"` - TemplateArgs []string `codec:"template_args"` - LogLevel string `codec:"log_level"` - Verbosity string `codec:"verbosity"` - Volumes []string `codec:"volumes"` -} - -// TaskState is the state which is encoded in the handle returned in -// StartTask. This information is needed to rebuild the task state and handler -// during recovery. -type TaskState struct { - TaskConfig *drivers.TaskConfig - ContainerName string - StartedAt time.Time -} - -// NewLXCDriver returns a new DriverPlugin implementation -func NewLXCDriver(logger hclog.Logger) drivers.DriverPlugin { - ctx, cancel := context.WithCancel(context.Background()) - logger = logger.Named(pluginName) - return &Driver{ - eventer: eventer.NewEventer(ctx, logger), - config: &Config{}, - tasks: newTaskStore(), - ctx: ctx, - signalShutdown: cancel, - logger: logger, - } -} - -func (d *Driver) PluginInfo() (*base.PluginInfoResponse, error) { - return pluginInfo, nil -} - -func (d *Driver) ConfigSchema() (*hclspec.Spec, error) { - return configSpec, nil -} - -func (d *Driver) SetConfig(cfg *base.Config) error { - var config Config - if len(cfg.PluginConfig) != 0 { - if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil { - return err - } - } - - d.config = &config - if cfg.AgentConfig != nil { - d.nomadConfig = cfg.AgentConfig.Driver - } - - return nil -} - -func (d *Driver) Shutdown() { - d.signalShutdown() -} - -func (d *Driver) TaskConfigSchema() (*hclspec.Spec, error) { - return taskConfigSpec, nil -} - -func (d *Driver) Capabilities() (*drivers.Capabilities, error) { - return capabilities, nil -} - -func (d *Driver) Fingerprint(ctx context.Context) (<-chan *drivers.Fingerprint, error) { - ch := make(chan *drivers.Fingerprint) - go d.handleFingerprint(ctx, ch) - return ch, nil -} - -func (d *Driver) handleFingerprint(ctx context.Context, ch chan<- *drivers.Fingerprint) { - defer close(ch) - ticker := time.NewTimer(0) - for { - select { - case <-ctx.Done(): - return - case <-d.ctx.Done(): - return - case <-ticker.C: - ticker.Reset(fingerprintPeriod) - ch <- d.buildFingerprint() - } - } -} - -func (d *Driver) buildFingerprint() *drivers.Fingerprint { - var health drivers.HealthState - var desc string - attrs := map[string]*pstructs.Attribute{} - - lxcVersion := lxc.Version() - - if d.config.Enabled && lxcVersion != "" { - health = drivers.HealthStateHealthy - desc = "ready" - attrs["driver.lxc"] = pstructs.NewBoolAttribute(true) - attrs["driver.lxc.version"] = pstructs.NewStringAttribute(lxcVersion) - } else { - health = drivers.HealthStateUndetected - desc = "disabled" - } - - if d.config.AllowVolumes { - attrs["driver.lxc.volumes.enabled"] = pstructs.NewBoolAttribute(true) - } - - return &drivers.Fingerprint{ - Attributes: attrs, - Health: health, - HealthDescription: desc, - } -} - -func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { - if handle == nil { - return fmt.Errorf("error: handle cannot be nil") - } - - if _, ok := d.tasks.Get(handle.Config.ID); ok { - return nil - } - - var taskState TaskState - if err := handle.GetDriverState(&taskState); err != nil { - return fmt.Errorf("failed to decode task state from handle: %v", err) - } - - var driverConfig TaskConfig - if err := taskState.TaskConfig.DecodeDriverConfig(&driverConfig); err != nil { - return fmt.Errorf("failed to decode driver config: %v", err) - } - - c, err := lxc.NewContainer(taskState.ContainerName, d.lxcPath()) - if err != nil { - return fmt.Errorf("failed to create container ref: %v", err) - } - - initPid := c.InitPid() - h := &taskHandle{ - container: c, - initPid: initPid, - taskConfig: taskState.TaskConfig, - procState: drivers.TaskStateRunning, - startedAt: taskState.StartedAt, - exitResult: &drivers.ExitResult{}, - - totalCpuStats: stats.NewCpuStats(), - userCpuStats: stats.NewCpuStats(), - systemCpuStats: stats.NewCpuStats(), - } - - d.tasks.Set(taskState.TaskConfig.ID, h) - - go h.run() - return nil -} - -func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) { - if _, ok := d.tasks.Get(cfg.ID); ok { - return nil, nil, fmt.Errorf("task with ID %q already started", cfg.ID) - } - - var driverConfig TaskConfig - if err := cfg.DecodeDriverConfig(&driverConfig); err != nil { - return nil, nil, fmt.Errorf("failed to decode driver config: %v", err) - } - - d.logger.Info("starting lxc task", "driver_cfg", hclog.Fmt("%+v", driverConfig)) - handle := drivers.NewTaskHandle(pluginName) - handle.Config = cfg - - c, err := d.initializeContainer(cfg, driverConfig) - if err != nil { - return nil, nil, err - } - - opt := toLXCCreateOptions(driverConfig) - if err := c.Create(opt); err != nil { - return nil, nil, fmt.Errorf("unable to create container: %v", err) - } - - cleanup := func() { - if err := c.Destroy(); err != nil { - d.logger.Error("failed to clean up from an error in Start", "error", err) - } - } - - if err := d.configureContainerNetwork(c); err != nil { - cleanup() - return nil, nil, err - } - - if err := d.mountVolumes(c, cfg, driverConfig); err != nil { - cleanup() - return nil, nil, err - } - - if err := c.Start(); err != nil { - cleanup() - return nil, nil, fmt.Errorf("unable to start container: %v", err) - } - - if err := d.setResourceLimits(c, cfg); err != nil { - cleanup() - return nil, nil, err - } - - pid := c.InitPid() - - h := &taskHandle{ - container: c, - initPid: pid, - taskConfig: cfg, - procState: drivers.TaskStateRunning, - startedAt: time.Now().Round(time.Millisecond), - logger: d.logger, - - totalCpuStats: stats.NewCpuStats(), - userCpuStats: stats.NewCpuStats(), - systemCpuStats: stats.NewCpuStats(), - } - - driverState := TaskState{ - ContainerName: c.Name(), - TaskConfig: cfg, - StartedAt: h.startedAt, - } - - if err := handle.SetDriverState(&driverState); err != nil { - d.logger.Error("failed to start task, error setting driver state", "error", err) - cleanup() - return nil, nil, fmt.Errorf("failed to set driver state: %v", err) - } - - d.tasks.Set(cfg.ID, h) - - go h.run() - - return handle, nil, nil -} - -func (d *Driver) WaitTask(ctx context.Context, taskID string) (<-chan *drivers.ExitResult, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - ch := make(chan *drivers.ExitResult) - go d.handleWait(ctx, handle, ch) - - return ch, nil -} - -func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *drivers.ExitResult) { - defer close(ch) - - // - // Wait for process completion by polling status from handler. - // We cannot use the following alternatives: - // * Process.Wait() requires LXC container processes to be children - // of self process; but LXC runs container in separate PID hierarchy - // owned by PID 1. - // * lxc.Container.Wait() holds a write lock on container and prevents - // any other calls, including stats. - // - // Going with simplest approach of polling for handler to mark exit. - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-d.ctx.Done(): - return - case <-ticker.C: - s := handle.TaskStatus() - if s.State == drivers.TaskStateExited { - ch <- handle.exitResult - } - } - } -} - -func (d *Driver) StopTask(taskID string, timeout time.Duration, signal string) error { - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - if err := handle.shutdown(timeout); err != nil { - return fmt.Errorf("executor Shutdown failed: %v", err) - } - - return nil -} - -func (d *Driver) DestroyTask(taskID string, force bool) error { - handle, ok := d.tasks.Get(taskID) - if !ok { - return drivers.ErrTaskNotFound - } - - if handle.IsRunning() && !force { - return fmt.Errorf("cannot destroy running task") - } - - if handle.IsRunning() { - // grace period is chosen arbitrary here - if err := handle.shutdown(1 * time.Minute); err != nil { - handle.logger.Error("failed to destroy executor", "err", err) - } - } - - d.tasks.Delete(taskID) - return nil -} - -func (d *Driver) InspectTask(taskID string) (*drivers.TaskStatus, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - return handle.TaskStatus(), nil -} - -func (d *Driver) TaskStats(taskID string) (*drivers.TaskResourceUsage, error) { - handle, ok := d.tasks.Get(taskID) - if !ok { - return nil, drivers.ErrTaskNotFound - } - - return handle.stats() -} - -func (d *Driver) TaskEvents(ctx context.Context) (<-chan *drivers.TaskEvent, error) { - return d.eventer.TaskEvents(ctx) -} - -func (d *Driver) SignalTask(taskID string, signal string) error { - return fmt.Errorf("LXC driver does not support signals") -} - -func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) { - return nil, fmt.Errorf("LXC driver does not support exec") -} diff --git a/drivers/lxc/driver_test.go b/drivers/lxc/driver_test.go deleted file mode 100644 index 313c9e480..000000000 --- a/drivers/lxc/driver_test.go +++ /dev/null @@ -1,280 +0,0 @@ -// +build linux,lxc - -package lxc - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - "github.com/hashicorp/hcl2/hcl" - ctestutil "github.com/hashicorp/nomad/client/testutil" - "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/plugins/drivers" - dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" - "github.com/hashicorp/nomad/plugins/shared/hclspec" - "github.com/hashicorp/nomad/plugins/shared/hclutils" - "github.com/hashicorp/nomad/testutil" - "github.com/stretchr/testify/require" - lxc "gopkg.in/lxc/go-lxc.v2" -) - -func TestLXCDriver_Fingerprint(t *testing.T) { - t.Parallel() - requireLXC(t) - - require := require.New(t) - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = true - harness := dtestutil.NewDriverHarness(t, d) - - fingerCh, err := harness.Fingerprint(context.Background()) - require.NoError(err) - select { - case finger := <-fingerCh: - require.Equal(drivers.HealthStateHealthy, finger.Health) - require.True(finger.Attributes["driver.lxc"].GetBool()) - require.NotEmpty(finger.Attributes["driver.lxc.version"].GetString()) - case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): - require.Fail("timeout receiving fingerprint") - } -} - -func TestLXCDriver_FingerprintNotEnabled(t *testing.T) { - t.Parallel() - requireLXC(t) - - require := require.New(t) - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = false - harness := dtestutil.NewDriverHarness(t, d) - - fingerCh, err := harness.Fingerprint(context.Background()) - require.NoError(err) - select { - case finger := <-fingerCh: - require.Equal(drivers.HealthStateUndetected, finger.Health) - require.Empty(finger.Attributes["driver.lxc"]) - require.Empty(finger.Attributes["driver.lxc.version"]) - case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): - require.Fail("timeout receiving fingerprint") - } -} - -func TestLXCDriver_Start_Wait(t *testing.T) { - if !testutil.IsTravis() { - t.Parallel() - } - requireLXC(t) - ctestutil.RequireRoot(t) - - require := require.New(t) - - // prepare test file - testFileContents := []byte("this should be visible under /mnt/tmp") - tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait") - if err != nil { - t.Fatalf("error writing temp file: %v", err) - } - defer os.Remove(tmpFile.Name()) - if _, err := tmpFile.Write(testFileContents); err != nil { - t.Fatalf("error writing temp file: %v", err) - } - if err := tmpFile.Close(); err != nil { - t.Fatalf("error closing temp file: %v", err) - } - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = true - d.config.AllowVolumes = true - - harness := dtestutil.NewDriverHarness(t, d) - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - Name: "test", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - }, - LinuxResources: &drivers.LinuxResources{ - CPUShares: 1024, - MemoryLimitBytes: 2 * 1024, - }, - }, - } - taskConfig := map[string]interface{}{ - "template": "/usr/share/lxc/templates/lxc-busybox", - "volumes": []string{"/tmp/:mnt/tmp"}, - } - encodeDriverHelper(require, task, taskConfig) - - cleanup := harness.MkAllocDir(task, false) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - require.NotNil(handle) - - lxcHandle, ok := d.tasks.Get(task.ID) - require.True(ok) - - container := lxcHandle.container - - // Destroy container after test - defer func() { - container.Stop() - container.Destroy() - }() - - // Test that container is running - testutil.WaitForResult(func() (bool, error) { - state := container.State() - if state == lxc.RUNNING { - return true, nil - } - return false, fmt.Errorf("container in state: %v", state) - }, func(err error) { - t.Fatalf("container failed to start: %v", err) - }) - - // Test that directories are mounted in their proper location - containerName := container.Name() - for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} { - fullpath := filepath.Join(d.lxcPath(), containerName, "rootfs", mnt) - stat, err := os.Stat(fullpath) - require.NoError(err) - require.True(stat.IsDir()) - } - - // Test bind mount volumes exist in container: - mountedContents, err := exec.Command("lxc-attach", - "-n", containerName, "--", - "cat", filepath.Join("/mnt/", tmpFile.Name()), - ).Output() - require.NoError(err) - require.Equal(string(testFileContents), string(mountedContents)) - - // Test that killing container marks container as stopped - require.NoError(container.Stop()) - - testutil.WaitForResult(func() (bool, error) { - status, err := d.InspectTask(task.ID) - if err == nil && status.State == drivers.TaskStateExited { - return true, nil - } - return false, fmt.Errorf("task in state: %v", status.State) - }, func(err error) { - t.Fatalf("task was not marked as stopped: %v", err) - }) -} - -func TestLXCDriver_Start_Stop(t *testing.T) { - if !testutil.IsTravis() { - t.Parallel() - } - requireLXC(t) - ctestutil.RequireRoot(t) - - require := require.New(t) - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = true - d.config.AllowVolumes = true - - harness := dtestutil.NewDriverHarness(t, d) - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - Name: "test", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - }, - LinuxResources: &drivers.LinuxResources{ - CPUShares: 1024, - MemoryLimitBytes: 2 * 1024, - }, - }, - } - taskConfig := map[string]interface{}{ - "template": "/usr/share/lxc/templates/lxc-busybox", - } - encodeDriverHelper(require, task, taskConfig) - - cleanup := harness.MkAllocDir(task, false) - defer cleanup() - - handle, _, err := harness.StartTask(task) - require.NoError(err) - require.NotNil(handle) - - lxcHandle, ok := d.tasks.Get(task.ID) - require.True(ok) - - container := lxcHandle.container - - // Destroy container after test - defer func() { - container.Stop() - container.Destroy() - }() - - // Test that container is running - testutil.WaitForResult(func() (bool, error) { - state := container.State() - if state == lxc.RUNNING { - return true, nil - } - return false, fmt.Errorf("container in state: %v", state) - }, func(err error) { - t.Fatalf("container failed to start: %v", err) - }) - - require.NoError(d.StopTask(task.ID, 5*time.Second, "kill")) - - testutil.WaitForResult(func() (bool, error) { - status, err := d.InspectTask(task.ID) - if err == nil && status.State == drivers.TaskStateExited { - return true, nil - } - return false, fmt.Errorf("task in state: %v", status.State) - }, func(err error) { - t.Fatalf("task was not marked as stopped: %v", err) - }) -} - -func requireLXC(t *testing.T) { - if lxc.Version() == "" { - t.Skip("skipping, lxc not present") - } -} - -func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) { - evalCtx := &hcl.EvalContext{ - Functions: hclutils.GetStdlibFuncs(), - } - spec, diag := hclspec.Convert(taskConfigSpec) - require.False(diag.HasErrors()) - taskConfigCtyVal, diag := hclutils.ParseHclInterface(taskConfig, spec, evalCtx) - require.False(diag.HasErrors()) - err := task.EncodeDriverConfig(taskConfigCtyVal) - require.Nil(err) -} diff --git a/drivers/lxc/handle.go b/drivers/lxc/handle.go deleted file mode 100644 index 059f5d3db..000000000 --- a/drivers/lxc/handle.go +++ /dev/null @@ -1,199 +0,0 @@ -//+build linux,lxc - -package lxc - -import ( - "fmt" - "strconv" - "strings" - "sync" - "time" - - hclog "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/client/stats" - "github.com/hashicorp/nomad/plugins/drivers" - lxc "gopkg.in/lxc/go-lxc.v2" -) - -type taskHandle struct { - container *lxc.Container - initPid int - logger hclog.Logger - - totalCpuStats *stats.CpuStats - userCpuStats *stats.CpuStats - systemCpuStats *stats.CpuStats - - // stateLock syncs access to all fields below - stateLock sync.RWMutex - - taskConfig *drivers.TaskConfig - procState drivers.TaskState - startedAt time.Time - completedAt time.Time - exitResult *drivers.ExitResult -} - -var ( - LXCMeasuredCpuStats = []string{"System Mode", "User Mode", "Percent"} - - LXCMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"} -) - -func (h *taskHandle) TaskStatus() *drivers.TaskStatus { - h.stateLock.RLock() - defer h.stateLock.RUnlock() - - return &drivers.TaskStatus{ - ID: h.taskConfig.ID, - Name: h.taskConfig.Name, - State: h.procState, - StartedAt: h.startedAt, - CompletedAt: h.completedAt, - ExitResult: h.exitResult, - DriverAttributes: map[string]string{ - "pid": strconv.Itoa(h.initPid), - }, - } -} - -func (h *taskHandle) IsRunning() bool { - h.stateLock.RLock() - defer h.stateLock.RUnlock() - return h.procState == drivers.TaskStateRunning -} - -func (h *taskHandle) run() { - h.stateLock.Lock() - if h.exitResult == nil { - h.exitResult = &drivers.ExitResult{} - } - h.stateLock.Unlock() - - if ok, err := waitTillStopped(h.container); !ok { - h.logger.Error("failed to find container process", "error", err) - return - } - - h.stateLock.Lock() - defer h.stateLock.Unlock() - - h.procState = drivers.TaskStateExited - h.exitResult.ExitCode = 0 - h.exitResult.Signal = 0 - h.completedAt = time.Now() - - // TODO: detect if the task OOMed -} - -func (h *taskHandle) stats() (*drivers.TaskResourceUsage, error) { - cpuStats, err := h.container.CPUStats() - if err != nil { - h.logger.Error("failed to get container cpu stats", "error", err) - return nil, nil - } - total, err := h.container.CPUTime() - if err != nil { - h.logger.Error("failed to get container cpu time", "error", err) - return nil, nil - } - - t := time.Now() - - // Get the cpu stats - system := cpuStats["system"] - user := cpuStats["user"] - cs := &drivers.CpuStats{ - SystemMode: h.systemCpuStats.Percent(float64(system)), - UserMode: h.systemCpuStats.Percent(float64(user)), - Percent: h.totalCpuStats.Percent(float64(total)), - TotalTicks: float64(user + system), - Measured: LXCMeasuredCpuStats, - } - - // Get the Memory Stats - memData := map[string]uint64{ - "rss": 0, - "cache": 0, - "swap": 0, - } - rawMemStats := h.container.CgroupItem("memory.stat") - for _, rawMemStat := range rawMemStats { - key, val, err := keysToVal(rawMemStat) - if err != nil { - h.logger.Error("failed to get stat", "line", rawMemStat, "error", err) - continue - } - if _, ok := memData[key]; ok { - memData[key] = val - - } - } - ms := &drivers.MemoryStats{ - RSS: memData["rss"], - Cache: memData["cache"], - Swap: memData["swap"], - Measured: LXCMeasuredMemStats, - } - - mu := h.container.CgroupItem("memory.max_usage_in_bytes") - for _, rawMemMaxUsage := range mu { - val, err := strconv.ParseUint(rawMemMaxUsage, 10, 64) - if err != nil { - h.logger.Error("failed to get max memory usage", "error", err) - continue - } - ms.MaxUsage = val - } - ku := h.container.CgroupItem("memory.kmem.usage_in_bytes") - for _, rawKernelUsage := range ku { - val, err := strconv.ParseUint(rawKernelUsage, 10, 64) - if err != nil { - h.logger.Error("failed to get kernel memory usage", "error", err) - continue - } - ms.KernelUsage = val - } - - mku := h.container.CgroupItem("memory.kmem.max_usage_in_bytes") - for _, rawMaxKernelUsage := range mku { - val, err := strconv.ParseUint(rawMaxKernelUsage, 10, 64) - if err != nil { - h.logger.Error("failed tog get max kernel memory usage", "error", err) - continue - } - ms.KernelMaxUsage = val - } - - taskResUsage := drivers.TaskResourceUsage{ - ResourceUsage: &drivers.ResourceUsage{ - CpuStats: cs, - MemoryStats: ms, - }, - Timestamp: t.UTC().UnixNano(), - } - - return &taskResUsage, nil - -} - -func keysToVal(line string) (string, uint64, error) { - tokens := strings.Split(line, " ") - if len(tokens) != 2 { - return "", 0, fmt.Errorf("line isn't a k/v pair") - } - key := tokens[0] - val, err := strconv.ParseUint(tokens[1], 10, 64) - return key, val, err -} - -// shutdown shuts down the container, with `timeout` grace period -// before killing the container with SIGKILL. -func (h *taskHandle) shutdown(timeout time.Duration) error { - err := h.container.Shutdown(timeout) - if err == nil { - return nil - } - - return h.container.Stop() -} diff --git a/drivers/lxc/lxc.go b/drivers/lxc/lxc.go deleted file mode 100644 index c1431591b..000000000 --- a/drivers/lxc/lxc.go +++ /dev/null @@ -1,253 +0,0 @@ -//+build linux,lxc - -package lxc - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "syscall" - "time" - - "github.com/hashicorp/nomad/plugins/drivers" - ldevices "github.com/opencontainers/runc/libcontainer/devices" - "gopkg.in/lxc/go-lxc.v2" -) - -var ( - verbosityLevels = map[string]lxc.Verbosity{ - "": lxc.Quiet, - "verbose": lxc.Verbose, - "quiet": lxc.Quiet, - } - - logLevels = map[string]lxc.LogLevel{ - "": lxc.ERROR, - "debug": lxc.DEBUG, - "error": lxc.ERROR, - "info": lxc.INFO, - "trace": lxc.TRACE, - "warn": lxc.WARN, - } -) - -const ( - // containerMonitorIntv is the interval at which the driver checks if the - // container is still alive - containerMonitorIntv = 2 * time.Second -) - -func (d *Driver) lxcPath() string { - lxcPath := d.config.Path - if lxcPath == "" { - lxcPath = lxc.DefaultConfigPath() - } - return lxcPath - -} -func (d *Driver) initializeContainer(cfg *drivers.TaskConfig, taskConfig TaskConfig) (*lxc.Container, error) { - lxcPath := d.lxcPath() - - containerName := fmt.Sprintf("%s-%s", cfg.Name, cfg.AllocID) - c, err := lxc.NewContainer(containerName, lxcPath) - if err != nil { - return nil, fmt.Errorf("failed to initialize container: %v", err) - } - - if v, ok := verbosityLevels[taskConfig.Verbosity]; ok { - c.SetVerbosity(v) - } else { - return nil, fmt.Errorf("lxc driver config 'verbosity' can only be either quiet or verbose") - } - - if v, ok := logLevels[taskConfig.LogLevel]; ok { - c.SetLogLevel(v) - } else { - return nil, fmt.Errorf("lxc driver config 'log_level' can only be trace, debug, info, warn or error") - } - - logFile := filepath.Join(cfg.TaskDir().Dir, fmt.Sprintf("%v-lxc.log", cfg.Name)) - c.SetLogFile(logFile) - - return c, nil -} - -func (d *Driver) configureContainerNetwork(c *lxc.Container) error { - // Set the network type to none - if err := c.SetConfigItem(networkTypeConfigKey(), "none"); err != nil { - return fmt.Errorf("error setting network type configuration: %v", err) - } - return nil -} - -func networkTypeConfigKey() string { - if lxc.VersionAtLeast(2, 1, 0) { - return "lxc.net.0.type" - } - - // prior to 2.1, network used - return "lxc.network.type" -} - -func (d *Driver) mountVolumes(c *lxc.Container, cfg *drivers.TaskConfig, taskConfig TaskConfig) error { - mounts, err := d.mountEntries(cfg, taskConfig) - if err != nil { - return err - } - - devCgroupAllows, err := d.devicesCgroupEntries(cfg) - if err != nil { - return err - } - - for _, mnt := range mounts { - if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil { - return fmt.Errorf("error setting bind mount %q error: %v", mnt, err) - } - } - - for _, cgroupDev := range devCgroupAllows { - if err := c.SetConfigItem("lxc.cgroup.devices.allow", cgroupDev); err != nil { - return fmt.Errorf("error setting cgroup permission %q error: %v", cgroupDev, err) - } - } - - return nil -} - -// mountEntries compute the mount entries to be set on the container -func (d *Driver) mountEntries(cfg *drivers.TaskConfig, taskConfig TaskConfig) ([]string, error) { - // Bind mount the shared alloc dir and task local dir in the container - mounts := []string{ - fmt.Sprintf("%s local none rw,bind,create=dir", cfg.TaskDir().LocalDir), - fmt.Sprintf("%s alloc none rw,bind,create=dir", cfg.TaskDir().SharedAllocDir), - fmt.Sprintf("%s secrets none rw,bind,create=dir", cfg.TaskDir().SecretsDir), - } - - mounts = append(mounts, d.formatTaskMounts(cfg.Mounts)...) - mounts = append(mounts, d.formatTaskDevices(cfg.Devices)...) - - volumesEnabled := d.config.AllowVolumes - - for _, volDesc := range taskConfig.Volumes { - // the format was checked in Validate() - paths := strings.Split(volDesc, ":") - - if filepath.IsAbs(paths[0]) { - if !volumesEnabled { - return nil, fmt.Errorf("absolute bind-mount volume in config but volumes are disabled") - } - } else { - // Relative source paths are treated as relative to alloc dir - paths[0] = filepath.Join(cfg.TaskDir().Dir, paths[0]) - } - - // LXC assumes paths are relative with respect to rootfs - target := strings.TrimLeft(paths[1], "/") - mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], target)) - } - - return mounts, nil - -} - -func (d *Driver) devicesCgroupEntries(cfg *drivers.TaskConfig) ([]string, error) { - entries := make([]string, len(cfg.Devices)) - - for i, d := range cfg.Devices { - hd, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions) - if err != nil { - return nil, err - } - - entries[i] = hd.CgroupString() - } - - return entries, nil -} - -func (d *Driver) formatTaskMounts(mounts []*drivers.MountConfig) []string { - result := make([]string, len(mounts)) - - for i, m := range mounts { - result[i] = d.formatMount(m.HostPath, m.TaskPath, m.Readonly) - } - - return result -} - -func (d *Driver) formatTaskDevices(devices []*drivers.DeviceConfig) []string { - result := make([]string, len(devices)) - - for i, m := range devices { - result[i] = d.formatMount(m.HostPath, m.TaskPath, - !strings.Contains(m.Permissions, "w")) - } - - return result -} - -func (d *Driver) formatMount(hostPath, taskPath string, readOnly bool) string { - typ := "dir" - s, err := os.Stat(hostPath) - if err != nil { - d.logger.Warn("failed to find mount host path type, defaulting to dir type", "path", hostPath, "error", err) - } else if !s.IsDir() { - typ = "file" - } - - perm := "rw" - if readOnly { - perm = "ro" - } - - // LXC assumes paths are relative with respect to rootfs - target := strings.TrimLeft(taskPath, "/") - return fmt.Sprintf("%s %s none %s,bind,create=%s", hostPath, target, perm, typ) -} - -func (d *Driver) setResourceLimits(c *lxc.Container, cfg *drivers.TaskConfig) error { - if err := c.SetMemoryLimit(lxc.ByteSize(cfg.Resources.NomadResources.Memory.MemoryMB) * lxc.MB); err != nil { - return fmt.Errorf("unable to set memory limits: %v", err) - } - - if err := c.SetCgroupItem("cpu.shares", strconv.FormatInt(cfg.Resources.LinuxResources.CPUShares, 10)); err != nil { - return fmt.Errorf("unable to set cpu shares: %v", err) - } - - return nil -} - -func toLXCCreateOptions(taskConfig TaskConfig) lxc.TemplateOptions { - return lxc.TemplateOptions{ - Template: taskConfig.Template, - Distro: taskConfig.Distro, - Release: taskConfig.Release, - Arch: taskConfig.Arch, - FlushCache: taskConfig.FlushCache, - DisableGPGValidation: taskConfig.DisableGPGValidation, - ExtraArgs: taskConfig.TemplateArgs, - } -} - -// waitTillStopped blocks and returns true when container stops; -// returns false with an error message if the container processes cannot be identified. -// -// Use this in preference to c.Wait() - lxc Wait() function holds a write lock on the container -// blocking any other operation on container, including looking up container stats -func waitTillStopped(c *lxc.Container) (bool, error) { - ps, err := os.FindProcess(c.InitPid()) - if err != nil { - return false, err - } - - for { - if err := ps.Signal(syscall.Signal(0)); err != nil { - return true, nil - } - - time.Sleep(containerMonitorIntv) - } -} diff --git a/drivers/lxc/lxc_test.go b/drivers/lxc/lxc_test.go deleted file mode 100644 index 2aed490a7..000000000 --- a/drivers/lxc/lxc_test.go +++ /dev/null @@ -1,95 +0,0 @@ -//+build linux,lxc - -package lxc - -import ( - "testing" - - "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/plugins/drivers" - "github.com/stretchr/testify/require" -) - -func TestLXCDriver_Mounts(t *testing.T) { - t.Parallel() - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - Name: "test", - Resources: &drivers.Resources{ - NomadResources: &structs.AllocatedTaskResources{ - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2, - }, - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - }, - LinuxResources: &drivers.LinuxResources{ - CPUShares: 1024, - MemoryLimitBytes: 2 * 1024, - }, - }, - Mounts: []*drivers.MountConfig{ - {HostPath: "/dev", TaskPath: "/task-mounts/dev-path"}, - {HostPath: "/bin/sh", TaskPath: "/task-mounts/task-path-ro", Readonly: true}, - }, - Devices: []*drivers.DeviceConfig{ - {HostPath: "/dev", TaskPath: "/task-devices/dev-path", Permissions: "rw"}, - {HostPath: "/bin/sh", TaskPath: "/task-devices/task-path-ro", Permissions: "ro"}, - }, - } - taskConfig := TaskConfig{ - Template: "busybox", - Volumes: []string{ - "relative/path:/usr-config/container/path", - "relative/path2:usr-config/container/relative", - }, - } - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = true - - entries, err := d.mountEntries(task, taskConfig) - require.NoError(t, err) - - expectedEntries := []string{ - "test/relative/path usr-config/container/path none rw,bind,create=dir", - "test/relative/path2 usr-config/container/relative none rw,bind,create=dir", - "/dev task-mounts/dev-path none rw,bind,create=dir", - "/bin/sh task-mounts/task-path-ro none ro,bind,create=file", - "/dev task-devices/dev-path none rw,bind,create=dir", - "/bin/sh task-devices/task-path-ro none ro,bind,create=file", - } - - for _, e := range expectedEntries { - require.Contains(t, entries, e) - } -} - -func TestLXCDriver_DevicesCgroup(t *testing.T) { - t.Parallel() - - task := &drivers.TaskConfig{ - ID: uuid.Generate(), - Name: "test", - Devices: []*drivers.DeviceConfig{ - {HostPath: "/dev/random", TaskPath: "/task-devices/devrandom", Permissions: "rw"}, - {HostPath: "/dev/null", TaskPath: "/task-devices/devnull", Permissions: "rwm"}, - }, - } - - d := NewLXCDriver(testlog.HCLogger(t)).(*Driver) - d.config.Enabled = true - - cgroupEntries, err := d.devicesCgroupEntries(task) - require.NoError(t, err) - - expected := []string{ - "c 1:8 rw", - "c 1:3 rwm", - } - require.EqualValues(t, expected, cgroupEntries) -} diff --git a/drivers/lxc/state.go b/drivers/lxc/state.go deleted file mode 100644 index c316a6cc6..000000000 --- a/drivers/lxc/state.go +++ /dev/null @@ -1,35 +0,0 @@ -//+build linux,lxc - -package lxc - -import ( - "sync" -) - -type taskStore struct { - store map[string]*taskHandle - lock sync.RWMutex -} - -func newTaskStore() *taskStore { - return &taskStore{store: map[string]*taskHandle{}} -} - -func (ts *taskStore) Set(id string, handle *taskHandle) { - ts.lock.Lock() - defer ts.lock.Unlock() - ts.store[id] = handle -} - -func (ts *taskStore) Get(id string) (*taskHandle, bool) { - ts.lock.RLock() - defer ts.lock.RUnlock() - t, ok := ts.store[id] - return t, ok -} - -func (ts *taskStore) Delete(id string) { - ts.lock.Lock() - defer ts.lock.Unlock() - delete(ts.store, id) -} diff --git a/plugins/shared/catalog/register_lxc.go b/plugins/shared/catalog/register_lxc.go deleted file mode 100644 index 6a029ec3f..000000000 --- a/plugins/shared/catalog/register_lxc.go +++ /dev/null @@ -1,14 +0,0 @@ -//+build linux,lxc - -package catalog - -import ( - "github.com/hashicorp/nomad/drivers/lxc" -) - -// This file is where all builtin plugins should be registered in the catalog. -// Plugins with build restrictions should be placed in the appropriate -// register_XXX.go file. -func init() { - RegisterDeferredConfig(lxc.PluginID, lxc.PluginConfig, lxc.PluginLoader) -} diff --git a/scripts/travis-linux.sh b/scripts/travis-linux.sh index aeb022189..6f25b9b3c 100644 --- a/scripts/travis-linux.sh +++ b/scripts/travis-linux.sh @@ -10,8 +10,7 @@ sudo service docker restart # true errors would fail in the apt-get install phase apt-get update || true -apt-get install -y liblxc1 lxc-dev lxc lxc-templates shellcheck -apt-get install -y qemu +apt-get install -y qemu shellcheck bash ./scripts/travis-rkt.sh bash ./scripts/travis-consul.sh bash ./scripts/travis-vault.sh diff --git a/scripts/vagrant-linux-priv-config.sh b/scripts/vagrant-linux-priv-config.sh index 4c23f41b5..7c8b38c52 100755 --- a/scripts/vagrant-linux-priv-config.sh +++ b/scripts/vagrant-linux-priv-config.sh @@ -26,11 +26,8 @@ apt-get install -y \ build-essential \ git \ libc6-dev-i386 \ - liblxc1 \ libpcre3-dev \ linux-libc-dev:i386 \ - lxc-dev \ - lxc-templates \ pkg-config \ zip