mirror of
https://github.com/kemko/nomad.git
synced 2026-01-06 18:35:44 +03:00
Moved packages around
This commit is contained in:
Diptanu Choudhury
277
client/driver/executor/executor_linux.go
Normal file
277
client/driver/executor/executor_linux.go
Normal file
@@ -0,0 +1,277 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups"
|
||||
cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
|
||||
"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
|
||||
cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
|
||||
|
||||
"github.com/hashicorp/nomad/client/allocdir"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
)
|
||||
|
||||
var (
|
||||
// A mapping of directories on the host OS to attempt to embed inside each
|
||||
// task's chroot.
|
||||
chrootEnv = map[string]string{
|
||||
"/bin": "/bin",
|
||||
"/etc": "/etc",
|
||||
"/lib": "/lib",
|
||||
"/lib32": "/lib32",
|
||||
"/lib64": "/lib64",
|
||||
"/usr/bin": "/usr/bin",
|
||||
"/usr/lib": "/usr/lib",
|
||||
"/usr/share": "/usr/share",
|
||||
}
|
||||
)
|
||||
|
||||
func (e *UniversalExecutor) configureIsolation() error {
|
||||
if e.ctx.FSIsolation {
|
||||
if err := e.configureChroot(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if e.ctx.ResourceLimits {
|
||||
if err := e.configureCgroups(e.ctx.TaskResources); err != nil {
|
||||
return fmt.Errorf("error creating cgroups: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) applyLimits() error {
|
||||
if !e.ctx.ResourceLimits {
|
||||
return nil
|
||||
}
|
||||
manager := e.getCgroupManager(e.groups)
|
||||
if err := manager.Apply(e.cmd.Process.Pid); err != nil {
|
||||
e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
|
||||
if err := e.Exit(); err != nil {
|
||||
e.logger.Printf("[ERROR] unable to kill process: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// configureCgroups converts a Nomad Resources specification into the equivalent
|
||||
// cgroup configuration. It returns an error if the resources are invalid.
|
||||
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
|
||||
e.groups = &cgroupConfig.Cgroup{}
|
||||
e.groups.Resources = &cgroupConfig.Resources{}
|
||||
e.groups.Name = structs.GenerateUUID()
|
||||
|
||||
// TODO: verify this is needed for things like network access
|
||||
e.groups.Resources.AllowAllDevices = true
|
||||
|
||||
if resources.MemoryMB > 0 {
|
||||
// Total amount of memory allowed to consume
|
||||
e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
|
||||
// Disable swap to avoid issues on the machine
|
||||
e.groups.Resources.MemorySwap = int64(-1)
|
||||
}
|
||||
|
||||
if resources.CPU < 2 {
|
||||
return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
|
||||
}
|
||||
|
||||
// Set the relative CPU shares for this cgroup.
|
||||
e.groups.Resources.CpuShares = int64(resources.CPU)
|
||||
|
||||
if resources.IOPS != 0 {
|
||||
// Validate it is in an acceptable range.
|
||||
if resources.IOPS < 10 || resources.IOPS > 1000 {
|
||||
return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
|
||||
}
|
||||
|
||||
e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// runAs takes a user id as a string and looks up the user, and sets the command
|
||||
// to execute as that user.
|
||||
func (e *UniversalExecutor) runAs(userid string) error {
|
||||
u, err := user.Lookup(userid)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to identify user %v: %v", userid, err)
|
||||
}
|
||||
|
||||
// Convert the uid and gid
|
||||
uid, err := strconv.ParseUint(u.Uid, 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to convert userid to uint32: %s", err)
|
||||
}
|
||||
gid, err := strconv.ParseUint(u.Gid, 10, 32)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
|
||||
}
|
||||
|
||||
// Set the command to run as that user and group.
|
||||
if e.cmd.SysProcAttr == nil {
|
||||
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
|
||||
}
|
||||
if e.cmd.SysProcAttr.Credential == nil {
|
||||
e.cmd.SysProcAttr.Credential = &syscall.Credential{}
|
||||
}
|
||||
e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
|
||||
e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pathExists is a helper function to check if the path exists.
|
||||
func (e *UniversalExecutor) pathExists(path string) bool {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (e *UniversalExecutor) configureChroot() error {
|
||||
allocDir := e.ctx.AllocDir
|
||||
if err := allocDir.MountSharedDir(e.ctx.TaskName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := allocDir.Embed(e.ctx.TaskName, chrootEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Mount dev
|
||||
dev := filepath.Join(e.taskDir, "dev")
|
||||
if !e.pathExists(dev) {
|
||||
if err := os.Mkdir(dev, 0777); err != nil {
|
||||
return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
|
||||
}
|
||||
|
||||
if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
|
||||
return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Mount proc
|
||||
proc := filepath.Join(e.taskDir, "proc")
|
||||
if !e.pathExists(proc) {
|
||||
if err := os.Mkdir(proc, 0777); err != nil {
|
||||
return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
|
||||
}
|
||||
|
||||
if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
|
||||
return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set the tasks AllocDir environment variable.
|
||||
e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
|
||||
|
||||
if e.cmd.SysProcAttr == nil {
|
||||
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
|
||||
}
|
||||
|
||||
e.cmd.SysProcAttr.Chroot = e.taskDir
|
||||
e.cmd.Dir = "/"
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanTaskDir is an idempotent operation to clean the task directory and
|
||||
// should be called when tearing down the task.
|
||||
func (e *UniversalExecutor) removeChrootMounts() error {
|
||||
// Prevent a race between Wait/ForceStop
|
||||
e.lock.Lock()
|
||||
defer e.lock.Unlock()
|
||||
|
||||
// Unmount dev.
|
||||
errs := new(multierror.Error)
|
||||
dev := filepath.Join(e.taskDir, "dev")
|
||||
if e.pathExists(dev) {
|
||||
if err := syscall.Unmount(dev, 0); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(dev); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
|
||||
}
|
||||
}
|
||||
|
||||
// Unmount
|
||||
// proc.
|
||||
proc := filepath.Join(e.taskDir, "proc")
|
||||
if e.pathExists(proc) {
|
||||
if err := syscall.Unmount(proc, 0); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(proc); err != nil {
|
||||
errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
|
||||
}
|
||||
}
|
||||
|
||||
return errs.ErrorOrNil()
|
||||
}
|
||||
|
||||
// destroyCgroup kills all processes in the cgroup and removes the cgroup
|
||||
// configuration from the host.
|
||||
func (e *UniversalExecutor) destroyCgroup() error {
|
||||
if e.groups == nil {
|
||||
return fmt.Errorf("Can't destroy: cgroup configuration empty")
|
||||
}
|
||||
|
||||
// Prevent a race between Wait/ForceStop
|
||||
e.lock.Lock()
|
||||
defer e.lock.Unlock()
|
||||
|
||||
manager := e.getCgroupManager(e.groups)
|
||||
pids, err := manager.GetPids()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
|
||||
}
|
||||
|
||||
errs := new(multierror.Error)
|
||||
for _, pid := range pids {
|
||||
process, err := os.FindProcess(pid)
|
||||
if err != nil {
|
||||
multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if err := process.Kill(); err != nil && err.Error() != "os: process already finished" {
|
||||
multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the cgroup.
|
||||
if err := manager.Destroy(); err != nil {
|
||||
multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
|
||||
}
|
||||
|
||||
if len(errs.Errors) != 0 {
|
||||
return fmt.Errorf("Failed to destroy cgroup: %v", errs)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getCgroupManager returns the correct libcontainer cgroup manager.
|
||||
func (e *UniversalExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
|
||||
var manager cgroups.Manager
|
||||
manager = &cgroupFs.Manager{Cgroups: groups}
|
||||
if systemd.UseSystemd() {
|
||||
manager = &systemd.Manager{Cgroups: groups}
|
||||
}
|
||||
return manager
|
||||
}
|
||||
Reference in New Issue
Block a user