drivers/exec+java: Add task configuration to restore previous PID/IPC isolation behavior

This PR adds pid_mode and ipc_mode options to the exec and java task
driver config options. By default these will defer to the default_pid_mode
and default_ipc_mode agent plugin options created in #9969. Setting
these values to "host" mode disables isolation for the task. Doing so
is not recommended, but may be necessary to support legacy job configurations.

Closes #9970
This commit is contained in:
Seth Hoenig
2021-02-08 10:36:11 -06:00
parent 6c376fc4a2
commit 836ee9e4a2
16 changed files with 330 additions and 37 deletions

View File

@@ -6,7 +6,7 @@ FEATURES:
IMPROVEMENTS:
* cli: Improved `scaling policy` commands with -verbose, auto-completion, and prefix-matching [[GH-9964](https://github.com/hashicorp/nomad/issues/9964)]
* consul/connect: Made handling of sidecar task container image URLs consistent with the `docker` task driver. [[GH-9580](https://github.com/hashicorp/nomad/issues/9580)]
* drivers/exec+java: Added client plugin configuration to re-enable previous PID/IPC namespace behavior [[GH-9982](https://github.com/hashicorp/nomad/pull/9982)]
* drivers/exec+java: Added client plugin and task configuration options to re-enable previous PID/IPC namespace behavior [[GH-9982](https://github.com/hashicorp/nomad/pull/9982)] [[GH-9990](https://github.com/hashicorp/nomad/pull/9990)]
BUG FIXES:
* consul: Fixed a bug where failing tasks with group services would only cause the allocation to restart once instead of respecting the `restart` field. [[GH-9869](https://github.com/hashicorp/nomad/issues/9869)]

View File

@@ -78,8 +78,10 @@ var (
// taskConfigSpec is the hcl specification for the driver config section of
// a task within a job. It is returned in the TaskConfigSchema RPC
taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
"command": hclspec.NewAttr("command", "string", true),
"args": hclspec.NewAttr("args", "list(string)", false),
"command": hclspec.NewAttr("command", "string", true),
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
@@ -158,8 +160,35 @@ func (c *Config) validate() error {
// TaskConfig is the driver configuration of a task within a job
type TaskConfig struct {
Command string `codec:"command"`
Args []string `codec:"args"`
// Command is the thing to exec.
Command string `codec:"command"`
// Args are passed along to Command.
Args []string `codec:"args"`
// ModePID indicates whether PID namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModePID string `codec:"pid_mode"`
// ModeIPC indicates whether IPC namespace isolation is enabled for the task.
// Must be "private" or "host" if set.
ModeIPC string `codec:"ipc_mode"`
}
func (tc *TaskConfig) validate() error {
switch tc.ModePID {
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
default:
return fmt.Errorf("pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModePID)
}
switch tc.ModeIPC {
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
default:
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}
return nil
}
// TaskState is the state which is encoded in the handle returned in
@@ -374,6 +403,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
}
if err := driverConfig.validate(); err != nil {
return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
}
d.logger.Info("starting task", "driver_cfg", hclog.Fmt("%+v", driverConfig))
handle := drivers.NewTaskHandle(taskHandleVersion)
handle.Config = cfg
@@ -419,8 +452,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
Mounts: cfg.Mounts,
Devices: cfg.Devices,
NetworkIsolation: cfg.NetworkIsolation,
DefaultModePID: d.config.DefaultModePID,
DefaultModeIPC: d.config.DefaultModeIPC,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
}
ps, err := exec.Launch(execCmd)

View File

@@ -781,3 +781,25 @@ func TestDriver_Config_validate(t *testing.T) {
}).validate())
}
}
func TestDriver_TaskConfig_validate(t *testing.T) {
for _, tc := range []struct {
pidMode, ipcMode string
exp error
}{
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "host", ipcMode: "private", exp: nil},
{pidMode: "host", ipcMode: "", exp: nil},
{pidMode: "host", ipcMode: "other", exp: errors.New(`ipc_mode must be "private" or "host", got "other"`)},
{pidMode: "host", ipcMode: "host", exp: nil},
{pidMode: "private", ipcMode: "host", exp: nil},
{pidMode: "", ipcMode: "host", exp: nil},
{pidMode: "other", ipcMode: "host", exp: errors.New(`pid_mode must be "private" or "host", got "other"`)},
} {
require.Equal(t, tc.exp, (&TaskConfig{
ModePID: tc.pidMode,
ModeIPC: tc.ipcMode,
}).validate())
}
}

View File

@@ -85,6 +85,8 @@ var (
"jar_path": hclspec.NewAttr("jar_path", "string", false),
"jvm_options": hclspec.NewAttr("jvm_options", "list(string)", false),
"args": hclspec.NewAttr("args", "list(string)", false),
"pid_mode": hclspec.NewAttr("pid_mode", "string", false),
"ipc_mode": hclspec.NewAttr("ipc_mode", "string", false),
})
// capabilities is returned by the Capabilities RPC and indicates what
@@ -144,6 +146,25 @@ type TaskConfig struct {
JarPath string `codec:"jar_path"`
JvmOpts []string `codec:"jvm_options"`
Args []string `codec:"args"` // extra arguments to java executable
ModePID string `codec:"pid_mode"`
ModeIPC string `codec:"ipc_mode"`
}
func (tc *TaskConfig) validate() error {
switch tc.ModePID {
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
default:
return fmt.Errorf("pid_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModePID)
}
switch tc.ModeIPC {
case "", executor.IsolationModePrivate, executor.IsolationModeHost:
default:
return fmt.Errorf("ipc_mode must be %q or %q, got %q", executor.IsolationModePrivate, executor.IsolationModeHost, tc.ModeIPC)
}
return nil
}
// TaskState is the state which is encoded in the handle returned in
@@ -369,6 +390,10 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
return nil, nil, fmt.Errorf("failed to decode driver config: %v", err)
}
if err := driverConfig.validate(); err != nil {
return nil, nil, fmt.Errorf("failed driver config validation: %v", err)
}
if driverConfig.Class == "" && driverConfig.JarPath == "" {
return nil, nil, fmt.Errorf("jar_path or class must be specified")
}
@@ -425,8 +450,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
Mounts: cfg.Mounts,
Devices: cfg.Devices,
NetworkIsolation: cfg.NetworkIsolation,
DefaultModePID: d.config.DefaultModePID,
DefaultModeIPC: d.config.DefaultModeIPC,
ModePID: executor.IsolationMode(d.config.DefaultModePID, driverConfig.ModePID),
ModeIPC: executor.IsolationMode(d.config.DefaultModeIPC, driverConfig.ModeIPC),
}
ps, err := exec.Launch(execCmd)

View File

@@ -45,8 +45,8 @@ func (c *grpcExecutorClient) Launch(cmd *ExecCommand) (*ProcessState, error) {
Mounts: drivers.MountsToProto(cmd.Mounts),
Devices: drivers.DevicesToProto(cmd.Devices),
NetworkIsolation: drivers.NetworkIsolationSpecToProto(cmd.NetworkIsolation),
DefaultPidMode: cmd.DefaultModePID,
DefaultIpcMode: cmd.DefaultModeIPC,
DefaultPidMode: cmd.ModePID,
DefaultIpcMode: cmd.ModeIPC,
}
resp, err := c.client.Launch(ctx, req)
if err != nil {

View File

@@ -141,11 +141,11 @@ type ExecCommand struct {
// NetworkIsolation is the network isolation configuration.
NetworkIsolation *drivers.NetworkIsolationSpec
// DefaultModePID is the default PID isolation mode
DefaultModePID string
// ModePID is the PID isolation mode (private or host).
ModePID string
// DefaultModeIPC is the default IPC isolation mode
DefaultModeIPC string
// ModeIPC is the IPC isolation mode (private or host).
ModeIPC string
}
// SetWriters sets the writer for the process stdout and stderr. This should

View File

@@ -590,7 +590,7 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error {
cfg.NoPivotRoot = command.NoPivotRoot
// set up default namespaces as configured
cfg.Namespaces = configureNamespaces(command.DefaultModePID, command.DefaultModeIPC)
cfg.Namespaces = configureNamespaces(command.ModePID, command.ModeIPC)
if command.NetworkIsolation != nil {
cfg.Namespaces = append(cfg.Namespaces, lconfigs.Namespace{

View File

@@ -129,8 +129,8 @@ func TestExecutor_Isolation_PID_and_IPC_hostMode(t *testing.T) {
defer allocDir.Destroy()
execCmd.ResourceLimits = true
execCmd.DefaultModePID = "host" // disable PID namespace
execCmd.DefaultModeIPC = "host" // disable IPC namespace
execCmd.ModePID = "host" // disable PID namespace
execCmd.ModeIPC = "host" // disable IPC namespace
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)
@@ -170,8 +170,8 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) {
defer allocDir.Destroy()
execCmd.ResourceLimits = true
execCmd.DefaultModePID = "private"
execCmd.DefaultModeIPC = "private"
execCmd.ModePID = "private"
execCmd.ModeIPC = "private"
executor := NewExecutorWithIsolation(testlog.HCLogger(t))
defer executor.Shutdown("SIGKILL", 0)

View File

@@ -35,8 +35,8 @@ func (s *grpcExecutorServer) Launch(ctx context.Context, req *proto.LaunchReques
Mounts: drivers.MountsFromProto(req.Mounts),
Devices: drivers.DevicesFromProto(req.Devices),
NetworkIsolation: drivers.NetworkIsolationSpecFromProto(req.NetworkIsolation),
DefaultModePID: req.DefaultPidMode,
DefaultModeIPC: req.DefaultIpcMode,
ModePID: req.DefaultPidMode,
ModeIPC: req.DefaultIpcMode,
})
if err != nil {

View File

@@ -139,3 +139,13 @@ func processStateFromProto(pb *proto.ProcessState) (*ProcessState, error) {
Time: timestamp,
}, nil
}
// IsolationMode returns the namespace isolation mode as determined from agent
// plugin configuration and task driver configuration. The task configuration
// takes precedence, if it is configured.
func IsolationMode(plugin, task string) string {
if task != "" {
return task
}
return plugin
}

View File

@@ -0,0 +1,28 @@
package executor
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestUtils_IsolationMode(t *testing.T) {
private := IsolationModePrivate
host := IsolationModeHost
blank := ""
for _, tc := range []struct {
plugin, task, exp string
}{
{plugin: private, task: private, exp: private},
{plugin: private, task: host, exp: host},
{plugin: private, task: blank, exp: private}, // default to private
{plugin: host, task: private, exp: private},
{plugin: host, task: host, exp: host},
{plugin: host, task: blank, exp: host}, // default to host
} {
result := IsolationMode(tc.plugin, tc.task)
require.Equal(t, tc.exp, result)
}
}

View File

@@ -0,0 +1,40 @@
job "exec" {
datacenters = ["dc1"]
type = "batch"
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "exec" {
task "exec" {
driver = "exec"
config {
command = "bash"
args = [
"-c", "local/pid.sh"
]
pid_mode = "host"
ipc_mode = "host"
}
template {
data = <<EOF
#!/usr/bin/env bash
echo my pid is $BASHPID
EOF
destination = "local/pid.sh"
perms = "777"
change_mode = "noop"
}
resources {
cpu = 100
memory = 64
}
}
}
}

View File

@@ -0,0 +1,41 @@
job "java_pid" {
datacenters = ["dc1"]
type = "batch"
group "java" {
task "build" {
lifecycle {
hook = "prestart"
sidecar = false
}
driver = "exec"
config {
command = "javac"
args = ["-d", "${NOMAD_ALLOC_DIR}", "local/Pid.java"]
}
template {
destination = "local/Pid.java"
data = <<EOH
public class Pid {
public static void main(String... s) throws Exception {
System.out.println("my pid is " + ProcessHandle.current().pid());
}
}
EOH
}
}
task "pid" {
driver = "java"
config {
class_path = "${NOMAD_ALLOC_DIR}"
class = "Pid"
pid_mode = "host"
ipc_mode = "host"
}
}
}
}

View File

@@ -46,15 +46,15 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F)
t.Skip("no Linux clients")
}
uuid := uuid.Generate()
jobID := "isolation-pid-namespace-" + uuid[0:8]
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/exec.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
tc.jobIDs = append(tc.jobIDs, jobID)
defer func() {
tc.Nomad().Jobs().Deregister(jobID, true, nil)
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
@@ -66,6 +66,36 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F)
require.Contains(t, out, "my pid is 1\n")
}
func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) {
t := f.T()
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
require.Nil(t, err)
if len(clientNodes) == 0 {
t.Skip("no Linux clients")
}
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/exec_host.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
tc.jobIDs = append(tc.jobIDs, jobID)
defer func() {
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
out, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut)
require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
require.NotContains(t, out, "my pid is 1\n")
}
func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) {
t := f.T()
@@ -76,14 +106,14 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *fr
t.Skip("no Linux clients")
}
uuid := uuid.Generate()
jobID := "isolation-pid-namespace-" + uuid[0:8]
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/alloc_exec.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
defer func() {
tc.Nomad().Jobs().Deregister(jobID, true, nil)
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
@@ -131,15 +161,15 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F)
t.Skip("no Linux clients")
}
uuid := uuid.Generate()
jobID := "isolation-pid-namespace-" + uuid[0:8]
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/java.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
tc.jobIDs = append(tc.jobIDs, jobID)
defer func() {
tc.Nomad().Jobs().Deregister(jobID, true, nil)
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
@@ -151,6 +181,36 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F)
require.Contains(t, out, "my pid is 1\n")
}
func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) {
t := f.T()
clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())
require.Nil(t, err)
if len(clientNodes) == 0 {
t.Skip("no Linux clients")
}
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/java_host.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
tc.jobIDs = append(tc.jobIDs, jobID)
defer func() {
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
e2eutil.WaitForAllocStopped(t, tc.Nomad(), allocID)
out, err := e2eutil.AllocTaskLogs(allocID, "pid", e2eutil.LogsStdOut)
require.NoError(t, err, fmt.Sprintf("could not get logs for alloc %s", allocID))
require.NotContains(t, out, "my pid is 1\n")
}
func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) {
t := f.T()
@@ -161,14 +221,14 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *fr
t.Skip("no Linux clients")
}
uuid := uuid.Generate()
jobID := "isolation-pid-namespace-" + uuid[0:8]
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/alloc_exec_java.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
defer func() {
tc.Nomad().Jobs().Deregister(jobID, true, nil)
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID
@@ -216,15 +276,15 @@ func (tc *IsolationTest) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framewo
t.Skip("no Linux clients")
}
uuid := uuid.Generate()
jobID := "isolation-pid-namespace-" + uuid[0:8]
jobID := "isolation-pid-namespace-" + uuid.Short()
file := "isolation/input/raw_exec.nomad"
allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), file, jobID, "")
require.Equal(t, len(allocs), 1, fmt.Sprintf("failed to register %s", jobID))
defer func() {
tc.Nomad().Jobs().Deregister(jobID, true, nil)
_, _, err = tc.Nomad().Jobs().Deregister(jobID, true, nil)
require.NoError(t, err)
}()
allocID := allocs[0].ID

View File

@@ -41,6 +41,20 @@ The `exec` driver supports the following configuration in the job spec:
variables](/docs/runtime/interpolation) will be interpreted before
launching the task.
- `pid_mode` - (Optional) Set to `"private"` to enable PID namespace isolation for
this task, or `"host"` to disable isolation. If left unset, the behavior is
determined from the [`default_pid_mode`][default_pid_mode] in plugin configuration.
!> **Warning:** If set to `"host"`, other processes running as the same user will
be able to access sensitive process information like environment variables.
- `ipc_mode` - (Optional) Set to `"private"` to enable IPC namespace isolation for
this task, or `"host"` to disable isolation. If left unset, the behavior is
determined from the [`default_ipc_mode`][default_ipc_mode] in plugin configuration.
!> **Warning:** If set to `"host"`, other processes running as the same user will be
able to make use of IPC features, like sending unexpected POSIX signals.
## Examples
To run a binary present on the Node:
@@ -184,3 +198,6 @@ create.
This list is configurable through the agent client
[configuration file](/docs/configuration/client#chroot_env).
[default_pid_mode]: /docs/drivers/exec#default_pid_mode
[default_ipc_mode]: /docs/drivers/exec#default_ipc_mode

View File

@@ -48,6 +48,20 @@ The `java` driver supports the following configuration in the job spec:
- `jvm_options` - (Optional) A list of JVM options to be passed while invoking
java. These options are passed without being validated in any way by Nomad.
- `pid_mode` - (Optional) Set to `"private"` to enable PID namespace isolation for
this task, or `"host"` to disable isolation. If left unset, the behavior is
determined from the [`default_pid_mode`][default_pid_mode] in plugin configuration.
!> **Warning:** If set to `"host"`, other processes running as the same user will
be able to access sensitive process information like environment variables.
- `ipc_mode` - (Optional) Set to `"private"` to enable IPC namespace isolation for
this task, or `"host"` to disable isolation. If left unset, the behavior is
determined from the [`default_ipc_mode`][default_ipc_mode] in plugin configuration.
!> **Warning:** If set to `"host"`, other processes running as the same user will be
able to make use of IPC features, like sending unexpected POSIX signals.
## Examples
A simple config block to run a Java Jar:
@@ -192,3 +206,6 @@ create.
This list is configurable through the agent client
[configuration file](/docs/configuration/client#chroot_env).
[default_pid_mode]: /docs/drivers/java#default_pid_mode
[default_ipc_mode]: /docs/drivers/java#default_ipc_mode