From a15bdc130dc44660400a310c77eeaa04022e7455 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 15 Nov 2019 09:31:34 -0500 Subject: [PATCH] Add tests for orphaned processes --- drivers/exec/driver_test.go | 101 ++++++++++++++++++++++++++ drivers/rawexec/driver_unix_test.go | 107 ++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) diff --git a/drivers/exec/driver_test.go b/drivers/exec/driver_test.go index e0680eb49..295924821 100644 --- a/drivers/exec/driver_test.go +++ b/drivers/exec/driver_test.go @@ -7,9 +7,12 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "runtime" + "strconv" "strings" "sync" + "syscall" "testing" "time" @@ -251,6 +254,104 @@ func TestExecDriver_StartWaitRecover(t *testing.T) { require.NoError(harness.DestroyTask(task.ID, true)) } +func TestExecDriver_DestroyKillsAll(t *testing.T) { + t.Parallel() + require := require.New(t) + ctestutils.ExecCompatible(t) + + d := NewExecDriver(testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "test", + } + + cleanup := harness.MkAllocDir(task, true) + defer cleanup() + + taskConfig := map[string]interface{}{} + taskConfig["command"] = "/bin/sh" + taskConfig["args"] = []string{"-c", fmt.Sprintf(`sleep 3600 & echo "SLEEP_PID=$!"`)} + + require.NoError(task.EncodeConcreteDriverConfig(&taskConfig)) + + handle, _, err := harness.StartTask(task) + require.NoError(err) + defer harness.DestroyTask(task.ID, true) + + ch, err := harness.WaitTask(context.Background(), handle.Config.ID) + require.NoError(err) + + select { + case result := <-ch: + require.True(result.Successful(), "command failed: %#v", result) + case <-time.After(10 * time.Second): + require.Fail("timeout waiting for task to shutdown") + } + + sleepPid := 0 + + // Ensure that the task is marked as dead, but account + // for WaitTask() closing channel before internal state is updated + testutil.WaitForResult(func() (bool, error) { + stdout, err := ioutil.ReadFile(filepath.Join(task.TaskDir().LogDir, "test.stdout.0")) + if err != nil { + return false, fmt.Errorf("failed to output pid file: %v", err) + } + + pidMatch := regexp.MustCompile(`SLEEP_PID=(\d+)`).FindStringSubmatch(string(stdout)) + if len(pidMatch) != 2 { + return false, fmt.Errorf("failed to find pid in %s", string(stdout)) + } + + pid, err := strconv.Atoi(pidMatch[1]) + if err != nil { + return false, fmt.Errorf("pid parts aren't int: %s", pidMatch[1]) + } + + sleepPid = pid + return true, nil + }, func(err error) { + require.NoError(err) + }) + + // isProcessRunning returns an error if process is not running + isProcessRunning := func(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return fmt.Errorf("failed to find process: %s", err) + } + + err = process.Signal(syscall.Signal(0)) + if err != nil { + return fmt.Errorf("failed to signal process: %s", err) + } + + return nil + } + + require.NoError(isProcessRunning(sleepPid)) + + require.NoError(harness.DestroyTask(task.ID, true)) + + testutil.WaitForResult(func() (bool, error) { + err := isProcessRunning(sleepPid) + if err == nil { + return false, fmt.Errorf("child process is still running") + } + + if !strings.Contains(err.Error(), "failed to signal process") { + return false, fmt.Errorf("unexpected error: %v", err) + } + + return true, nil + }, func(err error) { + require.NoError(err) + }) +} + func TestExecDriver_Stats(t *testing.T) { t.Parallel() require := require.New(t) diff --git a/drivers/rawexec/driver_unix_test.go b/drivers/rawexec/driver_unix_test.go index 8845d75d8..1ffcbda55 100644 --- a/drivers/rawexec/driver_unix_test.go +++ b/drivers/rawexec/driver_unix_test.go @@ -4,7 +4,11 @@ package rawexec import ( "context" + "os" + "regexp" "runtime" + "strconv" + "syscall" "testing" "fmt" @@ -196,6 +200,109 @@ func TestRawExecDriver_StartWaitStop(t *testing.T) { require.NoError(harness.DestroyTask(task.ID, true)) } +func TestRawExecDriver_DestroyKillsAll(t *testing.T) { + t.Parallel() + + // This only works reliably with cgroup PID tracking, happens in linux only + if runtime.GOOS != "linux" { + t.Skip("Linux only test") + } + + require := require.New(t) + + d := newEnabledRawExecDriver(t) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "test", + } + + cleanup := harness.MkAllocDir(task, true) + defer cleanup() + + taskConfig := map[string]interface{}{} + taskConfig["command"] = "/bin/sh" + taskConfig["args"] = []string{"-c", fmt.Sprintf(`sleep 3600 & echo "SLEEP_PID=$!"`)} + + require.NoError(task.EncodeConcreteDriverConfig(&taskConfig)) + + handle, _, err := harness.StartTask(task) + require.NoError(err) + defer harness.DestroyTask(task.ID, true) + + ch, err := harness.WaitTask(context.Background(), handle.Config.ID) + require.NoError(err) + + select { + case result := <-ch: + require.True(result.Successful(), "command failed: %#v", result) + case <-time.After(10 * time.Second): + require.Fail("timeout waiting for task to shutdown") + } + + sleepPid := 0 + + // Ensure that the task is marked as dead, but account + // for WaitTask() closing channel before internal state is updated + testutil.WaitForResult(func() (bool, error) { + stdout, err := ioutil.ReadFile(filepath.Join(task.TaskDir().LogDir, "test.stdout.0")) + if err != nil { + return false, fmt.Errorf("failed to output pid file: %v", err) + } + + pidMatch := regexp.MustCompile(`SLEEP_PID=(\d+)`).FindStringSubmatch(string(stdout)) + if len(pidMatch) != 2 { + return false, fmt.Errorf("failed to find pid in %s", string(stdout)) + } + + pid, err := strconv.Atoi(pidMatch[1]) + if err != nil { + return false, fmt.Errorf("pid parts aren't int: %s", pidMatch[1]) + } + + sleepPid = pid + return true, nil + }, func(err error) { + require.NoError(err) + }) + + // isProcessRunning returns an error if process is not running + isProcessRunning := func(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return fmt.Errorf("failed to find process: %s", err) + } + + err = process.Signal(syscall.Signal(0)) + if err != nil { + return fmt.Errorf("failed to signal process: %s", err) + } + + return nil + } + + require.NoError(isProcessRunning(sleepPid)) + + require.NoError(harness.DestroyTask(task.ID, true)) + + testutil.WaitForResult(func() (bool, error) { + err := isProcessRunning(sleepPid) + if err == nil { + return false, fmt.Errorf("child process is still running") + } + + if !strings.Contains(err.Error(), "failed to signal process") { + return false, fmt.Errorf("unexpected error: %v", err) + } + + return true, nil + }, func(err error) { + require.NoError(err) + }) +} + func TestRawExec_ExecTaskStreaming(t *testing.T) { t.Parallel() if runtime.GOOS == "darwin" {