From 08951ac759de110143474710f00a4669a1e8659c Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 30 Jan 2020 15:33:20 -0600 Subject: [PATCH] client: additional test cases around failures in SIDS hook --- .../allocrunner/taskrunner/sids_hook_test.go | 79 ++++++++++++++++--- .../taskrunner/task_runner_test.go | 56 +++++++++++++ 2 files changed, 126 insertions(+), 9 deletions(-) diff --git a/client/allocrunner/taskrunner/sids_hook_test.go b/client/allocrunner/taskrunner/sids_hook_test.go index 5052fe9e2..5499df7d5 100644 --- a/client/allocrunner/taskrunner/sids_hook_test.go +++ b/client/allocrunner/taskrunner/sids_hook_test.go @@ -4,6 +4,7 @@ import ( "context" "io/ioutil" "os" + "path/filepath" "testing" "time" @@ -43,7 +44,6 @@ func TestSIDSHook_recoverToken(t *testing.T) { defer cleanupDir(t, secrets) taskName, taskKind := sidecar("foo") - h := newSIDSHook(sidsHookConfig{ task: &structs.Task{ Name: taskName, @@ -69,7 +69,6 @@ func TestSIDSHook_recoverToken_empty(t *testing.T) { defer cleanupDir(t, secrets) taskName, taskKind := sidecar("foo") - h := newSIDSHook(sidsHookConfig{ task: &structs.Task{ Name: taskName, @@ -83,15 +82,81 @@ func TestSIDSHook_recoverToken_empty(t *testing.T) { r.Empty(token) } -func TestSIDSHook_deriveSIToken(t *testing.T) { +func TestSIDSHook_recoverToken_unReadable(t *testing.T) { t.Parallel() r := require.New(t) secrets := tmpDir(t) defer cleanupDir(t, secrets) - taskName, taskKind := sidecar("task1") + err := os.Chmod(secrets, 0000) + r.NoError(err) + taskName, taskKind := sidecar("foo") + h := newSIDSHook(sidsHookConfig{ + task: &structs.Task{ + Name: taskName, + Kind: taskKind, + }, + logger: testlog.HCLogger(t), + }) + + _, err = h.recoverToken(secrets) + r.Error(err) +} + +func TestSIDSHook_writeToken(t *testing.T) { + t.Parallel() + r := require.New(t) + + secrets := tmpDir(t) + defer cleanupDir(t, secrets) + + id := uuid.Generate() + h := new(sidsHook) + err := h.writeToken(secrets, id) + r.NoError(err) + + content, err := ioutil.ReadFile(filepath.Join(secrets, sidsTokenFile)) + r.NoError(err) + r.Equal(id, string(content)) +} + +func TestSIDSHook_writeToken_unWritable(t *testing.T) { + t.Parallel() + r := require.New(t) + + secrets := tmpDir(t) + defer cleanupDir(t, secrets) + + err := os.Chmod(secrets, 0000) + r.NoError(err) + + id := uuid.Generate() + h := new(sidsHook) + err = h.writeToken(secrets, id) + r.Error(err) +} + +func Test_SIDSHook_writeToken_nonExistent(t *testing.T) { + t.Parallel() + r := require.New(t) + + base := tmpDir(t) + defer cleanupDir(t, base) + secrets := filepath.Join(base, "does/not/exist") + + id := uuid.Generate() + h := new(sidsHook) + err := h.writeToken(secrets, id) + r.Error(err) +} + +func TestSIDSHook_deriveSIToken(t *testing.T) { + t.Parallel() + r := require.New(t) + + taskName, taskKind := sidecar("task1") h := newSIDSHook(sidsHookConfig{ alloc: &structs.Allocation{ID: "a1"}, task: &structs.Task{ @@ -112,11 +177,6 @@ func TestSIDSHook_deriveSIToken_timeout(t *testing.T) { t.Parallel() r := require.New(t) - secrets := tmpDir(t) - defer cleanupDir(t, secrets) - - taskName, taskKind := sidecar("task1") - siClient := consul.NewMockServiceIdentitiesClient() siClient.DeriveTokenFn = func(allocation *structs.Allocation, strings []string) (m map[string]string, err error) { select { @@ -124,6 +184,7 @@ func TestSIDSHook_deriveSIToken_timeout(t *testing.T) { } } + taskName, taskKind := sidecar("task1") h := newSIDSHook(sidsHookConfig{ alloc: &structs.Allocation{ID: "a1"}, task: &structs.Task{ diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 15b0aee31..3dbafab54 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1231,6 +1231,62 @@ func TestTaskRunner_DeriveSIToken_Retry(t *testing.T) { r.Equal(token, string(data)) } +func TestTaskRunner_DeriveSIToken_UnWritableTokenFile(t *testing.T) { + t.Parallel() + r := require.New(t) + + alloc := mock.BatchConnectAlloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Config = map[string]interface{}{ + "run_for": "0s", + } + + trConfig, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + // make the si_token file un-writable, triggering a failure after a + // successful token derivation + secrets := tmpDir(t) + defer cleanupDir(t, secrets) + trConfig.TaskDir.SecretsDir = secrets + err := ioutil.WriteFile(filepath.Join(secrets, sidsTokenFile), nil, 0400) + r.NoError(err) + + // derive token works just fine + deriveFn := func(*structs.Allocation, []string) (map[string]string, error) { + return map[string]string{task.Name: uuid.Generate()}, nil + } + siClient := trConfig.ConsulSI.(*consulapi.MockServiceIdentitiesClient) + siClient.DeriveTokenFn = deriveFn + + // start the task runner + tr, err := NewTaskRunner(trConfig) + r.NoError(err) + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + useMockEnvoyBootstrapHook(tr) // mock the envoy bootstrap + + go tr.Run() + + // wait for task runner to finish running + select { + case <-tr.WaitCh(): + case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second): + r.Fail("timed out waiting for task runner") + } + + // assert task exited un-successfully + finalState := tr.TaskState() + r.Equal(structs.TaskStateDead, finalState.State) + r.True(finalState.Failed) // should have failed to write SI token + r.Contains(finalState.Events[2].DisplayMessage, "failed to write SI token") + + // assert the token is *not* on disk, as secrets dir was un-writable + tokenPath := filepath.Join(trConfig.TaskDir.SecretsDir, sidsTokenFile) + token, err := ioutil.ReadFile(tokenPath) + r.NoError(err) + r.Empty(token) +} + // TestTaskRunner_DeriveSIToken_Unrecoverable asserts that an unrecoverable error // from deriving a service identity token will fail a task. func TestTaskRunner_DeriveSIToken_Unrecoverable(t *testing.T) {