client: additional test cases around failures in SIDS hook

client/allocrunner/taskrunner/sids_hook_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -43,7 +44,6 @@ func TestSIDSHook_recoverToken(t *testing.T) {
 	defer cleanupDir(t, secrets)
 
 	taskName, taskKind := sidecar("foo")
-
 	h := newSIDSHook(sidsHookConfig{
 		task: &structs.Task{
 			Name: taskName,
@@ -69,7 +69,6 @@ func TestSIDSHook_recoverToken_empty(t *testing.T) {
 	defer cleanupDir(t, secrets)
 
 	taskName, taskKind := sidecar("foo")
-
 	h := newSIDSHook(sidsHookConfig{
 		task: &structs.Task{
 			Name: taskName,
@@ -83,15 +82,81 @@ func TestSIDSHook_recoverToken_empty(t *testing.T) {
 	r.Empty(token)
 }
 
-func TestSIDSHook_deriveSIToken(t *testing.T) {
+func TestSIDSHook_recoverToken_unReadable(t *testing.T) {
 	t.Parallel()
 	r := require.New(t)
 
 	secrets := tmpDir(t)
 	defer cleanupDir(t, secrets)
 
-	taskName, taskKind := sidecar("task1")
+	err := os.Chmod(secrets, 0000)
+	r.NoError(err)
 
+	taskName, taskKind := sidecar("foo")
+	h := newSIDSHook(sidsHookConfig{
+		task: &structs.Task{
+			Name: taskName,
+			Kind: taskKind,
+		},
+		logger: testlog.HCLogger(t),
+	})
+
+	_, err = h.recoverToken(secrets)
+	r.Error(err)
+}
+
+func TestSIDSHook_writeToken(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	secrets := tmpDir(t)
+	defer cleanupDir(t, secrets)
+
+	id := uuid.Generate()
+	h := new(sidsHook)
+	err := h.writeToken(secrets, id)
+	r.NoError(err)
+
+	content, err := ioutil.ReadFile(filepath.Join(secrets, sidsTokenFile))
+	r.NoError(err)
+	r.Equal(id, string(content))
+}
+
+func TestSIDSHook_writeToken_unWritable(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	secrets := tmpDir(t)
+	defer cleanupDir(t, secrets)
+
+	err := os.Chmod(secrets, 0000)
+	r.NoError(err)
+
+	id := uuid.Generate()
+	h := new(sidsHook)
+	err = h.writeToken(secrets, id)
+	r.Error(err)
+}
+
+func Test_SIDSHook_writeToken_nonExistent(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	base := tmpDir(t)
+	defer cleanupDir(t, base)
+	secrets := filepath.Join(base, "does/not/exist")
+
+	id := uuid.Generate()
+	h := new(sidsHook)
+	err := h.writeToken(secrets, id)
+	r.Error(err)
+}
+
+func TestSIDSHook_deriveSIToken(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	taskName, taskKind := sidecar("task1")
 	h := newSIDSHook(sidsHookConfig{
 		alloc: &structs.Allocation{ID: "a1"},
 		task: &structs.Task{
@@ -112,11 +177,6 @@ func TestSIDSHook_deriveSIToken_timeout(t *testing.T) {
 	t.Parallel()
 	r := require.New(t)
 
-	secrets := tmpDir(t)
-	defer cleanupDir(t, secrets)
-
-	taskName, taskKind := sidecar("task1")
-
 	siClient := consul.NewMockServiceIdentitiesClient()
 	siClient.DeriveTokenFn = func(allocation *structs.Allocation, strings []string) (m map[string]string, err error) {
 		select {
@@ -124,6 +184,7 @@ func TestSIDSHook_deriveSIToken_timeout(t *testing.T) {
 		}
 	}
 
+	taskName, taskKind := sidecar("task1")
 	h := newSIDSHook(sidsHookConfig{
 		alloc: &structs.Allocation{ID: "a1"},
 		task: &structs.Task{

client/allocrunner/taskrunner/task_runner_test.go

@@ -1231,6 +1231,62 @@ func TestTaskRunner_DeriveSIToken_Retry(t *testing.T) {
 	r.Equal(token, string(data))
 }
 
+func TestTaskRunner_DeriveSIToken_UnWritableTokenFile(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	alloc := mock.BatchConnectAlloc()
+	task := alloc.Job.TaskGroups[0].Tasks[0]
+	task.Config = map[string]interface{}{
+		"run_for": "0s",
+	}
+
+	trConfig, cleanup := testTaskRunnerConfig(t, alloc, task.Name)
+	defer cleanup()
+
+	// make the si_token file un-writable, triggering a failure after a
+	// successful token derivation
+	secrets := tmpDir(t)
+	defer cleanupDir(t, secrets)
+	trConfig.TaskDir.SecretsDir = secrets
+	err := ioutil.WriteFile(filepath.Join(secrets, sidsTokenFile), nil, 0400)
+	r.NoError(err)
+
+	// derive token works just fine
+	deriveFn := func(*structs.Allocation, []string) (map[string]string, error) {
+		return map[string]string{task.Name: uuid.Generate()}, nil
+	}
+	siClient := trConfig.ConsulSI.(*consulapi.MockServiceIdentitiesClient)
+	siClient.DeriveTokenFn = deriveFn
+
+	// start the task runner
+	tr, err := NewTaskRunner(trConfig)
+	r.NoError(err)
+	defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup"))
+	useMockEnvoyBootstrapHook(tr) // mock the envoy bootstrap
+
+	go tr.Run()
+
+	// wait for task runner to finish running
+	select {
+	case <-tr.WaitCh():
+	case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second):
+		r.Fail("timed out waiting for task runner")
+	}
+
+	// assert task exited un-successfully
+	finalState := tr.TaskState()
+	r.Equal(structs.TaskStateDead, finalState.State)
+	r.True(finalState.Failed) // should have failed to write SI token
+	r.Contains(finalState.Events[2].DisplayMessage, "failed to write SI token")
+
+	// assert the token is *not* on disk, as secrets dir was un-writable
+	tokenPath := filepath.Join(trConfig.TaskDir.SecretsDir, sidsTokenFile)
+	token, err := ioutil.ReadFile(tokenPath)
+	r.NoError(err)
+	r.Empty(token)
+}
+
 // TestTaskRunner_DeriveSIToken_Unrecoverable asserts that an unrecoverable error
 // from deriving a service identity token will fail a task.
 func TestTaskRunner_DeriveSIToken_Unrecoverable(t *testing.T) {