Add unit test to simulate alloc runner creation failure

client/client_test.go

@@ -725,6 +725,76 @@ func TestClient_RestoreError(t *testing.T) {
 
 }
 
+func TestClient_AddAllocError(t *testing.T) {
+	t.Parallel()
+	require := require.New(t)
+
+	s1, _ := testServer(t, nil)
+	defer s1.Shutdown()
+	testutil.WaitForLeader(t, s1.RPC)
+
+	c1, cleanup := TestClient(t, func(c *config.Config) {
+		c.DevMode = false
+		c.RPCHandler = s1
+	})
+	defer cleanup()
+
+	// Wait until the node is ready
+	waitTilNodeReady(c1, t)
+
+	// Create mock allocation with invalid task group name
+	job := mock.Job()
+	alloc1 := mock.Alloc()
+	alloc1.NodeID = c1.Node().ID
+	alloc1.Job = job
+	alloc1.JobID = job.ID
+	alloc1.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver"
+	alloc1.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
+		"run_for": "10s",
+	}
+	alloc1.ClientStatus = structs.AllocClientStatusPending
+
+	state := s1.State()
+	err := state.UpsertJob(100, job)
+	require.Nil(err)
+
+	err = state.UpsertJobSummary(101, mock.JobSummary(alloc1.JobID))
+	require.Nil(err)
+
+	err = state.UpsertAllocs(102, []*structs.Allocation{alloc1})
+	require.Nil(err)
+
+	// Manipulate state store alloc to make its task group invalid
+	stateStoreAlloc, _ := s1.State().AllocByID(nil, alloc1.ID)
+	stateStoreAlloc.TaskGroup = "invalid"
+
+	// Ensure the allocation has been marked as invalid and failed on the server
+	testutil.WaitForResult(func() (bool, error) {
+		c1.allocLock.RLock()
+		ar := c1.allocs[alloc1.ID]
+		_, isInvalid := c1.invalidAllocs[alloc1.ID]
+		c1.allocLock.RUnlock()
+		if ar != nil {
+			return false, fmt.Errorf("expected nil alloc runner")
+		}
+		if !isInvalid {
+			return false, fmt.Errorf("expected alloc to be marked as invalid")
+		}
+		// Make the task group name valid again so that the client update to mark it failed works
+		stateStoreAlloc.TaskGroup = "web"
+		alloc, err := s1.State().AllocByID(nil, alloc1.ID)
+		require.Nil(err)
+		failed := alloc.ClientStatus == structs.AllocClientStatusFailed
+		if !failed {
+			return false, fmt.Errorf("Expected failed client status, but got %v", alloc.ClientStatus)
+		}
+		return true, nil
+	}, func(err error) {
+		require.NoError(err)
+	})
+
+}
+
 func TestClient_Init(t *testing.T) {
 	t.Parallel()
 	dir, err := ioutil.TempDir("", "nomad")