System jobs should be running until stopped

Prior to this commit they would be marked as dead if they had no
currently running allocations -- even though they would spring back to
life (running) if the cluster state changed such that a new eval+alloc
was created.

nomad/state/state_store.go

@@ -2124,6 +2124,22 @@ func (s *StateStore) getJobStatus(txn *memdb.Txn, job *structs.Job, evalDelete b
 		}
 	}
 
+	// system jobs are running until explicitly stopped (which is handled elsewhere)
+	if job.Type == structs.JobTypeSystem {
+		if job.Stop {
+			return structs.JobStatusDead, nil
+		}
+
+		if hasEval {
+			// At least one completed eval
+			return structs.JobStatusRunning, nil
+		}
+
+		// Pending until at least one eval has completed
+		return structs.JobStatusPending, nil
+
+	}
+
 	// The job is dead if all the allocations and evals are terminal or if there
 	// are no evals because of garbage collection.
 	if evalDelete || hasEval || hasAlloc {

nomad/state/state_store_test.go

@@ -4464,6 +4464,43 @@ func TestStateStore_SetJobStatus_PendingEval(t *testing.T) {
 	}
 }
 
+// TestStateStore_SetJobStatus_SystemJob asserts that system jobs are still
+// considered running until explicitly stopped.
+func TestStateStore_SetJobStatus_SystemJob(t *testing.T) {
+	state := testStateStore(t)
+	job := mock.SystemJob()
+
+	// Create a mock eval that is pending.
+	eval := mock.Eval()
+	eval.JobID = job.ID
+	eval.Type = job.Type
+	eval.Status = structs.EvalStatusComplete
+	if err := state.UpsertEvals(1000, []*structs.Evaluation{eval}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	txn := state.db.Txn(false)
+	status, err := state.getJobStatus(txn, job, true)
+	if err != nil {
+		t.Fatalf("getJobStatus() failed: %v", err)
+	}
+
+	if expected := structs.JobStatusRunning; status != expected {
+		t.Fatalf("getJobStatus() returned %v; expected %v", status, expected)
+	}
+
+	// Stop the job
+	job.Stop = true
+	status, err = state.getJobStatus(txn, job, true)
+	if err != nil {
+		t.Fatalf("getJobStatus() failed: %v", err)
+	}
+
+	if expected := structs.JobStatusDead; status != expected {
+		t.Fatalf("getJobStatus() returned %v; expected %v", status, expected)
+	}
+}
+
 func TestStateJobSummary_UpdateJobCount(t *testing.T) {
 	state := testStateStore(t)
 	alloc := mock.Alloc()