deployments: fix data races (#14121)

* deployments: fix data races

Both priority and state related fields may be mutated concurrently and
need to be accessed with the lock acquired.

nomad/deploymentwatcher/deployment_watcher.go

@@ -840,10 +840,12 @@ func (w *deploymentWatcher) getEval() *structs.Evaluation {
 	// on the previous version that are then "watched" on a leader that's on
 	// the new version. This would result in an eval with its priority set to
 	// zero which would be bad. This therefore protects against that.
+	w.l.Lock()
 	priority := w.d.EvalPriority
 	if priority == 0 {
 		priority = w.j.Priority
 	}
+	w.l.Unlock()
 
 	return &structs.Evaluation{
 		ID:           uuid.Generate(),

nomad/deploymentwatcher/deployments_watcher.go

@@ -193,7 +193,12 @@ func (w *Watcher) watchDeployments(ctx context.Context) {
 
 // getDeploys retrieves all deployments blocking at the given index.
 func (w *Watcher) getDeploys(ctx context.Context, minIndex uint64) ([]*structs.Deployment, uint64, error) {
-	resp, index, err := w.state.BlockingQuery(w.getDeploysImpl, minIndex, ctx)
+	// state can be updated concurrently
+	w.l.Lock()
+	stateStore := w.state
+	w.l.Unlock()
+
+	resp, index, err := stateStore.BlockingQuery(w.getDeploysImpl, minIndex, ctx)
 	if err != nil {
 		return nil, 0, err
 	}