nomad: emit more detailed error

Avoid returning context.DeadlineExceeded as it lacks helpful information
and is often ignored or handled specially by callers.

nomad/worker.go

@@ -231,6 +231,12 @@ func (w *Worker) snapshotAfter(waitIndex uint64, timeout time.Duration) (*state.
 	snap, err := w.srv.fsm.State().SnapshotAfter(ctx, waitIndex)
 	cancel()
 	metrics.MeasureSince([]string{"nomad", "worker", "wait_for_index"}, start)
+
+	// Wrap error to ensure callers don't disregard timeouts.
+	if err == context.DeadlineExceeded {
+		err = fmt.Errorf("timed out after %s waiting for index=%d", timeout, waitIndex)
+	}
+
 	return snap, err
 }
 

nomad/worker_test.go

@@ -1,7 +1,7 @@
 package nomad
 
 import (
-	"context"
+	"fmt"
 	"reflect"
 	"sync"
 	"testing"
@@ -304,9 +304,12 @@ func TestWorker_waitForIndex(t *testing.T) {
 	require.NoError(t, <-errCh)
 
 	// Cause a timeout
-	snap, err = w.snapshotAfter(index+100, 10*time.Millisecond)
+	waitIndex := index + 100
+	timeout := 10 * time.Millisecond
+	snap, err = w.snapshotAfter(index+100, timeout)
 	require.Nil(t, snap)
-	require.EqualError(t, err, context.DeadlineExceeded.Error())
+	require.EqualError(t, err,
+		fmt.Sprintf("timed out after %s waiting for index=%d", timeout, waitIndex))
 }
 
 func TestWorker_invokeScheduler(t *testing.T) {