Add remaining pprof profiles to nomad operator debug (#10748)

* Add remaining pprof profiles to debug dump
* Refactor pprof profile capture
* Add WaitForFilesUntil and WaitForResultUntil utility functions
* Add CHANGELOG entry

testutil/wait.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/kr/pretty"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -35,6 +34,23 @@ func WaitForResultRetries(retries int64, test testFn, error errorFn) {
 	}
 }
 
+// WaitForResultUntil waits the duration for the test to pass.
+// Otherwise error is called after the deadline expires.
+func WaitForResultUntil(until time.Duration, test testFn, errorFunc errorFn) {
+	var success bool
+	var err error
+	deadline := time.Now().Add(until)
+	for time.Now().Before(deadline) {
+		success, err = test()
+		if success {
+			return
+		}
+		// Sleep some arbitrary fraction of the deadline
+		time.Sleep(until / 30)
+	}
+	errorFunc(err)
+}
+
 // AssertUntil asserts the test function passes throughout the given duration.
 // Otherwise error is called on failure.
 func AssertUntil(until time.Duration, test testFn, error errorFn) {
@@ -217,16 +233,28 @@ func WaitForRunning(t testing.TB, rpc rpcFn, job *structs.Job) []*structs.AllocL
 
 // WaitForFiles blocks until all the files in the slice are present
 func WaitForFiles(t testing.TB, files []string) {
-	assert := assert.New(t)
 	WaitForResult(func() (bool, error) {
-		for _, f := range files {
-			exists := assert.FileExists(f)
-			if !exists {
-				return false, fmt.Errorf("expected file to exist %s", f)
-			}
-		}
-		return true, nil
+		return FilesExist(files), nil
 	}, func(err error) {
 		t.Fatalf("missing expected files: %v", err)
 	})
 }
+
+// WaitForFilesUntil blocks until duration or all the files in the slice are present
+func WaitForFilesUntil(t testing.TB, files []string, until time.Duration) {
+	WaitForResultUntil(until, func() (bool, error) {
+		return FilesExist(files), nil
+	}, func(err error) {
+		t.Fatalf("missing expected files: %v", err)
+	})
+}
+
+// FilesExist verifies all files in the slice are present
+func FilesExist(files []string) bool {
+	for _, f := range files {
+		if _, err := os.Stat(f); os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}

testutil/wait_test.go

@@ -1 +1,43 @@
 package testutil
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWait_WaitForFilesUntil(t *testing.T) {
+
+	var files []string
+	for i := 1; i < 10; i++ {
+		filename := fmt.Sprintf("test%d.txt", i)
+		filepath := filepath.Join(os.TempDir(), filename)
+		files = append(files, filepath)
+
+		defer os.Remove(filepath)
+	}
+
+	go func() {
+		for _, filepath := range files {
+			t.Logf("Creating file %s...", filepath)
+			fh, err := os.Create(filepath)
+			fh.Close()
+
+			require.NoError(t, err, "error creating test file")
+			require.FileExists(t, filepath)
+
+			time.Sleep(250 * time.Millisecond)
+		}
+	}()
+
+	duration := 5 * time.Second
+	t.Log("Waiting 5 seconds for files...")
+	WaitForFilesUntil(t, files, duration)
+
+	t.Log("done")
+
+}