e2e: Ensure test resources are cleaned. (#25611)

I couldn't find any reason the exec2 HTTP jobs were not being run
with a generated cleanup function, so I added this.

The deletion of the DHV ACL policy does not seem like it would
have any negative impact.

e2e/dynamic_host_volumes/dynamic_host_volumes_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/nomad/e2e/e2eutil"
 	"github.com/hashicorp/nomad/e2e/v3/jobs3"
 	"github.com/hashicorp/nomad/e2e/v3/volumes3"
+	"github.com/shoenig/test"
 	"github.com/shoenig/test/must"
 	"github.com/shoenig/test/wait"
 )
@@ -60,6 +61,11 @@ func TestDynamicHostVolumes_RegisterWorkflow(t *testing.T) {
 	must.NoError(t, err)
 	t.Logf("[%v] ACL policy for job %q created", time.Since(start), submitted.JobID())
 
+	t.Cleanup(func() {
+		_, err := nomad.ACLPolicies().Delete("register-volumes-policy", nil)
+		test.NoError(t, err)
+	})
+
 	must.NoError(t, e2eutil.Dispatch(submitted.JobID(),
 		map[string]string{
 			"vol_name": "registered-volume",

e2e/exec2/exec2_test.go

@@ -81,15 +81,11 @@ func testCountdash(t *testing.T) {
 }
 
 func testHTTP(t *testing.T) {
-	job, _ := jobs3.Submit(t,
-		"./input/http.hcl",
-		jobs3.DisableCleanup(),
-	)
+	job, httpCleanup := jobs3.Submit(t, "./input/http.hcl")
+	t.Cleanup(httpCleanup)
 
-	job2, _ := jobs3.Submit(t,
-		"./input/http_curl.hcl",
-		jobs3.DisableCleanup(),
-	)
+	job2, httpCurlCleanup := jobs3.Submit(t, "./input/http_curl.hcl")
+	t.Cleanup(httpCurlCleanup)
 
 	logs := job.TaskLogs("backend", "http")
 	must.StrContains(t, logs.Stderr, `"GET / HTTP/1.1" 200 -`)        // healthcheck