test: add E2E vaultcompat test for JWT auth flow (#18822)

Test the JWT auth flow using real Nomad and Vault agents.

command/job_plan_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/testutil"
 	"github.com/mitchellh/cli"
 	"github.com/shoenig/test/must"
@@ -188,7 +189,7 @@ func TestPlanCommand_From_Files(t *testing.T) {
 	s := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
 		c.Vault.Address = v.HTTPAddr
 		c.Vault.Enabled = true
-		c.Vault.AllowUnauthenticated = false
+		c.Vault.AllowUnauthenticated = pointer.Of(false)
 		c.Vault.Token = v.RootToken
 	})
 	defer s.Stop()

command/job_validate_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/testutil"
 	"github.com/mitchellh/cli"
 	"github.com/stretchr/testify/require"
@@ -29,7 +30,7 @@ func TestValidateCommand_Files(t *testing.T) {
 	s := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
 		c.Vault.Address = v.HTTPAddr
 		c.Vault.Enabled = true
-		c.Vault.AllowUnauthenticated = false
+		c.Vault.AllowUnauthenticated = pointer.Of(false)
 		c.Vault.Token = v.RootToken
 	})
 	defer s.Stop()

e2e/vaultcompat/cluster_setup_test.go

@@ -0,0 +1,84 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package vaultcompat
+
+import "fmt"
+
+const (
+	// jwtPath is where the JWT auth method is mounted in Vault.
+	// Use a non-default value for a more realistic scenario.
+	jwtPath = "nomad_jwt"
+)
+
+// roleLegacy is the legacy recommendation for nomad cluster role.
+var roleLegacy = map[string]interface{}{
+	"disallowed_policies": "nomad-server",
+	"explicit_max_ttl":    0, // use old name for vault compatibility
+	"name":                "nomad-cluster",
+	"orphan":              false,
+	"period":              259200, // use old name for vault compatibility
+	"renewable":           true,
+}
+
+// authConfigJWT is the configuration for the JWT auth method used by Nomad.
+func authConfigJWT(jwksURL string) map[string]any {
+	return map[string]any{
+		"jwks_url":           jwksURL,
+		"jwt_supported_algs": []string{"EdDSA"},
+		"default_role":       "nomad-workloads",
+	}
+}
+
+// roleWID is the recommended role for Nomad workloads when using JWT and
+// workload identity.
+func roleWID(policies []string) map[string]any {
+	return map[string]any{
+		"role_type":               "jwt",
+		"bound_audiences":         "vault.io",
+		"user_claim":              "/nomad_job_id",
+		"user_claim_json_pointer": true,
+		"claim_mappings": map[string]any{
+			"nomad_namespace": "nomad_namespace",
+			"nomad_job_id":    "nomad_job_id",
+		},
+		"token_ttl":      "30m",
+		"token_type":     "service",
+		"token_period":   "72h",
+		"token_policies": policies,
+	}
+}
+
+// policyWID is a templated Vault policy that grants tasks access to secret
+// paths prefixed by <namespace>/<job>.
+func policyWID(mountAccessor string) string {
+	return fmt.Sprintf(`
+path "secret/data/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/{{identity.entity.aliases.%[1]s.metadata.nomad_job_id}}/*" {
+  capabilities = ["read"]
+}
+
+path "secret/data/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/{{identity.entity.aliases.%[1]s.metadata.nomad_job_id}}" {
+  capabilities = ["read"]
+}
+
+path "secret/metadata/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/*" {
+  capabilities = ["list"]
+}
+
+path "secret/metadata/*" {
+  capabilities = ["list"]
+}
+`, mountAccessor)
+}
+
+// policyRestricted is Vault policy that only grants read access to a specific
+// path.
+const policyRestricted = `
+path "secret/data/restricted" {
+  capabilities = ["read"]
+}
+
+path "secret/metadata/restricted" {
+  capabilities = ["list"]
+}
+`

e2e/vaultcompat/input/cat_jwt.hcl

@@ -0,0 +1,111 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "cat_jwt" {
+  type = "batch"
+
+  // Tasks in this group are expected to succeed and run to completion.
+  group "success" {
+    vault {}
+
+    // Task default_identity uses the default workload identity injected by the
+    // server and the inherits the Vault configuration from the group.
+    task "default_identity" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/default/cat_jwt"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+    }
+
+    // Task custom_identity uses a custom workload identity configuration for
+    // Vault that exposes the JWT as a file and expand on the group Vault
+    // configuration.
+    task "custom_identity" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args = [
+          "${NOMAD_SECRETS_DIR}/secret.txt",
+          "${NOMAD_SECRETS_DIR}/nomad_vault_default.jwt",
+        ]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      vault {
+        role = "nomad-restricted"
+      }
+
+      identity {
+        name = "vault_default"
+        aud  = ["vault.io"]
+        ttl  = "10m"
+        file = true
+      }
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+  }
+
+  // Tasks in this group are expected to fail or never complete.
+  group "fail" {
+
+    // Task unauthorized fails to access secrets it doesn't have access to.
+    task "unauthorized" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      vault {}
+    }
+
+    // Task missing_vault fails to access the Vault token because it doesn't
+    // have a vault block, so Nomad doesn't derive a token.
+    task "missing_vault" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/vault_token"]
+      }
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+  }
+}

e2e/vaultcompat/role_test.go

@@ -1,14 +0,0 @@
-// Copyright (c) HashiCorp, Inc.
-// SPDX-License-Identifier: BUSL-1.1
-
-package vaultcompat
-
-// role is the recommended nomad cluster role
-var role = map[string]interface{}{
-	"disallowed_policies": "nomad-server",
-	"explicit_max_ttl":    0, // use old name for vault compatibility
-	"name":                "nomad-cluster",
-	"orphan":              false,
-	"period":              259200, // use old name for vault compatibility
-	"renewable":           true,
-}

e2e/vaultcompat/vaultcompat_test.go

@@ -11,14 +11,18 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-set/v2"
 	"github.com/hashicorp/go-version"
+	goversion "github.com/hashicorp/go-version"
 	nomadapi "github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/helper/uuid"
 	"github.com/hashicorp/nomad/testutil"
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/shoenig/test/must"
@@ -26,8 +30,18 @@ import (
 )
 
 const (
-	binDir  = "vault-bins"
-	envGate = "NOMAD_E2E_VAULTCOMPAT"
+	binDir     = "vault-bins"
+	envGate    = "NOMAD_E2E_VAULTCOMPAT"
+	envBaseDir = "NOMAD_E2E_VAULTCOMPAT_BASEDIR"
+)
+
+var (
+	// minJWTVersion is the first version where the Nomad workload identity
+	// auth flow is supported.
+	//
+	// 1.11.0 is when Vault added support for `user_claim_json_pointer`.
+	// https://github.com/hashicorp/vault/pull/15593
+	minJWTVersion = goversion.Must(goversion.NewVersion("1.11.0"))
 )
 
 func TestVaultCompat(t *testing.T) {
@@ -47,22 +61,120 @@ func testVaultVersions(t *testing.T) {
 }
 
 func testVaultBuild(t *testing.T, b build) {
-	t.Run("vault("+b.Version+")", func(t *testing.T) {
-		vStop, vc := startVault(t, b)
-		defer vStop()
-		setupVault(t, vc)
+	version, err := goversion.NewVersion(b.Version)
+	must.NoError(t, err)
 
-		nStop, nc := startNomad(t, vc)
-		defer nStop()
-		runCatJob(t, nc)
+	t.Run("vault("+b.Version+")", func(t *testing.T) {
+		t.Run("legacy", func(t *testing.T) {
+			testVaultLegacy(t, b)
+		})
+
+		if version.GreaterThanOrEqual(minJWTVersion) {
+			t.Run("jwt", func(t *testing.T) {
+				testVaultJWT(t, b)
+			})
+		}
 
 		// give nomad and vault time to stop
 		defer func() { time.Sleep(5 * time.Second) }()
 	})
 }
 
-func runCatJob(t *testing.T, nc *nomadapi.Client) {
-	b, err := os.ReadFile("input/cat.hcl")
+func testVaultLegacy(t *testing.T, b build) {
+	vStop, vc := startVault(t, b)
+	defer vStop()
+	setupVaultLegacy(t, vc)
+
+	nStop, nc := startNomad(t, configureNomadVaultLegacy(vc))
+	defer nStop()
+	runJob(t, nc, "input/cat.hcl", func(allocs []*nomadapi.AllocationListStub) error {
+		if n := len(allocs); n != 1 {
+			return fmt.Errorf("expected 1 alloc, got %d", n)
+		}
+		if s := allocs[0].ClientStatus; s != "complete" {
+			return fmt.Errorf("expected alloc status complete, got %s", s)
+		}
+		return nil
+	})
+}
+
+func testVaultJWT(t *testing.T, b build) {
+	vStop, vc := startVault(t, b)
+	defer vStop()
+
+	// Start Nomad without access to the Vault token.
+	vaultToken := vc.Token()
+	vc.SetToken("")
+	nStop, nc := startNomad(t, configureNomadVaultJWT(vc))
+	defer nStop()
+
+	// Restore token and configure Vault for JWT login.
+	vc.SetToken(vaultToken)
+	setupVaultJWT(t, vc, nc.Address()+"/.well-known/jwks.json")
+
+	// Write secrets for test job.
+	_, err := vc.KVv2("secret").Put(context.Background(), "default/cat_jwt", map[string]any{
+		"secret": "workload",
+	})
+	must.NoError(t, err)
+
+	_, err = vc.KVv2("secret").Put(context.Background(), "restricted", map[string]any{
+		"secret": "restricted",
+	})
+	must.NoError(t, err)
+
+	// Run test job.
+	runJob(t, nc, "input/cat_jwt.hcl", func(allocs []*nomadapi.AllocationListStub) error {
+		if n := len(allocs); n != 2 {
+			return fmt.Errorf("expected 2 allocs, got %d", n)
+		}
+
+		for _, alloc := range allocs {
+			switch alloc.TaskGroup {
+
+			// Verify all tasks in "success" group complete.
+			case "success":
+				if s := alloc.ClientStatus; s != "complete" {
+					return fmt.Errorf("expected alloc status complete, got %s", s)
+				}
+
+			// Verify all tasks in "fail" group fail for the expected reasons.
+			case "fail":
+				for task, state := range alloc.TaskStates {
+					switch task {
+
+					// Verify "unauthorized" task can't access Vault secret.
+					case "unauthorized":
+						hasEvent := false
+						for _, ev := range state.Events {
+							if strings.Contains(ev.DisplayMessage, "Missing: vault.read") {
+								hasEvent = true
+								break
+							}
+						}
+						if !hasEvent {
+							got := make([]string, 0, len(state.Events))
+							for _, ev := range state.Events {
+								got = append(got, ev.DisplayMessage)
+							}
+							return fmt.Errorf("missing expected event, got [%v]", strings.Join(got, ", "))
+						}
+
+					// Verify "missing_vault" task fails.
+					case "missing_vault":
+						if !state.Failed {
+							return fmt.Errorf("expected task to fail")
+						}
+					}
+				}
+			}
+		}
+		return nil
+	})
+}
+
+func runJob(t *testing.T, nc *nomadapi.Client, jobPath string, validateAllocs func([]*nomadapi.AllocationListStub) error) {
+	b, err := os.ReadFile(jobPath)
 	must.NoError(t, err)
 
 	jobs := nc.Jobs()
@@ -78,39 +190,37 @@ func runCatJob(t *testing.T, nc *nomadapi.Client) {
 			if err != nil {
 				return err
 			}
-			if n := len(allocs); n != 1 {
-				return fmt.Errorf("expected 1 alloc, got %d", n)
-			}
-			if s := allocs[0].ClientStatus; s != "complete" {
-				return fmt.Errorf("expected alloc status complete, got %s", s)
-			}
-			return nil
+			return validateAllocs(allocs)
 		}),
 		wait.Timeout(20*time.Second),
 		wait.Gap(1*time.Second),
 	))
 
-	t.Log("success running cat job")
+	t.Logf("success running job %s", *job.ID)
 
 	_, _, err = jobs.Deregister(*job.Name, true, nil)
 	must.NoError(t, err, must.Sprint("faild to deregister job"))
 }
 
 func startVault(t *testing.T, b build) (func(), *vaultapi.Client) {
-	path := filepath.Join(os.TempDir(), binDir, b.Version, "vault")
+	baseDir := os.Getenv(envBaseDir)
+	if baseDir == "" {
+		baseDir = os.TempDir()
+	}
+	path := filepath.Join(baseDir, binDir, b.Version, "vault")
 	vlt := testutil.NewTestVaultFromPath(t, path)
 	return vlt.Stop, vlt.Client
 }
 
-func setupVault(t *testing.T, vc *vaultapi.Client) {
-	policy, err := os.ReadFile("input/policy.hcl")
+func setupVaultLegacy(t *testing.T, vc *vaultapi.Client) {
+	policy, err := os.ReadFile("input/policy_legacy.hcl")
 	must.NoError(t, err)
 
 	sys := vc.Sys()
 	must.NoError(t, sys.PutPolicy("nomad-server", string(policy)))
 
 	log := vc.Logical()
-	log.Write("auth/token/roles/nomad-cluster", role)
+	log.Write("auth/token/roles/nomad-cluster", roleLegacy)
 
 	token := vc.Auth().Token()
 	secret, err := token.Create(&vaultapi.TokenCreateRequest{
@@ -123,31 +233,103 @@ func setupVault(t *testing.T, vc *vaultapi.Client) {
 	must.NotNil(t, secret.Auth)
 }
 
-func startNomad(t *testing.T, vc *vaultapi.Client) (func(), *nomadapi.Client) {
+func setupVaultJWT(t *testing.T, vc *vaultapi.Client, jwksURL string) {
+	logical := vc.Logical()
+	sys := vc.Sys()
+
+	// Enable JWT auth method and read back its accessor ID.
+	err := sys.EnableAuthWithOptions(jwtPath, &vaultapi.MountInput{
+		Type: "jwt",
+	})
+	must.NoError(t, err)
+
+	secret, err := logical.Read(fmt.Sprintf("sys/auth/%s", jwtPath))
+	must.NoError(t, err)
+	must.NotNil(t, secret)
+
+	jwtAuthAccessor := secret.Data["accessor"].(string)
+	must.NotEq(t, "", jwtAuthAccessor)
+
+	// Write JWT auth method config.
+	_, err = logical.Write(fmt.Sprintf("auth/%s/config", jwtPath), authConfigJWT(jwksURL))
+	must.NoError(t, err)
+
+	// Write policies for general Nomad workloads and for restricted secrets.
+	err = sys.PutPolicy("nomad-workloads", policyWID(jwtAuthAccessor))
+	must.NoError(t, err)
+
+	err = sys.PutPolicy("nomad-restricted", policyRestricted)
+	must.NoError(t, err)
+
+	// Write roles for each of the policies.
+	rolePath := fmt.Sprintf("auth/%s/role/nomad-workloads", jwtPath)
+	_, err = logical.Write(rolePath, roleWID([]string{"nomad-workloads"}))
+	must.NoError(t, err)
+
+	rolePath = fmt.Sprintf("auth/%s/role/nomad-restricted", jwtPath)
+	_, err = logical.Write(rolePath, roleWID([]string{"nomad-restricted"}))
+	must.NoError(t, err)
+}
+
+func startNomad(t *testing.T, cb func(*testutil.TestServerConfig)) (func(), *nomadapi.Client) {
+	bootstrapToken := uuid.Generate()
 	ts := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
-		c.Vault = &testutil.VaultConfig{
-			Enabled:              true,
-			Address:              vc.Address(),
-			Token:                vc.Token(),
-			Role:                 "nomad-cluster",
-			AllowUnauthenticated: true,
-		}
+		c.ACL.Enabled = true
+		c.ACL.BootstrapToken = bootstrapToken
 		c.DevMode = true
 		c.Client = &testutil.ClientConfig{
 			Enabled:      true,
 			TotalCompute: 1000,
 		}
 		c.LogLevel = testlog.HCLoggerTestLevel().String()
+
+		if cb != nil {
+			cb(c)
+		}
 	})
 	nc, err := nomadapi.NewClient(&nomadapi.Config{
 		Address: "http://" + ts.HTTPAddr,
 	})
 	must.NoError(t, err, must.Sprint("unable to create nomad api client"))
+	nc.SetSecretID(bootstrapToken)
 	return ts.Stop, nc
 }
 
+func configureNomadVaultLegacy(vc *vaultapi.Client) func(*testutil.TestServerConfig) {
+	return func(c *testutil.TestServerConfig) {
+		c.Vault = &testutil.VaultConfig{
+			Enabled:              true,
+			Address:              vc.Address(),
+			Token:                vc.Token(),
+			Role:                 "nomad-cluster",
+			AllowUnauthenticated: pointer.Of(true),
+		}
+	}
+}
+
+func configureNomadVaultJWT(vc *vaultapi.Client) func(*testutil.TestServerConfig) {
+	return func(c *testutil.TestServerConfig) {
+		c.Vault = &testutil.VaultConfig{
+			Enabled: true,
+			// Server configs.
+			DefaultIdentity: &testutil.WorkloadIdentityConfig{
+				Audience: []string{"vault.io"},
+				TTL:      "10m",
+			},
+
+			// Client configs.
+			Address:            vc.Address(),
+			JWTAuthBackendPath: jwtPath,
+		}
+	}
+}
+
 func downloadVaultBuild(t *testing.T, b build) {
-	path := filepath.Join(os.TempDir(), binDir, b.Version)
+	baseDir := os.Getenv(envBaseDir)
+	if baseDir == "" {
+		baseDir = os.TempDir()
+	}
+	path := filepath.Join(baseDir, binDir, b.Version)
 	must.NoError(t, os.MkdirAll(path, 0755))
 
 	if _, err := os.Stat(filepath.Join(path, "vault")); !os.IsNotExist(err) {

testutil/server.go

@@ -27,6 +27,7 @@ import (
 	cleanhttp "github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/helper/discover"
+	"github.com/hashicorp/nomad/helper/pointer"
 	testing "github.com/mitchellh/go-testing-interface"
 )
 
@@ -84,11 +85,14 @@ type ClientConfig struct {
 
 // VaultConfig is used to configure Vault
 type VaultConfig struct {
-	Enabled              bool   `json:"enabled"`
-	Address              string `json:"address"`
-	AllowUnauthenticated bool   `json:"allow_unauthenticated"`
-	Token                string `json:"token"`
-	Role                 string `json:"role"`
+	Name                 string                  `json:"name,omitempty"`
+	Enabled              bool                    `json:"enabled"`
+	Address              string                  `json:"address"`
+	AllowUnauthenticated *bool                   `json:"allow_unauthenticated,omitempty"`
+	Token                string                  `json:"token,omitemtpy"`
+	Role                 string                  `json:"role,omitempty"`
+	JWTAuthBackendPath   string                  `json:"jwt_auth_backend_path,omitempty"`
+	DefaultIdentity      *WorkloadIdentityConfig `json:"default_identity,omitempty"`
 }
 
 // ACLConfig is used to configure ACLs
@@ -97,6 +101,14 @@ type ACLConfig struct {
 	BootstrapToken string `json:"-"` // not in the real config
 }
 
+// WorkloadIdentityConfig is the configuration for default workload identities.
+type WorkloadIdentityConfig struct {
+	Audience []string `json:"aud"`
+	Env      bool     `json:"env"`
+	File     bool     `json:"file"`
+	TTL      string   `json:"ttl"`
+}
+
 // ServerConfigCallback is a function interface which can be
 // passed to NewTestServerConfig to modify the server config.
 type ServerConfigCallback func(c *TestServerConfig)
@@ -123,7 +135,7 @@ func defaultServerConfig() *TestServerConfig {
 		},
 		Vault: &VaultConfig{
 			Enabled:              false,
-			AllowUnauthenticated: true,
+			AllowUnauthenticated: pointer.Of(true),
 		},
 		ACL: &ACLConfig{
 			Enabled: false,

testutil/vault.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"time"
 
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/helper/useragent"
@@ -27,6 +28,10 @@ import (
 // and offers and easy API to tear itself down on test end. The only
 // prerequisite is that the Vault binary is on the $PATH.
 
+const (
+	envVaultLogLevel = "NOMAD_TEST_VAULT_LOG_LEVEL"
+)
+
 // TestVault wraps a test Vault server launched in dev mode, suitable for
 // testing.
 type TestVault struct {
@@ -48,13 +53,25 @@ func NewTestVaultFromPath(t testing.T, binary string) *TestVault {
 		t.Skipf("Skipping test %s, Vault binary %q not found in path.", t.Name(), binary)
 	}
 
+	// Define which log level to use. Default to the same as Nomad but allow a
+	// custom value for Vault. Since Vault doesn't support "off", cap it to
+	// "error".
+	logLevel := testlog.HCLoggerTestLevel().String()
+	if vaultLogLevel := os.Getenv(envVaultLogLevel); vaultLogLevel != "" {
+		logLevel = vaultLogLevel
+	}
+	if logLevel == hclog.Off.String() {
+		logLevel = hclog.Error.String()
+	}
+
 	port := ci.PortAllocator.Grab(1)[0]
 	token := uuid.Generate()
 	bind := fmt.Sprintf("-dev-listen-address=127.0.0.1:%d", port)
 	http := fmt.Sprintf("http://127.0.0.1:%d", port)
 	root := fmt.Sprintf("-dev-root-token-id=%s", token)
+	log := fmt.Sprintf("-log-level=%s", logLevel)
 
-	cmd := exec.Command(binary, "server", "-dev", bind, root)
+	cmd := exec.Command(binary, "server", "-dev", bind, root, log)
 	cmd.Stdout = testlog.NewWriter(t)
 	cmd.Stderr = testlog.NewWriter(t)