diff --git a/.changelog/25155.txt b/.changelog/25155.txt new file mode 100644 index 000000000..b07748c42 --- /dev/null +++ b/.changelog/25155.txt @@ -0,0 +1,5 @@ +```release-note:breaking-change +vault: The deprecated token-based authentication workflow for allocations has been removed. Please +see [Nomad's upgrade guide](https://developer.hashicorp.com/nomad/docs/upgrade/upgrade-specific) for +more detail. +``` diff --git a/api/jobs.go b/api/jobs.go index 54344bd35..4957bd1a3 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -549,7 +549,7 @@ func (j *Jobs) Dispatch(jobID string, meta map[string]string, // enforceVersion is set, the job is only reverted if the current version is at // the passed version. func (j *Jobs) Revert(jobID string, version uint64, enforcePriorVersion *uint64, - q *WriteOptions, consulToken, vaultToken string) (*JobRegisterResponse, *WriteMeta, error) { + q *WriteOptions, consulToken, _ string) (*JobRegisterResponse, *WriteMeta, error) { var resp JobRegisterResponse req := &JobRevertRequest{ @@ -557,7 +557,6 @@ func (j *Jobs) Revert(jobID string, version uint64, enforcePriorVersion *uint64, JobVersion: version, EnforcePriorVersion: enforcePriorVersion, ConsulToken: consulToken, - VaultToken: vaultToken, } wm, err := j.client.put("/v1/job/"+url.PathEscape(jobID)+"/revert", req, &resp, q) if err != nil { @@ -1103,7 +1102,6 @@ type Job struct { Migrate *MigrateStrategy `hcl:"migrate,block"` Meta map[string]string `hcl:"meta,block"` ConsulToken *string `mapstructure:"consul_token" hcl:"consul_token,optional"` - VaultToken *string `mapstructure:"vault_token" hcl:"vault_token,optional"` UI *JobUIConfig `hcl:"ui,block"` /* Fields set by server, not sourced from job config file */ @@ -1179,9 +1177,6 @@ func (j *Job) Canonicalize() { if j.ConsulNamespace == nil { j.ConsulNamespace = pointerOf("") } - if j.VaultToken == nil { - j.VaultToken = pointerOf("") - } if j.VaultNamespace == nil { j.VaultNamespace = pointerOf("") } @@ -1459,12 +1454,6 @@ type JobRevertRequest struct { // token and is not stored after the Job revert. ConsulToken string `json:",omitempty"` - // VaultToken is the Vault token that proves the submitter of the job revert - // has access to any Vault policies specified in the targeted job version. This - // field is only used to authorize the revert and is not stored after the Job - // revert. - VaultToken string `json:",omitempty"` - WriteRequest } diff --git a/api/jobs_test.go b/api/jobs_test.go index 1e22fa80e..e98f8bd09 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -288,7 +288,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Status: pointerOf(""), @@ -387,7 +386,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Status: pointerOf(""), @@ -469,7 +467,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), @@ -645,7 +642,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), @@ -818,7 +814,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), @@ -912,7 +907,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), @@ -1096,7 +1090,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), @@ -1275,7 +1268,6 @@ func TestJobs_Canonicalize(t *testing.T) { AllAtOnce: pointerOf(false), ConsulToken: pointerOf(""), ConsulNamespace: pointerOf(""), - VaultToken: pointerOf(""), VaultNamespace: pointerOf(""), NomadTokenID: pointerOf(""), Stop: pointerOf(false), diff --git a/client/allocrunner/taskrunner/task_runner_linux_test.go b/client/allocrunner/taskrunner/task_runner_linux_test.go index 7a115a221..67452245e 100644 --- a/client/allocrunner/taskrunner/task_runner_linux_test.go +++ b/client/allocrunner/taskrunner/task_runner_linux_test.go @@ -30,19 +30,18 @@ func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) { "run_for": "0s", } task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } // Setup a test Vault client. token := "1234" - handler := func(*structs.Allocation, []string) (map[string]string, error) { - return map[string]string{task.Name: token}, nil + handler := func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { + return token, true, nil } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.DeriveTokenFn = handler + vaultClient.SetDeriveTokenWithJWTFn(handler) conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() @@ -75,7 +74,7 @@ func TestTaskRunner_DisableFileForVaultToken_UpgradePath(t *testing.T) { must.Eq(t, structs.TaskStateDead, finalState.State) must.False(t, finalState.Failed) - // Verfiry token is in secrets dir. + // Verify token is in secrets dir. tokenPath = filepath.Join(conf.TaskDir.SecretsDir, vaultTokenFile) data, err := os.ReadFile(tokenPath) must.NoError(t, err) diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 5ec2f7c5d..689b14a7d 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -1635,26 +1635,43 @@ func TestTaskRunner_BlockForVaultToken(t *testing.T) { "run_for": "0s", } task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } // Control when we get a Vault token token := "1234" waitCh := make(chan struct{}) - handler := func(*structs.Allocation, []string) (map[string]string, error) { + handler := func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { <-waitCh - return map[string]string{task.Name: token}, nil + return token, true, nil } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.DeriveTokenFn = handler + vaultClient.SetDeriveTokenWithJWTFn(handler) conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -1728,23 +1745,40 @@ func TestTaskRunner_DisableFileForVaultToken(t *testing.T) { } task.Vault = &structs.Vault{ Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, DisableFile: true, } // Setup a test Vault client token := "1234" - handler := func(*structs.Allocation, []string) (map[string]string, error) { - return map[string]string{task.Name: token}, nil + handler := func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { + return token, true, nil } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.DeriveTokenFn = handler + vaultClient.SetDeriveTokenWithJWTFn(handler) conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + // Start task runner and wait for it to complete. tr, err := NewTaskRunner(conf) must.NoError(t, err) @@ -1778,29 +1812,46 @@ func TestTaskRunner_DeriveToken_Retry(t *testing.T) { alloc := mock.BatchAlloc() task := alloc.Job.TaskGroups[0].Tasks[0] task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } // Fail on the first attempt to derive a vault token token := "1234" count := 0 - handler := func(*structs.Allocation, []string) (map[string]string, error) { + handler := func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { if count > 0 { - return map[string]string{task.Name: token}, nil + return token, true, nil } count++ - return nil, structs.NewRecoverableError(fmt.Errorf("Want a retry"), true) + return "", false, structs.NewRecoverableError(fmt.Errorf("want a retry"), true) } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.DeriveTokenFn = handler + vaultClient.SetDeriveTokenWithJWTFn(handler) conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -1861,20 +1912,40 @@ func TestTaskRunner_DeriveToken_Unrecoverable(t *testing.T) { "run_for": "0s", } task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } // Error the token derivation vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) - vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.SetDeriveTokenError( - alloc.ID, []string{task.Name}, fmt.Errorf("Non recoverable")) - conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) + vc.(*vaultclient.MockVaultClient).SetDeriveTokenWithJWTFn( + func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { + return "", false, errors.New("unrecoverable") + }, + ) + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vc) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -1883,7 +1954,7 @@ func TestTaskRunner_DeriveToken_Unrecoverable(t *testing.T) { // Wait for the task to die select { case <-tr.WaitCh(): - case <-time.After(time.Duration(testutil.TestMultiplier()*15) * time.Second): + case <-time.After(time.Duration(testutil.TestMultiplier()*30) * time.Second): require.Fail(t, "timed out waiting for task runner to fail") } @@ -2178,25 +2249,42 @@ func TestTaskRunner_RestartSignalTask_NotRunning(t *testing.T) { // Use vault to block the start task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } // Control when we get a Vault token waitCh := make(chan struct{}, 1) defer close(waitCh) - handler := func(*structs.Allocation, []string) (map[string]string, error) { + handler := func(ctx context.Context, req vaultclient.JWTLoginRequest) (string, bool, error) { <-waitCh - return map[string]string{task.Name: "1234"}, nil + return "1234", true, nil } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) must.NoError(t, err) vaultClient := vc.(*vaultclient.MockVaultClient) - vaultClient.DeriveTokenFn = handler + vaultClient.SetDeriveTokenWithJWTFn(handler) conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -2343,8 +2431,7 @@ func TestTaskRunner_Template_BlockingPreStart(t *testing.T) { } task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, nil) @@ -2553,8 +2640,7 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { }, } task.Vault = &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, + Cluster: structs.VaultDefaultCluster, } vc, err := vaultclient.NewMockVaultClient(structs.VaultDefaultCluster) @@ -2564,6 +2650,24 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -2623,8 +2727,9 @@ func TestTaskRunner_Template_NewVaultToken(t *testing.T) { } -// TestTaskRunner_VaultManager_Restart asserts that the alloc is restarted when the alloc -// derived vault token expires, when task is configured with Restart change mode +// TestTaskRunner_VaultManager_Restart asserts that the alloc is restarted when +// the alloc derived vault token expires, when task is configured with Restart +// change mode. func TestTaskRunner_VaultManager_Restart(t *testing.T) { ci.Parallel(t) @@ -2635,7 +2740,6 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { } task.Vault = &structs.Vault{ Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, ChangeMode: structs.VaultChangeModeRestart, } @@ -2646,6 +2750,24 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) @@ -2700,8 +2822,9 @@ func TestTaskRunner_VaultManager_Restart(t *testing.T) { }) } -// TestTaskRunner_VaultManager_Signal asserts that the alloc is signalled when the alloc -// derived vault token expires, when task is configured with signal change mode +// TestTaskRunner_VaultManager_Signal asserts that the alloc is signalled when +// the alloc derived vault token expires, when task is configured with signal +// change mode. func TestTaskRunner_VaultManager_Signal(t *testing.T) { ci.Parallel(t) @@ -2712,7 +2835,6 @@ func TestTaskRunner_VaultManager_Signal(t *testing.T) { } task.Vault = &structs.Vault{ Cluster: structs.VaultDefaultCluster, - Policies: []string{"default"}, ChangeMode: structs.VaultChangeModeSignal, ChangeSignal: "SIGUSR1", } @@ -2723,6 +2845,24 @@ func TestTaskRunner_VaultManager_Signal(t *testing.T) { conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name, vaultClient) defer cleanup() + // The test triggers the task runner Vault hook which performs a call to + // the WI manager. We therefore need to seed the WI manager with data and + // use the mock implementation for this. The data itself doesn't matter, we + // just care about the lookup success. + mockIDManager := widmgr.NewMockIdentityManager() + mockIDManager.(*widmgr.MockIdentityManager).SetIdentity( + structs.WIHandle{ + IdentityName: task.Vault.IdentityName(), + WorkloadIdentifier: task.Name, + WorkloadType: structs.WorkloadTypeTask, + }, &structs.SignedWorkloadIdentity{ + WorkloadIdentityRequest: structs.WorkloadIdentityRequest{}, + JWT: "", + }, + ) + + conf.WIDMgr = mockIDManager + tr, err := NewTaskRunner(conf) require.NoError(t, err) defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) diff --git a/client/allocrunner/taskrunner/vault_hook.go b/client/allocrunner/taskrunner/vault_hook.go index 0d196e7e4..44764e12c 100644 --- a/client/allocrunner/taskrunner/vault_hook.go +++ b/client/allocrunner/taskrunner/vault_hook.go @@ -44,9 +44,6 @@ type vaultTokenUpdateHandler interface { updatedVaultToken(token string) } -// deriveTokenFunc is the signature of a function used to derive Vault tokens. -type deriveTokenFunc func() (string, error) - func (tr *TaskRunner) updatedVaultToken(token string) { // Update the task runner and environment tr.setVaultToken(token) @@ -120,9 +117,6 @@ type vaultHook struct { // widName is the workload identity name to use to retrieve signed JWTs. widName string - // deriveTokenFunc is the function used to derive Vault tokens. - deriveTokenFunc deriveTokenFunc - // allowTokenExpiration determines if a renew loop should be run allowTokenExpiration bool @@ -146,19 +140,11 @@ func newVaultHook(config *vaultHookConfig) *vaultHook { cancel: cancel, future: newTokenFuture(), widmgr: config.widmgr, + widName: config.task.Vault.IdentityName(), allowTokenExpiration: config.vaultBlock.AllowTokenExpiration, } h.logger = config.logger.Named(h.Name()) - h.widName = config.task.Vault.IdentityName() - wid := config.task.GetIdentity(h.widName) - switch { - case wid != nil: - h.deriveTokenFunc = h.deriveVaultTokenJWT - default: - h.deriveTokenFunc = h.deriveVaultTokenLegacy - } - return h } @@ -376,21 +362,11 @@ func (h *vaultHook) deriveVaultToken() (string, bool) { var attempts uint64 var backoff time.Duration for { - token, err := h.deriveTokenFunc() + token, err := h.deriveVaultTokenJWT() if err == nil { return token, false } - // Check if this is a server side error - if structs.IsServerSide(err) { - h.logger.Error("failed to derive Vault token", "error", err, "server_side", true) - h.lifecycle.Kill(h.ctx, - structs.NewTaskEvent(structs.TaskKilling). - SetFailsTask(). - SetDisplayMessage(fmt.Sprintf("Vault: server failed to derive vault token: %v", err))) - return "", true - } - // Check if we can't recover from the error if !structs.IsRecoverable(err) { h.logger.Error("failed to derive Vault token", "error", err, "recoverable", false) @@ -464,19 +440,6 @@ func (h *vaultHook) deriveVaultTokenJWT() (string, error) { return token, nil } -// deriveVaultTokenLegacy returns a Vault ACL token using the legacy flow where -// Nomad clients request Vault tokens from Nomad servers. -// -// Deprecated: This authentication flow will be removed Nomad 1.9. -func (h *vaultHook) deriveVaultTokenLegacy() (string, error) { - tokens, err := h.client.DeriveToken(h.alloc, []string{h.task.Name}) - if err != nil { - return "", err - } - - return tokens[h.task.Name], nil -} - // writeToken writes the given token to disk func (h *vaultHook) writeToken(token string) error { // Handle upgrade path by first checking if the tasks private directory diff --git a/client/allocrunner/taskrunner/vault_hook_test.go b/client/allocrunner/taskrunner/vault_hook_test.go index 36e62e442..1cf243c7a 100644 --- a/client/allocrunner/taskrunner/vault_hook_test.go +++ b/client/allocrunner/taskrunner/vault_hook_test.go @@ -117,18 +117,8 @@ func TestTaskRunner_VaultHook(t *testing.T) { configs map[string]*sconfig.VaultConfig configNonrenewable bool expectRole string - expectLegacy bool expectNoRenew bool }{ - { - name: "legacy flow", - task: &structs.Task{ - Vault: &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - }, - }, - expectLegacy: true, - }, { name: "jwt flow", task: &structs.Task{ @@ -284,22 +274,18 @@ func TestTaskRunner_VaultHook(t *testing.T) { // Token must have been derived. var token string client := hook.client.(*vaultclient.MockVaultClient) - if tc.expectLegacy { - tokens := client.LegacyTokens() - must.MapLen(t, 1, tokens) - token = tokens[tc.task.Name] - } else { - tokens := client.JWTTokens() - must.MapLen(t, 1, tokens) - swid, err := hook.widmgr.Get(structs.WIHandle{ - IdentityName: tc.task.Vault.IdentityName(), - WorkloadIdentifier: tc.task.Name, - WorkloadType: structs.WorkloadTypeTask, - }) - must.NoError(t, err) - token = tokens[swid.JWT] - } + tokens := client.JWTTokens() + must.MapLen(t, 1, tokens) + + swid, err := hook.widmgr.Get(structs.WIHandle{ + IdentityName: tc.task.Vault.IdentityName(), + WorkloadIdentifier: tc.task.Name, + WorkloadType: structs.WorkloadTypeTask, + }) + must.NoError(t, err) + token = tokens[swid.JWT] + must.NotEq(t, "", token) // Token must be derived with correct role. @@ -443,7 +429,6 @@ func TestTaskRunner_VaultHook_recover(t *testing.T) { // Verify token was recovered and not derived. client := hook.client.(*vaultclient.MockVaultClient) must.MapLen(t, 0, client.JWTTokens()) - must.MapLen(t, 0, client.LegacyTokens()) }) } } diff --git a/client/client.go b/client/client.go index 608efcb5c..465417b87 100644 --- a/client/client.go +++ b/client/client.go @@ -67,7 +67,6 @@ import ( nconfig "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/plugins/csi" "github.com/hashicorp/nomad/plugins/device" - vaultapi "github.com/hashicorp/vault/api" "github.com/shirou/gopsutil/v3/host" ) @@ -2871,7 +2870,7 @@ func (c *Client) setupVaultClients() error { c.vaultClients = map[string]vaultclient.VaultClient{} vaultConfigs := c.GetConfig().GetVaultConfigs(c.logger) for _, vaultConfig := range vaultConfigs { - vaultClient, err := vaultclient.NewVaultClient(vaultConfig, c.logger, c.deriveToken) + vaultClient, err := vaultclient.NewVaultClient(vaultConfig, c.logger) if err != nil { return err } @@ -2917,92 +2916,6 @@ func (c *Client) setupNomadServiceRegistrationHandler() { c.nomadService = nsd.NewServiceRegistrationHandler(c.logger, &cfg) } -// deriveToken takes in an allocation and a set of tasks and derives vault -// tokens for each of the tasks, unwraps all of them using the supplied vault -// client and returns a map of unwrapped tokens, indexed by the task name. -func (c *Client) deriveToken(alloc *structs.Allocation, taskNames []string, vclient *vaultapi.Client) (map[string]string, error) { - vlogger := c.logger.Named("vault") - - verifiedTasks, err := verifiedTasks(vlogger, alloc, taskNames) - if err != nil { - return nil, err - } - - // DeriveVaultToken of nomad server can take in a set of tasks and - // creates tokens for all the tasks. - req := &structs.DeriveVaultTokenRequest{ - NodeID: c.NodeID(), - SecretID: c.secretNodeID(), - AllocID: alloc.ID, - Tasks: verifiedTasks, - QueryOptions: structs.QueryOptions{ - Region: c.Region(), - AllowStale: false, - MinQueryIndex: alloc.CreateIndex, - AuthToken: c.secretNodeID(), - }, - } - - // Derive the tokens - // namespace is handled via nomad/vault - var resp structs.DeriveVaultTokenResponse - if err := c.RPC("Node.DeriveVaultToken", &req, &resp); err != nil { - vlogger.Error("error making derive token RPC", "error", err) - return nil, fmt.Errorf("DeriveVaultToken RPC failed: %v", err) - } - if resp.Error != nil { - vlogger.Error("error deriving vault tokens", "error", resp.Error) - return nil, structs.NewWrappedServerError(resp.Error) - } - if resp.Tasks == nil { - vlogger.Error("error derivng vault token", "error", "invalid response") - return nil, fmt.Errorf("failed to derive vault tokens: invalid response") - } - - unwrappedTokens := make(map[string]string) - - // Retrieve the wrapped tokens from the response and unwrap it - for _, taskName := range verifiedTasks { - // Get the wrapped token - wrappedToken, ok := resp.Tasks[taskName] - if !ok { - vlogger.Error("wrapped token missing for task", "task_name", taskName) - return nil, fmt.Errorf("wrapped token missing for task %q", taskName) - } - - // Unwrap the vault token - unwrapResp, err := vclient.Logical().Unwrap(wrappedToken) - if err != nil { - if structs.VaultUnrecoverableError.MatchString(err.Error()) { - return nil, err - } - - // The error is recoverable - return nil, structs.NewRecoverableError( - fmt.Errorf("failed to unwrap the token for task %q: %v", taskName, err), true) - } - - // Validate the response - var validationErr error - if unwrapResp == nil { - validationErr = fmt.Errorf("Vault returned nil secret when unwrapping") - } else if unwrapResp.Auth == nil { - validationErr = fmt.Errorf("Vault returned unwrap secret with nil Auth. Secret warnings: %v", unwrapResp.Warnings) - } else if unwrapResp.Auth.ClientToken == "" { - validationErr = fmt.Errorf("Vault returned unwrap secret with empty Auth.ClientToken. Secret warnings: %v", unwrapResp.Warnings) - } - if validationErr != nil { - vlogger.Warn("error unwrapping token", "error", err) - return nil, structs.NewRecoverableError(validationErr, true) - } - - // Append the unwrapped token to the return value - unwrappedTokens[taskName] = unwrapResp.Auth.ClientToken - } - - return unwrappedTokens, nil -} - // deriveSIToken takes an allocation and a set of tasks and derives Consul // Service Identity tokens for each of the tasks by requesting them from the // Nomad Server. diff --git a/client/vaultclient/vaultclient.go b/client/vaultclient/vaultclient.go index f83bf13c2..88a107bef 100644 --- a/client/vaultclient/vaultclient.go +++ b/client/vaultclient/vaultclient.go @@ -16,7 +16,6 @@ import ( hclog "github.com/hashicorp/go-hclog" metrics "github.com/hashicorp/go-metrics/compat" "github.com/hashicorp/nomad/helper/useragent" - "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" vaultapi "github.com/hashicorp/vault/api" ) @@ -25,11 +24,6 @@ import ( // by cluster name. This function is injected into the allocrunner/taskrunner type VaultClientFunc func(string) (VaultClient, error) -// TokenDeriverFunc takes in an allocation and a set of tasks and derives a -// wrapped token for all the tasks, from the nomad server. All the derived -// wrapped tokens will be unwrapped using the vault API client. -type TokenDeriverFunc func(*structs.Allocation, []string, *vaultapi.Client) (map[string]string, error) - // JWTLoginRequest is used to derive a Vault ACL token using a JWT login // request. type JWTLoginRequest struct { @@ -55,11 +49,6 @@ type VaultClient interface { // Stop terminates the renewal loop for tokens and secrets Stop() - // DeriveToken contacts the nomad server and fetches wrapped tokens for - // a set of tasks. The wrapped tokens will be unwrapped using vault and - // returned. - DeriveToken(*structs.Allocation, []string) (map[string]string, error) - // DeriveTokenWithJWT returns a Vault ACL token using the JWT login // endpoint, along with whether or not the token is renewable. DeriveTokenWithJWT(context.Context, JWTLoginRequest) (string, bool, error) @@ -76,11 +65,6 @@ type VaultClient interface { // Implementation of VaultClient interface to interact with vault and perform // token and lease renewals periodically. type vaultClient struct { - // tokenDeriver is a function pointer passed in by the client to derive - // tokens by making RPC calls to the nomad server. The wrapped tokens - // returned by the nomad server will be unwrapped by this function - // using the vault API client. - tokenDeriver TokenDeriverFunc // running indicates if the renewal loop is active or not running bool @@ -142,7 +126,7 @@ type vaultClientHeap struct { type vaultDataHeapImp []*vaultClientHeapEntry // NewVaultClient returns a new vault client from the given config. -func NewVaultClient(config *config.VaultConfig, logger hclog.Logger, tokenDeriver TokenDeriverFunc) (*vaultClient, error) { +func NewVaultClient(config *config.VaultConfig, logger hclog.Logger) (*vaultClient, error) { if config == nil { return nil, fmt.Errorf("nil vault config") } @@ -150,13 +134,11 @@ func NewVaultClient(config *config.VaultConfig, logger hclog.Logger, tokenDerive logger = logger.Named("vault").With("name", config.Name) c := &vaultClient{ - config: config, - stopCh: make(chan struct{}), - // Update channel should be a buffered channel - updateCh: make(chan struct{}, 1), - heap: newVaultClientHeap(), - logger: logger, - tokenDeriver: tokenDeriver, + config: config, + stopCh: make(chan struct{}), + updateCh: make(chan struct{}, 1), // Update channel should be buffered. + heap: newVaultClientHeap(), + logger: logger, } if !config.IsEnabled() { @@ -254,33 +236,6 @@ func (c *vaultClient) unlockAndUnset() { c.lock.Unlock() } -// DeriveToken takes in an allocation and a set of tasks and for each of the -// task, it derives a vault token from nomad server and unwraps it using vault. -// The return value is a map containing all the unwrapped tokens indexed by the -// task name. -func (c *vaultClient) DeriveToken(alloc *structs.Allocation, taskNames []string) (map[string]string, error) { - if !c.config.IsEnabled() { - return nil, fmt.Errorf("vault client not enabled") - } - if !c.isRunning() { - return nil, fmt.Errorf("vault client is not running") - } - - c.lock.Lock() - defer c.unlockAndUnset() - - // Use the token supplied to interact with vault - c.client.SetToken("") - - tokens, err := c.tokenDeriver(alloc, taskNames, c.client) - if err != nil { - c.logger.Error("error deriving token", "error", err, "alloc_id", alloc.ID, "task_names", taskNames) - return nil, err - } - - return tokens, nil -} - // DeriveTokenWithJWT returns a Vault ACL token using the JWT login endpoint. func (c *vaultClient) DeriveTokenWithJWT(ctx context.Context, req JWTLoginRequest) (string, bool, error) { if !c.config.IsEnabled() { diff --git a/client/vaultclient/vaultclient_test.go b/client/vaultclient/vaultclient_test.go index cc74cd3bb..2b222f608 100644 --- a/client/vaultclient/vaultclient_test.go +++ b/client/vaultclient/vaultclient_test.go @@ -11,7 +11,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "strings" "testing" "text/template" "time" @@ -211,7 +210,7 @@ func TestVaultClient_DeriveTokenWithJWT(t *testing.T) { v.Config.ConnectionRetryIntv = 100 * time.Millisecond v.Config.JWTAuthBackendPath = jwtAuthMountPathTest - c, err := NewVaultClient(v.Config, logger, nil) + c, err := NewVaultClient(v.Config, logger) must.NoError(t, err) c.Start() @@ -268,94 +267,9 @@ func TestVaultClient_DeriveTokenWithJWT(t *testing.T) { must.ErrorContains(t, err, `role "test" could not be found`) } -func TestVaultClient_TokenRenewals(t *testing.T) { - ci.Parallel(t) - - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - v.Config.TaskTokenTTL = "4s" - c, err := NewVaultClient(v.Config, logger, nil) - if err != nil { - t.Fatalf("failed to build vault Vault: %v", err) - } - - c.Start() - defer c.Stop() - - // Sleep a little while to ensure that the renewal loop is active - time.Sleep(time.Duration(testutil.TestMultiplier()) * time.Second) - - tcr := &vaultapi.TokenCreateRequest{ - Policies: []string{"foo", "bar"}, - TTL: "2s", - DisplayName: "derived-for-task", - Renewable: new(bool), - } - *tcr.Renewable = true - - num := 5 - tokens := make([]string, num) - for i := 0; i < num; i++ { - c.client.SetToken(v.Config.Token) - - if err := c.client.SetAddress(v.Config.Addr); err != nil { - t.Fatal(err) - } - - secret, err := c.client.Auth().Token().Create(tcr) - if err != nil { - t.Fatalf("failed to create vault token: %v", err) - } - - if secret == nil || secret.Auth == nil || secret.Auth.ClientToken == "" { - t.Fatal("failed to derive a wrapped vault token") - } - - tokens[i] = secret.Auth.ClientToken - - errCh, err := c.RenewToken(tokens[i], secret.Auth.LeaseDuration) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - go func(errCh <-chan error) { - for { - select { - case err := <-errCh: - must.NoError(t, err, must.Sprintf("unexpected error while renewing vault token")) - } - } - }(errCh) - } - - c.lock.Lock() - length := c.heap.Length() - c.lock.Unlock() - if length != num { - t.Fatalf("bad: Heap length: expected: %d, actual: %d", num, length) - } - - time.Sleep(time.Duration(testutil.TestMultiplier()) * time.Second) - - for i := 0; i < num; i++ { - if err := c.StopRenewToken(tokens[i]); err != nil { - must.NoError(t, err) - } - } - - c.lock.Lock() - length = c.heap.Length() - c.lock.Unlock() - if length != 0 { - t.Fatalf("bad: Heap length: expected: 0, actual: %d", length) - } -} - -// TestVaultClient_NamespaceSupport tests that the Vault namespace Config, if present, will result in the -// namespace header being set on the created Vault Vault. +// TestVaultClient_NamespaceSupport tests that the Vault namespace Config, if +// present, will result in the namespace header being set on the created Vault +// client. func TestVaultClient_NamespaceSupport(t *testing.T) { ci.Parallel(t) @@ -366,9 +280,8 @@ func TestVaultClient_NamespaceSupport(t *testing.T) { conf := structsc.DefaultVaultConfig() conf.Enabled = &tr - conf.Token = "testvaulttoken" conf.Namespace = testNs - c, err := NewVaultClient(conf, logger, nil) + c, err := NewVaultClient(conf, logger) must.NoError(t, err) must.Eq(t, testNs, c.client.Headers().Get(structs.VaultNamespaceHeaderName)) } @@ -379,11 +292,9 @@ func TestVaultClient_Heap(t *testing.T) { tr := true conf := structsc.DefaultVaultConfig() conf.Enabled = &tr - conf.Token = "testvaulttoken" - conf.TaskTokenTTL = "10s" logger := testlog.HCLogger(t) - c, err := NewVaultClient(conf, logger, nil) + c, err := NewVaultClient(conf, logger) if err != nil { t.Fatal(err) } @@ -480,91 +391,6 @@ func TestVaultClient_Heap(t *testing.T) { } -func TestVaultClient_RenewNonRenewableLease(t *testing.T) { - ci.Parallel(t) - - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - v.Config.TaskTokenTTL = "4s" - c, err := NewVaultClient(v.Config, logger, nil) - if err != nil { - t.Fatalf("failed to build vault Vault: %v", err) - } - - c.Start() - defer c.Stop() - - // Sleep a little while to ensure that the renewal loop is active - time.Sleep(time.Duration(testutil.TestMultiplier()) * time.Second) - - tcr := &vaultapi.TokenCreateRequest{ - Policies: []string{"foo", "bar"}, - TTL: "2s", - DisplayName: "derived-for-task", - Renewable: new(bool), - } - - c.client.SetToken(v.Config.Token) - - if err := c.client.SetAddress(v.Config.Addr); err != nil { - t.Fatal(err) - } - - secret, err := c.client.Auth().Token().Create(tcr) - if err != nil { - t.Fatalf("failed to create vault token: %v", err) - } - - if secret == nil || secret.Auth == nil || secret.Auth.ClientToken == "" { - t.Fatal("failed to derive a wrapped vault token") - } - - _, err = c.RenewToken(secret.Auth.ClientToken, secret.Auth.LeaseDuration) - if err == nil { - t.Fatalf("expected error, got nil") - } else if !strings.Contains(err.Error(), "lease is not renewable") { - t.Fatalf("expected \"%s\" in error message, got \"%v\"", "lease is not renewable", err) - } -} - -func TestVaultClient_RenewNonexistentLease(t *testing.T) { - ci.Parallel(t) - - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - v.Config.TaskTokenTTL = "4s" - c, err := NewVaultClient(v.Config, logger, nil) - if err != nil { - t.Fatalf("failed to build vault Vault: %v", err) - } - - c.Start() - defer c.Stop() - - // Sleep a little while to ensure that the renewal loop is active - time.Sleep(time.Duration(testutil.TestMultiplier()) * time.Second) - - c.client.SetToken(v.Config.Token) - - if err := c.client.SetAddress(v.Config.Addr); err != nil { - t.Fatal(err) - } - - _, err = c.RenewToken(c.client.Token(), 10) - if err == nil { - t.Fatalf("expected error, got nil") - // The Vault error message changed between 0.10.2 and 1.0.1 - } else if !strings.Contains(err.Error(), "lease not found") && !strings.Contains(err.Error(), "lease is not renewable") { - t.Fatalf("expected \"%s\" or \"%s\" in error message, got \"%v\"", "lease not found", "lease is not renewable", err.Error()) - } -} - // TestVaultClient_RenewalTime_Long asserts that for leases over 1m the renewal // time is jittered. func TestVaultClient_RenewalTime_Long(t *testing.T) { @@ -612,7 +438,7 @@ func TestVaultClient_SetUserAgent(t *testing.T) { conf := structsc.DefaultVaultConfig() conf.Enabled = pointer.Of(true) logger := testlog.HCLogger(t) - c, err := NewVaultClient(conf, logger, nil) + c, err := NewVaultClient(conf, logger) must.NoError(t, err) ua := c.client.Headers().Get("User-Agent") @@ -651,7 +477,7 @@ func TestVaultClient_RenewalConcurrent(t *testing.T) { conf.Addr = ts.URL conf.Enabled = pointer.Of(true) - vc, err := NewVaultClient(conf, testlog.HCLogger(t), nil) + vc, err := NewVaultClient(conf, testlog.HCLogger(t)) must.NoError(t, err) vc.Start() @@ -694,7 +520,7 @@ func TestVaultClient_NamespaceReset(t *testing.T) { for _, ns := range []string{"", "foo"} { conf.Namespace = ns - vc, err := NewVaultClient(conf, testlog.HCLogger(t), nil) + vc, err := NewVaultClient(conf, testlog.HCLogger(t)) must.NoError(t, err) vc.Start() diff --git a/client/vaultclient/vaultclient_testing.go b/client/vaultclient/vaultclient_testing.go index a21d0ec45..2516ac40d 100644 --- a/client/vaultclient/vaultclient_testing.go +++ b/client/vaultclient/vaultclient_testing.go @@ -9,14 +9,11 @@ import ( "sync" "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/structs" ) // MockVaultClient is used for testing the vaultclient integration and is safe // for concurrent access. type MockVaultClient struct { - // legacyTokens stores the tokens per task derived using the legacy flow. - legacyTokens map[string]string // jwtTokens stores the tokens derived using the JWT flow. jwtTokens map[string]string @@ -36,11 +33,6 @@ type MockVaultClient struct { // token is derived deriveTokenErrors map[string]map[string]error - // DeriveTokenFn allows the caller to control the DeriveToken function. If - // not set an error is returned if found in DeriveTokenErrors and otherwise - // a token is generated and returned - DeriveTokenFn func(a *structs.Allocation, tasks []string) (map[string]string, error) - // deriveTokenWithJWTFn allows the caller to control the DeriveTokenWithJWT // function. deriveTokenWithJWTFn func(context.Context, JWTLoginRequest) (string, bool, error) @@ -76,29 +68,6 @@ func (vc *MockVaultClient) DeriveTokenWithJWT(ctx context.Context, req JWTLoginR return token, vc.renewable, nil } -func (vc *MockVaultClient) DeriveToken(a *structs.Allocation, tasks []string) (map[string]string, error) { - vc.mu.Lock() - defer vc.mu.Unlock() - - if vc.DeriveTokenFn != nil { - return vc.DeriveTokenFn(a, tasks) - } - - tokens := make(map[string]string, len(tasks)) - for _, task := range tasks { - if tasks, ok := vc.deriveTokenErrors[a.ID]; ok { - if err, ok := tasks[task]; ok { - return nil, err - } - } - - tokens[task] = uuid.Generate() - } - - vc.legacyTokens = tokens - return tokens, nil -} - func (vc *MockVaultClient) SetDeriveTokenError(allocID string, tasks []string, err error) { vc.mu.Lock() defer vc.mu.Unlock() @@ -161,14 +130,7 @@ func (vc *MockVaultClient) SetRenewable(renewable bool) { vc.renewable = renewable } -// LegacyTokens returns the tokens generated using the legacy flow. -func (vc *MockVaultClient) LegacyTokens() map[string]string { - vc.mu.Lock() - defer vc.mu.Unlock() - return vc.legacyTokens -} - -// JWTTotkens returns the tokens generated suing the JWT flow. +// JWTTokens returns the tokens generated suing the JWT flow. func (vc *MockVaultClient) JWTTokens() map[string]string { vc.mu.Lock() defer vc.mu.Unlock() @@ -198,22 +160,6 @@ func (vc *MockVaultClient) RenewTokenErrCh(token string) chan error { return vc.renewTokens[token] } -// RenewTokenErrors is used to return an error when the RenewToken is called -// with the given token -func (vc *MockVaultClient) RenewTokenErrors() map[string]error { - vc.mu.Lock() - defer vc.mu.Unlock() - return vc.renewTokenErrors -} - -// DeriveTokenErrors maps an allocation ID and tasks to an error when the -// token is derived -func (vc *MockVaultClient) DeriveTokenErrors() map[string]map[string]error { - vc.mu.Lock() - defer vc.mu.Unlock() - return vc.deriveTokenErrors -} - // SetDeriveTokenWithJWTFn sets the function used to derive tokens using JWT. func (vc *MockVaultClient) SetDeriveTokenWithJWTFn(f func(context.Context, JWTLoginRequest) (string, bool, error)) { vc.mu.Lock() diff --git a/client/widmgr/mock.go b/client/widmgr/mock.go index 0202c9291..84bdab454 100644 --- a/client/widmgr/mock.go +++ b/client/widmgr/mock.go @@ -8,6 +8,7 @@ import ( "crypto/rsa" "fmt" "slices" + "sync" "time" "github.com/go-jose/go-jose/v3" @@ -113,3 +114,55 @@ func (m *MockWIDSigner) SignIdentities(minIndex uint64, req []*structs.WorkloadI } return swids, nil } + +type MockIdentityManager struct { + lastToken map[structs.WIHandle]*structs.SignedWorkloadIdentity + lastTokenLock sync.RWMutex +} + +// NewMockIdentityManager returns an implementation of the IdentityManager +// interface which supports data manipulation for testing. +func NewMockIdentityManager() IdentityManager { + return &MockIdentityManager{ + lastToken: make(map[structs.WIHandle]*structs.SignedWorkloadIdentity), + } +} + +// Get implements the IdentityManager.Get functionality. This should be used +// along with SetIdentity for testing. +func (m *MockIdentityManager) Get(handle structs.WIHandle) (*structs.SignedWorkloadIdentity, error) { + m.lastTokenLock.RLock() + defer m.lastTokenLock.RUnlock() + + token := m.lastToken[handle] + if token == nil { + return nil, fmt.Errorf("no token for handle name:%s wid:%s type:%v", + handle.IdentityName, handle.WorkloadIdentifier, handle.WorkloadType) + } + + return token, nil +} + +// Run implements the IdentityManager.Run functionality. It currently does +// nothing. +func (m *MockIdentityManager) Run() error { return nil } + +// Watch implements the IdentityManager.Watch functionality. It currently does +// nothing. +func (m *MockIdentityManager) Watch(_ structs.WIHandle) (<-chan *structs.SignedWorkloadIdentity, func()) { + return nil, nil +} + +// Shutdown implements the IdentityManager.Shutdown functionality. It currently +// does nothing. +func (m *MockIdentityManager) Shutdown() {} + +// SetIdentity is a helper function that allows testing callers to set custom +// identity information. The constructor function returns the interface name, +// therefore to call this you will need assert the type like +// ".(*widmgr.MockIdentityManager).SetIdentity(...)". +func (m *MockIdentityManager) SetIdentity(handle structs.WIHandle, token *structs.SignedWorkloadIdentity) { + m.lastTokenLock.Lock() + m.lastToken[handle] = token + m.lastTokenLock.Unlock() +} diff --git a/command/agent/agent_endpoint.go b/command/agent/agent_endpoint.go index 5d8f20367..8c50f70a0 100644 --- a/command/agent/agent_endpoint.go +++ b/command/agent/agent_endpoint.go @@ -84,12 +84,6 @@ func (s *HTTPServer) AgentSelfRequest(resp http.ResponseWriter, req *http.Reques self.Config = s.agent.GetConfig().Copy() - for _, vaultConfig := range self.Config.Vaults { - if vaultConfig.Token != "" { - vaultConfig.Token = "" - } - } - if self.Config != nil && self.Config.ACL != nil && self.Config.ACL.ReplicationToken != "" { self.Config.ACL.ReplicationToken = "" } diff --git a/command/agent/agent_endpoint_test.go b/command/agent/agent_endpoint_test.go index e5972ebdc..c840648fd 100644 --- a/command/agent/agent_endpoint_test.go +++ b/command/agent/agent_endpoint_test.go @@ -56,17 +56,6 @@ func TestHTTP_AgentSelf(t *testing.T) { require.NotNil(self.Config.ACL) require.NotEmpty(self.Stats) - // Check the Vault config - require.Empty(self.Config.defaultVault().Token) - - // Assign a Vault token and require it is redacted. - s.Config.defaultVault().Token = "badc0deb-adc0-deba-dc0d-ebadc0debadc" - respW = httptest.NewRecorder() - obj, err = s.Server.AgentSelfRequest(respW, req) - require.NoError(err) - self = obj.(agentSelf) - require.Equal("", self.Config.defaultVault().Token) - // Assign a ReplicationToken token and require it is redacted. s.Config.ACL.ReplicationToken = "badc0deb-adc0-deba-dc0d-ebadc0debadc" respW = httptest.NewRecorder() diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index b53fd2b96..aac7d6191 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -1320,7 +1320,6 @@ func TestServer_Reload_VaultConfig(t *testing.T) { c.Vaults[0] = &config.VaultConfig{ Name: "default", Enabled: pointer.Of(true), - Token: "vault-token", Namespace: "vault-namespace", Addr: "https://vault.consul:8200", } @@ -1331,7 +1330,6 @@ func TestServer_Reload_VaultConfig(t *testing.T) { newConfig.Vaults[0] = &config.VaultConfig{ Name: "default", Enabled: pointer.Of(true), - Token: "vault-token", Namespace: "another-namespace", Addr: "https://vault.consul:8200", } diff --git a/command/agent/command.go b/command/agent/command.go index d0f354edc..9059446b0 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -183,11 +183,6 @@ func (c *Command) readConfig() *Config { defaultVault.Enabled = &b return nil }), "vault-enabled", "") - flags.Var((flaghelper.FuncBoolVar)(func(b bool) error { - defaultVault.AllowUnauthenticated = &b - return nil - }), "vault-allow-unauthenticated", "") - flags.StringVar(&defaultVault.Token, "vault-token", "", "") flags.StringVar(&defaultVault.Addr, "vault-address", "", "") flags.StringVar(&defaultVault.Namespace, "vault-namespace", "", "") flags.StringVar(&defaultVault.Role, "vault-create-from-role", "", "") @@ -302,11 +297,6 @@ func (c *Command) readConfig() *Config { // configuration sources have been merged. defaultVault = config.defaultVault() - // Check to see if we should read the Vault token from the environment - if defaultVault.Token == "" { - defaultVault.Token = os.Getenv("VAULT_TOKEN") - } - // Check to see if we should read the Vault namespace from the environment if defaultVault.Namespace == "" { defaultVault.Namespace = os.Getenv("VAULT_NAMESPACE") @@ -653,12 +643,6 @@ func (c *Command) setupAgent(config *Config, logger hclog.InterceptLogger, logOu } c.httpServers = httpServers - for _, vault := range config.Vaults { - if vault.Token != "" { - logger.Warn("Setting a Vault token in the agent configuration is deprecated and will be removed in Nomad 1.10. Migrate your Vault configuration to use workload identity.", "cluster", vault.Name) - } - } - // If DisableUpdateCheck is not enabled, set up update checking // (DisableUpdateCheck is false by default) if config.DisableUpdateCheck != nil && !*config.DisableUpdateCheck { diff --git a/command/agent/command_test.go b/command/agent/command_test.go index dc3620b9e..6b25c4020 100644 --- a/command/agent/command_test.go +++ b/command/agent/command_test.go @@ -536,7 +536,6 @@ client { } vault { - token = "token-from-config" namespace = "ns-from-config" } `, @@ -555,46 +554,39 @@ vault { checkFn func(*testing.T, *Config) }{ { - name: "vault token and namespace from env var", + name: "namespace from env var", args: []string{ "-config", path.Join(configDir, "base.hcl"), }, env: map[string]string{ - "VAULT_TOKEN": "token-from-env", "VAULT_NAMESPACE": "ns-from-env", }, checkFn: func(t *testing.T, c *Config) { - must.Eq(t, "token-from-env", c.Vaults[0].Token) must.Eq(t, "ns-from-env", c.Vaults[0].Namespace) }, }, { - name: "vault token and namespace from config takes precedence over env var", + name: "namespace from config takes precedence over env var", args: []string{ "-config", path.Join(configDir, "vault.hcl"), }, env: map[string]string{ - "VAULT_TOKEN": "token-from-env", "VAULT_NAMESPACE": "ns-from-env", }, checkFn: func(t *testing.T, c *Config) { - must.Eq(t, "token-from-config", c.Vaults[0].Token) must.Eq(t, "ns-from-config", c.Vaults[0].Namespace) }, }, { - name: "vault token and namespace from flag takes precedence over env var and config", + name: "namespace from flag takes precedence over env var and config", args: []string{ "-config", path.Join(configDir, "vault.hcl"), - "-vault-token", "secret-from-cli", "-vault-namespace", "ns-from-cli", }, env: map[string]string{ - "VAULT_TOKEN": "secret-from-env", "VAULT_NAMESPACE": "ns-from-env", }, checkFn: func(t *testing.T, c *Config) { - must.Eq(t, "secret-from-cli", c.Vaults[0].Token) must.Eq(t, "ns-from-cli", c.Vaults[0].Namespace) }, }, diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index c22f74cb1..b13ce9163 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -260,21 +260,18 @@ var basicConfig = &Config{ }, }}, Vaults: []*config.VaultConfig{{ - Name: structs.VaultDefaultCluster, - Addr: "127.0.0.1:9500", - JWTAuthBackendPath: "nomad_jwt", - ConnectionRetryIntv: 30 * time.Second, - AllowUnauthenticated: &trueValue, - Enabled: &falseValue, - Role: "test_role", - TLSCaFile: "/path/to/ca/file", - TLSCaPath: "/path/to/ca", - TLSCertFile: "/path/to/cert/file", - TLSKeyFile: "/path/to/key/file", - TLSServerName: "foobar", - TLSSkipVerify: &trueValue, - TaskTokenTTL: "1s", - Token: "12345", + Name: structs.VaultDefaultCluster, + Addr: "127.0.0.1:9500", + JWTAuthBackendPath: "nomad_jwt", + ConnectionRetryIntv: 30 * time.Second, + Enabled: &falseValue, + Role: "test_role", + TLSCaFile: "/path/to/ca/file", + TLSCaPath: "/path/to/ca", + TLSCertFile: "/path/to/cert/file", + TLSKeyFile: "/path/to/key/file", + TLSServerName: "foobar", + TLSSkipVerify: &trueValue, DefaultIdentity: &config.WorkloadIdentityConfig{ Audience: []string{"vault.io", "nomad.io"}, Env: pointer.Of(false), @@ -876,13 +873,12 @@ var sample1 = &Config{ VerifySSL: pointer.Of(true), }}, Vaults: []*config.VaultConfig{{ - Name: structs.VaultDefaultCluster, - Enabled: pointer.Of(true), - Role: "nomad-cluster", - Addr: "http://host.example.com:8200", - JWTAuthBackendPath: "jwt-nomad", - ConnectionRetryIntv: 30 * time.Second, - AllowUnauthenticated: pointer.Of(true), + Name: structs.VaultDefaultCluster, + Enabled: pointer.Of(true), + Role: "nomad-cluster", + Addr: "http://host.example.com:8200", + JWTAuthBackendPath: "jwt-nomad", + ConnectionRetryIntv: 30 * time.Second, }}, TLSConfig: &config.TLSConfig{ EnableHTTP: true, @@ -1025,7 +1021,6 @@ func TestConfig_MultipleVault(t *testing.T) { must.Equal(t, config.DefaultVaultConfig(), defaultVault) must.Nil(t, defaultVault.Enabled) // unset must.Eq(t, "https://vault.service.consul:8200", defaultVault.Addr) - must.Eq(t, "", defaultVault.Token) must.Eq(t, "jwt-nomad", defaultVault.JWTAuthBackendPath) // merge in the user's configuration @@ -1040,7 +1035,6 @@ func TestConfig_MultipleVault(t *testing.T) { must.False(t, *defaultVault.Enabled) must.Eq(t, "127.0.0.1:9500", defaultVault.Addr) must.Eq(t, "nomad_jwt", defaultVault.JWTAuthBackendPath) - must.Eq(t, "12345", defaultVault.Token) // add an extra Vault config and override fields in the default fc, err = LoadConfig("testdata/extra-vault." + suffix) @@ -1053,12 +1047,10 @@ func TestConfig_MultipleVault(t *testing.T) { must.Eq(t, structs.VaultDefaultCluster, defaultVault.Name) must.True(t, *defaultVault.Enabled) must.Eq(t, "127.0.0.1:9500", defaultVault.Addr) - must.Eq(t, "abracadabra", defaultVault.Token) must.Eq(t, "alternate", cfg.Vaults[1].Name) must.True(t, *cfg.Vaults[1].Enabled) must.Eq(t, "127.0.0.1:9501", cfg.Vaults[1].Addr) - must.Eq(t, "xyzzy", cfg.Vaults[1].Token) must.Eq(t, "other", cfg.Vaults[2].Name) must.Nil(t, cfg.Vaults[2].Enabled) diff --git a/command/agent/config_test.go b/command/agent/config_test.go index e7e3768ef..b84a9a88f 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -189,17 +189,14 @@ func TestConfig_Merge(t *testing.T) { "Access-Control-Allow-Origin": "*", }, Vaults: []*config.VaultConfig{{ - Name: structs.VaultDefaultCluster, - Token: "1", - AllowUnauthenticated: &falseValue, - TaskTokenTTL: "1", - Addr: "1", - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: &falseValue, - TLSServerName: "1", + Name: structs.VaultDefaultCluster, + Addr: "1", + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: &falseValue, + TLSServerName: "1", }}, Consuls: []*config.ConsulConfig{{ ServerServiceName: "1", @@ -415,19 +412,16 @@ func TestConfig_Merge(t *testing.T) { "Access-Control-Allow-Methods": "GET, POST, OPTIONS", }, Vaults: []*config.VaultConfig{{ - Name: structs.VaultDefaultCluster, - Token: "2", - AllowUnauthenticated: &trueValue, - TaskTokenTTL: "2", - Addr: "2", - TLSCaFile: "2", - TLSCaPath: "2", - TLSCertFile: "2", - TLSKeyFile: "2", - TLSSkipVerify: &trueValue, - TLSServerName: "2", - ConnectionRetryIntv: time.Duration(30000000000), - JWTAuthBackendPath: "jwt", + Name: structs.VaultDefaultCluster, + Addr: "2", + TLSCaFile: "2", + TLSCaPath: "2", + TLSCertFile: "2", + TLSKeyFile: "2", + TLSSkipVerify: &trueValue, + TLSServerName: "2", + ConnectionRetryIntv: time.Duration(30000000000), + JWTAuthBackendPath: "jwt", }}, Consuls: []*config.ConsulConfig{{ Name: "default", diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 4bf7a52ba..db5289f50 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1148,7 +1148,6 @@ func ApiJobToStructJob(job *api.Job) *structs.Job { Payload: job.Payload, Meta: job.Meta, ConsulToken: *job.ConsulToken, - VaultToken: *job.VaultToken, VaultNamespace: *job.VaultNamespace, Constraints: ApiConstraintsToStructs(job.Constraints), Affinities: ApiAffinitiesToStructs(job.Affinities), @@ -1472,7 +1471,6 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup, if apiTask.Vault != nil { structsTask.Vault = &structs.Vault{ Role: apiTask.Vault.Role, - Policies: apiTask.Vault.Policies, Namespace: *apiTask.Vault.Namespace, Cluster: apiTask.Vault.Cluster, Env: *apiTask.Vault.Env, diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 808fcaf46..a50e5419e 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -3019,7 +3019,6 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, }, ConsulToken: pointer.Of("abc123"), - VaultToken: pointer.Of("def456"), VaultNamespace: pointer.Of("ghi789"), Status: pointer.Of("status"), StatusDescription: pointer.Of("status_desc"), @@ -3435,7 +3434,6 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { Role: "nomad-task", Namespace: "ns1", Cluster: structs.VaultDefaultCluster, - Policies: []string{"a", "b", "c"}, Env: true, DisableFile: false, ChangeMode: "c", @@ -3478,7 +3476,6 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, ConsulToken: "abc123", - VaultToken: "def456", } structsJob := ApiJobToStructJob(apiJob) diff --git a/command/agent/operator_endpoint_test.go b/command/agent/operator_endpoint_test.go index 869d698ed..b1054073e 100644 --- a/command/agent/operator_endpoint_test.go +++ b/command/agent/operator_endpoint_test.go @@ -727,8 +727,7 @@ func TestOperator_UpgradeCheckRequest_VaultWorkloadIdentity(t *testing.T) { // Create a test job with a Vault block but without an identity. job := mock.Job() job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Cluster: "default", - Policies: []string{"test"}, + Cluster: "default", } args := structs.JobRegisterRequest{ diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index 5934b971a..00c22317f 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -270,10 +270,7 @@ consul { vault { address = "127.0.0.1:9500" - allow_unauthenticated = true - task_token_ttl = "1s" enabled = false - token = "12345" ca_file = "/path/to/ca/file" ca_path = "/path/to/ca" cert_file = "/path/to/cert/file" diff --git a/command/job_plan.go b/command/job_plan.go index f28dea2ed..4e58fec7a 100644 --- a/command/job_plan.go +++ b/command/job_plan.go @@ -5,7 +5,6 @@ package command import ( "fmt" - "os" "sort" "strings" "time" @@ -67,10 +66,6 @@ Alias: nomad plan * 1: Allocations created or destroyed. * 255: Error determining plan results. - The plan command will set the vault_token of the job based on the following - precedence, going from highest to lowest: the -vault-token flag, the - $VAULT_TOKEN environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the 'submit-job' capability for the job's namespace. @@ -97,18 +92,6 @@ Plan Options: -policy-override Sets the flag to force override any soft mandatory Sentinel policies. - -vault-token - Used to validate if the user submitting the job has permission to run the job - according to its Vault policies. A Vault token must be supplied if the vault - block allow_unauthenticated is disabled in the Nomad server configuration. - If the -vault-token flag is set, the passed Vault token is added to the jobspec - before sending to the Nomad servers. This allows passing the Vault token - without storing it in the job file. This overrides the token found in the - $VAULT_TOKEN environment variable and the vault_token field in the job file. - This token is cleared from the job after validating and cannot be used within - the job executing environment. Use the vault block when templating in a job - with a Vault token. - -vault-namespace If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -137,7 +120,6 @@ func (c *JobPlanCommand) AutocompleteFlags() complete.Flags { "-verbose": complete.PredictNothing, "-json": complete.PredictNothing, "-hcl2-strict": complete.PredictNothing, - "-vault-token": complete.PredictAnything, "-vault-namespace": complete.PredictAnything, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), @@ -155,7 +137,7 @@ func (c *JobPlanCommand) AutocompleteArgs() complete.Predictor { func (c *JobPlanCommand) Name() string { return "job plan" } func (c *JobPlanCommand) Run(args []string) int { var diff, policyOverride, verbose bool - var vaultToken, vaultNamespace string + var vaultNamespace string flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) flagSet.Usage = func() { c.Ui.Output(c.Help()) } @@ -164,7 +146,6 @@ func (c *JobPlanCommand) Run(args []string) int { flagSet.BoolVar(&verbose, "verbose", false, "") flagSet.BoolVar(&c.JobGetter.JSON, "json", false, "") flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "") - flagSet.StringVar(&vaultToken, "vault-token", "", "") flagSet.StringVar(&vaultNamespace, "vault-namespace", "", "") flagSet.Var(&c.JobGetter.Vars, "var", "") flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") @@ -211,21 +192,6 @@ func (c *JobPlanCommand) Run(args []string) int { client.SetNamespace(*n) } - // Parse the Vault token. - if vaultToken == "" { - // Check the environment variable - vaultToken = os.Getenv("VAULT_TOKEN") - } - - if vaultToken != "" { - job.VaultToken = pointer.Of(vaultToken) - } - - // Set the vault token. - if vaultToken != "" { - job.VaultToken = pointer.Of(vaultToken) - } - // Set the vault namespace. if vaultNamespace != "" { job.VaultNamespace = pointer.Of(vaultNamespace) diff --git a/command/job_plan_test.go b/command/job_plan_test.go index 5d1b767fa..e81fd8ff4 100644 --- a/command/job_plan_test.go +++ b/command/job_plan_test.go @@ -188,27 +188,7 @@ func TestPlanCommand_From_Files(t *testing.T) { cmd := &JobPlanCommand{Meta: Meta{Ui: ui}} args := []string{"-address", "http://" + s.HTTPAddr, "testdata/example-vault.nomad"} code := cmd.Run(args) - must.Eq(t, 255, code) - must.StrContains(t, ui.ErrorWriter.String(), "* Vault used in the job but missing Vault token") - }) - - t.Run("vault bad token via flag", func(t *testing.T) { - ui := cli.NewMockUi() - cmd := &JobPlanCommand{Meta: Meta{Ui: ui}} - args := []string{"-address", "http://" + s.HTTPAddr, "-vault-token=abc123", "testdata/example-vault.nomad"} - code := cmd.Run(args) - must.Eq(t, 255, code) - must.StrContains(t, ui.ErrorWriter.String(), "* bad token") - }) - - t.Run("vault bad token via env", func(t *testing.T) { - t.Setenv("VAULT_TOKEN", "abc123") - ui := cli.NewMockUi() - cmd := &JobPlanCommand{Meta: Meta{Ui: ui}} - args := []string{"-address", "http://" + s.HTTPAddr, "testdata/example-vault.nomad"} - code := cmd.Run(args) - must.Eq(t, 255, code) - must.StrContains(t, ui.ErrorWriter.String(), "* bad token") + must.One(t, code) // no client running, fail to place }) } diff --git a/command/job_revert.go b/command/job_revert.go index db5572964..8167dbb81 100644 --- a/command/job_revert.go +++ b/command/job_revert.go @@ -49,10 +49,6 @@ Revert Options: The Consul token used to verify that the caller has access to the Service Identity policies associated in the targeted version of the job. - -vault-token - The Vault token used to verify that the caller has access to the Vault - policies in the targeted version of the job. - -verbose Display full information. ` @@ -90,14 +86,13 @@ func (c *JobRevertCommand) Name() string { return "job revert" } func (c *JobRevertCommand) Run(args []string) int { var detach, verbose bool - var consulToken, vaultToken string + var consulToken string flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } flags.BoolVar(&detach, "detach", false, "") flags.BoolVar(&verbose, "verbose", false, "") flags.StringVar(&consulToken, "consul-token", "", "") - flags.StringVar(&vaultToken, "vault-token", "", "") if err := flags.Parse(args); err != nil { return 1 @@ -130,12 +125,6 @@ func (c *JobRevertCommand) Run(args []string) int { consulToken = os.Getenv("CONSUL_HTTP_TOKEN") } - // Parse the Vault token - if vaultToken == "" { - // Check the environment variable - vaultToken = os.Getenv("VAULT_TOKEN") - } - // Parse the job version or version tag var revertVersion uint64 @@ -161,7 +150,7 @@ func (c *JobRevertCommand) Run(args []string) int { // Prefix lookup matched a single job q := &api.WriteOptions{Namespace: namespace} - resp, _, err := client.Jobs().Revert(jobID, revertVersion, nil, q, consulToken, vaultToken) + resp, _, err := client.Jobs().Revert(jobID, revertVersion, nil, q, consulToken, "") if err != nil { c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s", err)) return 1 diff --git a/command/job_run.go b/command/job_run.go index 1a9a216ce..5bed2ce42 100644 --- a/command/job_run.go +++ b/command/job_run.go @@ -59,10 +59,6 @@ Alias: nomad run precedence, going from highest to lowest: the -consul-token flag, the $CONSUL_HTTP_TOKEN environment variable and finally the value in the job file. - The run command will set the vault_token of the job based on the following - precedence, going from highest to lowest: the -vault-token flag, the - $VAULT_TOKEN environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the 'submit-job' capability for the job's namespace. Jobs that mount CSI volumes require a token with the 'csi-mount-volume' capability for the volume's @@ -126,18 +122,6 @@ Run Options: Nomad server configuration, then a Consul token must be supplied with appropriate service and KV Consul ACL policy permissions. - -vault-token - Used to validate if the user submitting the job has permission to run the job - according to its Vault policies. A Vault token must be supplied if the vault - block allow_unauthenticated is disabled in the Nomad server configuration. - If the -vault-token flag is set, the passed Vault token is added to the jobspec - before sending to the Nomad servers. This allows passing the Vault token - without storing it in the job file. This overrides the token found in the - $VAULT_TOKEN environment variable and the vault_token field in the job file. - This token is cleared from the job after validating and cannot be used within - the job executing environment. Use the vault block when templating in a job - with a Vault token. - -vault-namespace If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -166,7 +150,6 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags { "-verbose": complete.PredictNothing, "-consul-token": complete.PredictNothing, "-consul-namespace": complete.PredictAnything, - "-vault-token": complete.PredictAnything, "-vault-namespace": complete.PredictAnything, "-output": complete.PredictNothing, "-policy-override": complete.PredictNothing, @@ -191,7 +174,7 @@ func (c *JobRunCommand) Name() string { return "job run" } func (c *JobRunCommand) Run(args []string) int { var detach, verbose, output, override, preserveCounts bool - var checkIndexStr, consulToken, consulNamespace, vaultToken, vaultNamespace string + var checkIndexStr, consulToken, consulNamespace, vaultNamespace string var evalPriority int flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -206,7 +189,6 @@ func (c *JobRunCommand) Run(args []string) int { flagSet.StringVar(&checkIndexStr, "check-index", "", "") flagSet.StringVar(&consulToken, "consul-token", "", "") flagSet.StringVar(&consulNamespace, "consul-namespace", "", "") - flagSet.StringVar(&vaultToken, "vault-token", "", "") flagSet.StringVar(&vaultNamespace, "vault-namespace", "", "") flagSet.Var(&c.JobGetter.Vars, "var", "") flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") @@ -278,16 +260,6 @@ func (c *JobRunCommand) Run(args []string) int { job.ConsulNamespace = pointer.Of(consulNamespace) } - // Parse the Vault token - if vaultToken == "" { - // Check the environment variable - vaultToken = os.Getenv("VAULT_TOKEN") - } - - if vaultToken != "" { - job.VaultToken = pointer.Of(vaultToken) - } - if vaultNamespace != "" { job.VaultNamespace = pointer.Of(vaultNamespace) } diff --git a/command/job_validate.go b/command/job_validate.go index 9c75ef417..cf3d563a3 100644 --- a/command/job_validate.go +++ b/command/job_validate.go @@ -5,7 +5,6 @@ package command import ( "fmt" - "os" "strings" "github.com/hashicorp/go-multierror" @@ -33,10 +32,6 @@ Alias: nomad validate it is read from the file at the supplied path or downloaded and read from URL specified. - The run command will set the vault_token of the job based on the following - precedence, going from highest to lowest: the -vault-token flag, the - $VAULT_TOKEN environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the 'read-job' capability for the job's namespace. @@ -56,18 +51,6 @@ Validate Options: has been supplied which is not defined within the root variables. Defaults to true. - -vault-token - Used to validate if the user submitting the job has permission to run the job - according to its Vault policies. A Vault token must be supplied if the vault - block allow_unauthenticated is disabled in the Nomad server configuration. - If the -vault-token flag is set, the passed Vault token is added to the jobspec - before sending to the Nomad servers. This allows passing the Vault token - without storing it in the job file. This overrides the token found in the - $VAULT_TOKEN environment variable and the vault_token field in the job file. - This token is cleared from the job after validating and cannot be used within - the job executing environment. Use the vault block when templating in a job - with a Vault token. - -vault-namespace If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -88,7 +71,6 @@ func (c *JobValidateCommand) Synopsis() string { func (c *JobValidateCommand) AutocompleteFlags() complete.Flags { return complete.Flags{ "-hcl2-strict": complete.PredictNothing, - "-vault-token": complete.PredictAnything, "-vault-namespace": complete.PredictAnything, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), @@ -106,13 +88,12 @@ func (c *JobValidateCommand) AutocompleteArgs() complete.Predictor { func (c *JobValidateCommand) Name() string { return "job validate" } func (c *JobValidateCommand) Run(args []string) int { - var vaultToken, vaultNamespace string + var vaultNamespace string flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) flagSet.Usage = func() { c.Ui.Output(c.Help()) } flagSet.BoolVar(&c.JobGetter.JSON, "json", false, "") flagSet.BoolVar(&c.JobGetter.Strict, "hcl2-strict", true, "") - flagSet.StringVar(&vaultToken, "vault-token", "", "") flagSet.StringVar(&vaultNamespace, "vault-namespace", "", "") flagSet.Var(&c.JobGetter.Vars, "var", "") flagSet.Var(&c.JobGetter.VarFiles, "var-file", "") @@ -153,16 +134,6 @@ func (c *JobValidateCommand) Run(args []string) int { client.SetRegion(*r) } - // Parse the Vault token - if vaultToken == "" { - // Check the environment variable - vaultToken = os.Getenv("VAULT_TOKEN") - } - - if vaultToken != "" { - job.VaultToken = pointer.Of(vaultToken) - } - if vaultNamespace != "" { job.VaultNamespace = pointer.Of(vaultNamespace) } diff --git a/command/job_validate_test.go b/command/job_validate_test.go index 5a6253c7a..643b32cc4 100644 --- a/command/job_validate_test.go +++ b/command/job_validate_test.go @@ -48,27 +48,7 @@ func TestValidateCommand_Files(t *testing.T) { cmd := &JobValidateCommand{Meta: Meta{Ui: ui}} args := []string{"-address", "http://" + s.HTTPAddr, "testdata/example-vault.nomad"} code := cmd.Run(args) - must.StrContains(t, ui.ErrorWriter.String(), "* Vault used in the job but missing Vault token") - must.One(t, code) - }) - - t.Run("vault bad token via flag", func(t *testing.T) { - ui := cli.NewMockUi() - cmd := &JobValidateCommand{Meta: Meta{Ui: ui}} - args := []string{"-address", "http://" + s.HTTPAddr, "-vault-token=abc123", "testdata/example-vault.nomad"} - code := cmd.Run(args) - must.StrContains(t, ui.ErrorWriter.String(), "* bad token") - must.One(t, code) - }) - - t.Run("vault token bad via env", func(t *testing.T) { - t.Setenv("VAULT_TOKEN", "abc123") - ui := cli.NewMockUi() - cmd := &JobValidateCommand{Meta: Meta{Ui: ui}} - args := []string{"-address", "http://" + s.HTTPAddr, "testdata/example-vault.nomad"} - code := cmd.Run(args) - must.StrContains(t, ui.ErrorWriter.String(), "* bad token") - must.One(t, code) + must.Zero(t, code) }) } diff --git a/command/setup_vault_test.go b/command/setup_vault_test.go index 36adf8494..db30f566c 100644 --- a/command/setup_vault_test.go +++ b/command/setup_vault_test.go @@ -109,7 +109,7 @@ test default batch pending } ], "OutdatedNodes": [], - "VaultTokens": [] + "VaultTokens": null } `, *job.CreateIndex, *job.ModifyIndex, *job.SubmitTime), }, diff --git a/go.mod b/go.mod index e2012619f..4c2ba323d 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,6 @@ require ( google.golang.org/grpc v1.69.4 google.golang.org/protobuf v1.36.3 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 - gopkg.in/tomb.v2 v2.0.0-20140626144623-14b3d72120e8 oss.indeed.com/go/libtime v1.6.0 ) diff --git a/go.sum b/go.sum index d7ccaac5b..84c9381e9 100644 --- a/go.sum +++ b/go.sum @@ -2498,8 +2498,6 @@ gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/tomb.v2 v2.0.0-20140626144623-14b3d72120e8 h1:EQ3aCG3c3nkUNxx6quE0Ux47RYExj7cJyRMxUXqPf6I= -gopkg.in/tomb.v2 v2.0.0-20140626144623-14b3d72120e8/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/helper/raftutil/fsm.go b/helper/raftutil/fsm.go index 1cff0b0da..0c009de52 100644 --- a/helper/raftutil/fsm.go +++ b/helper/raftutil/fsm.go @@ -214,7 +214,6 @@ func StateAsMap(store *state.StateStore) map[string][]interface{} { "SITokenAccessors": toArray(store.SITokenAccessors(nil)), "ScalingEvents": toArray(store.ScalingEvents(nil)), "ScalingPolicies": toArray(store.ScalingPolicies(nil)), - "VaultAccessors": toArray(store.VaultAccessors(nil)), } insertEnterpriseState(result, store) diff --git a/nomad/fsm.go b/nomad/fsm.go index 1fa20e010..b81b633af 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -35,7 +35,6 @@ const ( AllocSnapshot SnapshotType = 4 PeriodicLaunchSnapshot SnapshotType = 6 JobSummarySnapshot SnapshotType = 7 - VaultAccessorSnapshot SnapshotType = 8 JobVersionSnapshot SnapshotType = 9 DeploymentSnapshot SnapshotType = 10 ACLPolicySnapshot SnapshotType = 11 @@ -63,6 +62,11 @@ const ( // Deprecated: Nomad no longer supports TimeTable snapshots since 1.9.2 TimeTableSnapshot SnapshotType = 5 + // VaultAccessorSnapshot + // Deprecated: Nomad no longer supports the Vault legacy token based + // workflow and therefore accessor snapshots since 1.10.0. + VaultAccessorSnapshot SnapshotType = 8 + // EventSinkSnapshot // Deprecated: Nomad no longer supports EventSink snapshots since 1.0 EventSinkSnapshot SnapshotType = 20 @@ -282,9 +286,9 @@ func (n *nomadFSM) Apply(log *raft.Log) interface{} { case structs.ReconcileJobSummariesRequestType: return n.applyReconcileSummaries(buf[1:], log.Index) case structs.VaultAccessorRegisterRequestType: - return n.applyUpsertVaultAccessor(buf[1:], log.Index) + return nil case structs.VaultAccessorDeregisterRequestType: - return n.applyDeregisterVaultAccessor(buf[1:], log.Index) + return nil case structs.ApplyPlanResultsRequestType: return n.applyPlanResults(msgType, buf[1:], log.Index) case structs.DeploymentStatusUpdateRequestType: @@ -1034,39 +1038,6 @@ func (n *nomadFSM) applyUpsertNodeEvent(msgType structs.MessageType, buf []byte, return nil } -// applyUpsertVaultAccessor stores the Vault accessors for a given allocation -// and task -func (n *nomadFSM) applyUpsertVaultAccessor(buf []byte, index uint64) interface{} { - defer metrics.MeasureSince([]string{"nomad", "fsm", "upsert_vault_accessor"}, time.Now()) - var req structs.VaultAccessorsRequest - if err := structs.Decode(buf, &req); err != nil { - panic(fmt.Errorf("failed to decode request: %v", err)) - } - - if err := n.state.UpsertVaultAccessor(index, req.Accessors); err != nil { - n.logger.Error("UpsertVaultAccessor failed", "error", err) - return err - } - - return nil -} - -// applyDeregisterVaultAccessor deregisters a set of Vault accessors -func (n *nomadFSM) applyDeregisterVaultAccessor(buf []byte, index uint64) interface{} { - defer metrics.MeasureSince([]string{"nomad", "fsm", "deregister_vault_accessor"}, time.Now()) - var req structs.VaultAccessorsRequest - if err := structs.Decode(buf, &req); err != nil { - panic(fmt.Errorf("failed to decode request: %v", err)) - } - - if err := n.state.DeleteVaultAccessors(index, req.Accessors); err != nil { - n.logger.Error("DeregisterVaultAccessor failed", "error", err) - return err - } - - return nil -} - func (n *nomadFSM) applyUpsertSIAccessor(buf []byte, index uint64) interface{} { defer metrics.MeasureSince([]string{"nomad", "fsm", "upsert_si_accessor"}, time.Now()) var request structs.SITokenAccessorsRequest @@ -1669,15 +1640,13 @@ func (n *nomadFSM) restoreImpl(old io.ReadCloser, filter *FSMFilter) error { } case VaultAccessorSnapshot: + // COMPAT: Nomad 1.10.0 removed the Vault accessor table. This case + // kept to gracefully handle snapshot requests which include an + // object from this. accessor := new(structs.VaultAccessor) if err := dec.Decode(accessor); err != nil { return err } - if filter.Include(accessor) { - if err := restore.VaultAccessorRestore(accessor); err != nil { - return err - } - } case ServiceIdentityTokenAccessorSnapshot: accessor := new(structs.SITokenAccessor) @@ -2512,10 +2481,6 @@ func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error { sink.Cancel() return err } - if err := s.persistVaultAccessors(sink, encoder); err != nil { - sink.Cancel() - return err - } if err := s.persistSITokenAccessors(sink, encoder); err != nil { sink.Cancel() return err @@ -2820,31 +2785,6 @@ func (s *nomadSnapshot) persistJobSummaries(sink raft.SnapshotSink, return nil } -func (s *nomadSnapshot) persistVaultAccessors(sink raft.SnapshotSink, - encoder *codec.Encoder) error { - - ws := memdb.NewWatchSet() - accessors, err := s.snap.VaultAccessors(ws) - if err != nil { - return err - } - - for { - raw := accessors.Next() - if raw == nil { - break - } - - accessor := raw.(*structs.VaultAccessor) - - sink.Write([]byte{byte(VaultAccessorSnapshot)}) - if err := encoder.Encode(accessor); err != nil { - return err - } - } - return nil -} - func (s *nomadSnapshot) persistSITokenAccessors(sink raft.SnapshotSink, encoder *codec.Encoder) error { ws := memdb.NewWatchSet() accessors, err := s.snap.SITokenAccessors(ws) diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 875cba715..0e3fbb217 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -1552,87 +1552,6 @@ func TestFSM_UpdateAllocDesiredTransition(t *testing.T) { require.True(*out2.DesiredTransition.Migrate) } -func TestFSM_UpsertVaultAccessor(t *testing.T) { - ci.Parallel(t) - fsm := testFSM(t) - fsm.blockedEvals.SetEnabled(true) - - va := mock.VaultAccessor() - va2 := mock.VaultAccessor() - req := structs.VaultAccessorsRequest{ - Accessors: []*structs.VaultAccessor{va, va2}, - } - buf, err := structs.Encode(structs.VaultAccessorRegisterRequestType, req) - if err != nil { - t.Fatalf("err: %v", err) - } - - resp := fsm.Apply(makeLog(buf)) - if resp != nil { - t.Fatalf("resp: %v", resp) - } - - // Verify we are registered - ws := memdb.NewWatchSet() - out1, err := fsm.State().VaultAccessor(ws, va.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - if out1 == nil { - t.Fatalf("not found!") - } - if out1.CreateIndex != 1 { - t.Fatalf("bad index: %d", out1.CreateIndex) - } - out2, err := fsm.State().VaultAccessor(ws, va2.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - if out2 == nil { - t.Fatalf("not found!") - } - if out1.CreateIndex != 1 { - t.Fatalf("bad index: %d", out2.CreateIndex) - } -} - -func TestFSM_DeregisterVaultAccessor(t *testing.T) { - ci.Parallel(t) - fsm := testFSM(t) - fsm.blockedEvals.SetEnabled(true) - - va := mock.VaultAccessor() - va2 := mock.VaultAccessor() - accessors := []*structs.VaultAccessor{va, va2} - - // Insert the accessors - if err := fsm.State().UpsertVaultAccessor(1000, accessors); err != nil { - t.Fatalf("bad: %v", err) - } - - req := structs.VaultAccessorsRequest{ - Accessors: accessors, - } - buf, err := structs.Encode(structs.VaultAccessorDeregisterRequestType, req) - if err != nil { - t.Fatalf("err: %v", err) - } - - resp := fsm.Apply(makeLog(buf)) - if resp != nil { - t.Fatalf("resp: %v", resp) - } - - ws := memdb.NewWatchSet() - out1, err := fsm.State().VaultAccessor(ws, va.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - if out1 != nil { - t.Fatalf("not deleted!") - } -} - func TestFSM_UpsertSITokenAccessor(t *testing.T) { ci.Parallel(t) r := require.New(t) @@ -2604,29 +2523,6 @@ func TestFSM_SnapshotRestore_JobSummary(t *testing.T) { } } -func TestFSM_SnapshotRestore_VaultAccessors(t *testing.T) { - ci.Parallel(t) - // Add some state - fsm := testFSM(t) - state := fsm.State() - a1 := mock.VaultAccessor() - a2 := mock.VaultAccessor() - state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a1, a2}) - - // Verify the contents - fsm2 := testSnapshotRestore(t, fsm) - state2 := fsm2.State() - ws := memdb.NewWatchSet() - out1, _ := state2.VaultAccessor(ws, a1.Accessor) - out2, _ := state2.VaultAccessor(ws, a2.Accessor) - if !reflect.DeepEqual(a1, out1) { - t.Fatalf("bad: \n%#v\n%#v", out1, a1) - } - if !reflect.DeepEqual(a2, out2) { - t.Fatalf("bad: \n%#v\n%#v", out2, a2) - } -} - func TestFSM_SnapshotRestore_JobVersions(t *testing.T) { ci.Parallel(t) // Add some state diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index cdb73632e..e056be7a4 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -312,9 +312,6 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis } } - // Clear the Vault token - args.Job.VaultToken = "" - // Clear the Consul token args.Job.ConsulToken = "" @@ -653,7 +650,6 @@ func (j *Job) Revert(args *structs.JobRevertRequest, reply *structs.JobRegisterR // Build the register request revJob := jobV.Copy() - revJob.VaultToken = args.VaultToken // use vault token from revert to perform (re)registration revJob.ConsulToken = args.ConsulToken // use consul token from revert to perform (re)registration // Clear out the VersionTag to prevent tag duplication diff --git a/nomad/job_endpoint_hook_vault.go b/nomad/job_endpoint_hook_vault.go index 8244c24a1..060cbd385 100644 --- a/nomad/job_endpoint_hook_vault.go +++ b/nomad/job_endpoint_hook_vault.go @@ -4,15 +4,9 @@ package nomad import ( - "context" - "errors" "fmt" - "slices" - "strings" - "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" - vapi "github.com/hashicorp/vault/api" ) // jobVaultHook is an job registration admission controller for Vault blocks. @@ -30,7 +24,6 @@ func (h jobVaultHook) Validate(job *structs.Job) ([]error, error) { return nil, nil } - requiresToken := false for _, tg := range vaultBlocks { for _, vaultBlock := range tg { vconf := h.srv.config.VaultConfigs[vaultBlock.Cluster] @@ -38,78 +31,13 @@ func (h jobVaultHook) Validate(job *structs.Job) ([]error, error) { return nil, fmt.Errorf("Vault %q not enabled but used in the job", vaultBlock.Cluster) } - if vconf.DefaultIdentity == nil && !vconf.AllowsUnauthenticated() { - requiresToken = true - } } } - err := h.validateClustersForNamespace(job, vaultBlocks) - if err != nil { + // Check namespaces. + if err := h.validateNamespaces(vaultBlocks); err != nil { return nil, err } - // Return early if Vault configuration doesn't require authentication. - if !requiresToken { - return nil, nil - } - - // At this point the job has a vault block and the server requires - // authentication, so check if the user has the right permissions. - if job.VaultToken == "" { - return nil, fmt.Errorf("Vault used in the job but missing Vault token") - } - - warnings := []error{ - errors.New("Setting a Vault token when submitting a job is deprecated and will be removed in Nomad 1.9. Migrate your Vault configuration to use workload identity")} - - tokenSecret, err := h.srv.vault.LookupToken(context.Background(), job.VaultToken) - if err != nil { - return warnings, fmt.Errorf("failed to lookup Vault token: %v", err) - } - - // Check namespaces. - err = h.validateNamespaces(vaultBlocks, tokenSecret) - if err != nil { - return warnings, err - } - - // Check policies. - err = h.validatePolicies(vaultBlocks, tokenSecret) - if err != nil { - return warnings, err - } - - return warnings, nil -} - -// validatePolicies returns an error if the job contains Vault blocks that -// require policies that the request token is not allowed to access. -func (jobVaultHook) validatePolicies( - blocks map[string]map[string]*structs.Vault, - token *vapi.Secret, -) error { - - jobPolicies := structs.VaultPoliciesSet(blocks) - if len(jobPolicies) == 0 { - return nil - } - - allowedPolicies, err := token.TokenPolicies() - if err != nil { - return fmt.Errorf("failed to lookup Vault token policies: %v", err) - } - - // If we are given a root token it can access all policies - if slices.Contains(allowedPolicies, "root") { - return nil - } - - subset, offending := helper.IsSubset(allowedPolicies, jobPolicies) - if !subset { - return fmt.Errorf("Vault token doesn't allow access to the following policies: %s", - strings.Join(offending, ", ")) - } - - return nil + return nil, h.validateClustersForNamespace(job, vaultBlocks) } diff --git a/nomad/job_endpoint_hook_vault_ce.go b/nomad/job_endpoint_hook_vault_ce.go index 2bcfa25e9..d4949323f 100644 --- a/nomad/job_endpoint_hook_vault_ce.go +++ b/nomad/job_endpoint_hook_vault_ce.go @@ -11,15 +11,10 @@ import ( "strings" "github.com/hashicorp/nomad/nomad/structs" - vapi "github.com/hashicorp/vault/api" ) -// validateNamespaces returns an error if the job contains multiple Vault -// namespaces. -func (jobVaultHook) validateNamespaces( - blocks map[string]map[string]*structs.Vault, - token *vapi.Secret, -) error { +// validateNamespaces returns an error if the job contains any Vault namespaces. +func (jobVaultHook) validateNamespaces(blocks map[string]map[string]*structs.Vault) error { requestedNamespaces := structs.VaultNamespaceSet(blocks) if len(requestedNamespaces) > 0 { diff --git a/nomad/job_endpoint_hook_vault_ce_test.go b/nomad/job_endpoint_hook_vault_ce_test.go index 7cd5014db..f5a3f4d00 100644 --- a/nomad/job_endpoint_hook_vault_ce_test.go +++ b/nomad/job_endpoint_hook_vault_ce_test.go @@ -24,7 +24,6 @@ func TestJobEndpointHook_VaultCE(t *testing.T) { srv, cleanup := TestServer(t, func(c *Config) { c.NumSchedulers = 0 c.VaultConfigs[structs.VaultDefaultCluster].Enabled = pointer.Of(true) - c.VaultConfigs[structs.VaultDefaultCluster].AllowUnauthenticated = pointer.Of(false) c.VaultConfigs[structs.VaultDefaultCluster].DefaultIdentity = &config.WorkloadIdentityConfig{ Name: "vault_default", Audience: []string{"vault.io"}, diff --git a/nomad/job_endpoint_hooks.go b/nomad/job_endpoint_hooks.go index 3d2d78d94..e32781481 100644 --- a/nomad/job_endpoint_hooks.go +++ b/nomad/job_endpoint_hooks.go @@ -581,23 +581,9 @@ func (v *jobValidate) validateVaultIdentity(t *structs.Task, okForIdentity bool) ) } - // Tasks using the default cluster but without a Vault identity will - // use the legacy flow. - if len(t.Vault.Policies) == 0 { - return warnings, fmt.Errorf("Task %s has a Vault block with an empty list of policies", t.Name) - } - return warnings, nil } - // Warn if tasks is using identity-based flow with the deprecated policies - // field. - if len(t.Vault.Policies) > 0 { - warnings = append(warnings, fmt.Errorf( - "Task %s has a Vault block with policies but uses workload identity to authenticate with Vault, policies will be ignored", - t.Name, - )) - } return warnings, nil } diff --git a/nomad/job_endpoint_hooks_test.go b/nomad/job_endpoint_hooks_test.go index 886156fd1..d0e83c8a1 100644 --- a/nomad/job_endpoint_hooks_test.go +++ b/nomad/job_endpoint_hooks_test.go @@ -192,21 +192,6 @@ func Test_jobValidate_Validate_vault(t *testing.T) { TTL: time.Hour, }}, }, - { - name: "no error when task uses legacy flow with default cluster", - inputTaskVault: &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"nomad-workload"}, - }, - }, - { - name: "error when not using vault identity and vault block is missing policies", - inputTaskVault: &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - }, - inputTaskIdentities: nil, - expectedErr: "Vault block with an empty list of policies", - }, { name: "error when no identity is available for non-default cluster", inputTaskVault: &structs.Vault{ @@ -219,36 +204,6 @@ func Test_jobValidate_Validate_vault(t *testing.T) { }, expectedErr: "does not have an identity named vault_other", }, - { - name: "warn when using default vault identity but task has vault policies", - inputTaskVault: &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"nomad-workload"}, - }, - inputTaskIdentities: nil, - inputConfig: map[string]*config.VaultConfig{ - structs.VaultDefaultCluster: { - DefaultIdentity: &config.WorkloadIdentityConfig{ - Audience: []string{"vault.io"}, - TTL: pointer.Of(time.Hour), - }, - }, - }, - expectedWarns: []string{"policies will be ignored"}, - }, - { - name: "warn when using task vault identity but task has vault policies", - inputTaskVault: &structs.Vault{ - Cluster: structs.VaultDefaultCluster, - Policies: []string{"nomad-workload"}, - }, - inputTaskIdentities: []*structs.WorkloadIdentity{{ - Name: "vault_default", - Audience: []string{"vault.io"}, - TTL: time.Hour, - }}, - expectedWarns: []string{"policies will be ignored"}, - }, { name: "warn when vault identity is provided but task does not have vault block", inputTaskVault: nil, diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index cd7861574..99d5acd68 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -1557,10 +1557,9 @@ func TestJobEndpoint_Register_Vault_Disabled(t *testing.T) { codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) - // Create the register request with a job asking for a vault policy + // Create the register request with a job using Vault. job := mock.Job() job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{"foo"}, ChangeMode: structs.VaultChangeModeRestart, } req := &structs.JobRegisterRequest{ @@ -1579,63 +1578,6 @@ func TestJobEndpoint_Register_Vault_Disabled(t *testing.T) { } } -// TestJobEndpoint_Register_Vault_AllowUnauthenticated asserts submitting a job -// with a Vault policy but without a Vault token is *succeeds* if -// allow_unauthenticated=true. -func TestJobEndpoint_Register_Vault_AllowUnauthenticated(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault and allow authenticated - tr := true - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &tr - - // Replace the Vault Client on the server - s1.vault = &TestVaultClient{} - - // Create the register request with a job asking for a vault policy - job := mock.Job() - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{"foo"}, - ChangeMode: structs.VaultChangeModeRestart, - } - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) - if err != nil { - t.Fatalf("bad: %v", err) - } - - // Check for the job in the FSM - state := s1.fsm.State() - ws := memdb.NewWatchSet() - out, err := state.JobByID(ws, job.Namespace, job.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out == nil { - t.Fatalf("expected job") - } - if out.CreateIndex != resp.JobModifyIndex { - t.Fatalf("index mis-match") - } -} - // TestJobEndpoint_Register_Vault_OverrideConstraint asserts that job // submitters can specify their own Vault constraint to override the // automatically injected one. @@ -1652,15 +1594,10 @@ func TestJobEndpoint_Register_Vault_OverrideConstraint(t *testing.T) { // Enable vault and allow authenticated tr := true s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &tr - - // Replace the Vault Client on the server - s1.vault = &TestVaultClient{} // Create the register request with a job asking for a vault policy job := mock.Job() job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{"foo"}, ChangeMode: structs.VaultChangeModeRestart, } @@ -1701,187 +1638,6 @@ func TestJobEndpoint_Register_Vault_OverrideConstraint(t *testing.T) { must.True(t, job.TaskGroups[0].Tasks[0].Constraints[0].Equal(outConstraints[0])) } -func TestJobEndpoint_Register_Vault_NoToken(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f - - // Replace the Vault Client on the server - s1.vault = &TestVaultClient{} - - // Create the register request with a job asking for a vault policy but - // don't send a Vault token - job := mock.Job() - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{"foo"}, - ChangeMode: structs.VaultChangeModeRestart, - } - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) - if err == nil || !strings.Contains(err.Error(), "missing Vault token") { - t.Fatalf("expected Vault not enabled error: %v", err) - } -} - -func TestJobEndpoint_Register_Vault_Policies(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - // Add three tokens: one that allows the requesting policy, one that does - // not and one that returns an error - policy := "foo" - - badToken := uuid.Generate() - badPolicies := []string{"a", "b", "c"} - tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) - - goodToken := uuid.Generate() - goodPolicies := []string{"foo", "bar", "baz"} - tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) - - rootToken := uuid.Generate() - rootPolicies := []string{"root"} - tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) - - errToken := uuid.Generate() - expectedErr := fmt.Errorf("return errors from vault") - tvc.SetLookupTokenError(errToken, expectedErr) - - // Create the register request with a job asking for a vault policy but - // send the bad Vault token - job := mock.Job() - job.VaultToken = badToken - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{policy}, - ChangeMode: structs.VaultChangeModeRestart, - } - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) - if err == nil || !strings.Contains(err.Error(), - "doesn't allow access to the following policies: "+policy) { - t.Fatalf("expected permission denied error: %v", err) - } - - // Use the err token - job.VaultToken = errToken - err = msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp) - if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { - t.Fatalf("expected permission denied error: %v", err) - } - - // Use the good token - job.VaultToken = goodToken - - // Fetch the response - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - - // Check for the job in the FSM - state := s1.fsm.State() - ws := memdb.NewWatchSet() - out, err := state.JobByID(ws, job.Namespace, job.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out == nil { - t.Fatalf("expected job") - } - if out.CreateIndex != resp.JobModifyIndex { - t.Fatalf("index mis-match") - } - if out.VaultToken != "" { - t.Fatalf("vault token not cleared") - } - - // Check that an implicit constraints were created for Vault and Consul. - constraints := out.TaskGroups[0].Constraints - if l := len(constraints); l != 2 { - t.Fatalf("Unexpected number of tests: %v", l) - } - - require.ElementsMatch(t, constraints, []*structs.Constraint{consulServiceDiscoveryConstraint, vaultConstraint}) - - // Create the register request with another job asking for a vault policy but - // send the root Vault token - job2 := mock.Job() - job2.VaultToken = rootToken - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{policy}, - ChangeMode: structs.VaultChangeModeRestart, - } - req = &structs.JobRegisterRequest{ - Job: job2, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - - // Check for the job in the FSM - out, err = state.JobByID(ws, job2.Namespace, job2.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out == nil { - t.Fatalf("expected job") - } - if out.CreateIndex != resp.JobModifyIndex { - t.Fatalf("index mis-match") - } - if out.VaultToken != "" { - t.Fatalf("vault token not cleared") - } -} - func TestJobEndpoint_Register_Vault_MultiNamespaces(t *testing.T) { ci.Parallel(t) @@ -1893,25 +1649,13 @@ func TestJobEndpoint_Register_Vault_MultiNamespaces(t *testing.T) { testutil.WaitForLeader(t, s1.RPC) // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - goodToken := uuid.Generate() - goodPolicies := []string{"foo", "bar", "baz"} - tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) + s1.config.GetDefaultVault().Enabled = pointer.Of(true) // Create the register request with a job asking for a vault policy but // don't send a Vault token job := mock.Job() - job.VaultToken = goodToken job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ Namespace: "ns1", - Policies: []string{"foo"}, ChangeMode: structs.VaultChangeModeRestart, } req := &structs.JobRegisterRequest{ @@ -2749,219 +2493,6 @@ func TestJobEndpoint_Revert(t *testing.T) { } } -func TestJobEndpoint_Revert_Vault_NoToken(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - // Add three tokens: one that allows the requesting policy, one that does - // not and one that returns an error - policy := "foo" - - goodToken := uuid.Generate() - goodPolicies := []string{"foo", "bar", "baz"} - tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) - - // Create the initial register request - job := mock.Job() - job.VaultToken = goodToken - job.Priority = 100 - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{policy}, - ChangeMode: structs.VaultChangeModeRestart, - } - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Reregister again to get another version - job2 := job.Copy() - job2.Priority = 1 - req = &structs.JobRegisterRequest{ - Job: job2, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - revertReq := &structs.JobRevertRequest{ - JobID: job.ID, - JobVersion: 1, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) - if err == nil || !strings.Contains(err.Error(), "current version") { - t.Fatalf("expected current version err: %v", err) - } - - // Create revert request and enforcing it be at version 1 - revertReq = &structs.JobRevertRequest{ - JobID: job.ID, - JobVersion: 0, - EnforcePriorVersion: pointer.Of(uint64(1)), - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) - if err == nil || !strings.Contains(err.Error(), "missing Vault token") { - t.Fatalf("expected Vault not enabled error: %v", err) - } -} - -// TestJobEndpoint_Revert_Vault_Policies asserts that job revert uses the -// revert request's Vault token when authorizing policies. -func TestJobEndpoint_Revert_Vault_Policies(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - // Add three tokens: one that allows the requesting policy, one that does - // not and one that returns an error - policy := "foo" - - badToken := uuid.Generate() - badPolicies := []string{"a", "b", "c"} - tvc.SetLookupTokenAllowedPolicies(badToken, badPolicies) - - registerGoodToken := uuid.Generate() - goodPolicies := []string{"foo", "bar", "baz"} - tvc.SetLookupTokenAllowedPolicies(registerGoodToken, goodPolicies) - - revertGoodToken := uuid.Generate() - revertGoodPolicies := []string{"foo", "bar_revert", "baz_revert"} - tvc.SetLookupTokenAllowedPolicies(revertGoodToken, revertGoodPolicies) - - rootToken := uuid.Generate() - rootPolicies := []string{"root"} - tvc.SetLookupTokenAllowedPolicies(rootToken, rootPolicies) - - errToken := uuid.Generate() - expectedErr := fmt.Errorf("return errors from vault") - tvc.SetLookupTokenError(errToken, expectedErr) - - // Create the initial register request - job := mock.Job() - job.VaultToken = registerGoodToken - job.Priority = 100 - job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{policy}, - ChangeMode: structs.VaultChangeModeRestart, - } - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Reregister again to get another version - job2 := job.Copy() - job2.Priority = 1 - req = &structs.JobRegisterRequest{ - Job: job2, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - if err := msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Create the revert request with the bad Vault token - revertReq := &structs.JobRevertRequest{ - JobID: job.ID, - JobVersion: 0, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - VaultToken: badToken, - } - - // Fetch the response - err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) - if err == nil || !strings.Contains(err.Error(), - "doesn't allow access to the following policies: "+policy) { - t.Fatalf("expected permission denied error: %v", err) - } - - // Use the err token - revertReq.VaultToken = errToken - err = msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp) - if err == nil || !strings.Contains(err.Error(), expectedErr.Error()) { - t.Fatalf("expected permission denied error: %v", err) - } - - // Use a good token - revertReq.VaultToken = revertGoodToken - if err := msgpackrpc.CallWithCodec(codec, "Job.Revert", revertReq, &resp); err != nil { - t.Fatalf("bad: %v", err) - } -} - func TestJobEndpoint_Revert_ACL(t *testing.T) { ci.Parallel(t) require := require.New(t) @@ -6659,24 +6190,11 @@ func TestJobEndpoint_ImplicitConstraints_Vault(t *testing.T) { testutil.WaitForLeader(t, s1.RPC) // Enable vault - tr, f := true, false - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &f + s1.config.GetDefaultVault().Enabled = pointer.Of(true) - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - policy := "foo" - goodToken := uuid.Generate() - goodPolicies := []string{"foo", "bar", "baz"} - tvc.SetLookupTokenAllowedPolicies(goodToken, goodPolicies) - - // Create the register request with a job asking for a vault policy + // Create the register request with a job using Vault. job := mock.Job() - job.VaultToken = goodToken job.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Policies: []string{policy}, ChangeMode: structs.VaultChangeModeRestart, } req := &structs.JobRegisterRequest{ diff --git a/nomad/leader.go b/nomad/leader.go index 0ef08b74c..877c19026 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -405,9 +405,6 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { } } - // Activate the vault client - s.vault.SetActive(true) - // Enable the periodic dispatcher, since we are now the leader. s.periodicDispatcher.SetEnabled(true) @@ -500,11 +497,6 @@ func (s *Server) establishLeadership(stopCh chan struct{}) error { return err } - // Cleanup orphaned Vault token accessors - if err := s.revokeVaultAccessorsOnRestore(); err != nil { - return err - } - // Cleanup orphaned Service Identity token accessors if err := s.revokeSITokenAccessorsOnRestore(); err != nil { return err @@ -831,60 +823,6 @@ func (s *Server) restoreEvals() error { return nil } -// revokeVaultAccessorsOnRestore is used to restore Vault accessors that should be -// revoked. -func (s *Server) revokeVaultAccessorsOnRestore() error { - // An accessor should be revoked if its allocation or node is terminal - ws := memdb.NewWatchSet() - state := s.fsm.State() - iter, err := state.VaultAccessors(ws) - if err != nil { - return fmt.Errorf("failed to get vault accessors: %v", err) - } - - var revoke []*structs.VaultAccessor - for { - raw := iter.Next() - if raw == nil { - break - } - - va := raw.(*structs.VaultAccessor) - - // Check the allocation - alloc, err := state.AllocByID(ws, va.AllocID) - if err != nil { - return fmt.Errorf("failed to lookup allocation %q: %v", va.AllocID, err) - } - if alloc == nil || alloc.Terminated() { - // No longer running and should be revoked - revoke = append(revoke, va) - continue - } - - // Check the node - node, err := state.NodeByID(ws, va.NodeID) - if err != nil { - return fmt.Errorf("failed to lookup node %q: %v", va.NodeID, err) - } - if node == nil || node.TerminalStatus() { - // Node is terminal so any accessor from it should be revoked - revoke = append(revoke, va) - continue - } - } - - if len(revoke) != 0 { - s.logger.Info("revoking vault accessors after becoming leader", "accessors", len(revoke)) - - if err := s.vault.MarkForRevocation(revoke); err != nil { - return fmt.Errorf("failed to revoke tokens: %v", err) - } - } - - return nil -} - // revokeSITokenAccessorsOnRestore is used to revoke Service Identity token // accessors on behalf of allocs that are now gone / terminal. func (s *Server) revokeSITokenAccessorsOnRestore() error { @@ -1510,9 +1448,6 @@ func (s *Server) revokeLeadership() error { // Disable the periodic dispatcher, since it is only useful as a leader s.periodicDispatcher.SetEnabled(false) - // Disable the Vault client as it is only useful as a leader. - s.vault.SetActive(false) - // Disable the deployment watcher as it is only useful as a leader. s.deploymentWatcher.SetEnabled(false, nil) diff --git a/nomad/leader_test.go b/nomad/leader_test.go index a816100a3..bed7c18ad 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -788,61 +788,6 @@ func TestLeader_ReapDuplicateEval(t *testing.T) { }) } -func TestLeader_revokeVaultAccessorsOnRestore(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 - }) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) - - // Insert a vault accessor that should be revoked - fsmState := s1.fsm.State() - va := mock.VaultAccessor() - if err := fsmState.UpsertVaultAccessor(100, []*structs.VaultAccessor{va}); err != nil { - t.Fatalf("bad: %v", err) - } - - // Swap the Vault client - tvc := &TestVaultClient{} - s1.vault = tvc - - // Do a restore - if err := s1.revokeVaultAccessorsOnRestore(); err != nil { - t.Fatalf("Failed to restore: %v", err) - } - - if len(tvc.RevokedTokens) != 1 && tvc.RevokedTokens[0].Accessor != va.Accessor { - t.Fatalf("Bad revoked accessors: %v", tvc.RevokedTokens) - } -} - -func TestLeader_revokeVaultAccessorsOnRestore_workloadIdentity(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 - }) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) - - // Insert a Vault accessor that should be revoked - fsmState := s1.fsm.State() - va := mock.VaultAccessor() - err := fsmState.UpsertVaultAccessor(100, []*structs.VaultAccessor{va}) - must.NoError(t, err) - - // Do a restore - err = s1.revokeVaultAccessorsOnRestore() - must.NoError(t, err) - - // Verify accessor was removed from state. - got, err := fsmState.VaultAccessor(nil, va.Accessor) - must.NoError(t, err) - must.Nil(t, got) -} - func TestLeader_revokeSITokenAccessorsOnRestore(t *testing.T) { ci.Parallel(t) r := require.New(t) diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 306928032..4fe1854ce 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -148,16 +148,6 @@ func JobSysBatchSummary(jobID string) *structs.JobSummary { } } -func VaultAccessor() *structs.VaultAccessor { - return &structs.VaultAccessor{ - Accessor: uuid.Generate(), - NodeID: uuid.Generate(), - AllocID: uuid.Generate(), - CreationTTL: 86400, - Task: "foo", - } -} - func SITokenAccessor() *structs.SITokenAccessor { return &structs.SITokenAccessor{ NodeID: uuid.Generate(), diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index ed9ca3b91..d937c9a86 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -17,7 +17,6 @@ import ( "github.com/hashicorp/go-memdb" metrics "github.com/hashicorp/go-metrics/compat" "github.com/hashicorp/go-multierror" - vapi "github.com/hashicorp/vault/api" "golang.org/x/sync/errgroup" "github.com/hashicorp/nomad/acl" @@ -475,18 +474,6 @@ func (n *Node) deregister(args *structs.NodeBatchDeregisterRequest, return err } - // Determine if there are any Vault accessors on the node - if accessors, err := snap.VaultAccessorsByNode(nil, nodeID); err != nil { - n.logger.Error("looking up vault accessors for node failed", "node_id", nodeID, "error", err) - return err - } else if l := len(accessors); l > 0 { - n.logger.Debug("revoking vault accessors on node due to deregister", "num_accessors", l, "node_id", nodeID) - if err := n.srv.vault.RevokeTokens(context.Background(), accessors, true); err != nil { - n.logger.Error("revoking vault accessors for node failed", "node_id", nodeID, "error", err) - return err - } - } - // Determine if there are any SI token accessors on the node if accessors, err := snap.SITokenAccessorsByNode(nil, nodeID); err != nil { n.logger.Error("looking up si accessors for node failed", "node_id", nodeID, "error", err) @@ -679,18 +666,6 @@ func (n *Node) UpdateStatus(args *structs.NodeUpdateStatusRequest, reply *struct // Check if we need to setup a heartbeat switch args.Status { case structs.NodeStatusDown: - // Determine if there are any Vault accessors on the node to cleanup - if accessors, err := n.srv.State().VaultAccessorsByNode(ws, args.NodeID); err != nil { - n.logger.Error("looking up vault accessors for node failed", "node_id", args.NodeID, "error", err) - return err - } else if l := len(accessors); l > 0 { - n.logger.Debug("revoking vault accessors on node due to down state", "num_accessors", l, "node_id", args.NodeID) - if err := n.srv.vault.RevokeTokens(context.Background(), accessors, true); err != nil { - n.logger.Error("revoking vault accessors for node failed", "node_id", args.NodeID, "error", err) - return err - } - } - // Determine if there are any SI token accessors on the node to cleanup if accessors, err := n.srv.State().SITokenAccessorsByNode(ws, args.NodeID); err != nil { n.logger.Error("looking up SI accessors for node failed", "node_id", args.NodeID, "error", err) @@ -1516,11 +1491,9 @@ func (n *Node) batchUpdate(future *structs.BatchFuture, updates []*structs.Alloc } // For each allocation we are updating, check if we should revoke any - // - Vault token accessors // - Service Identity token accessors var ( - revokeVault []*structs.VaultAccessor - revokeSI []*structs.SITokenAccessor + revokeSI []*structs.SITokenAccessor ) for _, alloc := range updates { @@ -1531,14 +1504,6 @@ func (n *Node) batchUpdate(future *structs.BatchFuture, updates []*structs.Alloc ws := memdb.NewWatchSet() - // Determine if there are any orphaned Vault accessors for the allocation - if accessors, err := n.srv.State().VaultAccessorsByAlloc(ws, alloc.ID); err != nil { - n.logger.Error("looking up vault accessors for alloc failed", "alloc_id", alloc.ID, "error", err) - mErr.Errors = append(mErr.Errors, err) - } else { - revokeVault = append(revokeVault, accessors...) - } - // Determine if there are any orphaned SI accessors for the allocation if accessors, err := n.srv.State().SITokenAccessorsByAlloc(ws, alloc.ID); err != nil { n.logger.Error("looking up si accessors for alloc failed", "alloc_id", alloc.ID, "error", err) @@ -1548,15 +1513,6 @@ func (n *Node) batchUpdate(future *structs.BatchFuture, updates []*structs.Alloc } } - // Revoke any orphaned Vault token accessors - if l := len(revokeVault); l > 0 { - n.logger.Debug("revoking vault accessors due to terminal allocations", "num_accessors", l) - if err := n.srv.vault.RevokeTokens(context.Background(), revokeVault, true); err != nil { - n.logger.Error("batched vault accessor revocation failed", "error", err) - mErr.Errors = append(mErr.Errors, err) - } - } - // Revoke any orphaned SI token accessors if l := len(revokeSI); l > 0 { n.logger.Debug("revoking si accessors due to terminal allocations", "num_accessors", l) @@ -1772,232 +1728,6 @@ func (n *Node) createNodeEvals(node *structs.Node, nodeIndex uint64) ([]string, return evalIDs, evalIndex, nil } -// DeriveVaultToken is used by the clients to request wrapped Vault tokens for -// tasks -func (n *Node) DeriveVaultToken(args *structs.DeriveVaultTokenRequest, reply *structs.DeriveVaultTokenResponse) error { - - authErr := n.srv.Authenticate(n.ctx, args) - - setError := func(e error, recoverable bool) { - if e != nil { - if re, ok := e.(*structs.RecoverableError); ok { - reply.Error = re // No need to wrap if error is already a RecoverableError - } else { - reply.Error = structs.NewRecoverableError(e, recoverable).(*structs.RecoverableError) - } - n.logger.Error("DeriveVaultToken failed", "recoverable", recoverable, "error", e) - } - } - - if done, err := n.srv.forward("Node.DeriveVaultToken", args, args, reply); done { - setError(err, structs.IsRecoverable(err) || err == structs.ErrNoLeader) - return nil - } - n.srv.MeasureRPCRate("node", structs.RateMetricWrite, args) - if authErr != nil { - return structs.ErrPermissionDenied - } - defer metrics.MeasureSince([]string{"nomad", "client", "derive_vault_token"}, time.Now()) - - // Verify the arguments - if args.NodeID == "" { - setError(fmt.Errorf("missing node ID"), false) - return nil - } - if args.SecretID == "" { - setError(fmt.Errorf("missing node SecretID"), false) - return nil - } - if args.AllocID == "" { - setError(fmt.Errorf("missing allocation ID"), false) - return nil - } - if len(args.Tasks) == 0 { - setError(fmt.Errorf("no tasks specified"), false) - return nil - } - - // Verify the following: - // * The Node exists and has the correct SecretID - // * The Allocation exists on the specified Node - // * The Allocation contains the given tasks and they each require Vault - // tokens - snap, err := n.srv.fsm.State().Snapshot() - if err != nil { - setError(err, false) - return nil - } - ws := memdb.NewWatchSet() - node, err := snap.NodeByID(ws, args.NodeID) - if err != nil { - setError(err, false) - return nil - } - if node == nil { - setError(fmt.Errorf("Node %q does not exist", args.NodeID), false) - return nil - } - if node.SecretID != args.SecretID { - setError(fmt.Errorf("SecretID mismatch"), false) - return nil - } - - alloc, err := snap.AllocByID(ws, args.AllocID) - if err != nil { - setError(err, false) - return nil - } - if alloc == nil { - setError(fmt.Errorf("Allocation %q does not exist", args.AllocID), false) - return nil - } - if alloc.NodeID != args.NodeID { - setError(fmt.Errorf("Allocation %q not running on Node %q", args.AllocID, args.NodeID), false) - return nil - } - if alloc.ClientTerminalStatus() { - setError(fmt.Errorf("Can't request Vault token for terminal allocation"), false) - return nil - } - - // Check if alloc has Vault - vaultBlocks := alloc.Job.Vault() - if vaultBlocks == nil { - setError(fmt.Errorf("Job does not require Vault token"), false) - return nil - } - tg, ok := vaultBlocks[alloc.TaskGroup] - if !ok { - setError(fmt.Errorf("Task group does not require Vault token"), false) - return nil - } - - var unneeded []string - for _, task := range args.Tasks { - taskVault := tg[task] - if taskVault == nil || len(taskVault.Policies) == 0 { - unneeded = append(unneeded, task) - } - } - - if len(unneeded) != 0 { - e := fmt.Errorf("Requested Vault tokens for tasks without defined Vault policies: %s", - strings.Join(unneeded, ", ")) - setError(e, false) - return nil - } - - // At this point the request is valid and we should contact Vault for - // tokens. - - // Create an error group where we will spin up a fixed set of goroutines to - // handle deriving tokens but where if any fails the whole group is - // canceled. - g, ctx := errgroup.WithContext(context.Background()) - - // Cap the handlers - handlers := len(args.Tasks) - if handlers > maxParallelRequestsPerDerive { - handlers = maxParallelRequestsPerDerive - } - - // Create the Vault Tokens - input := make(chan string, handlers) - results := make(map[string]*vapi.Secret, len(args.Tasks)) - for i := 0; i < handlers; i++ { - g.Go(func() error { - for { - select { - case task, ok := <-input: - if !ok { - return nil - } - - secret, err := n.srv.vault.CreateToken(ctx, alloc, task) - if err != nil { - return err - } - - results[task] = secret - case <-ctx.Done(): - return nil - } - } - }) - } - - // Send the input - go func() { - defer close(input) - for _, task := range args.Tasks { - select { - case <-ctx.Done(): - return - case input <- task: - } - } - }() - - // Wait for everything to complete or for an error - createErr := g.Wait() - - // Retrieve the results - accessors := make([]*structs.VaultAccessor, 0, len(results)) - tokens := make(map[string]string, len(results)) - for task, secret := range results { - w := secret.WrapInfo - tokens[task] = w.Token - accessor := &structs.VaultAccessor{ - Accessor: w.WrappedAccessor, - Task: task, - NodeID: alloc.NodeID, - AllocID: alloc.ID, - CreationTTL: w.TTL, - } - - accessors = append(accessors, accessor) - } - - // If there was an error revoke the created tokens - if createErr != nil { - n.logger.Error("Vault token creation for alloc failed", "alloc_id", alloc.ID, "error", createErr) - - if revokeErr := n.srv.vault.RevokeTokens(context.Background(), accessors, false); revokeErr != nil { - n.logger.Error("Vault token revocation for alloc failed", "alloc_id", alloc.ID, "error", revokeErr) - } - - if rerr, ok := createErr.(*structs.RecoverableError); ok { - reply.Error = rerr - } else { - reply.Error = structs.NewRecoverableError(createErr, false).(*structs.RecoverableError) - } - - return nil - } - - // Commit to Raft before returning any of the tokens - req := structs.VaultAccessorsRequest{Accessors: accessors} - _, index, err := n.srv.raftApply(structs.VaultAccessorRegisterRequestType, &req) - if err != nil { - n.logger.Error("registering Vault accessors for alloc failed", "alloc_id", alloc.ID, "error", err) - - // Determine if we can recover from the error - retry := false - switch err { - case raft.ErrNotLeader, raft.ErrLeadershipLost, raft.ErrRaftShutdown, raft.ErrEnqueueTimeout: - retry = true - } - - setError(err, retry) - return nil - } - - reply.Index = index - reply.Tasks = tokens - n.srv.setQueryMeta(&reply.QueryMeta) - return nil -} - type connectTask struct { TaskKind structs.TaskKind TaskName string diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 8de03e22b..a5ae4e6f0 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -26,7 +26,6 @@ import ( "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" - vapi "github.com/hashicorp/vault/api" "github.com/kr/pretty" "github.com/shoenig/test/must" "github.com/shoenig/test/wait" @@ -614,118 +613,6 @@ func TestClientEndpoint_Deregister_ACL(t *testing.T) { } } -func TestClientEndpoint_Deregister_Vault(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the register request - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - - // Fetch the response - var resp structs.GenericResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Swap the servers Vault Client - tvc := &TestVaultClient{} - s1.vault = tvc - - // Put some Vault accessors in the state store for that node - state := s1.fsm.State() - va1 := mock.VaultAccessor() - va1.NodeID = node.ID - va2 := mock.VaultAccessor() - va2.NodeID = node.ID - state.UpsertVaultAccessor(100, []*structs.VaultAccessor{va1, va2}) - - // Deregister - dereg := &structs.NodeBatchDeregisterRequest{ - NodeIDs: []string{node.ID}, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.GenericResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.BatchDeregister", dereg, &resp2); err != nil { - t.Fatalf("err: %v", err) - } - if resp2.Index == 0 { - t.Fatalf("bad index: %d", resp2.Index) - } - - // Check for the node in the FSM - ws := memdb.NewWatchSet() - out, err := state.NodeByID(ws, node.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out != nil { - t.Fatalf("unexpected node") - } - - // Check that the endpoint revoked the tokens - if l := len(tvc.RevokedTokens); l != 2 { - t.Fatalf("Deregister revoked %d tokens; want 2", l) - } -} - -func TestClientEndpoint_Deregister_Vault_WorkloadIdentity(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - // Enable Vault config and don't set any connection info to use the - // workload identity flow. - c.VaultConfigs[structs.VaultDefaultCluster].Enabled = pointer.Of(true) - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Register mock node. - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - var resp structs.GenericResponse - err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp) - must.NoError(t, err) - - // Put some Vault accessors in the state store for that node - var accessors []*structs.VaultAccessor - for i := 0; i < 3; i++ { - va := mock.VaultAccessor() - va.NodeID = node.ID - accessors = append(accessors, va) - } - state := s1.fsm.State() - state.UpsertVaultAccessor(100, accessors) - - // Deregister the mock node and verify no error happens when Vault tokens - // are revoked. - dereg := &structs.NodeDeregisterRequest{ - NodeID: node.ID, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.GenericResponse - err = msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg, &resp2) - must.NoError(t, err) - - // Verify accessors are removed from state. - for _, va := range accessors { - got, err := state.VaultAccessor(nil, va.Accessor) - must.NoError(t, err) - must.Nil(t, got) - } -} - func TestClientEndpoint_UpdateStatus(t *testing.T) { ci.Parallel(t) require := require.New(t) @@ -806,116 +693,6 @@ func TestClientEndpoint_UpdateStatus(t *testing.T) { }) } -func TestClientEndpoint_UpdateStatus_Vault(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the register request - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - - // Fetch the response - var resp structs.NodeUpdateResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Check for heartbeat interval - ttl := resp.HeartbeatTTL - if ttl < s1.config.MinHeartbeatTTL || ttl > 2*s1.config.MinHeartbeatTTL { - t.Fatalf("bad: %#v", ttl) - } - - // Swap the servers Vault Client - tvc := &TestVaultClient{} - s1.vault = tvc - - // Put some Vault accessors in the state store for that node - state := s1.fsm.State() - va1 := mock.VaultAccessor() - va1.NodeID = node.ID - va2 := mock.VaultAccessor() - va2.NodeID = node.ID - state.UpsertVaultAccessor(100, []*structs.VaultAccessor{va1, va2}) - - // Update the status to be down - dereg := &structs.NodeUpdateStatusRequest{ - NodeID: node.ID, - Status: structs.NodeStatusDown, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.NodeUpdateResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.UpdateStatus", dereg, &resp2); err != nil { - t.Fatalf("err: %v", err) - } - if resp2.Index == 0 { - t.Fatalf("bad index: %d", resp2.Index) - } - - // Check that the endpoint revoked the tokens - if l := len(tvc.RevokedTokens); l != 2 { - t.Fatalf("Deregister revoked %d tokens; want 2", l) - } -} - -func TestClientEndpoint_UpdateStatus_Vault_WorkloadIdentity(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - // Enable Vault config and don't set any connection info to use the - // workload identity flow. - c.VaultConfigs[structs.VaultDefaultCluster].Enabled = pointer.Of(true) - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Register mock node. - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - var resp structs.NodeUpdateResponse - err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp) - must.NoError(t, err) - - // Put some Vault accessors in the state store for the node. - var accessors []*structs.VaultAccessor - for i := 0; i < 3; i++ { - va := mock.VaultAccessor() - va.NodeID = node.ID - accessors = append(accessors, va) - } - state := s1.fsm.State() - state.UpsertVaultAccessor(100, accessors) - - // Update the status to be down and verify no error when Vault tokens are - // revoked. - updateReq := &structs.NodeUpdateStatusRequest{ - NodeID: node.ID, - Status: structs.NodeStatusDown, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.NodeUpdateResponse - err = msgpackrpc.CallWithCodec(codec, "Node.UpdateStatus", updateReq, &resp2) - must.NoError(t, err) - - // Verify accessors are removed from state. - for _, va := range accessors { - got, err := state.VaultAccessor(nil, va.Accessor) - must.NoError(t, err) - must.Nil(t, got) - } -} - func TestClientEndpoint_UpdateStatus_Reconnect(t *testing.T) { ci.Parallel(t) @@ -3370,160 +3147,6 @@ func TestClientEndpoint_BatchUpdate(t *testing.T) { } } -func TestClientEndpoint_UpdateAlloc_Vault(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the register request - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - - // Fetch the response - var resp structs.GenericResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // Swap the servers Vault Client - tvc := &TestVaultClient{} - s1.vault = tvc - - // Inject fake allocation and vault accessor - alloc := mock.Alloc() - alloc.NodeID = node.ID - state := s1.fsm.State() - state.UpsertJobSummary(99, mock.JobSummary(alloc.JobID)) - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 100, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - va := mock.VaultAccessor() - va.NodeID = node.ID - va.AllocID = alloc.ID - if err := state.UpsertVaultAccessor(101, []*structs.VaultAccessor{va}); err != nil { - t.Fatalf("err: %v", err) - } - - // Inject mock job - job := mock.Job() - job.ID = alloc.JobID - err := state.UpsertJob(structs.MsgTypeTestSetup, 101, nil, job) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Attempt update - clientAlloc := new(structs.Allocation) - *clientAlloc = *alloc - clientAlloc.ClientStatus = structs.AllocClientStatusFailed - - // Update the alloc - update := &structs.AllocUpdateRequest{ - Alloc: []*structs.Allocation{clientAlloc}, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.NodeAllocsResponse - start := time.Now() - if err := msgpackrpc.CallWithCodec(codec, "Node.UpdateAlloc", update, &resp2); err != nil { - t.Fatalf("err: %v", err) - } - if resp2.Index == 0 { - t.Fatalf("Bad index: %d", resp2.Index) - } - if diff := time.Since(start); diff < batchUpdateInterval { - t.Fatalf("too fast: %v", diff) - } - - // Lookup the alloc - ws := memdb.NewWatchSet() - out, err := state.AllocByID(ws, alloc.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - if out.ClientStatus != structs.AllocClientStatusFailed { - t.Fatalf("Bad: %#v", out) - } - - if l := len(tvc.RevokedTokens); l != 1 { - t.Fatalf("Deregister revoked %d tokens; want 1", l) - } -} - -func TestClientEndpoint_UpdateAlloc_VaultWorkloadIdentity(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, func(c *Config) { - // Enable Vault config and don't set any connection info to use the - // workload identity flow. - c.VaultConfigs[structs.VaultDefaultCluster].Enabled = pointer.Of(true) - }) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the node register request. - node := mock.Node() - reg := &structs.NodeRegisterRequest{ - Node: node, - WriteRequest: structs.WriteRequest{Region: "global"}, - } - var resp structs.GenericResponse - err := msgpackrpc.CallWithCodec(codec, "Node.Register", reg, &resp) - must.NoError(t, err) - - // Inject allocation and a few Vault accessors. - alloc := mock.Alloc() - alloc.NodeID = node.ID - state := s1.fsm.State() - state.UpsertJobSummary(99, mock.JobSummary(alloc.JobID)) - err = state.UpsertAllocs(structs.MsgTypeTestSetup, 100, []*structs.Allocation{alloc}) - must.NoError(t, err) - - var accessors []*structs.VaultAccessor - for i := 0; i < 3; i++ { - va := mock.VaultAccessor() - va.NodeID = node.ID - va.AllocID = alloc.ID - accessors = append(accessors, va) - } - err = state.UpsertVaultAccessor(101, accessors) - must.NoError(t, err) - - // Inject mock job. - job := mock.Job() - job.ID = alloc.JobID - err = state.UpsertJob(structs.MsgTypeTestSetup, 101, nil, job) - must.NoError(t, err) - - // Update alloc status and verify no error happens when the orphaned Vault - // tokens are revoked. - clientAlloc := new(structs.Allocation) - *clientAlloc = *alloc - clientAlloc.ClientStatus = structs.AllocClientStatusFailed - - update := &structs.AllocUpdateRequest{ - Alloc: []*structs.Allocation{clientAlloc}, - WriteRequest: structs.WriteRequest{Region: "global", AuthToken: node.SecretID}, - } - var resp2 structs.NodeAllocsResponse - err = msgpackrpc.CallWithCodec(codec, "Node.UpdateAlloc", update, &resp2) - must.NoError(t, err) - - // Verify accessors are removed from state. - for _, va := range accessors { - got, err := state.VaultAccessor(nil, va.Accessor) - must.NoError(t, err) - must.Nil(t, got) - } -} - func TestClientEndpoint_CreateNodeEvals(t *testing.T) { ci.Parallel(t) @@ -4175,244 +3798,6 @@ func TestClientEndpoint_ListNodes_Blocking(t *testing.T) { } } -func TestClientEndpoint_DeriveVaultToken_Bad(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - state := s1.fsm.State() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the node - node := mock.Node() - if err := state.UpsertNode(structs.MsgTypeTestSetup, 2, node); err != nil { - t.Fatalf("err: %v", err) - } - - // Create an alloc - alloc := mock.Alloc() - task := alloc.Job.TaskGroups[0].Tasks[0] - tasks := []string{task.Name} - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 3, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - badSecret := uuid.Generate() - req := &structs.DeriveVaultTokenRequest{ - NodeID: node.ID, - SecretID: badSecret, - AllocID: alloc.ID, - Tasks: tasks, - QueryOptions: structs.QueryOptions{ - Region: "global", - AuthToken: badSecret, - }, - } - - var resp structs.DeriveVaultTokenResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - - if resp.Error == nil || !strings.Contains(resp.Error.Error(), "SecretID mismatch") { - t.Fatalf("Expected SecretID mismatch: %v", resp.Error) - } - - // Put the correct SecretID - req.SecretID = node.SecretID - - // Now we should get an error about the allocation not running on the node - if err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - if resp.Error == nil || !strings.Contains(resp.Error.Error(), "not running on Node") { - t.Fatalf("Expected not running on node error: %v", resp.Error) - } - - // Update to be running on the node - alloc.NodeID = node.ID - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 4, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - // Now we should get an error about the job not needing any Vault secrets - if err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - if resp.Error == nil || !strings.Contains(resp.Error.Error(), "does not require") { - t.Fatalf("Expected no policies error: %v", resp.Error) - } - - // Update to be client-terminal - alloc.ClientStatus = structs.AllocClientStatusFailed - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 5, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - // Now we should get an error about the job not needing any Vault secrets - if err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - if resp.Error == nil || !strings.Contains(resp.Error.Error(), "terminal") { - t.Fatalf("Expected terminal allocation error: %v", resp.Error) - } - -} - -func TestClientEndpoint_DeriveVaultToken(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - state := s1.fsm.State() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault and allow authenticated - tr := true - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &tr - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - // Create the node - node := mock.Node() - if err := state.UpsertNode(structs.MsgTypeTestSetup, 2, node); err != nil { - t.Fatalf("err: %v", err) - } - - // Create an alloc an allocation that has vault policies required - alloc := mock.Alloc() - alloc.NodeID = node.ID - task := alloc.Job.TaskGroups[0].Tasks[0] - tasks := []string{task.Name} - task.Vault = &structs.Vault{Policies: []string{"a", "b"}} - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 3, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - // Return a secret for the task - token := uuid.Generate() - accessor := uuid.Generate() - ttl := 10 - secret := &vapi.Secret{ - WrapInfo: &vapi.SecretWrapInfo{ - Token: token, - WrappedAccessor: accessor, - TTL: ttl, - }, - } - tvc.SetCreateTokenSecret(alloc.ID, task.Name, secret) - - req := &structs.DeriveVaultTokenRequest{ - NodeID: node.ID, - SecretID: node.SecretID, - AllocID: alloc.ID, - Tasks: tasks, - QueryOptions: structs.QueryOptions{ - Region: "global", - AuthToken: node.SecretID, - }, - } - - var resp structs.DeriveVaultTokenResponse - if err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp); err != nil { - t.Fatalf("bad: %v", err) - } - if resp.Error != nil { - t.Fatalf("bad: %v", resp.Error) - } - - // Check the state store and ensure that we created a VaultAccessor - ws := memdb.NewWatchSet() - va, err := state.VaultAccessor(ws, accessor) - if err != nil { - t.Fatalf("bad: %v", err) - } - if va == nil { - t.Fatalf("bad: %v", va) - } - - if va.CreateIndex == 0 { - t.Fatalf("bad: %v", va) - } - - va.CreateIndex = 0 - expected := &structs.VaultAccessor{ - AllocID: alloc.ID, - Task: task.Name, - NodeID: alloc.NodeID, - Accessor: accessor, - CreationTTL: ttl, - } - - if !reflect.DeepEqual(expected, va) { - t.Fatalf("Got %#v; want %#v", va, expected) - } -} - -func TestClientEndpoint_DeriveVaultToken_VaultError(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - state := s1.fsm.State() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Enable vault and allow authenticated - tr := true - s1.config.GetDefaultVault().Enabled = &tr - s1.config.GetDefaultVault().AllowUnauthenticated = &tr - - // Replace the Vault Client on the server - tvc := &TestVaultClient{} - s1.vault = tvc - - // Create the node - node := mock.Node() - if err := state.UpsertNode(structs.MsgTypeTestSetup, 2, node); err != nil { - t.Fatalf("err: %v", err) - } - - // Create an alloc an allocation that has vault policies required - alloc := mock.Alloc() - alloc.NodeID = node.ID - task := alloc.Job.TaskGroups[0].Tasks[0] - tasks := []string{task.Name} - task.Vault = &structs.Vault{Policies: []string{"a", "b"}} - if err := state.UpsertAllocs(structs.MsgTypeTestSetup, 3, []*structs.Allocation{alloc}); err != nil { - t.Fatalf("err: %v", err) - } - - // Return an error when creating the token - tvc.SetCreateTokenError(alloc.ID, task.Name, - structs.NewRecoverableError(fmt.Errorf("recover"), true)) - - req := &structs.DeriveVaultTokenRequest{ - NodeID: node.ID, - SecretID: node.SecretID, - AllocID: alloc.ID, - Tasks: tasks, - QueryOptions: structs.QueryOptions{ - Region: "global", - AuthToken: node.SecretID, - }, - } - - var resp structs.DeriveVaultTokenResponse - err := msgpackrpc.CallWithCodec(codec, "Node.DeriveVaultToken", req, &resp) - if err != nil { - t.Fatalf("bad: %v", err) - } - if resp.Error == nil || !resp.Error.IsRecoverable() { - t.Fatalf("bad: %+v", resp.Error) - } -} - func TestClientEndpoint_taskUsesConnect(t *testing.T) { ci.Parallel(t) diff --git a/nomad/operator_endpoint.go b/nomad/operator_endpoint.go index b374ffd4b..1c54fe157 100644 --- a/nomad/operator_endpoint.go +++ b/nomad/operator_endpoint.go @@ -861,18 +861,6 @@ func (op *Operator) UpgradeCheckVaultWorkloadIdentity( } reply.OutdatedNodes = nodes - // Retrieve Vault tokens that were created by Nomad servers. - vaultTokensIter, err := op.srv.State().VaultAccessors(ws) - if err != nil { - return fmt.Errorf("failed to retrieve Vault token accessors: %w", err) - } - - vaultTokens := []*structs.VaultAccessor{} - for raw := vaultTokensIter.Next(); raw != nil; raw = vaultTokensIter.Next() { - vaultTokens = append(vaultTokens, raw.(*structs.VaultAccessor)) - } - reply.VaultTokens = vaultTokens - reply.QueryMeta.Index, _ = op.srv.State().LatestIndex() op.srv.setQueryMeta(&reply.QueryMeta) diff --git a/nomad/operator_endpoint_test.go b/nomad/operator_endpoint_test.go index 1afcb44b2..58a0f0f8f 100644 --- a/nomad/operator_endpoint_test.go +++ b/nomad/operator_endpoint_test.go @@ -18,7 +18,6 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp/cmpopts" "github.com/hashicorp/go-msgpack/v2/codec" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2" "github.com/hashicorp/nomad/acl" @@ -1281,183 +1280,3 @@ func TestOperator_SnapshotRestore_ACL(t *testing.T) { }) } } - -func TestOperator_UpgradeCheckRequest_VaultWorkloadIdentity(t *testing.T) { - ci.Parallel(t) - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) - - codec := rpcClient(t, s1) - state := s1.fsm.State() - - // Register mock nodes, one pre-1.7. - node := mock.Node() - node.Attributes["nomad.version"] = "1.7.2" - err := state.UpsertNode(structs.MsgTypeTestSetup, 1000, node) - must.NoError(t, err) - - outdatedNode := mock.Node() - outdatedNode.Attributes["nomad.version"] = "1.6.4" - err = state.UpsertNode(structs.MsgTypeTestSetup, 1001, outdatedNode) - must.NoError(t, err) - - // Create non-default namespace. - ns := mock.Namespace() - state.UpsertNamespaces(1002, []*structs.Namespace{ns}) - - // Register Vault jobs, one with and another without workload identity. - jobNoWID := mock.Job() - jobNoWID.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Cluster: "default", - Policies: []string{"test"}, - } - // Add multiple tasks and groups to make sure we don't have duplicate jobs - // in the result. - jobNoWID.TaskGroups[0].Tasks = append(jobNoWID.TaskGroups[0].Tasks, jobNoWID.TaskGroups[0].Tasks[0].Copy()) - jobNoWID.TaskGroups[0].Tasks[1].Name = "task-1" - jobNoWID.TaskGroups = append(jobNoWID.TaskGroups, jobNoWID.TaskGroups[0].Copy()) - jobNoWID.TaskGroups[1].Name = "tg-1" - - err = state.UpsertJob(structs.MsgTypeTestSetup, 1003, nil, jobNoWID) - must.NoError(t, err) - - jobNoWIDNonDefaultNS := mock.Job() - jobNoWIDNonDefaultNS.Namespace = ns.Name - jobNoWIDNonDefaultNS.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Cluster: "default", - Policies: []string{"test"}, - } - err = state.UpsertJob(structs.MsgTypeTestSetup, 1004, nil, jobNoWIDNonDefaultNS) - must.NoError(t, err) - - jobWithWID := mock.Job() - jobWithWID.TaskGroups[0].Tasks[0].Vault = &structs.Vault{ - Cluster: "default", - } - jobWithWID.TaskGroups[0].Tasks[0].Identities = []*structs.WorkloadIdentity{{ - Name: "vault_default", - }} - err = state.UpsertJob(structs.MsgTypeTestSetup, 1005, nil, jobWithWID) - must.NoError(t, err) - - // Create allocs for the jobs. - allocJobNoWID := mock.Alloc() - allocJobNoWID.Job = jobNoWID - allocJobNoWID.JobID = jobNoWID.ID - allocJobNoWID.NodeID = node.ID - - allocJobWithWID := mock.Alloc() - allocJobWithWID.Job = jobWithWID - allocJobWithWID.JobID = jobWithWID.ID - allocJobWithWID.NodeID = node.ID - - err = state.UpsertAllocs(structs.MsgTypeTestSetup, 1006, []*structs.Allocation{allocJobNoWID, allocJobWithWID}) - must.NoError(t, err) - - // Create Vault token accessor for job without Vault identity and one that - // is no longer used. - tokenJobNoWID := mock.VaultAccessor() - tokenJobNoWID.AllocID = allocJobNoWID.ID - tokenJobNoWID.NodeID = node.ID - - tokenUnused := mock.VaultAccessor() - err = state.UpsertVaultAccessor(1007, []*structs.VaultAccessor{tokenJobNoWID, tokenUnused}) - must.NoError(t, err) - - // Make request. - args := &structs.UpgradeCheckVaultWorkloadIdentityRequest{ - QueryOptions: structs.QueryOptions{ - Region: "global", - AuthToken: node.SecretID, - }, - } - var resp structs.UpgradeCheckVaultWorkloadIdentityResponse - err = msgpackrpc.CallWithCodec(codec, "Operator.UpgradeCheckVaultWorkloadIdentity", args, &resp) - must.NoError(t, err) - must.Eq(t, 1007, resp.Index) - - // Verify only jobs without Vault identity are returned. - must.Len(t, 2, resp.JobsWithoutVaultIdentity) - must.SliceContains(t, resp.JobsWithoutVaultIdentity, jobNoWID.Stub(nil, nil), must.Cmp(cmpopts.IgnoreFields( - structs.JobListStub{}, - "Status", - "ModifyIndex", - ))) - must.SliceContains(t, resp.JobsWithoutVaultIdentity, jobNoWIDNonDefaultNS.Stub(nil, nil), must.Cmp(cmpopts.IgnoreFields( - structs.JobListStub{}, - "Status", - "ModifyIndex", - ))) - - // Verify only outdated nodes are returned. - must.Len(t, 1, resp.OutdatedNodes) - must.SliceContains(t, resp.OutdatedNodes, outdatedNode.Stub(nil)) - - // Verify Vault ACL tokens are returned. - must.Len(t, 2, resp.VaultTokens) - must.SliceContains(t, resp.VaultTokens, tokenJobNoWID) - must.SliceContains(t, resp.VaultTokens, tokenUnused) -} - -func TestOperator_UpgradeCheckRequest_VaultWorkloadIdentity_ACL(t *testing.T) { - ci.Parallel(t) - - s1, root, cleanupS1 := TestACLServer(t, nil) - defer cleanupS1() - testutil.WaitForLeader(t, s1.RPC) - - codec := rpcClient(t, s1) - state := s1.fsm.State() - - // Create test tokens and policies. - allowed := mock.CreatePolicyAndToken(t, state, 1000, "allowed", `operator {policy = "read"}`) - notAllowed := mock.CreatePolicyAndToken(t, state, 1002, "not-allowed", mock.NamespacePolicy("default", "write", nil)) - - testCases := []struct { - name string - token string - expectedErr string - }{ - { - name: "root token is allowed", - token: root.SecretID, - expectedErr: "", - }, - { - name: "operator read token is allowed", - token: allowed.SecretID, - expectedErr: "", - }, - { - name: "token not allowed", - token: notAllowed.SecretID, - expectedErr: structs.ErrPermissionDenied.Error(), - }, - { - name: "missing token not allowed", - token: "", - expectedErr: structs.ErrPermissionDenied.Error(), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - // Make request. - args := &structs.UpgradeCheckVaultWorkloadIdentityRequest{ - QueryOptions: structs.QueryOptions{ - Region: "global", - AuthToken: tc.token, - }, - } - var resp structs.UpgradeCheckVaultWorkloadIdentityResponse - err := msgpackrpc.CallWithCodec(codec, "Operator.UpgradeCheckVaultWorkloadIdentity", args, &resp) - if tc.expectedErr == "" { - must.NoError(t, err) - } else { - must.ErrorContains(t, err, tc.expectedErr) - } - }) - } -} diff --git a/nomad/server.go b/nomad/server.go index 2682b7b80..c3dbb171a 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -258,9 +258,6 @@ type Server struct { // consulACLs is used for managing Consul Service Identity tokens. consulACLs ConsulACLsAPI - // vault is the client for communicating with Vault. - vault VaultClient - // Worker used for processing workers []*Worker workerLock sync.RWMutex @@ -405,13 +402,6 @@ func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulConfigFunc // Setup Consul (more) s.setupConsul(consulConfigFunc, consulACLs) - // Setup Vault - if err := s.setupVaultClient(); err != nil { - s.Shutdown() - s.logger.Error("failed to setup Vault client", "error", err) - return nil, fmt.Errorf("Failed to setup Vault client: %v", err) - } - // Set up the keyring keystorePath := filepath.Join(s.config.DataDir, "keystore") if s.config.DevMode && s.config.DataDir == "" { @@ -535,9 +525,6 @@ func NewServer(config *Config, consulCatalog consul.CatalogAPI, consulConfigFunc // Emit metrics for the blocked eval tracker. go s.blockedEvals.EmitStats(time.Second, s.shutdownCh) - // Emit metrics for the Vault client. - go s.vault.EmitStats(time.Second, s.shutdownCh) - // Emit metrics go s.heartbeatStats() @@ -760,11 +747,6 @@ func (s *Server) Shutdown() error { s.fsm.Close() } - // Stop Vault token renewal and revocations - if s.vault != nil { - s.vault.Stop() - } - // Stop the Consul ACLs token revocations s.consulACLs.Stop() @@ -893,30 +875,6 @@ func (s *Server) Reload(newConfig *Config) error { var mErr multierror.Error - // Handle the Vault reload. Vault should never be nil but just guard. - if s.vault != nil { - vconfig := newConfig.GetDefaultVault() - - // Verify if the new configuration would cause the client type to - // change. - var err error - switch s.vault.(type) { - case *NoopVault: - if vconfig != nil && vconfig.Token != "" { - err = fmt.Errorf("setting a Vault token requires restarting the Nomad agent") - } - case *vaultClient: - if vconfig != nil && vconfig.Token == "" { - err = fmt.Errorf("removing the Vault token requires restarting the Nomad agent") - } - } - if err != nil { - _ = multierror.Append(&mErr, err) - } else if err := s.vault.SetConfig(newConfig.GetDefaultVault()); err != nil { - _ = multierror.Append(&mErr, err) - } - } - shouldReloadTLS, err := tlsutil.ShouldReloadRPCConnections(s.config.TLSConfig, newConfig.TLSConfig) if err != nil { s.logger.Error("error checking whether to reload TLS configuration", "error", err) @@ -1208,23 +1166,6 @@ func (s *Server) setupConsul(consulConfigFunc consul.ConfigAPIFunc, consulACLs c s.consulACLs = NewConsulACLsAPI(consulACLs, s.logger, s.purgeSITokenAccessors) } -// setupVaultClient is used to set up the Vault API client. -func (s *Server) setupVaultClient() error { - vconfig := s.config.GetDefaultVault() - if vconfig != nil && vconfig.Token == "" { - s.vault = NewNoopVault(vconfig, s.logger, s.purgeVaultAccessors) - return nil - } - - delegate := s.entVaultDelegate() - v, err := NewVaultClient(vconfig, s.logger, s.purgeVaultAccessors, delegate) - if err != nil { - return err - } - s.vault = v - return nil -} - // setupRPC is used to setup the RPC listener func (s *Server) setupRPC(tlsWrap tlsutil.RegionWrapper) error { // Populate the static RPC server @@ -2065,7 +2006,6 @@ func (s *Server) Stats() map[string]map[string]string { "raft": s.raft.Stats(), "serf": s.serf.Stats(), "runtime": goruntime.RuntimeStats(), - "vault": s.vault.Stats(), } return stats diff --git a/nomad/server_setup_ce.go b/nomad/server_setup_ce.go index 6dc903cc9..bcde03a14 100644 --- a/nomad/server_setup_ce.go +++ b/nomad/server_setup_ce.go @@ -35,7 +35,3 @@ func (s *Server) setupEnterprise(config *Config) error { return nil } func (s *Server) startEnterpriseBackground() {} - -func (s *Server) entVaultDelegate() *VaultNoopDelegate { - return &VaultNoopDelegate{} -} diff --git a/nomad/server_test.go b/nomad/server_test.go index a8358582b..4893e3cf6 100644 --- a/nomad/server_test.go +++ b/nomad/server_test.go @@ -14,7 +14,6 @@ import ( msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc/v2" "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/nomad/structs/config" @@ -202,36 +201,6 @@ func TestServer_Regions(t *testing.T) { }) } -func TestServer_Reload_Vault(t *testing.T) { - ci.Parallel(t) - - token := uuid.Generate() - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.Region = "global" - c.GetDefaultVault().Token = token - }) - defer cleanupS1() - - must.False(t, s1.vault.Running()) - - tr := true - config := DefaultConfig() - config.GetDefaultVault().Enabled = &tr - config.GetDefaultVault().Token = token - config.GetDefaultVault().Namespace = "nondefault" - - err := s1.Reload(config) - must.NoError(t, err) - - must.True(t, s1.vault.Running()) - must.Eq(t, "nondefault", s1.vault.GetConfig().Namespace) - - // Removing the token requires agent restart. - config.GetDefaultVault().Token = "" - err = s1.Reload(config) - must.ErrorContains(t, err, "requires restarting the Nomad agent") -} - func connectionReset(msg string) bool { return strings.Contains(msg, "EOF") || strings.Contains(msg, "connection reset by peer") } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 91a8dfb56..acadfdc09 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -4619,126 +4619,6 @@ func (s *StateStore) allocsByNamespaceImpl(ws memdb.WatchSet, txn *txn, namespac return iter, nil } -// UpsertVaultAccessor is used to register a set of Vault Accessors. -func (s *StateStore) UpsertVaultAccessor(index uint64, accessors []*structs.VaultAccessor) error { - txn := s.db.WriteTxn(index) - defer txn.Abort() - - for _, accessor := range accessors { - // Set the create index - accessor.CreateIndex = index - - // Insert the accessor - if err := txn.Insert("vault_accessors", accessor); err != nil { - return fmt.Errorf("accessor insert failed: %v", err) - } - } - - if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { - return fmt.Errorf("index update failed: %v", err) - } - - return txn.Commit() -} - -// DeleteVaultAccessors is used to delete a set of Vault Accessors -func (s *StateStore) DeleteVaultAccessors(index uint64, accessors []*structs.VaultAccessor) error { - txn := s.db.WriteTxn(index) - defer txn.Abort() - - // Lookup the accessor - for _, accessor := range accessors { - // Delete the accessor - if err := txn.Delete("vault_accessors", accessor); err != nil { - return fmt.Errorf("accessor delete failed: %v", err) - } - } - - if err := txn.Insert("index", &IndexEntry{"vault_accessors", index}); err != nil { - return fmt.Errorf("index update failed: %v", err) - } - - return txn.Commit() -} - -// VaultAccessor returns the given Vault accessor -func (s *StateStore) VaultAccessor(ws memdb.WatchSet, accessor string) (*structs.VaultAccessor, error) { - txn := s.db.ReadTxn() - - watchCh, existing, err := txn.FirstWatch("vault_accessors", "id", accessor) - if err != nil { - return nil, fmt.Errorf("accessor lookup failed: %v", err) - } - - ws.Add(watchCh) - - if existing != nil { - return existing.(*structs.VaultAccessor), nil - } - - return nil, nil -} - -// VaultAccessors returns an iterator of Vault accessors. -func (s *StateStore) VaultAccessors(ws memdb.WatchSet) (memdb.ResultIterator, error) { - txn := s.db.ReadTxn() - - iter, err := txn.Get("vault_accessors", "id") - if err != nil { - return nil, err - } - - ws.Add(iter.WatchCh()) - - return iter, nil -} - -// VaultAccessorsByAlloc returns all the Vault accessors by alloc id -func (s *StateStore) VaultAccessorsByAlloc(ws memdb.WatchSet, allocID string) ([]*structs.VaultAccessor, error) { - txn := s.db.ReadTxn() - - // Get an iterator over the accessors - iter, err := txn.Get("vault_accessors", "alloc_id", allocID) - if err != nil { - return nil, err - } - - ws.Add(iter.WatchCh()) - - var out []*structs.VaultAccessor - for { - raw := iter.Next() - if raw == nil { - break - } - out = append(out, raw.(*structs.VaultAccessor)) - } - return out, nil -} - -// VaultAccessorsByNode returns all the Vault accessors by node id -func (s *StateStore) VaultAccessorsByNode(ws memdb.WatchSet, nodeID string) ([]*structs.VaultAccessor, error) { - txn := s.db.ReadTxn() - - // Get an iterator over the accessors - iter, err := txn.Get("vault_accessors", "node_id", nodeID) - if err != nil { - return nil, err - } - - ws.Add(iter.WatchCh()) - - var out []*structs.VaultAccessor - for { - raw := iter.Next() - if raw == nil { - break - } - out = append(out, raw.(*structs.VaultAccessor)) - } - return out, nil -} - func indexEntry(table string, index uint64) *IndexEntry { return &IndexEntry{ Key: table, diff --git a/nomad/state/state_store_restore.go b/nomad/state/state_store_restore.go index 41a571b71..457d19d12 100644 --- a/nomad/state/state_store_restore.go +++ b/nomad/state/state_store_restore.go @@ -117,14 +117,6 @@ func (r *StateRestore) DeploymentRestore(deployment *structs.Deployment) error { return nil } -// VaultAccessorRestore is used to restore a vault accessor -func (r *StateRestore) VaultAccessorRestore(accessor *structs.VaultAccessor) error { - if err := r.txn.Insert("vault_accessors", accessor); err != nil { - return fmt.Errorf("vault accessor insert failed: %v", err) - } - return nil -} - // SITokenAccessorRestore is used to restore an SI token accessor func (r *StateRestore) SITokenAccessorRestore(accessor *structs.SITokenAccessor) error { if err := r.txn.Insert(siTokenAccessorTable, accessor); err != nil { diff --git a/nomad/state/state_store_restore_test.go b/nomad/state/state_store_restore_test.go index c28681f32..99ebe58a3 100644 --- a/nomad/state/state_store_restore_test.go +++ b/nomad/state/state_store_restore_test.go @@ -253,26 +253,6 @@ func TestStateStore_RestoreAlloc(t *testing.T) { must.False(t, watchFired(ws)) } -func TestStateStore_RestoreVaultAccessor(t *testing.T) { - ci.Parallel(t) - - state := testStateStore(t) - a := mock.VaultAccessor() - - restore, err := state.Restore() - must.NoError(t, err) - - err = restore.VaultAccessorRestore(a) - must.NoError(t, err) - must.NoError(t, restore.Commit()) - - ws := memdb.NewWatchSet() - out, err := state.VaultAccessor(ws, a.Accessor) - must.NoError(t, err) - must.Eq(t, a, out) - must.False(t, watchFired(ws)) -} - func TestStateStore_RestoreSITokenAccessor(t *testing.T) { ci.Parallel(t) diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 72c5f2b48..9a940e505 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -9084,239 +9084,6 @@ func TestStateStore_UpsertDeploymentAllocHealth(t *testing.T) { } } -func TestStateStore_UpsertVaultAccessors(t *testing.T) { - ci.Parallel(t) - - state := testStateStore(t) - a := mock.VaultAccessor() - a2 := mock.VaultAccessor() - - ws := memdb.NewWatchSet() - if _, err := state.VaultAccessor(ws, a.Accessor); err != nil { - t.Fatalf("err: %v", err) - } - - if _, err := state.VaultAccessor(ws, a2.Accessor); err != nil { - t.Fatalf("err: %v", err) - } - - err := state.UpsertVaultAccessor(1000, []*structs.VaultAccessor{a, a2}) - if err != nil { - t.Fatalf("err: %v", err) - } - - if !watchFired(ws) { - t.Fatalf("bad") - } - - ws = memdb.NewWatchSet() - out, err := state.VaultAccessor(ws, a.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - - if !reflect.DeepEqual(a, out) { - t.Fatalf("bad: %#v %#v", a, out) - } - - out, err = state.VaultAccessor(ws, a2.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - - if !reflect.DeepEqual(a2, out) { - t.Fatalf("bad: %#v %#v", a2, out) - } - - iter, err := state.VaultAccessors(ws) - if err != nil { - t.Fatalf("err: %v", err) - } - - count := 0 - for { - raw := iter.Next() - if raw == nil { - break - } - - count++ - accessor := raw.(*structs.VaultAccessor) - - if !reflect.DeepEqual(accessor, a) && !reflect.DeepEqual(accessor, a2) { - t.Fatalf("bad: %#v", accessor) - } - } - - if count != 2 { - t.Fatalf("bad: %d", count) - } - - index, err := state.Index("vault_accessors") - if err != nil { - t.Fatalf("err: %v", err) - } - if index != 1000 { - t.Fatalf("bad: %d", index) - } - - if watchFired(ws) { - t.Fatalf("bad") - } -} - -func TestStateStore_DeleteVaultAccessors(t *testing.T) { - ci.Parallel(t) - - state := testStateStore(t) - a1 := mock.VaultAccessor() - a2 := mock.VaultAccessor() - accessors := []*structs.VaultAccessor{a1, a2} - - err := state.UpsertVaultAccessor(1000, accessors) - if err != nil { - t.Fatalf("err: %v", err) - } - - ws := memdb.NewWatchSet() - if _, err := state.VaultAccessor(ws, a1.Accessor); err != nil { - t.Fatalf("err: %v", err) - } - - err = state.DeleteVaultAccessors(1001, accessors) - if err != nil { - t.Fatalf("err: %v", err) - } - - if !watchFired(ws) { - t.Fatalf("bad") - } - - ws = memdb.NewWatchSet() - out, err := state.VaultAccessor(ws, a1.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - if out != nil { - t.Fatalf("bad: %#v %#v", a1, out) - } - out, err = state.VaultAccessor(ws, a2.Accessor) - if err != nil { - t.Fatalf("err: %v", err) - } - if out != nil { - t.Fatalf("bad: %#v %#v", a2, out) - } - - index, err := state.Index("vault_accessors") - if err != nil { - t.Fatalf("err: %v", err) - } - if index != 1001 { - t.Fatalf("bad: %d", index) - } - - if watchFired(ws) { - t.Fatalf("bad") - } -} - -func TestStateStore_VaultAccessorsByAlloc(t *testing.T) { - ci.Parallel(t) - - state := testStateStore(t) - alloc := mock.Alloc() - var accessors []*structs.VaultAccessor - var expected []*structs.VaultAccessor - - for i := 0; i < 5; i++ { - accessor := mock.VaultAccessor() - accessor.AllocID = alloc.ID - expected = append(expected, accessor) - accessors = append(accessors, accessor) - } - - for i := 0; i < 10; i++ { - accessor := mock.VaultAccessor() - accessors = append(accessors, accessor) - } - - err := state.UpsertVaultAccessor(1000, accessors) - if err != nil { - t.Fatalf("err: %v", err) - } - - ws := memdb.NewWatchSet() - out, err := state.VaultAccessorsByAlloc(ws, alloc.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(expected) != len(out) { - t.Fatalf("bad: %#v %#v", len(expected), len(out)) - } - - index, err := state.Index("vault_accessors") - if err != nil { - t.Fatalf("err: %v", err) - } - if index != 1000 { - t.Fatalf("bad: %d", index) - } - - if watchFired(ws) { - t.Fatalf("bad") - } -} - -func TestStateStore_VaultAccessorsByNode(t *testing.T) { - ci.Parallel(t) - - state := testStateStore(t) - node := mock.Node() - var accessors []*structs.VaultAccessor - var expected []*structs.VaultAccessor - - for i := 0; i < 5; i++ { - accessor := mock.VaultAccessor() - accessor.NodeID = node.ID - expected = append(expected, accessor) - accessors = append(accessors, accessor) - } - - for i := 0; i < 10; i++ { - accessor := mock.VaultAccessor() - accessors = append(accessors, accessor) - } - - err := state.UpsertVaultAccessor(1000, accessors) - if err != nil { - t.Fatalf("err: %v", err) - } - - ws := memdb.NewWatchSet() - out, err := state.VaultAccessorsByNode(ws, node.ID) - if err != nil { - t.Fatalf("err: %v", err) - } - - if len(expected) != len(out) { - t.Fatalf("bad: %#v %#v", len(expected), len(out)) - } - - index, err := state.Index("vault_accessors") - if err != nil { - t.Fatalf("err: %v", err) - } - if index != 1000 { - t.Fatalf("bad: %d", index) - } - - if watchFired(ws) { - t.Fatalf("bad") - } -} - func TestStateStore_UpsertSITokenAccessors(t *testing.T) { ci.Parallel(t) r := require.New(t) diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index 52f34a046..1185be269 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -95,43 +95,20 @@ type VaultConfig struct { // matches this block `name` parameter. DefaultIdentity *WorkloadIdentityConfig `mapstructure:"default_identity"` - // Deprecated fields. - - // Token is the Vault token given to Nomad such that it can - // derive child tokens. Nomad will renew this token at half its lease - // lifetime. - // - // Deprecated: Nomad 1.7.0 is able to derive Vault tokens from workload - // identities. This field will be removed in a future release. + // Token is used by the Vault Transit Keyring implementation only. It was + // previously used by the now removed Nomad server derive child token + // workflow. Token string `mapstructure:"token"` - - // AllowUnauthenticated allows users to submit jobs requiring Vault tokens - // without providing a Vault token proving they have access to these - // policies. - // - // Deprecated: Nomad 1.7.0 no longer requires a Vault token for job - // operations. This field will be removed in a future release. - AllowUnauthenticated *bool `mapstructure:"allow_unauthenticated"` - - // TaskTokenTTL is the TTL of the tokens created by Nomad Servers and used - // by the client. There should be a minimum time value such that the client - // does not have to renew with Vault at a very high frequency - // - // Deprecated: Nomad 1.7.0 derives tokens from workload identities that - // receive their TTL configuration from the Vault role used. This field - // will be removed in a future release. - TaskTokenTTL string `mapstructure:"task_token_ttl"` } // DefaultVaultConfig returns the canonical defaults for the Nomad // `vault` configuration. func DefaultVaultConfig() *VaultConfig { return &VaultConfig{ - Name: "default", - Addr: "https://vault.service.consul:8200", - JWTAuthBackendPath: "jwt-nomad", - ConnectionRetryIntv: DefaultVaultConnectRetryIntv, - AllowUnauthenticated: pointer.Of(true), + Name: "default", + Addr: "https://vault.service.consul:8200", + JWTAuthBackendPath: "jwt-nomad", + ConnectionRetryIntv: DefaultVaultConnectRetryIntv, } } @@ -143,12 +120,6 @@ func (c *VaultConfig) IsEnabled() bool { return c.Enabled != nil && *c.Enabled } -// AllowsUnauthenticated returns whether the config allows unauthenticated -// access to Vault -func (c *VaultConfig) AllowsUnauthenticated() bool { - return c.AllowUnauthenticated != nil && *c.AllowUnauthenticated -} - // Merge merges two Vault configurations together. func (c *VaultConfig) Merge(b *VaultConfig) *VaultConfig { result := *c @@ -201,16 +172,6 @@ func (c *VaultConfig) Merge(b *VaultConfig) *VaultConfig { result.DefaultIdentity = result.DefaultIdentity.Merge(b.DefaultIdentity) } - if b.Token != "" { - result.Token = b.Token - } - if b.AllowUnauthenticated != nil { - result.AllowUnauthenticated = b.AllowUnauthenticated - } - if b.TaskTokenTTL != "" { - result.TaskTokenTTL = b.TaskTokenTTL - } - return &result } @@ -305,15 +266,5 @@ func (c *VaultConfig) Equal(b *VaultConfig) bool { return false } - if c.Token != b.Token { - return false - } - if !pointer.Eq(c.AllowUnauthenticated, b.AllowUnauthenticated) { - return false - } - if c.TaskTokenTTL != b.TaskTokenTTL { - return false - } - return true } diff --git a/nomad/structs/config/vault_test.go b/nomad/structs/config/vault_test.go index 68b216c12..216aefb36 100644 --- a/nomad/structs/config/vault_test.go +++ b/nomad/structs/config/vault_test.go @@ -17,36 +17,30 @@ func TestVaultConfig_Merge(t *testing.T) { ci.Parallel(t) c1 := &VaultConfig{ - Enabled: pointer.Of(false), - Token: "1", - Role: "1", - AllowUnauthenticated: pointer.Of(true), - TaskTokenTTL: "1", - Addr: "1", - JWTAuthBackendPath: "jwt", - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "1", - DefaultIdentity: nil, + Enabled: pointer.Of(false), + Role: "1", + Addr: "1", + JWTAuthBackendPath: "jwt", + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "1", + DefaultIdentity: nil, } c2 := &VaultConfig{ - Enabled: pointer.Of(true), - Token: "2", - Role: "2", - AllowUnauthenticated: pointer.Of(false), - TaskTokenTTL: "2", - Addr: "2", - JWTAuthBackendPath: "jwt2", - TLSCaFile: "2", - TLSCaPath: "2", - TLSCertFile: "2", - TLSKeyFile: "2", - TLSSkipVerify: nil, - TLSServerName: "2", + Enabled: pointer.Of(true), + Role: "2", + Addr: "2", + JWTAuthBackendPath: "jwt2", + TLSCaFile: "2", + TLSCaPath: "2", + TLSCertFile: "2", + TLSKeyFile: "2", + TLSSkipVerify: nil, + TLSServerName: "2", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.dev"}, Env: pointer.Of(true), @@ -55,19 +49,16 @@ func TestVaultConfig_Merge(t *testing.T) { } e := &VaultConfig{ - Enabled: pointer.Of(true), - Token: "2", - Role: "2", - AllowUnauthenticated: pointer.Of(false), - TaskTokenTTL: "2", - Addr: "2", - JWTAuthBackendPath: "jwt2", - TLSCaFile: "2", - TLSCaPath: "2", - TLSCertFile: "2", - TLSKeyFile: "2", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "2", + Enabled: pointer.Of(true), + Role: "2", + Addr: "2", + JWTAuthBackendPath: "jwt2", + TLSCaFile: "2", + TLSCaPath: "2", + TLSCertFile: "2", + TLSKeyFile: "2", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "2", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.dev"}, Env: pointer.Of(true), @@ -85,21 +76,18 @@ func TestVaultConfig_Equals(t *testing.T) { ci.Parallel(t) c1 := &VaultConfig{ - Enabled: pointer.Of(false), - Token: "1", - Role: "1", - Namespace: "1", - AllowUnauthenticated: pointer.Of(true), - TaskTokenTTL: "1", - Addr: "1", - JWTAuthBackendPath: "jwt", - ConnectionRetryIntv: time.Second, - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "1", + Enabled: pointer.Of(false), + Role: "1", + Namespace: "1", + Addr: "1", + JWTAuthBackendPath: "jwt", + ConnectionRetryIntv: time.Second, + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "1", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.dev"}, Env: pointer.Of(true), @@ -108,21 +96,18 @@ func TestVaultConfig_Equals(t *testing.T) { } c2 := &VaultConfig{ - Enabled: pointer.Of(false), - Token: "1", - Role: "1", - Namespace: "1", - AllowUnauthenticated: pointer.Of(true), - TaskTokenTTL: "1", - Addr: "1", - JWTAuthBackendPath: "jwt", - ConnectionRetryIntv: time.Second, - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "1", + Enabled: pointer.Of(false), + Role: "1", + Namespace: "1", + Addr: "1", + JWTAuthBackendPath: "jwt", + ConnectionRetryIntv: time.Second, + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "1", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.dev"}, Env: pointer.Of(true), @@ -133,20 +118,17 @@ func TestVaultConfig_Equals(t *testing.T) { must.Equal(t, c1, c2) c3 := &VaultConfig{ - Enabled: pointer.Of(true), - Token: "1", - Role: "1", - Namespace: "1", - AllowUnauthenticated: pointer.Of(true), - TaskTokenTTL: "1", - Addr: "1", - ConnectionRetryIntv: time.Second, - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "1", + Enabled: pointer.Of(true), + Role: "1", + Namespace: "1", + Addr: "1", + ConnectionRetryIntv: time.Second, + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "1", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.dev"}, Env: pointer.Of(true), @@ -155,20 +137,17 @@ func TestVaultConfig_Equals(t *testing.T) { } c4 := &VaultConfig{ - Enabled: pointer.Of(false), - Token: "1", - Role: "1", - Namespace: "1", - AllowUnauthenticated: pointer.Of(true), - TaskTokenTTL: "1", - Addr: "1", - ConnectionRetryIntv: time.Second, - TLSCaFile: "1", - TLSCaPath: "1", - TLSCertFile: "1", - TLSKeyFile: "1", - TLSSkipVerify: pointer.Of(true), - TLSServerName: "1", + Enabled: pointer.Of(false), + Role: "1", + Namespace: "1", + Addr: "1", + ConnectionRetryIntv: time.Second, + TLSCaFile: "1", + TLSCaPath: "1", + TLSCertFile: "1", + TLSKeyFile: "1", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "1", DefaultIdentity: &WorkloadIdentityConfig{ Audience: []string{"vault.io"}, Env: pointer.Of(false), diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index c49f152bb..8fb454f42 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -1976,11 +1976,9 @@ func vaultDiff(old, new *Vault, contextual bool) *ObjectDiff { if reflect.DeepEqual(old, new) { return nil } else if old == nil { - old = &Vault{} diff.Type = DiffTypeAdded newPrimitiveFlat = flatmap.Flatten(new, nil, true) } else if new == nil { - new = &Vault{} diff.Type = DiffTypeDeleted oldPrimitiveFlat = flatmap.Flatten(old, nil, true) } else { @@ -1992,11 +1990,6 @@ func vaultDiff(old, new *Vault, contextual bool) *ObjectDiff { // Diff the primitive fields. diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) - // Policies diffs - if setDiff := stringSetDiff(old.Policies, new.Policies, "Policies", contextual); setDiff != nil { - diff.Objects = append(diff.Objects, setDiff) - } - return diff } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index e394545ad..c6c5656bd 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -1729,21 +1729,6 @@ func TestJobDiff(t *testing.T) { }, }, }, - { - // VaultToken is filtered - Old: &Job{ - ID: "vault-job", - VaultToken: "secret", - }, - New: &Job{ - ID: "vault-job", - VaultToken: "new-secret", - }, - Expected: &JobDiff{ - Type: DiffTypeNone, - ID: "vault-job", - }, - }, // UI block is added { @@ -8454,169 +8439,12 @@ func TestTaskDiff(t *testing.T) { }, }, }, - - { - Name: "Vault added", - Old: &Task{}, - New: &Task{ - Vault: &Vault{ - Role: "nomad-task", - Policies: []string{"foo", "bar"}, - Env: true, - DisableFile: true, - ChangeMode: "signal", - ChangeSignal: "SIGUSR1", - }, - }, - Expected: &TaskDiff{ - Type: DiffTypeEdited, - Objects: []*ObjectDiff{ - { - Type: DiffTypeAdded, - Name: "Vault", - Fields: []*FieldDiff{ - { - Type: DiffTypeAdded, - Name: "AllowTokenExpiration", - Old: "", - New: "false", - }, - { - Type: DiffTypeAdded, - Name: "ChangeMode", - Old: "", - New: "signal", - }, - { - Type: DiffTypeAdded, - Name: "ChangeSignal", - Old: "", - New: "SIGUSR1", - }, - { - Type: DiffTypeAdded, - Name: "DisableFile", - Old: "", - New: "true", - }, - { - Type: DiffTypeAdded, - Name: "Env", - Old: "", - New: "true", - }, - { - Type: DiffTypeAdded, - Name: "Role", - Old: "", - New: "nomad-task", - }, - }, - Objects: []*ObjectDiff{ - { - Type: DiffTypeAdded, - Name: "Policies", - Fields: []*FieldDiff{ - { - Type: DiffTypeAdded, - Name: "Policies", - Old: "", - New: "bar", - }, - { - Type: DiffTypeAdded, - Name: "Policies", - Old: "", - New: "foo", - }, - }, - }, - }, - }, - }, - }, - }, - { - Name: "Vault deleted", - Old: &Task{ - Vault: &Vault{ - Policies: []string{"foo", "bar"}, - Env: true, - DisableFile: true, - ChangeMode: "signal", - ChangeSignal: "SIGUSR1", - }, - }, - New: &Task{}, - Expected: &TaskDiff{ - Type: DiffTypeEdited, - Objects: []*ObjectDiff{ - { - Type: DiffTypeDeleted, - Name: "Vault", - Fields: []*FieldDiff{ - { - Type: DiffTypeDeleted, - Name: "AllowTokenExpiration", - Old: "false", - New: "", - }, - { - Type: DiffTypeDeleted, - Name: "ChangeMode", - Old: "signal", - New: "", - }, - { - Type: DiffTypeDeleted, - Name: "ChangeSignal", - Old: "SIGUSR1", - New: "", - }, - { - Type: DiffTypeDeleted, - Name: "DisableFile", - Old: "true", - New: "", - }, - { - Type: DiffTypeDeleted, - Name: "Env", - Old: "true", - New: "", - }, - }, - Objects: []*ObjectDiff{ - { - Type: DiffTypeDeleted, - Name: "Policies", - Fields: []*FieldDiff{ - { - Type: DiffTypeDeleted, - Name: "Policies", - Old: "bar", - New: "", - }, - { - Type: DiffTypeDeleted, - Name: "Policies", - Old: "foo", - New: "", - }, - }, - }, - }, - }, - }, - }, - }, { Name: "Vault edited", Old: &Task{ Vault: &Vault{ Role: "nomad-task", Namespace: "ns1", - Policies: []string{"foo", "bar"}, Env: true, DisableFile: true, ChangeMode: "signal", @@ -8628,7 +8456,6 @@ func TestTaskDiff(t *testing.T) { Vault: &Vault{ Role: "nomad-task-2", Namespace: "ns2", - Policies: []string{"bar", "baz"}, Env: false, DisableFile: false, ChangeMode: "restart", @@ -8686,141 +8513,6 @@ func TestTaskDiff(t *testing.T) { New: "nomad-task-2", }, }, - Objects: []*ObjectDiff{ - { - Type: DiffTypeEdited, - Name: "Policies", - Fields: []*FieldDiff{ - { - Type: DiffTypeAdded, - Name: "Policies", - Old: "", - New: "baz", - }, - { - Type: DiffTypeDeleted, - Name: "Policies", - Old: "foo", - New: "", - }, - }, - }, - }, - }, - }, - }, - }, - { - Name: "Vault edited with context", - Contextual: true, - Old: &Task{ - Vault: &Vault{ - Role: "nomad-task", - Namespace: "ns1", - Cluster: VaultDefaultCluster, - Policies: []string{"foo", "bar"}, - Env: true, - DisableFile: true, - ChangeMode: "signal", - ChangeSignal: "SIGUSR1", - AllowTokenExpiration: true, - }, - }, - New: &Task{ - Vault: &Vault{ - Role: "nomad-task", - Namespace: "ns1", - Cluster: VaultDefaultCluster, - Policies: []string{"bar", "baz"}, - Env: true, - DisableFile: true, - ChangeMode: "signal", - ChangeSignal: "SIGUSR1", - AllowTokenExpiration: true, - }, - }, - Expected: &TaskDiff{ - Type: DiffTypeEdited, - Objects: []*ObjectDiff{ - { - Type: DiffTypeEdited, - Name: "Vault", - Fields: []*FieldDiff{ - { - Type: DiffTypeNone, - Name: "AllowTokenExpiration", - Old: "true", - New: "true", - }, - { - Type: DiffTypeNone, - Name: "ChangeMode", - Old: "signal", - New: "signal", - }, - { - Type: DiffTypeNone, - Name: "ChangeSignal", - Old: "SIGUSR1", - New: "SIGUSR1", - }, - { - Type: DiffTypeNone, - Name: "Cluster", - Old: VaultDefaultCluster, - New: VaultDefaultCluster, - }, - { - Type: DiffTypeNone, - Name: "DisableFile", - Old: "true", - New: "true", - }, - { - Type: DiffTypeNone, - Name: "Env", - Old: "true", - New: "true", - }, - { - Type: DiffTypeNone, - Name: "Namespace", - Old: "ns1", - New: "ns1", - }, - { - Type: DiffTypeNone, - Name: "Role", - Old: "nomad-task", - New: "nomad-task", - }, - }, - Objects: []*ObjectDiff{ - { - Type: DiffTypeEdited, - Name: "Policies", - Fields: []*FieldDiff{ - { - Type: DiffTypeAdded, - Name: "Policies", - Old: "", - New: "baz", - }, - { - Type: DiffTypeNone, - Name: "Policies", - Old: "bar", - New: "bar", - }, - { - Type: DiffTypeDeleted, - Name: "Policies", - Old: "foo", - New: "", - }, - }, - }, - }, }, }, }, diff --git a/nomad/structs/funcs.go b/nomad/structs/funcs.go index 0b0278ecf..32000467b 100644 --- a/nomad/structs/funcs.go +++ b/nomad/structs/funcs.go @@ -359,28 +359,14 @@ func CopySliceNodeScoreMeta(s []*NodeScoreMeta) []*NodeScoreMeta { return c } -// VaultPoliciesSet takes the structure returned by VaultPolicies and returns -// the set of required policies -func VaultPoliciesSet(policies map[string]map[string]*Vault) []string { +// VaultNamespaceSet takes the structure returned by job.Vault() and returns a +// set of required namespaces. +func VaultNamespaceSet(blocks map[string]map[string]*Vault) []string { s := set.New[string](10) - for _, tgp := range policies { - for _, tp := range tgp { - if tp != nil { - s.InsertSlice(tp.Policies) - } - } - } - return s.Slice() -} - -// VaultNamespaceSet takes the structure returned by VaultPolicies and -// returns a set of required namespaces -func VaultNamespaceSet(policies map[string]map[string]*Vault) []string { - s := set.New[string](10) - for _, tgp := range policies { - for _, tp := range tgp { - if tp != nil && tp.Namespace != "" { - s.Insert(tp.Namespace) + for _, taskGroupVault := range blocks { + for _, taskVault := range taskGroupVault { + if taskVault != nil && taskVault.Namespace != "" { + s.Insert(taskVault.Namespace) } } } diff --git a/nomad/structs/funcs_test.go b/nomad/structs/funcs_test.go index b27262db0..1f27e729d 100644 --- a/nomad/structs/funcs_test.go +++ b/nomad/structs/funcs_test.go @@ -846,64 +846,6 @@ func TestGenerateMigrateToken(t *testing.T) { assert.True(CompareMigrateToken("x", nodeSecret, token2)) } -func TestVaultPoliciesSet(t *testing.T) { - input := map[string]map[string]*Vault{ - "tg1": { - "task1": { - Policies: []string{"policy1-1"}, - }, - "task2": { - Policies: []string{"policy1-2"}, - }, - }, - "tg2": { - "task1": { - Policies: []string{"policy2"}, - }, - "task2": { - Policies: []string{"policy2"}, - }, - }, - "tg3": { - "task1": { - Policies: []string{"policy3-1"}, - }, - }, - "tg4": { - "task1": nil, - }, - "tg5": { - "task1": { - Policies: []string{"policy2"}, - }, - }, - "tg6": { - "task1": {}, - }, - "tg7": { - "task1": { - Policies: []string{"policy7", "policy7"}, - }, - }, - "tg8": { - "task1": { - Policies: []string{"policy8-1-1", "policy8-1-2"}, - }, - }, - } - expected := []string{ - "policy1-1", - "policy1-2", - "policy2", - "policy3-1", - "policy7", - "policy8-1-1", - "policy8-1-2", - } - got := VaultPoliciesSet(input) - require.ElementsMatch(t, expected, got) -} - func TestVaultNamespaceSet(t *testing.T) { input := map[string]map[string]*Vault{ "tg1": { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index e4401369b..833cc108b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -960,12 +960,6 @@ type JobRevertRequest struct { // token and is not stored after the Job revert. ConsulToken string - // VaultToken is the Vault token that proves the submitter of the job revert - // has access to any Vault policies specified in the targeted job version. This - // field is only used to transfer the token and is not stored after the Job - // revert. - VaultToken string - WriteRequest } @@ -1268,21 +1262,6 @@ type ClusterMetadata struct { CreateTime int64 } -// DeriveVaultTokenRequest is used to request wrapped Vault tokens for the -// following tasks in the given allocation -type DeriveVaultTokenRequest struct { - NodeID string - SecretID string - AllocID string - Tasks []string - QueryOptions -} - -// VaultAccessorsRequest is used to operate on a set of Vault accessors -type VaultAccessorsRequest struct { - Accessors []*VaultAccessor -} - // VaultAccessor is a reference to a created Vault token on behalf of // an allocation's task. type VaultAccessor struct { @@ -1296,18 +1275,6 @@ type VaultAccessor struct { CreateIndex uint64 } -// DeriveVaultTokenResponse returns the wrapped tokens for each requested task -type DeriveVaultTokenResponse struct { - // Tasks is a mapping between the task name and the wrapped token - Tasks map[string]string - - // Error stores any error that occurred. Errors are stored here so we can - // communicate whether it is retryable - Error *RecoverableError - - QueryMeta -} - // GenericRequest is used to request where no // specific information is needed. type GenericRequest struct { @@ -4560,11 +4527,6 @@ type Job struct { // ConsulNamespace is the Consul namespace ConsulNamespace string - // VaultToken is the Vault token that proves the submitter of the job has - // access to the specified Vault policies. This field is only used to - // transfer the token and is not stored after Job submission. - VaultToken string - // VaultNamespace is the Vault namespace VaultNamespace string @@ -10400,9 +10362,6 @@ type Vault struct { // cluster default role. Role string - // Policies is the set of policies that the task needs access to - Policies []string - // Namespace is the vault namespace that should be used. Namespace string @@ -10442,8 +10401,6 @@ func (v *Vault) Equal(o *Vault) bool { switch { case v.Role != o.Role: return false - case !slices.Equal(v.Policies, o.Policies): - return false case v.Namespace != o.Namespace: return false case v.Cluster != o.Cluster: @@ -10493,11 +10450,6 @@ func (v *Vault) Validate() error { } var mErr multierror.Error - for _, p := range v.Policies { - if p == "root" { - _ = multierror.Append(&mErr, fmt.Errorf("Can not specify \"root\" policy")) - } - } switch v.ChangeMode { case VaultChangeModeSignal: diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 4f44a8b37..eab6d433a 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -973,21 +973,13 @@ func TestJob_Vault(t *testing.T) { e0 := make(map[string]map[string]*Vault, 0) vj1 := &Vault{ - Policies: []string{ - "p1", - "p2", - }, + Role: "role-1", } vj2 := &Vault{ - Policies: []string{ - "p3", - "p4", - }, + Role: "role-2", } vj3 := &Vault{ - Policies: []string{ - "p5", - }, + Role: "role-3", } j1 := &Job{ TaskGroups: []*TaskGroup{ @@ -1123,11 +1115,9 @@ func TestJob_RequiredSignals(t *testing.T) { e0 := make(map[string]map[string][]string, 0) vj1 := &Vault{ - Policies: []string{"p1"}, ChangeMode: VaultChangeModeNoop, } vj2 := &Vault{ - Policies: []string{"p1"}, ChangeMode: VaultChangeModeSignal, ChangeSignal: "SIGUSR1", } @@ -6405,7 +6395,6 @@ func TestVault_Validate(t *testing.T) { v := &Vault{ Env: true, ChangeMode: VaultChangeModeSignal, - Policies: []string{"foo", "root"}, } err := v.Validate() @@ -6416,14 +6405,10 @@ func TestVault_Validate(t *testing.T) { if !strings.Contains(err.Error(), "Signal must") { t.Fatalf("Expected signal empty error") } - if !strings.Contains(err.Error(), "root") { - t.Fatalf("Expected root error") - } } func TestVault_Copy(t *testing.T) { v := &Vault{ - Policies: []string{"policy1", "policy2"}, Namespace: "ns1", Env: false, ChangeMode: "noop", @@ -6432,7 +6417,6 @@ func TestVault_Copy(t *testing.T) { // Copy and modify. vc := v.Copy() - vc.Policies[0] = "policy0" vc.Namespace = "ns2" vc.Env = true vc.ChangeMode = "signal" @@ -8088,7 +8072,6 @@ func TestVault_Equal(t *testing.T) { must.StructEqual(t, &Vault{ Role: "nomad-task", - Policies: []string{"one"}, Namespace: "global", Env: true, ChangeMode: "signal", @@ -8096,9 +8079,6 @@ func TestVault_Equal(t *testing.T) { }, []must.Tweak[*Vault]{{ Field: "Role", Apply: func(v *Vault) { v.Role = "nomad-task-2" }, - }, { - Field: "Policies", - Apply: func(v *Vault) { v.Policies = []string{"two"} }, }, { Field: "Namespace", Apply: func(v *Vault) { v.Namespace = "regional" }, diff --git a/nomad/vault.go b/nomad/vault.go deleted file mode 100644 index 71e70c890..000000000 --- a/nomad/vault.go +++ /dev/null @@ -1,1470 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/useragent" - tomb "gopkg.in/tomb.v2" - - log "github.com/hashicorp/go-hclog" - metrics "github.com/hashicorp/go-metrics/compat" - multierror "github.com/hashicorp/go-multierror" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/nomad/structs/config" - vapi "github.com/hashicorp/vault/api" - - "golang.org/x/sync/errgroup" - "golang.org/x/time/rate" -) - -const ( - // vaultTokenCreateTTL is the duration the wrapped token for the client is - // valid for. The units are in seconds. - vaultTokenCreateTTL = "60s" - - // minimumTokenTTL is the minimum Token TTL allowed for child tokens. - minimumTokenTTL = 5 * time.Minute - - // defaultTokenTTL is the default Token TTL used when the passed token is a - // root token such that child tokens aren't being created against a role - // that has defined a TTL - defaultTokenTTL = "72h" - - // requestRateLimit is the maximum number of requests per second Nomad will - // make against Vault - requestRateLimit rate.Limit = 500.0 - - // maxParallelRevokes is the maximum number of parallel Vault - // token revocation requests - maxParallelRevokes = 64 - - // vaultRevocationIntv is the interval at which Vault tokens that failed - // initial revocation are retried - vaultRevocationIntv = 5 * time.Minute - - // vaultCapabilitiesLookupPath is the path to lookup the capabilities of - // ones token. - vaultCapabilitiesLookupPath = "sys/capabilities-self" - - // vaultTokenRenewPath is the path used to renew our token - vaultTokenRenewPath = "auth/token/renew-self" - - // vaultTokenLookupPath is the path used to lookup a token - vaultTokenLookupPath = "auth/token/lookup" - - // vaultTokenRevokePath is the path used to revoke a token - vaultTokenRevokePath = "auth/token/revoke-accessor" - - // vaultRoleLookupPath is the path to lookup a role - vaultRoleLookupPath = "auth/token/roles/%s" - - // vaultRoleCreatePath is the path to create a token from a role - vaultTokenRoleCreatePath = "auth/token/create/%s" -) - -var ( - // vaultCapabilitiesCapability is the expected capability of Nomad's Vault - // token on the path. The token must have at least one of the - // capabilities. - vaultCapabilitiesCapability = []string{"update", "root"} - - // vaultTokenRenewCapability is the expected capability Nomad's - // Vault token should have on the path. The token must have at least one of - // the capabilities. - vaultTokenRenewCapability = []string{"update", "root"} - - // vaultTokenLookupCapability is the expected capability Nomad's - // Vault token should have on the path. The token must have at least one of - // the capabilities. - vaultTokenLookupCapability = []string{"update", "root"} - - // vaultTokenRevokeCapability is the expected capability Nomad's - // Vault token should have on the path. The token must have at least one of - // the capabilities. - vaultTokenRevokeCapability = []string{"update", "root"} - - // vaultRoleLookupCapability is the expected capability Nomad's Vault - // token should have on the path. The token must have at least one of the - // capabilities. - vaultRoleLookupCapability = []string{"read", "root"} - - // vaultTokenRoleCreateCapability is the expected capability Nomad's Vault - // token should have on the path. The token must have at least one of the - // capabilities. - vaultTokenRoleCreateCapability = []string{"update", "root"} -) - -// VaultClient is the Servers interface for interfacing with Vault -type VaultClient interface { - // SetActive activates or de-activates the Vault client. When active, token - // creation/lookup/revocation operation are allowed. - SetActive(active bool) - - // SetConfig updates the config used by the Vault client - SetConfig(config *config.VaultConfig) error - - // GetConfig returns a copy of the config used by the Vault client, for - // testing - GetConfig() *config.VaultConfig - - // CreateToken takes an allocation and task and returns an appropriate Vault - // Secret - CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) - - // LookupToken takes a token string and returns its capabilities. - LookupToken(ctx context.Context, token string) (*vapi.Secret, error) - - // RevokeTokens takes a set of tokens accessor and revokes the tokens - RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error - - // MarkForRevocation revokes the tokens in background - MarkForRevocation(accessors []*structs.VaultAccessor) error - - // Stop is used to stop token renewal - Stop() - - // Running returns whether the Vault client is running - Running() bool - - // Stats returns the Vault clients statistics - Stats() map[string]string - - // EmitStats emits that clients statistics at the given period until stopCh - // is called. - EmitStats(period time.Duration, stopCh <-chan struct{}) -} - -// VaultStats returns all the stats about Vault tokens created and managed by -// Nomad. -type VaultStats struct { - // TrackedForRevoke is the count of tokens that are being tracked to be - // revoked since they could not be immediately revoked. - TrackedForRevoke int - - // TokenTTL is the time-to-live duration for the current token - TokenTTL time.Duration - - // TokenExpiry is the recorded expiry time of the current token - TokenExpiry time.Time - - // LastRenewalTime is the time since the token was last renewed - LastRenewalTime time.Time - TimeFromLastRenewal time.Duration - - // NextRenewalTime is the time the token will attempt to renew - NextRenewalTime time.Time - TimeToNextRenewal time.Duration -} - -// PurgeVaultAccessorFn is called to remove VaultAccessors from the system. If -// the function returns an error, the token will still be tracked and revocation -// will retry till there is a success -type PurgeVaultAccessorFn func(accessors []*structs.VaultAccessor) error - -// vaultClient is the Servers implementation of the VaultClient interface. The -// client renews the PeriodicToken given in the Vault configuration and provides -// the Server with the ability to create child tokens and lookup the permissions -// of tokens. -type vaultClient struct { - // limiter is used to rate limit requests to Vault - limiter *rate.Limiter - - // client is the Vault API client used for Namespace-relative integrations - // with the Vault API (anything except `/v1/sys`). If this server is not - // configured to reference a Vault namespace, this will point to the same - // client as clientSys - client *vapi.Client - - // clientSys is the Vault API client used for non-Namespace-relative integrations - // with the Vault API (anything involving `/v1/sys`). This client is never configured - // with a Vault namespace, because these endpoints may return errors if a namespace - // header is provided - clientSys *vapi.Client - - // auth is the Vault token auth API client - auth *vapi.TokenAuth - - // config is the user passed Vault config - config *config.VaultConfig - - // connEstablished marks whether we have an established connection to Vault. - connEstablished bool - - // connEstablishedErr marks an error that can occur when establishing a - // connection - connEstablishedErr error - - // token is the raw token used by the client - token string - - // tokenData is the data of the passed Vault token - tokenData *structs.VaultTokenData - - // revoking tracks the VaultAccessors that must be revoked - revoking map[*structs.VaultAccessor]time.Time - purgeFn PurgeVaultAccessorFn - revLock sync.Mutex - - // active indicates whether the vaultClient is active. It should be - // accessed using a helper and updated atomically - active int32 - - // running indicates whether the vault client is started. - running bool - - // renewLoopActive indicates whether the renewal goroutine is running - // It should be accessed and updated atomically - // used for testing purposes only - renewLoopActive int32 - - // childTTL is the TTL for child tokens. - childTTL string - - // currentExpiration is the time the current token lease expires - currentExpiration time.Time - currentExpirationLock sync.Mutex - lastRenewalTime time.Time - nextRenewalTime time.Time - renewalTimeLock sync.Mutex - - tomb *tomb.Tomb - logger log.Logger - - // l is used to lock the configuration aspects of the client such that - // multiple callers can't cause conflicting config updates - l sync.Mutex - - // setConfigLock serializes access to the SetConfig method - setConfigLock sync.Mutex - - // consts as struct fields for overriding in tests - maxRevokeBatchSize int - revocationIntv time.Duration - - entHandler taskClientHandler -} - -type taskClientHandler interface { - clientForTask(v *vaultClient, namespace string) (*vapi.Client, error) -} - -// NewVaultClient returns a Vault client from the given config. If the client -// couldn't be made an error is returned. -func NewVaultClient(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVaultAccessorFn, delegate taskClientHandler) (*vaultClient, error) { - if c == nil { - return nil, fmt.Errorf("must pass valid VaultConfig") - } - - if logger == nil { - return nil, fmt.Errorf("must pass valid logger") - } - if purgeFn == nil { - purgeFn = func(accessors []*structs.VaultAccessor) error { return nil } - } - if delegate == nil { - delegate = &VaultNoopDelegate{} - } - - v := &vaultClient{ - config: c, - logger: logger.Named("vault"), - limiter: rate.NewLimiter(requestRateLimit, int(requestRateLimit)), - revoking: make(map[*structs.VaultAccessor]time.Time), - purgeFn: purgeFn, - tomb: &tomb.Tomb{}, - maxRevokeBatchSize: maxVaultRevokeBatchSize, - revocationIntv: vaultRevocationIntv, - entHandler: delegate, - } - - if v.config.IsEnabled() { - if err := v.buildClient(); err != nil { - return nil, err - } - - // Launch the required goroutines - v.tomb.Go(wrapNilError(v.establishConnection)) - v.tomb.Go(wrapNilError(v.revokeDaemon)) - - v.running = true - } - - return v, nil -} - -func (v *vaultClient) Stop() { - v.l.Lock() - running := v.running - v.running = false - v.l.Unlock() - - if running { - v.tomb.Kill(nil) - v.tomb.Wait() - v.flush() - } -} - -func (v *vaultClient) Running() bool { - v.l.Lock() - defer v.l.Unlock() - return v.running -} - -// SetActive activates or de-activates the Vault client. When active, token -// creation/lookup/revocation operation are allowed. All queued revocations are -// cancelled if set un-active as it is assumed another instances is taking over -func (v *vaultClient) SetActive(active bool) { - if active { - atomic.StoreInt32(&v.active, 1) - } else { - atomic.StoreInt32(&v.active, 0) - } - - // Clear out the revoking tokens - v.revLock.Lock() - v.revoking = make(map[*structs.VaultAccessor]time.Time) - v.revLock.Unlock() -} - -// flush is used to reset the state of the vault client -func (v *vaultClient) flush() { - v.l.Lock() - defer v.l.Unlock() - v.revLock.Lock() - defer v.revLock.Unlock() - - v.client = nil - v.clientSys = nil - v.auth = nil - v.connEstablished = false - v.connEstablishedErr = nil - v.token = "" - v.tokenData = nil - v.revoking = make(map[*structs.VaultAccessor]time.Time) - v.childTTL = "" - v.tomb = &tomb.Tomb{} -} - -// GetConfig returns a copy of this vault client's configuration, for testing. -func (v *vaultClient) GetConfig() *config.VaultConfig { - v.setConfigLock.Lock() - defer v.setConfigLock.Unlock() - return v.config.Copy() -} - -// SetConfig is used to update the Vault config being used. A temporary outage -// may occur after calling as it re-establishes a connection to Vault -func (v *vaultClient) SetConfig(config *config.VaultConfig) error { - if config == nil { - return fmt.Errorf("must pass valid VaultConfig") - } - v.setConfigLock.Lock() - defer v.setConfigLock.Unlock() - - v.l.Lock() - defer v.l.Unlock() - - // If reloading the same config, no-op - if v.config.Equal(config) { - return nil - } - - // Kill any background routines - if v.running { - // Kill any background routine - v.tomb.Kill(nil) - - // Locking around tomb.Wait can deadlock with - // establishConnection exiting, so we must unlock here. - v.l.Unlock() - v.tomb.Wait() - v.l.Lock() - - // Stop accepting any new requests - v.connEstablished = false - v.tomb = &tomb.Tomb{} - v.running = false - } - - // Store the new config - v.config = config - - // Check if we should relaunch - if v.config.IsEnabled() { - // Rebuild the client - if err := v.buildClient(); err != nil { - return err - } - - // Launch the required goroutines - v.tomb.Go(wrapNilError(v.establishConnection)) - v.tomb.Go(wrapNilError(v.revokeDaemon)) - v.running = true - } - - return nil -} - -// buildClient is used to build a Vault client based on the stored Vault config -func (v *vaultClient) buildClient() error { - // Validate we have the required fields. - if v.config.Token == "" { - return errors.New("Vault token must be set") - } else if v.config.Addr == "" { - return errors.New("Vault address must be set") - } - - // Parse the TTL if it is set - if v.config.TaskTokenTTL != "" { - d, err := time.ParseDuration(v.config.TaskTokenTTL) - if err != nil { - return fmt.Errorf("failed to parse TaskTokenTTL %q: %v", v.config.TaskTokenTTL, err) - } - - if d.Nanoseconds() < minimumTokenTTL.Nanoseconds() { - return fmt.Errorf("ChildTokenTTL is less than minimum allowed of %v", minimumTokenTTL) - } - - v.childTTL = v.config.TaskTokenTTL - } else { - // Default the TaskTokenTTL - v.childTTL = defaultTokenTTL - } - - // Get the Vault API configuration - apiConf, err := v.config.ApiConfig() - if err != nil { - return fmt.Errorf("Failed to create Vault API config: %v", err) - } - - // Create the Vault API client - client, err := vapi.NewClient(apiConf) - if err != nil { - v.logger.Error("failed to create Vault client and not retrying", "error", err) - return err - } - useragent.SetHeaders(client) - - // Store the client, create/assign the /sys client - v.client = client - if v.config.Namespace != "" { - v.logger.Debug("configuring Vault namespace", "namespace", v.config.Namespace) - v.clientSys, err = vapi.NewClient(apiConf) - if err != nil { - v.logger.Error("failed to create Vault sys client and not retrying", "error", err) - return err - } - useragent.SetHeaders(v.clientSys) - client.SetNamespace(v.config.Namespace) - } else { - v.clientSys = client - } - - // Set the token - v.token = v.config.Token - client.SetToken(v.token) - v.auth = client.Auth().Token() - - return nil -} - -// establishConnection is used to make first contact with Vault. This should be -// called in a go-routine since the connection is retried until the Vault Client -// is stopped or the connection is successfully made at which point the renew -// loop is started. -func (v *vaultClient) establishConnection() { - // Create the retry timer and set initial duration to zero so it fires - // immediately - retryTimer := time.NewTimer(0) - initStatus := false -OUTER: - for { - select { - case <-v.tomb.Dying(): - return - case <-retryTimer.C: - // Retry validating the token till success - if err := v.parseSelfToken(); err != nil { - // if parsing token fails, try to distinguish legitimate token error from transient Vault initialization/connection issue - if !initStatus { - if _, err := v.clientSys.Sys().Health(); err != nil { - v.logger.Warn("failed to contact Vault API", "retry", v.config.ConnectionRetryIntv, "error", err) - retryTimer.Reset(v.config.ConnectionRetryIntv) - continue OUTER - } - initStatus = true - } - - v.logger.Error("failed to validate self token/role", "retry", v.config.ConnectionRetryIntv, "error", err) - retryTimer.Reset(v.config.ConnectionRetryIntv) - v.l.Lock() - v.connEstablished = true - v.connEstablishedErr = fmt.Errorf("failed to establish connection to Vault: %v", err) - v.l.Unlock() - continue OUTER - } - - break OUTER - } - } - - // Set the wrapping function such that token creation is wrapped now - // that we know our role - v.client.SetWrappingLookupFunc(v.getWrappingFn()) - - // If we are given a non-root token, start renewing it - if v.tokenData.Root() && v.tokenData.CreationTTL == 0 { - v.logger.Debug("not renewing token as it is root") - } else { - v.logger.Debug("starting renewal loop", "creation_ttl", time.Duration(v.tokenData.CreationTTL)*time.Second) - v.tomb.Go(wrapNilError(v.renewalLoop)) - } - - v.l.Lock() - v.connEstablished = true - v.connEstablishedErr = nil - v.l.Unlock() -} - -func (v *vaultClient) isRenewLoopActive() bool { - return atomic.LoadInt32(&v.renewLoopActive) == 1 -} - -// renewalLoop runs the renew loop. This should only be called if we are given a -// non-root token. -func (v *vaultClient) renewalLoop() { - atomic.StoreInt32(&v.renewLoopActive, 1) - defer atomic.StoreInt32(&v.renewLoopActive, 0) - - // Create the renewal timer and set initial duration to zero so it fires - // immediately - authRenewTimer := time.NewTimer(0) - - // Backoff is to reduce the rate we try to renew with Vault under error - // situations - backoff := 0.0 - - for { - select { - case <-v.tomb.Dying(): - return - case <-authRenewTimer.C: - // Renew the token and determine the new expiration - recoverable, err := v.renew() - v.currentExpirationLock.Lock() - currentExpiration := v.currentExpiration - v.currentExpirationLock.Unlock() - - // Successfully renewed - if err == nil { - // Attempt to renew the token at half the expiration time - durationUntilRenew := time.Until(currentExpiration) / 2 - v.renewalTimeLock.Lock() - now := time.Now() - v.lastRenewalTime = now - v.nextRenewalTime = now.Add(durationUntilRenew) - v.renewalTimeLock.Unlock() - - v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) - authRenewTimer.Reset(durationUntilRenew) - - // Reset any backoff - backoff = 0 - break - } - - metrics.IncrCounter([]string{"nomad", "vault", "renew_failed"}, 1) - v.logger.Warn("got error or bad auth, so backing off", "error", err, "recoverable", recoverable) - - if !recoverable { - return - } - - backoff = nextBackoff(backoff, currentExpiration) - if backoff < 0 { - // We have failed to renew the token past its expiration. Stop - // renewing with Vault. - v.logger.Error("failed to renew Vault token before lease expiration. Shutting down Vault client", - "error", err) - v.l.Lock() - v.connEstablished = false - v.connEstablishedErr = err - v.l.Unlock() - return - } - - durationUntilRetry := time.Duration(backoff) * time.Second - v.renewalTimeLock.Lock() - v.nextRenewalTime = time.Now().Add(durationUntilRetry) - v.renewalTimeLock.Unlock() - v.logger.Info("backing off renewal", "retry", durationUntilRetry) - - authRenewTimer.Reset(durationUntilRetry) - } - } -} - -// nextBackoff returns the delay for the next auto renew interval, in seconds. -// Returns negative value if past expiration -// -// It should increase the amount of backoff each time, with the following rules: -// -// - If token expired already despite earlier renewal attempts, -// back off for 1 minute + jitter -// - If we have an existing authentication that is going to expire, -// -// never back off more than half of the amount of time remaining -// until expiration (with 5s floor) -// * Never back off more than 30 seconds multiplied by a random -// value between 1 and 2 -// * Use randomness so that many clients won't keep hitting Vault -// at the same time -func nextBackoff(backoff float64, expiry time.Time) float64 { - maxBackoff := time.Until(expiry) / 2 - - if maxBackoff < 0 { - // expiry passed - return 60 * (1.0 + rand.Float64()) - } - - switch { - case backoff >= 24: - backoff = 30 - default: - backoff = backoff * 1.25 - } - - // Add randomness - backoff = backoff * (1.0 + rand.Float64()) - - if backoff > maxBackoff.Seconds() { - backoff = maxBackoff.Seconds() - } - - if backoff < 5 { - backoff = 5 - } - - return backoff -} - -// renew attempts to renew our Vault token. If the renewal fails, an error is -// returned. The boolean indicates whether it's safe to attempt to renew again. -// This method updates the currentExpiration time -func (v *vaultClient) renew() (bool, error) { - // Track how long the request takes - defer metrics.MeasureSince([]string{"nomad", "vault", "renew"}, time.Now()) - - // Attempt to renew the token - secret, err := v.auth.RenewSelf(v.tokenData.CreationTTL) - if err != nil { - // Check if there is a permission denied - recoverable := !structs.VaultUnrecoverableError.MatchString(err.Error()) - return recoverable, fmt.Errorf("failed to renew the vault token: %v", err) - } - - if secret == nil { - // It's possible for RenewSelf to return (nil, nil) if the - // response body from Vault is empty. - return true, fmt.Errorf("renewal failed: empty response from vault") - } - - // these treated as transient errors, where can keep renewing - auth := secret.Auth - if auth == nil { - return true, fmt.Errorf("renewal successful but not auth information returned") - } else if auth.LeaseDuration == 0 { - return true, fmt.Errorf("renewal successful but no lease duration returned") - } - - v.extendExpiration(auth.LeaseDuration) - - v.logger.Debug("successfully renewed server token") - return true, nil -} - -// getWrappingFn returns an appropriate wrapping function for Nomad Servers -func (v *vaultClient) getWrappingFn() func(operation, path string) string { - createPath := "auth/token/create" - role := v.getRole() - if role != "" { - createPath = fmt.Sprintf("auth/token/create/%s", role) - } - - return func(operation, path string) string { - // Only wrap the token create operation - if operation != "POST" || path != createPath { - return "" - } - - return vaultTokenCreateTTL - } -} - -// parseSelfToken looks up the Vault token in Vault and parses its data storing -// it in the client. If the token is not valid for Nomads purposes an error is -// returned. -func (v *vaultClient) parseSelfToken() error { - // Try looking up the token using the self endpoint - secret, err := v.lookupSelf() - if err != nil { - return err - } - - // Read and parse the fields - var data structs.VaultTokenData - if err := structs.DecodeVaultSecretData(secret, &data); err != nil { - return fmt.Errorf("failed to parse Vault token's data block: %v", err) - } - v.tokenData = &data - v.extendExpiration(data.TTL) - - // The criteria that must be met for the token to be valid are as follows: - // 1) If token is non-root or is but has a creation ttl - // a) The token must be renewable - // b) Token must have a non-zero TTL - // 2) Must have update capability for "auth/token/lookup/" (used to verify incoming tokens) - // 3) Must have update capability for "/auth/token/revoke-accessor/" (used to revoke unneeded tokens) - // 4) If configured to create tokens against a role: - // a) Must have read capability for "auth/token/roles/" - // c) Role must: - // 1) Must allow tokens to be renewed - // 2) Must not have an explicit max TTL - // 3) Must have non-zero period - // 5) If not configured against a role, the token must be root - - var mErr multierror.Error - role := v.getRole() - if !data.Root() { - // All non-root tokens must be renewable - if !data.Renewable { - _ = multierror.Append(&mErr, fmt.Errorf("Vault token is not renewable or root")) - } - - // All non-root tokens must have a lease duration - if data.CreationTTL == 0 { - _ = multierror.Append(&mErr, fmt.Errorf("invalid lease duration of zero")) - } - - // The lease duration can not be expired - if data.TTL == 0 { - _ = multierror.Append(&mErr, fmt.Errorf("token TTL is zero")) - } - - // There must be a valid role since we aren't root - if role == "" { - _ = multierror.Append(&mErr, fmt.Errorf("token role name must be set when not using a root token")) - } - - } else if data.CreationTTL != 0 { - // If the root token has a TTL it must be renewable - if !data.Renewable { - _ = multierror.Append(&mErr, fmt.Errorf("Vault token has a TTL but is not renewable")) - } else if data.TTL == 0 { - // If the token has a TTL make sure it has not expired - _ = multierror.Append(&mErr, fmt.Errorf("token TTL is zero")) - } - } - - // Check we have the correct capabilities - if err := v.validateCapabilities(role, data.Root()); err != nil { - _ = multierror.Append(&mErr, err) - } - - // If given a role validate it - if role != "" { - if err := v.validateRole(role); err != nil { - _ = multierror.Append(&mErr, err) - } - } - - return mErr.ErrorOrNil() -} - -// lookupSelf is a helper function that looks up latest self lease info. -func (v *vaultClient) lookupSelf() (*vapi.Secret, error) { - // Get the initial lease duration - auth := v.client.Auth().Token() - - secret, err := auth.LookupSelf() - if err == nil && secret != nil && secret.Data != nil { - return secret, nil - } - - // Try looking up our token directly, even when we get an empty response, - // in case of an unexpected event - a true failure would occur in this lookup again - secret, err = auth.Lookup(v.client.Token()) - switch { - case err != nil: - return nil, fmt.Errorf("failed to lookup Vault periodic token: %v", err) - case secret == nil || secret.Data == nil: - return nil, fmt.Errorf("failed to lookup Vault periodic token: got empty response") - default: - return secret, nil - } -} - -// getRole returns the role name to be used when creating tokens -func (v *vaultClient) getRole() string { - if v.config.Role != "" { - return v.config.Role - } - - return v.tokenData.Role -} - -// validateCapabilities checks that Nomad's Vault token has the correct -// capabilities. -func (v *vaultClient) validateCapabilities(role string, root bool) error { - // Check if the token can lookup capabilities. - var mErr multierror.Error - _, _, err := v.hasCapability(vaultCapabilitiesLookupPath, vaultCapabilitiesCapability) - if err != nil { - // Check if there is a permission denied - if structs.VaultUnrecoverableError.MatchString(err.Error()) { - // Since we can't read permissions, we just log a warning that we - // can't tell if the Vault token will work - msg := fmt.Sprintf("can not lookup token capabilities. "+ - "As such certain operations may fail in the future. "+ - "Please give Nomad a Vault token with one of the following "+ - "capabilities %q on %q so that the required capabilities can be verified", - vaultCapabilitiesCapability, vaultCapabilitiesLookupPath) - v.logger.Warn(msg) - return nil - } else { - _ = multierror.Append(&mErr, err) - } - } - - // verify is a helper function that verifies the token has one of the - // capabilities on the given path and adds an issue to the error - verify := func(path string, requiredCaps []string) { - ok, caps, err := v.hasCapability(path, requiredCaps) - if err != nil { - _ = multierror.Append(&mErr, err) - } else if !ok { - _ = multierror.Append(&mErr, - fmt.Errorf("token must have one of the following capabilities %q on %q; has %v", requiredCaps, path, caps)) - } - } - - // Check if we are verifying incoming tokens - if !v.config.AllowsUnauthenticated() { - verify(vaultTokenLookupPath, vaultTokenLookupCapability) - } - - // Verify we can renew our selves tokens - verify(vaultTokenRenewPath, vaultTokenRenewCapability) - - // Verify we can revoke tokens - verify(vaultTokenRevokePath, vaultTokenRevokeCapability) - - // If we are using a role verify the capability - if role != "" { - // Verify we can read the role - verify(fmt.Sprintf(vaultRoleLookupPath, role), vaultRoleLookupCapability) - - // Verify we can create from the role - verify(fmt.Sprintf(vaultTokenRoleCreatePath, role), vaultTokenRoleCreateCapability) - } - - return mErr.ErrorOrNil() -} - -// hasCapability takes a path and returns whether the token has at least one of -// the required capabilities on the given path. It also returns the set of -// capabilities the token does have as well as any error that occurred. -func (v *vaultClient) hasCapability(path string, required []string) (bool, []string, error) { - caps, err := v.client.Sys().CapabilitiesSelf(path) - if err != nil { - return false, nil, err - } - for _, c := range caps { - for _, r := range required { - if c == r { - return true, caps, nil - } - } - } - return false, caps, nil -} - -// validateRole contacts Vault and checks that the given Vault role is valid for -// the purposes of being used by Nomad -func (v *vaultClient) validateRole(role string) error { - if role == "" { - return fmt.Errorf("Invalid empty role name") - } - - // Validate the role - rsecret, err := v.client.Logical().Read(fmt.Sprintf("auth/token/roles/%s", role)) - if err != nil { - return fmt.Errorf("failed to lookup role %q: %v", role, err) - } - if rsecret == nil { - return fmt.Errorf("Role %q does not exist", role) - } - - // Read and parse the fields - var data structs.VaultTokenRoleData - if err := structs.DecodeVaultSecretData(rsecret, &data); err != nil { - return fmt.Errorf("failed to parse Vault role's data block: %v", err) - } - - // Validate the role is acceptable - var mErr multierror.Error - if !data.Renewable { - _ = multierror.Append(&mErr, fmt.Errorf("Role must allow tokens to be renewed")) - } - - if data.ExplicitMaxTtl != 0 || data.TokenExplicitMaxTtl != 0 { - _ = multierror.Append(&mErr, fmt.Errorf("Role can not use an explicit max ttl. Token must be periodic.")) - } - - if data.Period == 0 && data.TokenPeriod == 0 { - _ = multierror.Append(&mErr, fmt.Errorf("Role must have a non-zero period to make tokens periodic.")) - } - - return mErr.ErrorOrNil() -} - -// ConnectionEstablished returns whether a connection to Vault has been -// established and any error that potentially caused it to be false -func (v *vaultClient) ConnectionEstablished() (bool, error) { - v.l.Lock() - defer v.l.Unlock() - return v.connEstablished, v.connEstablishedErr -} - -// Enabled returns whether the client is active -func (v *vaultClient) Enabled() bool { - v.l.Lock() - defer v.l.Unlock() - return v.config.IsEnabled() -} - -// Active returns whether the client is active -func (v *vaultClient) Active() bool { - return atomic.LoadInt32(&v.active) == 1 -} - -// CreateToken takes the allocation and task and returns an appropriate Vault -// token. The call is rate limited and may be canceled with the passed policy. -// When the error is recoverable, it will be of type RecoverableError -func (v *vaultClient) CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) { - if !v.Enabled() { - return nil, fmt.Errorf("Vault integration disabled") - } - if !v.Active() { - return nil, structs.NewRecoverableError(fmt.Errorf("Vault client not active"), true) - } - // Check if we have established a connection with Vault - if established, err := v.ConnectionEstablished(); !established && err == nil { - return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) - } else if err != nil { - return nil, err - } - - // Track how long the request takes - defer metrics.MeasureSince([]string{"nomad", "vault", "create_token"}, time.Now()) - - // Retrieve the Vault block for the task - vaultBlocks := a.Job.Vault() - if vaultBlocks == nil { - return nil, fmt.Errorf("Job does not require Vault token") - } - tg, ok := vaultBlocks[a.TaskGroup] - if !ok { - return nil, fmt.Errorf("Task group does not require Vault token") - } - taskVault, ok := tg[task] - if !ok { - return nil, fmt.Errorf("Task does not require Vault token") - } - - // Set namespace for task - namespaceForTask := v.config.Namespace - if taskVault.Namespace != "" { - namespaceForTask = taskVault.Namespace - } - - // Build the creation request - req := &vapi.TokenCreateRequest{ - Policies: taskVault.Policies, - Metadata: map[string]string{ - "AllocationID": a.ID, - "JobID": a.JobID, - "TaskGroup": a.TaskGroup, - "Task": task, - "NodeID": a.NodeID, - "Namespace": namespaceForTask, - }, - TTL: v.childTTL, - DisplayName: fmt.Sprintf("%s-%s", a.ID, task), - } - - // Ensure we are under our rate limit - if err := v.limiter.Wait(ctx); err != nil { - return nil, err - } - - // Make the request and switch depending on whether we are using a root - // token or a role based token - var secret *vapi.Secret - var err error - role := v.getRole() - - // Fetch client for task - taskClient, err := v.entHandler.clientForTask(v, namespaceForTask) - if err != nil { - return nil, err - } - - if v.tokenData.Root() && role == "" { - req.Period = v.childTTL - secret, err = taskClient.Auth().Token().Create(req) - } else { - // Make the token using the role - secret, err = taskClient.Auth().Token().CreateWithRole(req, role) - } - - // Determine whether it is unrecoverable - if err != nil { - err = fmt.Errorf("failed to create an alloc vault token: %v", err) - if structs.VaultUnrecoverableError.MatchString(err.Error()) { - return secret, err - } - - // The error is recoverable - return nil, structs.NewRecoverableError(err, true) - } - - // Validate the response - var validationErr error - if secret == nil { - validationErr = fmt.Errorf("Vault returned nil Secret") - } else if secret.WrapInfo == nil { - validationErr = fmt.Errorf("Vault returned Secret with nil WrapInfo. Secret warnings: %v", secret.Warnings) - } else if secret.WrapInfo.WrappedAccessor == "" { - validationErr = fmt.Errorf("Vault returned WrapInfo without WrappedAccessor. Secret warnings: %v", secret.Warnings) - } - if validationErr != nil { - v.logger.Warn("failed to CreateToken", "error", validationErr) - return nil, structs.NewRecoverableError(validationErr, true) - } - - // Got a valid response - return secret, nil -} - -// LookupToken takes a Vault token and does a lookup against Vault. The call is -// rate limited and may be canceled with passed context. -func (v *vaultClient) LookupToken(ctx context.Context, token string) (*vapi.Secret, error) { - if !v.Enabled() { - return nil, fmt.Errorf("Vault integration disabled") - } - - if !v.Active() { - return nil, fmt.Errorf("Vault client not active") - } - - // Check if we have established a connection with Vault - if established, err := v.ConnectionEstablished(); !established && err == nil { - return nil, structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) - } else if err != nil { - return nil, err - } - - // Track how long the request takes - defer metrics.MeasureSince([]string{"nomad", "vault", "lookup_token"}, time.Now()) - - // Ensure we are under our rate limit - if err := v.limiter.Wait(ctx); err != nil { - return nil, err - } - - // Lookup the token - return v.auth.Lookup(token) -} - -// RevokeTokens revokes the passed set of accessors. If committed is set, the -// purge function passed to the client is called. If there is an error purging -// either because of Vault failures or because of the purge function, the -// revocation is retried until the tokens TTL. -func (v *vaultClient) RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error { - if !v.Enabled() { - return nil - } - - if !v.Active() { - return fmt.Errorf("Vault client not active") - } - - // Track how long the request takes - defer metrics.MeasureSince([]string{"nomad", "vault", "revoke_tokens"}, time.Now()) - - // Check if we have established a connection with Vault. If not just add it - // to the queue - if established, err := v.ConnectionEstablished(); !established && err == nil { - // Only bother tracking it for later revocation if the accessor was - // committed - if committed { - v.storeForRevocation(accessors) - } - - // Track that we are abandoning these accessors. - metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_abandoned"}, float32(len(accessors))) - return nil - } - - // Attempt to revoke immediately and if it fails, add it to the revoke queue - err := v.parallelRevoke(ctx, accessors) - if err != nil { - // If it is uncommitted, it is a best effort revoke as it will shortly - // TTL within the cubbyhole and has not been leaked to any outside - // system - if !committed { - metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_abandoned"}, float32(len(accessors))) - return nil - } - - v.logger.Warn("failed to revoke tokens. Will reattempt until TTL", "error", err) - v.storeForRevocation(accessors) - return nil - } else if !committed { - // Mark that it was revoked but there is nothing to purge so exit - metrics.IncrCounter([]string{"nomad", "vault", "undistributed_tokens_revoked"}, float32(len(accessors))) - return nil - } - - if err := v.purgeFn(accessors); err != nil { - v.logger.Error("failed to purge Vault accessors", "error", err) - v.storeForRevocation(accessors) - return nil - } - - // Track that it was revoked successfully - metrics.IncrCounter([]string{"nomad", "vault", "distributed_tokens_revoked"}, float32(len(accessors))) - - return nil -} - -func (v *vaultClient) MarkForRevocation(accessors []*structs.VaultAccessor) error { - if !v.Enabled() { - return nil - } - - if !v.Active() { - return fmt.Errorf("Vault client not active") - } - - v.storeForRevocation(accessors) - return nil -} - -// storeForRevocation stores the passed set of accessors for revocation. It -// captures their effective TTL by storing their create TTL plus the current -// time. -func (v *vaultClient) storeForRevocation(accessors []*structs.VaultAccessor) { - v.revLock.Lock() - - now := time.Now() - for _, a := range accessors { - if _, ok := v.revoking[a]; !ok { - v.revoking[a] = now.Add(time.Duration(a.CreationTTL) * time.Second) - } - } - v.revLock.Unlock() -} - -// parallelRevoke revokes the passed VaultAccessors in parallel. -func (v *vaultClient) parallelRevoke(ctx context.Context, accessors []*structs.VaultAccessor) error { - if !v.Enabled() { - return fmt.Errorf("Vault integration disabled") - } - - if !v.Active() { - return fmt.Errorf("Vault client not active") - } - - // Check if we have established a connection with Vault - if established, err := v.ConnectionEstablished(); !established && err == nil { - return structs.NewRecoverableError(fmt.Errorf("Connection to Vault has not been established"), true) - } else if err != nil { - return err - } - - g, pCtx := errgroup.WithContext(ctx) - - // Cap the handlers - handlers := len(accessors) - if handlers > maxParallelRevokes { - handlers = maxParallelRevokes - } - - // Revoke the Vault Token Accessors - input := make(chan *structs.VaultAccessor, handlers) - for i := 0; i < handlers; i++ { - g.Go(func() error { - for { - select { - case va, ok := <-input: - if !ok { - return nil - } - - err := v.auth.RevokeAccessor(va.Accessor) - if err != nil && !strings.Contains(err.Error(), "invalid accessor") { - return fmt.Errorf("failed to revoke token (alloc: %q, node: %q, task: %q): %v", va.AllocID, va.NodeID, va.Task, err) - } - case <-pCtx.Done(): - return nil - } - } - }) - } - - // Send the input - go func() { - defer close(input) - for _, va := range accessors { - select { - case <-pCtx.Done(): - return - case input <- va: - } - } - - }() - - // Wait for everything to complete - return g.Wait() -} - -// maxVaultRevokeBatchSize is the maximum tokens a revokeDaemon should revoke -// and purge at any given time. -// -// Limiting the revocation batch size is beneficial for few reasons: -// - A single revocation failure of any entry in batch result into retrying the whole batch; -// the larger the batch is the higher likelihood of such failure -// - Smaller batch sizes result into more co-operativeness: provides hooks for -// reconsidering token TTL and leadership steps down. -// - Batches limit the size of the Raft message purging tokens. Due to bugs -// pre-0.11.3, expired tokens were not properly purged, so users upgrading from -// older versions may have huge numbers (millions) of expired tokens to purge. -const maxVaultRevokeBatchSize = 1000 - -// revokeDaemon should be called in a goroutine and is used to periodically -// revoke Vault accessors that failed the original revocation -func (v *vaultClient) revokeDaemon() { - ticker := time.NewTicker(v.revocationIntv) - defer ticker.Stop() - - for { - select { - case <-v.tomb.Dying(): - return - case now := <-ticker.C: - if established, err := v.ConnectionEstablished(); !established || err != nil { - continue - } - - v.revLock.Lock() - - // Fast path - if len(v.revoking) == 0 { - v.revLock.Unlock() - continue - } - - // Build the list of accessors that need to be revoked while pruning any TTL'd checks - toRevoke := len(v.revoking) - if toRevoke > v.maxRevokeBatchSize { - v.logger.Info("batching tokens to be revoked", - "to_revoke", toRevoke, "batch_size", v.maxRevokeBatchSize, - "batch_interval", v.revocationIntv) - toRevoke = v.maxRevokeBatchSize - } - revoking := make([]*structs.VaultAccessor, 0, toRevoke) - ttlExpired := []*structs.VaultAccessor{} - for va, ttl := range v.revoking { - if now.After(ttl) { - ttlExpired = append(ttlExpired, va) - } else { - revoking = append(revoking, va) - } - - // Batches should consider tokens to be revoked - // as well as expired tokens to ensure the Raft - // message is reasonably sized. - if len(revoking)+len(ttlExpired) >= toRevoke { - break - } - } - - if err := v.parallelRevoke(context.Background(), revoking); err != nil { - v.logger.Warn("background token revocation errored", "error", err) - v.revLock.Unlock() - continue - } - - // Unlock before a potentially expensive operation - v.revLock.Unlock() - - // purge all explicitly revoked as well as ttl expired tokens - // and only remove them locally on purge success - revoking = append(revoking, ttlExpired...) - - // Call the passed in token revocation function - if err := v.purgeFn(revoking); err != nil { - // Can continue since revocation is idempotent - v.logger.Error("token revocation errored", "error", err) - continue - } - - // Track that tokens were revoked successfully - metrics.IncrCounter([]string{"nomad", "vault", "distributed_tokens_revoked"}, float32(len(revoking))) - - // Can delete from the tracked list now that we have purged - v.revLock.Lock() - for _, va := range revoking { - delete(v.revoking, va) - } - v.revLock.Unlock() - - } - } -} - -// purgeVaultAccessors creates a Raft transaction to remove the passed Vault -// Accessors -func (s *Server) purgeVaultAccessors(accessors []*structs.VaultAccessor) error { - // Commit this update via Raft - req := structs.VaultAccessorsRequest{Accessors: accessors} - _, _, err := s.raftApply(structs.VaultAccessorDeregisterRequestType, req) - return err -} - -// wrapNilError is a helper that returns a wrapped function that returns a nil -// error -func wrapNilError(f func()) func() error { - return func() error { - f() - return nil - } -} - -// setLimit is used to update the rate limit -func (v *vaultClient) setLimit(l rate.Limit) { - v.l.Lock() - defer v.l.Unlock() - v.limiter = rate.NewLimiter(l, int(l)) -} - -func (v *vaultClient) Stats() map[string]string { - stat := v.stats() - - expireTimeStr := "" - if !stat.TokenExpiry.IsZero() { - expireTimeStr = stat.TokenExpiry.Format(time.RFC3339) - } - - lastRenewTimeStr := "" - if !stat.LastRenewalTime.IsZero() { - lastRenewTimeStr = stat.LastRenewalTime.Format(time.RFC3339) - } - - nextRenewTimeStr := "" - if !stat.NextRenewalTime.IsZero() { - nextRenewTimeStr = stat.NextRenewalTime.Format(time.RFC3339) - } - - return map[string]string{ - "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), - "token_ttl": stat.TokenTTL.Round(time.Second).String(), - "token_expire_time": expireTimeStr, - "token_last_renewal_time": lastRenewTimeStr, - "token_next_renewal_time": nextRenewTimeStr, - } -} - -func (v *vaultClient) stats() *VaultStats { - // Allocate a new stats struct - stats := new(VaultStats) - - v.revLock.Lock() - stats.TrackedForRevoke = len(v.revoking) - v.revLock.Unlock() - - v.currentExpirationLock.Lock() - stats.TokenExpiry = v.currentExpiration - v.currentExpirationLock.Unlock() - - v.renewalTimeLock.Lock() - stats.NextRenewalTime = v.nextRenewalTime - stats.LastRenewalTime = v.lastRenewalTime - v.renewalTimeLock.Unlock() - - if !stats.TokenExpiry.IsZero() { - stats.TokenTTL = time.Until(stats.TokenExpiry) - } - - if !stats.LastRenewalTime.IsZero() { - stats.TimeFromLastRenewal = time.Since(stats.LastRenewalTime) - } - if !stats.NextRenewalTime.IsZero() { - stats.TimeToNextRenewal = time.Until(stats.NextRenewalTime) - } - - return stats -} - -// EmitStats is used to export metrics about the blocked eval tracker while enabled -func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { - timer, stop := helper.NewSafeTimer(period) - defer stop() - - for { - timer.Reset(period) - - select { - case <-timer.C: - stats := v.stats() - metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) - metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) - metrics.SetGauge([]string{"nomad", "vault", "token_last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond)) - metrics.SetGauge([]string{"nomad", "vault", "token_next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond)) - - case <-stopCh: - return - } - } -} - -// extendExpiration sets the current auth token expiration record to ttLSeconds seconds from now -func (v *vaultClient) extendExpiration(ttlSeconds int) { - v.currentExpirationLock.Lock() - v.currentExpiration = time.Now().Add(time.Duration(ttlSeconds) * time.Second) - v.currentExpirationLock.Unlock() -} - -// VaultVaultNoopDelegate returns the default vault api auth token handler -type VaultNoopDelegate struct{} - -func (e *VaultNoopDelegate) clientForTask(v *vaultClient, namespace string) (*vapi.Client, error) { - return v.client, nil -} diff --git a/nomad/vault_noop.go b/nomad/vault_noop.go deleted file mode 100644 index 57222fe16..000000000 --- a/nomad/vault_noop.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "context" - "errors" - "sync" - "time" - - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/nomad/structs/config" - vapi "github.com/hashicorp/vault/api" -) - -type NoopVault struct { - l sync.Mutex - - config *config.VaultConfig - logger log.Logger - purgeFn PurgeVaultAccessorFn -} - -func NewNoopVault(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVaultAccessorFn) *NoopVault { - return &NoopVault{ - config: c, - logger: logger.Named("vault-noop"), - purgeFn: purgeFn, - } -} - -func (v *NoopVault) SetActive(_ bool) {} - -func (v *NoopVault) SetConfig(c *config.VaultConfig) error { - v.l.Lock() - defer v.l.Unlock() - - v.config = c - return nil -} - -func (v *NoopVault) GetConfig() *config.VaultConfig { - v.l.Lock() - defer v.l.Unlock() - - return v.config.Copy() -} - -func (v *NoopVault) CreateToken(_ context.Context, _ *structs.Allocation, _ string) (*vapi.Secret, error) { - return nil, errors.New("Nomad server is not configured to create tokens") -} - -func (v *NoopVault) LookupToken(_ context.Context, _ string) (*vapi.Secret, error) { - return nil, errors.New("Nomad server is not configured to lookup tokens") -} - -func (v *NoopVault) RevokeTokens(_ context.Context, tokens []*structs.VaultAccessor, _ bool) error { - for _, t := range tokens { - v.logger.Debug("Vault token is no longer used, but Nomad is not able to revoke it. The token may need to be revoked manually or will expire once its TTL reaches zero.", "accessor", t.Accessor, "ttl", t.CreationTTL) - } - - if err := v.purgeFn(tokens); err != nil { - v.logger.Error("failed to purge Vault accessors", "error", err) - } - - return nil -} - -func (v *NoopVault) MarkForRevocation(tokens []*structs.VaultAccessor) error { - for _, t := range tokens { - v.logger.Debug("Vault token is no longer used, but Nomad is not able to mark it for revocation. The token may need to be revoked manually or will expire once its TTL reaches zero.", "accessor", t.Accessor, "ttl", t.CreationTTL) - } - - if err := v.purgeFn(tokens); err != nil { - v.logger.Error("failed to purge Vault accessors", "error", err) - } - - return nil -} - -func (v *NoopVault) Stop() {} - -func (v *NoopVault) Running() bool { return true } - -func (v *NoopVault) Stats() map[string]string { return nil } - -func (v *NoopVault) EmitStats(_ time.Duration, _ <-chan struct{}) {} diff --git a/nomad/vault_test.go b/nomad/vault_test.go deleted file mode 100644 index 539a782e6..000000000 --- a/nomad/vault_test.go +++ /dev/null @@ -1,1830 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "math/rand" - "reflect" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/hashicorp/nomad/ci" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/time/rate" - - "github.com/hashicorp/nomad/helper/pointer" - "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/helper/uuid" - "github.com/hashicorp/nomad/nomad/mock" - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/nomad/structs/config" - "github.com/hashicorp/nomad/testutil" - vapi "github.com/hashicorp/vault/api" -) - -const ( - // nomadRoleManagementPolicy is a policy that allows nomad to manage tokens - nomadRoleManagementPolicy = ` -path "auth/token/renew-self" { - capabilities = ["update"] -} - -path "auth/token/lookup" { - capabilities = ["update"] -} - -path "auth/token/roles/test" { - capabilities = ["read"] -} - -path "auth/token/revoke-accessor" { - capabilities = ["update"] -} -` - - // tokenLookupPolicy allows a token to be looked up - tokenLookupPolicy = ` -path "auth/token/lookup" { - capabilities = ["update"] -} -` - - // nomadRoleCreatePolicy gives the ability to create the role and derive tokens - // from the test role - nomadRoleCreatePolicy = ` -path "auth/token/create/test" { - capabilities = ["create", "update"] -} -` - - // secretPolicy gives access to the secret mount - secretPolicy = ` -path "secret/*" { - capabilities = ["create", "read", "update", "delete", "list"] -} -` -) - -// defaultTestVaultAllowlistRoleAndToken creates a test Vault role and returns a token -// created in that role -func defaultTestVaultAllowlistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "nomad-role-management": nomadRoleManagementPolicy, - } - d := make(map[string]interface{}, 2) - d["allowed_policies"] = "nomad-role-create,nomad-role-management" - d["period"] = rolePeriod - return testVaultRoleAndToken(v, t, vaultPolicies, d, - []string{"nomad-role-create", "nomad-role-management"}) -} - -// defaultTestVaultDenylistRoleAndToken creates a test Vault role using -// disallowed_policies and returns a token created in that role -func defaultTestVaultDenylistRoleAndToken(v *testutil.TestVault, t *testing.T, rolePeriod int) string { - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "nomad-role-management": nomadRoleManagementPolicy, - "secrets": secretPolicy, - } - - // Create the role - d := make(map[string]interface{}, 2) - d["disallowed_policies"] = "nomad-role-create" - d["period"] = rolePeriod - testVaultRoleAndToken(v, t, vaultPolicies, d, []string{"default"}) - - // Create a token that can use the role - a := v.Client.Auth().Token() - req := &vapi.TokenCreateRequest{ - Policies: []string{"nomad-role-create", "nomad-role-management"}, - } - s, err := a.Create(req) - if err != nil { - t.Fatalf("failed to create child token: %v", err) - } - - if s == nil || s.Auth == nil { - t.Fatalf("bad secret response: %+v", s) - } - - return s.Auth.ClientToken -} - -// testVaultRoleAndToken writes the vaultPolicies to vault and then creates a -// test role with the passed data. After that it derives a token from the role -// with the tokenPolicies -func testVaultRoleAndToken(v *testutil.TestVault, t *testing.T, vaultPolicies map[string]string, - data map[string]interface{}, tokenPolicies []string) string { - // Write the policies - sys := v.Client.Sys() - for p, data := range vaultPolicies { - if err := sys.PutPolicy(p, data); err != nil { - t.Fatalf("failed to create %q policy: %v", p, err) - } - } - - // Build a role - l := v.Client.Logical() - l.Write("auth/token/roles/test", data) - - // Create a new token with the role - a := v.Client.Auth().Token() - req := vapi.TokenCreateRequest{ - Policies: tokenPolicies, - } - s, err := a.CreateWithRole(&req, "test") - if err != nil { - t.Fatalf("failed to create child token: %v", err) - } - - // Get the client token - if s == nil || s.Auth == nil { - t.Fatalf("bad secret response: %+v", s) - } - - return s.Auth.ClientToken -} - -func TestVaultClient_BadConfig(t *testing.T) { - ci.Parallel(t) - conf := &config.VaultConfig{} - logger := testlog.HCLogger(t) - - // Should be no error since Vault is not enabled - _, err := NewVaultClient(nil, logger, nil, nil) - if err == nil || !strings.Contains(err.Error(), "valid") { - t.Fatalf("expected config error: %v", err) - } - - tr := true - conf.Enabled = &tr - _, err = NewVaultClient(conf, logger, nil, nil) - if err == nil || !strings.Contains(err.Error(), "token must be set") { - t.Fatalf("Expected token unset error: %v", err) - } - - conf.Token = "123" - _, err = NewVaultClient(conf, logger, nil, nil) - if err == nil || !strings.Contains(err.Error(), "address must be set") { - t.Fatalf("Expected address unset error: %v", err) - } -} - -// TestVaultClient_WithNamespaceSupport tests that the Vault namespace config, if present, will result in the -// namespace header being set on the created Vault client. -func TestVaultClient_WithNamespaceSupport(t *testing.T) { - ci.Parallel(t) - require := require.New(t) - tr := true - testNs := "test-namespace" - conf := &config.VaultConfig{ - Addr: "https://vault.service.consul:8200", - Enabled: &tr, - Token: "testvaulttoken", - Namespace: testNs, - } - logger := testlog.HCLogger(t) - - // Should be no error since Vault is not enabled - c, err := NewVaultClient(conf, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - - require.Equal(testNs, c.client.Headers().Get(structs.VaultNamespaceHeaderName)) - require.Equal("", c.clientSys.Headers().Get(structs.VaultNamespaceHeaderName)) - require.NotEqual(c.clientSys, c.client) -} - -// TestVaultClient_WithoutNamespaceSupport tests that the Vault namespace config, if present, will result in the -// namespace header being set on the created Vault client. -func TestVaultClient_WithoutNamespaceSupport(t *testing.T) { - ci.Parallel(t) - require := require.New(t) - tr := true - conf := &config.VaultConfig{ - Addr: "https://vault.service.consul:8200", - Enabled: &tr, - Token: "testvaulttoken", - Namespace: "", - } - logger := testlog.HCLogger(t) - - // Should be no error since Vault is not enabled - c, err := NewVaultClient(conf, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - - require.Equal("", c.client.Headers().Get(structs.VaultNamespaceHeaderName)) - require.Equal("", c.clientSys.Headers().Get(structs.VaultNamespaceHeaderName)) - require.Equal(c.clientSys, c.client) -} - -// started separately. -// Test that the Vault Client can establish a connection even if it is started -// before Vault is available. -func TestVaultClient_EstablishConnection(t *testing.T) { - ci.Parallel(t) - for i := 10; i >= 0; i-- { - v := testutil.NewTestVaultDelayed(t) - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - - // Sleep a little while and check that no connection has been established. - time.Sleep(100 * time.Duration(testutil.TestMultiplier()) * time.Millisecond) - if established, _ := client.ConnectionEstablished(); established { - t.Fatalf("ConnectionEstablished() returned true before Vault server started") - } - - // Start Vault - if err := v.Start(); err != nil { - v.Stop() - client.Stop() - - if i == 0 { - t.Fatalf("Failed to start vault: %v", err) - } - - wait := time.Duration(rand.Int31n(2000)) * time.Millisecond - time.Sleep(wait) - continue - } - - var waitErr error - testutil.WaitForResult(func() (bool, error) { - return client.ConnectionEstablished() - }, func(err error) { - waitErr = err - }) - - v.Stop() - client.Stop() - if waitErr != nil { - if i == 0 { - t.Fatalf("Failed to start vault: %v", err) - } - - wait := time.Duration(rand.Int31n(2000)) * time.Millisecond - time.Sleep(wait) - continue - } - - break - } -} - -func TestVaultClient_ValidateRole(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "nomad-role-management": nomadRoleManagementPolicy, - } - data := map[string]interface{}{ - "allowed_policies": "default,root", - "orphan": true, - "renewable": true, - "token_explicit_max_ttl": 10, - } - v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - client, err := NewVaultClient(v.Config, logger, nil, nil) - require.NoError(t, err) - - defer client.Stop() - - // Wait for an error - var conn bool - var connErr error - testutil.WaitForResult(func() (bool, error) { - conn, connErr = client.ConnectionEstablished() - if !conn { - return false, fmt.Errorf("Should connect") - } - - if connErr == nil { - return false, fmt.Errorf("expect an error") - } - - return true, nil - }, func(err error) { - require.NoError(t, err) - }) - - require.Contains(t, connErr.Error(), "explicit max ttl") - require.Contains(t, connErr.Error(), "non-zero period") -} - -// TestVaultClient_ValidateRole_Success asserts that a valid token role -// gets marked as valid -func TestVaultClient_ValidateRole_Success(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "nomad-role-management": nomadRoleManagementPolicy, - } - data := map[string]interface{}{ - "allowed_policies": "default,root", - "orphan": true, - "renewable": true, - "token_period": 1000, - } - v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - client, err := NewVaultClient(v.Config, logger, nil, nil) - require.NoError(t, err) - - defer client.Stop() - - // Wait for an error - var conn bool - var connErr error - testutil.WaitForResult(func() (bool, error) { - conn, connErr = client.ConnectionEstablished() - if !conn { - return false, fmt.Errorf("Should connect") - } - - if connErr != nil { - return false, connErr - } - - return true, nil - }, func(err error) { - require.NoError(t, err) - }) -} - -// TestVaultClient_ValidateRole_Deprecated_Success asserts that a valid token -// role gets marked as valid, even if it uses deprecated field, period -func TestVaultClient_ValidateRole_Deprecated_Success(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "nomad-role-management": nomadRoleManagementPolicy, - } - data := map[string]interface{}{ - "allowed_policies": "default,root", - "orphan": true, - "renewable": true, - "period": 1000, - } - v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, nil) - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - client, err := NewVaultClient(v.Config, logger, nil, nil) - require.NoError(t, err) - - defer client.Stop() - - // Wait for an error - var conn bool - var connErr error - testutil.WaitForResult(func() (bool, error) { - conn, connErr = client.ConnectionEstablished() - if !conn { - return false, fmt.Errorf("Should connect") - } - - if connErr != nil { - return false, connErr - } - - return true, nil - }, func(err error) { - require.NoError(t, err) - }) -} - -func TestVaultClient_ValidateRole_NonExistent(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - v.Config.Token = v.RootToken - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - v.Config.Role = "test-nonexistent" - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - // Wait for an error - var conn bool - var connErr error - testutil.WaitForResult(func() (bool, error) { - conn, connErr = client.ConnectionEstablished() - if !conn { - return false, fmt.Errorf("Should connect") - } - - if connErr == nil { - return false, fmt.Errorf("expect an error") - } - - return true, nil - }, func(err error) { - t.Fatalf("bad: %v", err) - }) - - errStr := connErr.Error() - if !strings.Contains(errStr, "does not exist") { - t.Fatalf("Expect does not exist error") - } -} - -func TestVaultClient_ValidateToken(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - vaultPolicies := map[string]string{ - "nomad-role-create": nomadRoleCreatePolicy, - "token-lookup": tokenLookupPolicy, - } - data := map[string]interface{}{ - "allowed_policies": "token-lookup,nomad-role-create", - "period": 10, - } - v.Config.Token = testVaultRoleAndToken(v, t, vaultPolicies, data, []string{"token-lookup", "nomad-role-create"}) - - logger := testlog.HCLogger(t) - v.Config.ConnectionRetryIntv = 100 * time.Millisecond - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - // Wait for an error - var conn bool - var connErr error - testutil.WaitForResult(func() (bool, error) { - conn, connErr = client.ConnectionEstablished() - if !conn { - return false, fmt.Errorf("Should connect") - } - - if connErr == nil { - return false, fmt.Errorf("expect an error") - } - - return true, nil - }, func(err error) { - t.Fatalf("bad: %v", err) - }) - - errStr := connErr.Error() - if !strings.Contains(errStr, vaultTokenRevokePath) { - t.Fatalf("Expect revoke error") - } - if !strings.Contains(errStr, fmt.Sprintf(vaultRoleLookupPath, "test")) { - t.Fatalf("Expect explicit max ttl error") - } - if !strings.Contains(errStr, "token must have one of the following") { - t.Fatalf("Expect explicit max ttl error") - } -} - -func TestVaultClient_SetActive(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - waitForConnection(client, t) - - // Do a lookup and expect an error about not being active - _, err = client.LookupToken(context.Background(), "123") - if err == nil || !strings.Contains(err.Error(), "not active") { - t.Fatalf("Expected not-active error: %v", err) - } - - client.SetActive(true) - - // Do a lookup of ourselves - _, err = client.LookupToken(context.Background(), v.RootToken) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } -} - -// Test that we can update the config and things keep working -func TestVaultClient_SetConfig(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - v2 := testutil.NewTestVault(t) - defer v2.Stop() - - // Set the configs token in a new test role - v2.Config.Token = defaultTestVaultAllowlistRoleAndToken(v2, t, 20) - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - waitForConnection(client, t) - - if client.tokenData == nil || len(client.tokenData.Policies) != 1 { - t.Fatalf("unexpected token: %v", client.tokenData) - } - - // Update the config - if err := client.SetConfig(v2.Config); err != nil { - t.Fatalf("SetConfig failed: %v", err) - } - - waitForConnection(client, t) - - if client.tokenData == nil || len(client.tokenData.Policies) != 3 { - t.Fatalf("unexpected token: %v", client.tokenData) - } - - // Test that when SetConfig is called with the same configuration, it is a - // no-op - failCh := make(chan struct{}, 1) - go func() { - tomb := client.tomb - select { - case <-tomb.Dying(): - close(failCh) - case <-time.After(1 * time.Second): - return - } - }() - - // Update the config - if err := client.SetConfig(v2.Config); err != nil { - t.Fatalf("SetConfig failed: %v", err) - } - - select { - case <-failCh: - t.Fatalf("Tomb shouldn't have exited") - case <-time.After(1 * time.Second): - return - } -} - -// TestVaultClient_SetConfig_Deadlock asserts that calling SetConfig -// concurrently with establishConnection does not deadlock. -func TestVaultClient_SetConfig_Deadlock(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - v2 := testutil.NewTestVault(t) - defer v2.Stop() - - // Set the configs token in a new test role - v2.Config.Token = defaultTestVaultAllowlistRoleAndToken(v2, t, 20) - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - for i := 0; i < 100; i++ { - // Alternate configs to cause updates - conf := v.Config - if i%2 == 0 { - conf = v2.Config - } - if err := client.SetConfig(conf); err != nil { - t.Fatalf("SetConfig failed: %v", err) - } - } -} - -// Test that we can disable vault -func TestVaultClient_SetConfig_Disable(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - waitForConnection(client, t) - - if client.tokenData == nil || len(client.tokenData.Policies) != 1 { - t.Fatalf("unexpected token: %v", client.tokenData) - } - - // Disable vault - f := false - config := config.VaultConfig{ - Enabled: &f, - } - - // Update the config - if err := client.SetConfig(&config); err != nil { - t.Fatalf("SetConfig failed: %v", err) - } - - if client.Enabled() || client.Running() { - t.Fatalf("SetConfig should have stopped client") - } -} - -func TestVaultClient_RenewalLoop(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - // Sleep 8 seconds and ensure we have a non-zero TTL - time.Sleep(8 * time.Second) - - // Get the current TTL - a := v.Client.Auth().Token() - s2, err := a.Lookup(v.Config.Token) - if err != nil { - t.Fatalf("failed to lookup token: %v", err) - } - - ttl := parseTTLFromLookup(s2, t) - if ttl == 0 { - t.Fatalf("token renewal failed; ttl %v", ttl) - } - - if client.currentExpiration.Before(time.Now()) { - t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) - } -} - -func TestVaultClientRenewUpdatesExpiration(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - // Get the current TTL - a := v.Client.Auth().Token() - s2, err := a.Lookup(v.Config.Token) - if err != nil { - t.Fatalf("failed to lookup token: %v", err) - } - exp0 := time.Now().Add(time.Duration(parseTTLFromLookup(s2, t)) * time.Second) - - time.Sleep(1 * time.Second) - - _, err = client.renew() - require.NoError(t, err) - exp1 := client.currentExpiration - require.True(t, exp0.Before(exp1)) - - time.Sleep(1 * time.Second) - - _, err = client.renew() - require.NoError(t, err) - exp2 := client.currentExpiration - require.True(t, exp1.Before(exp2)) -} - -func TestVaultClient_StopsAfterPermissionError(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 2) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - time.Sleep(500 * time.Millisecond) - - assert.True(t, client.isRenewLoopActive()) - - // Get the current TTL - a := v.Client.Auth().Token() - assert.NoError(t, a.RevokeSelf("")) - - testutil.WaitForResult(func() (bool, error) { - if !client.isRenewLoopActive() { - return true, nil - } else { - return false, errors.New("renew loop should terminate after token is revoked") - } - }, func(err error) { - t.Fatalf("err: %v", err) - }) -} -func TestVaultClient_LoopsUntilCannotRenew(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - defer client.Stop() - - // Sleep 8 seconds and ensure we have a non-zero TTL - time.Sleep(8 * time.Second) - - // Get the current TTL - a := v.Client.Auth().Token() - s2, err := a.Lookup(v.Config.Token) - if err != nil { - t.Fatalf("failed to lookup token: %v", err) - } - - ttl := parseTTLFromLookup(s2, t) - if ttl == 0 { - t.Fatalf("token renewal failed; ttl %v", ttl) - } - - if client.currentExpiration.Before(time.Now()) { - t.Fatalf("found current expiration to be in past %s", time.Until(client.currentExpiration)) - } -} - -func parseTTLFromLookup(s *vapi.Secret, t *testing.T) int64 { - if s == nil { - t.Fatalf("nil secret") - } else if s.Data == nil { - t.Fatalf("nil data block in secret") - } - - ttlRaw, ok := s.Data["ttl"] - if !ok { - t.Fatalf("no ttl") - } - - ttlNumber, ok := ttlRaw.(json.Number) - if !ok { - t.Fatalf("failed to convert ttl %q to json Number", ttlRaw) - } - - ttl, err := ttlNumber.Int64() - if err != nil { - t.Fatalf("Failed to get ttl from json.Number: %v", err) - } - - return ttl -} - -func TestVaultClient_LookupToken_Invalid(t *testing.T) { - ci.Parallel(t) - tr := true - conf := &config.VaultConfig{ - Enabled: &tr, - Addr: "http://foobar:12345", - Token: uuid.Generate(), - } - - // Enable vault but use a bad address so it never establishes a conn - logger := testlog.HCLogger(t) - client, err := NewVaultClient(conf, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - _, err = client.LookupToken(context.Background(), "foo") - if err == nil || !strings.Contains(err.Error(), "established") { - t.Fatalf("Expected error because connection to Vault hasn't been made: %v", err) - } -} - -func TestVaultClient_LookupToken_Root(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Lookup ourselves - s, err := client.LookupToken(context.Background(), v.Config.Token) - if err != nil { - t.Fatalf("self lookup failed: %v", err) - } - - policies, err := s.TokenPolicies() - if err != nil { - t.Fatalf("failed to parse policies: %v", err) - } - - expected := []string{"root"} - if !reflect.DeepEqual(policies, expected) { - t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) - } - - // Create a token with a different set of policies - expected = []string{"default"} - req := vapi.TokenCreateRequest{ - Policies: expected, - } - s, err = v.Client.Auth().Token().Create(&req) - if err != nil { - t.Fatalf("failed to create child token: %v", err) - } - - // Get the client token - if s == nil || s.Auth == nil { - t.Fatalf("bad secret response: %+v", s) - } - - // Lookup new child - s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) - if err != nil { - t.Fatalf("self lookup failed: %v", err) - } - - policies, err = s.TokenPolicies() - if err != nil { - t.Fatalf("failed to parse policies: %v", err) - } - - if !reflect.DeepEqual(policies, expected) { - t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) - } -} - -func TestVaultClient_LookupToken_Role(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Lookup ourselves - s, err := client.LookupToken(context.Background(), v.Config.Token) - if err != nil { - t.Fatalf("self lookup failed: %v", err) - } - - policies, err := s.TokenPolicies() - if err != nil { - t.Fatalf("failed to parse policies: %v", err) - } - - expected := []string{"default", "nomad-role-create", "nomad-role-management"} - if !reflect.DeepEqual(policies, expected) { - t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) - } - - // Create a token with a different set of policies - expected = []string{"default"} - req := vapi.TokenCreateRequest{ - Policies: expected, - } - s, err = v.Client.Auth().Token().Create(&req) - if err != nil { - t.Fatalf("failed to create child token: %v", err) - } - - // Get the client token - if s == nil || s.Auth == nil { - t.Fatalf("bad secret response: %+v", s) - } - - // Lookup new child - s, err = client.LookupToken(context.Background(), s.Auth.ClientToken) - if err != nil { - t.Fatalf("self lookup failed: %v", err) - } - - policies, err = s.TokenPolicies() - if err != nil { - t.Fatalf("failed to parse policies: %v", err) - } - - if !reflect.DeepEqual(policies, expected) { - t.Fatalf("Unexpected policies; got %v; want %v", policies, expected) - } -} - -func TestVaultClient_LookupToken_RateLimit(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - client.setLimit(rate.Limit(1.0)) - testRateLimit(t, 20, client, func(ctx context.Context) error { - // Lookup ourselves - _, err := client.LookupToken(ctx, v.Config.Token) - return err - }) -} - -func TestVaultClient_CreateToken_Root(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"default"}} - - s, err := client.CreateToken(context.Background(), a, task.Name) - if err != nil { - t.Fatalf("CreateToken failed: %v", err) - } - - // Ensure that created secret is a wrapped token - if s == nil || s.WrapInfo == nil { - t.Fatalf("Bad secret: %#v", s) - } - - d, err := time.ParseDuration(vaultTokenCreateTTL) - if err != nil { - t.Fatalf("bad: %v", err) - } - - if s.WrapInfo.WrappedAccessor == "" { - t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.Token == "" { - t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.TTL != int(d.Seconds()) { - t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) - } -} - -func TestVaultClient_CreateToken_Allowlist_Role(t *testing.T) { - ci.Parallel(t) - - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"default"}} - - s, err := client.CreateToken(context.Background(), a, task.Name) - if err != nil { - t.Fatalf("CreateToken failed: %v", err) - } - - // Ensure that created secret is a wrapped token - if s == nil || s.WrapInfo == nil { - t.Fatalf("Bad secret: %#v", s) - } - - d, err := time.ParseDuration(vaultTokenCreateTTL) - if err != nil { - t.Fatalf("bad: %v", err) - } - - if s.WrapInfo.WrappedAccessor == "" { - t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.Token == "" { - t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.TTL != int(d.Seconds()) { - t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) - } -} - -func TestVaultClient_CreateToken_Root_Target_Role(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Create the test role - defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Target the test role - v.Config.Role = "test" - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"default"}} - - s, err := client.CreateToken(context.Background(), a, task.Name) - if err != nil { - t.Fatalf("CreateToken failed: %v", err) - } - - // Ensure that created secret is a wrapped token - if s == nil || s.WrapInfo == nil { - t.Fatalf("Bad secret: %#v", s) - } - - d, err := time.ParseDuration(vaultTokenCreateTTL) - if err != nil { - t.Fatalf("bad: %v", err) - } - - if s.WrapInfo.WrappedAccessor == "" { - t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.Token == "" { - t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.TTL != int(d.Seconds()) { - t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) - } -} - -func TestVaultClient_CreateToken_Denylist_Role(t *testing.T) { - ci.Parallel(t) - - v := testutil.NewTestVault(t) - defer v.Stop() - - // Need to skip if test is 0.6.4 - version, err := testutil.VaultVersion() - if err != nil { - t.Fatalf("failed to determine version: %v", err) - } - - if strings.Contains(version, "v0.6.4") { - t.Skipf("Vault has a regression in v0.6.4 that this test hits") - } - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultDenylistRoleAndToken(v, t, 5) - v.Config.Role = "test" - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"secrets"}} - - s, err := client.CreateToken(context.Background(), a, task.Name) - if err != nil { - t.Fatalf("CreateToken failed: %v", err) - } - - // Ensure that created secret is a wrapped token - if s == nil || s.WrapInfo == nil { - t.Fatalf("Bad secret: %#v", s) - } - - d, err := time.ParseDuration(vaultTokenCreateTTL) - if err != nil { - t.Fatalf("bad: %v", err) - } - - if s.WrapInfo.WrappedAccessor == "" { - t.Fatalf("Bad accessor: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.Token == "" { - t.Fatalf("Bad token: %v", s.WrapInfo.WrappedAccessor) - } else if s.WrapInfo.TTL != int(d.Seconds()) { - t.Fatalf("Bad ttl: %v", s.WrapInfo.WrappedAccessor) - } -} - -func TestVaultClient_CreateToken_Role_InvalidToken(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - defaultTestVaultAllowlistRoleAndToken(v, t, 5) - v.Config.Token = "foo-bar" - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - testutil.WaitForResult(func() (bool, error) { - established, err := client.ConnectionEstablished() - if !established { - return false, fmt.Errorf("Should establish") - } - return err != nil, nil - }, func(err error) { - t.Fatalf("Connection not established") - }) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"default"}} - - _, err = client.CreateToken(context.Background(), a, task.Name) - if err == nil || !strings.Contains(err.Error(), "failed to establish connection to Vault") { - t.Fatalf("CreateToken should have failed: %v", err) - } -} - -func TestVaultClient_CreateToken_Role_Unrecoverable(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Start the client - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"unknown_policy"}} - - _, err = client.CreateToken(context.Background(), a, task.Name) - if err == nil { - t.Fatalf("CreateToken should have failed: %v", err) - } - - _, ok := err.(structs.Recoverable) - if ok { - t.Fatalf("CreateToken should not be a recoverable error type: %v (%T)", err, err) - } -} - -func TestVaultClient_CreateToken_Prestart(t *testing.T) { - ci.Parallel(t) - vconfig := &config.VaultConfig{ - Enabled: pointer.Of(true), - Token: uuid.Generate(), - Addr: "http://127.0.0.1:0", - } - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(vconfig, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - // Create an allocation that requires a Vault policy - a := mock.Alloc() - task := a.Job.TaskGroups[0].Tasks[0] - task.Vault = &structs.Vault{Policies: []string{"default"}} - - _, err = client.CreateToken(context.Background(), a, task.Name) - if err == nil { - t.Fatalf("CreateToken should have failed: %v", err) - } - - if rerr, ok := err.(*structs.RecoverableError); !ok { - t.Fatalf("Err should have been type recoverable error") - } else if ok && !rerr.IsRecoverable() { - t.Fatalf("Err should have been recoverable") - } -} - -func TestVaultClient_MarkForRevocation(t *testing.T) { - vconfig := &config.VaultConfig{ - Enabled: pointer.Of(true), - Token: uuid.Generate(), - Addr: "http://127.0.0.1:0", - } - logger := testlog.HCLogger(t) - client, err := NewVaultClient(vconfig, logger, nil, nil) - require.NoError(t, err) - - client.SetActive(true) - defer client.Stop() - - // Create some VaultAccessors - vas := []*structs.VaultAccessor{ - mock.VaultAccessor(), - mock.VaultAccessor(), - } - - err = client.MarkForRevocation(vas) - require.NoError(t, err) - - // Wasn't committed - require.Len(t, client.revoking, 2) - require.Equal(t, 2, client.stats().TrackedForRevoke) - -} -func TestVaultClient_RevokeTokens_PreEstablishs(t *testing.T) { - ci.Parallel(t) - vconfig := &config.VaultConfig{ - Enabled: pointer.Of(true), - Token: uuid.Generate(), - Addr: "http://127.0.0.1:0", - } - logger := testlog.HCLogger(t) - client, err := NewVaultClient(vconfig, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - // Create some VaultAccessors - vas := []*structs.VaultAccessor{ - mock.VaultAccessor(), - mock.VaultAccessor(), - } - - if err := client.RevokeTokens(context.Background(), vas, false); err != nil { - t.Fatalf("RevokeTokens failed: %v", err) - } - - // Wasn't committed - if len(client.revoking) != 0 { - t.Fatalf("didn't add to revoke loop") - } - - if err := client.RevokeTokens(context.Background(), vas, true); err != nil { - t.Fatalf("RevokeTokens failed: %v", err) - } - - // Was committed - if len(client.revoking) != 2 { - t.Fatalf("didn't add to revoke loop") - } - - if client.stats().TrackedForRevoke != 2 { - t.Fatalf("didn't add to revoke loop") - } -} - -// TestVaultClient_RevokeTokens_Failures_TTL asserts that -// the registered TTL doesn't get extended on retries -func TestVaultClient_RevokeTokens_Failures_TTL(t *testing.T) { - ci.Parallel(t) - vconfig := &config.VaultConfig{ - Enabled: pointer.Of(true), - Token: uuid.Generate(), - Addr: "http://127.0.0.1:0", - } - logger := testlog.HCLogger(t) - client, err := NewVaultClient(vconfig, logger, nil, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - // Create some VaultAccessors - vas := []*structs.VaultAccessor{ - mock.VaultAccessor(), - mock.VaultAccessor(), - } - - err = client.RevokeTokens(context.Background(), vas, true) - require.NoError(t, err) - - // Was committed - require.Len(t, client.revoking, 2) - - // set TTL - ttl := time.Now().Add(50 * time.Second) - client.revoking[vas[0]] = ttl - client.revoking[vas[1]] = ttl - - // revoke again and ensure that TTL isn't extended - err = client.RevokeTokens(context.Background(), vas, true) - require.NoError(t, err) - - require.Len(t, client.revoking, 2) - expected := map[*structs.VaultAccessor]time.Time{ - vas[0]: ttl, - vas[1]: ttl, - } - require.Equal(t, expected, client.revoking) -} - -func TestVaultClient_RevokeTokens_Root(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - purged := 0 - purge := func(accessors []*structs.VaultAccessor) error { - purged += len(accessors) - return nil - } - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, purge, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create some vault tokens - auth := v.Client.Auth().Token() - req := vapi.TokenCreateRequest{ - Policies: []string{"default"}, - } - t1, err := auth.Create(&req) - if err != nil { - t.Fatalf("Failed to create vault token: %v", err) - } - if t1 == nil || t1.Auth == nil { - t.Fatalf("bad secret response: %+v", t1) - } - t2, err := auth.Create(&req) - if err != nil { - t.Fatalf("Failed to create vault token: %v", err) - } - if t2 == nil || t2.Auth == nil { - t.Fatalf("bad secret response: %+v", t2) - } - - // Create two VaultAccessors - vas := []*structs.VaultAccessor{ - {Accessor: t1.Auth.Accessor}, - {Accessor: t2.Auth.Accessor}, - } - - // Issue a token revocation - if err := client.RevokeTokens(context.Background(), vas, true); err != nil { - t.Fatalf("RevokeTokens failed: %v", err) - } - - // Lookup the token and make sure we get an error - if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { - t.Fatalf("Revoked token lookup didn't fail: %+v", s) - } - if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { - t.Fatalf("Revoked token lookup didn't fail: %+v", s) - } - - if purged != 2 { - t.Fatalf("Expected purged 2; got %d", purged) - } -} - -func TestVaultClient_RevokeTokens_Role(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - purged := 0 - purge := func(accessors []*structs.VaultAccessor) error { - purged += len(accessors) - return nil - } - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, purge, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create some vault tokens - auth := v.Client.Auth().Token() - req := vapi.TokenCreateRequest{ - Policies: []string{"default"}, - } - t1, err := auth.Create(&req) - if err != nil { - t.Fatalf("Failed to create vault token: %v", err) - } - if t1 == nil || t1.Auth == nil { - t.Fatalf("bad secret response: %+v", t1) - } - t2, err := auth.Create(&req) - if err != nil { - t.Fatalf("Failed to create vault token: %v", err) - } - if t2 == nil || t2.Auth == nil { - t.Fatalf("bad secret response: %+v", t2) - } - - // Create two VaultAccessors - vas := []*structs.VaultAccessor{ - {Accessor: t1.Auth.Accessor}, - {Accessor: t2.Auth.Accessor}, - } - - // Issue a token revocation - if err := client.RevokeTokens(context.Background(), vas, true); err != nil { - t.Fatalf("RevokeTokens failed: %v", err) - } - - // Lookup the token and make sure we get an error - if purged != 2 { - t.Fatalf("Expected purged 2; got %d", purged) - } - if s, err := auth.Lookup(t1.Auth.ClientToken); err == nil { - t.Fatalf("Revoked token lookup didn't fail: %+v", s) - } - if s, err := auth.Lookup(t2.Auth.ClientToken); err == nil { - t.Fatalf("Revoked token lookup didn't fail: %+v", s) - } -} - -// TestVaultClient_RevokeTokens_Idempotent asserts that token revocation -// is idempotent, and can cope with cases if token was deleted out of band. -func TestVaultClient_RevokeTokens_Idempotent(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - purged := map[string]struct{}{} - purge := func(accessors []*structs.VaultAccessor) error { - for _, accessor := range accessors { - purged[accessor.Accessor] = struct{}{} - } - return nil - } - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(v.Config, logger, purge, nil) - if err != nil { - t.Fatalf("failed to build vault client: %v", err) - } - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create some vault tokens - auth := v.Client.Auth().Token() - req := vapi.TokenCreateRequest{ - Policies: []string{"default"}, - } - t1, err := auth.Create(&req) - require.NoError(t, err) - require.NotNil(t, t1) - require.NotNil(t, t1.Auth) - - t2, err := auth.Create(&req) - require.NoError(t, err) - require.NotNil(t, t2) - require.NotNil(t, t2.Auth) - - t3, err := auth.Create(&req) - require.NoError(t, err) - require.NotNil(t, t3) - require.NotNil(t, t3.Auth) - - // revoke t3 out of band - err = auth.RevokeAccessor(t3.Auth.Accessor) - require.NoError(t, err) - - // Create two VaultAccessors - vas := []*structs.VaultAccessor{ - {Accessor: t1.Auth.Accessor}, - {Accessor: t2.Auth.Accessor}, - {Accessor: t3.Auth.Accessor}, - } - - // Issue a token revocation - err = client.RevokeTokens(context.Background(), vas, true) - require.NoError(t, err) - require.Empty(t, client.revoking) - - // revoke token again - err = client.RevokeTokens(context.Background(), vas, true) - require.NoError(t, err) - require.Empty(t, client.revoking) - - // Lookup the token and make sure we get an error - require.Len(t, purged, 3) - require.Contains(t, purged, t1.Auth.Accessor) - require.Contains(t, purged, t2.Auth.Accessor) - require.Contains(t, purged, t3.Auth.Accessor) - s, err := auth.Lookup(t1.Auth.ClientToken) - require.Errorf(t, err, "failed to purge token: %v", s) - s, err = auth.Lookup(t2.Auth.ClientToken) - require.Errorf(t, err, "failed to purge token: %v", s) -} - -// TestVaultClient_RevokeDaemon_Bounded asserts that token revocation -// batches are bounded in size. -func TestVaultClient_RevokeDaemon_Bounded(t *testing.T) { - ci.Parallel(t) - v := testutil.NewTestVault(t) - defer v.Stop() - - // Set the configs token in a new test role - v.Config.Token = defaultTestVaultAllowlistRoleAndToken(v, t, 5) - - // Disable client until we can change settings for testing - conf := v.Config.Copy() - conf.Enabled = pointer.Of(false) - - const ( - batchSize = 100 - batches = 3 - ) - resultCh := make(chan error, batches) - var totalPurges int64 - - // Purge function asserts batches are always < batchSize - purge := func(vas []*structs.VaultAccessor) error { - if len(vas) > batchSize { - resultCh <- fmt.Errorf("too many Vault accessors in batch: %d > %d", len(vas), batchSize) - } else { - resultCh <- nil - } - atomic.AddInt64(&totalPurges, int64(len(vas))) - - return nil - } - - logger := testlog.HCLogger(t) - client, err := NewVaultClient(conf, logger, purge, nil) - require.NoError(t, err) - - // Override settings for testing and then enable client - client.maxRevokeBatchSize = batchSize - client.revocationIntv = 3 * time.Millisecond - conf = v.Config.Copy() - conf.Enabled = pointer.Of(true) - require.NoError(t, client.SetConfig(conf)) - - client.SetActive(true) - defer client.Stop() - - waitForConnection(client, t) - - // Create more tokens in Nomad than can fit in a batch; they don't need - // to exist in Vault. - accessors := make([]*structs.VaultAccessor, batchSize*batches) - for i := 0; i < len(accessors); i++ { - accessors[i] = &structs.VaultAccessor{Accessor: "abcd"} - } - - // Mark for revocation - require.NoError(t, client.MarkForRevocation(accessors)) - - // Wait for tokens to be revoked - for i := 0; i < batches; i++ { - select { - case err := <-resultCh: - require.NoError(t, err) - case <-time.After(10 * time.Second): - // 10 seconds should be plenty long to process 3 - // batches at a 3ms tick interval! - t.Errorf("timed out processing %d batches. %d/%d complete in 10s", - batches, i, batches) - } - } - - require.Equal(t, int64(len(accessors)), atomic.LoadInt64(&totalPurges)) -} - -func waitForConnection(v *vaultClient, t *testing.T) { - testutil.WaitForResult(func() (bool, error) { - return v.ConnectionEstablished() - }, func(err error) { - t.Fatalf("Connection not established") - }) -} - -func TestVaultClient_nextBackoff(t *testing.T) { - ci.Parallel(t) - - simpleCases := []struct { - name string - initBackoff float64 - - // define range of acceptable backoff values accounting for random factor - rangeMin float64 - rangeMax float64 - }{ - {"simple case", 7.0, 8.7, 17.60}, - {"too low", 2.0, 5.0, 10.0}, - {"too large", 100, 30.0, 60.0}, - } - - for _, c := range simpleCases { - t.Run(c.name, func(t *testing.T) { - b := nextBackoff(c.initBackoff, time.Now().Add(10*time.Hour)) - if !(c.rangeMin <= b && b <= c.rangeMax) { - t.Fatalf("Expected backoff within [%v, %v] but found %v", c.rangeMin, c.rangeMax, b) - } - }) - } - - // some edge cases - t.Run("close to expiry", func(t *testing.T) { - b := nextBackoff(20, time.Now().Add(1100*time.Millisecond)) - if b != 5.0 { - t.Fatalf("Expected backoff is 5 but found %v", b) - } - }) - - t.Run("past expiry", func(t *testing.T) { - b := nextBackoff(20, time.Now().Add(-1100*time.Millisecond)) - if !(60 <= b && b <= 120) { - t.Fatalf("Expected backoff within [%v, %v] but found %v", 60, 120, b) - } - }) -} - -func testRateLimit(t *testing.T, count int, client *vaultClient, fn func(context.Context) error) { - // Spin up many requests. These should block - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - cancels := 0 - unblock := make(chan struct{}) - for i := 0; i < count; i++ { - go func() { - err := fn(ctx) - if err != nil { - if err == context.Canceled { - cancels += 1 - return - } - t.Errorf("request failed: %v", err) - return - } - - // Cancel the context - close(unblock) - }() - } - - select { - case <-time.After(5 * time.Second): - t.Fatalf("timeout") - case <-unblock: - cancel() - } - - desired := count - 1 - testutil.WaitForResult(func() (bool, error) { - if desired-cancels > 2 { - return false, fmt.Errorf("Incorrect number of cancels; got %d; want %d", cancels, desired) - } - - return true, nil - }, func(err error) { - t.Fatal(err) - }) -} diff --git a/nomad/vault_testing.go b/nomad/vault_testing.go deleted file mode 100644 index 5675cf1b8..000000000 --- a/nomad/vault_testing.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomad - -import ( - "context" - "time" - - "github.com/hashicorp/nomad/nomad/structs" - "github.com/hashicorp/nomad/nomad/structs/config" - vapi "github.com/hashicorp/vault/api" -) - -// TestVaultClient is a Vault client appropriate for use during testing. Its -// behavior is programmable such that endpoints can be tested under various -// circumstances. -type TestVaultClient struct { - // LookupTokenErrors maps a token to an error that will be returned by the - // LookupToken call - LookupTokenErrors map[string]error - - // LookupTokenSecret maps a token to the Vault secret that will be returned - // by the LookupToken call - LookupTokenSecret map[string]*vapi.Secret - - // CreateTokenErrors maps a token to an error that will be returned by the - // CreateToken call - CreateTokenErrors map[string]map[string]error - - // CreateTokenSecret maps a token to the Vault secret that will be returned - // by the CreateToken call - CreateTokenSecret map[string]map[string]*vapi.Secret - - RevokedTokens []*structs.VaultAccessor -} - -func (v *TestVaultClient) LookupToken(ctx context.Context, token string) (*vapi.Secret, error) { - var secret *vapi.Secret - var err error - - if v.LookupTokenSecret != nil { - secret = v.LookupTokenSecret[token] - } - if v.LookupTokenErrors != nil { - err = v.LookupTokenErrors[token] - } - - return secret, err -} - -// SetLookupTokenError sets the error that will be returned by the token -// lookup -func (v *TestVaultClient) SetLookupTokenError(token string, err error) { - if v.LookupTokenErrors == nil { - v.LookupTokenErrors = make(map[string]error) - } - - v.LookupTokenErrors[token] = err -} - -// SetLookupTokenSecret sets the secret that will be returned by the token -// lookup -func (v *TestVaultClient) SetLookupTokenSecret(token string, secret *vapi.Secret) { - if v.LookupTokenSecret == nil { - v.LookupTokenSecret = make(map[string]*vapi.Secret) - } - - v.LookupTokenSecret[token] = secret -} - -// SetLookupTokenAllowedPolicies is a helper that adds a secret that allows the -// given policies -func (v *TestVaultClient) SetLookupTokenAllowedPolicies(token string, policies []string) { - s := &vapi.Secret{ - Data: map[string]interface{}{ - "policies": policies, - }, - } - - v.SetLookupTokenSecret(token, s) -} - -func (v *TestVaultClient) CreateToken(ctx context.Context, a *structs.Allocation, task string) (*vapi.Secret, error) { - var secret *vapi.Secret - var err error - - if v.CreateTokenSecret != nil { - tasks := v.CreateTokenSecret[a.ID] - if tasks != nil { - secret = tasks[task] - } - } - if v.CreateTokenErrors != nil { - tasks := v.CreateTokenErrors[a.ID] - if tasks != nil { - err = tasks[task] - } - } - - return secret, err -} - -// SetCreateTokenError sets the error that will be returned by the token -// creation -func (v *TestVaultClient) SetCreateTokenError(allocID, task string, err error) { - if v.CreateTokenErrors == nil { - v.CreateTokenErrors = make(map[string]map[string]error) - } - - tasks := v.CreateTokenErrors[allocID] - if tasks == nil { - tasks = make(map[string]error) - v.CreateTokenErrors[allocID] = tasks - } - - v.CreateTokenErrors[allocID][task] = err -} - -// SetCreateTokenSecret sets the secret that will be returned by the token -// creation -func (v *TestVaultClient) SetCreateTokenSecret(allocID, task string, secret *vapi.Secret) { - if v.CreateTokenSecret == nil { - v.CreateTokenSecret = make(map[string]map[string]*vapi.Secret) - } - - tasks := v.CreateTokenSecret[allocID] - if tasks == nil { - tasks = make(map[string]*vapi.Secret) - v.CreateTokenSecret[allocID] = tasks - } - - v.CreateTokenSecret[allocID][task] = secret -} - -func (v *TestVaultClient) RevokeTokens(ctx context.Context, accessors []*structs.VaultAccessor, committed bool) error { - v.RevokedTokens = append(v.RevokedTokens, accessors...) - return nil -} - -func (v *TestVaultClient) MarkForRevocation(accessors []*structs.VaultAccessor) error { - v.RevokedTokens = append(v.RevokedTokens, accessors...) - return nil -} - -func (v *TestVaultClient) Stop() {} -func (v *TestVaultClient) SetActive(enabled bool) {} -func (v *TestVaultClient) GetConfig() *config.VaultConfig { return nil } -func (v *TestVaultClient) SetConfig(config *config.VaultConfig) error { return nil } -func (v *TestVaultClient) Running() bool { return true } -func (v *TestVaultClient) Stats() map[string]string { return map[string]string{} } -func (v *TestVaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) {} diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 387e0a97e..2d2193ce9 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -313,10 +313,6 @@ func TestTasksUpdated(t *testing.T) { j14.TaskGroups[0].Networks[0].ReservedPorts = []structs.Port{{Label: "foo", Value: 1312}} must.True(t, tasksUpdated(j1, j14, name).modified) - j15 := mock.Job() - j15.TaskGroups[0].Tasks[0].Vault = &structs.Vault{Policies: []string{"foo"}} - must.True(t, tasksUpdated(j1, j15, name).modified) - j16 := mock.Job() j16.TaskGroups[0].EphemeralDisk.Sticky = true must.True(t, tasksUpdated(j1, j16, name).modified) diff --git a/testutil/vault.go b/testutil/vault.go index 6576467bb..70c2bb823 100644 --- a/testutil/vault.go +++ b/testutil/vault.go @@ -97,7 +97,6 @@ func NewTestVaultFromPath(t testing.T, binary string) *TestVault { Config: &config.VaultConfig{ Name: structs.VaultDefaultCluster, Enabled: &enable, - Token: token, Addr: http, }, } @@ -179,7 +178,6 @@ func NewTestVaultDelayedFromPath(t testing.T, binary string) *TestVault { Client: client, Config: &config.VaultConfig{ Enabled: &enable, - Token: token, Addr: http, }, } diff --git a/website/content/api-docs/jobs.mdx b/website/content/api-docs/jobs.mdx index 1b948de60..c0b5062a2 100644 --- a/website/content/api-docs/jobs.mdx +++ b/website/content/api-docs/jobs.mdx @@ -300,7 +300,6 @@ $ curl \ "TaskGroups": null, "Type": "service", "Update": null, - "VaultToken": "", "Version": 0 } ``` @@ -536,7 +535,6 @@ $ curl \ "foo": "bar", "baz": "pipe" }, - "VaultToken": "", "Status": "running", "StatusDescription": "", "CreateIndex": 7, @@ -821,7 +819,6 @@ $ curl \ "Stagger": 30000000000 }, "VaultNamespace": "", - "VaultToken": "", "Version": 0 } ] @@ -1040,7 +1037,6 @@ $ curl \ "Stagger": 30000000000 }, "VaultNamespace": "", - "VaultToken": "", "Version": 1 }, { @@ -1208,7 +1204,6 @@ $ curl \ "Stagger": 30000000000 }, "VaultNamespace": "", - "VaultToken": "", "Version": 0 } ] @@ -1883,7 +1878,7 @@ The table below shows this endpoint's support for - `JobID` `(string: )` - Specifies the ID of the job. This is specified as part of the path. -- `JobVersion` `(integer: 0)` - Specifies the job version to revert to. Use either +- `JobVersion` `(integer: 0)` - Specifies the job version to revert to. Use either this parameter or `TaggedVersion`, but do not use both. - `TaggedVersion` `(string: "")` - Specifies the tag name of the job version you @@ -1896,9 +1891,6 @@ The table below shows this endpoint's support for - `ConsulToken` `(string:"")` - Optional value specifying the [consul token](/nomad/docs/commands/job/revert) used for Consul [service identity polity authentication checking](/nomad/docs/configuration/consul#allow_unauthenticated). -- `VaultToken` `(string: "")` - Optional value specifying the [vault token](/nomad/docs/commands/job/revert) - used for Vault [policy authentication checking](/nomad/docs/configuration/vault#allow_unauthenticated). - - `namespace` `(string: "default")` - Specifies the target namespace. If ACL is enabled, this value must match a namespace that the token is allowed to access. This is specified as a query string parameter. diff --git a/website/content/docs/commands/job/plan.mdx b/website/content/docs/commands/job/plan.mdx index 8573ebc6d..de7e50c0e 100644 --- a/website/content/docs/commands/job/plan.mdx +++ b/website/content/docs/commands/job/plan.mdx @@ -48,10 +48,6 @@ Plan will return one of the following exit codes: - 1: Allocations created or destroyed. - 255: Error determining plan results. -The plan command will set the `vault_token` of the job based on the following -precedence, going from highest to lowest: the `-vault-token` flag, the -`$VAULT_TOKEN` environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the `submit-job` capability for the job's namespace. @@ -75,17 +71,6 @@ capability for the job's namespace. a variable has been supplied which is not defined within the root variables. Defaults to true. -- `-vault-token`: Used to validate if the user submitting the job has - permission to run the job according to its Vault policies. A Vault token must - be supplied if the [`vault` block `allow_unauthenticated`] is disabled in - the Nomad server configuration. If the `-vault-token` flag is set, the passed - Vault token is added to the jobspec before sending to the Nomad servers. This - allows passing the Vault token without storing it in the job file. This - overrides the token found in the `$VAULT_TOKEN` environment variable and the - [`vault_token`] field in the job file. This token is cleared from the job - after planning and cannot be used within the job executing environment. Use - the `vault` block when templating in a job with a Vault token. - - `-vault-namespace`: If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -256,5 +241,3 @@ if a change is detected. [`go-getter`]: https://github.com/hashicorp/go-getter [`nomad job run -check-index`]: /nomad/docs/commands/job/run#check-index [`tee`]: https://man7.org/linux/man-pages/man1/tee.1.html -[`vault` block `allow_unauthenticated`]: /nomad/docs/configuration/vault#allow_unauthenticated -[`vault_token`]: /nomad/docs/job-specification/job#vault_token diff --git a/website/content/docs/commands/job/revert.mdx b/website/content/docs/commands/job/revert.mdx index 0c95d693d..7c4f7942e 100644 --- a/website/content/docs/commands/job/revert.mdx +++ b/website/content/docs/commands/job/revert.mdx @@ -18,13 +18,6 @@ persisted, it must be provided to revert if the targeted job version includes Consul Connect enabled services and the Nomad servers were configured to require [consul service identity] authentication. -The revert command will use a Vault token with the following preference: -first the `-vault-token` flag, then the `$VAULT_TOKEN` environment variable. -Because the vault token used to [run] the targeted job version was not -persisted, it must be provided to revert if the targeted job version includes -Vault policies and the Nomad servers were configured to require [vault policy] -authentication. - ## Usage ```shell-session @@ -57,10 +50,6 @@ not used. request to the Nomad servers. This overrides the token found in the `$CONSUL_HTTP_TOKEN` environment variable. -- `-vault-token`: If set, the passed Vault token is sent along with the revert - request to the Nomad servers. This overrides the token found in the - `$VAULT_TOKEN` environment variable. - - `-verbose`: Show full information. ## Examples @@ -126,5 +115,4 @@ Submit Date = 07/25/17 21:27:18 UTC [`job history`]: /nomad/docs/commands/job/history [eval status]: /nomad/docs/commands/eval/status [consul service identity]: /nomad/docs/configuration/consul#allow_unauthenticated -[vault policy]: /nomad/docs/configuration/vault#allow_unauthenticated [run]: /nomad/docs/commands/job/run diff --git a/website/content/docs/commands/job/run.mdx b/website/content/docs/commands/job/run.mdx index b8dad76e8..9f2b01766 100644 --- a/website/content/docs/commands/job/run.mdx +++ b/website/content/docs/commands/job/run.mdx @@ -44,10 +44,6 @@ The run command will set the `consul_token` of the job based on the following precedence, going from highest to lowest: the `-consul-token` flag, the `$CONSUL_HTTP_TOKEN` environment variable and finally the value in the job file. -The run command will set the `vault_token` of the job based on the following -precedence, going from highest to lowest: the `-vault-token` flag, the -`$VAULT_TOKEN` environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the `submit-job` capability for the job's namespace. Jobs that mount CSI volumes require a token with the `csi-mount-volume` capability for the volume's namespace. Jobs @@ -103,17 +99,6 @@ that volume. a Consul token must be supplied with appropriate service and KV Consul ACL policy permissions. -- `-vault-token`: Used to validate if the user submitting the job has - permission to run the job according to its Vault policies. A Vault token must - be supplied if the [`vault` block `allow_unauthenticated`] is disabled in - the Nomad server configuration. If the `-vault-token` flag is set, the passed - Vault token is added to the jobspec before sending to the Nomad servers. This - allows passing the Vault token without storing it in the job file. This - overrides the token found in the `$VAULT_TOKEN` environment variable and the - [`vault_token`] field in the job file. This token is cleared from the job - after validating and cannot be used within the job executing environment. Use - the `vault` block when templating in a job with a Vault token. - - `-vault-namespace`: If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -242,5 +227,3 @@ $ nomad job run example.nomad.hcl [job specification]: /nomad/docs/job-specification [JSON jobs]: /nomad/api-docs/json-jobs [`system`]: /nomad/docs/schedulers#system -[`vault` block `allow_unauthenticated`]: /nomad/docs/configuration/vault#allow_unauthenticated -[`vault_token`]: /nomad/docs/job-specification/job#vault_token diff --git a/website/content/docs/commands/job/validate.mdx b/website/content/docs/commands/job/validate.mdx index b388afddf..c2b5001c3 100644 --- a/website/content/docs/commands/job/validate.mdx +++ b/website/content/docs/commands/job/validate.mdx @@ -29,10 +29,6 @@ supports `go-getter` syntax. On successful validation, exit code 0 will be returned, otherwise an exit code of 1 indicates an error. -The run command will set the `vault_token` of the job based on the following -precedence, going from highest to lowest: the `-vault-token` flag, the -`$VAULT_TOKEN` environment variable and finally the value in the job file. - When ACLs are enabled, this command requires a token with the `read-job` capability for the job's namespace. @@ -50,17 +46,6 @@ capability for the job's namespace. a variable has been supplied which is not defined within the root variables. Defaults to true. -- `-vault-token`: Used to validate if the user submitting the job has - permission to run the job according to its Vault policies. A Vault token must - be supplied if the [`vault` block `allow_unauthenticated`] is disabled in - the Nomad server configuration. If the `-vault-token` flag is set, the passed - Vault token is added to the jobspec before sending to the Nomad servers. This - allows passing the Vault token without storing it in the job file. This - overrides the token found in the `$VAULT_TOKEN` environment variable and the - [`vault_token`] field in the job file. This token is cleared from the job - after validating and cannot be used within the job executing environment. Use - the `vault` block when templating in a job with a Vault token. - - `-vault-namespace`: If set, the passed Vault namespace is stored in the job before sending to the Nomad servers. @@ -95,5 +80,3 @@ Job validation successful [`go-getter`]: https://github.com/hashicorp/go-getter [job specification]: /nomad/docs/job-specification -[`vault` block `allow_unauthenticated`]: /nomad/docs/configuration/vault#allow_unauthenticated -[`vault_token`]: /nomad/docs/job-specification/job#vault_token diff --git a/website/content/docs/configuration/vault.mdx b/website/content/docs/configuration/vault.mdx index 4a768ea48..b3b7a71ba 100644 --- a/website/content/docs/configuration/vault.mdx +++ b/website/content/docs/configuration/vault.mdx @@ -22,12 +22,6 @@ identity claims. When configured, job tasks can use [workload identities][workload_id] to receive Vault ACL tokens automatically. -Alternatively, Nomad servers may be configured with a high-privileged Vault -token that is used to derive fine-grained tokens for tasks. Refer to -[Token-Based Authentication](#token-based-authentication) for configuration -details. However, we strongly recommend using workload identities since -token-based authentication is deprecated. - Refer to the [Nomad and Vault Integration][nomad-vault] page for more information about the Vault integration. @@ -57,11 +51,6 @@ agents running as clients, servers, or in all agents. Parameters are safely ignored if placed in a configuration file where they are not expected to be defined. -The placement also depends on the authentication strategy used. This section -describes the parameter organization for workload identity authentication. -Refer to the [Token-based Authentication](#token-based-authentication) section -if using Vault tokens directly. - ### Parameters for Nomad Clients and Servers These parameters should be defined in the configuration file of all Nomad @@ -75,24 +64,6 @@ agents. - `enabled` `(bool: false)` - Specifies if the Vault integration should be activated. -- `create_from_role` `(string: "")` - Specifies the role to create tokens from. - - When using workload identities this field defines the role used to derive - task tokens when the job does not define a value for - [`vault.role`][jobspec_vault_role]. If empty, the default Vault cluster role - is used. - - For token-based authentication, the Nomad servers derive tokens - using this role. The token given to Nomad does not have to be created from - this role but must have `update` capability on - `auth/token/create/` path in Vault. If this value is unset - and the token is created from a role, the value is defaulted to the role the - token is from. - - -> **Note:** This used to be a server-only field, but it is client-only when - using workload identities. It should be set in both while transitioning from - token-based authentication to workload identities. - ### Parameters for Nomad Clients These parameters should only be defined in the configuration file of Nomad @@ -125,6 +96,11 @@ agents with [`client.enabled`] set to `true`. [tls_require_and_verify_client_cert](/vault/docs/configuration/listener/tcp#tls_require_and_verify_client_cert) is enabled in Vault. + - `create_from_role` `(string: "")` - Specifies the role to create tokens + from. This field defines the role used to derive task tokens when the job does + not define a value for [`vault.role`][jobspec_vault_role]. If empty, the + default Vault cluster role is used. + - `key_file` `(string: "")` - Specifies the path to the private key used for Vault communication. If this is set then you need to also set `cert_file`. This must be set if @@ -153,30 +129,6 @@ agents with [`server.enabled`] set to `true`. [`name`](#name) parameter. Setting a default identity causes the value of `allow_unauthenticated` to be ignored. -### Deprecated Parameters - -These parameters are used by the deprecated token-based authentication flow and -will be removed in a future release. - -- `allow_unauthenticated` `(bool: true)` - Specifies if users submitting jobs to - the Nomad server should be required to provide their own Vault token, proving - they have access to the policies listed in the job. This option should be - disabled in an untrusted environment. - -- `task_token_ttl` `(string: "72h")` - Specifies the TTL of created tokens when - using a root token. This is specified using a label suffix like "30s" or "1h". - -- `token` `(string: "")` - Specifies the parent Vault token to use to derive - child tokens for jobs requesting tokens. Only required on Nomad servers. - Nomad client agents use the allocation's token when contacting Vault. - Refer to the [Vault Integration Guide](/nomad/docs/integrations/vault-integration) - for how to generate an appropriate token in Vault. - - !> It is **strongly discouraged** to place the token as a configuration - parameter like this, since the token could be checked into source control - accidentally. Users should set the `VAULT_TOKEN` environment variable when - starting the agent instead. - ### `default_identity` Parameters - `aud` `(array: [])` - List of valid recipients for this workload @@ -219,35 +171,6 @@ will be removed in a future release. - `${vault.namespace}` - The Vault namespace. - `${vault.role}` - The Vault role. -### Token-Based Authentication - -~> **Warning:** The token-based authentication flow is deprecated and will be - removed in a future release. It is highly recommended to migrate and use the - workload identity flow instead. - -When using token-based authentication the `vault` block, define parameters as in -the following examples. - -#### Parameters for Nomad Clients and Servers - -- [`address`](#address) -- [`ca_file`](#ca_file) -- [`ca_path`](#ca_path) -- [`cert_file`](#cert_file) -- [`enabled`](#enabled) -- [`key_file`](#key_file) -- [`name`](#name) -- [`namespace`](#namespace) -- [`tls_server_name`](#tls_server_name) -- [`tls_skip_verify`](#tls_skip_verify) - -#### Parameters for Nomad Servers - -- [`allow_unauthenticated`](#allow_unauthenticated) -- [`create_from_role`](#create_from_role) -- [`token`](#token) -- [`task_token_ttl`](#task_token_ttl) - ## `vault` Examples The following examples only show the `vault` blocks. Remember that the @@ -267,10 +190,6 @@ server { vault { enabled = true - # Only needed in servers when transitioning from the token-based flow to - # workload identities. - create_from_role = "nomad-cluster" - # Provide a default workload identity configuration so jobs don't need to # specify one. default_identity { @@ -282,37 +201,6 @@ vault { } ``` -This example shows a Vault configuration for a Nomad server using the -deprecated token-based authentication flow. - -```hcl -server { - enabled = true - # ... -} - -vault { - enabled = true - ca_path = "/etc/certs/ca" - cert_file = "/var/certs/vault.crt" - key_file = "/var/certs/vault.key" - - # Address to communicate with Vault. The following is the default address if - # unspecified. - address = "https://vault.service.consul:8200" - - # Embedding the token in the configuration is discouraged. Instead users - # should set the VAULT_TOKEN environment variable when starting the Nomad - # agent - token = "debecfdc-9ed7-ea22-c6ee-948f22cdd474" - - # Setting the create_from_role option causes Nomad to create tokens for tasks - # via the provided role. This allows the role to manage what policies are - # allowed and disallowed for use by tasks. - create_from_role = "nomad-cluster" -} -``` - ### Nomad Client This example shows a Vault configuration for a Nomad client. diff --git a/website/content/docs/integrations/vault/acl.mdx b/website/content/docs/integrations/vault/acl.mdx index 685c9b70a..5dce26cb9 100644 --- a/website/content/docs/integrations/vault/acl.mdx +++ b/website/content/docs/integrations/vault/acl.mdx @@ -11,10 +11,8 @@ properly configured in order for the Vault and Nomad integrations to work. ## Nomad Workload Identities -Starting in Nomad 1.7, Nomad clients can use a task's [Workload Identity][] to -authenticate to Vault and obtain a Vault ACL token specific to the task. When -using Nomad workload identities, you no longer need to pass in a Vault ACL -token to submit a job. +Starting in Nomad 1.10.0, Nomad clients use a task's [Workload Identity][] to +authenticate to Vault and obtain a Vault ACL token specific to the task. By default, Nomad only generates a workload identity for tasks that can be used to access Nomad itself, such as for reading [Variables][] from a [`template`][] @@ -332,373 +330,6 @@ The [`nomad setup vault`][nomad_cli_setup_vault] command and the module can help you automate the process of applying configuration to a Vault cluster. -## Authentication Without Workload Identity (Legacy) - -To use the legacy Vault integration, Nomad servers must be provided a Vault -token. This token can either be a root token or a periodic token with -permissions to create from a token role. The root token is the easiest way to -get started, but we recommend a token role based token for production -installations. Nomad servers will renew the token automatically. **Note that the -Nomad clients do not need to be provided with a Vault token.** - - - -This legacy workflow will be removed in Nomad 1.10. Before upgrading to Nomad -1.10, you need to configure authentication with Vault as described in [Nomad -Workload Identities](#nomad-workload-identities). - - - -See the [Enterprise specific section][ent] for configuring Vault Enterprise. - -### Root Token Integration - -If Nomad is given a [root token](/vault/docs/concepts/tokens#root-tokens), no -further configuration is needed as Nomad can derive a token for jobs using any -Vault policies. Best practices recommend using a periodic token with the minimal -permissions necessary instead of providing Nomad the root vault token. - -### Token Role based Integration - -Vault's [Token Authentication Backend][auth] supports a concept called "roles". -Token roles allow policies to be grouped together and token creation to be -delegated to a trusted service such as Nomad. By creating a token role, the set -of policies that tasks managed by Nomad can access may be limited compared to -giving Nomad a root token. Token roles allow both allowlist and denylist -management of policies accessible to the role. - -To configure Nomad and Vault to create tokens against a role, the following must -occur: - -1. Create a "nomad-server" policy used by Nomad to create and manage tokens. - -2. Create a Vault token role with the configuration described below. - -3. Configure Nomad to use the created token role. - -4. Give Nomad servers a periodic token with the "nomad-server" policy created - above. - -#### Required Vault Policies - -The token Nomad receives must have the capabilities listed below. An explanation -for the use of each capability is given. - -```hcl -# Allow creating tokens under "nomad-cluster" token role. The token role name -# should be updated if "nomad-cluster" is not used. -path "auth/token/create/nomad-cluster" { - capabilities = ["update"] -} - -# Allow looking up "nomad-cluster" token role. The token role name should be -# updated if "nomad-cluster" is not used. -path "auth/token/roles/nomad-cluster" { - capabilities = ["read"] -} - -# Allow looking up the token passed to Nomad to validate # the token has the -# proper capabilities. This is provided by the "default" policy. -path "auth/token/lookup-self" { - capabilities = ["read"] -} - -# Allow looking up incoming tokens to validate they have permissions to access -# the tokens they are requesting. This is only required if -# `allow_unauthenticated` is set to false. -path "auth/token/lookup" { - capabilities = ["update"] -} - -# Allow revoking tokens that should no longer exist. This allows revoking -# tokens for dead tasks. -path "auth/token/revoke-accessor" { - capabilities = ["update"] -} - -# Allow checking the capabilities of our own token. This is used to validate the -# token upon startup. Note this requires update permissions because the Vault API -# is a POST -path "sys/capabilities-self" { - capabilities = ["update"] -} - -# Allow our own token to be renewed. -path "auth/token/renew-self" { - capabilities = ["update"] -} -``` - -The above [`nomad-server` policy](https://nomadproject.io/data/vault/nomad-server-policy.hcl) is -available for download. Below is an example of writing this policy to Vault: - -```shell-session -# Download the policy -$ curl https://nomadproject.io/data/vault/nomad-server-policy.hcl -O -s -L - -# Write the policy to Vault -$ vault policy write nomad-server nomad-server-policy.hcl -``` - -#### Vault Token Role Configuration - -A Vault token role must be created for use by Nomad. The token role can be used -to manage what Vault policies are accessible by jobs submitted to Nomad. The -policies can be managed as a allowlist by using `allowed_policies` in the token -role definition or as a denylist by using `disallowed_policies`. - -If using `allowed_policies`, tasks may only request Vault policies that are in -the list. If `disallowed_policies` is used, task may request any policy that is -not in the `disallowed_policies` list. There are trade-offs to both approaches -but generally it is easier to use the denylist approach and add policies that -you would not like tasks to have access to into the `disallowed_policies` list. - -An example token role definition is given below: - -```json -{ - "disallowed_policies": "nomad-server", - "token_explicit_max_ttl": 0, - "name": "nomad-cluster", - "orphan": true, - "token_period": 259200, - "renewable": true -} -``` - -##### Token Role Requirements - -Nomad checks that token role has an appropriate configuration for use by the -cluster. Fields that are checked are documented below as well as descriptions of -the important fields. See Vault's [Token Authentication Backend][auth] -documentation for all possible fields and more complete documentation. - -- `allowed_policies` - Specifies the list of allowed policies as a - comma-separated string. This list should contain all policies that jobs running - under Nomad should have access to. - -- `disallowed_policies` - Specifies the list of disallowed policies as a - comma-separated string. This list should contain all policies that jobs running - under Nomad should **not** have access to. The policy created above that - grants Nomad the ability to generate tokens from the token role should be - included in list of disallowed policies. This prevents tokens created by - Nomad from generating new tokens with different policies than those granted - by Nomad. - - A regression occurred in Vault 0.6.4 when validating token creation using a - token role with `disallowed_policies` such that it is not usable with - Nomad. This was remedied in 0.6.5 and does not effect earlier versions - of Vault. - -- `token_explicit_max_ttl` - Specifies the max TTL of a token. **Must be set to `0`** to - allow periodic tokens. - -- `name` - Specifies the name of the policy. We recommend using the name - `nomad-cluster`. If a different name is chosen, replace the token role in the - above policy. - -- `orphan` - Specifies whether tokens created against this token role will be - orphaned and have no parents. Nomad does not enforce the value of this field - but understanding the implications of each value is important. - - If set to false, all tokens will be revoked when the Vault token given to - Nomad expires. This makes it easy to revoke all tokens generated by Nomad but - forces all Nomad servers to use the same Vault token, even through upgrades of - Nomad servers. If the Vault token that was given to Nomad and used to generate - a tasks token expires, the token used by the task will also be revoked which - is not ideal. - - When set to true, the tokens generated for tasks will not be revoked when - Nomad's token is revoked. However Nomad will still revoke tokens when the - allocation is no longer running, minimizing the lifetime of any task's token. - With orphaned enabled, each Nomad server may also use a unique Vault token, - making bootstrapping and upgrading simpler. As such, **setting `orphan = true` - is the recommended setting**. - -- `token_period` - Specifies the length the TTL is extended by each renewal in - seconds. It is suggested to set this value on the order of magnitude of 3 days - (259200 seconds) to avoid a large renewal request rate to Vault. **Must be set - to a positive value**. - -- `renewable` - Specifies whether created tokens are renewable. This allows - Nomad to renew tokens for tasks. Nomad clients will automatically detect when - tokens cannot be renewed and will not attempt to renew them (see - [`vault.allow_token_expiration`][]). - -The above [`nomad-cluster` token role](https://nomadproject.io/data/vault/nomad-cluster-role.json) is -available for download. Below is an example of writing this role to Vault: - -```shell-session -# Download the token role -$ curl https://nomadproject.io/data/vault/nomad-cluster-role.json -O -s -L - -# Create the token role with Vault -$ vault write /auth/token/roles/nomad-cluster @nomad-cluster-role.json -``` - -#### Example Configuration - -To make getting started easy, the basic [`nomad-server` -policy](https://nomadproject.io/data/vault/nomad-server-policy.hcl) and -[`nomad-cluster` role](https://nomadproject.io/data/vault/nomad-cluster-role.json) described above are -available for download. - -The below example assumes Vault is accessible, unsealed and the operator has -appropriate permissions. - -```shell-session -# Download the policy and token role -$ curl https://nomadproject.io/data/vault/nomad-server-policy.hcl -O -s -L -$ curl https://nomadproject.io/data/vault/nomad-cluster-role.json -O -s -L - -# Write the policy to Vault -$ vault policy write nomad-server nomad-server-policy.hcl - -# Create the token role with Vault -$ vault write /auth/token/roles/nomad-cluster @nomad-cluster-role.json -``` - -#### Retrieving the Token Role based Token - -After the token role is created, a token suitable for the Nomad servers may be -retrieved by issuing the following Vault command: - -```shell-session -$ vault token create -policy nomad-server -period 72h -orphan -Key Value ---- ----- -token f02f01c2-c0d1-7cb7-6b88-8a14fada58c0 -token_accessor 8cb7fcb3-9a4f-6fbf-0efc-83092bb0cb1c -token_duration 259200s -token_renewable true -token_policies [default nomad-server] -``` - -The `-orphan` flag is included when generating the Nomad server token above to -prevent revocation of the token when its parent expires. Vault typically -creates tokens with a parent-child relationship. When an ancestor token is -revoked, all of its descendant tokens and their associated leases are revoked -as well. - -When generating Nomad's Vault token, we need to ensure that revocation of the -parent token does not revoke Nomad's token. To prevent this behavior we -specify the `-orphan` flag when we create the Nomad's Vault token. All -other tokens generated by Nomad for jobs will be generated using the policy -default of `orphan = false`. - -More information about creating orphan tokens can be found in -[Vault's Token Hierarchies and Orphan Tokens documentation][tokenhierarchy]. - -The [`-period` flag](/vault/docs/commands/token/create#period) is required to allow the automatic renewal of the token. If this is left out, a [`vault token renew` command](/vault/docs/commands/token/renew) will need to be run manually to renew the token. - -The token can then be set in the server configuration's -[`vault` block][config], as a command-line flag, or via an environment -variable. - -```shell-session -$ VAULT_TOKEN=f02f01c2-c0d1-7cb7-6b88-8a14fada58c0 nomad agent -config /path/to/config -``` - -An example of what may be contained in the configuration is shown below. For -complete documentation please see the [Nomad agent Vault integration][config] -configuration. - -```hcl -vault { - enabled = true - ca_path = "/etc/certs/ca" - cert_file = "/var/certs/vault.crt" - key_file = "/var/certs/vault.key" - address = "https://vault.service.consul:8200" - create_from_role = "nomad-cluster" -} -``` - -### Troubleshooting Legacy Authentication - -#### Invalid Vault token - -Upon startup, Nomad will attempt to connect to the specified Vault server. Nomad -will lookup the passed token and if the token is from a token role, the token -role will be validated. Nomad will not shutdown if given an invalid Vault token, -but will log the reasons the token is invalid and disable Vault integration. - -#### No Secret Exists - -Vault has two APIs for secrets, [`v1` and `v2`][vault-secrets-version]. Each version -has different paths, and Nomad does not abstract this for you. As such you will -need to specify the path as reflected by Vault's HTTP API, rather than the path -used in the `vault kv` command. - -You can see examples of `v1` and `v2` syntax in the -[template documentation][vault-kv-templates]. - -## Enterprise Configuration - - - -Nomad Enterprise allows jobs to use multiple [Vault Namespaces][]. There are a -few configuration settings to consider when using this functionality. - -### Example Configuration - -Below is an example for creating two Namespaces within Vault. - -```shell-session -# Create a namespace "engineering" within Vault -$ vault namespace create engineering - -# Create a child namespace "frontend" under "engineering" -$ vault namespace create -namespace=engineering frontend -``` - -### Required Vault Policies - -Policies are configured per Vault namespace. We will apply the policy in the -example above to each namespace—engineering and engineering/frontend. - -```shell-session -# Create the "nomad-server" policy in the "engineering" namespace -$ vault policy write -namespace=engineering nomad-server nomad-server-policy.hcl - -# Create the "nomad-server" policy in the "engineering/frontend" namespace -$ vault policy write -namespace=engineering/frontend nomad-server nomad-server-policy.hcl -``` - -We will also configure the previously configured `nomad-workloads` role with each -Namespace - -```shell-session -# Create the "nomad-cluster" token role in the "engineering" namespace -$ vault write -namespace=engineering /auth/token/roles/nomad-workloads @nomad-workloads-role.json - -# Create the "nomad-cluster" token role in the "engineering/frontend" namespace -$ vault write -namespace=engineering/frontend /auth/token/roles/nomad-workloads @nomad-workloads-role.json -``` - -The [Nomad agent Vault integration][config] configuration supports specifying a -Vault Namespace, but since we will be using multiple it can be left blank. By -default Nomad will interact with Vault's root Namespace, but individual jobs may -specify other Vault Namespaces to use. - -```hcl -vault { - enabled = true - ca_path = "/etc/certs/ca" - cert_file = "/var/certs/vault.crt" - key_file = "/var/certs/vault.key" - address = "https://vault.service.consul:8200" - - default_identity { - aud = ["vault.io"] - } -} -``` - -For legacy authentication, the same steps can be taken to inject a Vault token -from the [Retrieving the Token Role based -Token](#retrieving-the-token-role-based-token) steps. - ### Submitting a job with a Vault Namespace The example job file below specifies to use the `engineering` Namespace in @@ -735,92 +366,6 @@ EOF } ``` -### Submitting a job with a Vault Namespace with Legacy Authentication - -For the legacy authentication, because [`allow_unauthenticated`][allow_unauth] -is set to `false` job submitters will need to provide a sufficiently privileged -token when submitting a job. A token that has access to an appropriate policy in -`engineering` namespace is needed: - -```shell-session -$ vault token create -policy access-kv -namespace=engineering -period 72h -orphan - -Key Value ---- ----- -token s.H39hfS7eHSbb1GpkdzOQLTmz.fvuLy -token_accessor VsKtJwaShwtTo1r9nWV9Rlad.fvuLy -token_duration 72h -token_renewable true -token_policies ["access-kv" "default"] -identity_policies [] -policies ["access-kv" "default"] -``` - -The token can then be submitted with our job - -```shell-session -$ VAULT_TOKEN=s.H39hfS7eHSbb1GpkdzOQLTmz.fvuLy nomad job run vault.nomad -``` - -## Migrating to Using Workload Identity with Vault - -Migrating from the legacy (pre-1.7) workflow where workloads use the agent's -Vault token requires configuration on your Vault cluster and your Nomad server -agents. - -Once the migration is fully complete, Nomad server will no longer have access -to Vault, as it was required in the deprecated legacy workflow. This also means -that they will no longer be able to fulfill some of their responsibilities from -the legacy workflow, such as generating and revoking Vault ACL tokens. - -Before removing Vault connectivity configuration from Nomad servers, you must -make sure the rest of the cluster is ready to support workload identities for -Vault. You can run the [`nomad setup vault -check`][nomad_cli_setup_vault] -command to verify what changes are still necessary. - -Before removing Nomad servers access to Vault you must: - - * Redeploy the jobs listed in the section `Jobs Without Workload Identity for - Vault` with an identity for Vault. You can specify this identity [directly - in the job][jobspec_identity_vault] or redeploy the job without changes to - use the default value from the server [`vault.default_identity`][] - configuration if set. - * Upgrade nodes listed in the section `Outdated Nodes` to a version of Nomad - above 1.7.0. - -There is not action required for the Vault ACL tokens listed under `Vault -Tokens`. Nomad will revoke them as you redeploy jobs to use workload identities -but there may be some leftovers. You can still proceed with the migration -process, but Nomad will not revoke them once access to Vault is removed from -Nomad servers. They will expire once their TTL reaches zero, or you may -manually revoke them if they are no longer needed by an allocation. - -The migration process can happen over time. As long as all servers are upgraded -to Nomad 1.7+ and still retain access to Vault, jobs can still use either the -new workload identity flow or the deprecated legacy flow. - -To summarize the migration process: - -* Enable [`vault.default_identity`][] blocks in your Nomad server agent - configurations, but **do not modify any of the existing Vault - configuration**. -* Upgrade your cluster following the documented [Upgrade - Process][docs_upgrade]. -* Create the Vault auth method, default role, and policies on your Vault - cluster. -* Run the `nomad setup vault -check` command to verify if the cluster is ready - to migrate to workload identity access to Vault. - * Resubmit Nomad jobs that need access to Vault to redeploy them with a new - workload identity for Vault. - * (Optionally) Add [`vault.role`][] fields to any Nomad jobs that will not - use the default role. - * (Optionally) add [`identity`][] blocks to your jobs if you want to use a - different identity because of how your auth method and roles are - configured. - * Upgrade any remaining clients to Nomad 1.7+. -* Remove parameters no longer used by the Nomad server agents from the - [`vault`][config] configuration block. - [Variables]: /nomad/docs/concepts/variables [Vault Namespaces]: /vault/docs/enterprise/namespaces [Workload Identity]: /nomad/docs/concepts/workload-identity @@ -853,10 +398,8 @@ To summarize the migration process: [nomad_jwks_url]: /nomad/api-docs/operator/keyring#list-active-public-keys [nomad_wid_claims]: /nomad/docs/concepts/workload-identity#workload-identity-claims [tf_nomad_setup_vault]: https://registry.terraform.io/modules/hashicorp-modules/nomad-setup/vault -[tokenhierarchy]: /vault/docs/concepts/tokens#token-hierarchies-and-orphan-tokens 'Vault Tokens - Token Hierarchies and Orphan Tokens' [tutorial_mtls]: /nomad/tutorials/transport-security/security-enable-tls [vault-kv-templates]: /nomad/docs/job-specification/template#vault-kv-api-v1 'Vault KV API v1' -[vault-secrets-version]: /vault/docs/secrets/kv 'KV Secrets Engine' [vault_auth]: /vault/docs/concepts/auth [vault_bound_claims]: /vault/docs/auth/jwt#bound-claims [vault_jwt_auth_method]: /vault/api-docs/auth/jwt @@ -865,4 +408,3 @@ To summarize the migration process: [vault_templated_policies]: /vault/tutorials/policies/policy-templating [vault_token_types]: /vault/tutorials/tokens/tokens#token-types [vault_tutorial_wid]: /nomad/tutorials/integrate-vault/vault-acl -[`vault.allow_token_expiration`]: /nomad/docs/job-specification/vault#allow_token_expiration diff --git a/website/content/docs/job-specification/vault.mdx b/website/content/docs/job-specification/vault.mdx index 2f95ff0a2..c9a818872 100644 --- a/website/content/docs/job-specification/vault.mdx +++ b/website/content/docs/job-specification/vault.mdx @@ -109,12 +109,6 @@ with Vault as well. to use for the task. The Nomad client will retrieve a Vault token that is scoped to this particular namespace. -- `policies` `(array: [])` - Specifies the set of Vault policies that - the task requires. The Nomad client will retrieve a Vault token that is - limited to those policies. This field may only be used with the legacy Vault - authentication workflow and not with JWT and workload identity. It is - deprecated in favor of the `role` field and will be removed in Nomad 1.10. - - `role` `(string: "")` - Specifies the Vault role used when retrieving a token from Vault using JWT and workload identity. If not specified the client's [`create_from_role`][] value is used. diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 3c4c7de9d..f27975c9e 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -52,6 +52,29 @@ incremented. If you were relying on the previous behavior to redistribute workloads, you can force a destructive update by changing fields that require one, such as the `meta` block. +#### Vault integration changes + +Nomad 1.10.0 removes the previously deprecated token-based Vault authentication +workflow. Nomad clients must now use a task's [Workload Identity][] to +authenticate to Vault and obtain a Vault token specific to the task. + +This table lists removed fields and the new workflow. + +| Field | Configuration | New Workflow | +| ------ | ------------ | ------------ | +| [`vault.allow_unauthenticated`][] | Agent | Tasks should use a workload identity. Do not use a Vault token. | +| [`vault.task_token_ttl`][] | Agent | With workload identity, tasks receive their TTL configuration from the Vault role. | +| [`vault.token`][] | Agent | Nomad agents use the workload identity when making requests to authenticated endpoints. | +| [`vault.policies`][] | Job specification | Configure and use a Vault role. | + +Before upgrading to Nomad 1.10, perform the following tasks: + +1. Configure Vault to work with workload identity. +1. Migrate all workloads to use workload identity. + +Refer to [Migrating to Using Workload Identity with Vault][] for more +details. + ## Nomad 1.9.5 #### CNI plugins