secrets: Add secrets block to job spec (#26076)

api/tasks.go

@@ -786,6 +786,7 @@ type Task struct {
 	KillSignal      string                 `mapstructure:"kill_signal" hcl:"kill_signal,optional"`
 	Kind            string                 `hcl:"kind,optional"`
 	ScalingPolicies []*ScalingPolicy       `hcl:"scaling,block"`
+	Secrets         []*Secret              `hcl:"secret,block"`
 
 	// Identity is the default Nomad Workload Identity and will be added to
 	// Identities with the name "default"
@@ -825,6 +826,9 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) {
 	for _, tmpl := range t.Templates {
 		tmpl.Canonicalize()
 	}
+	for _, s := range t.Secrets {
+		s.Canonicalize()
+	}
 	for _, s := range t.Services {
 		s.Canonicalize(t, tg, job)
 	}
@@ -1042,6 +1046,19 @@ func (v *Vault) Canonicalize() {
 	}
 }
 
+type Secret struct {
+	Name     string         `hcl:"name,label"`
+	Provider string         `hcl:"provider,optional"`
+	Path     string         `hcl:"path,optional"`
+	Config   map[string]any `hcl:"config,block"`
+}
+
+func (s *Secret) Canonicalize() {
+	if len(s.Config) == 0 {
+		s.Config = nil
+	}
+}
+
 // NewTask creates and initializes a new Task.
 func NewTask(name, driver string) *Task {
 	return &Task{

api/tasks_test.go

@@ -506,6 +506,27 @@ func TestTask_Canonicalize_Vault(t *testing.T) {
 	}
 }
 
+func TestTask_Canonicalize_Secret(t *testing.T) {
+	testutil.Parallel(t)
+
+	testSecret := &Secret{
+		Name:     "test-secret",
+		Provider: "test-provider",
+		Path:     "/test/path",
+		Config:   make(map[string]any),
+	}
+
+	expected := &Secret{
+		Name:     "test-secret",
+		Provider: "test-provider",
+		Path:     "/test/path",
+		Config:   nil,
+	}
+
+	testSecret.Canonicalize()
+	must.Eq(t, expected, testSecret)
+}
+
 // Ensures no regression on https://github.com/hashicorp/nomad/issues/3132
 func TestTaskGroup_Canonicalize_Update(t *testing.T) {
 	testutil.Parallel(t)

command/agent/job_endpoint.go

@@ -1466,6 +1466,18 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
 		}
 	}
 
+	if len(apiTask.Secrets) > 0 {
+		structsTask.Secrets = []*structs.Secret{}
+		for _, s := range apiTask.Secrets {
+			structsTask.Secrets = append(structsTask.Secrets, &structs.Secret{
+				Name:     s.Name,
+				Provider: s.Provider,
+				Path:     s.Path,
+				Config:   s.Config,
+			})
+		}
+	}
+
 	if apiTask.Consul != nil {
 		structsTask.Consul = apiConsulToStructs(apiTask.Consul)
 	}

jobspec2/hcl_conversions.go

@@ -270,7 +270,8 @@ func decodeTaskGroup(body hcl.Body, ctx *hcl.EvalContext, val interface{}) hcl.D
 	diags = append(diags, moreDiags...)
 
 	tgExtra := struct {
-		Vault *api.Vault `hcl:"vault,block"`
+		Vault   *api.Vault    `hcl:"vault,block"`
+		Secrets []*api.Secret `hcl:"secret,block"`
 	}{}
 
 	extra, _ := gohcl.ImpliedBodySchema(tgExtra)
@@ -286,6 +287,14 @@ func decodeTaskGroup(body hcl.Body, ctx *hcl.EvalContext, val interface{}) hcl.D
 			diags = append(diags, hclDecoder.DecodeBody(b.Body, ctx, v)...)
 			tgExtra.Vault = v
 		}
+		if b.Type == "secret" {
+			v := &api.Secret{}
+			diags = append(diags, hclDecoder.DecodeBody(b.Body, ctx, v)...)
+			if len(b.Labels) == 1 {
+				v.Name = b.Labels[0]
+			}
+			tgExtra.Secrets = append(tgExtra.Secrets, v)
+		}
 	}
 
 	d := newHCLDecoder()
@@ -304,6 +313,16 @@ func decodeTaskGroup(body hcl.Body, ctx *hcl.EvalContext, val interface{}) hcl.D
 		}
 	}
 
+	if len(tgExtra.Secrets) > 0 {
+		for _, t := range tg.Tasks {
+			if len(t.Secrets) == 0 {
+				t.Secrets = tgExtra.Secrets
+			} else {
+				t.Secrets = append(t.Secrets, t.Secrets...)
+			}
+		}
+	}
+
 	if tg.Scaling != nil {
 		if tg.Scaling.Type == "" {
 			tg.Scaling.Type = "horizontal"

jobspec2/parse.go

@@ -133,6 +133,7 @@ func decode(c *jobConfig) error {
 	diags = append(diags, decodeMapInterfaceType(&c.Job, c.EvalContext())...)
 	diags = append(diags, decodeMapInterfaceType(&c.Tasks, c.EvalContext())...)
 	diags = append(diags, decodeMapInterfaceType(&c.Vault, c.EvalContext())...)
+	diags = append(diags, decodeMapInterfaceType(&c.Secrets, c.EvalContext())...)
 
 	if diags.HasErrors() {
 		return diags

jobspec2/parse_job.go

@@ -54,6 +54,12 @@ func normalizeJob(jc *jobConfig) {
 				t.Vault = jc.Vault
 			}
 
+			if len(t.Secrets) == 0 {
+				t.Secrets = jc.Secrets
+			} else {
+				t.Secrets = append(t.Secrets, jc.Secrets...)
+			}
+
 			//COMPAT To preserve compatibility with pre-1.7 agents, move the default
 			//       identity to Task.Identity.
 			defaultIdx := -1

jobspec2/types.config.go

@@ -20,6 +20,7 @@ const (
 	localsLabel    = "locals"
 	vaultLabel     = "vault"
 	taskLabel      = "task"
+	secretLabel    = "secret"
 
 	inputVariablesAccessor = "var"
 	localsAccessor         = "local"
@@ -31,8 +32,9 @@ type jobConfig struct {
 
 	ParseConfig *ParseConfig
 
-	Vault *api.Vault  `hcl:"vault,block"`
+	Vault   *api.Vault    `hcl:"vault,block"`
-	Tasks []*api.Task `hcl:"task,block"`
+	Secrets []*api.Secret `hcl:"secret,block"`
+	Tasks   []*api.Task   `hcl:"task,block"`
 
 	InputVariables Variables
 	LocalVariables Variables
@@ -174,6 +176,13 @@ func (c *jobConfig) decodeTopLevelExtras(content *hcl.BodyContent, ctx *hcl.Eval
 				t.Name = b.Labels[0]
 				c.Tasks = append(c.Tasks, t)
 			}
+		} else if b.Type == secretLabel {
+			t := &api.Secret{}
+			diags = append(diags, hclDecoder.DecodeBody(b.Body, ctx, t)...)
+			if len(b.Labels) == 1 {
+				t.Name = b.Labels[0]
+				c.Secrets = append(c.Secrets, t)
+			}
 		}
 	}
 
@@ -277,6 +286,7 @@ func (c *jobConfig) decodeJob(content *hcl.BodyContent, ctx *hcl.EvalContext) hc
 		extra, remain, mdiags := body.PartialContent(&hcl.BodySchema{
 			Blocks: []hcl.BlockHeaderSchema{
 				{Type: "vault"},
+				{Type: "secret", LabelNames: []string{"name"}},
 				{Type: "task", LabelNames: []string{"name"}},
 			},
 		})

nomad/structs/diff.go

@@ -537,6 +537,11 @@ func (t *Task) Diff(other *Task, contextual bool) (*TaskDiff, error) {
 		diff.Objects = append(diff.Objects, vDiff)
 	}
 
+	secDiffs := secretsDiffs(t.Secrets, other.Secrets, contextual)
+	if secDiffs != nil {
+		diff.Objects = append(diff.Objects, secDiffs...)
+	}
+
 	// Consul diff
 	consulDiff := primitiveObjectDiff(t.Consul, other.Consul, nil, "Consul", contextual)
 	if consulDiff != nil {
@@ -578,6 +583,61 @@ func (t *Task) Diff(other *Task, contextual bool) (*TaskDiff, error) {
 	return diff, nil
 }
 
+func secretsDiff(old, new *Secret, contextual bool) *ObjectDiff {
+	diff := &ObjectDiff{Type: DiffTypeNone, Name: "Secret"}
+	if reflect.DeepEqual(old, new) {
+		return nil
+	} else if old == nil {
+		old = &Secret{}
+		diff.Type = DiffTypeAdded
+	} else if new == nil {
+		new = &Secret{}
+		diff.Type = DiffTypeDeleted
+	} else {
+		diff.Type = DiffTypeEdited
+	}
+
+	// Diff the primitive fields.
+	oldPrimitiveFlat := flatmap.Flatten(old, nil, false)
+	newPrimitiveFlat := flatmap.Flatten(new, nil, false)
+	diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual)
+	return diff
+}
+
+// secretsDiffs diffs a set of secrets. The comparator for whether a secret
+// is new/edited/deleted is the secret Name field.
+func secretsDiffs(old, new []*Secret, contextual bool) []*ObjectDiff {
+	var diffs []*ObjectDiff
+
+	oldMap := map[string]*Secret{}
+	newMap := map[string]*Secret{}
+
+	for _, o := range old {
+		oldMap[o.Name] = o
+	}
+	for _, n := range new {
+		newMap[n.Name] = n
+	}
+
+	for k, v := range oldMap {
+		if diff := secretsDiff(v, newMap[k], contextual); diff != nil {
+			diffs = append(diffs, diff)
+		}
+	}
+	for k, v := range newMap {
+		// diff any newly added secrets
+		if _, ok := oldMap[k]; !ok {
+			if diff := secretsDiff(nil, v, contextual); diff != nil {
+				diffs = append(diffs, diff)
+			}
+		}
+	}
+
+	sort.Sort(ObjectDiffs(diffs))
+
+	return diffs
+}
+
 func actionDiff(old, new *Action, contextual bool) *ObjectDiff {
 	diff := &ObjectDiff{Type: DiffTypeNone, Name: "Action"}
 	var oldPrimitiveFlat, newPrimitiveFlat map[string]string

nomad/structs/diff_test.go

@@ -9466,6 +9466,168 @@ func TestTaskDiff(t *testing.T) {
 				},
 			},
 		},
+		{
+			Name: "Secret edited",
+			Old: &Task{
+				Secrets: []*Secret{
+					{
+						Name:     "foo",
+						Provider: "bar",
+						Path:     "/foo/bar",
+						Config: map[string]any{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			New: &Task{
+				Secrets: []*Secret{
+					{
+						Name:     "foo",
+						Provider: "bar1",
+						Path:     "/foo/bar1",
+						Config: map[string]any{
+							"foo": "bar1",
+						},
+					},
+				},
+			},
+			Expected: &TaskDiff{
+				Type: DiffTypeEdited,
+				Objects: []*ObjectDiff{
+					{
+						Type: DiffTypeEdited,
+						Name: "Secret",
+						Fields: []*FieldDiff{
+							{
+								Type: DiffTypeEdited,
+								Name: "Config[foo]",
+								Old:  "bar",
+								New:  "bar1",
+							},
+							{
+								Type: DiffTypeEdited,
+								Name: "Path",
+								Old:  "/foo/bar",
+								New:  "/foo/bar1",
+							},
+							{
+								Type: DiffTypeEdited,
+								Name: "Provider",
+								Old:  "bar",
+								New:  "bar1",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			Name: "Secret added",
+			Old: &Task{
+				Secrets: []*Secret{},
+			},
+			New: &Task{
+				Secrets: []*Secret{
+					{
+						Name:     "foo",
+						Provider: "bar",
+						Path:     "/foo/bar",
+						Config: map[string]any{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			Expected: &TaskDiff{
+				Type: DiffTypeEdited,
+				Objects: []*ObjectDiff{
+					{
+						Type: DiffTypeAdded,
+						Name: "Secret",
+						Fields: []*FieldDiff{
+							{
+								Type: DiffTypeAdded,
+								Name: "Config[foo]",
+								Old:  "",
+								New:  "bar",
+							},
+							{
+								Type: DiffTypeAdded,
+								Name: "Name",
+								Old:  "",
+								New:  "foo",
+							},
+							{
+								Type: DiffTypeAdded,
+								Name: "Path",
+								Old:  "",
+								New:  "/foo/bar",
+							},
+							{
+								Type: DiffTypeAdded,
+								Name: "Provider",
+								Old:  "",
+								New:  "bar",
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			Name: "Secret deleted",
+			Old: &Task{
+				Secrets: []*Secret{
+					{
+						Name:     "foo",
+						Provider: "bar",
+						Path:     "/foo/bar",
+						Config: map[string]any{
+							"foo": "bar",
+						},
+					},
+				},
+			},
+			New: &Task{
+				Secrets: []*Secret{},
+			},
+			Expected: &TaskDiff{
+				Type: DiffTypeEdited,
+				Objects: []*ObjectDiff{
+					{
+						Type: DiffTypeDeleted,
+						Name: "Secret",
+						Fields: []*FieldDiff{
+							{
+								Type: DiffTypeDeleted,
+								Name: "Config[foo]",
+								Old:  "bar",
+								New:  "",
+							},
+							{
+								Type: DiffTypeDeleted,
+								Name: "Name",
+								Old:  "foo",
+								New:  "",
+							},
+							{
+								Type: DiffTypeDeleted,
+								Name: "Path",
+								Old:  "/foo/bar",
+								New:  "",
+							},
+							{
+								Type: DiffTypeDeleted,
+								Name: "Provider",
+								Old:  "bar",
+								New:  "",
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, c := range cases {

nomad/structs/structs.go

@@ -7789,6 +7789,9 @@ type Task struct {
 	// have access to.
 	Vault *Vault
 
+	// List of secrets for the task.
+	Secrets []*Secret
+
 	// Consul configuration specific to this task. If uset, falls back to the
 	// group's Consul field.
 	Consul *Consul
@@ -8288,6 +8291,19 @@ func (t *Task) Validate(jobType string, tg *TaskGroup) error {
 		}
 	}
 
+	secrets := make(map[string]bool)
+	for _, s := range t.Secrets {
+		if _, ok := secrets[s.Name]; ok {
+			mErr.Errors = append(mErr.Errors, fmt.Errorf("Duplicate secret %q found", s.Name))
+		} else {
+			secrets[s.Name] = true
+		}
+
+		if err := s.Validate(); err != nil {
+			mErr.Errors = append(mErr.Errors, fmt.Errorf("Secret %q is invalid: %w", s.Name, err))
+		}
+	}
+
 	return mErr.ErrorOrNil()
 }
 
@@ -10385,6 +10401,84 @@ func (v *Vault) Validate() error {
 	return mErr.ErrorOrNil()
 }
 
+type Secret struct {
+	Name     string
+	Provider string
+	Path     string
+	Config   map[string]any
+}
+
+func (s *Secret) Equal(o *Secret) bool {
+	if s == nil || o == nil {
+		return s == o
+	}
+
+	switch {
+	case s.Name != o.Name:
+		return false
+	case s.Provider != o.Provider:
+		return false
+	case s.Path != o.Path:
+		return false
+	case !maps.Equal(s.Config, o.Config):
+		return false
+	}
+
+	return true
+}
+
+func (s *Secret) Copy() *Secret {
+	if s == nil {
+		return nil
+	}
+
+	confCopy, err := copystructure.Copy(s.Config)
+	if err != nil {
+		// The default Copy() implementation should not return
+		// an error, so we should not reach this code path.
+		panic(err.Error())
+	}
+
+	return &Secret{
+		Name:     s.Name,
+		Provider: s.Provider,
+		Path:     s.Path,
+		Config:   confCopy.(map[string]any),
+	}
+}
+
+func (s *Secret) Validate() error {
+	if s == nil {
+		return nil
+	}
+
+	var mErr multierror.Error
+
+	if s.Name == "" {
+		_ = multierror.Append(&mErr, fmt.Errorf("Secret name cannot be empty"))
+	}
+
+	if s.Provider == "" {
+		_ = multierror.Append(&mErr, fmt.Errorf("Secret provider cannot be empty"))
+	}
+
+	if s.Path == "" {
+		_ = multierror.Append(&mErr, fmt.Errorf("Secret path cannot be empty"))
+	}
+
+	return mErr.ErrorOrNil()
+}
+
+func (s *Secret) Canonicalize() {
+	if s == nil {
+		return
+	}
+
+	if len(s.Config) == 0 {
+		s.Config = nil
+	}
+}
+
 const (
 	// DeploymentStatuses are the various states a deployment can be be in
 	DeploymentStatusRunning      = "running"

nomad/structs/structs_test.go

@@ -6459,6 +6459,100 @@ func TestVault_Canonicalize(t *testing.T) {
 	require.Equal(t, VaultChangeModeRestart, v.ChangeMode)
 }
 
+func TestSecrets_Copy(t *testing.T) {
+	ci.Parallel(t)
+	s := &Secret{
+		Name:     "test-secret",
+		Provider: "test-provider",
+		Path:     "/test/path",
+		Config: map[string]any{
+			"some-key": map[string]any{
+				"nested-key": "nested-value",
+			},
+		},
+	}
+	ns := s.Copy()
+
+	must.Eq(t, s.Name, ns.Name)
+	must.Eq(t, s.Provider, ns.Provider)
+	must.Eq(t, s.Path, ns.Path)
+	must.Eq(t, s.Config, ns.Config)
+
+	// make sure nested maps are copied correctly
+	s.Config["some-key"].(map[string]any)["nested-key"] = "new-value"
+
+	must.NotEq(t, s.Config, ns.Config)
+}
+
+func TestSecrets_Validate(t *testing.T) {
+	ci.Parallel(t)
+	testCases := []struct {
+		name      string
+		secret    *Secret
+		expectErr error
+	}{
+		{
+			name: "valid secret",
+			secret: &Secret{
+				Name:     "test-secret",
+				Provider: "test-provier",
+				Path:     "test-path",
+			},
+			expectErr: nil,
+		},
+		{
+			name: "missing name",
+			secret: &Secret{
+				Path:     "test-path",
+				Provider: "test-provider",
+			},
+			expectErr: fmt.Errorf("Secret name cannot be empty"),
+		},
+		{
+			name: "missing provider",
+			secret: &Secret{
+				Name: "test-secret",
+				Path: "test-path",
+			},
+			expectErr: fmt.Errorf("Secret provider cannot be empty"),
+		},
+		{
+			name: "missing path",
+			secret: &Secret{
+				Name:     "test-secret",
+				Provider: "test-provier",
+			},
+			expectErr: fmt.Errorf("Secret path cannot be empty"),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.secret.Validate()
+			if tc.expectErr != nil {
+				must.ErrorContains(t, err, tc.expectErr.Error())
+			} else {
+				must.NoError(t, err)
+			}
+		})
+	}
+
+}
+
+func TestSecrets_Canonicalize(t *testing.T) {
+	ci.Parallel(t)
+	s := &Secret{
+		Name:     "test-secret",
+		Provider: "test-provider",
+		Path:     "/test/path",
+		Config:   make(map[string]any),
+	}
+
+	s.Canonicalize()
+
+	must.Nil(t, s.Config)
+}
+
 func TestParameterizedJobConfig_Validate(t *testing.T) {
 	ci.Parallel(t)
 

scheduler/util.go

@@ -230,6 +230,9 @@ func tasksUpdated(jobA, jobB *structs.Job, taskGroup string) comparison {
 		if !at.Vault.Equal(bt.Vault) {
 			return difference("task vault", at.Vault, bt.Vault)
 		}
+		if !slices.EqualFunc(at.Secrets, bt.Secrets, func(a, b *structs.Secret) bool { return a.Equal(b) }) {
+			return difference("task secrets", at.Secrets, bt.Secrets)
+		}
 		if c := consulUpdated(at.Consul, bt.Consul); c.modified {
 			return c
 		}

scheduler/util_test.go

@@ -440,6 +440,22 @@ func TestTasksUpdated(t *testing.T) {
 	j32.TaskGroups[0].Tasks[0].VolumeMounts = nil
 
 	must.True(t, tasksUpdated(j31, j32, name).modified)
+
+	j33 := mock.Job()
+	j33 = j32.Copy()
+
+	must.False(t, tasksUpdated(j32, j33, name).modified)
+
+	// Add a task secret
+	j33.TaskGroups[0].Tasks[0].Secrets = append(j32.TaskGroups[0].Tasks[0].Secrets,
+		&structs.Secret{
+			Name:     "mysecret",
+			Provider: "nomad",
+			Path:     "/my/path",
+		})
+
+	must.True(t, tasksUpdated(j32, j33, name).modified)
+
 }
 
 func TestTasksUpdated_connectServiceUpdated(t *testing.T) {