diff --git a/api/services.go b/api/services.go index 47da17930..be987a8d0 100644 --- a/api/services.go +++ b/api/services.go @@ -135,6 +135,7 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) { type ConsulConnect struct { Native bool SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"` + SidecarTask *Task `mapstructure:"sidecar_task"` } // ConsulSidecarService represents a Consul Connect SidecarService jobspec diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index d05083bb9..2ff6e48bb 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1007,39 +1007,50 @@ func ApiServicesToStructs(in []*api.Service) []*structs.Service { } } - if s.Connect == nil { - continue + if s.Connect != nil { + out[i].Connect = ApiConsulConnectToStructs(s.Connect) } - out[i].Connect = &structs.ConsulConnect{ - Native: s.Connect.Native, + } + + return out +} + +func ApiConsulConnectToStructs(in *api.ConsulConnect) *structs.ConsulConnect { + if in == nil { + return nil + } + + out := &structs.ConsulConnect{ + Native: in.Native, + } + + if in.SidecarService != nil { + + out.SidecarService = &structs.ConsulSidecarService{ + Port: in.SidecarService.Port, } - if s.Connect.SidecarService == nil { - continue - } + if in.SidecarService.Proxy != nil { - out[i].Connect.SidecarService = &structs.ConsulSidecarService{ - Port: s.Connect.SidecarService.Port, - } - - if s.Connect.SidecarService.Proxy == nil { - continue - } - - out[i].Connect.SidecarService.Proxy = &structs.ConsulProxy{ - Config: s.Connect.SidecarService.Proxy.Config, - } - - upstreams := make([]*structs.ConsulUpstream, len(s.Connect.SidecarService.Proxy.Upstreams)) - for i, p := range s.Connect.SidecarService.Proxy.Upstreams { - upstreams[i] = &structs.ConsulUpstream{ - DestinationName: p.DestinationName, - LocalBindPort: p.LocalBindPort, + out.SidecarService.Proxy = &structs.ConsulProxy{ + Config: in.SidecarService.Proxy.Config, } - } - out[i].Connect.SidecarService.Proxy.Upstreams = upstreams + upstreams := make([]structs.ConsulUpstream, len(in.SidecarService.Proxy.Upstreams)) + for i, p := range in.SidecarService.Proxy.Upstreams { + upstreams[i] = structs.ConsulUpstream{ + DestinationName: p.DestinationName, + LocalBindPort: p.LocalBindPort, + } + } + + out.SidecarService.Proxy.Upstreams = upstreams + } + } + + if in.SidecarTask != nil { + ApiTaskToStructsTask(in.SidecarTask, out.SidecarTask) } return out diff --git a/jobspec/parse.go b/jobspec/parse.go index 3c6ce4329..3cff938e5 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -8,8 +8,6 @@ import ( "path/filepath" "regexp" "strconv" - "strings" - "time" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl" @@ -84,427 +82,6 @@ func ParseFile(path string) (*api.Job, error) { return Parse(f) } -func parseJob(result *api.Job, list *ast.ObjectList) error { - if len(list.Items) != 1 { - return fmt.Errorf("only one 'job' block allowed") - } - list = list.Children() - if len(list.Items) != 1 { - return fmt.Errorf("'job' block missing name") - } - - // Get our job object - obj := list.Items[0] - - // Decode the full thing into a map[string]interface for ease - var m map[string]interface{} - if err := hcl.DecodeObject(&m, obj.Val); err != nil { - return err - } - delete(m, "constraint") - delete(m, "affinity") - delete(m, "meta") - delete(m, "migrate") - delete(m, "parameterized") - delete(m, "periodic") - delete(m, "reschedule") - delete(m, "update") - delete(m, "vault") - delete(m, "spread") - - // Set the ID and name to the object key - result.ID = helper.StringToPtr(obj.Keys[0].Token.Value().(string)) - result.Name = helper.StringToPtr(*result.ID) - - // Decode the rest - if err := mapstructure.WeakDecode(m, result); err != nil { - return err - } - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("job '%s' value: should be an object", *result.ID) - } - - // Check for invalid keys - valid := []string{ - "all_at_once", - "constraint", - "affinity", - "spread", - "datacenters", - "group", - "id", - "meta", - "migrate", - "name", - "namespace", - "parameterized", - "periodic", - "priority", - "region", - "reschedule", - "task", - "type", - "update", - "vault", - "vault_token", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return multierror.Prefix(err, "job:") - } - - // Parse constraints - if o := listVal.Filter("constraint"); len(o.Items) > 0 { - if err := parseConstraints(&result.Constraints, o); err != nil { - return multierror.Prefix(err, "constraint ->") - } - } - - // Parse affinities - if o := listVal.Filter("affinity"); len(o.Items) > 0 { - if err := parseAffinities(&result.Affinities, o); err != nil { - return multierror.Prefix(err, "affinity ->") - } - } - - // If we have an update strategy, then parse that - if o := listVal.Filter("update"); len(o.Items) > 0 { - if err := parseUpdate(&result.Update, o); err != nil { - return multierror.Prefix(err, "update ->") - } - } - - // If we have a periodic definition, then parse that - if o := listVal.Filter("periodic"); len(o.Items) > 0 { - if err := parsePeriodic(&result.Periodic, o); err != nil { - return multierror.Prefix(err, "periodic ->") - } - } - - // Parse spread - if o := listVal.Filter("spread"); len(o.Items) > 0 { - if err := parseSpread(&result.Spreads, o); err != nil { - return multierror.Prefix(err, "spread ->") - } - } - - // If we have a parameterized definition, then parse that - if o := listVal.Filter("parameterized"); len(o.Items) > 0 { - if err := parseParameterizedJob(&result.ParameterizedJob, o); err != nil { - return multierror.Prefix(err, "parameterized ->") - } - } - - // If we have a reschedule stanza, then parse that - if o := listVal.Filter("reschedule"); len(o.Items) > 0 { - if err := parseReschedulePolicy(&result.Reschedule, o); err != nil { - return multierror.Prefix(err, "reschedule ->") - } - } - - // If we have a migration strategy, then parse that - if o := listVal.Filter("migrate"); len(o.Items) > 0 { - if err := parseMigrate(&result.Migrate, o); err != nil { - return multierror.Prefix(err, "migrate ->") - } - } - - // Parse out meta fields. These are in HCL as a list so we need - // to iterate over them and merge them. - if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { - for _, o := range metaO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &result.Meta); err != nil { - return err - } - } - } - - // If we have tasks outside, create TaskGroups for them - if o := listVal.Filter("task"); len(o.Items) > 0 { - var tasks []*api.Task - if err := parseTasks(*result.Name, "", &tasks, o); err != nil { - return multierror.Prefix(err, "task:") - } - - result.TaskGroups = make([]*api.TaskGroup, len(tasks), len(tasks)*2) - for i, t := range tasks { - result.TaskGroups[i] = &api.TaskGroup{ - Name: helper.StringToPtr(t.Name), - Tasks: []*api.Task{t}, - } - } - } - - // Parse the task groups - if o := listVal.Filter("group"); len(o.Items) > 0 { - if err := parseGroups(result, o); err != nil { - return multierror.Prefix(err, "group:") - } - } - - // If we have a vault block, then parse that - if o := listVal.Filter("vault"); len(o.Items) > 0 { - jobVault := &api.Vault{ - Env: helper.BoolToPtr(true), - ChangeMode: helper.StringToPtr("restart"), - } - - if err := parseVault(jobVault, o); err != nil { - return multierror.Prefix(err, "vault ->") - } - - // Go through the task groups/tasks and if they don't have a Vault block, set it - for _, tg := range result.TaskGroups { - for _, task := range tg.Tasks { - if task.Vault == nil { - task.Vault = jobVault - } - } - } - } - - return nil -} - -func parseGroups(result *api.Job, list *ast.ObjectList) error { - list = list.Children() - if len(list.Items) == 0 { - return nil - } - - // Go through each object and turn it into an actual result. - collection := make([]*api.TaskGroup, 0, len(list.Items)) - seen := make(map[string]struct{}) - for _, item := range list.Items { - n := item.Keys[0].Token.Value().(string) - - // Make sure we haven't already found this - if _, ok := seen[n]; ok { - return fmt.Errorf("group '%s' defined more than once", n) - } - seen[n] = struct{}{} - - // We need this later - var listVal *ast.ObjectList - if ot, ok := item.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("group '%s': should be an object", n) - } - - // Check for invalid keys - valid := []string{ - "count", - "constraint", - "affinity", - "restart", - "meta", - "task", - "ephemeral_disk", - "update", - "reschedule", - "vault", - "migrate", - "spread", - "network", - "service", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s' ->", n)) - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, item.Val); err != nil { - return err - } - delete(m, "constraint") - delete(m, "affinity") - delete(m, "meta") - delete(m, "task") - delete(m, "restart") - delete(m, "ephemeral_disk") - delete(m, "update") - delete(m, "vault") - delete(m, "migrate") - delete(m, "spread") - delete(m, "network") - delete(m, "service") - - // Build the group with the basic decode - var g api.TaskGroup - g.Name = helper.StringToPtr(n) - if err := mapstructure.WeakDecode(m, &g); err != nil { - return err - } - - // Parse constraints - if o := listVal.Filter("constraint"); len(o.Items) > 0 { - if err := parseConstraints(&g.Constraints, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', constraint ->", n)) - } - } - - // Parse affinities - if o := listVal.Filter("affinity"); len(o.Items) > 0 { - if err := parseAffinities(&g.Affinities, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', affinity ->", n)) - } - } - - // Parse restart policy - if o := listVal.Filter("restart"); len(o.Items) > 0 { - if err := parseRestartPolicy(&g.RestartPolicy, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', restart ->", n)) - } - } - - // Parse spread - if o := listVal.Filter("spread"); len(o.Items) > 0 { - if err := parseSpread(&g.Spreads, o); err != nil { - return multierror.Prefix(err, "spread ->") - } - } - - // Parse network - if o := listVal.Filter("network"); len(o.Items) > 0 { - networks, err := parseNetwork(o) - if err != nil { - return err - } - g.Networks = []*api.NetworkResource{networks} - } - - // Parse reschedule policy - if o := listVal.Filter("reschedule"); len(o.Items) > 0 { - if err := parseReschedulePolicy(&g.ReschedulePolicy, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', reschedule ->", n)) - } - } - // Parse ephemeral disk - if o := listVal.Filter("ephemeral_disk"); len(o.Items) > 0 { - g.EphemeralDisk = &api.EphemeralDisk{} - if err := parseEphemeralDisk(&g.EphemeralDisk, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', ephemeral_disk ->", n)) - } - } - - // If we have an update strategy, then parse that - if o := listVal.Filter("update"); len(o.Items) > 0 { - if err := parseUpdate(&g.Update, o); err != nil { - return multierror.Prefix(err, "update ->") - } - } - - // If we have a migration strategy, then parse that - if o := listVal.Filter("migrate"); len(o.Items) > 0 { - if err := parseMigrate(&g.Migrate, o); err != nil { - return multierror.Prefix(err, "migrate ->") - } - } - - // Parse out meta fields. These are in HCL as a list so we need - // to iterate over them and merge them. - if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { - for _, o := range metaO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &g.Meta); err != nil { - return err - } - } - } - - // Parse tasks - if o := listVal.Filter("task"); len(o.Items) > 0 { - if err := parseTasks(*result.Name, *g.Name, &g.Tasks, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', task:", n)) - } - } - - // If we have a vault block, then parse that - if o := listVal.Filter("vault"); len(o.Items) > 0 { - tgVault := &api.Vault{ - Env: helper.BoolToPtr(true), - ChangeMode: helper.StringToPtr("restart"), - } - - if err := parseVault(tgVault, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', vault ->", n)) - } - - // Go through the tasks and if they don't have a Vault block, set it - for _, task := range g.Tasks { - if task.Vault == nil { - task.Vault = tgVault - } - } - } - - if o := listVal.Filter("service"); len(o.Items) > 0 { - if err := parseGroupServices(*result.Name, *g.Name, &g, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s',", n)) - } - } - - collection = append(collection, &g) - } - - result.TaskGroups = append(result.TaskGroups, collection...) - return nil -} - -func parseRestartPolicy(final **api.RestartPolicy, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'restart' block allowed") - } - - // Get our job object - obj := list.Items[0] - - // Check for invalid keys - valid := []string{ - "attempts", - "interval", - "delay", - "mode", - } - if err := helper.CheckHCLKeys(obj.Val, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, obj.Val); err != nil { - return err - } - - var result api.RestartPolicy - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &result, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - *final = &result - return nil -} - func parseReschedulePolicy(final **api.ReschedulePolicy, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) > 1 { @@ -706,39 +283,6 @@ func parseAffinities(result *[]*api.Affinity, list *ast.ObjectList) error { return nil } -func parseEphemeralDisk(result **api.EphemeralDisk, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'ephemeral_disk' block allowed") - } - - // Get our ephemeral_disk object - obj := list.Items[0] - - // Check for invalid keys - valid := []string{ - "sticky", - "size", - "migrate", - } - if err := helper.CheckHCLKeys(obj.Val, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, obj.Val); err != nil { - return err - } - - var ephemeralDisk api.EphemeralDisk - if err := mapstructure.WeakDecode(m, &ephemeralDisk); err != nil { - return err - } - *result = &ephemeralDisk - - return nil -} - func parseSpread(result *[]*api.Spread, list *ast.ObjectList) error { for _, o := range list.Elem().Items { // Check for invalid keys @@ -847,1052 +391,6 @@ func parseBool(value interface{}) (bool, error) { return enabled, err } -func parseTasks(jobName string, taskGroupName string, result *[]*api.Task, list *ast.ObjectList) error { - list = list.Children() - if len(list.Items) == 0 { - return nil - } - - // Go through each object and turn it into an actual result. - seen := make(map[string]struct{}) - for _, item := range list.Items { - n := item.Keys[0].Token.Value().(string) - - // Make sure we haven't already found this - if _, ok := seen[n]; ok { - return fmt.Errorf("task '%s' defined more than once", n) - } - seen[n] = struct{}{} - - // We need this later - var listVal *ast.ObjectList - if ot, ok := item.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("group '%s': should be an object", n) - } - - // Check for invalid keys - valid := []string{ - "artifact", - "config", - "constraint", - "affinity", - "dispatch_payload", - "driver", - "env", - "kill_timeout", - "leader", - "logs", - "meta", - "resources", - "service", - "shutdown_delay", - "template", - "user", - "vault", - "kill_signal", - "kind", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s' ->", n)) - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, item.Val); err != nil { - return err - } - delete(m, "artifact") - delete(m, "config") - delete(m, "constraint") - delete(m, "affinity") - delete(m, "dispatch_payload") - delete(m, "env") - delete(m, "logs") - delete(m, "meta") - delete(m, "resources") - delete(m, "service") - delete(m, "template") - delete(m, "vault") - - // Build the task - var t api.Task - t.Name = n - if taskGroupName == "" { - taskGroupName = n - } - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &t, - }) - - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - // If we have env, then parse them - if o := listVal.Filter("env"); len(o.Items) > 0 { - for _, o := range o.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &t.Env); err != nil { - return err - } - } - } - - if o := listVal.Filter("service"); len(o.Items) > 0 { - services, err := parseServices(jobName, taskGroupName, o) - if err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s',", n)) - } - - t.Services = services - } - - // If we have config, then parse that - if o := listVal.Filter("config"); len(o.Items) > 0 { - for _, o := range o.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - if err := mapstructure.WeakDecode(m, &t.Config); err != nil { - return err - } - } - } - - // Parse constraints - if o := listVal.Filter("constraint"); len(o.Items) > 0 { - if err := parseConstraints(&t.Constraints, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf( - "'%s', constraint ->", n)) - } - } - - // Parse affinities - if o := listVal.Filter("affinity"); len(o.Items) > 0 { - if err := parseAffinities(&t.Affinities, o); err != nil { - return multierror.Prefix(err, "affinity ->") - } - } - - // Parse out meta fields. These are in HCL as a list so we need - // to iterate over them and merge them. - if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { - for _, o := range metaO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &t.Meta); err != nil { - return err - } - } - } - - // If we have resources, then parse that - if o := listVal.Filter("resources"); len(o.Items) > 0 { - var r api.Resources - if err := parseResources(&r, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s',", n)) - } - - t.Resources = &r - } - - // If we have logs then parse that - if o := listVal.Filter("logs"); len(o.Items) > 0 { - if len(o.Items) > 1 { - return fmt.Errorf("only one logs block is allowed in a Task. Number of logs block found: %d", len(o.Items)) - } - var m map[string]interface{} - logsBlock := o.Items[0] - - // Check for invalid keys - valid := []string{ - "max_files", - "max_file_size", - } - if err := helper.CheckHCLKeys(logsBlock.Val, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', logs ->", n)) - } - - if err := hcl.DecodeObject(&m, logsBlock.Val); err != nil { - return err - } - - var log api.LogConfig - if err := mapstructure.WeakDecode(m, &log); err != nil { - return err - } - - t.LogConfig = &log - } - - // Parse artifacts - if o := listVal.Filter("artifact"); len(o.Items) > 0 { - if err := parseArtifacts(&t.Artifacts, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', artifact ->", n)) - } - } - - // Parse templates - if o := listVal.Filter("template"); len(o.Items) > 0 { - if err := parseTemplates(&t.Templates, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', template ->", n)) - } - } - - // If we have a vault block, then parse that - if o := listVal.Filter("vault"); len(o.Items) > 0 { - v := &api.Vault{ - Env: helper.BoolToPtr(true), - ChangeMode: helper.StringToPtr("restart"), - } - - if err := parseVault(v, o); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', vault ->", n)) - } - - t.Vault = v - } - - // If we have a dispatch_payload block parse that - if o := listVal.Filter("dispatch_payload"); len(o.Items) > 0 { - if len(o.Items) > 1 { - return fmt.Errorf("only one dispatch_payload block is allowed in a task. Number of dispatch_payload blocks found: %d", len(o.Items)) - } - var m map[string]interface{} - dispatchBlock := o.Items[0] - - // Check for invalid keys - valid := []string{ - "file", - } - if err := helper.CheckHCLKeys(dispatchBlock.Val, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', dispatch_payload ->", n)) - } - - if err := hcl.DecodeObject(&m, dispatchBlock.Val); err != nil { - return err - } - - t.DispatchPayload = &api.DispatchPayloadConfig{} - if err := mapstructure.WeakDecode(m, t.DispatchPayload); err != nil { - return err - } - } - - *result = append(*result, &t) - } - - return nil -} - -func parseArtifacts(result *[]*api.TaskArtifact, list *ast.ObjectList) error { - for _, o := range list.Elem().Items { - // Check for invalid keys - valid := []string{ - "source", - "options", - "mode", - "destination", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - delete(m, "options") - - var ta api.TaskArtifact - if err := mapstructure.WeakDecode(m, &ta); err != nil { - return err - } - - var optionList *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - optionList = ot.List - } else { - return fmt.Errorf("artifact should be an object") - } - - if oo := optionList.Filter("options"); len(oo.Items) > 0 { - options := make(map[string]string) - if err := parseArtifactOption(options, oo); err != nil { - return multierror.Prefix(err, "options: ") - } - ta.GetterOptions = options - } - - *result = append(*result, &ta) - } - - return nil -} - -func parseArtifactOption(result map[string]string, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'options' block allowed per artifact") - } - - // Get our resource object - o := list.Items[0] - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - if err := mapstructure.WeakDecode(m, &result); err != nil { - return err - } - - return nil -} - -func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error { - for _, o := range list.Elem().Items { - // Check for invalid keys - valid := []string{ - "change_mode", - "change_signal", - "data", - "destination", - "left_delimiter", - "perms", - "right_delimiter", - "source", - "splay", - "env", - "vault_grace", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - templ := &api.Template{ - ChangeMode: helper.StringToPtr("restart"), - Splay: helper.TimeToPtr(5 * time.Second), - Perms: helper.StringToPtr("0644"), - } - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: templ, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - *result = append(*result, templ) - } - - return nil -} - -//TODO(schmichael) combine with non-group services -func parseGroupServices(jobName string, taskGroupName string, g *api.TaskGroup, serviceObjs *ast.ObjectList) error { - g.Services = make([]*api.Service, len(serviceObjs.Items)) - for idx, o := range serviceObjs.Items { - // Check for invalid keys - valid := []string{ - "name", - "tags", - "canary_tags", - "port", - "check", - "address_mode", - "check_restart", - "connect", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("service (%d) ->", idx)) - } - - var service api.Service - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - delete(m, "check") - delete(m, "check_restart") - delete(m, "connect") - - if err := mapstructure.WeakDecode(m, &service); err != nil { - return err - } - - // Filter checks - var checkList *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - checkList = ot.List - } else { - return fmt.Errorf("service '%s': should be an object", service.Name) - } - - if co := checkList.Filter("check"); len(co.Items) > 0 { - if err := parseChecks(&service, co); err != nil { - return multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } - } - - // Filter check_restart - if cro := checkList.Filter("check_restart"); len(cro.Items) > 0 { - if len(cro.Items) > 1 { - return fmt.Errorf("check_restart '%s': cannot have more than 1 check_restart", service.Name) - } - if cr, err := parseCheckRestart(cro.Items[0]); err != nil { - return multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } else { - service.CheckRestart = cr - } - } - - // Filter connect - if co := checkList.Filter("connect"); len(co.Items) > 0 { - if len(co.Items) > 1 { - return fmt.Errorf("connect '%s': cannot have more than 1 connect", service.Name) - } - if c, err := parseConnect(co.Items[0]); err != nil { - return multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } else { - service.Connect = c - } - } - - g.Services[idx] = &service - } - - return nil -} - -func parseServices(jobName string, taskGroupName string, serviceObjs *ast.ObjectList) ([]*api.Service, error) { - services := make([]*api.Service, len(serviceObjs.Items)) - for idx, o := range serviceObjs.Items { - // Check for invalid keys - valid := []string{ - "name", - "tags", - "canary_tags", - "port", - "check", - "address_mode", - "check_restart", - "connect", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return nil, multierror.Prefix(err, fmt.Sprintf("service (%d) ->", idx)) - } - - var service api.Service - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return nil, err - } - - delete(m, "check") - delete(m, "check_restart") - delete(m, "connect") - - if err := mapstructure.WeakDecode(m, &service); err != nil { - return nil, err - } - - // Filter checks - var checkList *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - checkList = ot.List - } else { - return nil, fmt.Errorf("service '%s': should be an object", service.Name) - } - - if co := checkList.Filter("check"); len(co.Items) > 0 { - if err := parseChecks(&service, co); err != nil { - return nil, multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } - } - - // Filter check_restart - if cro := checkList.Filter("check_restart"); len(cro.Items) > 0 { - if len(cro.Items) > 1 { - return nil, fmt.Errorf("check_restart '%s': cannot have more than 1 check_restart", service.Name) - } - if cr, err := parseCheckRestart(cro.Items[0]); err != nil { - return nil, multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } else { - service.CheckRestart = cr - } - } - - // Filter connect - if co := checkList.Filter("connect"); len(co.Items) > 0 { - if len(co.Items) > 1 { - return nil, fmt.Errorf("connect '%s': cannot have more than 1 connect stanza", service.Name) - } - - c, err := parseConnect(co.Items[0]) - if err != nil { - return nil, multierror.Prefix(err, fmt.Sprintf("service: '%s',", service.Name)) - } - - service.Connect = c - } - - services[idx] = &service - } - - return services, nil -} - -func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { - valid := []string{ - "native", - "sidecar_service", - } - - if err := helper.CheckHCLKeys(co.Val, valid); err != nil { - return nil, multierror.Prefix(err, "connect ->") - } - - var connect api.ConsulConnect - var m map[string]interface{} - if err := hcl.DecodeObject(&m, co.Val); err != nil { - return nil, err - } - - delete(m, "sidecar_service") - - if err := mapstructure.WeakDecode(m, &connect); err != nil { - return nil, err - } - - var connectList *ast.ObjectList - if ot, ok := co.Val.(*ast.ObjectType); ok { - connectList = ot.List - } else { - return nil, fmt.Errorf("connect should be an object") - } - - // Parse the sidecar_service - o := connectList.Filter("sidecar_service") - if len(o.Items) == 0 { - return &connect, nil - } - if len(o.Items) > 1 { - return nil, fmt.Errorf("only one 'sidecar_service' block allowed per task") - } - - r, err := parseSidecarService(o.Items[0]) - if err != nil { - return nil, fmt.Errorf("sidecar_service, %v", err) - } - connect.SidecarService = r - - return &connect, nil -} - -func parseSidecarService(o *ast.ObjectItem) (*api.ConsulSidecarService, error) { - valid := []string{ - "port", - "proxy", - } - - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return nil, multierror.Prefix(err, "sidecar_service ->") - } - - var sidecar api.ConsulSidecarService - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return nil, err - } - - delete(m, "proxy") - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &sidecar, - }) - if err != nil { - return nil, err - } - if err := dec.Decode(m); err != nil { - return nil, fmt.Errorf("foo: %v", err) - } - - var proxyList *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - proxyList = ot.List - } else { - return nil, fmt.Errorf("sidecar_service: should be an object") - } - - // Parse the proxy - po := proxyList.Filter("proxy") - if len(po.Items) == 0 { - return &sidecar, nil - } - if len(po.Items) > 1 { - return nil, fmt.Errorf("only one 'proxy' block allowed per task") - } - - r, err := parseProxy(po.Items[0]) - if err != nil { - return nil, fmt.Errorf("proxy, %v", err) - } - sidecar.Proxy = r - - return &sidecar, nil -} - -func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { - valid := []string{ - "upstreams", - "config", - } - - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return nil, multierror.Prefix(err, "proxy ->") - } - - var proxy api.ConsulProxy - - var listVal *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return nil, fmt.Errorf("proxy: should be an object") - } - - // Parse the proxy - uo := listVal.Filter("upstreams") - proxy.Upstreams = make([]*api.ConsulUpstream, len(uo.Items)) - for i := range uo.Items { - u, err := parseUpstream(uo.Items[i]) - if err != nil { - return nil, err - } - - proxy.Upstreams[i] = u - } - - // If we have config, then parse that - if o := listVal.Filter("config"); len(o.Items) > 1 { - return nil, fmt.Errorf("only 1 meta object supported") - } else if len(o.Items) == 1 { - var mSlice []map[string]interface{} - if err := hcl.DecodeObject(&mSlice, o.Items[0].Val); err != nil { - return nil, err - } - - if len(mSlice) > 1 { - return nil, fmt.Errorf("only 1 meta object supported") - } - - m := mSlice[0] - - if err := mapstructure.WeakDecode(m, &proxy.Config); err != nil { - return nil, err - } - - proxy.Config = flattenMapSlice(proxy.Config) - } - - return &proxy, nil -} - -func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { - valid := []string{ - "destination_name", - "local_bind_port", - } - - if err := helper.CheckHCLKeys(uo.Val, valid); err != nil { - return nil, multierror.Prefix(err, "upstream ->") - } - - var upstream api.ConsulUpstream - var m map[string]interface{} - if err := hcl.DecodeObject(&m, uo.Val); err != nil { - return nil, err - } - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &upstream, - }) - if err != nil { - return nil, err - } - - if err := dec.Decode(m); err != nil { - return nil, err - } - - return &upstream, nil -} -func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error { - service.Checks = make([]api.ServiceCheck, len(checkObjs.Items)) - for idx, co := range checkObjs.Items { - // Check for invalid keys - valid := []string{ - "name", - "type", - "interval", - "timeout", - "path", - "protocol", - "port", - "command", - "args", - "initial_status", - "tls_skip_verify", - "header", - "method", - "check_restart", - "address_mode", - "grpc_service", - "grpc_use_tls", - } - if err := helper.CheckHCLKeys(co.Val, valid); err != nil { - return multierror.Prefix(err, "check ->") - } - - var check api.ServiceCheck - var cm map[string]interface{} - if err := hcl.DecodeObject(&cm, co.Val); err != nil { - return err - } - - // HCL allows repeating stanzas so merge 'header' into a single - // map[string][]string. - if headerI, ok := cm["header"]; ok { - headerRaw, ok := headerI.([]map[string]interface{}) - if !ok { - return fmt.Errorf("check -> header -> expected a []map[string][]string but found %T", headerI) - } - m := map[string][]string{} - for _, rawm := range headerRaw { - for k, vI := range rawm { - vs, ok := vI.([]interface{}) - if !ok { - return fmt.Errorf("check -> header -> %q expected a []string but found %T", k, vI) - } - for _, vI := range vs { - v, ok := vI.(string) - if !ok { - return fmt.Errorf("check -> header -> %q expected a string but found %T", k, vI) - } - m[k] = append(m[k], v) - } - } - } - - check.Header = m - - // Remove "header" as it has been parsed - delete(cm, "header") - } - - delete(cm, "check_restart") - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &check, - }) - if err != nil { - return err - } - if err := dec.Decode(cm); err != nil { - return err - } - - // Filter check_restart - var checkRestartList *ast.ObjectList - if ot, ok := co.Val.(*ast.ObjectType); ok { - checkRestartList = ot.List - } else { - return fmt.Errorf("check_restart '%s': should be an object", check.Name) - } - - if cro := checkRestartList.Filter("check_restart"); len(cro.Items) > 0 { - if len(cro.Items) > 1 { - return fmt.Errorf("check_restart '%s': cannot have more than 1 check_restart", check.Name) - } - if cr, err := parseCheckRestart(cro.Items[0]); err != nil { - return multierror.Prefix(err, fmt.Sprintf("check: '%s',", check.Name)) - } else { - check.CheckRestart = cr - } - } - - service.Checks[idx] = check - } - - return nil -} - -func parseCheckRestart(cro *ast.ObjectItem) (*api.CheckRestart, error) { - valid := []string{ - "limit", - "grace", - "ignore_warnings", - } - - if err := helper.CheckHCLKeys(cro.Val, valid); err != nil { - return nil, multierror.Prefix(err, "check_restart ->") - } - - var checkRestart api.CheckRestart - var crm map[string]interface{} - if err := hcl.DecodeObject(&crm, cro.Val); err != nil { - return nil, err - } - - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &checkRestart, - }) - if err != nil { - return nil, err - } - if err := dec.Decode(crm); err != nil { - return nil, err - } - - return &checkRestart, nil -} - -func parseResources(result *api.Resources, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) == 0 { - return nil - } - if len(list.Items) > 1 { - return fmt.Errorf("only one 'resource' block allowed per task") - } - - // Get our resource object - o := list.Items[0] - - // We need this later - var listVal *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("resource: should be an object") - } - - // Check for invalid keys - valid := []string{ - "cpu", - "iops", // COMPAT(0.10): Remove after one release to allow it to be removed from jobspecs - "disk", - "memory", - "network", - "device", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return multierror.Prefix(err, "resources ->") - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - delete(m, "network") - delete(m, "device") - - if err := mapstructure.WeakDecode(m, result); err != nil { - return err - } - - // Parse the network resources - if o := listVal.Filter("network"); len(o.Items) > 0 { - r, err := parseNetwork(o) - if err != nil { - return fmt.Errorf("resource, %v", err) - } - result.Networks = []*api.NetworkResource{r} - } - - // Parse the device resources - if o := listVal.Filter("device"); len(o.Items) > 0 { - result.Devices = make([]*api.RequestedDevice, len(o.Items)) - for idx, do := range o.Items { - if l := len(do.Keys); l == 0 { - return multierror.Prefix(fmt.Errorf("missing device name"), fmt.Sprintf("resources, device[%d]->", idx)) - } else if l > 1 { - return multierror.Prefix(fmt.Errorf("only one name may be specified"), fmt.Sprintf("resources, device[%d]->", idx)) - } - name := do.Keys[0].Token.Value().(string) - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := do.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("device should be an object") - } - - // Check for invalid keys - valid := []string{ - "name", - "count", - "affinity", - "constraint", - } - if err := helper.CheckHCLKeys(do.Val, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("resources, device[%d]->", idx)) - } - - // Set the name - var r api.RequestedDevice - r.Name = name - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, do.Val); err != nil { - return err - } - - delete(m, "constraint") - delete(m, "affinity") - - if err := mapstructure.WeakDecode(m, &r); err != nil { - return err - } - - // Parse constraints - if o := listVal.Filter("constraint"); len(o.Items) > 0 { - if err := parseConstraints(&r.Constraints, o); err != nil { - return multierror.Prefix(err, "constraint ->") - } - } - - // Parse affinities - if o := listVal.Filter("affinity"); len(o.Items) > 0 { - if err := parseAffinities(&r.Affinities, o); err != nil { - return multierror.Prefix(err, "affinity ->") - } - } - - result.Devices[idx] = &r - } - } - - return nil -} - -func parseNetwork(o *ast.ObjectList) (*api.NetworkResource, error) { - if len(o.Items) > 1 { - return nil, fmt.Errorf("only one 'network' resource allowed") - } - - // Check for invalid keys - valid := []string{ - "mode", - "mbits", - "port", - } - if err := helper.CheckHCLKeys(o.Items[0].Val, valid); err != nil { - return nil, multierror.Prefix(err, "network ->") - } - - var r api.NetworkResource - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Items[0].Val); err != nil { - return nil, err - } - if err := mapstructure.WeakDecode(m, &r); err != nil { - return nil, err - } - - var networkObj *ast.ObjectList - if ot, ok := o.Items[0].Val.(*ast.ObjectType); ok { - networkObj = ot.List - } else { - return nil, fmt.Errorf("should be an object") - } - if err := parsePorts(networkObj, &r); err != nil { - return nil, multierror.Prefix(err, "network, ports ->") - } - - return &r, nil -} - -func parsePorts(networkObj *ast.ObjectList, nw *api.NetworkResource) error { - // Check for invalid keys - valid := []string{ - "mbits", - "port", - "mode", - } - if err := helper.CheckHCLKeys(networkObj, valid); err != nil { - return err - } - - portsObjList := networkObj.Filter("port") - knownPortLabels := make(map[string]bool) - for _, port := range portsObjList.Items { - if len(port.Keys) == 0 { - return fmt.Errorf("ports must be named") - } - label := port.Keys[0].Token.Value().(string) - if !reDynamicPorts.MatchString(label) { - return errPortLabel - } - l := strings.ToLower(label) - if knownPortLabels[l] { - return fmt.Errorf("found a port label collision: %s", label) - } - var p map[string]interface{} - var res api.Port - if err := hcl.DecodeObject(&p, port.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(p, &res); err != nil { - return err - } - res.Label = label - if res.Value > 0 { - nw.ReservedPorts = append(nw.ReservedPorts, res) - } else { - nw.DynamicPorts = append(nw.DynamicPorts, res) - } - knownPortLabels[l] = true - } - return nil -} - func parseUpdate(result **api.UpdateStrategy, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) > 1 { @@ -1970,54 +468,6 @@ func parseMigrate(result **api.MigrateStrategy, list *ast.ObjectList) error { return dec.Decode(m) } -func parsePeriodic(result **api.PeriodicConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'periodic' block allowed per job") - } - - // Get our resource object - o := list.Items[0] - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - // Check for invalid keys - valid := []string{ - "enabled", - "cron", - "prohibit_overlap", - "time_zone", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return err - } - - if value, ok := m["enabled"]; ok { - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("periodic.enabled should be set to true or false; %v", err) - } - m["Enabled"] = enabled - } - - // If "cron" is provided, set the type to "cron" and store the spec. - if cron, ok := m["cron"]; ok { - m["SpecType"] = api.PeriodicSpecCron - m["Spec"] = cron - } - - // Build the constraint - var p api.PeriodicConfig - if err := mapstructure.WeakDecode(m, &p); err != nil { - return err - } - *result = &p - return nil -} - func parseVault(result *api.Vault, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) == 0 { @@ -2060,37 +510,3 @@ func parseVault(result *api.Vault, list *ast.ObjectList) error { return nil } - -func parseParameterizedJob(result **api.ParameterizedJobConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'parameterized' block allowed per job") - } - - // Get our resource object - o := list.Items[0] - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - // Check for invalid keys - valid := []string{ - "payload", - "meta_required", - "meta_optional", - } - if err := helper.CheckHCLKeys(o.Val, valid); err != nil { - return err - } - - // Build the parameterized job block - var d api.ParameterizedJobConfig - if err := mapstructure.WeakDecode(m, &d); err != nil { - return err - } - - *result = &d - return nil -} diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go new file mode 100644 index 000000000..222fca62b --- /dev/null +++ b/jobspec/parse_group.go @@ -0,0 +1,276 @@ +package jobspec + +import ( + "fmt" + + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func parseGroups(result *api.Job, list *ast.ObjectList) error { + list = list.Children() + if len(list.Items) == 0 { + return nil + } + + // Go through each object and turn it into an actual result. + collection := make([]*api.TaskGroup, 0, len(list.Items)) + seen := make(map[string]struct{}) + for _, item := range list.Items { + n := item.Keys[0].Token.Value().(string) + + // Make sure we haven't already found this + if _, ok := seen[n]; ok { + return fmt.Errorf("group '%s' defined more than once", n) + } + seen[n] = struct{}{} + + // We need this later + var listVal *ast.ObjectList + if ot, ok := item.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return fmt.Errorf("group '%s': should be an object", n) + } + + // Check for invalid keys + valid := []string{ + "count", + "constraint", + "affinity", + "restart", + "meta", + "task", + "ephemeral_disk", + "update", + "reschedule", + "vault", + "migrate", + "spread", + "network", + "service", + } + if err := helper.CheckHCLKeys(listVal, valid); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s' ->", n)) + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, item.Val); err != nil { + return err + } + delete(m, "constraint") + delete(m, "affinity") + delete(m, "meta") + delete(m, "task") + delete(m, "restart") + delete(m, "ephemeral_disk") + delete(m, "update") + delete(m, "vault") + delete(m, "migrate") + delete(m, "spread") + delete(m, "network") + delete(m, "service") + + // Build the group with the basic decode + var g api.TaskGroup + g.Name = helper.StringToPtr(n) + if err := mapstructure.WeakDecode(m, &g); err != nil { + return err + } + + // Parse constraints + if o := listVal.Filter("constraint"); len(o.Items) > 0 { + if err := parseConstraints(&g.Constraints, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', constraint ->", n)) + } + } + + // Parse affinities + if o := listVal.Filter("affinity"); len(o.Items) > 0 { + if err := parseAffinities(&g.Affinities, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', affinity ->", n)) + } + } + + // Parse restart policy + if o := listVal.Filter("restart"); len(o.Items) > 0 { + if err := parseRestartPolicy(&g.RestartPolicy, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', restart ->", n)) + } + } + + // Parse spread + if o := listVal.Filter("spread"); len(o.Items) > 0 { + if err := parseSpread(&g.Spreads, o); err != nil { + return multierror.Prefix(err, "spread ->") + } + } + + // Parse network + if o := listVal.Filter("network"); len(o.Items) > 0 { + networks, err := parseNetwork(o) + if err != nil { + return err + } + g.Networks = []*api.NetworkResource{networks} + } + + // Parse reschedule policy + if o := listVal.Filter("reschedule"); len(o.Items) > 0 { + if err := parseReschedulePolicy(&g.ReschedulePolicy, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', reschedule ->", n)) + } + } + // Parse ephemeral disk + if o := listVal.Filter("ephemeral_disk"); len(o.Items) > 0 { + g.EphemeralDisk = &api.EphemeralDisk{} + if err := parseEphemeralDisk(&g.EphemeralDisk, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', ephemeral_disk ->", n)) + } + } + + // If we have an update strategy, then parse that + if o := listVal.Filter("update"); len(o.Items) > 0 { + if err := parseUpdate(&g.Update, o); err != nil { + return multierror.Prefix(err, "update ->") + } + } + + // If we have a migration strategy, then parse that + if o := listVal.Filter("migrate"); len(o.Items) > 0 { + if err := parseMigrate(&g.Migrate, o); err != nil { + return multierror.Prefix(err, "migrate ->") + } + } + + // Parse out meta fields. These are in HCL as a list so we need + // to iterate over them and merge them. + if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { + for _, o := range metaO.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + if err := mapstructure.WeakDecode(m, &g.Meta); err != nil { + return err + } + } + } + + // Parse tasks + if o := listVal.Filter("task"); len(o.Items) > 0 { + if err := parseTasks(&g.Tasks, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', task:", n)) + } + } + + // If we have a vault block, then parse that + if o := listVal.Filter("vault"); len(o.Items) > 0 { + tgVault := &api.Vault{ + Env: helper.BoolToPtr(true), + ChangeMode: helper.StringToPtr("restart"), + } + + if err := parseVault(tgVault, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s', vault ->", n)) + } + + // Go through the tasks and if they don't have a Vault block, set it + for _, task := range g.Tasks { + if task.Vault == nil { + task.Vault = tgVault + } + } + } + + if o := listVal.Filter("service"); len(o.Items) > 0 { + if err := parseGroupServices(&g, o); err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s',", n)) + } + } + + collection = append(collection, &g) + } + + result.TaskGroups = append(result.TaskGroups, collection...) + return nil +} + +func parseEphemeralDisk(result **api.EphemeralDisk, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'ephemeral_disk' block allowed") + } + + // Get our ephemeral_disk object + obj := list.Items[0] + + // Check for invalid keys + valid := []string{ + "sticky", + "size", + "migrate", + } + if err := helper.CheckHCLKeys(obj.Val, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, obj.Val); err != nil { + return err + } + + var ephemeralDisk api.EphemeralDisk + if err := mapstructure.WeakDecode(m, &ephemeralDisk); err != nil { + return err + } + *result = &ephemeralDisk + + return nil +} + +func parseRestartPolicy(final **api.RestartPolicy, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'restart' block allowed") + } + + // Get our job object + obj := list.Items[0] + + // Check for invalid keys + valid := []string{ + "attempts", + "interval", + "delay", + "mode", + } + if err := helper.CheckHCLKeys(obj.Val, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, obj.Val); err != nil { + return err + } + + var result api.RestartPolicy + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &result, + }) + if err != nil { + return err + } + if err := dec.Decode(m); err != nil { + return err + } + + *final = &result + return nil +} diff --git a/jobspec/parse_job.go b/jobspec/parse_job.go new file mode 100644 index 000000000..e4f75404b --- /dev/null +++ b/jobspec/parse_job.go @@ -0,0 +1,284 @@ +package jobspec + +import ( + "fmt" + + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func parseJob(result *api.Job, list *ast.ObjectList) error { + if len(list.Items) != 1 { + return fmt.Errorf("only one 'job' block allowed") + } + list = list.Children() + if len(list.Items) != 1 { + return fmt.Errorf("'job' block missing name") + } + + // Get our job object + obj := list.Items[0] + + // Decode the full thing into a map[string]interface for ease + var m map[string]interface{} + if err := hcl.DecodeObject(&m, obj.Val); err != nil { + return err + } + delete(m, "constraint") + delete(m, "affinity") + delete(m, "meta") + delete(m, "migrate") + delete(m, "parameterized") + delete(m, "periodic") + delete(m, "reschedule") + delete(m, "update") + delete(m, "vault") + delete(m, "spread") + + // Set the ID and name to the object key + result.ID = helper.StringToPtr(obj.Keys[0].Token.Value().(string)) + result.Name = helper.StringToPtr(*result.ID) + + // Decode the rest + if err := mapstructure.WeakDecode(m, result); err != nil { + return err + } + + // Value should be an object + var listVal *ast.ObjectList + if ot, ok := obj.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return fmt.Errorf("job '%s' value: should be an object", *result.ID) + } + + // Check for invalid keys + valid := []string{ + "all_at_once", + "constraint", + "affinity", + "spread", + "datacenters", + "group", + "id", + "meta", + "migrate", + "name", + "namespace", + "parameterized", + "periodic", + "priority", + "region", + "reschedule", + "task", + "type", + "update", + "vault", + "vault_token", + } + if err := helper.CheckHCLKeys(listVal, valid); err != nil { + return multierror.Prefix(err, "job:") + } + + // Parse constraints + if o := listVal.Filter("constraint"); len(o.Items) > 0 { + if err := parseConstraints(&result.Constraints, o); err != nil { + return multierror.Prefix(err, "constraint ->") + } + } + + // Parse affinities + if o := listVal.Filter("affinity"); len(o.Items) > 0 { + if err := parseAffinities(&result.Affinities, o); err != nil { + return multierror.Prefix(err, "affinity ->") + } + } + + // If we have an update strategy, then parse that + if o := listVal.Filter("update"); len(o.Items) > 0 { + if err := parseUpdate(&result.Update, o); err != nil { + return multierror.Prefix(err, "update ->") + } + } + + // If we have a periodic definition, then parse that + if o := listVal.Filter("periodic"); len(o.Items) > 0 { + if err := parsePeriodic(&result.Periodic, o); err != nil { + return multierror.Prefix(err, "periodic ->") + } + } + + // Parse spread + if o := listVal.Filter("spread"); len(o.Items) > 0 { + if err := parseSpread(&result.Spreads, o); err != nil { + return multierror.Prefix(err, "spread ->") + } + } + + // If we have a parameterized definition, then parse that + if o := listVal.Filter("parameterized"); len(o.Items) > 0 { + if err := parseParameterizedJob(&result.ParameterizedJob, o); err != nil { + return multierror.Prefix(err, "parameterized ->") + } + } + + // If we have a reschedule stanza, then parse that + if o := listVal.Filter("reschedule"); len(o.Items) > 0 { + if err := parseReschedulePolicy(&result.Reschedule, o); err != nil { + return multierror.Prefix(err, "reschedule ->") + } + } + + // If we have a migration strategy, then parse that + if o := listVal.Filter("migrate"); len(o.Items) > 0 { + if err := parseMigrate(&result.Migrate, o); err != nil { + return multierror.Prefix(err, "migrate ->") + } + } + + // Parse out meta fields. These are in HCL as a list so we need + // to iterate over them and merge them. + if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { + for _, o := range metaO.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + if err := mapstructure.WeakDecode(m, &result.Meta); err != nil { + return err + } + } + } + + // If we have tasks outside, create TaskGroups for them + if o := listVal.Filter("task"); len(o.Items) > 0 { + var tasks []*api.Task + if err := parseTasks(&tasks, o); err != nil { + return multierror.Prefix(err, "task:") + } + + result.TaskGroups = make([]*api.TaskGroup, len(tasks), len(tasks)*2) + for i, t := range tasks { + result.TaskGroups[i] = &api.TaskGroup{ + Name: helper.StringToPtr(t.Name), + Tasks: []*api.Task{t}, + } + } + } + + // Parse the task groups + if o := listVal.Filter("group"); len(o.Items) > 0 { + if err := parseGroups(result, o); err != nil { + return multierror.Prefix(err, "group:") + } + } + + // If we have a vault block, then parse that + if o := listVal.Filter("vault"); len(o.Items) > 0 { + jobVault := &api.Vault{ + Env: helper.BoolToPtr(true), + ChangeMode: helper.StringToPtr("restart"), + } + + if err := parseVault(jobVault, o); err != nil { + return multierror.Prefix(err, "vault ->") + } + + // Go through the task groups/tasks and if they don't have a Vault block, set it + for _, tg := range result.TaskGroups { + for _, task := range tg.Tasks { + if task.Vault == nil { + task.Vault = jobVault + } + } + } + } + + return nil +} + +func parsePeriodic(result **api.PeriodicConfig, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'periodic' block allowed per job") + } + + // Get our resource object + o := list.Items[0] + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + // Check for invalid keys + valid := []string{ + "enabled", + "cron", + "prohibit_overlap", + "time_zone", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + if value, ok := m["enabled"]; ok { + enabled, err := parseBool(value) + if err != nil { + return fmt.Errorf("periodic.enabled should be set to true or false; %v", err) + } + m["Enabled"] = enabled + } + + // If "cron" is provided, set the type to "cron" and store the spec. + if cron, ok := m["cron"]; ok { + m["SpecType"] = api.PeriodicSpecCron + m["Spec"] = cron + } + + // Build the constraint + var p api.PeriodicConfig + if err := mapstructure.WeakDecode(m, &p); err != nil { + return err + } + *result = &p + return nil +} + +func parseParameterizedJob(result **api.ParameterizedJobConfig, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'parameterized' block allowed per job") + } + + // Get our resource object + o := list.Items[0] + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + // Check for invalid keys + valid := []string{ + "payload", + "meta_required", + "meta_optional", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + // Build the parameterized job block + var d api.ParameterizedJobConfig + if err := mapstructure.WeakDecode(m, &d); err != nil { + return err + } + + *result = &d + return nil +} diff --git a/jobspec/parse_network.go b/jobspec/parse_network.go new file mode 100644 index 000000000..b47844540 --- /dev/null +++ b/jobspec/parse_network.go @@ -0,0 +1,94 @@ +package jobspec + +import ( + "fmt" + "strings" + + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func parseNetwork(o *ast.ObjectList) (*api.NetworkResource, error) { + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one 'network' resource allowed") + } + + // Check for invalid keys + valid := []string{ + "mode", + "mbits", + "port", + } + if err := helper.CheckHCLKeys(o.Items[0].Val, valid); err != nil { + return nil, multierror.Prefix(err, "network ->") + } + + var r api.NetworkResource + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Items[0].Val); err != nil { + return nil, err + } + if err := mapstructure.WeakDecode(m, &r); err != nil { + return nil, err + } + + var networkObj *ast.ObjectList + if ot, ok := o.Items[0].Val.(*ast.ObjectType); ok { + networkObj = ot.List + } else { + return nil, fmt.Errorf("should be an object") + } + if err := parsePorts(networkObj, &r); err != nil { + return nil, multierror.Prefix(err, "network, ports ->") + } + + return &r, nil +} + +func parsePorts(networkObj *ast.ObjectList, nw *api.NetworkResource) error { + // Check for invalid keys + valid := []string{ + "mbits", + "port", + "mode", + } + if err := helper.CheckHCLKeys(networkObj, valid); err != nil { + return err + } + + portsObjList := networkObj.Filter("port") + knownPortLabels := make(map[string]bool) + for _, port := range portsObjList.Items { + if len(port.Keys) == 0 { + return fmt.Errorf("ports must be named") + } + label := port.Keys[0].Token.Value().(string) + if !reDynamicPorts.MatchString(label) { + return errPortLabel + } + l := strings.ToLower(label) + if knownPortLabels[l] { + return fmt.Errorf("found a port label collision: %s", label) + } + var p map[string]interface{} + var res api.Port + if err := hcl.DecodeObject(&p, port.Val); err != nil { + return err + } + if err := mapstructure.WeakDecode(p, &res); err != nil { + return err + } + res.Label = label + if res.Value > 0 { + nw.ReservedPorts = append(nw.ReservedPorts, res) + } else { + nw.DynamicPorts = append(nw.DynamicPorts, res) + } + knownPortLabels[l] = true + } + return nil +} diff --git a/jobspec/parse_service.go b/jobspec/parse_service.go new file mode 100644 index 000000000..d2a086b6d --- /dev/null +++ b/jobspec/parse_service.go @@ -0,0 +1,450 @@ +package jobspec + +import ( + "fmt" + + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func parseGroupServices(g *api.TaskGroup, serviceObjs *ast.ObjectList) error { + g.Services = make([]*api.Service, len(serviceObjs.Items)) + for idx, o := range serviceObjs.Items { + service, err := parseService(o) + if err != nil { + return multierror.Prefix(err, fmt.Sprintf("service (%d):", idx)) + } + g.Services[idx] = service + } + + return nil +} + +func parseServices(serviceObjs *ast.ObjectList) ([]*api.Service, error) { + services := make([]*api.Service, len(serviceObjs.Items)) + for idx, o := range serviceObjs.Items { + service, err := parseService(o) + if err != nil { + return nil, multierror.Prefix(err, fmt.Sprintf("service (%d):", idx)) + } + services[idx] = service + } + return services, nil +} +func parseService(o *ast.ObjectItem) (*api.Service, error) { + // Check for invalid keys + valid := []string{ + "name", + "tags", + "canary_tags", + "port", + "check", + "address_mode", + "check_restart", + "connect", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, err + } + + var service api.Service + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "check") + delete(m, "check_restart") + delete(m, "connect") + + if err := mapstructure.WeakDecode(m, &service); err != nil { + return nil, err + } + + // Filter checks + var checkList *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + checkList = ot.List + } else { + return nil, fmt.Errorf("'%s': should be an object", service.Name) + } + + if co := checkList.Filter("check"); len(co.Items) > 0 { + if err := parseChecks(&service, co); err != nil { + return nil, multierror.Prefix(err, fmt.Sprintf("'%s',", service.Name)) + } + } + + // Filter check_restart + if cro := checkList.Filter("check_restart"); len(cro.Items) > 0 { + if len(cro.Items) > 1 { + return nil, fmt.Errorf("check_restart '%s': cannot have more than 1 check_restart", service.Name) + } + cr, err := parseCheckRestart(cro.Items[0]) + if err != nil { + return nil, multierror.Prefix(err, fmt.Sprintf("'%s',", service.Name)) + } + service.CheckRestart = cr + + } + + // Filter connect + if co := checkList.Filter("connect"); len(co.Items) > 0 { + if len(co.Items) > 1 { + return nil, fmt.Errorf("connect '%s': cannot have more than 1 connect stanza", service.Name) + } + + c, err := parseConnect(co.Items[0]) + if err != nil { + return nil, multierror.Prefix(err, fmt.Sprintf("'%s',", service.Name)) + } + + service.Connect = c + } + + return &service, nil +} + +func parseConnect(co *ast.ObjectItem) (*api.ConsulConnect, error) { + valid := []string{ + "native", + "sidecar_service", + "sidecar_task", + } + + if err := helper.CheckHCLKeys(co.Val, valid); err != nil { + return nil, multierror.Prefix(err, "connect ->") + } + + var connect api.ConsulConnect + var m map[string]interface{} + if err := hcl.DecodeObject(&m, co.Val); err != nil { + return nil, err + } + + delete(m, "sidecar_service") + delete(m, "sidecar_task") + + if err := mapstructure.WeakDecode(m, &connect); err != nil { + return nil, err + } + + var connectList *ast.ObjectList + if ot, ok := co.Val.(*ast.ObjectType); ok { + connectList = ot.List + } else { + return nil, fmt.Errorf("connect should be an object") + } + + // Parse the sidecar_service + o := connectList.Filter("sidecar_service") + if len(o.Items) == 0 { + return &connect, nil + } + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one 'sidecar_service' block allowed per task") + } + + r, err := parseSidecarService(o.Items[0]) + if err != nil { + return nil, fmt.Errorf("sidecar_service, %v", err) + } + connect.SidecarService = r + + // Parse the sidecar_task + o = connectList.Filter("sidecar_task") + if len(o.Items) == 0 { + return &connect, nil + } + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one 'sidecar_task' block allowed per task") + } + + t, err := parseTask(o.Items[0]) + if err != nil { + return nil, fmt.Errorf("sidecar_task, %v", err) + } + connect.SidecarTask = t + + return &connect, nil +} + +func parseSidecarService(o *ast.ObjectItem) (*api.ConsulSidecarService, error) { + valid := []string{ + "port", + "proxy", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "sidecar_service ->") + } + + var sidecar api.ConsulSidecarService + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + delete(m, "proxy") + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &sidecar, + }) + if err != nil { + return nil, err + } + if err := dec.Decode(m); err != nil { + return nil, fmt.Errorf("foo: %v", err) + } + + var proxyList *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + proxyList = ot.List + } else { + return nil, fmt.Errorf("sidecar_service: should be an object") + } + + // Parse the proxy + po := proxyList.Filter("proxy") + if len(po.Items) == 0 { + return &sidecar, nil + } + if len(po.Items) > 1 { + return nil, fmt.Errorf("only one 'proxy' block allowed per task") + } + + r, err := parseProxy(po.Items[0]) + if err != nil { + return nil, fmt.Errorf("proxy, %v", err) + } + sidecar.Proxy = r + + return &sidecar, nil +} + +func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) { + valid := []string{ + "upstreams", + "config", + } + + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, multierror.Prefix(err, "proxy ->") + } + + var proxy api.ConsulProxy + + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("proxy: should be an object") + } + + // Parse the proxy + uo := listVal.Filter("upstreams") + proxy.Upstreams = make([]*api.ConsulUpstream, len(uo.Items)) + for i := range uo.Items { + u, err := parseUpstream(uo.Items[i]) + if err != nil { + return nil, err + } + + proxy.Upstreams[i] = u + } + + // If we have config, then parse that + if o := listVal.Filter("config"); len(o.Items) > 1 { + return nil, fmt.Errorf("only 1 meta object supported") + } else if len(o.Items) == 1 { + var mSlice []map[string]interface{} + if err := hcl.DecodeObject(&mSlice, o.Items[0].Val); err != nil { + return nil, err + } + + if len(mSlice) > 1 { + return nil, fmt.Errorf("only 1 meta object supported") + } + + m := mSlice[0] + + if err := mapstructure.WeakDecode(m, &proxy.Config); err != nil { + return nil, err + } + + proxy.Config = flattenMapSlice(proxy.Config) + } + + return &proxy, nil +} + +func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) { + valid := []string{ + "destination_name", + "local_bind_port", + } + + if err := helper.CheckHCLKeys(uo.Val, valid); err != nil { + return nil, multierror.Prefix(err, "upstream ->") + } + + var upstream api.ConsulUpstream + var m map[string]interface{} + if err := hcl.DecodeObject(&m, uo.Val); err != nil { + return nil, err + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &upstream, + }) + if err != nil { + return nil, err + } + + if err := dec.Decode(m); err != nil { + return nil, err + } + + return &upstream, nil +} +func parseChecks(service *api.Service, checkObjs *ast.ObjectList) error { + service.Checks = make([]api.ServiceCheck, len(checkObjs.Items)) + for idx, co := range checkObjs.Items { + // Check for invalid keys + valid := []string{ + "name", + "type", + "interval", + "timeout", + "path", + "protocol", + "port", + "command", + "args", + "initial_status", + "tls_skip_verify", + "header", + "method", + "check_restart", + "address_mode", + "grpc_service", + "grpc_use_tls", + } + if err := helper.CheckHCLKeys(co.Val, valid); err != nil { + return multierror.Prefix(err, "check ->") + } + + var check api.ServiceCheck + var cm map[string]interface{} + if err := hcl.DecodeObject(&cm, co.Val); err != nil { + return err + } + + // HCL allows repeating stanzas so merge 'header' into a single + // map[string][]string. + if headerI, ok := cm["header"]; ok { + headerRaw, ok := headerI.([]map[string]interface{}) + if !ok { + return fmt.Errorf("check -> header -> expected a []map[string][]string but found %T", headerI) + } + m := map[string][]string{} + for _, rawm := range headerRaw { + for k, vI := range rawm { + vs, ok := vI.([]interface{}) + if !ok { + return fmt.Errorf("check -> header -> %q expected a []string but found %T", k, vI) + } + for _, vI := range vs { + v, ok := vI.(string) + if !ok { + return fmt.Errorf("check -> header -> %q expected a string but found %T", k, vI) + } + m[k] = append(m[k], v) + } + } + } + + check.Header = m + + // Remove "header" as it has been parsed + delete(cm, "header") + } + + delete(cm, "check_restart") + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &check, + }) + if err != nil { + return err + } + if err := dec.Decode(cm); err != nil { + return err + } + + // Filter check_restart + var checkRestartList *ast.ObjectList + if ot, ok := co.Val.(*ast.ObjectType); ok { + checkRestartList = ot.List + } else { + return fmt.Errorf("check_restart '%s': should be an object", check.Name) + } + + if cro := checkRestartList.Filter("check_restart"); len(cro.Items) > 0 { + if len(cro.Items) > 1 { + return fmt.Errorf("check_restart '%s': cannot have more than 1 check_restart", check.Name) + } + cr, err := parseCheckRestart(cro.Items[0]) + if err != nil { + return multierror.Prefix(err, fmt.Sprintf("check: '%s',", check.Name)) + } + check.CheckRestart = cr + } + + service.Checks[idx] = check + } + + return nil +} + +func parseCheckRestart(cro *ast.ObjectItem) (*api.CheckRestart, error) { + valid := []string{ + "limit", + "grace", + "ignore_warnings", + } + + if err := helper.CheckHCLKeys(cro.Val, valid); err != nil { + return nil, multierror.Prefix(err, "check_restart ->") + } + + var checkRestart api.CheckRestart + var crm map[string]interface{} + if err := hcl.DecodeObject(&crm, cro.Val); err != nil { + return nil, err + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &checkRestart, + }) + if err != nil { + return nil, err + } + if err := dec.Decode(crm); err != nil { + return nil, err + } + + return &checkRestart, nil +} diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go new file mode 100644 index 000000000..425f95365 --- /dev/null +++ b/jobspec/parse_task.go @@ -0,0 +1,506 @@ +package jobspec + +import ( + "fmt" + "time" + + multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" +) + +func parseTasks(result *[]*api.Task, list *ast.ObjectList) error { + list = list.Children() + if len(list.Items) == 0 { + return nil + } + + // Go through each object and turn it into an actual result. + seen := make(map[string]struct{}) + for _, item := range list.Items { + n := item.Keys[0].Token.Value().(string) + + // Make sure we haven't already found this + if _, ok := seen[n]; ok { + return fmt.Errorf("task '%s' defined more than once", n) + } + seen[n] = struct{}{} + + t, err := parseTask(item) + if err != nil { + return multierror.Prefix(err, fmt.Sprintf("'%s',", n)) + } + + t.Name = n + + *result = append(*result, t) + } + + return nil +} + +func parseTask(item *ast.ObjectItem) (*api.Task, error) { + // We need this later + var listVal *ast.ObjectList + if ot, ok := item.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return nil, fmt.Errorf("should be an object") + } + + // Check for invalid keys + valid := []string{ + "artifact", + "config", + "constraint", + "affinity", + "dispatch_payload", + "driver", + "env", + "kill_timeout", + "leader", + "logs", + "meta", + "resources", + "service", + "shutdown_delay", + "template", + "user", + "vault", + "kill_signal", + "kind", + } + if err := helper.CheckHCLKeys(listVal, valid); err != nil { + return nil, err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, item.Val); err != nil { + return nil, err + } + delete(m, "artifact") + delete(m, "config") + delete(m, "constraint") + delete(m, "affinity") + delete(m, "dispatch_payload") + delete(m, "env") + delete(m, "logs") + delete(m, "meta") + delete(m, "resources") + delete(m, "service") + delete(m, "template") + delete(m, "vault") + + // Build the task + var t api.Task + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: &t, + }) + + if err != nil { + return nil, err + } + if err := dec.Decode(m); err != nil { + return nil, err + } + + // If we have env, then parse them + if o := listVal.Filter("env"); len(o.Items) > 0 { + for _, o := range o.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + if err := mapstructure.WeakDecode(m, &t.Env); err != nil { + return nil, err + } + } + } + + if o := listVal.Filter("service"); len(o.Items) > 0 { + services, err := parseServices(o) + if err != nil { + return nil, err + } + + t.Services = services + } + + // If we have config, then parse that + if o := listVal.Filter("config"); len(o.Items) > 0 { + for _, o := range o.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + + if err := mapstructure.WeakDecode(m, &t.Config); err != nil { + return nil, err + } + } + } + + // Parse constraints + if o := listVal.Filter("constraint"); len(o.Items) > 0 { + if err := parseConstraints(&t.Constraints, o); err != nil { + return nil, multierror.Prefix(err, "constraint ->") + } + } + + // Parse affinities + if o := listVal.Filter("affinity"); len(o.Items) > 0 { + if err := parseAffinities(&t.Affinities, o); err != nil { + return nil, multierror.Prefix(err, "affinity ->") + } + } + + // Parse out meta fields. These are in HCL as a list so we need + // to iterate over them and merge them. + if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { + for _, o := range metaO.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return nil, err + } + if err := mapstructure.WeakDecode(m, &t.Meta); err != nil { + return nil, err + } + } + } + + // If we have resources, then parse that + if o := listVal.Filter("resources"); len(o.Items) > 0 { + var r api.Resources + if err := parseResources(&r, o); err != nil { + return nil, multierror.Prefix(err, "resources ->") + } + + t.Resources = &r + } + + // If we have logs then parse that + if o := listVal.Filter("logs"); len(o.Items) > 0 { + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one logs block is allowed in a Task. Number of logs block found: %d", len(o.Items)) + } + var m map[string]interface{} + logsBlock := o.Items[0] + + // Check for invalid keys + valid := []string{ + "max_files", + "max_file_size", + } + if err := helper.CheckHCLKeys(logsBlock.Val, valid); err != nil { + return nil, multierror.Prefix(err, "logs ->") + } + + if err := hcl.DecodeObject(&m, logsBlock.Val); err != nil { + return nil, err + } + + var log api.LogConfig + if err := mapstructure.WeakDecode(m, &log); err != nil { + return nil, err + } + + t.LogConfig = &log + } + + // Parse artifacts + if o := listVal.Filter("artifact"); len(o.Items) > 0 { + if err := parseArtifacts(&t.Artifacts, o); err != nil { + return nil, multierror.Prefix(err, "artifact ->") + } + } + + // Parse templates + if o := listVal.Filter("template"); len(o.Items) > 0 { + if err := parseTemplates(&t.Templates, o); err != nil { + return nil, multierror.Prefix(err, "template ->") + } + } + + // If we have a vault block, then parse that + if o := listVal.Filter("vault"); len(o.Items) > 0 { + v := &api.Vault{ + Env: helper.BoolToPtr(true), + ChangeMode: helper.StringToPtr("restart"), + } + + if err := parseVault(v, o); err != nil { + return nil, multierror.Prefix(err, "vault ->") + } + + t.Vault = v + } + + // If we have a dispatch_payload block parse that + if o := listVal.Filter("dispatch_payload"); len(o.Items) > 0 { + if len(o.Items) > 1 { + return nil, fmt.Errorf("only one dispatch_payload block is allowed in a task. Number of dispatch_payload blocks found: %d", len(o.Items)) + } + var m map[string]interface{} + dispatchBlock := o.Items[0] + + // Check for invalid keys + valid := []string{ + "file", + } + if err := helper.CheckHCLKeys(dispatchBlock.Val, valid); err != nil { + return nil, multierror.Prefix(err, "dispatch_payload ->") + } + + if err := hcl.DecodeObject(&m, dispatchBlock.Val); err != nil { + return nil, err + } + + t.DispatchPayload = &api.DispatchPayloadConfig{} + if err := mapstructure.WeakDecode(m, t.DispatchPayload); err != nil { + return nil, err + } + } + + return &t, nil +} + +func parseArtifacts(result *[]*api.TaskArtifact, list *ast.ObjectList) error { + for _, o := range list.Elem().Items { + // Check for invalid keys + valid := []string{ + "source", + "options", + "mode", + "destination", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + delete(m, "options") + + var ta api.TaskArtifact + if err := mapstructure.WeakDecode(m, &ta); err != nil { + return err + } + + var optionList *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + optionList = ot.List + } else { + return fmt.Errorf("artifact should be an object") + } + + if oo := optionList.Filter("options"); len(oo.Items) > 0 { + options := make(map[string]string) + if err := parseArtifactOption(options, oo); err != nil { + return multierror.Prefix(err, "options: ") + } + ta.GetterOptions = options + } + + *result = append(*result, &ta) + } + + return nil +} + +func parseArtifactOption(result map[string]string, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'options' block allowed per artifact") + } + + // Get our resource object + o := list.Items[0] + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + if err := mapstructure.WeakDecode(m, &result); err != nil { + return err + } + + return nil +} + +func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error { + for _, o := range list.Elem().Items { + // Check for invalid keys + valid := []string{ + "change_mode", + "change_signal", + "data", + "destination", + "left_delimiter", + "perms", + "right_delimiter", + "source", + "splay", + "env", + "vault_grace", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + templ := &api.Template{ + ChangeMode: helper.StringToPtr("restart"), + Splay: helper.TimeToPtr(5 * time.Second), + Perms: helper.StringToPtr("0644"), + } + + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + WeaklyTypedInput: true, + Result: templ, + }) + if err != nil { + return err + } + if err := dec.Decode(m); err != nil { + return err + } + + *result = append(*result, templ) + } + + return nil +} + +func parseResources(result *api.Resources, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) == 0 { + return nil + } + if len(list.Items) > 1 { + return fmt.Errorf("only one 'resource' block allowed per task") + } + + // Get our resource object + o := list.Items[0] + + // We need this later + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return fmt.Errorf("resource: should be an object") + } + + // Check for invalid keys + valid := []string{ + "cpu", + "iops", // COMPAT(0.10): Remove after one release to allow it to be removed from jobspecs + "disk", + "memory", + "network", + "device", + } + if err := helper.CheckHCLKeys(listVal, valid); err != nil { + return multierror.Prefix(err, "resources ->") + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + delete(m, "network") + delete(m, "device") + + if err := mapstructure.WeakDecode(m, result); err != nil { + return err + } + + // Parse the network resources + if o := listVal.Filter("network"); len(o.Items) > 0 { + r, err := parseNetwork(o) + if err != nil { + return fmt.Errorf("resource, %v", err) + } + result.Networks = []*api.NetworkResource{r} + } + + // Parse the device resources + if o := listVal.Filter("device"); len(o.Items) > 0 { + result.Devices = make([]*api.RequestedDevice, len(o.Items)) + for idx, do := range o.Items { + if l := len(do.Keys); l == 0 { + return multierror.Prefix(fmt.Errorf("missing device name"), fmt.Sprintf("resources, device[%d]->", idx)) + } else if l > 1 { + return multierror.Prefix(fmt.Errorf("only one name may be specified"), fmt.Sprintf("resources, device[%d]->", idx)) + } + name := do.Keys[0].Token.Value().(string) + + // Value should be an object + var listVal *ast.ObjectList + if ot, ok := do.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return fmt.Errorf("device should be an object") + } + + // Check for invalid keys + valid := []string{ + "name", + "count", + "affinity", + "constraint", + } + if err := helper.CheckHCLKeys(do.Val, valid); err != nil { + return multierror.Prefix(err, fmt.Sprintf("resources, device[%d]->", idx)) + } + + // Set the name + var r api.RequestedDevice + r.Name = name + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, do.Val); err != nil { + return err + } + + delete(m, "constraint") + delete(m, "affinity") + + if err := mapstructure.WeakDecode(m, &r); err != nil { + return err + } + + // Parse constraints + if o := listVal.Filter("constraint"); len(o.Items) > 0 { + if err := parseConstraints(&r.Constraints, o); err != nil { + return multierror.Prefix(err, "constraint ->") + } + } + + // Parse affinities + if o := listVal.Filter("affinity"); len(o.Items) > 0 { + if err := parseAffinities(&r.Affinities, o); err != nil { + return multierror.Prefix(err, "affinity ->") + } + } + + result.Devices[idx] = &r + } + } + + return nil +} diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 0f10e7fd0..5e7e898c2 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -911,6 +911,15 @@ func TestParse(t *testing.T) { }, }, }, + SidecarTask: &api.Task{ + Resources: &api.Resources{ + CPU: helper.IntToPtr(500), + MemoryMB: helper.IntToPtr(1024), + }, + Env: map[string]string{ + "FOO": "abc", + }, + }, }, }, }, @@ -1004,7 +1013,7 @@ func TestIncorrectKey(t *testing.T) { t.Fatalf("Expected an error") } - if !strings.Contains(err.Error(), "* group: 'binsl', task: 'binstore', service: 'foo', check -> invalid key: nterval") { + if !strings.Contains(err.Error(), "* group: 'binsl', task: 'binstore', service (0): 'foo', check -> invalid key: nterval") { t.Fatalf("Expected key error; got %v", err) } } diff --git a/jobspec/test-fixtures/tg-network.hcl b/jobspec/test-fixtures/tg-network.hcl index 0abd67c95..25e3f6b02 100644 --- a/jobspec/test-fixtures/tg-network.hcl +++ b/jobspec/test-fixtures/tg-network.hcl @@ -9,7 +9,7 @@ job "foo" { to = 8080 } } - + service { name = "connect-service" tags = ["foo", "bar"] @@ -25,6 +25,16 @@ job "foo" { } } } + sidecar_task { + resources { + cpu = 500 + memory = 1024 + } + + env { + FOO = "abc" + } + } } } diff --git a/nomad/structs/services.go b/nomad/structs/services.go index abb53dddc..1e214de05 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -515,6 +515,9 @@ type ConsulConnect struct { // SidecarService is non-nil if a service requires a sidecar. SidecarService *ConsulSidecarService + + // SidecarTask is non-nil if sidecar overrides are set + SidecarTask *Task } // Copy the stanza recursively. Returns nil if nil. @@ -526,6 +529,7 @@ func (c *ConsulConnect) Copy() *ConsulConnect { return &ConsulConnect{ Native: c.Native, SidecarService: c.SidecarService.Copy(), + SidecarTask: c.SidecarTask.Copy(), } } @@ -595,7 +599,7 @@ func (s *ConsulSidecarService) Equals(o *ConsulSidecarService) bool { type ConsulProxy struct { // Upstreams configures the upstream services this service intends to // connect to. - Upstreams []*ConsulUpstream + Upstreams []ConsulUpstream // Config is a proxy configuration. It is opaque to Nomad and passed // directly to Consul. @@ -611,10 +615,10 @@ func (p *ConsulProxy) Copy() *ConsulProxy { newP := ConsulProxy{} if n := len(p.Upstreams); n > 0 { - newP.Upstreams = make([]*ConsulUpstream, n) + newP.Upstreams = make([]ConsulUpstream, n) for i := range p.Upstreams { - newP.Upstreams[i] = p.Upstreams[i].Copy() + newP.Upstreams[i] = *p.Upstreams[i].Copy() } } @@ -643,7 +647,7 @@ func (p *ConsulProxy) Equals(o *ConsulProxy) bool { OUTER: for _, up := range p.Upstreams { for _, innerUp := range o.Upstreams { - if up.Equals(innerUp) { + if up.Equals(&innerUp) { // Match; find next upstream continue OUTER } diff --git a/nomad/structs/services_test.go b/nomad/structs/services_test.go index b1f265999..00f27cd87 100644 --- a/nomad/structs/services_test.go +++ b/nomad/structs/services_test.go @@ -34,7 +34,7 @@ func TestConsulConnect_CopyEquals(t *testing.T) { SidecarService: &ConsulSidecarService{ Port: "9001", Proxy: &ConsulProxy{ - Upstreams: []*ConsulUpstream{ + Upstreams: []ConsulUpstream{ { DestinationName: "up1", LocalBindPort: 9002,