From 4182e3e141e555b1faeebed5f86b40ceb789d13c Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 14 Nov 2018 11:13:52 -0500 Subject: [PATCH 1/6] nomad: add flag to disable publishing of job_summary metrics for dispatched jobs --- command/agent/agent.go | 1 + command/agent/command.go | 9 ++ command/agent/config.go | 35 ++++++ command/agent/config_parse.go | 2 + command/agent/config_test.go | 44 +++++++ nomad/config.go | 4 + nomad/leader.go | 119 +++++++++--------- nomad/state/state_store.go | 8 +- nomad/structs/structs.go | 3 + .../docs/configuration/telemetry.html.md | 19 +++ 10 files changed, 185 insertions(+), 59 deletions(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 0dc02044c..50261a4d2 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -325,6 +325,7 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { // Setup telemetry related config conf.StatsCollectionInterval = agentConfig.Telemetry.collectionInterval conf.DisableTaggedMetrics = agentConfig.Telemetry.DisableTaggedMetrics + conf.DisableDispatchedJobSummaryMetrics = agentConfig.Telemetry.DisableDispatchedJobSummaryMetrics conf.BackwardsCompatibleMetrics = agentConfig.Telemetry.BackwardsCompatibleMetrics return conf, nil diff --git a/command/agent/command.go b/command/agent/command.go index 4ff44e4d4..fc910bc56 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -880,6 +880,14 @@ func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) { metricsConf.EnableHostname = true } + allowedPrefixes, blockedPrefixes, err := telConfig.PrefixFilters() + if err != nil { + return inm, err + } + + metricsConf.AllowedPrefixes = allowedPrefixes + metricsConf.BlockedPrefixes = blockedPrefixes + // Configure the statsite sink var fanout metrics.FanoutSink if telConfig.StatsiteAddr != "" { @@ -963,6 +971,7 @@ func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) { metricsConf.EnableHostname = false metrics.NewGlobal(metricsConf, inm) } + return inm, nil } diff --git a/command/agent/config.go b/command/agent/config.go index 818ff430c..e75013224 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -441,6 +441,15 @@ type Telemetry struct { // key/value structure as done in older versions of Nomad BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"` + // PrefixFilter allows for filtering out metrics from being collected + PrefixFilter []string `mapstructure:"prefix_filter"` + + // DisableDispatchedJobSummaryMetrics allows for ignore dispatched jobs when + // publishing Job summary metrics. This is useful in environment that produce + // high numbers of single count dispatch jobs as the metrics for each take up + // a small memory overhead. + DisableDispatchedJobSummaryMetrics bool `mapstructure:"disable_dispatched_job_summary_metrics"` + // Circonus: see https://github.com/circonus-labs/circonus-gometrics // for more details on the various configuration options. // Valid configuration combinations: @@ -513,6 +522,24 @@ type Telemetry struct { CirconusBrokerSelectTag string `mapstructure:"circonus_broker_select_tag"` } +// PrefixFilters parses the PrefixFilter field and returns a list of allowed and blocked filters +func (t *Telemetry) PrefixFilters() (allowed, blocked []string, err error) { + for _, rule := range t.PrefixFilter { + if rule == "" { + continue + } + switch rule[0] { + case '+': + allowed = append(allowed, rule[1:]) + case '-': + blocked = append(blocked, rule[1:]) + default: + return nil, nil, fmt.Errorf("Filter rule must begin with either '+' or '-': %q", rule) + } + } + return allowed, blocked, nil +} + // Ports encapsulates the various ports we bind to for network services. If any // are not specified then the defaults are used instead. type Ports struct { @@ -1289,6 +1316,14 @@ func (a *Telemetry) Merge(b *Telemetry) *Telemetry { result.BackwardsCompatibleMetrics = b.BackwardsCompatibleMetrics } + if b.PrefixFilter != nil { + result.PrefixFilter = b.PrefixFilter + } + + if b.DisableDispatchedJobSummaryMetrics { + result.DisableDispatchedJobSummaryMetrics = b.DisableDispatchedJobSummaryMetrics + } + return &result } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index cfbd32723..c7c9aa3d4 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -734,6 +734,8 @@ func parseTelemetry(result **Telemetry, list *ast.ObjectList) error { "circonus_broker_select_tag", "disable_tagged_metrics", "backwards_compatible_metrics", + "prefix_filter", + "disable_dispatched_job_summary_metrics", } if err := helper.CheckHCLKeys(listVal, valid); err != nil { return err diff --git a/command/agent/config_test.go b/command/agent/config_test.go index ca3681415..baf9b7f22 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -77,6 +77,7 @@ func TestConfig_Merge(t *testing.T) { CirconusCheckTags: "cat1:tag1,cat2:tag2", CirconusBrokerID: "0", CirconusBrokerSelectTag: "dc:dc1", + PrefixFilter: []string{"filter1", "filter2"}, }, Client: &ClientConfig{ Enabled: false, @@ -225,6 +226,7 @@ func TestConfig_Merge(t *testing.T) { CirconusCheckTags: "cat1:tag1,cat2:tag2", CirconusBrokerID: "1", CirconusBrokerSelectTag: "dc:dc2", + PrefixFilter: []string{"prefix1", "prefix2"}, }, Client: &ClientConfig{ Enabled: true, @@ -990,3 +992,45 @@ func TestMergeServerJoin(t *testing.T) { require.Equal(result.RetryInterval, retryInterval) } } + +func TestTelemetry_PrefixFilters(t *testing.T) { + t.Parallel() + cases := []struct { + in []string + expAllow []string + expBlock []string + expErr bool + }{ + { + in: []string{"+foo"}, + expAllow: []string{"foo"}, + }, + { + in: []string{"-foo"}, + expBlock: []string{"foo"}, + }, + { + in: []string{"+a.b.c", "-x.y.z"}, + expAllow: []string{"a.b.c"}, + expBlock: []string{"x.y.z"}, + }, + { + in: []string{"+foo", "bad", "-bar"}, + expErr: true, + }, + } + + for i, c := range cases { + t.Run(fmt.Sprintf("PrefixCase%d", i), func(t *testing.T) { + require := require.New(t) + tel := &Telemetry{ + PrefixFilter: c.in, + } + + allow, block, err := tel.PrefixFilters() + require.Exactly(c.expAllow, allow) + require.Exactly(c.expBlock, block) + require.Equal(c.expErr, err != nil) + }) + } +} diff --git a/nomad/config.go b/nomad/config.go index c712e5833..d67241757 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -280,6 +280,10 @@ type Config struct { // key/value/tag format, or simply a key/value format DisableTaggedMetrics bool + // DisableDispatchedJobSummaryMetrics allows for ignore dispatched jobs when + // publishing Job summary metrics + DisableDispatchedJobSummaryMetrics bool + // BackwardsCompatibleMetrics determines whether to show methods of // displaying metrics for older versions, or to only show the new format BackwardsCompatibleMetrics bool diff --git a/nomad/leader.go b/nomad/leader.go index 3d6759d69..3eebb7b52 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -624,68 +624,75 @@ func (s *Server) publishJobSummaryMetrics(stopCh chan struct{}) { break } summary := raw.(*structs.JobSummary) - for name, tgSummary := range summary.Summary { - if !s.config.DisableTaggedMetrics { - labels := []metrics.Label{ - { - Name: "job", - Value: summary.JobID, - }, - { - Name: "task_group", - Value: name, - }, - } - - if strings.Contains(summary.JobID, "/dispatch-") { - jobInfo := strings.Split(summary.JobID, "/dispatch-") - labels = append(labels, metrics.Label{ - Name: "parent_id", - Value: jobInfo[0], - }, metrics.Label{ - Name: "dispatch_id", - Value: jobInfo[1], - }) - } - - if strings.Contains(summary.JobID, "/periodic-") { - jobInfo := strings.Split(summary.JobID, "/periodic-") - labels = append(labels, metrics.Label{ - Name: "parent_id", - Value: jobInfo[0], - }, metrics.Label{ - Name: "periodic_id", - Value: jobInfo[1], - }) - } - - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "queued"}, - float32(tgSummary.Queued), labels) - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "complete"}, - float32(tgSummary.Complete), labels) - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "failed"}, - float32(tgSummary.Failed), labels) - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "running"}, - float32(tgSummary.Running), labels) - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "starting"}, - float32(tgSummary.Starting), labels) - metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "lost"}, - float32(tgSummary.Lost), labels) - } - if s.config.BackwardsCompatibleMetrics { - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "queued"}, float32(tgSummary.Queued)) - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "complete"}, float32(tgSummary.Complete)) - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "failed"}, float32(tgSummary.Failed)) - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "running"}, float32(tgSummary.Running)) - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "starting"}, float32(tgSummary.Starting)) - metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "lost"}, float32(tgSummary.Lost)) - } + if s.config.DisableDispatchedJobSummaryMetrics && summary.Dispatched { + continue } + s.iterateJobSummaryMetrics(summary) } } } } +func (s *Server) iterateJobSummaryMetrics(summary *structs.JobSummary) { + for name, tgSummary := range summary.Summary { + if !s.config.DisableTaggedMetrics { + labels := []metrics.Label{ + { + Name: "job", + Value: summary.JobID, + }, + { + Name: "task_group", + Value: name, + }, + } + + if strings.Contains(summary.JobID, "/dispatch-") { + jobInfo := strings.Split(summary.JobID, "/dispatch-") + labels = append(labels, metrics.Label{ + Name: "parent_id", + Value: jobInfo[0], + }, metrics.Label{ + Name: "dispatch_id", + Value: jobInfo[1], + }) + } + + if strings.Contains(summary.JobID, "/periodic-") { + jobInfo := strings.Split(summary.JobID, "/periodic-") + labels = append(labels, metrics.Label{ + Name: "parent_id", + Value: jobInfo[0], + }, metrics.Label{ + Name: "periodic_id", + Value: jobInfo[1], + }) + } + + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "queued"}, + float32(tgSummary.Queued), labels) + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "complete"}, + float32(tgSummary.Complete), labels) + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "failed"}, + float32(tgSummary.Failed), labels) + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "running"}, + float32(tgSummary.Running), labels) + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "starting"}, + float32(tgSummary.Starting), labels) + metrics.SetGaugeWithLabels([]string{"nomad", "job_summary", "lost"}, + float32(tgSummary.Lost), labels) + } + if s.config.BackwardsCompatibleMetrics { + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "queued"}, float32(tgSummary.Queued)) + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "complete"}, float32(tgSummary.Complete)) + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "failed"}, float32(tgSummary.Failed)) + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "running"}, float32(tgSummary.Running)) + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "starting"}, float32(tgSummary.Starting)) + metrics.SetGauge([]string{"nomad", "job_summary", summary.JobID, name, "lost"}, float32(tgSummary.Lost)) + } + } +} + // revokeLeadership is invoked once we step down as leader. // This is used to cleanup any state that may be specific to a leader. func (s *Server) revokeLeadership() error { diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index da5987301..0e1ee64bb 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -3050,9 +3050,10 @@ func (s *StateStore) ReconcileJobSummaries(index uint64) error { // Create a job summary for the job summary := &structs.JobSummary{ - JobID: job.ID, - Namespace: job.Namespace, - Summary: make(map[string]structs.TaskGroupSummary), + JobID: job.ID, + Namespace: job.Namespace, + Summary: make(map[string]structs.TaskGroupSummary), + Dispatched: job.Dispatched, } for _, tg := range job.TaskGroups { summary.Summary[tg.Name] = structs.TaskGroupSummary{} @@ -3349,6 +3350,7 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, Summary: make(map[string]structs.TaskGroupSummary), Children: new(structs.JobChildrenSummary), CreateIndex: index, + Dispatched: job.Dispatched, } hasSummaryChanged = true } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ff2db8dee..cb0b2cb28 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -3593,6 +3593,9 @@ type JobSummary struct { // Children contains a summary for the children of this job. Children *JobChildrenSummary + // Dispatched is true if this job is dispatched from a parameterized job + Dispatched bool + // Raft Indexes CreateIndex uint64 ModifyIndex uint64 diff --git a/website/source/docs/configuration/telemetry.html.md b/website/source/docs/configuration/telemetry.html.md index 861b4a8dc..675da6d83 100644 --- a/website/source/docs/configuration/telemetry.html.md +++ b/website/source/docs/configuration/telemetry.html.md @@ -70,7 +70,26 @@ The following options are available on all telemetry configurations. 0.7. Note that this option is used to transition monitoring to tagged metrics and will eventually be deprecated. +- `prefix_filter` `(list: [])` - This is a list of filter rules to apply for + allowing/blocking metrics by prefix. A leading "+" will enable any + metrics with the given prefix, and a leading "-" will block them. If + there is overlap between two rules, the more specific rule will take + precedence. Blocking will take priority if the same prefix is listed multiple + times. +```javascript + [ + "-nomad.raft", + "+nomad.raft.apply", + "-nomad.memberlist", + ] +``` + +- `disable_dispatched_job_summary_metrics` `(bool: false)` - Specifies if Nomad + should ignore jobs dispatched from a parameterized job when publishing job + summary statistics. Since each job has a small memory overhead for tracking + summary statistics, it is sometimes desired to trade these statistics for + more memory when dispatching high volumes of jobs. ### `statsite` From af3f535f0a888e2f01277717af48af1289559cfc Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 14 Nov 2018 15:02:49 -0500 Subject: [PATCH 2/6] agent: suppose filter_default telemetry option --- command/agent/command.go | 4 ++++ command/agent/config.go | 8 ++++++++ command/agent/config_parse.go | 1 + website/source/docs/configuration/telemetry.html.md | 6 +++++- 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/command/agent/command.go b/command/agent/command.go index fc910bc56..c9cdab3df 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -888,6 +888,10 @@ func (c *Command) setupTelemetry(config *Config) (*metrics.InmemSink, error) { metricsConf.AllowedPrefixes = allowedPrefixes metricsConf.BlockedPrefixes = blockedPrefixes + if telConfig.FilterDefault != nil { + metricsConf.FilterDefault = *telConfig.FilterDefault + } + // Configure the statsite sink var fanout metrics.FanoutSink if telConfig.StatsiteAddr != "" { diff --git a/command/agent/config.go b/command/agent/config.go index e75013224..9c899d60b 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -444,6 +444,10 @@ type Telemetry struct { // PrefixFilter allows for filtering out metrics from being collected PrefixFilter []string `mapstructure:"prefix_filter"` + // FilterDefault controls whether to allow metrics that have not been specified + // by the filter + FilterDefault *bool `mapstructure:"filter_default"` + // DisableDispatchedJobSummaryMetrics allows for ignore dispatched jobs when // publishing Job summary metrics. This is useful in environment that produce // high numbers of single count dispatch jobs as the metrics for each take up @@ -1320,6 +1324,10 @@ func (a *Telemetry) Merge(b *Telemetry) *Telemetry { result.PrefixFilter = b.PrefixFilter } + if b.FilterDefault != nil { + result.FilterDefault = b.FilterDefault + } + if b.DisableDispatchedJobSummaryMetrics { result.DisableDispatchedJobSummaryMetrics = b.DisableDispatchedJobSummaryMetrics } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index c7c9aa3d4..5f5cdcbb7 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -735,6 +735,7 @@ func parseTelemetry(result **Telemetry, list *ast.ObjectList) error { "disable_tagged_metrics", "backwards_compatible_metrics", "prefix_filter", + "filter_default", "disable_dispatched_job_summary_metrics", } if err := helper.CheckHCLKeys(listVal, valid); err != nil { diff --git a/website/source/docs/configuration/telemetry.html.md b/website/source/docs/configuration/telemetry.html.md index 675da6d83..ba7186e53 100644 --- a/website/source/docs/configuration/telemetry.html.md +++ b/website/source/docs/configuration/telemetry.html.md @@ -64,12 +64,16 @@ The following options are available on all telemetry configurations. only be added to tagged metrics. Note that this option is used to transition monitoring to tagged metrics and will eventually be deprecated. - - `disable_tagged_metrics` `(bool: false)` - Specifies if Nomad should not emit tagged metrics and only emit metrics compatible with versions below Nomad 0.7. Note that this option is used to transition monitoring to tagged metrics and will eventually be deprecated. +- `filter_default` `(bool: true)` - This controls whether to allow metrics that + have not been specified by the filter. Defaults to true, which will allow all + metrics when no filters are provided. When set to false with no filters, no + metrics will be sent. + - `prefix_filter` `(list: [])` - This is a list of filter rules to apply for allowing/blocking metrics by prefix. A leading "+" will enable any metrics with the given prefix, and a leading "-" will block them. If From 8de254c5c50d6c978d9ac4d6dea1fcd863eca27e Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 14 Nov 2018 15:42:35 -0500 Subject: [PATCH 3/6] nomad: lookup job instead of adding Dispatched to summary --- nomad/leader.go | 6 +++++- nomad/state/state_store.go | 8 +++----- nomad/structs/structs.go | 3 --- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nomad/leader.go b/nomad/leader.go index 3eebb7b52..c77b53876 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -624,7 +624,11 @@ func (s *Server) publishJobSummaryMetrics(stopCh chan struct{}) { break } summary := raw.(*structs.JobSummary) - if s.config.DisableDispatchedJobSummaryMetrics && summary.Dispatched { + job, err := state.JobByID(ws, summary.Namespace, summary.JobID) + if err != nil { + s.logger.Printf("[ERR] nomad: failed to lookup job for summary: %v", err) + } + if s.config.DisableDispatchedJobSummaryMetrics && job.Dispatched { continue } s.iterateJobSummaryMetrics(summary) diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 0e1ee64bb..da5987301 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -3050,10 +3050,9 @@ func (s *StateStore) ReconcileJobSummaries(index uint64) error { // Create a job summary for the job summary := &structs.JobSummary{ - JobID: job.ID, - Namespace: job.Namespace, - Summary: make(map[string]structs.TaskGroupSummary), - Dispatched: job.Dispatched, + JobID: job.ID, + Namespace: job.Namespace, + Summary: make(map[string]structs.TaskGroupSummary), } for _, tg := range job.TaskGroups { summary.Summary[tg.Name] = structs.TaskGroupSummary{} @@ -3350,7 +3349,6 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, Summary: make(map[string]structs.TaskGroupSummary), Children: new(structs.JobChildrenSummary), CreateIndex: index, - Dispatched: job.Dispatched, } hasSummaryChanged = true } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index cb0b2cb28..ff2db8dee 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -3593,9 +3593,6 @@ type JobSummary struct { // Children contains a summary for the children of this job. Children *JobChildrenSummary - // Dispatched is true if this job is dispatched from a parameterized job - Dispatched bool - // Raft Indexes CreateIndex uint64 ModifyIndex uint64 From 825c2ffca98bc22d9677218b2b9374ca1953cf0c Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 14 Nov 2018 16:04:03 -0500 Subject: [PATCH 4/6] nomad: only lookup job is disable_dispatched_job_summary_metrics is set --- nomad/leader.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/nomad/leader.go b/nomad/leader.go index c77b53876..83faa99a1 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -624,12 +624,15 @@ func (s *Server) publishJobSummaryMetrics(stopCh chan struct{}) { break } summary := raw.(*structs.JobSummary) - job, err := state.JobByID(ws, summary.Namespace, summary.JobID) - if err != nil { - s.logger.Printf("[ERR] nomad: failed to lookup job for summary: %v", err) - } - if s.config.DisableDispatchedJobSummaryMetrics && job.Dispatched { - continue + if s.config.DisableDispatchedJobSummaryMetrics { + job, err := state.JobByID(ws, summary.Namespace, summary.JobID) + if err != nil { + s.logger.Printf("[ERR] nomad: failed to lookup job for summary: %v", err) + continue + } + if job.Dispatched { + continue + } } s.iterateJobSummaryMetrics(summary) } From 19c260a4a55b29c90aeed7a79ea940ee8a10954d Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 14 Nov 2018 16:04:33 -0500 Subject: [PATCH 5/6] command/agent: additional tests for telemetry config parsing --- command/agent/config_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/command/agent/config_test.go b/command/agent/config_test.go index baf9b7f22..dae2353ba 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -227,6 +227,8 @@ func TestConfig_Merge(t *testing.T) { CirconusBrokerID: "1", CirconusBrokerSelectTag: "dc:dc2", PrefixFilter: []string{"prefix1", "prefix2"}, + DisableDispatchedJobSummaryMetrics: true, + FilterDefault: helper.BoolToPtr(false), }, Client: &ClientConfig{ Enabled: true, @@ -1034,3 +1036,26 @@ func TestTelemetry_PrefixFilters(t *testing.T) { }) } } + +func TestTelemetry_Parse(t *testing.T) { + require := require.New(t) + dir, err := ioutil.TempDir("", "nomad") + require.NoError(err) + defer os.RemoveAll(dir) + + file1 := filepath.Join(dir, "config1.hcl") + err = ioutil.WriteFile(file1, []byte(`telemetry{ + prefix_filter = ["+nomad.raft"] + filter_default = false + disable_dispatched_job_summary_metrics = true + }`), 0600) + require.NoError(err) + + // Works on config dir + config, err := LoadConfig(dir) + require.NoError(err) + + require.False(*config.Telemetry.FilterDefault) + require.Exactly([]string{"+nomad.raft"}, config.Telemetry.PrefixFilter) + require.True(config.Telemetry.DisableDispatchedJobSummaryMetrics) +} From 69e6b0ea215cafca87afbc178b2002e2b68ece3c Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 29 Nov 2018 22:27:39 -0500 Subject: [PATCH 6/6] nomad: fix hclog usage --- command/agent/config.go | 4 ++-- nomad/leader.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/command/agent/config.go b/command/agent/config.go index 9c899d60b..6716ccc03 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -448,8 +448,8 @@ type Telemetry struct { // by the filter FilterDefault *bool `mapstructure:"filter_default"` - // DisableDispatchedJobSummaryMetrics allows for ignore dispatched jobs when - // publishing Job summary metrics. This is useful in environment that produce + // DisableDispatchedJobSummaryMetrics allows ignoring dispatched jobs when + // publishing Job summary metrics. This is useful in environments that produce // high numbers of single count dispatch jobs as the metrics for each take up // a small memory overhead. DisableDispatchedJobSummaryMetrics bool `mapstructure:"disable_dispatched_job_summary_metrics"` diff --git a/nomad/leader.go b/nomad/leader.go index 83faa99a1..f9aa35208 100644 --- a/nomad/leader.go +++ b/nomad/leader.go @@ -627,7 +627,7 @@ func (s *Server) publishJobSummaryMetrics(stopCh chan struct{}) { if s.config.DisableDispatchedJobSummaryMetrics { job, err := state.JobByID(ws, summary.Namespace, summary.JobID) if err != nil { - s.logger.Printf("[ERR] nomad: failed to lookup job for summary: %v", err) + s.logger.Error("error getting job for summary", "error", err) continue } if job.Dispatched {