From c3006c685f9adee25c64bbb258f3cb9005537381 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Wed, 6 Sep 2017 18:23:58 +0000 Subject: [PATCH] improve documentation move metrics to telemetry; copy to client config --- client/client.go | 47 +++++++------------ client/client_test.go | 6 +-- command/agent/agent.go | 12 ++--- command/agent/agent_test.go | 24 ++++++++++ command/agent/config-test-fixtures/basic.hcl | 4 +- command/agent/testagent.go | 2 +- .../agent/configuration/telemetry.html.md | 13 +++-- 7 files changed, 60 insertions(+), 48 deletions(-) diff --git a/client/client.go b/client/client.go index bf109380f..d657d9224 100644 --- a/client/client.go +++ b/client/client.go @@ -157,9 +157,6 @@ type Client struct { // baseLabels are used when emitting tagged metrics. All client metrics will // have these tags, and optionally more. baseLabels []metrics.Label - - // Subset of global telemetry configuration options for the client - clientTelemetry *ClientTelemetry } var ( @@ -169,15 +166,8 @@ var ( noServersErr = errors.New("no servers") ) -// ClientTelemetry is a subset of global telemetry configuration options that -// are relevant for the client -type ClientTelemetry struct { - DisableTaggedMetrics bool - BackwardsCompatibleMetrics bool -} - // NewClient is used to create a new client from the given configuration -func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger, telemetry *ClientTelemetry) (*Client, error) { +func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger) (*Client, error) { // Create the tls wrapper var tlsWrap tlsutil.RegionWrapper if cfg.TLSConfig.EnableRPC { @@ -190,7 +180,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic // Create the client c := &Client{ - clientTelemetry: telemetry, config: cfg, consulCatalog: consulCatalog, consulService: consulService, @@ -1890,14 +1879,14 @@ func (c *Client) emitStats() { // setGaugeForMemoryStats proxies metrics for memory specific statistics func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats) { - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "total"}, float32(hStats.Memory.Total), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "available"}, float32(hStats.Memory.Available), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "used"}, float32(hStats.Memory.Used), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "free"}, float32(hStats.Memory.Free), c.baseLabels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "host", "memory", nodeID, "total"}, float32(hStats.Memory.Total)) metrics.SetGauge([]string{"client", "host", "memory", nodeID, "available"}, float32(hStats.Memory.Available)) metrics.SetGauge([]string{"client", "host", "memory", nodeID, "used"}, float32(hStats.Memory.Used)) @@ -1908,7 +1897,7 @@ func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats) // setGaugeForCPUStats proxies metrics for CPU specific statistics func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) { for _, cpu := range hStats.CPU { - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { labels := append(c.baseLabels, metrics.Label{"cpu", cpu.CPU}) metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "total"}, float32(cpu.Total), labels) @@ -1917,7 +1906,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) { metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "system"}, float32(cpu.System), labels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "total"}, float32(cpu.Total)) metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "user"}, float32(cpu.User)) metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "idle"}, float32(cpu.Idle)) @@ -1929,7 +1918,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) { // setGaugeForDiskStats proxies metrics for disk specific statistics func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) { for _, disk := range hStats.DiskStats { - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { labels := append(c.baseLabels, metrics.Label{"disk", disk.Device}) metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "size"}, float32(disk.Size), labels) @@ -1939,7 +1928,7 @@ func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) { metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "inodes_percent"}, float32(disk.InodesUsedPercent), labels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "size"}, float32(disk.Size)) metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "used"}, float32(disk.Used)) metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "available"}, float32(disk.Available)) @@ -1958,14 +1947,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) { allocated := c.getAllocatedResources(node) // Emit allocated - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { metrics.SetGaugeWithLabels([]string{"client", "allocated", "memory"}, float32(allocated.MemoryMB), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "allocated", "disk"}, float32(allocated.DiskMB), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "allocated", "cpu"}, float32(allocated.CPU), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "allocated", "iops"}, float32(allocated.IOPS), c.baseLabels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "allocated", "memory", nodeID}, float32(allocated.MemoryMB)) metrics.SetGauge([]string{"client", "allocated", "disk", nodeID}, float32(allocated.DiskMB)) metrics.SetGauge([]string{"client", "allocated", "cpu", nodeID}, float32(allocated.CPU)) @@ -1973,12 +1962,12 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) { } for _, n := range allocated.Networks { - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { labels := append(c.baseLabels, metrics.Label{"device", n.Device}) metrics.SetGaugeWithLabels([]string{"client", "allocated", "network"}, float32(n.MBits), labels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "allocated", "network", n.Device, nodeID}, float32(n.MBits)) } } @@ -1989,14 +1978,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) { unallocatedCpu := total.CPU - res.CPU - allocated.CPU unallocatedIops := total.IOPS - res.IOPS - allocated.IOPS - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { metrics.SetGaugeWithLabels([]string{"client", "unallocated", "memory"}, float32(unallocatedMem), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "unallocated", "disk"}, float32(unallocatedDisk), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "unallocated", "cpu"}, float32(unallocatedCpu), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "unallocated", "iops"}, float32(unallocatedIops), c.baseLabels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "unallocated", "memory", nodeID}, float32(unallocatedMem)) metrics.SetGauge([]string{"client", "unallocated", "disk", nodeID}, float32(unallocatedDisk)) metrics.SetGauge([]string{"client", "unallocated", "cpu", nodeID}, float32(unallocatedCpu)) @@ -2014,7 +2003,7 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) { unallocatedMbits := totalMbits - n.MBits - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { labels := append(c.baseLabels, metrics.Label{"device", n.Device}) metrics.SetGaugeWithLabels([]string{"client", "unallocated", "network"}, float32(unallocatedMbits), labels) } @@ -2027,10 +2016,10 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) { // No lables are required so we emit with only a key/value syntax func (c *Client) setGaugeForUptime(hStats *stats.HostStats) { - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { metrics.SetGaugeWithLabels([]string{"uptime"}, float32(hStats.Uptime), c.baseLabels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"uptime"}, float32(hStats.Uptime)) } } @@ -2073,7 +2062,7 @@ func (c *Client) emitClientMetrics() { } } - if !c.clientTelemetry.DisableTaggedMetrics { + if !c.config.DisableTaggedMetrics { metrics.SetGaugeWithLabels([]string{"client", "allocations", "migrating"}, float32(migrating), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "allocations", "blocked"}, float32(blocked), c.baseLabels) metrics.SetGaugeWithLabels([]string{"client", "allocations", "pending"}, float32(pending), c.baseLabels) @@ -2081,7 +2070,7 @@ func (c *Client) emitClientMetrics() { metrics.SetGaugeWithLabels([]string{"client", "allocations", "terminal"}, float32(terminal), c.baseLabels) } - if c.clientTelemetry.BackwardsCompatibleMetrics { + if c.config.BackwardsCompatibleMetrics { metrics.SetGauge([]string{"client", "allocations", "migrating", nodeID}, float32(migrating)) metrics.SetGauge([]string{"client", "allocations", "blocked", nodeID}, float32(blocked)) metrics.SetGauge([]string{"client", "allocations", "pending", nodeID}, float32(pending)) diff --git a/client/client_test.go b/client/client_test.go index d85a10f96..fafd338ad 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -122,8 +122,7 @@ func testClient(t *testing.T, cb func(c *config.Config)) *Client { catalog := consul.NewMockCatalog(logger) mockService := newMockConsulServiceClient() mockService.logger = logger - telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false} - client, err := NewClient(conf, catalog, mockService, logger, telemetry) + client, err := NewClient(conf, catalog, mockService, logger) if err != nil { t.Fatalf("err: %v", err) } @@ -795,8 +794,7 @@ func TestClient_SaveRestoreState(t *testing.T) { catalog := consul.NewMockCatalog(logger) mockService := newMockConsulServiceClient() mockService.logger = logger - telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false} - c2, err := NewClient(c1.config, catalog, mockService, logger, telemetry) + c2, err := NewClient(c1.config, catalog, mockService, logger) if err != nil { t.Fatalf("err: %v", err) } diff --git a/command/agent/agent.go b/command/agent/agent.go index fdcad7502..50409c09b 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -335,9 +335,13 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { conf.ConsulConfig = a.config.Consul conf.VaultConfig = a.config.Vault + + // Set up Telemetry configuration conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics + conf.DisableTaggedMetrics = a.config.Telemetry.DisableTaggedMetrics + conf.BackwardsCompatibleMetrics = a.config.Telemetry.BackwardsCompatibleMetrics // Set the TLS related configs conf.TLSConfig = a.config.TLSConfig @@ -493,13 +497,7 @@ func (a *Agent) setupClient() error { } } - // Create the client - clientTelemetry := &client.ClientTelemetry{ - DisableTaggedMetrics: a.config.Telemetry.DisableTaggedMetrics, - BackwardsCompatibleMetrics: a.config.Telemetry.BackwardsCompatibleMetrics, - } - - client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger, clientTelemetry) + client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger) if err != nil { return fmt.Errorf("client setup failed: %v", err) } diff --git a/command/agent/agent_test.go b/command/agent/agent_test.go index bbfc0956d..310fc8a00 100644 --- a/command/agent/agent_test.go +++ b/command/agent/agent_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/helper" sconfig "github.com/hashicorp/nomad/nomad/structs/config" + "github.com/stretchr/testify/assert" ) func getPort() int { @@ -316,6 +317,29 @@ func TestAgent_ClientConfig(t *testing.T) { } } +// Clients should inherit telemetry configuration +func TestAget_Client_TelemetryConfiguration(t *testing.T) { + assert := assert.New(t) + + conf := DefaultConfig() + conf.DevMode = true + conf.Telemetry.DisableTaggedMetrics = true + conf.Telemetry.BackwardsCompatibleMetrics = true + + a := &Agent{config: conf} + + c, err := a.clientConfig() + assert.Nil(err) + + telemetry := conf.Telemetry + + assert.Equal(c.StatsCollectionInterval, telemetry.collectionInterval) + assert.Equal(c.PublishNodeMetrics, telemetry.PublishNodeMetrics) + assert.Equal(c.PublishAllocationMetrics, telemetry.PublishAllocationMetrics) + assert.Equal(c.DisableTaggedMetrics, telemetry.DisableTaggedMetrics) + assert.Equal(c.BackwardsCompatibleMetrics, telemetry.BackwardsCompatibleMetrics) +} + // TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP // API health check depending on configuration. func TestAgent_HTTPCheck(t *testing.T) { diff --git a/command/agent/config-test-fixtures/basic.hcl b/command/agent/config-test-fixtures/basic.hcl index e1ba443e7..2e451932e 100644 --- a/command/agent/config-test-fixtures/basic.hcl +++ b/command/agent/config-test-fixtures/basic.hcl @@ -96,8 +96,8 @@ telemetry { collection_interval = "3s" publish_allocation_metrics = true publish_node_metrics = true - disable_tagged_metrics = true - backwards_compatible_metrics = true + disable_tagged_metrics = true + backwards_compatible_metrics = true } leave_on_interrupt = true leave_on_terminate = true diff --git a/command/agent/testagent.go b/command/agent/testagent.go index b6133fd3d..76394353a 100644 --- a/command/agent/testagent.go +++ b/command/agent/testagent.go @@ -192,7 +192,7 @@ func (a *TestAgent) start() (*Agent, error) { metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm) if inm == nil { - return nil, fmt.Errorf("unable to set up in memory metrics neede for agent initialization") + return nil, fmt.Errorf("unable to set up in memory metrics needed for agent initialization") } agent, err := NewAgent(a.Config, a.LogOutput, inm) diff --git a/website/source/docs/agent/configuration/telemetry.html.md b/website/source/docs/agent/configuration/telemetry.html.md index 55b565c1f..013c030b4 100644 --- a/website/source/docs/agent/configuration/telemetry.html.md +++ b/website/source/docs/agent/configuration/telemetry.html.md @@ -60,11 +60,17 @@ The following options are available on all telemetry configurations. - `backwards_compatible_metrics` `(bool: false)` - Specifies if Nomad should publish metrics that are backwards compatible with versions below 0.7, as - post version 0.7, Nomad emits tagged metrics. + post version 0.7, Nomad emits tagged metrics. and all new metrics will + only be added to tagged metrics. Note that this option is used to transition + monitoring to tagged metrics and will eventually be deprecated. + - `disable_tagged_metrics` `(bool: false)` - Specifies if Nomad should not emit tagged metrics and only emit metrics compatible with versions below Nomad - 0.7. + 0.7. Note that this option is used to transition monitoring to tagged + metrics and will eventually be deprecated. + + ### `statsite` @@ -170,6 +176,3 @@ These `telemetry` parameters apply to best use of this is to as a hint for which broker should be used based on *where* this particular instance is running (e.g. a specific geographic location or datacenter, dc:sfo). - - -