kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-06 10:25:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

improve documentation

Browse Source 
move metrics to telemetry; copy to client config
This commit is contained in:
Chelsea Holland Komlo
2017-09-06 18:23:58 +00:00
parent 1df5310c6e
commit c3006c685f
7 changed files with 60 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
client/client.go
View File

@@ -157,9 +157,6 @@ type Client struct {
// baseLabels are used when emitting tagged metrics. All client metrics will
// have these tags, and optionally more.
baseLabels []metrics.Label
// Subset of global telemetry configuration options for the client
clientTelemetry *ClientTelemetry
}
var (
@@ -169,15 +166,8 @@ var (
noServersErr = errors.New("no servers")
)
// ClientTelemetry is a subset of global telemetry configuration options that
// are relevant for the client
type ClientTelemetry struct {
DisableTaggedMetrics bool
BackwardsCompatibleMetrics bool
}
// NewClient is used to create a new client from the given configuration
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger, telemetry *ClientTelemetry) (*Client, error) {
func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulService ConsulServiceAPI, logger *log.Logger) (*Client, error) {
// Create the tls wrapper
var tlsWrap tlsutil.RegionWrapper
if cfg.TLSConfig.EnableRPC {
@@ -190,7 +180,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic
// Create the client
c := &Client{
clientTelemetry: telemetry,
config: cfg,
consulCatalog: consulCatalog,
consulService: consulService,
@@ -1890,14 +1879,14 @@ func (c *Client) emitStats() {
// setGaugeForMemoryStats proxies metrics for memory specific statistics
func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats) {
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "total"}, float32(hStats.Memory.Total), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "available"}, float32(hStats.Memory.Available), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "used"}, float32(hStats.Memory.Used), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "host", "memory", "free"}, float32(hStats.Memory.Free), c.baseLabels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "total"}, float32(hStats.Memory.Total))
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "available"}, float32(hStats.Memory.Available))
metrics.SetGauge([]string{"client", "host", "memory", nodeID, "used"}, float32(hStats.Memory.Used))
@@ -1908,7 +1897,7 @@ func (c *Client) setGaugeForMemoryStats(nodeID string, hStats *stats.HostStats)
// setGaugeForCPUStats proxies metrics for CPU specific statistics
func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
for _, cpu := range hStats.CPU {
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"cpu", cpu.CPU})
metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "total"}, float32(cpu.Total), labels)
@@ -1917,7 +1906,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
metrics.SetGaugeWithLabels([]string{"client", "host", "cpu", "system"}, float32(cpu.System), labels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "total"}, float32(cpu.Total))
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "user"}, float32(cpu.User))
metrics.SetGauge([]string{"client", "host", "cpu", nodeID, cpu.CPU, "idle"}, float32(cpu.Idle))
@@ -1929,7 +1918,7 @@ func (c *Client) setGaugeForCPUStats(nodeID string, hStats *stats.HostStats) {
// setGaugeForDiskStats proxies metrics for disk specific statistics
func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
for _, disk := range hStats.DiskStats {
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"disk", disk.Device})
metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "size"}, float32(disk.Size), labels)
@@ -1939,7 +1928,7 @@ func (c *Client) setGaugeForDiskStats(nodeID string, hStats *stats.HostStats) {
metrics.SetGaugeWithLabels([]string{"client", "host", "disk", "inodes_percent"}, float32(disk.InodesUsedPercent), labels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "size"}, float32(disk.Size))
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "used"}, float32(disk.Used))
metrics.SetGauge([]string{"client", "host", "disk", nodeID, disk.Device, "available"}, float32(disk.Available))
@@ -1958,14 +1947,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
allocated := c.getAllocatedResources(node)
// Emit allocated
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "allocated", "memory"}, float32(allocated.MemoryMB), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "disk"}, float32(allocated.DiskMB), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "cpu"}, float32(allocated.CPU), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocated", "iops"}, float32(allocated.IOPS), c.baseLabels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocated", "memory", nodeID}, float32(allocated.MemoryMB))
metrics.SetGauge([]string{"client", "allocated", "disk", nodeID}, float32(allocated.DiskMB))
metrics.SetGauge([]string{"client", "allocated", "cpu", nodeID}, float32(allocated.CPU))
@@ -1973,12 +1962,12 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
}
for _, n := range allocated.Networks {
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"device", n.Device})
metrics.SetGaugeWithLabels([]string{"client", "allocated", "network"}, float32(n.MBits), labels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocated", "network", n.Device, nodeID}, float32(n.MBits))
}
}
@@ -1989,14 +1978,14 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
unallocatedCpu := total.CPU - res.CPU - allocated.CPU
unallocatedIops := total.IOPS - res.IOPS - allocated.IOPS
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "memory"}, float32(unallocatedMem), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "disk"}, float32(unallocatedDisk), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "cpu"}, float32(unallocatedCpu), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "iops"}, float32(unallocatedIops), c.baseLabels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "unallocated", "memory", nodeID}, float32(unallocatedMem))
metrics.SetGauge([]string{"client", "unallocated", "disk", nodeID}, float32(unallocatedDisk))
metrics.SetGauge([]string{"client", "unallocated", "cpu", nodeID}, float32(unallocatedCpu))
@@ -2014,7 +2003,7 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
unallocatedMbits := totalMbits - n.MBits
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
labels := append(c.baseLabels, metrics.Label{"device", n.Device})
metrics.SetGaugeWithLabels([]string{"client", "unallocated", "network"}, float32(unallocatedMbits), labels)
}
@@ -2027,10 +2016,10 @@ func (c *Client) setGaugeForAllocationStats(nodeID string) {
// No lables are required so we emit with only a key/value syntax
func (c *Client) setGaugeForUptime(hStats *stats.HostStats) {
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"uptime"}, float32(hStats.Uptime), c.baseLabels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"uptime"}, float32(hStats.Uptime))
}
}
@@ -2073,7 +2062,7 @@ func (c *Client) emitClientMetrics() {
}
}
if !c.clientTelemetry.DisableTaggedMetrics {
if !c.config.DisableTaggedMetrics {
metrics.SetGaugeWithLabels([]string{"client", "allocations", "migrating"}, float32(migrating), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "blocked"}, float32(blocked), c.baseLabels)
metrics.SetGaugeWithLabels([]string{"client", "allocations", "pending"}, float32(pending), c.baseLabels)
@@ -2081,7 +2070,7 @@ func (c *Client) emitClientMetrics() {
metrics.SetGaugeWithLabels([]string{"client", "allocations", "terminal"}, float32(terminal), c.baseLabels)
}
if c.clientTelemetry.BackwardsCompatibleMetrics {
if c.config.BackwardsCompatibleMetrics {
metrics.SetGauge([]string{"client", "allocations", "migrating", nodeID}, float32(migrating))
metrics.SetGauge([]string{"client", "allocations", "blocked", nodeID}, float32(blocked))
metrics.SetGauge([]string{"client", "allocations", "pending", nodeID}, float32(pending))

6
client/client_test.go
View File

@@ -122,8 +122,7 @@ func testClient(t *testing.T, cb func(c *config.Config)) *Client {
catalog := consul.NewMockCatalog(logger)
mockService := newMockConsulServiceClient()
mockService.logger = logger
telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
client, err := NewClient(conf, catalog, mockService, logger, telemetry)
client, err := NewClient(conf, catalog, mockService, logger)
if err != nil {
t.Fatalf("err: %v", err)
}
@@ -795,8 +794,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
catalog := consul.NewMockCatalog(logger)
mockService := newMockConsulServiceClient()
mockService.logger = logger
telemetry := &ClientTelemetry{DisableTaggedMetrics: false, BackwardsCompatibleMetrics: false}
c2, err := NewClient(c1.config, catalog, mockService, logger, telemetry)
c2, err := NewClient(c1.config, catalog, mockService, logger)
if err != nil {
t.Fatalf("err: %v", err)
}

12
command/agent/agent.go
View File

@@ -335,9 +335,13 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) {
conf.ConsulConfig = a.config.Consul
conf.VaultConfig = a.config.Vault
// Set up Telemetry configuration
conf.StatsCollectionInterval = a.config.Telemetry.collectionInterval
conf.PublishNodeMetrics = a.config.Telemetry.PublishNodeMetrics
conf.PublishAllocationMetrics = a.config.Telemetry.PublishAllocationMetrics
conf.DisableTaggedMetrics = a.config.Telemetry.DisableTaggedMetrics
conf.BackwardsCompatibleMetrics = a.config.Telemetry.BackwardsCompatibleMetrics
// Set the TLS related configs
conf.TLSConfig = a.config.TLSConfig
@@ -493,13 +497,7 @@ func (a *Agent) setupClient() error {
}
}
// Create the client
clientTelemetry := &client.ClientTelemetry{
DisableTaggedMetrics: a.config.Telemetry.DisableTaggedMetrics,
BackwardsCompatibleMetrics: a.config.Telemetry.BackwardsCompatibleMetrics,
}
client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger, clientTelemetry)
client, err := client.NewClient(conf, a.consulCatalog, a.consulService, a.logger)
if err != nil {
return fmt.Errorf("client setup failed: %v", err)
}

24
command/agent/agent_test.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/hashicorp/nomad/helper"
sconfig "github.com/hashicorp/nomad/nomad/structs/config"
"github.com/stretchr/testify/assert"
)
func getPort() int {
@@ -316,6 +317,29 @@ func TestAgent_ClientConfig(t *testing.T) {
}
}
// Clients should inherit telemetry configuration
func TestAget_Client_TelemetryConfiguration(t *testing.T) {
assert := assert.New(t)
conf := DefaultConfig()
conf.DevMode = true
conf.Telemetry.DisableTaggedMetrics = true
conf.Telemetry.BackwardsCompatibleMetrics = true
a := &Agent{config: conf}
c, err := a.clientConfig()
assert.Nil(err)
telemetry := conf.Telemetry
assert.Equal(c.StatsCollectionInterval, telemetry.collectionInterval)
assert.Equal(c.PublishNodeMetrics, telemetry.PublishNodeMetrics)
assert.Equal(c.PublishAllocationMetrics, telemetry.PublishAllocationMetrics)
assert.Equal(c.DisableTaggedMetrics, telemetry.DisableTaggedMetrics)
assert.Equal(c.BackwardsCompatibleMetrics, telemetry.BackwardsCompatibleMetrics)
}
// TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP
// API health check depending on configuration.
func TestAgent_HTTPCheck(t *testing.T) {

4
command/agent/config-test-fixtures/basic.hcl
View File

@@ -96,8 +96,8 @@ telemetry {
collection_interval = "3s"
publish_allocation_metrics = true
publish_node_metrics = true
disable_tagged_metrics = true
backwards_compatible_metrics = true
disable_tagged_metrics = true
backwards_compatible_metrics = true
}
leave_on_interrupt = true
leave_on_terminate = true

2
command/agent/testagent.go
View File

@@ -192,7 +192,7 @@ func (a *TestAgent) start() (*Agent, error) {
metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm)
if inm == nil {
return nil, fmt.Errorf("unable to set up in memory metrics neede for agent initialization")
return nil, fmt.Errorf("unable to set up in memory metrics needed for agent initialization")
}
agent, err := NewAgent(a.Config, a.LogOutput, inm)

13
website/source/docs/agent/configuration/telemetry.html.md
View File

@@ -60,11 +60,17 @@ The following options are available on all telemetry configurations.
- `backwards_compatible_metrics` `(bool: false)` - Specifies if Nomad should
publish metrics that are backwards compatible with versions below 0.7, as
post version 0.7, Nomad emits tagged metrics.
post version 0.7, Nomad emits tagged metrics. and all new metrics will
only be added to tagged metrics. Note that this option is used to transition
monitoring to tagged metrics and will eventually be deprecated.
- `disable_tagged_metrics` `(bool: false)` - Specifies if Nomad should not emit
tagged metrics and only emit metrics compatible with versions below Nomad
0.7.
0.7. Note that this option is used to transition monitoring to tagged
metrics and will eventually be deprecated.
### `statsite`
@@ -170,6 +176,3 @@ These `telemetry` parameters apply to
best use of this is to as a hint for which broker should be used based on
*where* this particular instance is running (e.g. a specific geographic location or
datacenter, dc:sfo).