From 5af9296bb49cf3e535b299a1a3ead85e3d1ad340 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Tue, 13 Nov 2018 11:49:14 -0500 Subject: [PATCH] Expose Device Stats in /client/stats API endpoint --- client/client.go | 32 +++++++++++++------------- client/stats/host.go | 40 ++++++++++++++++++++++++--------- plugins/shared/structs/stats.go | 16 ++++++------- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/client/client.go b/client/client.go index 923ab9fa9..60c1d32a3 100644 --- a/client/client.go +++ b/client/client.go @@ -274,22 +274,6 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic return nil, fmt.Errorf("failed to initialize ACL state: %v", err) } - // Add the stats collector - statsCollector := stats.NewHostStatsCollector(c.logger, c.config.AllocDir) - c.hostStatsCollector = statsCollector - - // Add the garbage collector - gcConfig := &GCConfig{ - MaxAllocs: cfg.GCMaxAllocs, - DiskUsageThreshold: cfg.GCDiskUsageThreshold, - InodeUsageThreshold: cfg.GCInodeUsageThreshold, - Interval: cfg.GCInterval, - ParallelDestroys: cfg.GCParallelDestroys, - ReservedDiskMB: cfg.Node.Reserved.DiskMB, - } - c.garbageCollector = NewAllocGarbageCollector(c.logger, statsCollector, c, gcConfig) - go c.garbageCollector.Run() - // Setup the node if err := c.setupNode(); err != nil { return nil, fmt.Errorf("node setup failed: %v", err) @@ -324,6 +308,22 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic c.devicemanager = devicemanager.New(devConfig) go c.devicemanager.Run() + // Add the stats collector + statsCollector := stats.NewHostStatsCollector(c.logger, c.config.AllocDir, c.devicemanager.AllStats) + c.hostStatsCollector = statsCollector + + // Add the garbage collector + gcConfig := &GCConfig{ + MaxAllocs: cfg.GCMaxAllocs, + DiskUsageThreshold: cfg.GCDiskUsageThreshold, + InodeUsageThreshold: cfg.GCInodeUsageThreshold, + Interval: cfg.GCInterval, + ParallelDestroys: cfg.GCParallelDestroys, + ReservedDiskMB: cfg.Node.Reserved.DiskMB, + } + c.garbageCollector = NewAllocGarbageCollector(c.logger, statsCollector, c, gcConfig) + go c.garbageCollector.Run() + // Set the preconfigured list of static servers c.configLock.RLock() if len(c.configCopy.Servers) > 0 { diff --git a/client/stats/host.go b/client/stats/host.go index c3cb90cbb..7a5be3322 100644 --- a/client/stats/host.go +++ b/client/stats/host.go @@ -8,6 +8,7 @@ import ( "time" hclog "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/plugins/device" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/disk" "github.com/shirou/gopsutil/host" @@ -20,6 +21,7 @@ type HostStats struct { CPU []*CPUStats DiskStats []*DiskStats AllocDirStats *DiskStats + DeviceStats []*DeviceGroupStats Uptime uint64 Timestamp int64 CPUTicksConsumed float64 @@ -53,6 +55,12 @@ type DiskStats struct { InodesUsedPercent float64 } +// DeviceGroupStats represents stats related to device group +type DeviceGroupStats = device.DeviceGroupStats + +// DeviceStatsCollector is used to retrieve all the latest statistics for all devices. +type DeviceStatsCollector func() []*DeviceGroupStats + // NodeStatsCollector is an interface which is used for the purposes of mocking // the HostStatsCollector in the tests type NodeStatsCollector interface { @@ -62,11 +70,12 @@ type NodeStatsCollector interface { // HostStatsCollector collects host resource usage stats type HostStatsCollector struct { - numCores int - statsCalculator map[string]*HostCpuStatsCalculator - hostStats *HostStats - hostStatsLock sync.RWMutex - allocDir string + numCores int + statsCalculator map[string]*HostCpuStatsCalculator + hostStats *HostStats + hostStatsLock sync.RWMutex + allocDir string + deviceStatsCollector DeviceStatsCollector // badParts is a set of partitions whose usage cannot be read; used to // squelch logspam. @@ -78,16 +87,17 @@ type HostStatsCollector struct { // NewHostStatsCollector returns a HostStatsCollector. The allocDir is passed in // so that we can present the disk related statistics for the mountpoint where // the allocation directory lives -func NewHostStatsCollector(logger hclog.Logger, allocDir string) *HostStatsCollector { +func NewHostStatsCollector(logger hclog.Logger, allocDir string, deviceStatsCollector DeviceStatsCollector) *HostStatsCollector { logger = logger.Named("host_stats") numCores := runtime.NumCPU() statsCalculator := make(map[string]*HostCpuStatsCalculator) collector := &HostStatsCollector{ - statsCalculator: statsCalculator, - numCores: numCores, - logger: logger, - allocDir: allocDir, - badParts: make(map[string]struct{}), + statsCalculator: statsCalculator, + numCores: numCores, + logger: logger, + allocDir: allocDir, + badParts: make(map[string]struct{}), + deviceStatsCollector: deviceStatsCollector, } return collector } @@ -140,6 +150,10 @@ func (h *HostStatsCollector) collectLocked() error { } hs.AllocDirStats = h.toDiskStats(usage, nil) + // Collect devices stats + deviceStats := h.collectDeviceGroupStats() + hs.DeviceStats = deviceStats + // Update the collected status object. h.hostStats = hs @@ -189,6 +203,10 @@ func (h *HostStatsCollector) collectDiskStats() ([]*DiskStats, error) { return diskStats, nil } +func (h *HostStatsCollector) collectDeviceGroupStats() []*DeviceGroupStats { + return h.deviceStatsCollector() +} + // Stats returns the host stats that has been collected func (h *HostStatsCollector) Stats() *HostStats { h.hostStatsLock.RLock() diff --git a/plugins/shared/structs/stats.go b/plugins/shared/structs/stats.go index 233991f9f..5ebc778f7 100644 --- a/plugins/shared/structs/stats.go +++ b/plugins/shared/structs/stats.go @@ -16,23 +16,23 @@ type StatObject struct { type StatValue struct { // FloatNumeratorVal exposes a floating point value. If denominator is set // it is assumed to be a fractional value, otherwise it is a scalar. - FloatNumeratorVal *float64 - FloatDenominatorVal *float64 + FloatNumeratorVal *float64 `json:",omitempty"` + FloatDenominatorVal *float64 `json:",omitempty"` // IntNumeratorVal exposes a int value. If denominator is set it is assumed // to be a fractional value, otherwise it is a scalar. - IntNumeratorVal *int64 - IntDenominatorVal *int64 + IntNumeratorVal *int64 `json:",omitempty"` + IntDenominatorVal *int64 `json:",omitempty"` // StringVal exposes a string value. These are likely annotations. - StringVal *string + StringVal *string `json:",omitempty"` // BoolVal exposes a boolean statistic. - BoolVal *bool + BoolVal *bool `json:",omitempty"` // Unit gives the unit type: °F, %, MHz, MB, etc. - Unit string + Unit string `json:",omitempty"` // Desc provides a human readable description of the statistic. - Desc string + Desc string `json:",omitempty"` }