Merge pull request #1247 from hashicorp/f-measured-resources

Annotate what statistics are measured

api/tasks.go

@@ -12,6 +12,7 @@ type MemoryStats struct {
 	MaxUsage       uint64
 	KernelUsage    uint64
 	KernelMaxUsage uint64
+	Measured       []string
 }
 
 // CpuStats holds cpu usage related stats
@@ -21,6 +22,7 @@ type CpuStats struct {
 	ThrottledPeriods uint64
 	ThrottledTime    uint64
 	Percent          float64
+	Measured         []string
 }
 
 // ResourceUsage holds information related to cpu and memory stats

client/driver/docker.go

@@ -32,6 +32,10 @@ var (
 	// We store the client globally to cache the connection to the docker daemon.
 	createClient sync.Once
 	client       *docker.Client
+
+	// The statistics the Docker driver exposes
+	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage"}
+	DockerMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
 )
 
 const (
@@ -851,6 +855,7 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er
 		h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err)
 	}
 
+	go h.collectStats()
 	go h.run()
 	return h, nil
 }
@@ -989,6 +994,7 @@ func (h *DockerHandle) collectStats() {
 					Cache:    s.MemoryStats.Stats.Cache,
 					Swap:     s.MemoryStats.Stats.Swap,
 					MaxUsage: s.MemoryStats.MaxUsage,
+					Measured: DockerMeasuredMemStats,
 				}
 
 				cs := &cstructs.CpuStats{
@@ -996,7 +1002,9 @@ func (h *DockerHandle) collectStats() {
 					UserMode:         float64(s.CPUStats.CPUUsage.UsageInKernelmode),
 					ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
 					ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
+					Measured:         DockerMeasuredCpuStats,
 				}
+
 				// Calculate percentage
 				cs.Percent = 0.0
 				cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage)
@@ -1004,6 +1012,7 @@ func (h *DockerHandle) collectStats() {
 				if cpuDelta > 0.0 && systemDelta > 0.0 {
 					cs.Percent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0
 				}
+
 				h.resourceUsageLock.Lock()
 				h.resourceUsage = &cstructs.TaskResourceUsage{
 					ResourceUsage: &cstructs.ResourceUsage{

client/driver/executor/executor.go

@@ -36,6 +36,12 @@ const (
 	pidScanInterval = 5 * time.Second
 )
 
+var (
+	// The statistics the basic executor exposes
+	ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"}
+	ExecutorBasicMeasuredCpuStats = []string{"SystemMode", "UserMode", "Percent"}
+)
+
 // Executor is the interface which allows a driver to launch and supervise
 // a process
 type Executor interface {
@@ -502,12 +508,14 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro
 		if memInfo, err := p.MemoryInfo(); err == nil {
 			ms.RSS = memInfo.RSS
 			ms.Swap = memInfo.Swap
+			ms.Measured = ExecutorBasicMeasuredMemStats
 		}
 
 		cs := &cstructs.CpuStats{}
 		if cpuStats, err := p.Times(); err == nil {
 			cs.SystemMode = cpuStats.System
 			cs.UserMode = cpuStats.User
+			cs.Measured = ExecutorBasicMeasuredCpuStats
 
 			// calculate cpu usage percent
 			cs.Percent = pid.cpuStats.Percent(cpuStats.Total())
@@ -766,11 +774,13 @@ func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstruct
 		SystemMode: systemModeCPU,
 		UserMode:   userModeCPU,
 		Percent:    percent,
+		Measured:   ExecutorBasicMeasuredMemStats,
 	}
 
 	totalMemory := &cstructs.MemoryStats{
-		RSS:  totalRSS,
-		Swap: totalSwap,
+		RSS:      totalRSS,
+		Swap:     totalSwap,
+		Measured: ExecutorBasicMeasuredCpuStats,
 	}
 
 	resourceUsage := cstructs.ResourceUsage{

client/driver/executor/executor_linux.go

@@ -24,7 +24,6 @@ import (
 )
 
 var (
-
 	// A mapping of directories on the host OS to attempt to embed inside each
 	// task's chroot.
 	chrootEnv = map[string]string{
@@ -38,9 +37,15 @@ var (
 		"/usr":            "/usr",
 	}
 
+	// clockTicks is the clocks per second of the machine
 	clockTicks = uint64(system.GetClockTicks())
 
+	// nanosecondsInSecond is the number of nanoseconds in a second.
 	nanosecondsInSecond = uint64(1000000000)
+
+	// The statistics the executor exposes when using cgroups
+	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage", "KernelUsage", "KernelMaxUsage"}
+	ExecutorCgroupMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"}
 )
 
 // configureIsolation configures chroot and creates cgroups
@@ -156,6 +161,7 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
 		MaxUsage:       maxUsage,
 		KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
 		KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
+		Measured:       ExecutorCgroupMeasuredMemStats,
 	}
 
 	// CPU Related Stats
@@ -171,6 +177,7 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
 		UserMode:         float64(umTicks),
 		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
 		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
+		Measured:         ExecutorCgroupMeasuredCpuStats,
 	}
 	if e.cpuStats != nil {
 		cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage / nanosecondsInSecond))

client/driver/structs/structs.go

@@ -93,6 +93,9 @@ type MemoryStats struct {
 	MaxUsage       uint64
 	KernelUsage    uint64
 	KernelMaxUsage uint64
+
+	// A list of fields whose values were actually sampled
+	Measured []string
 }
 
 // CpuStats holds cpu usage related stats
@@ -102,6 +105,9 @@ type CpuStats struct {
 	ThrottledPeriods uint64
 	ThrottledTime    uint64
 	Percent          float64
+
+	// A list of fields whose values were actually sampled
+	Measured []string
 }
 
 // ResourceUsage holds information related to cpu and memory stats