diff --git a/api/tasks.go b/api/tasks.go index c82db5ba2..14d0a39a1 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -12,6 +12,7 @@ type MemoryStats struct { MaxUsage uint64 KernelUsage uint64 KernelMaxUsage uint64 + Measured []string } // CpuStats holds cpu usage related stats @@ -21,6 +22,7 @@ type CpuStats struct { ThrottledPeriods uint64 ThrottledTime uint64 Percent float64 + Measured []string } // ResourceUsage holds information related to cpu and memory stats diff --git a/client/driver/docker.go b/client/driver/docker.go index 35c55fe5d..3cd6b9ee1 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -32,6 +32,10 @@ var ( // We store the client globally to cache the connection to the docker daemon. createClient sync.Once client *docker.Client + + // The statistics the Docker driver exposes + DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage"} + DockerMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"} ) const ( @@ -851,6 +855,7 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err) } + go h.collectStats() go h.run() return h, nil } @@ -989,6 +994,7 @@ func (h *DockerHandle) collectStats() { Cache: s.MemoryStats.Stats.Cache, Swap: s.MemoryStats.Stats.Swap, MaxUsage: s.MemoryStats.MaxUsage, + Measured: DockerMeasuredMemStats, } cs := &cstructs.CpuStats{ @@ -996,7 +1002,9 @@ func (h *DockerHandle) collectStats() { UserMode: float64(s.CPUStats.CPUUsage.UsageInKernelmode), ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, + Measured: DockerMeasuredCpuStats, } + // Calculate percentage cs.Percent = 0.0 cpuDelta := float64(s.CPUStats.CPUUsage.TotalUsage) - float64(s.PreCPUStats.CPUUsage.TotalUsage) @@ -1004,6 +1012,7 @@ func (h *DockerHandle) collectStats() { if cpuDelta > 0.0 && systemDelta > 0.0 { cs.Percent = (cpuDelta / systemDelta) * float64(len(s.CPUStats.CPUUsage.PercpuUsage)) * 100.0 } + h.resourceUsageLock.Lock() h.resourceUsage = &cstructs.TaskResourceUsage{ ResourceUsage: &cstructs.ResourceUsage{ diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 582a4cefd..57fbf9671 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -36,6 +36,12 @@ const ( pidScanInterval = 5 * time.Second ) +var ( + // The statistics the basic executor exposes + ExecutorBasicMeasuredMemStats = []string{"RSS", "Swap"} + ExecutorBasicMeasuredCpuStats = []string{"SystemMode", "UserMode", "Percent"} +) + // Executor is the interface which allows a driver to launch and supervise // a process type Executor interface { @@ -502,12 +508,14 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro if memInfo, err := p.MemoryInfo(); err == nil { ms.RSS = memInfo.RSS ms.Swap = memInfo.Swap + ms.Measured = ExecutorBasicMeasuredMemStats } cs := &cstructs.CpuStats{} if cpuStats, err := p.Times(); err == nil { cs.SystemMode = cpuStats.System cs.UserMode = cpuStats.User + cs.Measured = ExecutorBasicMeasuredCpuStats // calculate cpu usage percent cs.Percent = pid.cpuStats.Percent(cpuStats.Total()) @@ -766,11 +774,13 @@ func (e *UniversalExecutor) aggregatedResourceUsage(pidStats map[string]*cstruct SystemMode: systemModeCPU, UserMode: userModeCPU, Percent: percent, + Measured: ExecutorBasicMeasuredMemStats, } totalMemory := &cstructs.MemoryStats{ - RSS: totalRSS, - Swap: totalSwap, + RSS: totalRSS, + Swap: totalSwap, + Measured: ExecutorBasicMeasuredCpuStats, } resourceUsage := cstructs.ResourceUsage{ diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index 45f52cb38..cc838510c 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -24,7 +24,6 @@ import ( ) var ( - // A mapping of directories on the host OS to attempt to embed inside each // task's chroot. chrootEnv = map[string]string{ @@ -38,9 +37,15 @@ var ( "/usr": "/usr", } + // clockTicks is the clocks per second of the machine clockTicks = uint64(system.GetClockTicks()) + // nanosecondsInSecond is the number of nanoseconds in a second. nanosecondsInSecond = uint64(1000000000) + + // The statistics the executor exposes when using cgroups + ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "MaxUsage", "KernelUsage", "KernelMaxUsage"} + ExecutorCgroupMeasuredCpuStats = []string{"SystemMode", "UserMode", "ThrottledPeriods", "ThrottledTime", "Percent"} ) // configureIsolation configures chroot and creates cgroups @@ -156,6 +161,7 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { MaxUsage: maxUsage, KernelUsage: stats.MemoryStats.KernelUsage.Usage, KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage, + Measured: ExecutorCgroupMeasuredMemStats, } // CPU Related Stats @@ -171,6 +177,7 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { UserMode: float64(umTicks), ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods, ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime, + Measured: ExecutorCgroupMeasuredCpuStats, } if e.cpuStats != nil { cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage / nanosecondsInSecond)) diff --git a/client/driver/structs/structs.go b/client/driver/structs/structs.go index d089b9813..41be827ba 100644 --- a/client/driver/structs/structs.go +++ b/client/driver/structs/structs.go @@ -93,6 +93,9 @@ type MemoryStats struct { MaxUsage uint64 KernelUsage uint64 KernelMaxUsage uint64 + + // A list of fields whose values were actually sampled + Measured []string } // CpuStats holds cpu usage related stats @@ -102,6 +105,9 @@ type CpuStats struct { ThrottledPeriods uint64 ThrottledTime uint64 Percent float64 + + // A list of fields whose values were actually sampled + Measured []string } // ResourceUsage holds information related to cpu and memory stats