From 98068678f1d587922f89d3dc1c669fb6e82e8f38 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Fri, 20 May 2016 02:05:48 -0700 Subject: [PATCH] Implemented nomad cpu percentage calculator --- client/driver/executor/executor.go | 52 ++++++++++++------------ client/driver/executor/executor_basic.go | 2 +- client/driver/executor/executor_linux.go | 15 +++++-- client/stats/cpu.go | 40 +++++++----------- command/stats.go | 2 +- 5 files changed, 53 insertions(+), 58 deletions(-) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 74fdfa4a9..5a15ce763 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -130,6 +130,11 @@ type ProcessState struct { Time time.Time } +type NomadPid struct { + pid int + cpuStats *stats.CpuStats +} + // SyslogServerState holds the address and islation information of a launched // syslog server type SyslogServerState struct { @@ -154,7 +159,7 @@ type UniversalExecutor struct { ctx *ExecutorContext command *ExecCommand - pids []int + pids []*NomadPid taskDir string exitState *ProcessState processExited chan interface{} @@ -181,10 +186,7 @@ func NewExecutor(logger *log.Logger) Executor { exec := &UniversalExecutor{ logger: logger, processExited: make(chan interface{}), - } - - if cpuStats, err := stats.NewCpuStats(); err == nil { - exec.cpuStats = cpuStats + cpuStats: stats.NewCpuStats(logger), } return exec @@ -483,31 +485,27 @@ func (e *UniversalExecutor) PidStats() (map[int]*cstructs.TaskResourceUsage, err stats := make(map[int]*cstructs.TaskResourceUsage) ts := time.Now() for _, pid := range e.pids { - p, err := process.NewProcess(int32(pid)) + p, err := process.NewProcess(int32(pid.pid)) if err != nil { - e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid) + e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid.pid) continue } - memInfo, err := p.MemoryInfo() - if err != nil { - e.logger.Printf("[DEBUG] executor: unable to get memory stats for process: %v", pid) - } - cpuStats, err := p.Times() - if err != nil { - e.logger.Printf("[DEBUG] executor: unable to get cpu stats for process: %v", pid) - } - ms := &cstructs.MemoryStats{ - RSS: memInfo.RSS, - Swap: memInfo.Swap, + ms := &cstructs.MemoryStats{} + if memInfo, err := p.MemoryInfo(); err == nil { + ms.RSS = memInfo.RSS + ms.Swap = memInfo.Swap } - percent, _ := p.Percent(0) - cs := &cstructs.CpuUsage{ - SystemMode: cpuStats.System, - UserMode: cpuStats.User, - Percent: percent, + cs := &cstructs.CpuUsage{} + if cpuStats, err := p.Times(); err == nil { + cs.SystemMode = cpuStats.System + cs.UserMode = cpuStats.User + + // calculate cpu usage percent + cs.Percent = pid.cpuStats.Percent(cpuStats.Total()) + e.logger.Printf("DIPTANU CPU PERCENT for pid %v: %v", pid.pid, cs.Percent) } - stats[pid] = &cstructs.TaskResourceUsage{MemoryStats: ms, CpuStats: cs, Timestamp: ts} + stats[pid.pid] = &cstructs.TaskResourceUsage{MemoryStats: ms, CpuStats: cs, Timestamp: ts} } return stats, nil @@ -698,7 +696,7 @@ func (e *UniversalExecutor) collectPids() { // scanPids scans all the pids on the machine running the current executor and // returns the child processes of the executor. -func (e *UniversalExecutor) scanPids() ([]int, error) { +func (e *UniversalExecutor) scanPids() ([]*NomadPid, error) { processFamily := make(map[int]struct{}) processFamily[os.Getpid()] = struct{}{} pids, err := ps.Processes() @@ -712,9 +710,9 @@ func (e *UniversalExecutor) scanPids() ([]int, error) { processFamily[pid.Pid()] = struct{}{} } } - res := make([]int, 0, len(processFamily)) + res := make([]*NomadPid, 0, len(processFamily)) for pid := range processFamily { - res = append(res, pid) + res = append(res, &NomadPid{pid, stats.NewCpuStats(e.logger)}) } return res, nil } diff --git a/client/driver/executor/executor_basic.go b/client/driver/executor/executor_basic.go index cb87f0301..b458ea549 100644 --- a/client/driver/executor/executor_basic.go +++ b/client/driver/executor/executor_basic.go @@ -35,6 +35,6 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { return e.resourceUsagePids() } -func (e *UniversalExecutor) getAllPids() ([]int, error) { +func (e *UniversalExecutor) getAllPids() ([]*NomadPid, error) { return e.scanPids() } diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index 2a77ad907..56ae2f734 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" cstructs "github.com/hashicorp/nomad/client/driver/structs" + "github.com/hashicorp/nomad/client/stats" "github.com/hashicorp/nomad/nomad/structs" ) @@ -164,7 +165,7 @@ func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) { ThrottledTime: stats.CpuStats.ThrottlingData.ThrottledTime, } if e.cpuStats != nil { - cs.Percent = e.cpuStats.Percent(totalProcessCPUUsage) + cs.Percent = e.cpuStats.Percent(float64(totalProcessCPUUsage)) } return &cstructs.TaskResourceUsage{MemoryStats: ms, CpuStats: cs, Timestamp: time.Now()}, nil } @@ -239,10 +240,18 @@ func (e *UniversalExecutor) removeChrootMounts() error { return e.ctx.AllocDir.UnmountAll() } -func (e *UniversalExecutor) getAllPids() ([]int, error) { +func (e *UniversalExecutor) getAllPids() ([]*NomadPid, error) { if e.command.ResourceLimits { manager := getCgroupManager(e.groups, e.cgPaths) - return manager.GetAllPids() + pids, err := manager.GetAllPids() + if err != nil { + return nil, err + } + np := make([]*NomadPid, len(pids)) + for idx, pid := range pids { + np[idx] = &NomadPid{pid, stats.NewCpuStats(e.logger)} + } + return np, nil } return e.scanPids() } diff --git a/client/stats/cpu.go b/client/stats/cpu.go index 351f81dc5..42e1dea7b 100644 --- a/client/stats/cpu.go +++ b/client/stats/cpu.go @@ -1,41 +1,29 @@ package stats import ( - "github.com/shirou/gopsutil/cpu" + "log" + "runtime" + "time" ) type CpuStats struct { - prevSystemUsage float64 - prevProcessUsage uint64 + prevProcessUsage float64 + prevTime time.Time totalCpus int + logger *log.Logger } -func NewCpuStats() (*CpuStats, error) { - cpuInfo, err := cpu.Info() - if err != nil { - return nil, err - } - return &CpuStats{totalCpus: len(cpuInfo)}, nil +func NewCpuStats(logger *log.Logger) *CpuStats { + numCpus := runtime.NumCPU() + return &CpuStats{totalCpus: numCpus, logger: logger} } -func (c *CpuStats) Percent(currentProcessUsage uint64) float64 { - percent := 0.0 - - sysCPUStats, err := cpu.Times(false) - if err != nil { - return 0 - } - currentSysUsage := 0.0 - for _, cpuStat := range sysCPUStats { - currentSysUsage += cpuStat.Total() * 1000000000 - } - - delta := float64(currentProcessUsage) - float64(c.prevProcessUsage) - sysDelta := float64(currentSysUsage) - float64(c.prevSystemUsage) - - percent = (delta / sysDelta) * float64(c.totalCpus) * 100.0 - c.prevSystemUsage = currentSysUsage +func (c *CpuStats) Percent(currentProcessUsage float64) float64 { + procDelta := float64(currentProcessUsage) - float64(c.prevProcessUsage) + delta := (time.Now().Sub(c.prevTime).Seconds()) * float64(c.totalCpus) + percent := ((procDelta / delta) * 1000) * float64(c.totalCpus) c.prevProcessUsage = currentProcessUsage return percent + } diff --git a/command/stats.go b/command/stats.go index aa9dd6b31..c9b37b719 100644 --- a/command/stats.go +++ b/command/stats.go @@ -137,7 +137,7 @@ func (f *StatsCommand) printTaskResourceUsage(task string, resourceUsage map[str out = make([]string, 2) out[0] = "Percent|Throttled Periods|Throttled Time" percent := strconv.FormatFloat(tu.CpuStats.Percent, 'f', 2, 64) - out[1] = fmt.Sprintf("%v|%v|%v|%v", percent, + out[1] = fmt.Sprintf("%v|%v|%v", percent, tu.CpuStats.ThrottledPeriods, tu.CpuStats.ThrottledTime) f.Ui.Output(formatList(out)) }