mirror of
https://github.com/kemko/nomad.git
synced 2026-01-03 17:05:43 +03:00
374e987b9b
In cgroups v2, a different map of memory stats is available from the kernel than in v1. The Docker API reflects this change. But there are equivalent values in the map for RSS (anonymously mapped memory) and cache (filesystem cache and tmpfs), which the Docker driver is not currently emitting. Fallback to these alternate values when the cgroups v1 values are not available. Include the anonymous mapping in the "measured" allocation stats as "RSS" so that they both show up in allocation metrics. We can do this on both the `docker` driver and the Linux executor for `exec` and `java` drivers. Fixes: https://github.com/hashicorp/nomad/issues/19185 Ref: https://hashicorp.atlassian.net/browse/NMD-437 Ref: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files Ref: https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
89 lines
2.9 KiB
Go
89 lines
2.9 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
//go:build !windows
|
|
|
|
package util
|
|
|
|
import (
|
|
containerapi "github.com/docker/docker/api/types/container"
|
|
"github.com/hashicorp/nomad/client/lib/cpustats"
|
|
cstructs "github.com/hashicorp/nomad/client/structs"
|
|
)
|
|
|
|
var (
|
|
DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
|
|
|
|
// cgroup-v2 only exposes a subset of memory stats
|
|
DockerCgroupV1MeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"}
|
|
DockerCgroupV2MeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage"}
|
|
)
|
|
|
|
func DockerStatsToTaskResourceUsage(s *containerapi.StatsResponse, compute cpustats.Compute) *cstructs.TaskResourceUsage {
|
|
var (
|
|
totalCompute = compute.TotalCompute
|
|
totalCores = compute.NumCores
|
|
)
|
|
|
|
measuredMems := DockerCgroupV1MeasuredMemStats
|
|
|
|
// use a simple heuristic to check if cgroup-v2 is used.
|
|
// go-dockerclient doesn't distinguish between 0 and not-present value
|
|
if s.MemoryStats.MaxUsage == 0 && s.MemoryStats.Usage != 0 {
|
|
measuredMems = DockerCgroupV2MeasuredMemStats
|
|
}
|
|
|
|
cache := s.MemoryStats.Stats["cache"]
|
|
if cache == 0 {
|
|
// This is the equivalent stat for cgroups v2, including filesystem
|
|
// cache and tmpfs
|
|
cache = s.MemoryStats.Stats["file"]
|
|
}
|
|
rss := s.MemoryStats.Stats["rss"]
|
|
if rss == 0 {
|
|
// This is the equivalent stat of anonymous mappings for cgroups v2.
|
|
rss = s.MemoryStats.Stats["anon"]
|
|
}
|
|
|
|
// containerapi exposes memory stat file as a map. for the meaning of the
|
|
// source values, consult:
|
|
// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
|
|
// https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html#memory-interface-files
|
|
ms := &cstructs.MemoryStats{
|
|
MappedFile: s.MemoryStats.Stats["file_mapped"],
|
|
Cache: cache,
|
|
RSS: rss,
|
|
Swap: s.MemoryStats.Stats["swap"],
|
|
Usage: s.MemoryStats.Usage,
|
|
MaxUsage: s.MemoryStats.MaxUsage,
|
|
Measured: measuredMems,
|
|
}
|
|
|
|
cs := &cstructs.CpuStats{
|
|
ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
|
|
ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime,
|
|
Measured: DockerMeasuredCPUStats,
|
|
}
|
|
|
|
// Calculate percentage
|
|
cs.Percent = CalculateCPUPercent(
|
|
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
|
|
s.CPUStats.SystemUsage, s.PreCPUStats.SystemUsage, totalCores)
|
|
cs.SystemMode = CalculateCPUPercent(
|
|
s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
|
|
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, totalCores)
|
|
cs.UserMode = CalculateCPUPercent(
|
|
s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
|
|
s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, totalCores)
|
|
|
|
cs.TotalTicks = (cs.Percent / 100) * float64(totalCompute) / float64(totalCores)
|
|
|
|
return &cstructs.TaskResourceUsage{
|
|
ResourceUsage: &cstructs.ResourceUsage{
|
|
MemoryStats: ms,
|
|
CpuStats: cs,
|
|
},
|
|
Timestamp: s.Read.UTC().UnixNano(),
|
|
}
|
|
}
|