From cb3a03c8291090c8b30233352e0f9fd9c465fa97 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 2 Nov 2017 10:55:10 -0700 Subject: [PATCH] Make unable-to-gc log level adaptive WARNing when someone has over 50 non-terminal allocs was just too confusing. Tested manually with `gc_max_allocs = 10` and bumping a job from `count = 19` to `count = 21`: ``` 2017/11/02 17:54:21.076132 [INFO] client.gc: garbage collection due to number of allocations (19) is over the limit (10) skipped because no terminal allocations ... 2017/11/02 17:54:48.634529 [WARN] client.gc: garbage collection due to number of allocations (21) is over the limit (10) skipped because no terminal allocations ``` --- client/gc.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/gc.go b/client/gc.go index a7332ebc4..cdbf876e7 100644 --- a/client/gc.go +++ b/client/gc.go @@ -130,6 +130,7 @@ func (a *AllocGarbageCollector) keepUsageBelowThreshold() error { // See if we are below thresholds for used disk space and inode usage diskStats := a.statsCollector.Stats().AllocDirStats reason := "" + level := "WARN" liveAllocs := a.allocCounter.NumAllocs() @@ -141,6 +142,10 @@ func (a *AllocGarbageCollector) keepUsageBelowThreshold() error { reason = fmt.Sprintf("inode usage of %.0f is over gc threshold of %.0f", diskStats.InodesUsedPercent, a.config.InodeUsageThreshold) case liveAllocs > a.config.MaxAllocs: + // if we're unable to gc, don't WARN until at least 2x over limit + if liveAllocs < (a.config.MaxAllocs * 2) { + level = "INFO" + } reason = fmt.Sprintf("number of allocations (%d) is over the limit (%d)", liveAllocs, a.config.MaxAllocs) } @@ -152,7 +157,7 @@ func (a *AllocGarbageCollector) keepUsageBelowThreshold() error { // Collect an allocation gcAlloc := a.allocRunners.Pop() if gcAlloc == nil { - a.logger.Printf("[WARN] client.gc: garbage collection due to %s skipped because no terminal allocations", reason) + a.logger.Printf("[%s] client.gc: garbage collection due to %s skipped because no terminal allocations", level, reason) break }