kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-01 16:05:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

e2e: add reporting to flaky spread test (#9115)

Browse Source 
The spread test is infrequently flaky and it's hard to extract what's actually
happening. If the test fails, dump all the allocation metrics so that we can
debug the behavior.
This commit is contained in:
Tim Gross
2020-10-16 11:01:07 -04:00
committed by GitHub
parent 3e827ccb43
commit 2b11e52fe9
1 changed files with 29 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
e2e/spread/spread.go
View File

@@ -1,10 +1,14 @@
package spread
import (
"github.com/hashicorp/nomad/e2e/framework"
"fmt"
"strings"
"github.com/stretchr/testify/require"
"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/e2e/e2eutil"
"github.com/hashicorp/nomad/e2e/framework"
"github.com/hashicorp/nomad/helper/uuid"
)
@@ -66,6 +70,7 @@ func (tc *SpreadTest) TestMultipleSpreads(f *framework.F) {
jobAllocs := nomadClient.Allocations()
dcToAllocs := make(map[string]int)
rackToAllocs := make(map[string]int)
allocMetrics := make(map[string]*api.AllocationMetric)
require := require.New(f.T())
// Verify spread score and alloc distribution
@@ -74,6 +79,7 @@ func (tc *SpreadTest) TestMultipleSpreads(f *framework.F) {
require.Nil(err)
require.NotEmpty(alloc.Metrics.ScoreMetaData)
allocMetrics[allocStub.ID] = alloc.Metrics
node, _, err := nomadClient.Nodes().Info(alloc.NodeID, nil)
require.Nil(err)
@@ -87,15 +93,35 @@ func (tc *SpreadTest) TestMultipleSpreads(f *framework.F) {
expectedDcToAllocs := make(map[string]int)
expectedDcToAllocs["dc1"] = 5
expectedDcToAllocs["dc2"] = 5
require.Equal(expectedDcToAllocs, dcToAllocs)
require.Equal(expectedDcToAllocs, dcToAllocs, report(allocMetrics))
expectedRackToAllocs := make(map[string]int)
expectedRackToAllocs["r1"] = 7
expectedRackToAllocs["r2"] = 3
require.Equal(expectedRackToAllocs, rackToAllocs)
require.Equal(expectedRackToAllocs, rackToAllocs, report(allocMetrics))
}
func report(metrics map[string]*api.AllocationMetric) string {
var s strings.Builder
for allocID, m := range metrics {
s.WriteString("Alloc ID: " + allocID + "\n")
s.WriteString(fmt.Sprintf(" NodesEvaluated: %d\n", m.NodesEvaluated))
s.WriteString(fmt.Sprintf(" NodesAvailable: %#v\n", m.NodesAvailable))
s.WriteString(fmt.Sprintf(" ClassFiltered: %#v\n", m.ClassFiltered))
s.WriteString(fmt.Sprintf(" ConstraintFiltered: %#v\n", m.ConstraintFiltered))
s.WriteString(fmt.Sprintf(" NodesExhausted: %d\n", m.NodesExhausted))
s.WriteString(fmt.Sprintf(" ClassExhausted: %#v\n", m.ClassExhausted))
s.WriteString(fmt.Sprintf(" DimensionExhausted: %#v\n", m.DimensionExhausted))
s.WriteString(fmt.Sprintf(" QuotaExhausted: %#v\n", m.QuotaExhausted))
for _, nodeMeta := range m.ScoreMetaData {
s.WriteString(fmt.Sprintf(" NodeID: %s, NormScore: %f, Scores: %#v\n",
nodeMeta.NodeID, nodeMeta.NormScore, nodeMeta.Scores))
}
}
return s.String()
}
func (tc *SpreadTest) AfterEach(f *framework.F) {
nomadClient := tc.Nomad()
jobs := nomadClient.Jobs()