mirror of
https://github.com/kemko/nomad.git
synced 2026-01-01 16:05:42 +03:00
b4b682b7fa
Adds a stanza for both Host Volumes and CSI Volumes to the the CLI output for `nomad alloc status`. Mostly relies on information already in the API structs, but in the case where there are CSI Volumes we need to make extra API calls to get the volume status. To reduce overhead, these extra calls are hidden behind the `-verbose` flag.
464 lines
13 KiB
Go
464 lines
13 KiB
Go
package command
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hashicorp/nomad/command/agent"
|
|
"github.com/hashicorp/nomad/helper/uuid"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/testutil"
|
|
"github.com/mitchellh/cli"
|
|
"github.com/posener/complete"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestAllocStatusCommand_Implements(t *testing.T) {
|
|
t.Parallel()
|
|
var _ cli.Command = &AllocStatusCommand{}
|
|
}
|
|
|
|
func TestAllocStatusCommand_Fails(t *testing.T) {
|
|
t.Parallel()
|
|
srv, _, url := testServer(t, false, nil)
|
|
defer srv.Shutdown()
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
|
|
// Fails on misuse
|
|
if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
|
|
t.Fatalf("expected exit code 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) {
|
|
t.Fatalf("expected help output, got: %s", out)
|
|
}
|
|
ui.ErrorWriter.Reset()
|
|
|
|
// Fails on connection failure
|
|
if code := cmd.Run([]string{"-address=nope", "foobar"}); code != 1 {
|
|
t.Fatalf("expected exit code 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error querying allocation") {
|
|
t.Fatalf("expected failed query error, got: %s", out)
|
|
}
|
|
ui.ErrorWriter.Reset()
|
|
|
|
// Fails on missing alloc
|
|
if code := cmd.Run([]string{"-address=" + url, "26470238-5CF2-438F-8772-DC67CFB0705C"}); code != 1 {
|
|
t.Fatalf("expected exit 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, "No allocation(s) with prefix or id") {
|
|
t.Fatalf("expected not found error, got: %s", out)
|
|
}
|
|
ui.ErrorWriter.Reset()
|
|
|
|
// Fail on identifier with too few characters
|
|
if code := cmd.Run([]string{"-address=" + url, "2"}); code != 1 {
|
|
t.Fatalf("expected exit 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, "must contain at least two characters.") {
|
|
t.Fatalf("expected too few characters error, got: %s", out)
|
|
}
|
|
ui.ErrorWriter.Reset()
|
|
|
|
// Identifiers with uneven length should produce a query result
|
|
if code := cmd.Run([]string{"-address=" + url, "123"}); code != 1 {
|
|
t.Fatalf("expected exit 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, "No allocation(s) with prefix or id") {
|
|
t.Fatalf("expected not found error, got: %s", out)
|
|
}
|
|
ui.ErrorWriter.Reset()
|
|
|
|
// Failed on both -json and -t options are specified
|
|
if code := cmd.Run([]string{"-address=" + url, "-json", "-t", "{{.ID}}"}); code != 1 {
|
|
t.Fatalf("expected exit 1, got: %d", code)
|
|
}
|
|
if out := ui.ErrorWriter.String(); !strings.Contains(out, "Both json and template formatting are not allowed") {
|
|
t.Fatalf("expected getting formatter error, got: %s", out)
|
|
}
|
|
}
|
|
|
|
func TestAllocStatusCommand_Run(t *testing.T) {
|
|
t.Parallel()
|
|
srv, client, url := testServer(t, true, nil)
|
|
defer srv.Shutdown()
|
|
|
|
// Wait for a node to be ready
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
nodes, _, err := client.Nodes().List(nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, node := range nodes {
|
|
if _, ok := node.Drivers["mock_driver"]; ok &&
|
|
node.Status == structs.NodeStatusReady {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, fmt.Errorf("no ready nodes")
|
|
}, func(err error) {
|
|
t.Fatalf("err: %v", err)
|
|
})
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
|
|
jobID := "job1_sfx"
|
|
job1 := testJob(jobID)
|
|
resp, _, err := client.Jobs().Register(job1, nil)
|
|
if err != nil {
|
|
t.Fatalf("err: %s", err)
|
|
}
|
|
if code := waitForSuccess(ui, client, fullId, t, resp.EvalID); code != 0 {
|
|
t.Fatalf("status code non zero saw %d", code)
|
|
}
|
|
// get an alloc id
|
|
allocId1 := ""
|
|
nodeName := ""
|
|
if allocs, _, err := client.Jobs().Allocations(jobID, false, nil); err == nil {
|
|
if len(allocs) > 0 {
|
|
allocId1 = allocs[0].ID
|
|
nodeName = allocs[0].NodeName
|
|
}
|
|
}
|
|
if allocId1 == "" {
|
|
t.Fatal("unable to find an allocation")
|
|
}
|
|
|
|
if code := cmd.Run([]string{"-address=" + url, allocId1}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out := ui.OutputWriter.String()
|
|
if !strings.Contains(out, "Created") {
|
|
t.Fatalf("expected to have 'Created' but saw: %s", out)
|
|
}
|
|
|
|
if !strings.Contains(out, "Modified") {
|
|
t.Fatalf("expected to have 'Modified' but saw: %s", out)
|
|
}
|
|
|
|
nodeNameRegexpStr := fmt.Sprintf(`\nNode Name\s+= %s\n`, regexp.QuoteMeta(nodeName))
|
|
require.Regexp(t, regexp.MustCompile(nodeNameRegexpStr), out)
|
|
|
|
ui.OutputWriter.Reset()
|
|
|
|
if code := cmd.Run([]string{"-address=" + url, "-verbose", allocId1}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out = ui.OutputWriter.String()
|
|
if !strings.Contains(out, allocId1) {
|
|
t.Fatal("expected to find alloc id in output")
|
|
}
|
|
if !strings.Contains(out, "Created") {
|
|
t.Fatalf("expected to have 'Created' but saw: %s", out)
|
|
}
|
|
ui.OutputWriter.Reset()
|
|
|
|
// Try the query with an even prefix that includes the hyphen
|
|
if code := cmd.Run([]string{"-address=" + url, allocId1[:13]}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out = ui.OutputWriter.String()
|
|
if !strings.Contains(out, "Created") {
|
|
t.Fatalf("expected to have 'Created' but saw: %s", out)
|
|
}
|
|
ui.OutputWriter.Reset()
|
|
|
|
if code := cmd.Run([]string{"-address=" + url, "-verbose", allocId1}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out = ui.OutputWriter.String()
|
|
if !strings.Contains(out, allocId1) {
|
|
t.Fatal("expected to find alloc id in output")
|
|
}
|
|
ui.OutputWriter.Reset()
|
|
|
|
}
|
|
|
|
func TestAllocStatusCommand_RescheduleInfo(t *testing.T) {
|
|
t.Parallel()
|
|
srv, client, url := testServer(t, true, nil)
|
|
defer srv.Shutdown()
|
|
|
|
// Wait for a node to be ready
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
nodes, _, err := client.Nodes().List(nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, node := range nodes {
|
|
if node.Status == structs.NodeStatusReady {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, fmt.Errorf("no ready nodes")
|
|
}, func(err error) {
|
|
t.Fatalf("err: %v", err)
|
|
})
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
// Test reschedule attempt info
|
|
require := require.New(t)
|
|
state := srv.Agent.Server().State()
|
|
a := mock.Alloc()
|
|
a.Metrics = &structs.AllocMetric{}
|
|
nextAllocId := uuid.Generate()
|
|
a.NextAllocation = nextAllocId
|
|
a.RescheduleTracker = &structs.RescheduleTracker{
|
|
Events: []*structs.RescheduleEvent{
|
|
{
|
|
RescheduleTime: time.Now().Add(-2 * time.Minute).UTC().UnixNano(),
|
|
PrevAllocID: uuid.Generate(),
|
|
PrevNodeID: uuid.Generate(),
|
|
},
|
|
},
|
|
}
|
|
require.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))
|
|
|
|
if code := cmd.Run([]string{"-address=" + url, a.ID}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out := ui.OutputWriter.String()
|
|
require.Contains(out, "Replacement Alloc ID")
|
|
require.Regexp(regexp.MustCompile(".*Reschedule Attempts\\s*=\\s*1/2"), out)
|
|
}
|
|
|
|
func TestAllocStatusCommand_ScoreMetrics(t *testing.T) {
|
|
t.Parallel()
|
|
srv, client, url := testServer(t, true, nil)
|
|
defer srv.Shutdown()
|
|
|
|
// Wait for a node to be ready
|
|
testutil.WaitForResult(func() (bool, error) {
|
|
nodes, _, err := client.Nodes().List(nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
for _, node := range nodes {
|
|
if node.Status == structs.NodeStatusReady {
|
|
return true, nil
|
|
}
|
|
}
|
|
return false, fmt.Errorf("no ready nodes")
|
|
}, func(err error) {
|
|
t.Fatalf("err: %v", err)
|
|
})
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
// Test node metrics
|
|
require := require.New(t)
|
|
state := srv.Agent.Server().State()
|
|
a := mock.Alloc()
|
|
mockNode1 := mock.Node()
|
|
mockNode2 := mock.Node()
|
|
a.Metrics = &structs.AllocMetric{
|
|
ScoreMetaData: []*structs.NodeScoreMeta{
|
|
{
|
|
NodeID: mockNode1.ID,
|
|
Scores: map[string]float64{
|
|
"binpack": 0.77,
|
|
"node-affinity": 0.5,
|
|
},
|
|
},
|
|
{
|
|
NodeID: mockNode2.ID,
|
|
Scores: map[string]float64{
|
|
"binpack": 0.75,
|
|
"node-affinity": 0.33,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
require.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))
|
|
|
|
if code := cmd.Run([]string{"-address=" + url, "-verbose", a.ID}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out := ui.OutputWriter.String()
|
|
require.Contains(out, "Placement Metrics")
|
|
require.Contains(out, mockNode1.ID)
|
|
require.Contains(out, mockNode2.ID)
|
|
|
|
// assert we sort headers alphabetically
|
|
require.Contains(out, "binpack node-affinity")
|
|
require.Contains(out, "final score")
|
|
}
|
|
|
|
func TestAllocStatusCommand_AutocompleteArgs(t *testing.T) {
|
|
assert := assert.New(t)
|
|
t.Parallel()
|
|
|
|
srv, _, url := testServer(t, true, nil)
|
|
defer srv.Shutdown()
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}}
|
|
|
|
// Create a fake alloc
|
|
state := srv.Agent.Server().State()
|
|
a := mock.Alloc()
|
|
assert.Nil(state.UpsertAllocs(1000, []*structs.Allocation{a}))
|
|
|
|
prefix := a.ID[:5]
|
|
args := complete.Args{Last: prefix}
|
|
predictor := cmd.AutocompleteArgs()
|
|
|
|
res := predictor.Predict(args)
|
|
assert.Equal(1, len(res))
|
|
assert.Equal(a.ID, res[0])
|
|
}
|
|
|
|
func TestAllocStatusCommand_HostVolumes(t *testing.T) {
|
|
t.Parallel()
|
|
// We have to create a tempdir for the host volume even though we're
|
|
// not going to use it b/c the server validates the config on startup
|
|
tmpDir, err := ioutil.TempDir("", "vol0")
|
|
if err != nil {
|
|
t.Fatalf("unable to create tempdir for test: %v", err)
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
vol0 := uuid.Generate()
|
|
srv, _, url := testServer(t, true, func(c *agent.Config) {
|
|
c.Client.HostVolumes = []*structs.ClientHostVolumeConfig{
|
|
{
|
|
Name: vol0,
|
|
Path: tmpDir,
|
|
ReadOnly: false,
|
|
},
|
|
}
|
|
})
|
|
defer srv.Shutdown()
|
|
state := srv.Agent.Server().State()
|
|
|
|
// Upsert the job and alloc
|
|
node := mock.Node()
|
|
alloc := mock.Alloc()
|
|
alloc.Metrics = &structs.AllocMetric{}
|
|
alloc.NodeID = node.ID
|
|
job := alloc.Job
|
|
job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{
|
|
vol0: {
|
|
Name: vol0,
|
|
Type: structs.VolumeTypeHost,
|
|
Source: tmpDir,
|
|
},
|
|
}
|
|
job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{
|
|
{
|
|
Volume: vol0,
|
|
Destination: "/var/www",
|
|
ReadOnly: true,
|
|
PropagationMode: "private",
|
|
},
|
|
}
|
|
// fakes the placement enough so that we have something to iterate
|
|
// on in 'nomad alloc status'
|
|
alloc.TaskStates = map[string]*structs.TaskState{
|
|
"web": &structs.TaskState{
|
|
Events: []*structs.TaskEvent{
|
|
structs.NewTaskEvent("test event").SetMessage("test msg"),
|
|
},
|
|
},
|
|
}
|
|
summary := mock.JobSummary(alloc.JobID)
|
|
require.NoError(t, state.UpsertJobSummary(1004, summary))
|
|
require.NoError(t, state.UpsertAllocs(1005, []*structs.Allocation{alloc}))
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
if code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out := ui.OutputWriter.String()
|
|
require.Contains(t, out, "Host Volumes")
|
|
require.Contains(t, out, fmt.Sprintf("%s true", vol0))
|
|
require.NotContains(t, out, "CSI Volumes")
|
|
}
|
|
|
|
func TestAllocStatusCommand_CSIVolumes(t *testing.T) {
|
|
t.Parallel()
|
|
srv, _, url := testServer(t, true, nil)
|
|
defer srv.Shutdown()
|
|
state := srv.Agent.Server().State()
|
|
|
|
// Upsert the node, plugin, and volume
|
|
vol0 := uuid.Generate()
|
|
node := mock.Node()
|
|
node.CSINodePlugins = map[string]*structs.CSIInfo{
|
|
"minnie": {
|
|
PluginID: "minnie",
|
|
Healthy: true,
|
|
NodeInfo: &structs.CSINodeInfo{},
|
|
},
|
|
}
|
|
err := state.UpsertNode(1001, node)
|
|
require.NoError(t, err)
|
|
|
|
vols := []*structs.CSIVolume{{
|
|
ID: vol0,
|
|
Namespace: "notTheNamespace",
|
|
PluginID: "minnie",
|
|
AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter,
|
|
AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem,
|
|
Topologies: []*structs.CSITopology{{
|
|
Segments: map[string]string{"foo": "bar"},
|
|
}},
|
|
}}
|
|
err = state.CSIVolumeRegister(1002, vols)
|
|
require.NoError(t, err)
|
|
|
|
// Upsert the job and alloc
|
|
alloc := mock.Alloc()
|
|
alloc.Metrics = &structs.AllocMetric{}
|
|
alloc.NodeID = node.ID
|
|
job := alloc.Job
|
|
job.TaskGroups[0].Volumes = map[string]*structs.VolumeRequest{
|
|
vol0: {
|
|
Name: vol0,
|
|
Type: structs.VolumeTypeCSI,
|
|
Source: "/tmp/vol0",
|
|
},
|
|
}
|
|
job.TaskGroups[0].Tasks[0].VolumeMounts = []*structs.VolumeMount{
|
|
{
|
|
Volume: vol0,
|
|
Destination: "/var/www",
|
|
ReadOnly: true,
|
|
PropagationMode: "private",
|
|
},
|
|
}
|
|
// if we don't set a task state, there's nothing to iterate on alloc status
|
|
alloc.TaskStates = map[string]*structs.TaskState{
|
|
"web": &structs.TaskState{
|
|
Events: []*structs.TaskEvent{
|
|
structs.NewTaskEvent("test event").SetMessage("test msg"),
|
|
},
|
|
},
|
|
}
|
|
summary := mock.JobSummary(alloc.JobID)
|
|
require.NoError(t, state.UpsertJobSummary(1004, summary))
|
|
require.NoError(t, state.UpsertAllocs(1005, []*structs.Allocation{alloc}))
|
|
|
|
ui := new(cli.MockUi)
|
|
cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}}
|
|
if code := cmd.Run([]string{"-address=" + url, "-verbose", alloc.ID}); code != 0 {
|
|
t.Fatalf("expected exit 0, got: %d", code)
|
|
}
|
|
out := ui.OutputWriter.String()
|
|
require.Contains(t, out, "CSI Volumes")
|
|
require.Contains(t, out, fmt.Sprintf("%s minnie", vol0))
|
|
require.NotContains(t, out, "Host Volumes")
|
|
}
|