diff --git a/api/nodes.go b/api/nodes.go index 6a61ab90b..beb35070e 100644 --- a/api/nodes.go +++ b/api/nodes.go @@ -113,10 +113,32 @@ type Node struct { Status string StatusDescription string StatusUpdatedAt int64 + NodeEvents []*NodeEvent CreateIndex uint64 ModifyIndex uint64 } +// Subsystem denotes the subsystem where a node event took place. +type Subsystem string + +const ( + Drain Subsystem = "Drain" + Driver Subsystem = "Driver" + Heartbeat Subsystem = "Heartbeat" + Server Subsystem = "Server" + Cluster Subsystem = "Cluster" +) + +// NodeEvent is a single unit representing a node’s state change +type NodeEvent struct { + Message string + Subsystem Subsystem + Details map[string]string + Timestamp int64 + + CreateIndex uint64 +} + // HostStats represents resource usage stats of the host running a Nomad client type HostStats struct { Memory *HostMemoryStats diff --git a/api/nodes_test.go b/api/nodes_test.go index d020d84ae..7c9a8be97 100644 --- a/api/nodes_test.go +++ b/api/nodes_test.go @@ -134,6 +134,10 @@ func TestNodes_Info(t *testing.T) { if result.StatusUpdatedAt < startTime { t.Fatalf("start time: %v, status updated: %v", startTime, result.StatusUpdatedAt) } + + if len(result.NodeEvents) < 1 { + t.Fatalf("Expected at minimum the node register event to be populated: %+v", result) + } } func TestNodes_ToggleDrain(t *testing.T) { diff --git a/command/agent/node_endpoint_test.go b/command/agent/node_endpoint_test.go index 8bf56e328..113fa5de8 100644 --- a/command/agent/node_endpoint_test.go +++ b/command/agent/node_endpoint_test.go @@ -396,5 +396,11 @@ func TestHTTP_NodeQuery(t *testing.T) { if n.ID != node.ID { t.Fatalf("bad: %#v", n) } + if len(n.NodeEvents) != 1 { + t.Fatalf("Expected node registration event to be populated: %#v", n) + } + if n.NodeEvents[0].Message != "Node Registered" { + t.Fatalf("Expected node registration event to be first node event: %#v", n) + } }) } diff --git a/command/node_status.go b/command/node_status.go index 804e63d34..9b35ae758 100644 --- a/command/node_status.go +++ b/command/node_status.go @@ -332,6 +332,8 @@ func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { } c.Ui.Output(c.Colorize().Color(formatKV(basic))) + c.outputNodeStatusEvents(node) + // Get list of running allocations on the node runningAllocs, err := getRunningAllocs(client, node.ID) if err != nil { @@ -386,6 +388,34 @@ func (c *NodeStatusCommand) formatNode(client *api.Client, node *api.Node) int { } +func (c *NodeStatusCommand) outputNodeStatusEvents(node *api.Node) { + c.Ui.Output(c.Colorize().Color("\n[bold]Node Events ")) + c.outputNodeEvent(node.NodeEvents) +} + +func (c *NodeStatusCommand) outputNodeEvent(events []*api.NodeEvent) { + size := len(events) + nodeEvents := make([]string, size+1) + nodeEvents[0] = "Timestamp|Subsystem|Message|Details" + + for i, event := range events { + timestamp := formatUnixNanoTime(event.Timestamp) + subsystem := event.Subsystem + msg := event.Message + details := formatEventDetails(event.Details) + nodeEvents[size-i] = fmt.Sprintf("%s|%s|%s|%s", timestamp, subsystem, msg, details) + } + c.Ui.Output(formatList(nodeEvents)) +} + +func formatEventDetails(details map[string]string) string { + var output string + for k, v := range details { + output += fmt.Sprintf("%s: %s, ", k, v) + } + return output +} + func (c *NodeStatusCommand) formatAttributes(node *api.Node) { // Print the attributes keys := make([]string, len(node.Attributes)) diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index b0737f187..14b630fe5 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -112,7 +112,7 @@ func TestFSM_ApplyNodeEvent(t *testing.T) { first := node.NodeEvents[1] require.Equal(uint64(1), first.CreateIndex) - require.Equal("Registration failed", first.Message) + require.Equal("Heartbeating failed", first.Message) } func TestFSM_UpsertNode(t *testing.T) { diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 37d4d05cc..af8e7b075 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -984,6 +984,14 @@ func TestClientEndpoint_GetNode(t *testing.T) { t.Fatalf("bad: %#v \n %#v", node, resp2.Node) } + // assert that the node register event was set correctly + if len(resp2.Node.NodeEvents) != 1 { + t.Fatalf("Did not set node events: %#v", resp2.Node) + } + if resp2.Node.NodeEvents[0].Message != "Node Registered" { + t.Fatalf("Did not set node register event correctly: %#v", resp2.Node) + } + // Lookup non-existing node get.NodeID = "12345678-abcd-efab-cdef-123456789abc" if err := msgpackrpc.CallWithCodec(codec, "Node.GetNode", get, &resp2); err != nil { diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 41523f749..e2649cf78 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -526,6 +526,9 @@ func (s *StateStore) UpsertNode(index uint64, node *structs.Node) error { node.CreateIndex = exist.CreateIndex node.ModifyIndex = index node.Drain = exist.Drain // Retain the drain mode + + // retain node events that have already been set on the node + node.NodeEvents = exist.NodeEvents } else { // Because this is the first time the node is being registered, we should // also create a node registration event diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d320408f9..8ae65516e 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1178,9 +1178,18 @@ func (n *Node) Copy() *Node { nn.Reserved = nn.Reserved.Copy() nn.Links = helper.CopyMapStringString(nn.Links) nn.Meta = helper.CopyMapStringString(nn.Meta) + nn.NodeEvents = copyNodeEvents(n) return nn } +func copyNodeEvents(first *Node) []*NodeEvent { + nodeEvents := make([]*NodeEvent, 0) + for _, e := range first.NodeEvents { + nodeEvents = append(nodeEvents, e) + } + return nodeEvents +} + // TerminalStatus returns if the current status is terminal and // will no longer transition. func (n *Node) TerminalStatus() bool {