nomad: updating for new alloc representation

nomad/client_endpoint_test.go

@@ -271,7 +271,7 @@ func TestClientEndpoint_GetAllocs(t *testing.T) {
 	alloc := mock.Alloc()
 	alloc.NodeID = node.ID
 	state := s1.fsm.State()
-	err := state.UpdateAllocations(100, nil, []*structs.Allocation{alloc})
+	err := state.UpdateAllocations(100, []*structs.Allocation{alloc})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -334,7 +334,7 @@ func TestClientEndpoint_GetAllocs_Blocking(t *testing.T) {
 	start := time.Now()
 	go func() {
 		time.Sleep(100 * time.Millisecond)
-		err := state.UpdateAllocations(100, nil, []*structs.Allocation{alloc})
+		err := state.UpdateAllocations(100, []*structs.Allocation{alloc})
 		if err != nil {
 			t.Fatalf("err: %v", err)
 		}
@@ -376,7 +376,7 @@ func TestClientEndpoint_CreateNodeEvals(t *testing.T) {
 	// Inject fake evaluations
 	alloc := mock.Alloc()
 	state := s1.fsm.State()
-	err := state.UpdateAllocations(1, nil, []*structs.Allocation{alloc})
+	err := state.UpdateAllocations(1, []*structs.Allocation{alloc})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
@@ -443,7 +443,7 @@ func TestClientEndpoint_Evaluate(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	err = state.UpdateAllocations(2, nil, []*structs.Allocation{alloc})
+	err = state.UpdateAllocations(2, []*structs.Allocation{alloc})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

nomad/config.go

@@ -86,6 +86,9 @@ type Config struct {
 	// RaftConfig is the configuration used for Raft in the local DC
 	RaftConfig *raft.Config
 
+	// RaftTimeout is applied to any network traffic for raft. Defaults to 10s.
+	RaftTimeout time.Duration
+
 	// RequireTLS ensures that all RPC traffic is protected with TLS
 	RequireTLS bool
 
@@ -170,6 +173,7 @@ func DefaultConfig() *Config {
 		NodeName:          hostname,
 		ProtocolVersion:   ProtocolVersionMax,
 		RaftConfig:        raft.DefaultConfig(),
+		RaftTimeout:       10 * time.Second,
 		RPCAddr:           DefaultRPCAddr,
 		SerfConfig:        serf.DefaultConfig(),
 		NumSchedulers:     1,

nomad/core_sched_test.go

@@ -26,8 +26,8 @@ func TestCoreScheduler_EvalGC(t *testing.T) {
 	// Insert "dead" alloc
 	alloc := mock.Alloc()
 	alloc.EvalID = eval.ID
-	alloc.Status = structs.AllocStatusFailed
-	err = state.UpdateAllocations(1001, nil, []*structs.Allocation{alloc})
+	alloc.DesiredStatus = structs.AllocDesiredStatusFailed
+	err = state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}

nomad/fsm.go

@@ -254,7 +254,7 @@ func (n *nomadFSM) applyAllocUpdate(buf []byte, index uint64) interface{} {
 		panic(fmt.Errorf("failed to decode request: %v", err))
 	}
 
-	if err := n.state.UpdateAllocations(index, req.Evict, req.Alloc); err != nil {
+	if err := n.state.UpdateAllocations(index, req.Alloc); err != nil {
 		n.logger.Printf("[ERR] nomad.fsm: UpdateAllocations failed: %v", err)
 		return err
 	}

nomad/fsm_test.go

@@ -328,7 +328,6 @@ func TestFSM_UpdateAllocations(t *testing.T) {
 
 	alloc := mock.Alloc()
 	req := structs.AllocUpdateRequest{
-		Evict: nil,
 		Alloc: []*structs.Allocation{alloc},
 	}
 	buf, err := structs.Encode(structs.AllocUpdateRequestType, req)
@@ -352,8 +351,11 @@ func TestFSM_UpdateAllocations(t *testing.T) {
 		t.Fatalf("bad: %#v %#v", alloc, out)
 	}
 
+	evictAlloc := new(structs.Allocation)
+	*evictAlloc = *alloc
+	evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict
 	req2 := structs.AllocUpdateRequest{
-		Evict: []string{alloc.ID},
+		Alloc: []*structs.Allocation{evictAlloc},
 	}
 	buf, err = structs.Encode(structs.AllocUpdateRequestType, req2)
 	if err != nil {
@@ -370,7 +372,7 @@ func TestFSM_UpdateAllocations(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if out.Status != structs.AllocStatusEvict {
+	if out.DesiredStatus != structs.AllocDesiredStatusEvict {
 		t.Fatalf("alloc found!")
 	}
 }
@@ -471,9 +473,9 @@ func TestFSM_SnapshotRestore_Allocs(t *testing.T) {
 	fsm := testFSM(t)
 	state := fsm.State()
 	alloc1 := mock.Alloc()
-	state.UpdateAllocations(1000, nil, []*structs.Allocation{alloc1})
+	state.UpdateAllocations(1000, []*structs.Allocation{alloc1})
 	alloc2 := mock.Alloc()
-	state.UpdateAllocations(1001, nil, []*structs.Allocation{alloc2})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc2})
 
 	// Verify the contents
 	fsm2 := testSnapshotRestore(t, fsm)

nomad/plan_apply.go

@@ -52,7 +52,7 @@ func (s *Server) planApply() {
 		}
 
 		// Apply the plan if there is anything to do
-		if len(result.NodeEvict) != 0 || len(result.NodeAllocation) != 0 || len(result.FailedAllocs) != 0 {
+		if !result.IsNoOp() {
 			allocIndex, err := s.applyPlan(result)
 			if err != nil {
 				s.logger.Printf("[ERR] nomad: failed to apply plan: %v", err)
@@ -71,8 +71,8 @@ func (s *Server) planApply() {
 func (s *Server) applyPlan(result *structs.PlanResult) (uint64, error) {
 	defer metrics.MeasureSince([]string{"nomad", "plan", "apply"}, time.Now())
 	req := structs.AllocUpdateRequest{}
-	for _, evictList := range result.NodeEvict {
-		req.Evict = append(req.Evict, evictList...)
+	for _, updateList := range result.NodeUpdate {
+		req.Alloc = append(req.Alloc, updateList...)
 	}
 	for _, allocList := range result.NodeAllocation {
 		req.Alloc = append(req.Alloc, allocList...)
@@ -91,13 +91,22 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR
 
 	// Create a result holder for the plan
 	result := &structs.PlanResult{
-		NodeEvict:      make(map[string][]string),
+		NodeUpdate:     make(map[string][]*structs.Allocation),
 		NodeAllocation: make(map[string][]*structs.Allocation),
 		FailedAllocs:   plan.FailedAllocs,
 	}
 
-	// Check each allocation to see if it should be allowed
+	// Collect all the nodeIDs
+	nodeIDs := make(map[string]struct{})
+	for nodeID := range plan.NodeUpdate {
+		nodeIDs[nodeID] = struct{}{}
+	}
 	for nodeID := range plan.NodeAllocation {
+		nodeIDs[nodeID] = struct{}{}
+	}
+
+	// Check each allocation to see if it should be allowed
+	for nodeID := range nodeIDs {
 		// Evaluate the plan for this node
 		fit, err := evaluateNodePlan(snap, plan, nodeID)
 		if err != nil {
@@ -119,7 +128,7 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR
 			// If we require all-at-once scheduling, there is no point
 			// to continue the evaluation, as we've already failed.
 			if plan.AllAtOnce {
-				result.NodeEvict = nil
+				result.NodeUpdate = nil
 				result.NodeAllocation = nil
 				return result, nil
 			}
@@ -129,8 +138,8 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR
 		}
 
 		// Add this to the plan result
-		if nodeEvict := plan.NodeEvict[nodeID]; len(nodeEvict) > 0 {
-			result.NodeEvict[nodeID] = nodeEvict
+		if nodeUpdate := plan.NodeUpdate[nodeID]; len(nodeUpdate) > 0 {
+			result.NodeUpdate[nodeID] = nodeUpdate
 		}
 		if nodeAlloc := plan.NodeAllocation[nodeID]; len(nodeAlloc) > 0 {
 			result.NodeAllocation[nodeID] = nodeAlloc
@@ -172,13 +181,13 @@ func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID stri
 	// Determine the proposed allocation by first removing allocations
 	// that are planned evictions and adding the new allocations.
 	proposed := existingAlloc
-	var remove []string
-	if evict := plan.NodeEvict[nodeID]; len(evict) > 0 {
-		remove = append(remove, evict...)
+	var remove []*structs.Allocation
+	if update := plan.NodeUpdate[nodeID]; len(update) > 0 {
+		remove = append(remove, update...)
 	}
 	if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 {
 		for _, alloc := range updated {
-			remove = append(remove, alloc.ID)
+			remove = append(remove, alloc)
 		}
 	}
 	proposed = structs.RemoveAllocs(existingAlloc, remove)

nomad/plan_apply_test.go

@@ -39,9 +39,6 @@ func TestPlanApply_applyPlan(t *testing.T) {
 	alloc := mock.Alloc()
 	allocFail := mock.Alloc()
 	plan := &structs.PlanResult{
-		NodeEvict: map[string][]string{
-			node.ID: []string{},
-		},
 		NodeAllocation: map[string][]*structs.Allocation{
 			node.ID: []*structs.Allocation{alloc},
 		},
@@ -76,10 +73,13 @@ func TestPlanApply_applyPlan(t *testing.T) {
 	}
 
 	// Evict alloc, Register alloc2
+	allocEvict := new(structs.Allocation)
+	*allocEvict = *alloc
+	allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict
 	alloc2 := mock.Alloc()
 	plan = &structs.PlanResult{
-		NodeEvict: map[string][]string{
-			node.ID: []string{alloc.ID},
+		NodeUpdate: map[string][]*structs.Allocation{
+			node.ID: []*structs.Allocation{allocEvict},
 		},
 		NodeAllocation: map[string][]*structs.Allocation{
 			node.ID: []*structs.Allocation{alloc2},
@@ -100,8 +100,8 @@ func TestPlanApply_applyPlan(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if out.Status != structs.AllocStatusEvict {
-		t.Fatalf("should be evicted alloc")
+	if out.DesiredStatus != structs.AllocDesiredStatusEvict {
+		t.Fatalf("should be evicted alloc: %#v", out)
 	}
 
 	// Lookup the allocation
@@ -281,8 +281,7 @@ func TestPlanApply_EvalNodePlan_NodeFull(t *testing.T) {
 	node.Resources = alloc.Resources
 	node.Reserved = nil
 	state.RegisterNode(1000, node)
-	state.UpdateAllocations(1001, nil,
-		[]*structs.Allocation{alloc})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	snap, _ := state.Snapshot()
 
 	alloc2 := mock.Alloc()
@@ -310,8 +309,7 @@ func TestPlanApply_EvalNodePlan_UpdateExisting(t *testing.T) {
 	node.Resources = alloc.Resources
 	node.Reserved = nil
 	state.RegisterNode(1000, node)
-	state.UpdateAllocations(1001, nil,
-		[]*structs.Allocation{alloc})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	snap, _ := state.Snapshot()
 
 	plan := &structs.Plan{
@@ -337,14 +335,16 @@ func TestPlanApply_EvalNodePlan_NodeFull_Evict(t *testing.T) {
 	node.Resources = alloc.Resources
 	node.Reserved = nil
 	state.RegisterNode(1000, node)
-	state.UpdateAllocations(1001, nil,
-		[]*structs.Allocation{alloc})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	snap, _ := state.Snapshot()
 
+	allocEvict := new(structs.Allocation)
+	*allocEvict = *alloc
+	allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict
 	alloc2 := mock.Alloc()
 	plan := &structs.Plan{
-		NodeEvict: map[string][]string{
-			node.ID: []string{alloc.ID},
+		NodeUpdate: map[string][]*structs.Allocation{
+			node.ID: []*structs.Allocation{allocEvict},
 		},
 		NodeAllocation: map[string][]*structs.Allocation{
 			node.ID: []*structs.Allocation{alloc2},
@@ -365,12 +365,11 @@ func TestPlanApply_EvalNodePlan_NodeFull_AllocEvict(t *testing.T) {
 	state := testStateStore(t)
 	node := mock.Node()
 	alloc.NodeID = node.ID
-	alloc.Status = structs.AllocStatusEvict
+	alloc.DesiredStatus = structs.AllocDesiredStatusEvict
 	node.Resources = alloc.Resources
 	node.Reserved = nil
 	state.RegisterNode(1000, node)
-	state.UpdateAllocations(1001, nil,
-		[]*structs.Allocation{alloc})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	snap, _ := state.Snapshot()
 
 	alloc2 := mock.Alloc()
@@ -398,13 +397,15 @@ func TestPlanApply_EvalNodePlan_NodeMaint_EvictOnly(t *testing.T) {
 	node.Reserved = nil
 	node.Status = structs.NodeStatusMaint
 	state.RegisterNode(1000, node)
-	state.UpdateAllocations(1001, nil,
-		[]*structs.Allocation{alloc})
+	state.UpdateAllocations(1001, []*structs.Allocation{alloc})
 	snap, _ := state.Snapshot()
 
+	allocEvict := new(structs.Allocation)
+	*allocEvict = *alloc
+	allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict
 	plan := &structs.Plan{
-		NodeEvict: map[string][]string{
-			node.ID: []string{alloc.ID},
+		NodeUpdate: map[string][]*structs.Allocation{
+			node.ID: []*structs.Allocation{allocEvict},
 		},
 	}
 

nomad/rpc_test.go

@@ -15,6 +15,7 @@ func TestRPC_forwardLeader(t *testing.T) {
 	defer s2.Shutdown()
 	testJoin(t, s1, s2)
 	testutil.WaitForLeader(t, s1.RPC)
+	testutil.WaitForLeader(t, s2.RPC)
 
 	var out struct{}
 	err := s1.forwardLeader("Status.Ping", struct{}{}, &out)

nomad/server.go

@@ -394,7 +394,7 @@ func (s *Server) setupRaft() error {
 	}
 
 	// Create a transport layer
-	trans := raft.NewNetworkTransport(s.raftLayer, 3, 10*time.Second,
+	trans := raft.NewNetworkTransport(s.raftLayer, 3, s.config.RaftTimeout,
 		s.config.LogOutput)
 	s.raftTransport = trans
 

nomad/server_test.go

@@ -47,6 +47,7 @@ func testServer(t *testing.T, cb func(*Config)) *Server {
 	config.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
 	config.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
 	config.RaftConfig.ElectionTimeout = 40 * time.Millisecond
+	config.RaftTimeout = 500 * time.Millisecond
 
 	// Invoke the callback if any
 	if cb != nil {

nomad/structs/structs.go

@@ -217,9 +217,6 @@ type PlanRequest struct {
 // to cause evictions or to assign new allocaitons. Both can be done
 // within a single transaction
 type AllocUpdateRequest struct {
-	// Evict is the list of allocation IDs to evict
-	Evict []string
-
 	// Alloc is the list of new allocations to assign
 	Alloc []*Allocation
 }
@@ -725,7 +722,7 @@ type Allocation struct {
 // will no longer transition. This is not based on the current client status.
 func (a *Allocation) TerminalStatus() bool {
 	switch a.DesiredStatus {
-	case AllocDesiredStatusStop, AllocDesiredStatusEvict:
+	case AllocDesiredStatusStop, AllocDesiredStatusEvict, AllocDesiredStatusFailed:
 		return true
 	default:
 		return false
@@ -1008,6 +1005,11 @@ type PlanResult struct {
 	AllocIndex uint64
 }
 
+// IsNoOp checks if this plan result would do nothing
+func (p *PlanResult) IsNoOp() bool {
+	return len(p.NodeUpdate) == 0 && len(p.NodeAllocation) == 0 && len(p.FailedAllocs) == 0
+}
+
 // FullCommit is used to check if all the allocations in a plan
 // were committed as part of the result. Returns if there was
 // a match, and the number of expected and actual allocations.