diff --git a/nomad/client_endpoint_test.go b/nomad/client_endpoint_test.go index 933422595..90fb9b41d 100644 --- a/nomad/client_endpoint_test.go +++ b/nomad/client_endpoint_test.go @@ -271,7 +271,7 @@ func TestClientEndpoint_GetAllocs(t *testing.T) { alloc := mock.Alloc() alloc.NodeID = node.ID state := s1.fsm.State() - err := state.UpdateAllocations(100, nil, []*structs.Allocation{alloc}) + err := state.UpdateAllocations(100, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } @@ -334,7 +334,7 @@ func TestClientEndpoint_GetAllocs_Blocking(t *testing.T) { start := time.Now() go func() { time.Sleep(100 * time.Millisecond) - err := state.UpdateAllocations(100, nil, []*structs.Allocation{alloc}) + err := state.UpdateAllocations(100, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } @@ -376,7 +376,7 @@ func TestClientEndpoint_CreateNodeEvals(t *testing.T) { // Inject fake evaluations alloc := mock.Alloc() state := s1.fsm.State() - err := state.UpdateAllocations(1, nil, []*structs.Allocation{alloc}) + err := state.UpdateAllocations(1, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } @@ -443,7 +443,7 @@ func TestClientEndpoint_Evaluate(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - err = state.UpdateAllocations(2, nil, []*structs.Allocation{alloc}) + err = state.UpdateAllocations(2, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/config.go b/nomad/config.go index b5450d75f..cfc5d5b95 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -86,6 +86,9 @@ type Config struct { // RaftConfig is the configuration used for Raft in the local DC RaftConfig *raft.Config + // RaftTimeout is applied to any network traffic for raft. Defaults to 10s. + RaftTimeout time.Duration + // RequireTLS ensures that all RPC traffic is protected with TLS RequireTLS bool @@ -170,6 +173,7 @@ func DefaultConfig() *Config { NodeName: hostname, ProtocolVersion: ProtocolVersionMax, RaftConfig: raft.DefaultConfig(), + RaftTimeout: 10 * time.Second, RPCAddr: DefaultRPCAddr, SerfConfig: serf.DefaultConfig(), NumSchedulers: 1, diff --git a/nomad/core_sched_test.go b/nomad/core_sched_test.go index 416f12def..69669dddb 100644 --- a/nomad/core_sched_test.go +++ b/nomad/core_sched_test.go @@ -26,8 +26,8 @@ func TestCoreScheduler_EvalGC(t *testing.T) { // Insert "dead" alloc alloc := mock.Alloc() alloc.EvalID = eval.ID - alloc.Status = structs.AllocStatusFailed - err = state.UpdateAllocations(1001, nil, []*structs.Allocation{alloc}) + alloc.DesiredStatus = structs.AllocDesiredStatusFailed + err = state.UpdateAllocations(1001, []*structs.Allocation{alloc}) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/fsm.go b/nomad/fsm.go index 4eb7331fa..08ef84016 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -254,7 +254,7 @@ func (n *nomadFSM) applyAllocUpdate(buf []byte, index uint64) interface{} { panic(fmt.Errorf("failed to decode request: %v", err)) } - if err := n.state.UpdateAllocations(index, req.Evict, req.Alloc); err != nil { + if err := n.state.UpdateAllocations(index, req.Alloc); err != nil { n.logger.Printf("[ERR] nomad.fsm: UpdateAllocations failed: %v", err) return err } diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 787371815..a08470e40 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -328,7 +328,6 @@ func TestFSM_UpdateAllocations(t *testing.T) { alloc := mock.Alloc() req := structs.AllocUpdateRequest{ - Evict: nil, Alloc: []*structs.Allocation{alloc}, } buf, err := structs.Encode(structs.AllocUpdateRequestType, req) @@ -352,8 +351,11 @@ func TestFSM_UpdateAllocations(t *testing.T) { t.Fatalf("bad: %#v %#v", alloc, out) } + evictAlloc := new(structs.Allocation) + *evictAlloc = *alloc + evictAlloc.DesiredStatus = structs.AllocDesiredStatusEvict req2 := structs.AllocUpdateRequest{ - Evict: []string{alloc.ID}, + Alloc: []*structs.Allocation{evictAlloc}, } buf, err = structs.Encode(structs.AllocUpdateRequestType, req2) if err != nil { @@ -370,7 +372,7 @@ func TestFSM_UpdateAllocations(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - if out.Status != structs.AllocStatusEvict { + if out.DesiredStatus != structs.AllocDesiredStatusEvict { t.Fatalf("alloc found!") } } @@ -471,9 +473,9 @@ func TestFSM_SnapshotRestore_Allocs(t *testing.T) { fsm := testFSM(t) state := fsm.State() alloc1 := mock.Alloc() - state.UpdateAllocations(1000, nil, []*structs.Allocation{alloc1}) + state.UpdateAllocations(1000, []*structs.Allocation{alloc1}) alloc2 := mock.Alloc() - state.UpdateAllocations(1001, nil, []*structs.Allocation{alloc2}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc2}) // Verify the contents fsm2 := testSnapshotRestore(t, fsm) diff --git a/nomad/plan_apply.go b/nomad/plan_apply.go index 12d8ce83a..5d6a02a35 100644 --- a/nomad/plan_apply.go +++ b/nomad/plan_apply.go @@ -52,7 +52,7 @@ func (s *Server) planApply() { } // Apply the plan if there is anything to do - if len(result.NodeEvict) != 0 || len(result.NodeAllocation) != 0 || len(result.FailedAllocs) != 0 { + if !result.IsNoOp() { allocIndex, err := s.applyPlan(result) if err != nil { s.logger.Printf("[ERR] nomad: failed to apply plan: %v", err) @@ -71,8 +71,8 @@ func (s *Server) planApply() { func (s *Server) applyPlan(result *structs.PlanResult) (uint64, error) { defer metrics.MeasureSince([]string{"nomad", "plan", "apply"}, time.Now()) req := structs.AllocUpdateRequest{} - for _, evictList := range result.NodeEvict { - req.Evict = append(req.Evict, evictList...) + for _, updateList := range result.NodeUpdate { + req.Alloc = append(req.Alloc, updateList...) } for _, allocList := range result.NodeAllocation { req.Alloc = append(req.Alloc, allocList...) @@ -91,13 +91,22 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR // Create a result holder for the plan result := &structs.PlanResult{ - NodeEvict: make(map[string][]string), + NodeUpdate: make(map[string][]*structs.Allocation), NodeAllocation: make(map[string][]*structs.Allocation), FailedAllocs: plan.FailedAllocs, } - // Check each allocation to see if it should be allowed + // Collect all the nodeIDs + nodeIDs := make(map[string]struct{}) + for nodeID := range plan.NodeUpdate { + nodeIDs[nodeID] = struct{}{} + } for nodeID := range plan.NodeAllocation { + nodeIDs[nodeID] = struct{}{} + } + + // Check each allocation to see if it should be allowed + for nodeID := range nodeIDs { // Evaluate the plan for this node fit, err := evaluateNodePlan(snap, plan, nodeID) if err != nil { @@ -119,7 +128,7 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR // If we require all-at-once scheduling, there is no point // to continue the evaluation, as we've already failed. if plan.AllAtOnce { - result.NodeEvict = nil + result.NodeUpdate = nil result.NodeAllocation = nil return result, nil } @@ -129,8 +138,8 @@ func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanR } // Add this to the plan result - if nodeEvict := plan.NodeEvict[nodeID]; len(nodeEvict) > 0 { - result.NodeEvict[nodeID] = nodeEvict + if nodeUpdate := plan.NodeUpdate[nodeID]; len(nodeUpdate) > 0 { + result.NodeUpdate[nodeID] = nodeUpdate } if nodeAlloc := plan.NodeAllocation[nodeID]; len(nodeAlloc) > 0 { result.NodeAllocation[nodeID] = nodeAlloc @@ -172,13 +181,13 @@ func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID stri // Determine the proposed allocation by first removing allocations // that are planned evictions and adding the new allocations. proposed := existingAlloc - var remove []string - if evict := plan.NodeEvict[nodeID]; len(evict) > 0 { - remove = append(remove, evict...) + var remove []*structs.Allocation + if update := plan.NodeUpdate[nodeID]; len(update) > 0 { + remove = append(remove, update...) } if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 { for _, alloc := range updated { - remove = append(remove, alloc.ID) + remove = append(remove, alloc) } } proposed = structs.RemoveAllocs(existingAlloc, remove) diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 1a6a5aa03..8c83c044e 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -39,9 +39,6 @@ func TestPlanApply_applyPlan(t *testing.T) { alloc := mock.Alloc() allocFail := mock.Alloc() plan := &structs.PlanResult{ - NodeEvict: map[string][]string{ - node.ID: []string{}, - }, NodeAllocation: map[string][]*structs.Allocation{ node.ID: []*structs.Allocation{alloc}, }, @@ -76,10 +73,13 @@ func TestPlanApply_applyPlan(t *testing.T) { } // Evict alloc, Register alloc2 + allocEvict := new(structs.Allocation) + *allocEvict = *alloc + allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict alloc2 := mock.Alloc() plan = &structs.PlanResult{ - NodeEvict: map[string][]string{ - node.ID: []string{alloc.ID}, + NodeUpdate: map[string][]*structs.Allocation{ + node.ID: []*structs.Allocation{allocEvict}, }, NodeAllocation: map[string][]*structs.Allocation{ node.ID: []*structs.Allocation{alloc2}, @@ -100,8 +100,8 @@ func TestPlanApply_applyPlan(t *testing.T) { if err != nil { t.Fatalf("err: %v", err) } - if out.Status != structs.AllocStatusEvict { - t.Fatalf("should be evicted alloc") + if out.DesiredStatus != structs.AllocDesiredStatusEvict { + t.Fatalf("should be evicted alloc: %#v", out) } // Lookup the allocation @@ -281,8 +281,7 @@ func TestPlanApply_EvalNodePlan_NodeFull(t *testing.T) { node.Resources = alloc.Resources node.Reserved = nil state.RegisterNode(1000, node) - state.UpdateAllocations(1001, nil, - []*structs.Allocation{alloc}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc}) snap, _ := state.Snapshot() alloc2 := mock.Alloc() @@ -310,8 +309,7 @@ func TestPlanApply_EvalNodePlan_UpdateExisting(t *testing.T) { node.Resources = alloc.Resources node.Reserved = nil state.RegisterNode(1000, node) - state.UpdateAllocations(1001, nil, - []*structs.Allocation{alloc}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc}) snap, _ := state.Snapshot() plan := &structs.Plan{ @@ -337,14 +335,16 @@ func TestPlanApply_EvalNodePlan_NodeFull_Evict(t *testing.T) { node.Resources = alloc.Resources node.Reserved = nil state.RegisterNode(1000, node) - state.UpdateAllocations(1001, nil, - []*structs.Allocation{alloc}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc}) snap, _ := state.Snapshot() + allocEvict := new(structs.Allocation) + *allocEvict = *alloc + allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict alloc2 := mock.Alloc() plan := &structs.Plan{ - NodeEvict: map[string][]string{ - node.ID: []string{alloc.ID}, + NodeUpdate: map[string][]*structs.Allocation{ + node.ID: []*structs.Allocation{allocEvict}, }, NodeAllocation: map[string][]*structs.Allocation{ node.ID: []*structs.Allocation{alloc2}, @@ -365,12 +365,11 @@ func TestPlanApply_EvalNodePlan_NodeFull_AllocEvict(t *testing.T) { state := testStateStore(t) node := mock.Node() alloc.NodeID = node.ID - alloc.Status = structs.AllocStatusEvict + alloc.DesiredStatus = structs.AllocDesiredStatusEvict node.Resources = alloc.Resources node.Reserved = nil state.RegisterNode(1000, node) - state.UpdateAllocations(1001, nil, - []*structs.Allocation{alloc}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc}) snap, _ := state.Snapshot() alloc2 := mock.Alloc() @@ -398,13 +397,15 @@ func TestPlanApply_EvalNodePlan_NodeMaint_EvictOnly(t *testing.T) { node.Reserved = nil node.Status = structs.NodeStatusMaint state.RegisterNode(1000, node) - state.UpdateAllocations(1001, nil, - []*structs.Allocation{alloc}) + state.UpdateAllocations(1001, []*structs.Allocation{alloc}) snap, _ := state.Snapshot() + allocEvict := new(structs.Allocation) + *allocEvict = *alloc + allocEvict.DesiredStatus = structs.AllocDesiredStatusEvict plan := &structs.Plan{ - NodeEvict: map[string][]string{ - node.ID: []string{alloc.ID}, + NodeUpdate: map[string][]*structs.Allocation{ + node.ID: []*structs.Allocation{allocEvict}, }, } diff --git a/nomad/rpc_test.go b/nomad/rpc_test.go index 281c4a7e6..1fe24f5f6 100644 --- a/nomad/rpc_test.go +++ b/nomad/rpc_test.go @@ -15,6 +15,7 @@ func TestRPC_forwardLeader(t *testing.T) { defer s2.Shutdown() testJoin(t, s1, s2) testutil.WaitForLeader(t, s1.RPC) + testutil.WaitForLeader(t, s2.RPC) var out struct{} err := s1.forwardLeader("Status.Ping", struct{}{}, &out) diff --git a/nomad/server.go b/nomad/server.go index d70b54680..1aaa58844 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -394,7 +394,7 @@ func (s *Server) setupRaft() error { } // Create a transport layer - trans := raft.NewNetworkTransport(s.raftLayer, 3, 10*time.Second, + trans := raft.NewNetworkTransport(s.raftLayer, 3, s.config.RaftTimeout, s.config.LogOutput) s.raftTransport = trans diff --git a/nomad/server_test.go b/nomad/server_test.go index 52cadf12c..77a9355fc 100644 --- a/nomad/server_test.go +++ b/nomad/server_test.go @@ -47,6 +47,7 @@ func testServer(t *testing.T, cb func(*Config)) *Server { config.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond config.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond config.RaftConfig.ElectionTimeout = 40 * time.Millisecond + config.RaftTimeout = 500 * time.Millisecond // Invoke the callback if any if cb != nil { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 49b0eb3a3..b08eeca05 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -217,9 +217,6 @@ type PlanRequest struct { // to cause evictions or to assign new allocaitons. Both can be done // within a single transaction type AllocUpdateRequest struct { - // Evict is the list of allocation IDs to evict - Evict []string - // Alloc is the list of new allocations to assign Alloc []*Allocation } @@ -725,7 +722,7 @@ type Allocation struct { // will no longer transition. This is not based on the current client status. func (a *Allocation) TerminalStatus() bool { switch a.DesiredStatus { - case AllocDesiredStatusStop, AllocDesiredStatusEvict: + case AllocDesiredStatusStop, AllocDesiredStatusEvict, AllocDesiredStatusFailed: return true default: return false @@ -1008,6 +1005,11 @@ type PlanResult struct { AllocIndex uint64 } +// IsNoOp checks if this plan result would do nothing +func (p *PlanResult) IsNoOp() bool { + return len(p.NodeUpdate) == 0 && len(p.NodeAllocation) == 0 && len(p.FailedAllocs) == 0 +} + // FullCommit is used to check if all the allocations in a plan // were committed as part of the result. Returns if there was // a match, and the number of expected and actual allocations.