From 9895fa710c01ad96fc91d1e0246cfebd1689c8f2 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 2 Feb 2017 13:52:31 -0800 Subject: [PATCH] Bump protocol version and update numOtherPeers --- nomad/config.go | 5 +++++ nomad/heartbeat_test.go | 2 +- nomad/leader_test.go | 16 ++++++++-------- nomad/node_endpoint_test.go | 2 +- nomad/serf_test.go | 2 +- nomad/server.go | 29 ++++++++++++----------------- nomad/status_endpoint.go | 2 +- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/nomad/config.go b/nomad/config.go index 5f8ea1e16..3a54217d0 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -266,6 +266,11 @@ func DefaultConfig() *Config { // Disable shutdown on removal c.RaftConfig.ShutdownOnRemove = false + + // Enable interoperability with unversioned Raft library, and don't + // start using new ID-based features yet. + c.RaftConfig.ProtocolVersion = 1 + return c } diff --git a/nomad/heartbeat_test.go b/nomad/heartbeat_test.go index 7ab5495a8..9e9659b03 100644 --- a/nomad/heartbeat_test.go +++ b/nomad/heartbeat_test.go @@ -198,7 +198,7 @@ func TestServer_HeartbeatTTL_Failover(t *testing.T) { testJoin(t, s1, s2, s3) testutil.WaitForResult(func() (bool, error) { - peers, _ := s1.raftPeers.Peers() + peers, _ := s1.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") diff --git a/nomad/leader_test.go b/nomad/leader_test.go index 71b4e7878..7f524db7b 100644 --- a/nomad/leader_test.go +++ b/nomad/leader_test.go @@ -29,7 +29,7 @@ func TestLeader_LeftServer(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") @@ -48,7 +48,7 @@ func TestLeader_LeftServer(t *testing.T) { } for _, s := range servers[1:] { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 2, errors.New(fmt.Sprintf("%v", peers)) } @@ -76,7 +76,7 @@ func TestLeader_LeftLeader(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") @@ -102,7 +102,7 @@ func TestLeader_LeftLeader(t *testing.T) { continue } testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 2, errors.New(fmt.Sprintf("%v", peers)) }, func(err error) { t.Fatalf("should have 2 peers: %v", err) @@ -130,7 +130,7 @@ func TestLeader_MultiBootstrap(t *testing.T) { // Ensure we don't have multiple raft peers for _, s := range servers { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() if len(peers) != 1 { t.Fatalf("should only have 1 raft peer!") } @@ -155,7 +155,7 @@ func TestLeader_PlanQueue_Reset(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") @@ -232,7 +232,7 @@ func TestLeader_EvalBroker_Reset(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") @@ -309,7 +309,7 @@ func TestLeader_PeriodicDispatcher_Restore_Adds(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") diff --git a/nomad/node_endpoint_test.go b/nomad/node_endpoint_test.go index 749e48b83..6c21bfb50 100644 --- a/nomad/node_endpoint_test.go +++ b/nomad/node_endpoint_test.go @@ -536,7 +536,7 @@ func TestClientEndpoint_UpdateStatus_HeartbeatOnly(t *testing.T) { for _, s := range servers { testutil.WaitForResult(func() (bool, error) { - peers, _ := s.raftPeers.Peers() + peers, _ := s.numPeers() return len(peers) == 3, nil }, func(err error) { t.Fatalf("should have 3 peers") diff --git a/nomad/serf_test.go b/nomad/serf_test.go index c04dedb07..9574203e4 100644 --- a/nomad/serf_test.go +++ b/nomad/serf_test.go @@ -162,7 +162,7 @@ func TestNomad_BadExpect(t *testing.T) { // should still have no peers (because s2 is in expect=2 mode) testutil.WaitForResult(func() (bool, error) { for _, s := range servers { - p, _ := s.raftPeers.Peers() + p, _ := s.numPeers() if len(p) != 0 { return false, fmt.Errorf("%d", len(p)) } diff --git a/nomad/server.go b/nomad/server.go index d2c2582f9..51979ece6 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -11,7 +11,6 @@ import ( "reflect" "sort" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -358,7 +357,7 @@ func (s *Server) Leave() error { s.left = true // Check the number of known peers - numPeers, err := s.numOtherPeers() + numPeers, err := s.numPeers() if err != nil { s.logger.Printf("[ERR] nomad: failed to check raft peers: %v", err) return err @@ -369,7 +368,7 @@ func (s *Server) Leave() error { // not the leader, then we should issue our leave intention and wait to be removed // for some sane period of time. isLeader := s.IsLeader() - if isLeader && numPeers > 0 { + if isLeader && numPeers > 1 { future := s.raft.RemovePeer(s.raftTransport.LocalAddr()) if err := future.Error(); err != nil && err != raft.ErrUnknownPeer { s.logger.Printf("[ERR] nomad: failed to remove ourself as raft peer: %v", err) @@ -390,7 +389,7 @@ func (s *Server) Leave() error { limit := time.Now().Add(raftRemoveGracePeriod) for numPeers > 0 && time.Now().Before(limit) { // Update the number of peers - numPeers, err = s.numOtherPeers() + numPeers, err = s.numPeers() if err != nil { s.logger.Printf("[ERR] nomad: failed to check raft peers: %v", err) break @@ -490,7 +489,7 @@ func (s *Server) setupBootstrapHandler() error { // This Nomad Server has not been bootstrapped, reach // out to Consul if our peer list is less than // `bootstrap_expect`. - raftPeers, err := s.raftPeers.Peers() + raftPeers, err := s.numPeers() if err != nil { peersTimeout.Reset(peersPollInterval + lib.RandomStagger(peersPollInterval/peersPollJitterFactor)) return nil @@ -828,15 +827,15 @@ func (s *Server) setupWorkers() error { return nil } -// numOtherPeers is used to check on the number of known peers -// excluding the local node -func (s *Server) numOtherPeers() (int, error) { - peers, err := s.raftPeers.Peers() - if err != nil { +// numPeers is used to check on the number of known peers, including the local +// node. +func (s *Server) numPeers() (int, error) { + future := s.raft.GetConfiguration() + if err := future.Error(); err != nil { return 0, err } - otherPeers := raft.ExcludePeer(peers, s.raftTransport.LocalAddr()) - return len(otherPeers), nil + configuration := future.Configuration() + return len(configuration.Servers), nil } // IsLeader checks if this server is the cluster leader @@ -961,11 +960,7 @@ func (s *Server) Stats() map[string]map[string]string { "serf": s.serf.Stats(), "runtime": RuntimeStats(), } - if peers, err := s.raftPeers.Peers(); err == nil { - stats["raft"]["raft_peers"] = strings.Join(peers, ",") - } else { - s.logger.Printf("[DEBUG] server: error getting raft peers: %v", err) - } + return stats } diff --git a/nomad/status_endpoint.go b/nomad/status_endpoint.go index cdd871841..221dd49b1 100644 --- a/nomad/status_endpoint.go +++ b/nomad/status_endpoint.go @@ -53,7 +53,7 @@ func (s *Status) Peers(args *structs.GenericRequest, reply *[]string) error { return err } - peers, err := s.srv.raftPeers.Peers() + peers, err := s.srv.numPeers() if err != nil { return err }