close raft long-lived connections

nomad/raft_rpc.go

@@ -51,7 +51,7 @@ func (l *RaftLayer) Handoff(c net.Conn, ctx context.Context) error {
 	case <-l.closeCh:
 		return fmt.Errorf("Raft RPC layer closed")
 	case <-ctx.Done():
-		return fmt.Errorf("[INFO] nomad.rpc: Closing server RPC connection")
+		return fmt.Errorf("[INFO] nomad.rpc: Closing raft RPC connection")
 	}
 }
 

nomad/server.go

@@ -405,14 +405,16 @@ func (s *Server) ReloadTLSConnections(newTLSConfig *config.TLSConfig) error {
 	}
 	s.rpcListener = list
 
+	wrapper := tlsutil.RegionSpecificWrapper(s.config.Region, tlsWrap)
+	s.raftLayer.ReloadTLS(wrapper)
+	s.raftTransport.Reload()
+	time.Sleep(500 * time.Millisecond)
+
 	// reinitialize the cancel context
 	ctx, cancel := context.WithCancel(context.Background())
 	s.rpcCancel = cancel
 	go s.listen(ctx)
 
-	wrapper := tlsutil.RegionSpecificWrapper(s.config.Region, tlsWrap)
-	s.raftLayer.ReloadTLS(wrapper)
-
 	s.logger.Printf("[INFO] nomad: finished reloading server connections")
 	return nil
 }

vendor/github.com/hashicorp/raft/net_transport.go

@@ -177,6 +177,19 @@ func (n *NetworkTransport) Close() error {
 	return nil
 }
 
+func (n *NetworkTransport) Reload() {
+	n.shutdownLock.Lock()
+	defer n.shutdownLock.Unlock()
+
+	if !n.shutdown {
+		close(n.shutdownCh)
+		n.shutdown = true
+	}
+
+	time.Sleep(3 * time.Second)
+	n.shutdownCh = make(chan struct{})
+}
+
 // Consumer implements the Transport interface.
 func (n *NetworkTransport) Consumer() <-chan RPC {
 	return n.consumeCh