diff --git a/nomad/client_rpc.go b/nomad/client_rpc.go
index e1bbb6317..fb47ed325 100644
--- a/nomad/client_rpc.go
+++ b/nomad/client_rpc.go
@@ -40,6 +40,11 @@ func (s *Server) getNodeConn(nodeID string) (*nodeConnState, bool) {
 		}
 	}
 
+	// Shouldn't happen but rather be safe
+	if state == nil {
+		return nil, false
+	}
+
 	return state, ok
 }
 
@@ -108,13 +113,21 @@ func (s *Server) removeNodeConn(ctx *RPCContext) {
 	// dial various addresses that all route to the same server. The most common
 	// case for this is the original address the client uses to connect to the
 	// server differs from the advertised address sent by the heartbeat.
+	numConns := len(conns)
+	found := false
 	for i, conn := range conns {
 		if conn.Ctx.Conn.LocalAddr().String() == ctx.Conn.LocalAddr().String() &&
 			conn.Ctx.Conn.RemoteAddr().String() == ctx.Conn.RemoteAddr().String() {
 			s.nodeConns[ctx.NodeID] = append(s.nodeConns[ctx.NodeID][:i], s.nodeConns[ctx.NodeID][i+1:]...)
-			return
+			found = true
+			break
 		}
 	}
+
+	// If we just deleted the last conn, remove it from the map
+	if found && numConns == 1 {
+		delete(s.nodeConns, ctx.NodeID)
+	}
 }
 
 // serverWithNodeConn is used to determine which remote server has the most
diff --git a/nomad/client_rpc_test.go b/nomad/client_rpc_test.go
index d7edad6bc..e1a75c596 100644
--- a/nomad/client_rpc_test.go
+++ b/nomad/client_rpc_test.go
@@ -77,6 +77,9 @@ func TestServer_removeNodeConn_differentAddrs(t *testing.T) {
 	// Delete the second
 	s1.removeNodeConn(ctx2)
 	require.Len(s1.connectedNodes(), 0)
+
+	_, ok = s1.getNodeConn(nodeID)
+	require.False(ok)
 }
 
 func TestServerWithNodeConn_NoPath(t *testing.T) {