From 22a4bcd3ca80090decce5459fc3beafd2f56840a Mon Sep 17 00:00:00 2001
From: Chris Baker <cgbaker@hashicorp.com>
Date: Fri, 7 Dec 2018 20:11:46 +0000
Subject: [PATCH] rpc accept loop: added backoff on logging for failed
 connections, in case there is a fast fail loop (NMD-1173)

---
 nomad/rpc.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/nomad/rpc.go b/nomad/rpc.go
index 7695217f0..e65b73725 100644
--- a/nomad/rpc.go
+++ b/nomad/rpc.go
@@ -84,6 +84,7 @@ type RPCContext struct {
 // listen is used to listen for incoming RPC connections
 func (r *rpcHandler) listen(ctx context.Context) {
 	defer close(r.listenerCh)
+	var tempDelay time.Duration
 	for {
 		select {
 		case <-ctx.Done():
@@ -105,9 +106,21 @@ func (r *rpcHandler) listen(ctx context.Context) {
 			default:
 			}
 
-			r.logger.Error("failed to accept RPC conn", "error", err)
+			if ne, ok := err.(net.Error); ok && ne.Temporary() {
+				if tempDelay == 0 {
+					tempDelay = 5 * time.Millisecond
+				} else {
+					tempDelay *= 2
+				}
+				if max := 1 * time.Second; tempDelay > max {
+					tempDelay = max
+				}
+				r.logger.Error("failed to accept RPC conn", "error", err, "delay", tempDelay)
+				time.Sleep(tempDelay)
+			}
 			continue
 		}
+		tempDelay = 0
 
 		go r.handleConn(ctx, conn, &RPCContext{Conn: conn})
 		metrics.IncrCounter([]string{"nomad", "rpc", "accept_conn"}, 1)