mirror of
https://github.com/kemko/nomad.git
synced 2026-01-02 16:35:44 +03:00
f00bff09f1
We use capped exponential backoff in several places in the code when handling failures. The code we've copy-and-pasted all over has a check to see if the backoff is greater than the limit, but this check happens after the bitshift and we always increment the number of attempts. This causes an overflow with a fairly small number of failures (ex. at one place I tested it occurs after only 24 iterations), resulting in a negative backoff which then never recovers. The backoff becomes a tight loop consuming resources and/or DoS'ing a Nomad RPC handler or an external API such as Vault. Note this doesn't occur in places where we cap the number of iterations so the loop breaks (usually to return an error), so long as the number of iterations is reasonable. Introduce a helper with a check on the cap before the bitshift to avoid overflow in all places this can occur. Fixes: #18199 Co-authored-by: stswidwinski <stan.swidwinski@gmail.com>
93 lines
1.9 KiB
Go
93 lines
1.9 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package consul
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"time"
|
|
|
|
log "github.com/hashicorp/go-hclog"
|
|
version "github.com/hashicorp/go-version"
|
|
"github.com/hashicorp/nomad/helper"
|
|
)
|
|
|
|
// checkConsulTLSSkipVerify logs if Consul does not support TLSSkipVerify on
|
|
// checks and is intended to be run in a goroutine.
|
|
func checkConsulTLSSkipVerify(ctx context.Context, logger log.Logger, client AgentAPI, done chan struct{}) {
|
|
const (
|
|
baseline = time.Second
|
|
limit = 20 * time.Second
|
|
)
|
|
|
|
defer close(done)
|
|
|
|
timer, stop := helper.NewSafeTimer(limit)
|
|
defer stop()
|
|
|
|
var attempts uint64
|
|
var backoff time.Duration
|
|
|
|
for {
|
|
self, err := client.Self()
|
|
if err == nil {
|
|
if supportsTLSSkipVerify(self) {
|
|
logger.Trace("Consul supports TLSSkipVerify")
|
|
} else {
|
|
logger.Warn("Consul does NOT support TLSSkipVerify; please upgrade Consul",
|
|
"min_version", consulTLSSkipVerifyMinVersion)
|
|
}
|
|
return
|
|
}
|
|
|
|
backoff = helper.Backoff(baseline, limit, attempts)
|
|
attempts++
|
|
timer.Reset(backoff)
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
var consulTLSSkipVerifyMinVersion = version.Must(version.NewVersion("0.7.2"))
|
|
|
|
// supportsTLSSkipVerify returns true if Consul supports TLSSkipVerify.
|
|
func supportsTLSSkipVerify(self map[string]map[string]interface{}) bool {
|
|
member, ok := self["Member"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
tagsI, ok := member["Tags"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
tags, ok := tagsI.(map[string]interface{})
|
|
if !ok {
|
|
return false
|
|
}
|
|
buildI, ok := tags["build"]
|
|
if !ok {
|
|
return false
|
|
}
|
|
build, ok := buildI.(string)
|
|
if !ok {
|
|
return false
|
|
}
|
|
parts := strings.SplitN(build, ":", 2)
|
|
if len(parts) != 2 {
|
|
return false
|
|
}
|
|
v, err := version.NewVersion(parts[0])
|
|
if err != nil {
|
|
return false
|
|
}
|
|
if v.LessThan(consulTLSSkipVerifyMinVersion) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|