mirror of
https://github.com/kemko/nomad.git
synced 2026-01-01 16:05:42 +03:00
6c9f2fdd29
This changeset includes several adjustments to the upgrade testing scripts to reduce flakes and make problems more understandable: * When a node is drained prior to the 3rd client upgrade, it's entirely possible the 3rd client to be upgraded is the drained node. This results in miscounting the expected number of allocations because many of them will be "complete" (service/batch) or "pending" (system). Leave the system jobs running during drains and only count the running allocations at that point as the expected set. Move the inline script that gets this count into a script file for legibility. * When the last initial workload is deployed, it's possible for it to be briefly still in "pending" when we move to the next step. Poll for a short window for the expected count of jobs. * Make sure that any scripts that are being run right after a server or client is coming back up can handle temporary unavailability gracefully. * Change the debugging output of several scripts to avoid having the debug output run into the error message (Ex. "some allocs are not running" looked like the first allocation running was the missing allocation). * Add some notes to the README about running locally with `-dev` builds and tagging a cluster with your own name. Ref: https://hashicorp.atlassian.net/browse/NMD-162
60 lines
1.5 KiB
Bash
Executable File
60 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Copyright (c) HashiCorp, Inc.
|
|
# SPDX-License-Identifier: BUSL-1.1
|
|
|
|
set -euo pipefail
|
|
|
|
error_exit() {
|
|
printf 'Error: %s' "${1}"
|
|
exit 1
|
|
}
|
|
|
|
# Quality: "nomad_CLIENTS_status: A GET call to /v1/nodes returns the correct number of clients and they are all eligible and ready"
|
|
|
|
MAX_WAIT_TIME=30 # Maximum wait time in seconds
|
|
POLL_INTERVAL=2 # Interval between status checks
|
|
|
|
elapsed_time=0
|
|
ready_clients=
|
|
last_error=
|
|
|
|
checkReadyClients() {
|
|
local clients_length
|
|
|
|
ready_clients=$(nomad node status -json | jq '[.[] | select(.Status == "ready" and .SchedulingEligibility == "eligible")]') || {
|
|
last_error="Could not query node status"
|
|
return 1
|
|
}
|
|
|
|
clients_length=$(echo "$ready_clients" | jq 'length')
|
|
if [ "$clients_length" -eq "$CLIENT_COUNT" ]; then
|
|
last_error=
|
|
return 0
|
|
fi
|
|
|
|
last_error="Unexpected number of ready clients: $clients_length"
|
|
return 1
|
|
}
|
|
|
|
checkEligibleClients() {
|
|
echo "$ready_clients" | jq -e '
|
|
map(select(.SchedulingEligibility != "eligible")) | length == 0' && return 0
|
|
|
|
last_error=$(echo "$ready_clients" | jq -r '
|
|
map(select(.SchedulingEligibility != "eligible")) | "\(.[].ID) is ineligible"')
|
|
return 1
|
|
}
|
|
|
|
while true; do
|
|
checkReadyClients && checkEligibleClients && break
|
|
|
|
if [ "$elapsed_time" -ge "$MAX_WAIT_TIME" ]; then
|
|
error_exit "$last_error"
|
|
fi
|
|
|
|
sleep "$POLL_INTERVAL"
|
|
elapsed_time=$((elapsed_time + POLL_INTERVAL))
|
|
done
|
|
|
|
echo "All $CLIENT_COUNT clients are eligible and running."
|