e2e: Add Client Intro test framework and initial test. (#26639)

The new client intro test mimics the Consul and Vault compat tests
and uses local agents to perform the required setup. This method
allows us the flexibility moving forward to test when enforcement
mode is in strict.

The test suite will now be triggered from the test-e2e CI run
and can also be called by a make target.

.github/workflows/test-e2e.yml

@@ -82,10 +82,28 @@ jobs:
           sudo sed -i 's!Defaults!#Defaults!g' /etc/sudoers
           sudo -E env "PATH=$PATH" make integration-test-consul
 
+  test-e2e-client-intro:
+    runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux"]') || 'ubuntu-22.04' }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Git config token
+        if: endsWith(github.repository, '-enterprise')
+        run: git config --global url.'https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com'.insteadOf 'https://github.com'
+      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
+        with:
+          cache: ${{ contains(runner.name, 'Github Actions') }}
+          go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
+      - name: Client Introuction
+        run: |
+          make deps
+          make integration-test-client-intro
+
   handle-failure:
     needs:
       - test-e2e-vault
       - test-e2e-consul
+      - test-e2e-client-intro
     if: always() && github.event_name == 'push' && contains(needs.*.result, 'failure')
     uses: ./.github/workflows/test-failure-notification.yml
     secrets: inherit

GNUmakefile

@@ -351,6 +351,17 @@ integration-test-consul: dev ## Run Nomad integration tests
 		-tags "$(GO_TAGS)" \
 		github.com/hashicorp/nomad/e2e/consulcompat
 
+.PHONY: integration-test-client-intro
+integration-test-client-intro: dev ## Run Nomad's Client Intro integration tests
+	@echo "==> Running Nomad integration test suite for Client Introduction:"
+	NOMAD_E2E_CLIENT_INTRO=1 go test \
+		-v \
+		-race \
+		-timeout=120s \
+		-count=1 \
+		-tags "$(GO_TAGS)" \
+		github.com/hashicorp/nomad/e2e/client_intro
+
 .PHONY: clean
 clean: GOPATH=$(shell go env GOPATH)
 clean: ## Remove build artifacts
@@ -396,7 +407,7 @@ ember-dist: ## Build the static UI assets from source
 dev-ui: ember-dist static-assets ## Build a dev UI binary
 	@$(MAKE) NOMAD_UI_TAG="ui" dev ## Build a dev binary with the UI baked in
 
-HELP_FORMAT="    \033[36m%-25s\033[0m %s\n"
+HELP_FORMAT="    \033[36m%-32s\033[0m %s\n"
 .PHONY: help
 help: ## Display this usage information
 	@echo "Valid targets:"

e2e/client_intro/client_intro_test.go

@@ -0,0 +1,155 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package clientintro
+
+import (
+	"errors"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/e2e/execagent"
+	"github.com/hashicorp/nomad/helper/discover"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/shoenig/test/must"
+	"github.com/shoenig/test/wait"
+)
+
+const (
+	envGate = "NOMAD_E2E_CLIENT_INTRO"
+)
+
+func TestClientIntro(t *testing.T) {
+
+	// This test is gated behind an environment variable so it does not run when
+	// the e2e suite is triggered by CI.
+	if os.Getenv(envGate) != "1" {
+		t.Skip(envGate + " is not set; skipping")
+	}
+
+	t.Run("testClientIntroEnforcementWarn", testClientIntroEnforcementWarn)
+}
+
+func testClientIntroEnforcementWarn(t *testing.T) {
+
+	// Find the Nomad binary that will be used for all Nomad agents in this
+	// test.
+	nomadBinary, err := discover.NomadExecutable()
+	must.NoError(t, err)
+	must.FileExists(t, nomadBinary)
+
+	// Generate our server configuration file which sets the log level to warn,
+	// which ensures we include the client intro log lines.
+	serverCallbackFn := func(c *execagent.AgentTemplateVars) {
+		c.AgentName = "server-intro-" + uuid.Short()
+		c.LogLevel = hclog.Warn.String()
+	}
+
+	// Use our custom logger to capture the server output so we can inspect it
+	// later.
+	serverWriter := newCaptureLogger()
+
+	testServer, err := execagent.NewSingleModeAgent(
+		nomadBinary,
+		t.TempDir(),
+		execagent.ModeServer,
+		serverWriter,
+		serverCallbackFn,
+	)
+
+	must.NoError(t, testServer.Start())
+	t.Cleanup(func() { _ = testServer.Destroy() })
+
+	clientCallbackFn := func(c *execagent.AgentTemplateVars) {
+		c.AgentName = "client-intro-" + uuid.Short()
+		c.LogLevel = hclog.Warn.String()
+		c.RetryJoinAddrs = []string{"127.0.0.1" + ":" + strconv.Itoa(testServer.Vars.RPC)}
+	}
+
+	clientWriter := io.Writer(os.Stderr)
+
+	testClient, err := execagent.NewSingleModeAgent(
+		nomadBinary,
+		t.TempDir(),
+		execagent.ModeClient,
+		clientWriter,
+		clientCallbackFn,
+	)
+	must.NoError(t, err)
+	must.NotNil(t, testClient)
+
+	must.NoError(t, testClient.Start())
+	t.Cleanup(func() { _ = testClient.Destroy() })
+
+	// Create a Nomad API client to talk to the server. Do it here, so we only
+	// do this once.
+	nomadClient, err := testServer.Client()
+	must.NoError(t, err)
+	must.NotNil(t, nomadClient)
+
+	// Wait for the client to show up in the server's node list. We use the node
+	// name as the identifier to check for since it's unique.
+	must.Wait(t, wait.InitialSuccess(
+		wait.ErrorFunc(func() error {
+			nodeList, _, err := nomadClient.Nodes().List(nil)
+			if err != nil {
+				return err
+			}
+			for _, node := range nodeList {
+				if node.Name == testClient.Vars.AgentName {
+					return nil
+				}
+			}
+			return errors.New("node not found")
+		}),
+		wait.Timeout(20*time.Second),
+		wait.Gap(3*time.Second),
+	))
+
+	// If we reached this point, we are going to be reading the log lines, so
+	// we can now destroy the agents to avoid race conditions.
+	_ = testServer.Destroy()
+	_ = testClient.Destroy()
+
+	// The node has joined the cluster, now we need to check the server logs to
+	// ensurewe saw the expected warning about the client joining without an
+	// intro token.
+	must.SliceContainsFunc(
+		t, serverWriter.lines,
+		"[WARN]  nomad.client: node registration without introduction token",
+		func(a string, b string) bool {
+			return strings.Contains(a, b)
+		},
+	)
+}
+
+// caputreLogger is a simple logger that captures log lines in memory and also
+// writes them to stderr. It allows us to caputre output and inspect it for
+// testing.
+type captureLogger struct {
+	lines []string
+}
+
+func newCaptureLogger() *captureLogger {
+	return &captureLogger{
+		lines: make([]string, 0),
+	}
+}
+
+func (c *captureLogger) Write(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	totalLength := len(p)
+	buf := make([]byte, 0, totalLength)
+	buf = append(buf, p...)
+
+	c.lines = append(c.lines, string(buf))
+
+	return os.Stderr.Write(buf)
+}

e2e/client_intro/doc.go

@@ -0,0 +1,12 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package clientintro provides end-to-end tests for Nomad's client introduction
+// feature. This does not involve running jobs and does not run against the
+// nightly cluster. Instead it uses local agents to verify client introduction
+// behavior.
+//
+// In order to run this test suite only, from the e2e directory you can trigger
+// 'go test -v -run '^TestClientIntro$' ./client_intro' or from the top
+// level you can use the 'integration-test-client-intro' make target.
+package clientintro

e2e/execagent/execagent.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"text/template"
 
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad/api"
 )
 
@@ -37,7 +38,8 @@ var (
 
 	agentTemplate = template.Must(template.New("agent").Parse(`
 enable_debug = true
-log_level = "{{ or .LogLevel "DEBUG" }}"
+name         = "{{ or .AgentName "nomad-e2e-test-agent" }}"
+log_level    = "{{ or .LogLevel "DEBUG" }}"
 
 ports {
   http = {{.HTTP}}
@@ -58,6 +60,11 @@ client {
   options = {
     "driver.raw_exec.enable" = "1"
   }
+  {{- $retry_join_length := len .RetryJoinAddrs }}{{ if not (eq $retry_join_length 0) }}
+  server_join {
+    retry_join = [{{ range $index, $element := .RetryJoinAddrs }}{{if $index}}, {{end}}"{{$element}}"{{ end }}]
+  }
+  {{ end }}
 }
 {{ end }}
 `))
@@ -69,7 +76,22 @@ type AgentTemplateVars struct {
 	Serf         int
 	EnableClient bool
 	EnableServer bool
-	LogLevel     string
+
+	// AgentName is the name to apply to the Nomad agent. This is optional, but
+	// allows for multiple agents to be run on the same host. If not set, it
+	// will default to "nomad-e2e-test-agent".
+	AgentName string
+
+	LogLevel string
+
+	// NodePool is the Nomad node pool to assign the agent to when running with
+	// client mode enabled. This will default to the "default" node pool if not
+	// set.
+	NodePool string
+
+	// RetryJoinAddrs is a list of addresses to use for the retry_join config
+	// block.
+	RetryJoinAddrs []string
 }
 
 func newAgentTemplateVars() (*AgentTemplateVars, error) {
@@ -87,14 +109,32 @@ func newAgentTemplateVars() (*AgentTemplateVars, error) {
 	}
 
 	vars := AgentTemplateVars{
-		HTTP: httpPort,
-		RPC:  rpcPort,
-		Serf: serfPort,
+		HTTP:     httpPort,
+		RPC:      rpcPort,
+		Serf:     serfPort,
+		LogLevel: hclog.Warn.String(),
+		NodePool: "default",
 	}
 
 	return &vars, nil
 }
 
+// SetMode is a helper function to allow setting the agent mode (client, server,
+// or both).
+func (a *AgentTemplateVars) SetMode(mode AgentMode) {
+	switch mode {
+	case ModeClient:
+		a.EnableClient = true
+		a.EnableServer = false
+	case ModeServer:
+		a.EnableClient = false
+		a.EnableServer = true
+	case ModeBoth:
+		a.EnableClient = true
+		a.EnableServer = true
+	}
+}
+
 func writeConfig(path string, vars *AgentTemplateVars) error {
 	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
 	if err != nil {
@@ -222,6 +262,65 @@ func NewClientServerPair(bin string, serverOut, clientOut io.Writer) (
 	return
 }
 
+// TemplateVariableCallbackFunc is a callback function that allow callers to
+// modify the template variables before the config file is written out.
+type TemplateVariableCallbackFunc func(c *AgentTemplateVars)
+
+func NewSingleModeAgent(
+	bin, baseDir string,
+	mode AgentMode,
+	writer io.Writer,
+	varCallbackFn TemplateVariableCallbackFunc,
+) (*NomadAgent, error) {
+
+	templateVars, err := newAgentTemplateVars()
+	if err != nil {
+		return nil, err
+	}
+
+	// Allow the caller to modify the template variables before we write out the
+	// config file.
+	if varCallbackFn != nil {
+		varCallbackFn(templateVars)
+	}
+
+	// Set the mode (client, server, or both)
+	templateVars.SetMode(mode)
+
+	baseDataDir := BaseDir
+
+	if baseDir != "" {
+		baseDataDir = baseDir
+	}
+
+	if err := os.MkdirAll(baseDataDir, 0755); err != nil {
+		return nil, err
+	}
+
+	agentDir, err := os.MkdirTemp(baseDataDir, "agent")
+	if err != nil {
+		return nil, err
+	}
+
+	agentConfig := filepath.Join(agentDir, "agent.hcl")
+	if err := writeConfig(agentConfig, templateVars); err != nil {
+		return nil, err
+	}
+
+	nomadAgent := &NomadAgent{
+		BinPath:  bin,
+		DataDir:  agentDir,
+		ConfFile: agentConfig,
+		Vars:     templateVars,
+		Cmd:      exec.Command(bin, "agent", "-config", agentConfig, "-data-dir", agentDir),
+	}
+
+	nomadAgent.Cmd.Stdout = writer
+	nomadAgent.Cmd.Stderr = writer
+
+	return nomadAgent, nil
+}
+
 // Start the agent command.
 func (n *NomadAgent) Start() error {
 	return n.Cmd.Start()