From 1f957947b46084ea680b2ff6c18ee24557a015bc Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Fri, 10 Nov 2023 07:09:24 -0600 Subject: [PATCH] e2e: refactor nomadexec test suite (#19054) --- e2e/nomadexec/doc.go | 5 + e2e/nomadexec/exec.go | 146 ------------------ e2e/nomadexec/exec_test.go | 106 +++++++++++++ .../docker.nomad => input/busybox.hcl} | 11 +- e2e/v3/jobs3/jobs3.go | 19 +++ 5 files changed, 135 insertions(+), 152 deletions(-) create mode 100644 e2e/nomadexec/doc.go delete mode 100644 e2e/nomadexec/exec.go create mode 100644 e2e/nomadexec/exec_test.go rename e2e/nomadexec/{testdata/docker.nomad => input/busybox.hcl} (71%) diff --git a/e2e/nomadexec/doc.go b/e2e/nomadexec/doc.go new file mode 100644 index 000000000..b53ee1df8 --- /dev/null +++ b/e2e/nomadexec/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +// The nomadexec package contains tests around using the alloc exec command. +package nomadexec diff --git a/e2e/nomadexec/exec.go b/e2e/nomadexec/exec.go deleted file mode 100644 index 3ea4b22a3..000000000 --- a/e2e/nomadexec/exec.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 - -package nomadexec - -import ( - "bytes" - "context" - "fmt" - "io" - "reflect" - "regexp" - "testing" - "time" - - "github.com/hashicorp/nomad/api" - "github.com/hashicorp/nomad/e2e/e2eutil" - "github.com/hashicorp/nomad/e2e/framework" - "github.com/hashicorp/nomad/helper/uuid" - dtestutils "github.com/hashicorp/nomad/plugins/drivers/testutils" - "github.com/stretchr/testify/assert" -) - -type NomadExecE2ETest struct { - framework.TC - - name string - jobFilePath string - - jobID string - alloc api.Allocation -} - -func init() { - framework.AddSuites(&framework.TestSuite{ - Component: "Nomad exec", - CanRunLocal: true, - Cases: []framework.TestCase{ - newNomadExecE2eTest("docker", "./nomadexec/testdata/docker.nomad"), - }, - }) -} - -func newNomadExecE2eTest(name, jobFilePath string) *NomadExecE2ETest { - return &NomadExecE2ETest{ - name: name, - jobFilePath: jobFilePath, - } -} - -func (tc *NomadExecE2ETest) Name() string { - return fmt.Sprintf("%v (%v)", tc.TC.Name(), tc.name) -} - -func (tc *NomadExecE2ETest) BeforeAll(f *framework.F) { - // Ensure cluster has leader before running tests - e2eutil.WaitForLeader(f.T(), tc.Nomad()) - e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 1) - - // register a job for execing into - tc.jobID = "nomad-exec" + uuid.Generate()[:8] - allocs := e2eutil.RegisterAndWaitForAllocs(f.T(), tc.Nomad(), tc.jobFilePath, tc.jobID, "") - f.Len(allocs, 1) - - e2eutil.WaitForAllocRunning(f.T(), tc.Nomad(), allocs[0].ID) - - tc.alloc = api.Allocation{ - ID: allocs[0].ID, - Namespace: allocs[0].Namespace, - NodeID: allocs[0].NodeID, - } -} - -func (tc *NomadExecE2ETest) TestExecBasicResponses(f *framework.F) { - for _, c := range dtestutils.ExecTaskStreamingBasicCases { - f.T().Run(c.Name, func(t *testing.T) { - - stdin := newTestStdin(c.Tty, c.Stdin) - var stdout, stderr bytes.Buffer - - resizeCh := make(chan api.TerminalSize) - go func() { - resizeCh <- api.TerminalSize{Height: 100, Width: 100} - }() - - ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second) - defer cancelFn() - - exitCode, err := tc.Nomad().Allocations().Exec(ctx, - &tc.alloc, "task", c.Tty, - []string{"/bin/sh", "-c", c.Command}, - stdin, &stdout, &stderr, - resizeCh, nil) - - assert.NoError(t, err) - - assert.Equal(t, c.ExitCode, exitCode) - - switch s := c.Stdout.(type) { - case string: - assert.Equal(t, s, stdout.String()) - case *regexp.Regexp: - assert.Regexp(t, s, stdout.String()) - case nil: - assert.Empty(t, stdout.String()) - default: - assert.Fail(t, "unexpected stdout type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) - } - - switch s := c.Stderr.(type) { - case string: - assert.Equal(t, s, stderr.String()) - case *regexp.Regexp: - assert.Regexp(t, s, stderr.String()) - case nil: - assert.Empty(t, stderr.String()) - default: - assert.Fail(t, "unexpected stderr type", "found %v (%v), but expected string or regexp", s, reflect.TypeOf(s)) - } - }) - } -} - -func (tc *NomadExecE2ETest) AfterAll(f *framework.F) { - jobs := tc.Nomad().Jobs() - if tc.jobID != "" { - jobs.Deregister(tc.jobID, true, nil) - } - tc.Nomad().System().GarbageCollect() -} - -func newTestStdin(tty bool, d string) io.Reader { - pr, pw := io.Pipe() - go func() { - pw.Write([]byte(d)) - - // when testing TTY, leave connection open for the entire duration of command - // closing stdin may cause TTY session prematurely before command completes - if !tty { - pw.Close() - } - - }() - - return pr -} diff --git a/e2e/nomadexec/exec_test.go b/e2e/nomadexec/exec_test.go new file mode 100644 index 000000000..e14f10cd2 --- /dev/null +++ b/e2e/nomadexec/exec_test.go @@ -0,0 +1,106 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package nomadexec + +import ( + "bytes" + "context" + "io" + "regexp" + "testing" + "time" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + dtestutils "github.com/hashicorp/nomad/plugins/drivers/testutils" + "github.com/shoenig/test/must" +) + +func TestNomadExec(t *testing.T) { + cluster3.Establish(t, + cluster3.Leader(), + cluster3.LinuxClients(1), + ) + + t.Run("testDocker", testDocker) +} + +func getAlloc(t *testing.T, allocID string) *api.Allocation { + allocsAPI := e2eutil.NomadClient(t).Allocations() + info, _, err := allocsAPI.Info(allocID, nil) + must.NoError(t, err) + return info +} + +func testDocker(t *testing.T) { + job, jobCleanup := jobs3.Submit(t, "./input/busybox.hcl") + t.Cleanup(jobCleanup) + + alloc := getAlloc(t, job.AllocID("group")) + + for _, tc := range dtestutils.ExecTaskStreamingBasicCases { + t.Run(tc.Name, func(t *testing.T) { + stdin := newTestStdin(tc.Tty, tc.Stdin) + var stdout, stderr bytes.Buffer + + resizeCh := make(chan api.TerminalSize) + go func() { + resizeCh <- api.TerminalSize{Height: 100, Width: 100} + }() + + ctx, cancelFn := context.WithTimeout(context.Background(), 15*time.Second) + defer cancelFn() + + exitCode, err := e2eutil.NomadClient(t).Allocations().Exec( + ctx, + alloc, "task", tc.Tty, + []string{"/bin/sh", "-c", tc.Command}, + stdin, &stdout, &stderr, + resizeCh, nil, + ) + must.NoError(t, err) + must.Eq(t, tc.ExitCode, exitCode) + + switch s := tc.Stdout.(type) { + case string: + must.Eq(t, s, stdout.String()) + case *regexp.Regexp: + must.RegexMatch(t, s, stdout.String()) + case nil: + must.Eq(t, "", stdout.String()) + default: + must.Unreachable(t, must.Sprint("unexpected match type")) + } + + switch s := tc.Stderr.(type) { + case string: + must.Eq(t, s, stderr.String()) + case *regexp.Regexp: + must.RegexMatch(t, s, stderr.String()) + case nil: + must.Eq(t, "", stderr.String()) + default: + must.Unreachable(t, must.Sprint("unexpected match type")) + } + }) + } +} + +func newTestStdin(tty bool, d string) io.Reader { + pr, pw := io.Pipe() + go func() { + pw.Write([]byte(d)) + + // when testing TTY, leave connection open for the entire duration of command + // closing stdin may cause TTY session prematurely before command completes + if !tty { + pw.Close() + } + + }() + + return pr +} diff --git a/e2e/nomadexec/testdata/docker.nomad b/e2e/nomadexec/input/busybox.hcl similarity index 71% rename from e2e/nomadexec/testdata/docker.nomad rename to e2e/nomadexec/input/busybox.hcl index bd12d942b..906f258e5 100644 --- a/e2e/nomadexec/testdata/docker.nomad +++ b/e2e/nomadexec/input/busybox.hcl @@ -1,9 +1,7 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -job "nomadexec-docker" { - datacenters = ["dc1"] - +job "busybox" { constraint { attribute = "${attr.kernel.name}" value = "linux" @@ -14,15 +12,16 @@ job "nomadexec-docker" { driver = "docker" config { - image = "busybox:1.29.2" + image = "busybox:1" command = "/bin/sleep" - args = ["1000"] + args = ["infinity"] } resources { cpu = 500 - memory = 256 + memory = 128 } } } } + diff --git a/e2e/v3/jobs3/jobs3.go b/e2e/v3/jobs3/jobs3.go index 7d4eb65d0..50da9f84d 100644 --- a/e2e/v3/jobs3/jobs3.go +++ b/e2e/v3/jobs3/jobs3.go @@ -116,6 +116,25 @@ func (sub *Submission) JobID() string { return sub.jobID } +// AllocID returns the ID of an alloc of the given task group. If there is more than +// one allocation for the task group, an ID is chosen at random. If there is no +// allocation of the given task group the test assertion fails. +func (sub *Submission) AllocID(group string) string { + queryOpts := sub.queryOptions() + jobsAPI := sub.nomadClient.Jobs() + stubs, _, err := jobsAPI.Allocations(sub.jobID, false, queryOpts) + must.NoError(sub.t, err) + + for _, stub := range stubs { + if stub.TaskGroup == group { + return stub.ID + } + } + + must.Unreachable(sub.t, must.Sprintf("no alloc id found for group %q", group)) + panic("bug") +} + func (sub *Submission) logf(msg string, args ...any) { sub.t.Helper() util3.Log3(sub.t, sub.verbose, msg, args...)