From 159bf511200b8462d212910bd48e3bd716ec5177 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 12 Jul 2023 11:56:08 -0500 Subject: [PATCH] e2e: add some e2e tests for pledge task driver (#17909) * e2e: setup nomad for pledge driver * e2e: add some e2e tests for pledge task driver --- e2e/pledge/doc.go | 6 ++ e2e/pledge/input/bridge.hcl | 62 ++++++++++++++++++ e2e/pledge/input/curl.hcl | 41 ++++++++++++ e2e/pledge/input/sleep.hcl | 39 +++++++++++ e2e/pledge/input/unveil.hcl | 34 ++++++++++ e2e/pledge/pledge_test.go | 64 +++++++++++++++++++ e2e/terraform/etc/nomad.d/client-linux.hcl | 6 ++ e2e/terraform/packer/build | 1 + .../packer/ubuntu-jammy-amd64/setup.sh | 9 +++ 9 files changed, 262 insertions(+) create mode 100644 e2e/pledge/doc.go create mode 100644 e2e/pledge/input/bridge.hcl create mode 100644 e2e/pledge/input/curl.hcl create mode 100644 e2e/pledge/input/sleep.hcl create mode 100644 e2e/pledge/input/unveil.hcl create mode 100644 e2e/pledge/pledge_test.go diff --git a/e2e/pledge/doc.go b/e2e/pledge/doc.go new file mode 100644 index 000000000..5948dc541 --- /dev/null +++ b/e2e/pledge/doc.go @@ -0,0 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +// Package pledge tests the community pledge task driver. +// https://github.com/shoenig/nomad-pledge-driver +package pledge diff --git a/e2e/pledge/input/bridge.hcl b/e2e/pledge/input/bridge.hcl new file mode 100644 index 000000000..32ec097dc --- /dev/null +++ b/e2e/pledge/input/bridge.hcl @@ -0,0 +1,62 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +job "bridge" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + network { + mode = "bridge" + port "http" { to = 8181 } + } + + service { + provider = "nomad" + name = "pybridge" + port = "http" + tags = ["public=${attr.unique.platform.aws.public-ipv4}"] + check { + name = "up" + type = "http" + path = "/index.html" + interval = "6s" + timeout = "1s" + } + } + + task "python" { + driver = "pledge" + config { + command = "python3" + args = ["-m", "http.server", "8181", "--directory", "${NOMAD_TASK_DIR}"] + promises = "stdio rpath inet" + unveil = ["r:/etc/mime.types", "r:${NOMAD_TASK_DIR}"] + } + + template { + destination = "local/index.html" + data = < + + bridge mode +

Hello, pal!

+ +EOH + } + } + + restart { + attempts = 0 + mode = "fail" + } + + update { + min_healthy_time = "4s" + } + } +} + diff --git a/e2e/pledge/input/curl.hcl b/e2e/pledge/input/curl.hcl new file mode 100644 index 000000000..b9d4dfe82 --- /dev/null +++ b/e2e/pledge/input/curl.hcl @@ -0,0 +1,41 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +variable "address" { + type = string + description = "The address to cURL" +} + +job "curl" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + network { + mode = "host" + } + + reschedule { + attempts = 0 + unlimited = false + } + + restart { + attempts = 0 + mode = "fail" + } + + task "curl" { + driver = "pledge" + config { + command = "curl" + args = ["${var.address}"] + promises = "stdio rpath inet dns sendfd" + } + } + } +} diff --git a/e2e/pledge/input/sleep.hcl b/e2e/pledge/input/sleep.hcl new file mode 100644 index 000000000..c88021e52 --- /dev/null +++ b/e2e/pledge/input/sleep.hcl @@ -0,0 +1,39 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +job "sleep" { + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + update { + min_healthy_time = "4s" + } + + reschedule { + attempts = 0 + unlimited = false + } + + restart { + attempts = 0 + mode = "fail" + } + + task "task" { + driver = "pledge" + + config { + command = "sleep" + args = ["infinity"] + } + + resources { + cpu = 10 + memory = 32 + } + } + } +} diff --git a/e2e/pledge/input/unveil.hcl b/e2e/pledge/input/unveil.hcl new file mode 100644 index 000000000..2dd201352 --- /dev/null +++ b/e2e/pledge/input/unveil.hcl @@ -0,0 +1,34 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +job "unveil" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "group" { + + reschedule { + attempts = 0 + unlimited = false + } + + restart { + attempts = 0 + mode = "fail" + } + + task "cat" { + driver = "pledge" + config { + command = "cat" + args = ["/etc/passwd"] + promises = "stdio rpath" + unveil = ["r:/etc/passwd"] + } + } + } +} diff --git a/e2e/pledge/pledge_test.go b/e2e/pledge/pledge_test.go new file mode 100644 index 000000000..49fab7365 --- /dev/null +++ b/e2e/pledge/pledge_test.go @@ -0,0 +1,64 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package pledge + +import ( + "fmt" + "testing" + "time" + + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/shoenig/test/must" +) + +func TestPledge(t *testing.T) { + cluster3.Establish(t, + cluster3.Leader(), + cluster3.LinuxClients(1), + cluster3.Timeout(10*time.Second), + ) + + t.Run("testSleep", testSleep) + t.Run("testBridgeNetwork", testBridgeNetwork) + t.Run("testUnveil", testUnveil) +} + +func testSleep(t *testing.T) { + _, cleanup := jobs3.Submit(t, "./input/sleep.hcl") + t.Cleanup(cleanup) +} + +func testBridgeNetwork(t *testing.T) { + _, cleanup := jobs3.Submit(t, "./input/bridge.hcl") + t.Cleanup(cleanup) + + ip, port := findService(t, "pybridge") + address := fmt.Sprintf("http://%s:%d", ip, port) + + curlJob, curlCleanup := jobs3.Submit(t, "./input/curl.hcl", + jobs3.Var("address", address), + jobs3.WaitComplete("curl"), + ) + t.Cleanup(curlCleanup) + + logs := curlJob.TaskLogs("group", "curl") + must.StrContains(t, logs.Stdout, "bridge mode") +} + +func testUnveil(t *testing.T) { + job, cleanup := jobs3.Submit(t, "./input/unveil.hcl") + t.Cleanup(cleanup) + + logs := job.TaskLogs("group", "cat") + must.StrContains(t, logs.Stdout, "root:x:0:0") +} + +// findService returns the service address and port +func findService(t *testing.T, name string) (string, int) { + services, _, err := e2eutil.NomadClient(t).Services().Get(name, nil) + must.NoError(t, err) + return services[0].Address, services[0].Port +} diff --git a/e2e/terraform/etc/nomad.d/client-linux.hcl b/e2e/terraform/etc/nomad.d/client-linux.hcl index a28057814..f88c1392d 100644 --- a/e2e/terraform/etc/nomad.d/client-linux.hcl +++ b/e2e/terraform/etc/nomad.d/client-linux.hcl @@ -41,3 +41,9 @@ plugin "docker" { } } } + +plugin "nomad-pledge-driver" { + config { + pledge_executable = "/usr/local/bin/pledge" + } +} diff --git a/e2e/terraform/packer/build b/e2e/terraform/packer/build index b8b39b7da..c457e307f 100755 --- a/e2e/terraform/packer/build +++ b/e2e/terraform/packer/build @@ -11,6 +11,7 @@ Build an AMI for the target configuration Examples build ubuntu-jammy-amd64 + build windows-2016-amd64 EOF diff --git a/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh b/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh index 9cfc79c6c..47262f16b 100755 --- a/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh +++ b/e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh @@ -96,6 +96,15 @@ sudo apt-get -y install podman catatonit echo "Installing Podman Driver" sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version 0.4.2 nomad-driver-podman +# Pledge +echo "Installing Pledge Driver" +curl -fsSL -o /tmp/pledge-driver.tar.gz https://github.com/shoenig/nomad-pledge-driver/releases/download/v0.2.3/nomad-pledge-driver_0.2.3_linux_amd64.tar.gz +curl -fsSL -o /tmp/pledge https://github.com/shoenig/nomad-pledge-driver/releases/download/pledge-1.8.com/pledge-1.8.com +tar -C /tmp -xf /tmp/pledge-driver.tar.gz +sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR} +sudo mv /tmp/pledge /usr/local/bin +sudo chmod +x /usr/local/bin/pledge + # ECS if [ -a "/tmp/linux/nomad-driver-ecs" ]; then echo "Installing nomad-driver-ecs"