kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-01 16:05:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

e2e: add tests for exec2 task driver (#22406)

Browse Source 
* e2e: add tests for exec2 task driver

* e2e: use envoy 1.29.4 because consul

* e2e: add a bridge networking http test for exec driver

* e2e: split up http test so curl always starts after the server
This commit is contained in:
Seth Hoenig
2024-05-31 09:22:39 -05:00
committed by GitHub
parent 86ee56b8c5
commit 2054e87158
10 changed files with 472 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
e2e/exec2/doc.go Normal file
View File

@@ -0,0 +1,5 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
// Package exec2 contains test cases related to the exec2 task driver.
package exec2

100
e2e/exec2/exec2_test.go Normal file
View File

@@ -0,0 +1,100 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package exec2
import (
"fmt"
"regexp"
"testing"
"github.com/hashicorp/nomad/e2e/v3/cluster3"
"github.com/hashicorp/nomad/e2e/v3/jobs3"
"github.com/shoenig/test/must"
)
func TestExec2(t *testing.T) {
cluster3.Establish(t,
cluster3.Leader(),
cluster3.LinuxClients(1),
)
t.Run("testEnv", testEnv)
t.Run("testSecretsDir", testSecretsDir)
t.Run("testCountdash", testCountdash)
t.Run("testHTTP", testHTTP)
}
func testEnv(t *testing.T) {
job, cleanup := jobs3.Submit(t,
"./input/env.hcl",
jobs3.WaitComplete("group"),
)
t.Cleanup(cleanup)
logs := job.TaskLogs("group", "env")
// ensure the job id lines up
expect := fmt.Sprintf("NOMAD_JOB_ID=%s", job.JobID())
must.StrContains(t, logs.Stdout, expect)
// ensure dynamic user e.g.
// USER=nomad-85249
userRe := regexp.MustCompile(`nomad-\d+`)
must.RegexMatch(t, userRe, logs.Stdout)
}
func testSecretsDir(t *testing.T) {
job, cleanup := jobs3.Submit(t,
"./input/secrets.hcl",
jobs3.WaitComplete("group"),
)
t.Cleanup(cleanup)
// ensure we can read the workload identity token file
nomadTokenLogs := job.TaskLogs("group", "nomad-token")
tokenRe := regexp.MustCompile(`[\w_-]+`)
must.RegexMatch(t, tokenRe, nomadTokenLogs.Stdout)
// ensure we can read the written password.txt file
passwordLogs := job.TaskLogs("group", "password")
must.StrContains(t, passwordLogs.Stdout, "abc123")
}
func testCountdash(t *testing.T) {
job, cleanup := jobs3.Submit(t,
"./input/countdash.hcl",
)
t.Cleanup(cleanup)
apiEnvoyLogs := job.TaskLogs("api", "connect-proxy-count-api")
must.StrContains(t, apiEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
dashEnvoyLogs := job.TaskLogs("dashboard", "connect-proxy-count-dashboard")
must.StrContains(t, dashEnvoyLogs.Stderr, "all clusters initialized. initializing init manager")
apiLogs := job.TaskLogs("api", "backend")
must.StrContains(t, apiLogs.Stdout, "Serving at http://localhost:9001")
dashLogs := job.TaskLogs("dashboard", "dashboard")
must.StrContains(t, dashLogs.Stdout, "Using counting service at http://127.0.0.1:8080")
}
func testHTTP(t *testing.T) {
job, _ := jobs3.Submit(t,
"./input/http.hcl",
jobs3.DisableCleanup(),
)
job2, _ := jobs3.Submit(t,
"./input/http_curl.hcl",
jobs3.DisableCleanup(),
)
logs := job.TaskLogs("backend", "http")
must.StrContains(t, logs.Stderr, `"GET / HTTP/1.1" 200 -`) // healthcheck
must.StrContains(t, logs.Stderr, `"GET /hi.html HTTP/1.1" 200 -`) // curl
logs2 := job2.TaskLogs("client", "curl")
must.StrContains(t, logs2.Stdout, "<body><p>Hello, friend!</p></body>")
}

119
e2e/exec2/input/countdash.hcl Normal file
View File

@@ -0,0 +1,119 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This is a variation of countdash that uses exec2 for running the envoy
# proxies manually.
job "countdash" {
group "api" {
network {
mode = "bridge"
}
service {
name = "count-api"
port = "9001"
connect {
sidecar_service {}
sidecar_task {
driver = "exec2"
user = "nobody"
config {
command = "/opt/bin/envoy"
args = [
"-c",
"${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
"-l",
"${meta.connect.log_level}",
"--concurrency",
"${meta.connect.proxy_concurrency}",
"--disable-hot-restart"
]
# TODO(shoenig) should not need NOMAD_ values once
# https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
# fixed.
unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
}
resources {
cpu = 1000
memory = 256
}
}
}
}
task "backend" {
driver = "docker"
config {
image = "docker.io/hashicorpdev/counter-api:v3"
}
}
}
group "dashboard" {
network {
mode = "bridge"
port "http" {
static = 9002
to = 9002
}
}
service {
name = "count-dashboard"
port = "http"
connect {
sidecar_service {
proxy {
upstreams {
destination_name = "count-api"
local_bind_port = 8080
}
}
}
sidecar_task {
driver = "exec2"
user = "nobody"
config {
command = "/opt/bin/envoy"
args = [
"-c",
"${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
"-l",
"${meta.connect.log_level}",
"--concurrency",
"${meta.connect.proxy_concurrency}",
"--disable-hot-restart"
]
# TODO(shoenig) should not need NOMAD_ values once
# https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
# fixed.
unveil = ["rx:/opt/bin", "rwc:/dev/shm", "r:${NOMAD_TASK_DIR}", "r:${NOMAD_SECRETS_DIR}"]
}
resources {
cpu = 1000
memory = 256
}
}
}
}
task "dashboard" {
driver = "docker"
env {
COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}"
}
config {
image = "docker.io/hashicorpdev/counter-dashboard:v3"
}
}
}
}

38
e2e/exec2/input/env.hcl Normal file
View File

@@ -0,0 +1,38 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This is a simple env job using the exec2 task driver.
job "env" {
type = "batch"
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "group" {
reschedule {
attempts = 0
unlimited = false
}
restart {
attempts = 0
mode = "fail"
}
task "env" {
driver = "exec2"
config {
command = "env"
}
resources {
cpu = 100
memory = 64
}
}
}
}

74
e2e/exec2/input/http.hcl Normal file
View File

@@ -0,0 +1,74 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This job serves the NOMAD_TASK_DIR over http.
job "http" {
type = "service"
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "backend" {
network {
mode = "bridge"
port "http" {
to = "9999"
}
}
task "http" {
driver = "exec2"
service {
name = "python-http"
port = "http"
provider = "nomad"
check {
name = "hi"
type = "http"
path = "/"
interval = "3s"
timeout = "1s"
}
}
config {
command = "python3"
args = ["-m", "http.server", "9999", "--directory", "${NOMAD_TASK_DIR}"]
}
template {
destination = "local/hi.html"
data = <<EOH
<!doctype html>
<html>
<title>example</title>
<body><p>Hello, friend!</p></body>
</html>
EOH
}
resources {
cpu = 500
memory = 256
}
}
restart {
attempts = 0
mode = "fail"
}
reschedule {
attempts = 0
unlimited = false
}
update {
min_healthy_time = "5s"
}
}
}

43
e2e/exec2/input/http_curl.hcl Normal file
View File

@@ -0,0 +1,43 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This job makes requests to the "python-http" service.
job "http_curl" {
type = "service"
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "client" {
task "curl" {
driver = "exec2"
config {
command = "bash"
args = ["local/script.sh"]
}
template {
destination = "local/script.sh"
change_mode = "noop"
data = <<EOF
#!/usr/bin/env bash
while true
do
{{ range nomadService "python-http" }}
(curl -s -S -L "{{ .Address }}:{{ .Port }}/hi.html") || true
{{ end }}
sleep 2
done
EOF
}
}
}
}

67
e2e/exec2/input/secrets.hcl Normal file
View File

@@ -0,0 +1,67 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This job writes and reads the secrets directory.
job "secrets" {
type = "batch"
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "group" {
reschedule {
attempts = 0
unlimited = false
}
restart {
attempts = 0
mode = "fail"
}
task "nomad-token" {
driver = "exec2"
identity {
file = true
}
config {
command = "cat"
args = ["${NOMAD_SECRETS_DIR}/nomad_token"]
# TODO(shoenig) should not need NOMAD_ values once
# https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
# fixed.
unveil = ["r:${NOMAD_SECRETS_DIR}"]
}
resources {
cpu = 100
memory = 64
}
}
task "password" {
driver = "exec2"
lifecycle {
hook = "prestart"
sidecar = false
}
config {
command = "bash"
args = ["-c", "echo abc123 > ${NOMAD_SECRETS_DIR}/password.txt && cat ${NOMAD_SECRETS_DIR}/password.txt"]
# TODO(shoenig) should not need NOMAD_ values once
# https://github.com/hashicorp/nomad-driver-exec2/issues/29 is
# fixed.
unveil = ["rwc:${NOMAD_SECRETS_DIR}"]
}
resources {
cpu = 100
memory = 64
}
}
}
}

11
e2e/terraform/Makefile
View File

@@ -1,5 +1,6 @@
PKG_PATH = $(shell pwd)/../../pkg/linux_amd64/nomad
LICENSE_PATH ?=
NOMAD_LICENSE_PATH ?=
CONSUL_LICENSE_PATH ?=
# deploy for quick local development testing
@@ -8,14 +9,16 @@ plan:
-var="nomad_local_binary=$(PKG_PATH)" \
-var="volumes=false" \
-var="client_count_ubuntu_jammy_amd64=3" \
-var="client_count_windows_2016_amd64=0"
-var="client_count_windows_2016_amd64=0" \
-var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))"
apply:
terraform apply -auto-approve \
-var="nomad_local_binary=$(PKG_PATH)" \
-var="volumes=false" \
-var="client_count_ubuntu_jammy_amd64=3" \
-var="client_count_windows_2016_amd64=0"
-var="client_count_windows_2016_amd64=0" \
-var="consul_license=$(shell cat $(CONSUL_LICENSE_PATH))"
clean: destroy tidy
@@ -32,7 +35,7 @@ plan_full:
apply_full:
@terraform apply -auto-approve \
-var="nomad_license=$(shell cat $(LICENSE_PATH))"
-var="nomad_license=$(shell cat $(NOMAD_LICENSE_PATH))"
clean_full: destroy_full tidy

8
e2e/terraform/etc/nomad.d/client-linux.hcl
View File

@@ -51,3 +51,11 @@ plugin "nomad-pledge-driver" {
pledge_executable = "/usr/local/bin/pledge"
}
}
plugin "nomad-driver-exec2" {
config {
unveil_defaults = true
unveil_by_task = true
unveil_paths = ["r:/etc/mime.types"]
}
}

11
e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh
View File

@@ -19,6 +19,7 @@ export DEBIAN_FRONTEND=noninteractive
echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
mkdir_for_root /opt
mkdir_for_root /opt/bin # for envoy
mkdir_for_root /srv/data # for host volumes
mkdir_for_root /opt/cni/bin
@@ -132,6 +133,16 @@ sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR}
sudo mv /tmp/pledge /usr/local/bin
sudo chmod +x /usr/local/bin/pledge
# Exec2
echo "Installing Exec2 Driver"
sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version v0.1.0-alpha.2 nomad-driver-exec2
sudo chmod +x ${NOMAD_PLUGIN_DIR}/nomad-driver-exec2
# Envoy
echo "Installing Envoy"
sudo curl -s -S -L -o /opt/bin/envoy https://github.com/envoyproxy/envoy/releases/download/v1.29.4/envoy-1.29.4-linux-x86_64
sudo chmod +x /opt/bin/envoy
# ECS
if [ -a "/tmp/linux/nomad-driver-ecs" ]; then
echo "Installing nomad-driver-ecs"