e2e podman private registry (#17642)

* e2e: add tests for using private registry with podman driver

This PR adds e2e tests that stands up a private docker registry
and has a podman tasks run a container from an image in that private
registry.

Tests
 - user:password set in task config
 - auth_soft_fail works for public images when auth is set in driver
 - credentials helper is set in driver auth config
 - config auth.json file is set in driver auth config

* packer: use nomad-driver-podman v0.5.0

* e2e: eliminate unnecessary chmod

Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>

* cr: no need to install nomad twice

* cl: no need to install docker twice

---------

Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>

e2e/podman/input/auth_basic.hcl

@@ -0,0 +1,76 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job runs a podman task using a container stored in a private registry
+# configured with basic authentication. The registry.hcl job should be running
+# and healthy before running this job. The registry_address and registry_port
+# HCL variables must be provided.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+variable "registry_username" {
+  type        = string
+  description = "The Basic Auth username of the local registry"
+  default     = "auth_basic_user"
+}
+
+variable "registry_password" {
+  type        = string
+  description = "The Basic Auth password of the local registry"
+  default     = "auth_basic_pass"
+}
+
+locals {
+  registry_auth = base64encode("${var.registry_username}:${var.registry_password}")
+}
+
+job "auth_basic" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "basic" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "podman"
+
+      config {
+        image          = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_basic:private"
+        args           = ["echo", "The auth basic test is OK!"]
+        auth_soft_fail = true
+
+        auth {
+          username   = "${var.registry_username}"
+          password   = "${var.registry_password}"
+          tls_verify = false
+        }
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/podman/input/auth_helper.hcl

@@ -0,0 +1,58 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job runs a podman task using a container stored in a private registry
+# configured with credentials helper authentication. The registry.hcl job should
+# be running and healthy before running this job.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+job "auth_static" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "helper" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "podman"
+
+      config {
+        image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_helper:private"
+        args  = ["echo", "The credentials helper auth test is OK!"]
+
+        auth {
+          # usename and password come from [docker-credential-]test.sh found on
+          # $PATH as specified by "helper=test.sh" in plugin config
+          tls_verify = false
+        }
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/podman/input/auth_static.hcl

@@ -0,0 +1,68 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job runs a podman task using a container stored in a private registry
+# configured with file config static authentication. The registry.hcl job should
+# be running and healthy before running this job.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+job "auth_static" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "static" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "podman"
+
+      config {
+        image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_static:private"
+        args  = ["echo", "The static auth test is OK!"]
+
+        auth {
+          # usename and password come from auth.json in plugin config
+          tls_verify = false
+        }
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}
+
+# auth.json (must be pointed to by config=<path>/auth.json)
+#
+# {
+#   "auths": {
+#     "127.0.0.1:7511/docker.io/library/bash_auth_static": {
+#       "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz"
+#     }
+#   }
+# }
+

e2e/podman/input/redis.hcl

@@ -1,7 +1,9 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: MPL-2.0
 
-job "podman_basic" {
+# This is a simple redis job using the podman task driver.
+
+job "redis" {
 
   constraint {
     attribute = "${attr.kernel.name}"
@@ -19,8 +21,9 @@ job "podman_basic" {
       driver = "podman"
 
       config {
-        image = "redis:7"
-        ports = ["db"]
+        image          = "docker.io/library/redis:7"
+        ports          = ["db"]
+        auth_soft_fail = true
       }
 
       resources {

e2e/podman/input/registry-auths.hcl

@@ -0,0 +1,120 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job runs after the private registry is up and running, when we know
+# address and port provided by the bridge network. It is a sysbatch job
+# that writes these files on every linux client.
+#   - /usr/local/bin/docker-credential-test.sh
+#   - /etc/docker-registry-auth.json
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+}
+
+variable "auth_dir" {
+  type        = string
+  description = "The destination directory of the auth.json file."
+  default     = "/tmp"
+}
+
+variable "helper_dir" {
+  type        = string
+  description = "The directory in which test.sh will be written."
+  default     = "/tmp"
+}
+
+variable "user" {
+  type        = string
+  description = "The user to create files as. Should be root in e2e."
+  # no default because dealing with root files is annoying locally
+  # try -var=user=$USER for local development
+}
+
+job "registry-auths" {
+  type = "sysbatch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "create-files" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    # write out the test.sh file into var.helper_dir
+    task "create-helper-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/test.sh", "${var.helper_dir}/docker-credential-test.sh"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/test.sh", "rwc:${var.helper_dir}"]
+      }
+      template {
+        destination = "local/test.sh"
+        perms       = "755"
+        data        = <<EOH
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+value=$(cat /dev/stdin)
+
+username="auth_helper_user"
+password="auth_helper_pass"
+
+case "${value}" in
+  docker.io/*)
+    echo "must use local registry"
+    exit 3
+    ;;
+  *)
+    echo "{\"Username\": \"$username\", \"Secret\": \"$password\"}"
+    exit 0
+    ;;
+esac
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+
+    # write out the auth.json file into var.auth_dir
+    task "create-auth-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/auth.json", "${var.auth_dir}/auth.json"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/auth.json", "rwc:${var.auth_dir}"]
+      }
+      template {
+        perms       = "644"
+        destination = "local/auth.json"
+        data        = <<EOH
+{
+  "auths": {
+    "${var.registry_address}:/docker.io/library/bash_auth_static": {
+      "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz"
+    }
+  }
+}
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+  }
+}

e2e/podman/input/registry.hcl

@@ -0,0 +1,137 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+# This job stands up a private container registry for use in e2e tests.
+# In a post start task we then upload some default images for convenience.
+#
+# <address>:<port>/docker.io/library/bash_auth_basic:private
+#
+# Note that the <address>:<port> is dynamic and can be found using NSD.
+# Note that credentials are required (e.g. podman login), and are specific to
+# each image, e.g. "auth_basic_user" and "auth_basic_pass".
+#
+# To add a new username/password credential, run this container command
+# podman run --rm --entrypoint htpasswd registry:2.7.0 -Bbn <username> <password>
+# and add <username>:<hash> to the local/auth.txt file template below.
+
+job "registry" {
+  type = "service"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "registry-server" {
+
+    update {
+      min_healthy_time = "4s"
+    }
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    network {
+      mode = "host"
+      port "registryhttp" {}
+    }
+
+    service {
+      provider = "nomad"
+      name     = "registry"
+      port     = "registryhttp"
+      check {
+        name     = "registry-http"
+        type     = "http"
+        path     = "/"
+        interval = "10s"
+        timeout  = "3s"
+      }
+    }
+
+    task "registry" {
+      driver = "podman"
+
+      template {
+        data        = <<EOH
+e2euser:$2y$05$QpRvGkM/CMG.AG/G7Uh6guULMIlv1ZvjwfPa6dNjdkH.fhTzcpLDC
+auth_basic_user:$2y$05$b/lpKjGJhVMdgbpu1hxe0eAGegeHFrsWXH9g0JEO2gcWzPNgvesby
+auth_static_user:$2y$05$ZDOhbzsNe9pCcR0NslV72.gTrRLwI.05tq5yJMtFkD2LSS.G0wAYe
+auth_helper_user:$2y$05$sY4qctfzsjIhNyPD.zBEVumP0l6V5gU1f6GEThvHQ1cwupS8rogtu
+        EOH
+        destination = "local/auth.txt"
+      }
+
+      config {
+        image          = "docker.io/library/registry:2"
+        auth_soft_fail = true
+        ports          = ["registryhttp"]
+        network_mode   = "host"
+      }
+
+      env {
+        REGISTRY_HTTP_ADDR           = "${NOMAD_ADDR_registryhttp}"
+        REGISTRY_AUTH                = "htpasswd"
+        REGISTRY_AUTH_HTPASSWD_REALM = "Registry Realm"
+        REGISTRY_AUTH_HTPASSWD_PATH  = "local/auth.txt"
+      }
+
+      resources {
+        cpu    = 250
+        memory = 200
+      }
+    }
+
+
+    task "registry-preload" {
+      user   = "root"
+      driver = "raw_exec"
+
+      lifecycle {
+        hook    = "poststart"
+        sidecar = false
+      }
+
+      template {
+        data        = <<EOH
+{
+  "auths": {
+    "{{- env "NOMAD_ADDR_registryhttp" -}}": {
+      "auth": "ZTJldXNlcjplMmVwYXNzd29yZA=="
+    }
+  }
+}
+        EOH
+        destination = "local/auth.json"
+      }
+
+      template {
+        data        = <<EOH
+set -xeuo pipefail
+podman pull docker.io/library/bash:5
+podman push --tls-verify=false --authfile=local/auth.json docker.io/library/bash:5 {{env "NOMAD_ADDR_registryhttp" -}}/docker.io/library/bash_auth_basic:private
+podman push --tls-verify=false --authfile=local/auth.json docker.io/library/bash:5 {{env "NOMAD_ADDR_registryhttp" -}}/docker.io/library/bash_auth_static:private
+podman push --tls-verify=false --authfile=local/auth.json docker.io/library/bash:5 {{env "NOMAD_ADDR_registryhttp" -}}/docker.io/library/bash_auth_helper:private
+        EOH
+        destination = "local/script.sh"
+      }
+
+      config {
+        command = "bash"
+        args    = ["local/script.sh"]
+      }
+
+      resources {
+        cpu    = 200
+        memory = 100
+      }
+    }
+  }
+}

e2e/podman/podman_test.go

@@ -4,36 +4,117 @@
 package podman
 
 import (
+	"fmt"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/nomad/e2e/e2eutil"
-	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/e2e/v3/cluster3"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
 	"github.com/shoenig/test/must"
 )
 
+const (
+	registryService = "registry"
+)
+
 func TestPodman(t *testing.T) {
-	nomad := e2eutil.NomadClient(t)
+	cluster3.Establish(t,
+		cluster3.Leader(),
+		cluster3.LinuxClients(1),
+	)
 
-	e2eutil.WaitForLeader(t, nomad)
-	e2eutil.WaitForNodesReady(t, nomad, 1)
+	runRegistry(t)
 
-	t.Run("testBasic", testBasic)
+	t.Run("testRedis", testRedis)
+	t.Run("testAuthBasic", testAuthBasic)
+	t.Run("testAuthFileStatic", testAuthFileStatic)
+	t.Run("testAuthHelper", testAuthHelper)
 }
 
-func testBasic(t *testing.T) {
-	nomad := e2eutil.NomadClient(t)
-	jobID := "podman-basic-" + uuid.Short()
-	jobIDs := []string{jobID}
-	t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs))
-
-	// start job
-	e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/podman_basic.hcl", jobID, "")
-
-	// get alloc id
-	allocID := e2eutil.SingleAllocID(t, jobID, "", 0)
-
-	// check logs for redis startup
-	logs, err := e2eutil.AllocTaskLogs(allocID, "redis", e2eutil.LogsStdOut)
-	must.NoError(t, err)
-	must.StrContains(t, logs, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo")
+func findService(t *testing.T, name string) (string, int) {
+	services, _, err := e2eutil.NomadClient(t).Services().Get(name, nil)
+	must.NoError(t, err, must.Sprintf("failed to find %q service", name))
+	must.Len(t, 1, services, must.Sprintf("expected 1 %q service", name))
+	return services[0].Address, services[0].Port
+}
+
+func runRegistry(t *testing.T) {
+	_, regCleanup := jobs3.Submit(t,
+		"./input/registry.hcl",
+		jobs3.Timeout(40*time.Second), // pulls an image
+	)
+	t.Cleanup(regCleanup)
+
+	// lookup registry address
+	addr, port := findService(t, registryService)
+	address := fmt.Sprintf("%s:%d", addr, port)
+
+	// run the sed job to fixup the auth.json file with correct address
+	_, sedCleanup := jobs3.Submit(t,
+		"./input/registry-auths.hcl",
+		jobs3.Var("registry_address", address),
+		jobs3.Var("user", "root"),
+		jobs3.Var("helper_dir", "/usr/local/bin"),
+		jobs3.Var("auth_dir", "/etc"),
+		jobs3.WaitComplete("create-files"),
+		jobs3.Timeout(20*time.Second),
+	)
+	t.Cleanup(sedCleanup)
+}
+
+func testRedis(t *testing.T) {
+	job, cleanup := jobs3.Submit(t, "./input/redis.hcl")
+	t.Cleanup(cleanup)
+
+	logs := job.TaskLogs("cache", "redis")
+	must.StrContains(t, logs.Stdout, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo")
+}
+
+func testAuthBasic(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	// run the private bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_basic.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("basic"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("basic", "echo")
+	must.StrContains(t, logs.Stdout, "The auth basic test is OK!")
+}
+
+func testAuthFileStatic(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	// run the private _static bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_static.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("static"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("static", "echo")
+	must.StrContains(t, logs.Stdout, "The static auth test is OK!")
+}
+
+func testAuthHelper(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	t.Log("registry", regAddr, regPort)
+
+	// run the private _helper bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_helper.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("helper"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("helper", "echo")
+	must.StrContains(t, logs.Stdout, "The credentials helper auth test is OK!")
 }

e2e/terraform/etc/nomad.d/client-linux.hcl

@@ -15,6 +15,10 @@ plugin "nomad-driver-podman" {
     volumes {
       enabled = true
     }
+    auth {
+      helper = "test.sh"
+      config = "/etc/auth.json"
+    }
   }
 }
 

e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh

@@ -43,8 +43,19 @@ sudo chown root:root /usr/local/bin/sockaddr
 sudo ufw disable || echo "ufw not installed"
 
 echo "Install HashiCorp apt repositories"
-curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add -
-sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"
+wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+
+echo "Installing Docker apt repositories"
+sudo install -m 0755 -d /etc/apt/keyrings
+curl --insecure -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
+echo \
+  "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+echo "Refresh apt with third party repositories"
 sudo apt-get update
 
 echo "Install Consul and Nomad"
@@ -67,16 +78,11 @@ mkdir_for_root /opt/nomad
 mkdir_for_root $NOMAD_PLUGIN_DIR
 sudo mv /tmp/linux/nomad.service /etc/systemd/system/nomad.service
 
-echo "Installing third-party apt repositories"
+echo "Installing third-party tools"
 
 # Docker
-distro=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
-curl -fsSL https://download.docker.com/linux/debian/gpg | sudo apt-key add -
-sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/${distro} $(lsb_release -cs) stable"
-
-# Docker
-echo "Installing Docker"
-sudo apt-get install -y docker-ce
+echo "Installing Docker CE"
+sudo apt-get install -y docker-ce docker-ce-cli
 
 # Java
 echo "Installing Java"
@@ -94,12 +100,12 @@ echo "Installing Podman"
 sudo apt-get -y install podman catatonit
 
 echo "Installing Podman Driver"
-sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version 0.4.2 nomad-driver-podman
+sudo hc-install install --path ${NOMAD_PLUGIN_DIR} --version 0.5.0 nomad-driver-podman
 
 # Pledge
 echo "Installing Pledge Driver"
-curl -fsSL -o /tmp/pledge-driver.tar.gz https://github.com/shoenig/nomad-pledge-driver/releases/download/v0.2.3/nomad-pledge-driver_0.2.3_linux_amd64.tar.gz
-curl -fsSL -o /tmp/pledge https://github.com/shoenig/nomad-pledge-driver/releases/download/pledge-1.8.com/pledge-1.8.com
+curl -k -fsSL -o /tmp/pledge-driver.tar.gz https://github.com/shoenig/nomad-pledge-driver/releases/download/v0.2.3/nomad-pledge-driver_0.2.3_linux_amd64.tar.gz
+curl -k -fsSL -o /tmp/pledge https://github.com/shoenig/nomad-pledge-driver/releases/download/pledge-1.8.com/pledge-1.8.com
 tar -C /tmp -xf /tmp/pledge-driver.tar.gz
 sudo mv /tmp/nomad-pledge-driver ${NOMAD_PLUGIN_DIR}
 sudo mv /tmp/pledge /usr/local/bin