docker: fix a bug where auth for private registries wasn't parsed correctly (#24215)

In #23966 we introduced an official Docker client and did not notice that in
contrast to our previous 3rd party client, the official SDK PullOptions object
expects a base64 encoded JSON with username and password, instead of username/
password pair.

.changelog/24215.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+docker: Fix incorrect auth parsing for private registries
+```

drivers/docker/driver_test.go

@@ -2481,6 +2481,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) {
 		{
 			Repo: "redis:7",
 			AuthConfig: &registry.AuthConfig{
+				Auth:          "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6IjEyMzQifQ==",
 				Username:      "test",
 				Password:      "1234",
 				Email:         "",
@@ -2490,6 +2491,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) {
 		{
 			Repo: "quay.io/redis:7",
 			AuthConfig: &registry.AuthConfig{
+				Auth:          "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6IjU2NzgifQ==",
 				Username:      "test",
 				Password:      "5678",
 				Email:         "",
@@ -2499,6 +2501,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) {
 		{
 			Repo: "other.io/redis:7",
 			AuthConfig: &registry.AuthConfig{
+				Auth:          "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6ImFiY2QifQ==",
 				Username:      "test",
 				Password:      "abcd",
 				Email:         "",
@@ -2535,6 +2538,7 @@ func TestDockerDriver_AuthFromTaskConfig(t *testing.T) {
 				ServerAddr: "www.foobar.com",
 			},
 			AuthConfig: &registry.AuthConfig{
+				Auth:          "eyJ1c2VybmFtZSI6ImZvbyIsInBhc3N3b3JkIjoiYmFyIn0=",
 				Username:      "foo",
 				Password:      "bar",
 				Email:         "foo@bar.com",
@@ -2549,6 +2553,7 @@ func TestDockerDriver_AuthFromTaskConfig(t *testing.T) {
 				ServerAddr: "www.foobar.com",
 			},
 			AuthConfig: &registry.AuthConfig{
+				Auth:          "eyJ1c2VybmFtZSI6ImZvbyIsInBhc3N3b3JkIjoiYmFyIn0=",
 				Username:      "foo",
 				Password:      "bar",
 				ServerAddress: "www.foobar.com",

drivers/docker/utils.go

@@ -4,6 +4,7 @@
 package docker
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -102,12 +103,19 @@ func authFromTaskConfig(driverConfig *TaskConfig) authBackend {
 		if len(driverConfig.Auth.Username) == 0 && len(driverConfig.Auth.Password) == 0 && len(driverConfig.Auth.Email) == 0 && len(driverConfig.Auth.ServerAddr) == 0 {
 			return nil, nil
 		}
-		return &registrytypes.AuthConfig{
+
+		authConfig := &registrytypes.AuthConfig{
 			Username:      driverConfig.Auth.Username,
 			Password:      driverConfig.Auth.Password,
 			Email:         driverConfig.Auth.Email,
 			ServerAddress: driverConfig.Auth.ServerAddr,
-		}, nil
+		}
+
+		if err := encodeAuth(authConfig); err != nil {
+			return nil, err
+		}
+
+		return authConfig, nil
 	}
 }
 
@@ -140,6 +148,11 @@ func authFromDockerConfig(file string) authBackend {
 					IdentityToken: dockerAuthConfig.IdentityToken,
 					RegistryToken: dockerAuthConfig.RegistryToken,
 				}
+
+				if err := encodeAuth(auth); err != nil {
+					return nil, err
+				}
+
 				if authIsEmpty(auth) {
 					return nil, nil
 				}
@@ -187,6 +200,9 @@ func authFromHelper(helperName string) authBackend {
 			Username: response["Username"],
 			Password: response["Secret"],
 		}
+		if err := encodeAuth(auth); err != nil {
+			return nil, err
+		}
 
 		if authIsEmpty(auth) {
 			return nil, nil
@@ -195,6 +211,21 @@ func authFromHelper(helperName string) authBackend {
 	}
 }
 
+// some docker api calls require a base64 encoded basic auth string
+func encodeAuth(cfg *registrytypes.AuthConfig) error {
+	auth := &registrytypes.AuthConfig{
+		Username: cfg.Username,
+		Password: cfg.Password,
+	}
+	encodedJSON, err := json.Marshal(auth)
+	if err != nil {
+		return fmt.Errorf("error encoding basic auth: %v", err)
+	}
+
+	cfg.Auth = base64.URLEncoding.EncodeToString(encodedJSON)
+	return nil
+}
+
 // authIsEmpty returns if auth is nil or an empty structure
 func authIsEmpty(auth *registrytypes.AuthConfig) bool {
 	if auth == nil {

e2e/docker/doc.go

@@ -0,0 +1,5 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+// Package docker contains test cases related to the docker task driver.
+package docker

e2e/docker/docker_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package docker
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/cluster3"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/shoenig/test/must"
+)
+
+const (
+	registryService = "registry"
+)
+
+func TestDocker(t *testing.T) {
+	cluster3.Establish(t,
+		cluster3.Leader(),
+		cluster3.LinuxClients(1),
+	)
+
+	runRegistry(t)
+
+	t.Run("testRedis", testRedis)
+	t.Run("testAuthBasic", testAuthBasic)
+	t.Run("testAuthFileStatic", testAuthFileStatic)
+	t.Run("testAuthHelper", testAuthHelper)
+}
+
+func findService(t *testing.T, name string) (string, int) {
+	services, _, err := e2eutil.NomadClient(t).Services().Get(name, nil)
+	must.NoError(t, err, must.Sprintf("failed to find %q service", name))
+	must.Len(t, 1, services, must.Sprintf("expected 1 %q service", name))
+	return services[0].Address, services[0].Port
+}
+
+func runRegistry(t *testing.T) {
+	_, regCleanup := jobs3.Submit(t,
+		"../docker_registry/registry.hcl",
+		jobs3.Timeout(40*time.Second), // pulls an image
+	)
+	t.Cleanup(regCleanup)
+
+	// lookup registry address
+	addr, port := findService(t, registryService)
+	address := fmt.Sprintf("%s:%d", addr, port)
+
+	t.Logf("Setting up insecure private registry at %v", address)
+
+	// run the sed job to fixup the auth.json file with correct address and make
+	// sure the registry is marked as insecure for docker, otherwise pulls will
+	// fail
+	_, sedCleanup := jobs3.Submit(t,
+		"./input/registry-auths.hcl",
+		jobs3.Var("registry_address", address),
+		jobs3.Var("user", "root"),
+		jobs3.Var("helper_dir", "/usr/local/bin"),
+		jobs3.Var("auth_dir", "/etc"),
+		jobs3.Var("docker_conf_dir", "/etc/docker"),
+		jobs3.WaitComplete("create-files"),
+		jobs3.Timeout(20*time.Second),
+	)
+	t.Cleanup(sedCleanup)
+}
+
+func testRedis(t *testing.T) {
+	job, cleanup := jobs3.Submit(t, "./input/redis.hcl")
+	t.Cleanup(cleanup)
+
+	logs := job.TaskLogs("cache", "redis")
+	must.StrContains(t, logs.Stdout, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo")
+}
+
+func testAuthBasic(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	// run the private bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_basic.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("basic"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("basic", "echo")
+	must.StrContains(t, logs.Stdout, "The auth basic test is OK!")
+}
+
+func testAuthFileStatic(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	// run the private _static bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_static.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("static"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("static", "echo")
+	must.StrContains(t, logs.Stdout, "The static auth test is OK!")
+}
+
+func testAuthHelper(t *testing.T) {
+	// find the private registry service
+	regAddr, regPort := findService(t, "registry")
+
+	t.Log("registry", regAddr, regPort)
+
+	// run the private _helper bash image
+	bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_helper.hcl",
+		jobs3.Var("registry_address", regAddr),
+		jobs3.Var("registry_port", strconv.Itoa(regPort)),
+		jobs3.WaitComplete("helper"),
+	)
+	t.Cleanup(bashCleanup)
+	logs := bashJob.TaskLogs("helper", "echo")
+	must.StrContains(t, logs.Stdout, "The credentials helper auth test is OK!")
+}

e2e/docker/input/auth_basic.hcl

@@ -0,0 +1,75 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job runs a docker task using a container stored in a private registry
+# configured with basic authentication. The registry.hcl job should be running
+# and healthy before running this job. The registry_address and registry_port
+# HCL variables must be provided.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+variable "registry_username" {
+  type        = string
+  description = "The Basic Auth username of the local registry"
+  default     = "auth_basic_user"
+}
+
+variable "registry_password" {
+  type        = string
+  description = "The Basic Auth password of the local registry"
+  default     = "auth_basic_pass"
+}
+
+locals {
+  registry_auth = base64encode("${var.registry_username}:${var.registry_password}")
+}
+
+job "auth_basic" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "basic" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "docker"
+
+      config {
+        image          = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_basic:private"
+        args           = ["echo", "The auth basic test is OK!"]
+        auth_soft_fail = true
+
+        auth {
+          username = "${var.registry_username}"
+          password = "${var.registry_password}"
+        }
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/docker/input/auth_helper.hcl

@@ -0,0 +1,55 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job runs a docker task using a container stored in a private registry
+# configured with credentials helper authentication. The registry.hcl job should
+# be running and healthy before running this job.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+job "auth_static" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "helper" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "docker"
+
+      config {
+        image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_helper:private"
+        args  = ["echo", "The credentials helper auth test is OK!"]
+
+        # usename and password come from [docker-credential-]test.sh found on
+        # $PATH as specified by "helper=test.sh" in plugin config
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}

e2e/docker/input/auth_static.hcl

@@ -0,0 +1,65 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job runs a docker task using a container stored in a private registry
+# configured with file config static authentication. The registry.hcl job should
+# be running and healthy before running this job.
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+  default     = "localhost"
+}
+
+variable "registry_port" {
+  type        = number
+  description = "The HTTP port of the local registry"
+  default     = "7511"
+}
+
+job "auth_static" {
+  type = "batch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "static" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    network {
+      mode = "host"
+    }
+
+    task "echo" {
+      driver = "docker"
+
+      config {
+        image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_static:private"
+        args  = ["echo", "The static auth test is OK!"]
+
+        # usename and password come from auth.json in plugin config
+      }
+
+      resources {
+        cpu    = 100
+        memory = 64
+      }
+    }
+  }
+}
+
+# auth.json (must be pointed to by config=<path>/auth.json)
+#
+# {
+#   "auths": {
+#     "127.0.0.1:7511/docker.io/library/bash_auth_static": {
+#       "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz"
+#     }
+#   }
+# }
+

e2e/docker/input/redis.hcl

@@ -0,0 +1,35 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This is a simple redis job using the docker task driver.
+
+job "redis" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "cache" {
+    network {
+      port "db" {
+        to = 6379
+      }
+    }
+
+    task "redis" {
+      driver = "docker"
+
+      config {
+        image          = "docker.io/library/redis:7"
+        ports          = ["db"]
+        auth_soft_fail = true
+      }
+
+      resources {
+        cpu    = 50
+        memory = 128
+      }
+    }
+  }
+}

e2e/docker/input/registry-auths.hcl

@@ -0,0 +1,170 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# This job runs after the private registry is up and running, when we know
+# address and port provided by the bridge network. It is a sysbatch job
+# that writes these files on every linux client.
+#   - /usr/local/bin/docker-credential-test.sh
+#   - /etc/docker-registry-auth.json
+
+variable "registry_address" {
+  type        = string
+  description = "The HTTP address of the local registry"
+}
+
+variable "auth_dir" {
+  type        = string
+  description = "The destination directory of the auth.json file."
+  default     = "/tmp"
+}
+
+variable "helper_dir" {
+  type        = string
+  description = "The directory in which test.sh will be written."
+  default     = "/tmp"
+}
+
+variable "docker_conf_dir" {
+  type        = string
+  description = "The directory in which daemon.json will be written."
+  default     = "/tmp"
+}
+
+variable "user" {
+  type        = string
+  description = "The user to create files as. Should be root in e2e."
+  # no default because dealing with root files is annoying locally
+  # try -var=user=$USER for local development
+}
+
+job "registry-auths" {
+  type = "sysbatch"
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  group "create-files" {
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+
+    # write out the test.sh file into var.helper_dir
+    task "create-helper-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/test.sh", "${var.helper_dir}/docker-credential-test.sh"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/test.sh", "rwc:${var.helper_dir}"]
+      }
+
+      template {
+        destination = "local/test.sh"
+        perms       = "755"
+        data        = <<EOH
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+value=$(cat /dev/stdin)
+
+username="auth_helper_user"
+password="auth_helper_pass"
+
+case "${value}" in
+  ${var.registry_address}*)
+    echo "{\"Username\": \"$username\", \"Secret\": \"$password\"}"
+    exit 0
+    ;;
+  *)
+    echo "must use local registry"
+    exit 3
+    ;;
+esac
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+
+    # write out the auth.json file into var.auth_dir
+    task "create-auth-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/auth.json", "${var.auth_dir}/auth.json"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/auth.json", "rwc:${var.auth_dir}"]
+      }
+      template {
+        perms       = "644"
+        destination = "local/auth.json"
+        data        = <<EOH
+{
+  "auths": {
+    "${var.registry_address}:/docker.io/library/bash_auth_static": {
+      "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz"
+    }
+  }
+}
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+  }
+
+  group "create-conf" {
+    task "create-daemon-file" {
+      driver = "pledge"
+      user   = "${var.user}"
+
+      config {
+        command  = "cp"
+        args     = ["${NOMAD_TASK_DIR}/daemon.json", "${var.docker_conf_dir}/daemon.json"]
+        promises = "stdio rpath wpath cpath"
+        unveil   = ["r:${NOMAD_TASK_DIR}/daemon.json", "rwc:${var.docker_conf_dir}"]
+      }
+
+      template {
+        destination = "local/daemon.json"
+        perms       = "644"
+        data        = <<EOH
+{
+   "insecure-registries": [
+      "${var.registry_address}"
+   ]
+}
+EOH
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+
+    task "restart-docker" {
+      driver = "raw_exec" # TODO: see if this could be done with pledge?
+
+      config {
+        command = "service"
+        args    = ["docker", "restart"]
+      }
+      resources {
+        cpu    = 100
+        memory = 32
+      }
+    }
+  }
+}

e2e/e2e_test.go

@@ -35,6 +35,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/connect"
 	_ "github.com/hashicorp/nomad/e2e/consultemplate"
 	_ "github.com/hashicorp/nomad/e2e/disconnectedclients"
+	_ "github.com/hashicorp/nomad/e2e/docker"
 	_ "github.com/hashicorp/nomad/e2e/isolation"
 	_ "github.com/hashicorp/nomad/e2e/metrics"
 	_ "github.com/hashicorp/nomad/e2e/namespaces"

e2e/podman/podman_test.go

@@ -42,7 +42,7 @@ func findService(t *testing.T, name string) (string, int) {
 
 func runRegistry(t *testing.T) {
 	_, regCleanup := jobs3.Submit(t,
-		"./input/registry.hcl",
+		"../docker_registry/registry.hcl",
 		jobs3.Timeout(40*time.Second), // pulls an image
 	)
 	t.Cleanup(regCleanup)

e2e/terraform/etc/nomad.d/client-linux.hcl

@@ -43,6 +43,10 @@ plugin "docker" {
     volumes {
       enabled = true
     }
+    auth {
+      helper = "test.sh"
+      config = "/etc/auth.json"
+    }
   }
 }