nomad/client/allocrunner/taskrunner/envoy_version_hook_test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package taskrunner

import (
	"context"
	"errors"
	"testing"

	"github.com/hashicorp/nomad/ci"
	"github.com/hashicorp/nomad/client/allocdir"
	ifs "github.com/hashicorp/nomad/client/allocrunner/interfaces"
	clientconsul "github.com/hashicorp/nomad/client/consul"
	"github.com/hashicorp/nomad/client/taskenv"
	"github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/helper/envoy"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/drivers/fsisolation"
	"github.com/shoenig/test/must"
)

var (
	taskEnvDefault = taskenv.NewTaskEnv(nil, nil, nil, map[string]string{
		"meta.connect.sidecar_image": envoy.ImageFormat,
		"meta.connect.gateway_image": envoy.ImageFormat,
	}, "", "")
)

func TestEnvoyVersionHook_semver(t *testing.T) {
	ci.Parallel(t)

	t.Run("with v", func(t *testing.T) {
		result, err := semver("v1.2.3")
		must.NoError(t, err)
		must.Eq(t, "1.2.3", result)
	})

	t.Run("without v", func(t *testing.T) {
		result, err := semver("1.2.3")
		must.NoError(t, err)
		must.Eq(t, "1.2.3", result)
	})

	t.Run("unexpected", func(t *testing.T) {
		_, err := semver("foo")
		must.ErrorContains(t, err, "unexpected envoy version format: Malformed version: foo")
	})
}

func TestEnvoyVersionHook_taskImage(t *testing.T) {
	ci.Parallel(t)

	t.Run("absent", func(t *testing.T) {
		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
			// empty
		})
		must.Eq(t, envoy.ImageFormat, result)
	})

	t.Run("not a string", func(t *testing.T) {
		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
			"image": 7, // not a string
		})
		must.Eq(t, envoy.ImageFormat, result)
	})

	t.Run("normal", func(t *testing.T) {
		result := (*envoyVersionHook)(nil).taskImage(map[string]interface{}{
			"image": "custom/envoy:latest",
		})
		must.Eq(t, "custom/envoy:latest", result)
	})
}

func TestEnvoyVersionHook_tweakImage(t *testing.T) {
	ci.Parallel(t)

	image := envoy.ImageFormat

	t.Run("legacy", func(t *testing.T) {
		_, err := (*envoyVersionHook)(nil).tweakImage(image, nil)
		must.Error(t, err)
	})

	t.Run("unexpected", func(t *testing.T) {
		_, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
			"envoy": {"foo", "bar", "baz"},
		})
		must.ErrorContains(t, err, "unexpected envoy version format: Malformed version: foo")
	})

	t.Run("standard envoy", func(t *testing.T) {
		result, err := (*envoyVersionHook)(nil).tweakImage(image, map[string][]string{
			"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
		})
		must.NoError(t, err)
		must.Eq(t, "docker.io/envoyproxy/envoy:v1.15.0", result)
	})

	t.Run("custom image", func(t *testing.T) {
		custom := "custom-${NOMAD_envoy_version}/envoy:${NOMAD_envoy_version}"
		result, err := (*envoyVersionHook)(nil).tweakImage(custom, map[string][]string{
			"envoy": {"1.15.0", "1.14.4", "1.13.4", "1.12.6"},
		})
		must.NoError(t, err)
		must.Eq(t, "custom-1.15.0/envoy:1.15.0", result)
	})
}

func TestEnvoyVersionHook_interpolateImage(t *testing.T) {
	ci.Parallel(t)

	hook := (*envoyVersionHook)(nil)

	t.Run("default sidecar", func(t *testing.T) {
		task := &structs.Task{
			Config: map[string]interface{}{"image": envoy.SidecarConfigVar},
		}
		hook.interpolateImage(task, taskEnvDefault)
		must.Eq(t, envoy.ImageFormat, task.Config["image"])
	})

	t.Run("default gateway", func(t *testing.T) {
		task := &structs.Task{
			Config: map[string]interface{}{"image": envoy.GatewayConfigVar},
		}
		hook.interpolateImage(task, taskEnvDefault)
		must.Eq(t, envoy.ImageFormat, task.Config["image"])
	})

	t.Run("custom static", func(t *testing.T) {
		task := &structs.Task{
			Config: map[string]interface{}{"image": "custom/envoy"},
		}
		hook.interpolateImage(task, taskEnvDefault)
		must.Eq(t, "custom/envoy", task.Config["image"])
	})

	t.Run("custom interpolated", func(t *testing.T) {
		task := &structs.Task{
			Config: map[string]interface{}{"image": "${MY_ENVOY}"},
		}
		hook.interpolateImage(task, taskenv.NewTaskEnv(map[string]string{
			"MY_ENVOY": "my/envoy",
		}, map[string]string{
			"MY_ENVOY": "my/envoy",
		}, nil, nil, "", ""))
		must.Eq(t, "my/envoy", task.Config["image"])
	})

	t.Run("no image", func(t *testing.T) {
		task := &structs.Task{
			Config: map[string]interface{}{},
		}
		hook.interpolateImage(task, taskEnvDefault)
		must.MapEmpty(t, task.Config)
	})
}

func TestEnvoyVersionHook_skip(t *testing.T) {
	ci.Parallel(t)

	h := new(envoyVersionHook)

	t.Run("not docker", func(t *testing.T) {
		skip := h.skip(&ifs.TaskPrestartRequest{
			Task: &structs.Task{
				Driver: "exec",
				Config: nil,
			},
		})
		must.True(t, skip)
	})

	t.Run("not connect", func(t *testing.T) {
		skip := h.skip(&ifs.TaskPrestartRequest{
			Task: &structs.Task{
				Driver: "docker",
				Kind:   "",
			},
		})
		must.True(t, skip)
	})

	t.Run("version not needed", func(t *testing.T) {
		skip := h.skip(&ifs.TaskPrestartRequest{
			Task: &structs.Task{
				Driver: "docker",
				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
				Config: map[string]interface{}{
					"image": "custom/envoy:latest",
				},
			},
		})
		must.True(t, skip)
	})

	t.Run("version needed custom", func(t *testing.T) {
		skip := h.skip(&ifs.TaskPrestartRequest{
			Task: &structs.Task{
				Driver: "docker",
				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
				Config: map[string]interface{}{
					"image": "custom/envoy:v${NOMAD_envoy_version}",
				},
			},
		})
		must.False(t, skip)
	})

	t.Run("version needed standard", func(t *testing.T) {
		skip := h.skip(&ifs.TaskPrestartRequest{
			Task: &structs.Task{
				Driver: "docker",
				Kind:   structs.NewTaskKind(structs.ConnectProxyPrefix, "task"),
				Config: map[string]interface{}{
					"image": envoy.ImageFormat,
				},
			},
		})
		must.False(t, skip)
	})
}

func TestTaskRunner_EnvoyVersionHook_Prestart_standard(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Setup a mock for Consul API
	spAPI := consul.MockSupportedProxiesAPI{
		Value: map[string][]string{
			"envoy": {"1.15.0", "1.14.4"},
		},
		Error: nil,
	}
	spAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return spAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook
	must.NoError(t, h.Prestart(context.Background(), request, &response))

	// Assert the Task.Config[image] is concrete
	must.Eq(t, "docker.io/envoyproxy/envoy:v1.15.0", request.Task.Config["image"])
}

func TestTaskRunner_EnvoyVersionHook_Prestart_custom(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	alloc.Job.TaskGroups[0].Tasks[0].Driver = "podman"
	alloc.Job.TaskGroups[0].Tasks[0].Config["image"] = "custom-${NOMAD_envoy_version}:latest"
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Setup a mock for Consul API
	spAPI := consul.MockSupportedProxiesAPI{
		Value: map[string][]string{
			"envoy": {"1.14.1", "1.13.3"},
		},
		Error: nil,
	}
	spAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return spAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook
	must.NoError(t, h.Prestart(context.Background(), request, &response))

	// Assert the Task.Config[image] is concrete
	must.Eq(t, "custom-1.14.1:latest", request.Task.Config["image"])
}

func TestTaskRunner_EnvoyVersionHook_Prestart_skip(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	alloc.Job.TaskGroups[0].Tasks[0].Driver = "exec"
	alloc.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{
		"command": "/sidecar",
	}
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Setup a mock for Consul API
	spAPI := consul.MockSupportedProxiesAPI{
		Value: map[string][]string{
			"envoy": {"1.14.1", "1.13.3"},
		},
		Error: nil,
	}
	spAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return spAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook
	must.NoError(t, h.Prestart(context.Background(), request, &response))

	// Assert the Task.Config[image] does not get set
	must.MapNotContainsKey(t, request.Task.Config, "image")
}

func TestTaskRunner_EnvoyVersionHook_Prestart_no_fallback(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Setup a mock for Consul API
	spAPI := consul.MockSupportedProxiesAPI{
		Value: nil, // old consul, no .xDS.SupportedProxies
		Error: nil,
	}
	spAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return spAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook
	must.Error(t, h.Prestart(context.Background(), request, &response))
}

func TestTaskRunner_EnvoyVersionHook_Prestart_error(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Setup a mock for Consul API
	spAPI := consul.MockSupportedProxiesAPI{
		Value: nil,
		Error: errors.New("some consul error"),
	}
	spAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return spAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, spAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook, error should be recoverable
	err := h.Prestart(context.Background(), request, &response)
	must.ErrorContains(t, err, "error retrieving supported Envoy versions from Consul: some consul error")
}

func TestTaskRunner_EnvoyVersionHook_Prestart_restart(t *testing.T) {
	ci.Parallel(t)

	logger := testlog.HCLogger(t)

	// Setup an Allocation
	alloc := mock.ConnectAlloc()
	alloc.Job.TaskGroups[0].Tasks[0] = mock.ConnectSidecarTask()
	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyVersionHook", alloc.ID)
	defer cleanupDir()

	// Set up a mock for Consul API.
	mockProxiesAPI := consul.MockSupportedProxiesAPI{
		Value: map[string][]string{
			"envoy": {"1.15.0", "1.14.4"},
		},
		Error: nil,
	}
	mockProxiesAPIFunc := func(_ string) clientconsul.SupportedProxiesAPI { return mockProxiesAPI }

	// Run envoy_version hook
	h := newEnvoyVersionHook(newEnvoyVersionHookConfig(alloc, mockProxiesAPIFunc, logger))

	// Create a prestart request
	request := &ifs.TaskPrestartRequest{
		Task:    alloc.Job.TaskGroups[0].Tasks[0],
		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0]),
		TaskEnv: taskEnvDefault,
	}
	must.NoError(t, request.TaskDir.Build(fsisolation.None, nil, alloc.Job.TaskGroups[0].Tasks[0].User))

	// Prepare a response
	var response ifs.TaskPrestartResponse

	// Run the hook and ensure the tasks image has been modified.
	must.NoError(t, h.Prestart(context.Background(), request, &response))
	must.Eq(t, "docker.io/envoyproxy/envoy:v1.15.0", request.Task.Config["image"])

	// Overwrite the previously modified image. This is the same behaviour that
	// occurs when the server sends a non-destructive allocation update.
	request.Task.Config["image"] = "${meta.connect.sidecar_image}"

	// Run the Prestart hook function again, and ensure the image is updated.
	must.NoError(t, h.Prestart(context.Background(), request, &response))
	must.Eq(t, "docker.io/envoyproxy/envoy:v1.15.0", request.Task.Config["image"])

	// Run the hook again, and ensure the config is still the same mimicking
	// a non-user initiated restart.
	must.NoError(t, h.Prestart(context.Background(), request, &response))
	must.Eq(t, "docker.io/envoyproxy/envoy:v1.15.0", request.Task.Config["image"])
}