template: allow change_mode script to run after client restart (#23663)

For templates with `change_mode = "script"`, we set a driver handle in the
poststart method, so the template runner can execute the script inside the
task. But when the client is restarted and the template contents change during
that window, we trigger a change_mode in the prestart method. In that case, the
hook will not have the handle and so returns an errror trying to run the change
mode.

We restore the driver handle before we call any prestart hooks, so we can pass
that handle in the constructor whenever it's available. In the normal task start
case the handle will be empty but also won't be called.

The error messages are also misleading, as there's no capabilities check
happening here. Update the error messages to match.

Fixes: https://github.com/hashicorp/nomad/issues/15851
Ref: https://hashicorp.atlassian.net/browse/NET-9338

.changelog/23663.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+template: Fixed a bug where change_mode = "script" would not execute after a client restart
+```

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -126,6 +126,7 @@ func (tr *TaskRunner) initHooks() {
 			consulNamespace:     consulNamespace,
 			nomadNamespace:      tr.alloc.Job.Namespace,
 			renderOnTaskRestart: task.RestartPolicy.RenderTemplates,
+			driverHandle:        tr.handle,
 		}))
 	}
 

client/allocrunner/taskrunner/template/template.go

@@ -585,7 +585,7 @@ func (tm *TaskTemplateManager) processScript(script *structs.ChangeScript, wg *s
 
 	if tm.handle == nil {
 		failureMsg := fmt.Sprintf(
-			"Template failed to run script %v with arguments %v because task driver doesn't support the exec operation",
+			"Template failed to run script %v with arguments %v because task driver handle is not available",
 			script.Command,
 			script.Args,
 		)

client/allocrunner/taskrunner/template_hook.go

@@ -55,6 +55,11 @@ type templateHookConfig struct {
 
 	// hookResources are used to fetch Consul tokens
 	hookResources *cstructs.AllocHookResources
+
+	// driverHandle is the task driver executor used to run scripts when the
+	// template change mode is set to script. Typically this will be nil in this
+	// config struct, unless we're restoring a task after a client restart.
+	driverHandle ti.ScriptExecutor
 }
 
 type templateHook struct {
@@ -68,7 +73,10 @@ type templateHook struct {
 	managerLock     sync.Mutex
 
 	// driverHandle is the task driver executor used by the template manager to
-	// run scripts when the template change mode is set to script.
+	// run scripts when the template change mode is set to script. This value is
+	// set in the Poststart hook after the task has run, or passed in as
+	// configuration if this is a task that's being restored after a client
+	// restart.
 	//
 	// Must obtain a managerLock before changing. It may be nil.
 	driverHandle ti.ScriptExecutor
@@ -105,6 +113,7 @@ func newTemplateHook(config *templateHookConfig) *templateHook {
 		config:          config,
 		consulNamespace: config.consulNamespace,
 		logger:          config.logger.Named(templateHookName),
+		driverHandle:    config.driverHandle,
 	}
 }
 
@@ -206,7 +215,7 @@ func (h *templateHook) Poststart(_ context.Context, req *interfaces.TaskPoststar
 	} else {
 		for _, tmpl := range h.config.templates {
 			if tmpl.ChangeMode == structs.TemplateChangeModeScript {
-				return fmt.Errorf("template has change mode set to 'script' but the driver it uses does not provide exec capability")
+				return fmt.Errorf("template has change mode set to 'script' but task driver handle is not available")
 			}
 		}
 	}

client/allocrunner/taskrunner/template_hook_test.go

@@ -8,7 +8,9 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"path"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -269,3 +271,70 @@ func Test_templateHook_Prestart_Vault(t *testing.T) {
 		})
 	}
 }
+
+// TestTemplateHook_RestoreChangeModeScript exercises change_mode=script
+// behavior for a task restored after a client restart
+func TestTemplateHook_RestoreChangeModeScript(t *testing.T) {
+
+	logger := testlog.HCLogger(t)
+	tmpDir := t.TempDir()
+	destPath := filepath.Join(tmpDir, "foo.txt")
+	must.NoError(t, os.WriteFile(destPath, []byte("original-content"), 0755))
+
+	clientConfig := config.DefaultConfig()
+	clientConfig.TemplateConfig.DisableSandbox = true
+
+	alloc := mock.BatchAlloc()
+	task := alloc.Job.TaskGroups[0].Tasks[0]
+	envBuilder := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
+
+	lifecycle := trtesting.NewMockTaskHooks()
+	lifecycle.HasHandle = true
+
+	events := &trtesting.MockEmitter{}
+
+	executor := &simpleExec{
+		code: 117,
+		err:  fmt.Errorf("oh no"),
+	}
+
+	hook := newTemplateHook(&templateHookConfig{
+		alloc:     alloc,
+		logger:    logger,
+		lifecycle: lifecycle,
+		events:    events,
+		templates: []*structs.Template{{
+			DestPath:     destPath,
+			EmbeddedTmpl: "changed-content",
+			ChangeMode:   structs.TemplateChangeModeScript,
+			ChangeScript: &structs.ChangeScript{
+				Command: "echo",
+				Args:    []string{"foo"},
+			},
+		}},
+		clientConfig:  clientConfig,
+		envBuilder:    envBuilder,
+		hookResources: &cstructs.AllocHookResources{},
+		driverHandle:  executor,
+	})
+	req := &interfaces.TaskPrestartRequest{
+		Alloc:   alloc,
+		Task:    task,
+		TaskDir: &allocdir.TaskDir{Dir: tmpDir},
+	}
+
+	must.NoError(t, hook.Prestart(context.TODO(), req, nil))
+
+	// self-test the test by making sure we really changed the template file
+	out, err := os.ReadFile(destPath)
+	must.NoError(t, err)
+	must.Eq(t, "changed-content", string(out))
+
+	// verify our change script executed
+	gotEvents := events.Events()
+	must.Len(t, 1, gotEvents)
+	must.Eq(t, structs.TaskHookFailed, gotEvents[0].Type)
+	must.Eq(t, "Template failed to run script echo with arguments [foo] on change: oh no Exit code: 117",
+		gotEvents[0].DisplayMessage)
+
+}