Add OOM detection for exec driver (#19563)

* Add OomKilled field to executor proto format

* Teach linux executor to detect and report OOMs

* Teach exec driver to propagate OOMKill information

* Fix data race

* use tail /dev/zero to create oom condition

* use new test framework

* minor tweaks to executor test

* add cl entry

* remove type conversion

---------

Co-authored-by: Marvin Chin <marvinchin@users.noreply.github.com>
Co-authored-by: Seth Hoenig <shoenig@duck.com>

drivers/shared/executor/executor_linux_test.go

@@ -62,6 +62,7 @@ func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
 		"/bin/echo":         "/bin/echo",
 		"/bin/bash":         "/bin/bash",
 		"/bin/sleep":        "/bin/sleep",
+		"/bin/tail":         "/bin/tail",
 		"/foobar":           "/does/not/exist",
 	}
 
@@ -267,6 +268,40 @@ passwd`
 	}, func(err error) { t.Error(err) })
 }
 
+func TestExecutor_OOMKilled(t *testing.T) {
+	ci.Parallel(t)
+	testutil.ExecCompatible(t)
+	testutil.CgroupsCompatible(t)
+
+	testExecCmd := testExecutorCommandWithChroot(t)
+	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
+	execCmd.Cmd = "/bin/tail"
+	execCmd.Args = []string{"/dev/zero"}
+	defer allocDir.Destroy()
+
+	execCmd.ResourceLimits = true
+	execCmd.ModePID = "private"
+	execCmd.ModeIPC = "private"
+	execCmd.Resources.LinuxResources.MemoryLimitBytes = 10 * 1024 * 1024
+	execCmd.Resources.NomadResources.Memory.MemoryMB = 10
+
+	executor := NewExecutorWithIsolation(testlog.HCLogger(t), compute)
+	defer executor.Shutdown("SIGKILL", 0)
+
+	ps, err := executor.Launch(execCmd)
+	must.NoError(t, err)
+	must.Positive(t, ps.Pid)
+
+	estate, err := executor.Wait(context.Background())
+	must.NoError(t, err)
+	must.Positive(t, estate.ExitCode)
+	must.True(t, estate.OOMKilled)
+
+	// Shut down executor
+	must.NoError(t, executor.Shutdown("", 0))
+	executor.Wait(context.Background())
+}
+
 // TestExecutor_CgroupPaths asserts that process starts with independent cgroups
 // hierarchy created for this process
 func TestExecutor_CgroupPaths(t *testing.T) {