Add OOM detection for exec driver (#19563)

* Add OomKilled field to executor proto format

* Teach linux executor to detect and report OOMs

* Teach exec driver to propagate OOMKill information

* Fix data race

* use tail /dev/zero to create oom condition

* use new test framework

* minor tweaks to executor test

* add cl entry

* remove type conversion

---------

Co-authored-by: Marvin Chin <marvinchin@users.noreply.github.com>
Co-authored-by: Seth Hoenig <shoenig@duck.com>

.changelog/19563.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+drivers/exec: Added support for OOM detection in exec driver
+```

ci/slow.go

@@ -43,6 +43,7 @@ var TinyChroot = map[string]string{
 	"/usr/bin/dash":  "/bin/sh",
 	"/usr/bin/bash":  "/bin/bash",
 	"/usr/bin/cat":   "/bin/cat",
+	"/usr/bin/tail":  "/bin/tail",
 
 	// destination: /usr/bin
 	"/usr/bin/stty":   "/usr/bin/stty",

drivers/exec/driver.go

@@ -552,8 +552,9 @@ func (d *Driver) handleWait(ctx context.Context, handle *taskHandle, ch chan *dr
 		}
 	} else {
 		result = &drivers.ExitResult{
-			ExitCode: ps.ExitCode,
-			Signal:   ps.Signal,
+			ExitCode:  ps.ExitCode,
+			Signal:    ps.Signal,
+			OOMKilled: ps.OOMKilled,
 		}
 	}
 

drivers/exec/driver_test.go

@@ -32,6 +32,7 @@ import (
 	"github.com/hashicorp/nomad/plugins/drivers"
 	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
 	"github.com/hashicorp/nomad/testutil"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 )
 
@@ -788,6 +789,48 @@ func TestExecDriver_NoPivotRoot(t *testing.T) {
 	require.NoError(t, harness.DestroyTask(task.ID, true))
 }
 
+func TestExecDriver_OOMKilled(t *testing.T) {
+	ci.Parallel(t)
+	ctestutils.ExecCompatible(t)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	d := newExecDriverTest(t, ctx)
+	harness := dtestutil.NewDriverHarness(t, d)
+	allocID := uuid.Generate()
+	name := "oom-killed"
+	task := &drivers.TaskConfig{
+		AllocID:   allocID,
+		ID:        uuid.Generate(),
+		Name:      name,
+		Resources: testResources(allocID, name),
+	}
+	task.Resources.LinuxResources.MemoryLimitBytes = 10 * 1024 * 1024
+	task.Resources.NomadResources.Memory.MemoryMB = 10
+
+	tc := &TaskConfig{
+		Command: "/bin/tail",
+		Args:    []string{"/dev/zero"},
+	}
+	must.NoError(t, task.EncodeConcreteDriverConfig(&tc))
+
+	cleanup := harness.MkAllocDir(task, false)
+	defer cleanup()
+
+	handle, _, err := harness.StartTask(task)
+	must.NoError(t, err)
+
+	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
+	must.NoError(t, err)
+	result := <-ch
+	must.False(t, result.Successful(), must.Sprint("container should OOM"))
+	must.True(t, result.OOMKilled, must.Sprintf("got non-OOM error, code: %d, err: %v", result.ExitCode, result.Err))
+
+	t.Logf("Successfully killed by OOM killer")
+	must.NoError(t, harness.DestroyTask(task.ID, true))
+}
+
 func TestDriver_Config_validate(t *testing.T) {
 	ci.Parallel(t)
 	t.Run("pid/ipc", func(t *testing.T) {

drivers/exec/handle.go

@@ -76,7 +76,6 @@ func (h *taskHandle) run() {
 	h.procState = drivers.TaskStateExited
 	h.exitResult.ExitCode = ps.ExitCode
 	h.exitResult.Signal = ps.Signal
+	h.exitResult.OOMKilled = ps.OOMKilled
 	h.completedAt = ps.Time
-
-	// TODO: detect if the task OOMed
 }

drivers/shared/executor/executor.go

@@ -259,10 +259,11 @@ func (c *ExecCommand) Close() {
 
 // ProcessState holds information about the state of a user process.
 type ProcessState struct {
-	Pid      int
-	ExitCode int
-	Signal   int
-	Time     time.Time
+	Pid       int
+	ExitCode  int
+	Signal    int
+	OOMKilled bool
+	Time      time.Time
 }
 
 // ExecutorVersion is the version of the executor

drivers/shared/executor/executor_linux.go

@@ -15,6 +15,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -202,6 +203,23 @@ func (l *LibcontainerExecutor) Wait(ctx context.Context) (*ProcessState, error)
 func (l *LibcontainerExecutor) wait() {
 	defer close(l.userProcExited)
 
+	// Best effort detection of OOMs. It's possible for us to miss OOM notifications in
+	// the event that the wait returns before we read from the OOM notification channel
+	var oomKilled atomic.Bool
+	go func() {
+		oomCh, err := l.container.NotifyOOM()
+		if err != nil {
+			l.logger.Error("failed to get OOM notification channel for container(%s): %v", l.id, err)
+			return
+		}
+
+		for range oomCh {
+			oomKilled.Store(true)
+			// We can terminate this goroutine as soon as we've seen the first OOM
+			return
+		}
+	}()
+
 	ps, err := l.userProc.Wait()
 	if err != nil {
 		// If the process has exited before we called wait an error is returned
@@ -229,10 +247,11 @@ func (l *LibcontainerExecutor) wait() {
 	}
 
 	l.exitState = &ProcessState{
-		Pid:      ps.Pid(),
-		ExitCode: exitCode,
-		Signal:   signal,
-		Time:     time.Now(),
+		Pid:       ps.Pid(),
+		ExitCode:  exitCode,
+		Signal:    signal,
+		OOMKilled: oomKilled.Load(),
+		Time:      time.Now(),
 	}
 }
 

drivers/shared/executor/executor_linux_test.go

@@ -62,6 +62,7 @@ func testExecutorCommandWithChroot(t *testing.T) *testExecCmd {
 		"/bin/echo":         "/bin/echo",
 		"/bin/bash":         "/bin/bash",
 		"/bin/sleep":        "/bin/sleep",
+		"/bin/tail":         "/bin/tail",
 		"/foobar":           "/does/not/exist",
 	}
 
@@ -267,6 +268,40 @@ passwd`
 	}, func(err error) { t.Error(err) })
 }
 
+func TestExecutor_OOMKilled(t *testing.T) {
+	ci.Parallel(t)
+	testutil.ExecCompatible(t)
+	testutil.CgroupsCompatible(t)
+
+	testExecCmd := testExecutorCommandWithChroot(t)
+	execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir
+	execCmd.Cmd = "/bin/tail"
+	execCmd.Args = []string{"/dev/zero"}
+	defer allocDir.Destroy()
+
+	execCmd.ResourceLimits = true
+	execCmd.ModePID = "private"
+	execCmd.ModeIPC = "private"
+	execCmd.Resources.LinuxResources.MemoryLimitBytes = 10 * 1024 * 1024
+	execCmd.Resources.NomadResources.Memory.MemoryMB = 10
+
+	executor := NewExecutorWithIsolation(testlog.HCLogger(t), compute)
+	defer executor.Shutdown("SIGKILL", 0)
+
+	ps, err := executor.Launch(execCmd)
+	must.NoError(t, err)
+	must.Positive(t, ps.Pid)
+
+	estate, err := executor.Wait(context.Background())
+	must.NoError(t, err)
+	must.Positive(t, estate.ExitCode)
+	must.True(t, estate.OOMKilled)
+
+	// Shut down executor
+	must.NoError(t, executor.Shutdown("", 0))
+	executor.Wait(context.Background())
+}
+
 // TestExecutor_CgroupPaths asserts that process starts with independent cgroups
 // hierarchy created for this process
 func TestExecutor_CgroupPaths(t *testing.T) {

drivers/shared/executor/proto/executor.pb.go

@@ -791,6 +791,7 @@ type ProcessState struct {
 	ExitCode             int32                `protobuf:"varint,2,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"`
 	Signal               int32                `protobuf:"varint,3,opt,name=signal,proto3" json:"signal,omitempty"`
 	Time                 *timestamp.Timestamp `protobuf:"bytes,4,opt,name=time,proto3" json:"time,omitempty"`
+	OomKilled            bool                 `protobuf:"varint,5,opt,name=oom_killed,json=oomKilled,proto3" json:"oom_killed,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}             `json:"-"`
 	XXX_unrecognized     []byte               `json:"-"`
 	XXX_sizecache        int32                `json:"-"`
@@ -849,6 +850,13 @@ func (m *ProcessState) GetTime() *timestamp.Timestamp {
 	return nil
 }
 
+func (m *ProcessState) GetOomKilled() bool {
+	if m != nil {
+		return m.OomKilled
+	}
+	return false
+}
+
 func init() {
 	proto.RegisterType((*LaunchRequest)(nil), "hashicorp.nomad.plugins.executor.proto.LaunchRequest")
 	proto.RegisterType((*LaunchResponse)(nil), "hashicorp.nomad.plugins.executor.proto.LaunchResponse")
@@ -874,74 +882,75 @@ func init() {
 }
 
 var fileDescriptor_66b85426380683f3 = []byte{
-	// 1058 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x6d, 0x6f, 0x1b, 0x45,
-	0x10, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0x2e, 0xa8, 0x5c, 0x8d, 0x50, 0xcd, 0x21, 0x51,
-	0x0b, 0xca, 0x25, 0x4a, 0xdf, 0x90, 0x90, 0x28, 0x22, 0x2d, 0xa8, 0x52, 0x1a, 0x45, 0x97, 0x42,
-	0x25, 0x3e, 0x70, 0x6c, 0xee, 0xb6, 0xf6, 0x2a, 0xf6, 0xed, 0xb2, 0xbb, 0xe7, 0x04, 0x09, 0x89,
-	0x4f, 0xfc, 0x03, 0x90, 0xf8, 0x31, 0xfc, 0x38, 0xb4, 0x6f, 0x17, 0x3b, 0x2d, 0xd5, 0xb9, 0x88,
-	0x4f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x27, 0x17, 0x74, 0x41, 0x84,
-	0xdc, 0x95, 0x53, 0x2c, 0x48, 0xbe, 0x4b, 0x2e, 0x48, 0x56, 0x2a, 0x26, 0x76, 0xb9, 0x60, 0x8a,
-	0x55, 0xc7, 0xd8, 0x1c, 0xd1, 0xc7, 0x53, 0x2c, 0xa7, 0x34, 0x63, 0x82, 0xc7, 0x05, 0x9b, 0xe3,
-	0x3c, 0xe6, 0xb3, 0x72, 0x42, 0x0b, 0x19, 0xaf, 0xe2, 0x86, 0xb7, 0x26, 0x8c, 0x4d, 0x66, 0xc4,
-	0x3a, 0x39, 0x2d, 0x5f, 0xee, 0x2a, 0x3a, 0x27, 0x52, 0xe1, 0x39, 0x77, 0x80, 0xc8, 0x11, 0x77,
-	0x7d, 0x78, 0x1b, 0xce, 0x9e, 0x2c, 0x26, 0xfa, 0xbb, 0x09, 0xfd, 0x43, 0x5c, 0x16, 0xd9, 0x34,
-	0x21, 0x3f, 0x97, 0x44, 0x2a, 0x34, 0x80, 0x46, 0x36, 0xcf, 0xc3, 0x60, 0x14, 0x8c, 0x3b, 0x89,
-	0xfe, 0x44, 0x08, 0x36, 0xb1, 0x98, 0xc8, 0x70, 0x63, 0xd4, 0x18, 0x77, 0x12, 0xf3, 0x8d, 0x8e,
-	0xa0, 0x23, 0x88, 0x64, 0xa5, 0xc8, 0x88, 0x0c, 0x1b, 0xa3, 0x60, 0xdc, 0xdd, 0xdf, 0x8b, 0xff,
-	0x2d, 0x71, 0x17, 0xdf, 0x86, 0x8c, 0x13, 0xcf, 0x4b, 0x2e, 0x5d, 0xa0, 0x5b, 0xd0, 0x95, 0x2a,
-	0x67, 0xa5, 0x4a, 0x39, 0x56, 0xd3, 0x70, 0xd3, 0x44, 0x07, 0x6b, 0x3a, 0xc6, 0x6a, 0xea, 0x00,
-	0x44, 0x08, 0x0b, 0xd8, 0xaa, 0x00, 0x44, 0x08, 0x03, 0x18, 0x40, 0x83, 0x14, 0x8b, 0xb0, 0x69,
-	0x92, 0xd4, 0x9f, 0x3a, 0xef, 0x52, 0x12, 0x11, 0xb6, 0x0c, 0xd6, 0x7c, 0xa3, 0x9b, 0xd0, 0x56,
-	0x58, 0x9e, 0xa5, 0x39, 0x15, 0x61, 0xdb, 0xd8, 0x5b, 0xfa, 0xfc, 0x98, 0x0a, 0x74, 0x1b, 0x76,
-	0x7c, 0x3e, 0xe9, 0x8c, 0xce, 0xa9, 0x92, 0x61, 0x67, 0x14, 0x8c, 0xdb, 0xc9, 0xb6, 0x37, 0x1f,
-	0x1a, 0x2b, 0xda, 0x83, 0x77, 0x4f, 0xb1, 0xa4, 0x59, 0xca, 0x05, 0xcb, 0x88, 0x94, 0x69, 0x36,
-	0x11, 0xac, 0xe4, 0x21, 0x18, 0x34, 0x32, 0xff, 0x1d, 0xdb, 0xbf, 0x0e, 0xcc, 0x3f, 0xe8, 0x31,
-	0x34, 0xe7, 0xac, 0x2c, 0x94, 0x0c, 0xbb, 0xa3, 0xc6, 0xb8, 0xbb, 0x7f, 0xa7, 0xe6, 0x53, 0x3d,
-	0xd3, 0xa4, 0xc4, 0x71, 0xd1, 0xb7, 0xd0, 0xca, 0xc9, 0x82, 0xea, 0x17, 0xef, 0x19, 0x37, 0x9f,
-	0xd5, 0x74, 0xf3, 0xd8, 0xb0, 0x12, 0xcf, 0x46, 0x53, 0xb8, 0x5e, 0x10, 0x75, 0xce, 0xc4, 0x59,
-	0x4a, 0x25, 0x9b, 0x61, 0x45, 0x59, 0x11, 0xf6, 0x4d, 0x11, 0xbf, 0xa8, 0xe9, 0xf2, 0xc8, 0xf2,
-	0x9f, 0x7a, 0xfa, 0x09, 0x27, 0x59, 0x32, 0x28, 0xae, 0x58, 0x51, 0x04, 0xfd, 0x82, 0xa5, 0x9c,
-	0x2e, 0x98, 0x4a, 0x05, 0x63, 0x2a, 0xdc, 0x36, 0x6f, 0xd4, 0x2d, 0xd8, 0xb1, 0xb6, 0x25, 0x8c,
-	0x29, 0x34, 0x86, 0x41, 0x4e, 0x5e, 0xe2, 0x72, 0xa6, 0x52, 0x4e, 0xf3, 0x74, 0xce, 0x72, 0x12,
-	0xee, 0x98, 0xd2, 0x6c, 0x3b, 0xfb, 0x31, 0xcd, 0x9f, 0xb1, 0x9c, 0x2c, 0x23, 0x29, 0xcf, 0x2c,
-	0x72, 0xb0, 0x82, 0x7c, 0xca, 0x33, 0x83, 0xfc, 0x08, 0xfa, 0x19, 0x2f, 0x25, 0x51, 0xbe, 0x36,
-	0xd7, 0x0d, 0xac, 0x67, 0x8d, 0xae, 0x2a, 0x1f, 0x00, 0xe0, 0xd9, 0x8c, 0x9d, 0xa7, 0x19, 0xe6,
-	0x32, 0x44, 0xa6, 0x71, 0x3a, 0xc6, 0x72, 0x80, 0xb9, 0x44, 0x11, 0xf4, 0x32, 0xcc, 0xf1, 0x29,
-	0x9d, 0x51, 0x45, 0x89, 0x0c, 0xdf, 0x31, 0x80, 0x15, 0x5b, 0xf4, 0x13, 0x6c, 0xfb, 0xe9, 0x91,
-	0x9c, 0x15, 0x92, 0xa0, 0x23, 0x68, 0xb9, 0xb6, 0x30, 0x23, 0xd4, 0xdd, 0xbf, 0x17, 0xd7, 0x9b,
-	0xe7, 0xd8, 0xb5, 0xcc, 0x89, 0xc2, 0x8a, 0x24, 0xde, 0x49, 0xd4, 0x87, 0xee, 0x0b, 0x4c, 0x95,
-	0x9b, 0xce, 0xe8, 0x47, 0xe8, 0xd9, 0xe3, 0xff, 0x14, 0xee, 0x10, 0x76, 0x4e, 0xa6, 0xa5, 0xca,
-	0xd9, 0x79, 0xe1, 0x05, 0xe1, 0x06, 0x34, 0x25, 0x9d, 0x14, 0x78, 0xe6, 0x34, 0xc1, 0x9d, 0xd0,
-	0x87, 0xd0, 0x9b, 0x08, 0x9c, 0x91, 0x94, 0x13, 0x41, 0x59, 0x1e, 0x6e, 0x8c, 0x82, 0x71, 0x23,
-	0xe9, 0x1a, 0xdb, 0xb1, 0x31, 0x45, 0x08, 0x06, 0x97, 0xde, 0x6c, 0xc6, 0xd1, 0x14, 0x6e, 0x7c,
-	0xc7, 0x73, 0x1d, 0xb4, 0xd2, 0x01, 0x17, 0x68, 0x45, 0x53, 0x82, 0xff, 0xac, 0x29, 0xd1, 0x4d,
-	0x78, 0xef, 0x95, 0x48, 0x2e, 0x89, 0x01, 0x6c, 0x7f, 0x4f, 0x84, 0xa4, 0xcc, 0xdf, 0x32, 0xfa,
-	0x14, 0x76, 0x2a, 0x8b, 0x7b, 0xdb, 0x10, 0x5a, 0x0b, 0x6b, 0x72, 0x37, 0xf7, 0xc7, 0xe8, 0x13,
-	0xe8, 0xe9, 0x77, 0xab, 0x32, 0x1f, 0x42, 0x9b, 0x16, 0x8a, 0x88, 0x85, 0x7b, 0xa4, 0x46, 0x52,
-	0x9d, 0xa3, 0x17, 0xd0, 0x77, 0x58, 0xe7, 0xf6, 0x1b, 0xd8, 0x92, 0xda, 0xb0, 0xe6, 0x15, 0x9f,
-	0x63, 0x79, 0x66, 0x1d, 0x59, 0x7a, 0x74, 0x1b, 0xfa, 0x27, 0xa6, 0x12, 0xaf, 0x2f, 0xd4, 0x96,
-	0x2f, 0x94, 0xbe, 0xac, 0x07, 0xba, 0xeb, 0x9f, 0x41, 0xf7, 0xc9, 0x05, 0xc9, 0x3c, 0xf1, 0x01,
-	0xb4, 0x73, 0x82, 0xf3, 0x19, 0x2d, 0x88, 0x4b, 0x6a, 0x18, 0xdb, 0xe5, 0x12, 0xfb, 0xe5, 0x12,
-	0x3f, 0xf7, 0xcb, 0x25, 0xa9, 0xb0, 0x7e, 0x55, 0x6c, 0xbc, 0xba, 0x2a, 0x1a, 0x97, 0xab, 0x22,
-	0x3a, 0x80, 0x9e, 0x0d, 0xe6, 0xee, 0x7f, 0x03, 0x9a, 0xac, 0x54, 0xbc, 0x54, 0x26, 0x56, 0x2f,
-	0x71, 0x27, 0xf4, 0x3e, 0x74, 0xc8, 0x05, 0x55, 0x69, 0xa6, 0xc7, 0x7a, 0xc3, 0xdc, 0xa0, 0xad,
-	0x0d, 0x07, 0x2c, 0x27, 0xd1, 0xef, 0x01, 0xf4, 0x96, 0x3b, 0x56, 0xc7, 0xe6, 0x34, 0x77, 0x37,
-	0xd5, 0x9f, 0x6f, 0xe4, 0x2f, 0xbd, 0x4d, 0x63, 0xf9, 0x6d, 0x50, 0x0c, 0x9b, 0x7a, 0x6d, 0x9a,
-	0x85, 0xf3, 0xe6, 0x6b, 0x1b, 0xdc, 0xfe, 0x9f, 0x1d, 0x68, 0x3f, 0x71, 0x83, 0x84, 0x7e, 0x81,
-	0xa6, 0x9d, 0x7e, 0x74, 0xbf, 0xee, 0xd4, 0xad, 0xec, 0xda, 0xe1, 0x83, 0x75, 0x69, 0xae, 0x7e,
-	0xd7, 0x90, 0x84, 0x4d, 0xad, 0x03, 0xe8, 0x6e, 0x5d, 0x0f, 0x4b, 0x22, 0x32, 0xbc, 0xb7, 0x1e,
-	0xa9, 0x0a, 0xfa, 0x1b, 0xb4, 0xfd, 0x38, 0xa3, 0x87, 0x75, 0x7d, 0x5c, 0x91, 0x93, 0xe1, 0xe7,
-	0xeb, 0x13, 0xab, 0x04, 0xfe, 0x08, 0x60, 0xe7, 0xca, 0x48, 0xa3, 0x2f, 0xeb, 0xfa, 0x7b, 0xbd,
-	0xea, 0x0c, 0x1f, 0xbd, 0x35, 0xbf, 0x4a, 0xeb, 0x57, 0x68, 0x39, 0xed, 0x40, 0xb5, 0x2b, 0xba,
-	0x2a, 0x3f, 0xc3, 0x87, 0x6b, 0xf3, 0xaa, 0xe8, 0x17, 0xb0, 0x65, 0x74, 0x01, 0xd5, 0x2e, 0xeb,
-	0xb2, 0x76, 0x0d, 0xef, 0xaf, 0xc9, 0xf2, 0x71, 0xf7, 0x02, 0xdd, 0xff, 0x56, 0x58, 0xea, 0xf7,
-	0xff, 0x8a, 0x62, 0xd5, 0xef, 0xff, 0x2b, 0xfa, 0x65, 0xfa, 0x5f, 0x8f, 0x61, 0xfd, 0xfe, 0x5f,
-	0xd2, 0xbb, 0xfa, 0xfd, 0xbf, 0xac, 0x5b, 0xd1, 0x35, 0xf4, 0x57, 0x00, 0x7d, 0x6d, 0x3a, 0x51,
-	0x82, 0xe0, 0x39, 0x2d, 0x26, 0xe8, 0x51, 0x4d, 0xf1, 0xd6, 0x2c, 0x2b, 0xe0, 0x8e, 0xe9, 0x53,
-	0xf9, 0xea, 0xed, 0x1d, 0xf8, 0xb4, 0xc6, 0xc1, 0x5e, 0xf0, 0x75, 0xeb, 0x87, 0x2d, 0xab, 0x59,
-	0x4d, 0xf3, 0x73, 0xf7, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2e, 0x63, 0xc4, 0xd3, 0x74, 0x0c,
-	0x00, 0x00,
+	// 1079 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x55, 0x7d, 0x6f, 0x1b, 0xc5,
+	0x13, 0xee, 0xc5, 0x89, 0x5f, 0xc6, 0x76, 0xe2, 0xee, 0xef, 0xa7, 0x70, 0x35, 0x42, 0x35, 0x87,
+	0x44, 0x2d, 0x28, 0x97, 0x28, 0x7d, 0x43, 0x42, 0xa2, 0x88, 0xa4, 0xa0, 0x8a, 0x34, 0x8a, 0x2e,
+	0x85, 0x4a, 0xfc, 0xc1, 0xb1, 0xb9, 0xdb, 0xda, 0x2b, 0x9f, 0x6f, 0x8f, 0xdd, 0x3d, 0x27, 0x48,
+	0x48, 0x7c, 0x09, 0x90, 0xf8, 0x00, 0x7c, 0x0c, 0x3e, 0x1c, 0xda, 0xb7, 0x8b, 0x9d, 0x96, 0xea,
+	0x5c, 0xc4, 0x5f, 0xbe, 0x1d, 0x3f, 0xcf, 0xcc, 0xec, 0xce, 0xcc, 0x33, 0x70, 0x37, 0xe5, 0x74,
+	0x41, 0xb8, 0xd8, 0x13, 0x53, 0xcc, 0x49, 0xba, 0x47, 0x2e, 0x49, 0x52, 0x4a, 0xc6, 0xf7, 0x0a,
+	0xce, 0x24, 0xab, 0x8e, 0xa1, 0x3e, 0xa2, 0x0f, 0xa7, 0x58, 0x4c, 0x69, 0xc2, 0x78, 0x11, 0xe6,
+	0x6c, 0x8e, 0xd3, 0xb0, 0xc8, 0xca, 0x09, 0xcd, 0x45, 0xb8, 0x8a, 0x1b, 0xde, 0x9e, 0x30, 0x36,
+	0xc9, 0x88, 0x71, 0x72, 0x5e, 0xbe, 0xdc, 0x93, 0x74, 0x4e, 0x84, 0xc4, 0xf3, 0xc2, 0x02, 0x02,
+	0x4b, 0xdc, 0x73, 0xe1, 0x4d, 0x38, 0x73, 0x32, 0x98, 0xe0, 0xaf, 0x26, 0xf4, 0x8f, 0x71, 0x99,
+	0x27, 0xd3, 0x88, 0xfc, 0x54, 0x12, 0x21, 0xd1, 0x00, 0x1a, 0xc9, 0x3c, 0xf5, 0xbd, 0x91, 0x37,
+	0xee, 0x44, 0xea, 0x13, 0x21, 0xd8, 0xc4, 0x7c, 0x22, 0xfc, 0x8d, 0x51, 0x63, 0xdc, 0x89, 0xf4,
+	0x37, 0x3a, 0x81, 0x0e, 0x27, 0x82, 0x95, 0x3c, 0x21, 0xc2, 0x6f, 0x8c, 0xbc, 0x71, 0xf7, 0x60,
+	0x3f, 0xfc, 0xa7, 0xc4, 0x6d, 0x7c, 0x13, 0x32, 0x8c, 0x1c, 0x2f, 0xba, 0x72, 0x81, 0x6e, 0x43,
+	0x57, 0xc8, 0x94, 0x95, 0x32, 0x2e, 0xb0, 0x9c, 0xfa, 0x9b, 0x3a, 0x3a, 0x18, 0xd3, 0x29, 0x96,
+	0x53, 0x0b, 0x20, 0x9c, 0x1b, 0xc0, 0x56, 0x05, 0x20, 0x9c, 0x6b, 0xc0, 0x00, 0x1a, 0x24, 0x5f,
+	0xf8, 0x4d, 0x9d, 0xa4, 0xfa, 0x54, 0x79, 0x97, 0x82, 0x70, 0xbf, 0xa5, 0xb1, 0xfa, 0x1b, 0xdd,
+	0x82, 0xb6, 0xc4, 0x62, 0x16, 0xa7, 0x94, 0xfb, 0x6d, 0x6d, 0x6f, 0xa9, 0xf3, 0x11, 0xe5, 0xe8,
+	0x0e, 0xec, 0xb8, 0x7c, 0xe2, 0x8c, 0xce, 0xa9, 0x14, 0x7e, 0x67, 0xe4, 0x8d, 0xdb, 0xd1, 0xb6,
+	0x33, 0x1f, 0x6b, 0x2b, 0xda, 0x87, 0xff, 0x9f, 0x63, 0x41, 0x93, 0xb8, 0xe0, 0x2c, 0x21, 0x42,
+	0xc4, 0xc9, 0x84, 0xb3, 0xb2, 0xf0, 0x41, 0xa3, 0x91, 0xfe, 0xef, 0xd4, 0xfc, 0x75, 0xa8, 0xff,
+	0x41, 0x47, 0xd0, 0x9c, 0xb3, 0x32, 0x97, 0xc2, 0xef, 0x8e, 0x1a, 0xe3, 0xee, 0xc1, 0xdd, 0x9a,
+	0x4f, 0xf5, 0x4c, 0x91, 0x22, 0xcb, 0x45, 0x5f, 0x43, 0x2b, 0x25, 0x0b, 0xaa, 0x5e, 0xbc, 0xa7,
+	0xdd, 0x7c, 0x52, 0xd3, 0xcd, 0x91, 0x66, 0x45, 0x8e, 0x8d, 0xa6, 0x70, 0x33, 0x27, 0xf2, 0x82,
+	0xf1, 0x59, 0x4c, 0x05, 0xcb, 0xb0, 0xa4, 0x2c, 0xf7, 0xfb, 0xba, 0x88, 0x9f, 0xd5, 0x74, 0x79,
+	0x62, 0xf8, 0x4f, 0x1d, 0xfd, 0xac, 0x20, 0x49, 0x34, 0xc8, 0xaf, 0x59, 0x51, 0x00, 0xfd, 0x9c,
+	0xc5, 0x05, 0x5d, 0x30, 0x19, 0x73, 0xc6, 0xa4, 0xbf, 0xad, 0xdf, 0xa8, 0x9b, 0xb3, 0x53, 0x65,
+	0x8b, 0x18, 0x93, 0x68, 0x0c, 0x83, 0x94, 0xbc, 0xc4, 0x65, 0x26, 0xe3, 0x82, 0xa6, 0xf1, 0x9c,
+	0xa5, 0xc4, 0xdf, 0xd1, 0xa5, 0xd9, 0xb6, 0xf6, 0x53, 0x9a, 0x3e, 0x63, 0x29, 0x59, 0x46, 0xd2,
+	0x22, 0x31, 0xc8, 0xc1, 0x0a, 0xf2, 0x69, 0x91, 0x68, 0xe4, 0x07, 0xd0, 0x4f, 0x8a, 0x52, 0x10,
+	0xe9, 0x6a, 0x73, 0x53, 0xc3, 0x7a, 0xc6, 0x68, 0xab, 0xf2, 0x1e, 0x00, 0xce, 0x32, 0x76, 0x11,
+	0x27, 0xb8, 0x10, 0x3e, 0xd2, 0x8d, 0xd3, 0xd1, 0x96, 0x43, 0x5c, 0x08, 0x14, 0x40, 0x2f, 0xc1,
+	0x05, 0x3e, 0xa7, 0x19, 0x95, 0x94, 0x08, 0xff, 0x7f, 0x1a, 0xb0, 0x62, 0x0b, 0x7e, 0x84, 0x6d,
+	0x37, 0x3d, 0xa2, 0x60, 0xb9, 0x20, 0xe8, 0x04, 0x5a, 0xb6, 0x2d, 0xf4, 0x08, 0x75, 0x0f, 0xee,
+	0x87, 0xf5, 0xe6, 0x39, 0xb4, 0x2d, 0x73, 0x26, 0xb1, 0x24, 0x91, 0x73, 0x12, 0xf4, 0xa1, 0xfb,
+	0x02, 0x53, 0x69, 0xa7, 0x33, 0xf8, 0x01, 0x7a, 0xe6, 0xf8, 0x1f, 0x85, 0x3b, 0x86, 0x9d, 0xb3,
+	0x69, 0x29, 0x53, 0x76, 0x91, 0x3b, 0x41, 0xd8, 0x85, 0xa6, 0xa0, 0x93, 0x1c, 0x67, 0x56, 0x13,
+	0xec, 0x09, 0xbd, 0x0f, 0xbd, 0x09, 0xc7, 0x09, 0x89, 0x0b, 0xc2, 0x29, 0x4b, 0xfd, 0x8d, 0x91,
+	0x37, 0x6e, 0x44, 0x5d, 0x6d, 0x3b, 0xd5, 0xa6, 0x00, 0xc1, 0xe0, 0xca, 0x9b, 0xc9, 0x38, 0x98,
+	0xc2, 0xee, 0xb7, 0x45, 0xaa, 0x82, 0x56, 0x3a, 0x60, 0x03, 0xad, 0x68, 0x8a, 0xf7, 0xaf, 0x35,
+	0x25, 0xb8, 0x05, 0xef, 0xbc, 0x12, 0xc9, 0x26, 0x31, 0x80, 0xed, 0xef, 0x08, 0x17, 0x94, 0xb9,
+	0x5b, 0x06, 0x1f, 0xc3, 0x4e, 0x65, 0xb1, 0x6f, 0xeb, 0x43, 0x6b, 0x61, 0x4c, 0xf6, 0xe6, 0xee,
+	0x18, 0x7c, 0x04, 0x3d, 0xf5, 0x6e, 0x55, 0xe6, 0x43, 0x68, 0xd3, 0x5c, 0x12, 0xbe, 0xb0, 0x8f,
+	0xd4, 0x88, 0xaa, 0x73, 0xf0, 0x02, 0xfa, 0x16, 0x6b, 0xdd, 0x7e, 0x05, 0x5b, 0x42, 0x19, 0xd6,
+	0xbc, 0xe2, 0x73, 0x2c, 0x66, 0xc6, 0x91, 0xa1, 0x07, 0x77, 0xa0, 0x7f, 0xa6, 0x2b, 0xf1, 0xfa,
+	0x42, 0x6d, 0xb9, 0x42, 0xa9, 0xcb, 0x3a, 0xa0, 0xbd, 0xfe, 0x0c, 0xba, 0x4f, 0x2e, 0x49, 0xe2,
+	0x88, 0x0f, 0xa1, 0x9d, 0x12, 0x9c, 0x66, 0x34, 0x27, 0x36, 0xa9, 0x61, 0x68, 0x96, 0x4b, 0xe8,
+	0x96, 0x4b, 0xf8, 0xdc, 0x2d, 0x97, 0xa8, 0xc2, 0xba, 0x55, 0xb1, 0xf1, 0xea, 0xaa, 0x68, 0x5c,
+	0xad, 0x8a, 0xe0, 0x10, 0x7a, 0x26, 0x98, 0xbd, 0xff, 0x2e, 0x34, 0x59, 0x29, 0x8b, 0x52, 0xea,
+	0x58, 0xbd, 0xc8, 0x9e, 0xd0, 0xbb, 0xd0, 0x21, 0x97, 0x54, 0xc6, 0x89, 0x1a, 0xeb, 0x0d, 0x7d,
+	0x83, 0xb6, 0x32, 0x1c, 0xb2, 0x94, 0x04, 0x7f, 0x7a, 0xd0, 0x5b, 0xee, 0x58, 0x15, 0xbb, 0xa0,
+	0xa9, 0xbd, 0xa9, 0xfa, 0x7c, 0x23, 0x7f, 0xe9, 0x6d, 0x1a, 0xcb, 0x6f, 0x83, 0x42, 0xd8, 0x54,
+	0x6b, 0x53, 0x2f, 0x9c, 0x37, 0x5f, 0x5b, 0xe3, 0x94, 0x66, 0x30, 0x36, 0x8f, 0x67, 0x34, 0xcb,
+	0x48, 0xaa, 0xb7, 0x50, 0x3b, 0xea, 0x30, 0x36, 0xff, 0x46, 0x1b, 0x0e, 0x7e, 0xef, 0x40, 0xfb,
+	0x89, 0x9d, 0x33, 0xf4, 0x33, 0x34, 0x8d, 0x38, 0xa0, 0x07, 0x75, 0x87, 0x72, 0x65, 0x15, 0x0f,
+	0x1f, 0xae, 0x4b, 0xb3, 0xe5, 0xbd, 0x81, 0x04, 0x6c, 0x2a, 0x99, 0x40, 0xf7, 0xea, 0x7a, 0x58,
+	0xd2, 0x98, 0xe1, 0xfd, 0xf5, 0x48, 0x55, 0xd0, 0x5f, 0xa1, 0xed, 0xa6, 0x1d, 0x3d, 0xaa, 0xeb,
+	0xe3, 0x9a, 0xda, 0x0c, 0x3f, 0x5d, 0x9f, 0x58, 0x25, 0xf0, 0x9b, 0x07, 0x3b, 0xd7, 0x26, 0x1e,
+	0x7d, 0x5e, 0xd7, 0xdf, 0xeb, 0x45, 0x69, 0xf8, 0xf8, 0xad, 0xf9, 0x55, 0x5a, 0xbf, 0x40, 0xcb,
+	0x4a, 0x0b, 0xaa, 0x5d, 0xd1, 0x55, 0x75, 0x1a, 0x3e, 0x5a, 0x9b, 0x57, 0x45, 0xbf, 0x84, 0x2d,
+	0x2d, 0x1b, 0xa8, 0x76, 0x59, 0x97, 0xa5, 0x6d, 0xf8, 0x60, 0x4d, 0x96, 0x8b, 0xbb, 0xef, 0xa9,
+	0xfe, 0x37, 0xba, 0x53, 0xbf, 0xff, 0x57, 0x04, 0xad, 0x7e, 0xff, 0x5f, 0x93, 0x37, 0xdd, 0xff,
+	0x6a, 0x0c, 0xeb, 0xf7, 0xff, 0x92, 0x1c, 0xd6, 0xef, 0xff, 0x65, 0x59, 0x0b, 0x6e, 0xa0, 0x3f,
+	0x3c, 0xe8, 0x2b, 0xd3, 0x99, 0xe4, 0x04, 0xcf, 0x69, 0x3e, 0x41, 0x8f, 0x6b, 0x6a, 0xbb, 0x62,
+	0x19, 0x7d, 0xb7, 0x4c, 0x97, 0xca, 0x17, 0x6f, 0xef, 0xc0, 0xa5, 0x35, 0xf6, 0xf6, 0xbd, 0x2f,
+	0x5b, 0xdf, 0x6f, 0x19, 0x49, 0x6b, 0xea, 0x9f, 0x7b, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xd5,
+	0xc9, 0x00, 0x0a, 0x93, 0x0c, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.

drivers/shared/executor/proto/executor.proto

@@ -111,4 +111,5 @@ message ProcessState {
     int32 exit_code = 2;
     int32 signal = 3;
     google.protobuf.Timestamp time = 4;
+    bool oom_killed = 5;
 }

drivers/shared/executor/utils.go

@@ -111,10 +111,11 @@ func processStateToProto(ps *ProcessState) (*proto.ProcessState, error) {
 		return nil, err
 	}
 	pb := &proto.ProcessState{
-		Pid:      int32(ps.Pid),
-		ExitCode: int32(ps.ExitCode),
-		Signal:   int32(ps.Signal),
-		Time:     timestamp,
+		Pid:       int32(ps.Pid),
+		ExitCode:  int32(ps.ExitCode),
+		Signal:    int32(ps.Signal),
+		OomKilled: ps.OOMKilled,
+		Time:      timestamp,
 	}
 
 	return pb, nil
@@ -127,10 +128,11 @@ func processStateFromProto(pb *proto.ProcessState) (*ProcessState, error) {
 	}
 
 	return &ProcessState{
-		Pid:      int(pb.Pid),
-		ExitCode: int(pb.ExitCode),
-		Signal:   int(pb.Signal),
-		Time:     timestamp,
+		Pid:       int(pb.Pid),
+		ExitCode:  int(pb.ExitCode),
+		Signal:    int(pb.Signal),
+		OOMKilled: pb.OomKilled,
+		Time:      timestamp,
 	}, nil
 }