mirror of
https://github.com/kemko/nomad.git
synced 2026-01-07 19:05:42 +03:00
e168548341
Some of our allocrunner hooks require a task environment for interpolating values based on the node or allocation. But several of the hooks accept an already-built environment or builder and then keep that in memory. Both of these retain a copy of all the node attributes and allocation metadata, which balloons memory usage until the allocation is GC'd. While we'd like to look into ways to avoid keeping the allocrunner around entirely (see #25372), for now we can significantly reduce memory usage by creating the task environment on-demand when calling allocrunner methods, rather than persisting it in the allocrunner hooks. In doing so, we uncover two other bugs: * The WID manager, the group service hook, and the checks hook have to interpolate services for specific tasks. They mutated a taskenv builder to do so, but each time they mutate the builder, they write to the same environment map. When a group has multiple tasks, it's possible for one task to set an environment variable that would then be interpolated in the service definition for another task if that task did not have that environment variable. Only the service definition interpolation is impacted. This does not leak env vars across running tasks, as each taskrunner has its own builder. To fix this, we move the `UpdateTask` method off the builder and onto the taskenv as the `WithTask` method. This makes a shallow copy of the taskenv with a deep clone of the environment map used for interpolation, and then overwrites the environment from the task. * The checks hook interpolates Nomad native service checks only on `Prerun` and not on `Update`. This could cause unexpected deregistration and registration of checks during in-place updates. To fix this, we make sure we interpolate in the `Update` method. I also bumped into an incorrectly implemented interface in the CSI hook. I've pulled that and some better guardrails out to https://github.com/hashicorp/nomad/pull/25472. Fixes: https://github.com/hashicorp/nomad/issues/25269 Fixes: https://hashicorp.atlassian.net/browse/NET-12310 Ref: https://github.com/hashicorp/nomad/issues/25372
131 lines
3.4 KiB
Go
131 lines
3.4 KiB
Go
// Copyright (c) HashiCorp, Inc.
|
|
// SPDX-License-Identifier: BUSL-1.1
|
|
|
|
package allocrunner
|
|
|
|
import (
|
|
"bytes"
|
|
"net"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/hashicorp/nomad/ci"
|
|
"github.com/hashicorp/nomad/client/allocdir"
|
|
"github.com/hashicorp/nomad/helper/testlog"
|
|
"github.com/hashicorp/nomad/nomad/mock"
|
|
"github.com/hashicorp/nomad/nomad/structs"
|
|
"github.com/hashicorp/nomad/nomad/structs/config"
|
|
"github.com/shoenig/test/must"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestConsulSocketHook_PrerunPostrun_Ok(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
fakeConsul, err := net.Listen("tcp", "127.0.0.1:0")
|
|
require.NoError(t, err)
|
|
defer fakeConsul.Close()
|
|
|
|
consulConfigs := map[string]*config.ConsulConfig{
|
|
structs.ConsulDefaultCluster: {Addr: fakeConsul.Addr().String()},
|
|
}
|
|
|
|
alloc := mock.ConnectNativeAlloc("bridge")
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "ConnectNativeTask", alloc.ID)
|
|
defer cleanupDir()
|
|
|
|
// start unix socket proxy
|
|
h := newConsulHTTPSocketHook(logger, alloc, allocDir, consulConfigs)
|
|
require.NoError(t, h.Prerun(nil))
|
|
|
|
httpSocket := filepath.Join(allocDir.AllocDir, allocdir.AllocHTTPSocket)
|
|
taskCon, err := net.Dial("unix", httpSocket)
|
|
require.NoError(t, err)
|
|
|
|
// write to consul from task to ensure data is proxied out of the netns
|
|
input := bytes.Repeat([]byte{'X'}, 5*1024)
|
|
errCh := make(chan error, 1)
|
|
go func() {
|
|
_, err := taskCon.Write(input)
|
|
errCh <- err
|
|
}()
|
|
|
|
// accept the connection from inside the netns
|
|
consulConn, err := fakeConsul.Accept()
|
|
require.NoError(t, err)
|
|
defer consulConn.Close()
|
|
|
|
output := make([]byte, len(input))
|
|
_, err = consulConn.Read(output)
|
|
require.NoError(t, err)
|
|
require.NoError(t, <-errCh)
|
|
require.Equal(t, input, output)
|
|
|
|
// read from consul to ensure http response bodies can come back
|
|
input = bytes.Repeat([]byte{'Y'}, 5*1024)
|
|
go func() {
|
|
_, err := consulConn.Write(input)
|
|
errCh <- err
|
|
}()
|
|
|
|
output = make([]byte, len(input))
|
|
_, err = taskCon.Read(output)
|
|
require.NoError(t, err)
|
|
require.NoError(t, <-errCh)
|
|
require.Equal(t, input, output)
|
|
|
|
// stop the unix socket proxy
|
|
require.NoError(t, h.Postrun())
|
|
|
|
// consul reads should now error
|
|
n, err := consulConn.Read(output)
|
|
require.Error(t, err)
|
|
require.Zero(t, n)
|
|
|
|
// task reads and writes should error
|
|
n, err = taskCon.Write(input)
|
|
require.Error(t, err)
|
|
require.Zero(t, n)
|
|
n, err = taskCon.Read(output)
|
|
require.Error(t, err)
|
|
require.Zero(t, n)
|
|
}
|
|
|
|
func TestConsulHTTPSocketHook_Prerun_Error(t *testing.T) {
|
|
ci.Parallel(t)
|
|
|
|
logger := testlog.HCLogger(t)
|
|
|
|
consulConfigs := map[string]*config.ConsulConfig{
|
|
structs.ConsulDefaultCluster: new(config.ConsulConfig),
|
|
}
|
|
|
|
alloc := mock.Alloc()
|
|
connectNativeAlloc := mock.ConnectNativeAlloc("bridge")
|
|
|
|
allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "ConnectNativeTask", alloc.ID)
|
|
defer cleanupDir()
|
|
|
|
{
|
|
// an alloc without a connect native task should not return an error
|
|
h := newConsulHTTPSocketHook(logger, alloc, allocDir, consulConfigs)
|
|
must.NoError(t, h.Prerun(nil))
|
|
|
|
// postrun should be a noop
|
|
must.NoError(t, h.Postrun())
|
|
}
|
|
|
|
{
|
|
// an alloc with a native task should return an error when consul is not
|
|
// configured
|
|
h := newConsulHTTPSocketHook(logger, connectNativeAlloc, allocDir, consulConfigs)
|
|
must.ErrorContains(t, h.Prerun(nil), "consul address must be set on nomad client")
|
|
|
|
// Postrun should be a noop
|
|
must.NoError(t, h.Postrun())
|
|
}
|
|
}
|