kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-01 16:05:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

e2e: fixes for race conditions in testing (#6300)

Browse Source 
- In script checks, ensure we're running `Exec` against the new running
  allocation and not the earlier stopped one.
- In script checks, allow `Exec` calls to error due to lack of pty when
  we use the exec to kill the task.
- In `utils.go/RegisterAllocs`, force query for allocations to wait on
  wait index returned by registration call.
This commit is contained in:
Tim Gross
2019-09-10 13:45:16 -04:00
committed by GitHub
parent ae3488dbac
commit 31f91cf4c8
2 changed files with 29 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
e2e/consul/script_checks.go
View File

@@ -3,6 +3,7 @@ package consul
import (
"bytes"
"context"
"fmt"
"os"
"strings"
"time"
@@ -108,7 +109,7 @@ func (tc *ScriptChecksE2ETest) TestGroupScriptCheck(f *framework.F) {
requireDeregistered(require, consulClient, "group-service-2")
requireDeregistered(require, consulClient, "group-service-3")
// // Restore for next test
// Restore for next test
allocs = e2eutil.RegisterAndWaitForAllocs(f.T(),
nomadClient, "consul/input/checks_group.nomad", jobId)
require.Equal(2, len(allocs))
@@ -118,7 +119,9 @@ func (tc *ScriptChecksE2ETest) TestGroupScriptCheck(f *framework.F) {
// Crash a task: verify that checks become healthy again
_, _, err = exec(nomadClient, allocs, []string{"pkill", "sleep"})
require.NoError(err)
if err != nil && err.Error() != "plugin is shut down" {
require.FailNow("unexpected error: %v", err)
}
requireStatus(require, consulClient, "group-service-1", capi.HealthPassing)
requireStatus(require, consulClient, "group-service-2", capi.HealthWarning)
requireStatus(require, consulClient, "group-service-3", capi.HealthCritical)
@@ -173,7 +176,7 @@ func (tc *ScriptChecksE2ETest) TestTaskScriptCheck(f *framework.F) {
requireDeregistered(require, consulClient, "task-service-2")
requireDeregistered(require, consulClient, "task-service-3")
// // Restore for next test
// Restore for next test
allocs = e2eutil.RegisterAndWaitForAllocs(f.T(),
nomadClient, "consul/input/checks_task.nomad", jobId)
require.Equal(2, len(allocs))
@@ -183,7 +186,9 @@ func (tc *ScriptChecksE2ETest) TestTaskScriptCheck(f *framework.F) {
// Crash a task: verify that checks become healthy again
_, _, err = exec(nomadClient, allocs, []string{"pkill", "sleep"})
require.NoError(err)
if err != nil && err.Error() != "plugin is shut down" {
require.FailNow("unexpected error: %v", err)
}
requireStatus(require, consulClient, "task-service-1", capi.HealthPassing)
requireStatus(require, consulClient, "task-service-2", capi.HealthWarning)
requireStatus(require, consulClient, "task-service-3", capi.HealthCritical)
@@ -207,17 +212,27 @@ func exec(client *api.Client, allocs []*api.AllocationListStub, command []string
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()
// we're getting a list of from the registration call here but
// one of them might be stopped or stopping, which will return
// an error if we try to exec into it.
var alloc *api.Allocation
for _, stub := range allocs {
if stub.DesiredStatus == "run" {
alloc = &api.Allocation{
ID: stub.ID,
Namespace: stub.Namespace,
NodeID: stub.NodeID,
}
}
}
var stdout, stderr bytes.Buffer
alloc := &api.Allocation{
ID: allocs[0].ID,
Namespace: allocs[0].Namespace,
NodeID: allocs[0].NodeID,
if alloc == nil {
return stdout, stderr, fmt.Errorf("no allocation ready for exec")
}
_, err := client.Allocations().Exec(ctx,
alloc, "test", false,
command,
os.Stdin, &stdout, &stderr, // os.Stdout, os.Stderr,
os.Stdin, &stdout, &stderr,
make(chan api.TerminalSize), nil)
return stdout, stderr, err
}

6
e2e/e2eutil/utils.go
View File

@@ -62,18 +62,20 @@ func RegisterAllocs(t *testing.T, nomadClient *api.Client, jobFile string, jobID
job.ID = helper.StringToPtr(jobID)
// Register job
var idx uint64
jobs := nomadClient.Jobs()
testutil.WaitForResult(func() (bool, error) {
resp, _, err := jobs.Register(job, nil)
resp, meta, err := jobs.Register(job, nil)
if err != nil {
return false, err
}
idx = meta.LastIndex
return resp.EvalID != "", fmt.Errorf("expected EvalID:%s", pretty.Sprint(resp))
}, func(err error) {
require.NoError(err)
})
allocs, _, _ := jobs.Allocations(jobID, false, nil)
allocs, _, _ := jobs.Allocations(jobID, false, &api.QueryOptions{WaitIndex: idx})
return allocs
}