From 145827d8b7338482b39c95b551347eface0113e6 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 20 Dec 2018 00:53:44 -0500 Subject: [PATCH 1/5] fix tests that fail as a result of async client startup --- api/fs_test.go | 3 +++ client/client.go | 10 ++++++++++ client/node_updater.go | 5 ++++- command/job_status_test.go | 15 +++++++++++++++ nomad/client_stats_endpoint_test.go | 8 ++++++++ 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/api/fs_test.go b/api/fs_test.go index 790d2ca7c..3b38f66c9 100644 --- a/api/fs_test.go +++ b/api/fs_test.go @@ -46,6 +46,9 @@ func TestFS_Logs(t *testing.T) { if nodes[0].Status != "ready" { return false, fmt.Errorf("node not ready: %s", nodes[0].Status) } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } return true, nil }, func(err error) { t.Fatalf("err: %v", err) diff --git a/client/client.go b/client/client.go index c8a347632..7abadd425 100644 --- a/client/client.go +++ b/client/client.go @@ -237,6 +237,10 @@ type Client struct { // batchNodeUpdates is used to batch initial updates to the node batchNodeUpdates *batchNodeUpdates + + // fpInitialized chan is closed when the first batch of fingerprints are + // applied to the node and the server is updated + fpInitialized chan struct{} } var ( @@ -281,6 +285,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic triggerDiscoveryCh: make(chan struct{}), triggerNodeUpdate: make(chan struct{}, 8), triggerEmitNodeEvent: make(chan *structs.NodeEvent, 8), + fpInitialized: make(chan struct{}), } c.batchNodeUpdates = newBatchNodeUpdates( @@ -436,6 +441,11 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulServic return c, nil } +// Ready returns a chan that is closed when the client is fully initialized +func (c *Client) Ready() <-chan struct{} { + return c.fpInitialized +} + // init is used to initialize the client and perform any setup // needed before we begin starting its various components. func (c *Client) init() error { diff --git a/client/node_updater.go b/client/node_updater.go index 24e5f76bc..4367eeb4f 100644 --- a/client/node_updater.go +++ b/client/node_updater.go @@ -18,7 +18,8 @@ var ( ) // batchFirstFingerprints waits for the first fingerprint response from all -// plugin managers and sends a single Node update for all fingerprints +// plugin managers and sends a single Node update for all fingerprints. It +// should only ever be called once func (c *Client) batchFirstFingerprints() { ctx, cancel := context.WithTimeout(context.Background(), batchFirstFingerprintsTimeout) defer cancel() @@ -63,6 +64,8 @@ SEND_BATCH: if driverChanged || devicesChanged { c.updateNodeLocked() } + + close(c.fpInitialized) } // updateNodeFromDriver receives a DriverInfo struct for the driver and updates diff --git a/command/job_status_test.go b/command/job_status_test.go index 84aa86c54..bac0b8cf1 100644 --- a/command/job_status_test.go +++ b/command/job_status_test.go @@ -26,6 +26,21 @@ func TestJobStatusCommand_Run(t *testing.T) { t.Parallel() srv, client, url := testServer(t, true, nil) defer srv.Shutdown() + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + t.Fatalf("err: %s", err) + }) ui := new(cli.MockUi) cmd := &JobStatusCommand{Meta: Meta{Ui: ui}} diff --git a/nomad/client_stats_endpoint_test.go b/nomad/client_stats_endpoint_test.go index 78f667bd5..b1220cbf0 100644 --- a/nomad/client_stats_endpoint_test.go +++ b/nomad/client_stats_endpoint_test.go @@ -2,6 +2,7 @@ package nomad import ( "testing" + "time" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" @@ -188,6 +189,13 @@ func TestClientStats_Stats_Remote(t *testing.T) { }) defer cleanup() + // Wait for client initialization + select { + case <-c.Ready(): + case <-time.After(10 * time.Second): + require.Fail("client timedout on initialize") + } + testutil.WaitForResult(func() (bool, error) { nodes := s2.connectedNodes() return len(nodes) == 1, nil From 431d7af2812a80534a5b76d10e6dee29cd33fe28 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 20 Dec 2018 01:01:53 -0500 Subject: [PATCH 2/5] fix test --- command/job_status_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/command/job_status_test.go b/command/job_status_test.go index bac0b8cf1..e0bc40b56 100644 --- a/command/job_status_test.go +++ b/command/job_status_test.go @@ -262,6 +262,24 @@ func TestJobStatusCommand_WithAccessPolicy(t *testing.T) { token := srv.RootToken assert.NotNil(token, "failed to bootstrap ACL token") + // Wait for client ready + client.SetSecretID(token.SecretID) + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := client.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + t.Fatalf("err: %s", err) + }) + // Register a job j := testJob("job1_sfx") From 457ce6ca1ba0f659dc96e348321ec062f46c326f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 20 Dec 2018 01:05:17 -0500 Subject: [PATCH 3/5] fix tests --- command/alloc_status_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/command/alloc_status_test.go b/command/alloc_status_test.go index e8d8f996d..fbd199424 100644 --- a/command/alloc_status_test.go +++ b/command/alloc_status_test.go @@ -96,7 +96,8 @@ func TestAllocStatusCommand_Run(t *testing.T) { return false, err } for _, node := range nodes { - if node.Status == structs.NodeStatusReady { + if _, ok := node.Drivers["mock_driver"]; ok && + node.Status == structs.NodeStatusReady { return true, nil } } From 617d102ce4252582a194e1f0b7417f186b70910f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 20 Dec 2018 13:52:33 -0500 Subject: [PATCH 4/5] command: wait for drivers to be ready before test --- command/node_drain_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/command/node_drain_test.go b/command/node_drain_test.go index f75d0e503..11b06879e 100644 --- a/command/node_drain_test.go +++ b/command/node_drain_test.go @@ -113,6 +113,9 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { if len(nodes) == 0 { return false, fmt.Errorf("missing node") } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } nodeID = nodes[0].ID return true, nil }, func(err error) { @@ -186,6 +189,8 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { _, _, err = client.Jobs().Register(sysjob, nil) require.Nil(err) + evals, _, _ := client.Evaluations().List(nil) + var allocs []*api.Allocation testutil.WaitForResult(func() (bool, error) { allocs, _, err = client.Nodes().Allocations(nodeID, nil) From 0b235d89d9c4dd1830fe80ceb9f335d51c5c8343 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 20 Dec 2018 13:54:29 -0500 Subject: [PATCH 5/5] fix test --- command/node_drain_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/command/node_drain_test.go b/command/node_drain_test.go index 11b06879e..9156096e0 100644 --- a/command/node_drain_test.go +++ b/command/node_drain_test.go @@ -189,8 +189,6 @@ func TestNodeDrainCommand_Monitor(t *testing.T) { _, _, err = client.Jobs().Register(sysjob, nil) require.Nil(err) - evals, _, _ := client.Evaluations().List(nil) - var allocs []*api.Allocation testutil.WaitForResult(func() (bool, error) { allocs, _, err = client.Nodes().Allocations(nodeID, nil)