Merge pull request #2874 from hashicorp/f-command-agent-tests

Parallelize the command/agent tests and add new test agent

command/agent/agent_endpoint_test.go

@@ -13,7 +13,8 @@ import (
 )
 
 func TestHTTP_AgentSelf(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/agent/self", nil)
 		if err != nil {
@@ -39,7 +40,9 @@ func TestHTTP_AgentSelf(t *testing.T) {
 }
 
 func TestHTTP_AgentJoin(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	// TODO(alexdadgar)
+	// t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Determine the join address
 		member := s.Agent.Server().LocalMember()
 		addr := fmt.Sprintf("%s:%d", member.Addr, member.Port)
@@ -70,7 +73,8 @@ func TestHTTP_AgentJoin(t *testing.T) {
 }
 
 func TestHTTP_AgentMembers(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/agent/members", nil)
 		if err != nil {
@@ -93,7 +97,8 @@ func TestHTTP_AgentMembers(t *testing.T) {
 }
 
 func TestHTTP_AgentForceLeave(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("PUT", "/v1/agent/force-leave?node=foo", nil)
 		if err != nil {
@@ -110,7 +115,8 @@ func TestHTTP_AgentForceLeave(t *testing.T) {
 }
 
 func TestHTTP_AgentSetServers(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Establish a baseline number of servers
 		req, err := http.NewRequest("GET", "/v1/agent/servers", nil)
 		if err != nil {
@@ -183,11 +189,13 @@ func TestHTTP_AgentSetServers(t *testing.T) {
 }
 
 func TestHTTP_AgentListKeys(t *testing.T) {
+	t.Parallel()
+
 	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
 
 	httpTest(t, func(c *Config) {
 		c.Server.EncryptKey = key1
-	}, func(s *TestServer) {
+	}, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
 		if err != nil {
 			t.Fatalf("err: %s", err)
@@ -206,12 +214,15 @@ func TestHTTP_AgentListKeys(t *testing.T) {
 }
 
 func TestHTTP_AgentInstallKey(t *testing.T) {
+	// TODO(alexdadgar)
+	// t.Parallel()
+
 	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
 	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
 
 	httpTest(t, func(c *Config) {
 		c.Server.EncryptKey = key1
-	}, func(s *TestServer) {
+	}, func(s *TestAgent) {
 		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
 		if err != nil {
 			t.Fatalf("err: %v", err)
@@ -244,12 +255,15 @@ func TestHTTP_AgentInstallKey(t *testing.T) {
 }
 
 func TestHTTP_AgentRemoveKey(t *testing.T) {
+	// TODO(alexdadgar)
+	// t.Parallel()
+
 	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
 	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
 
 	httpTest(t, func(c *Config) {
 		c.Server.EncryptKey = key1
-	}, func(s *TestServer) {
+	}, func(s *TestAgent) {
 		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
 		if err != nil {
 			t.Fatalf("err: %v", err)

command/agent/agent_test.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/hashicorp/nomad/helper"
-	"github.com/hashicorp/nomad/nomad"
 	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
 )
 
@@ -38,61 +37,9 @@ func tmpDir(t testing.TB) string {
 	return dir
 }
 
-func makeAgent(t testing.TB, cb func(*Config)) (string, *Agent) {
-	dir := tmpDir(t)
-	conf := DevConfig()
-
-	// Customize the server configuration
-	config := nomad.DefaultConfig()
-	conf.NomadConfig = config
-
-	// Set the data_dir
-	conf.DataDir = dir
-	conf.NomadConfig.DataDir = dir
-
-	// Bind and set ports
-	conf.BindAddr = "127.0.0.1"
-	conf.Ports = &Ports{
-		HTTP: getPort(),
-		RPC:  getPort(),
-		Serf: getPort(),
-	}
-	conf.NodeName = fmt.Sprintf("Node %d", conf.Ports.RPC)
-	conf.Consul = sconfig.DefaultConsulConfig()
-	conf.Vault.Enabled = new(bool)
-
-	// Tighten the Serf timing
-	config.SerfConfig.MemberlistConfig.SuspicionMult = 2
-	config.SerfConfig.MemberlistConfig.RetransmitMult = 2
-	config.SerfConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
-	config.SerfConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
-	config.SerfConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
-
-	// Tighten the Raft timing
-	config.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
-	config.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
-	config.RaftConfig.ElectionTimeout = 40 * time.Millisecond
-	config.RaftConfig.StartAsLeader = true
-	config.RaftTimeout = 500 * time.Millisecond
-
-	if cb != nil {
-		cb(conf)
-	}
-
-	if err := conf.normalizeAddrs(); err != nil {
-		t.Fatalf("error normalizing config: %v", err)
-	}
-	agent, err := NewAgent(conf, os.Stderr)
-	if err != nil {
-		os.RemoveAll(dir)
-		t.Fatalf("err: %v", err)
-	}
-	return dir, agent
-}
-
 func TestAgent_RPCPing(t *testing.T) {
-	dir, agent := makeAgent(t, nil)
-	defer os.RemoveAll(dir)
+	t.Parallel()
+	agent := NewTestAgent(t.Name(), nil)
 	defer agent.Shutdown()
 
 	var out struct{}
@@ -102,6 +49,7 @@ func TestAgent_RPCPing(t *testing.T) {
 }
 
 func TestAgent_ServerConfig(t *testing.T) {
+	t.Parallel()
 	conf := DefaultConfig()
 	conf.DevMode = true // allow localhost for advertise addrs
 	a := &Agent{config: conf}
@@ -320,6 +268,7 @@ func TestAgent_ServerConfig(t *testing.T) {
 }
 
 func TestAgent_ClientConfig(t *testing.T) {
+	t.Parallel()
 	conf := DefaultConfig()
 	conf.Client.Enabled = true
 
@@ -365,6 +314,7 @@ func TestAgent_ClientConfig(t *testing.T) {
 // TestAgent_HTTPCheck asserts Agent.agentHTTPCheck properly alters the HTTP
 // API health check depending on configuration.
 func TestAgent_HTTPCheck(t *testing.T) {
+	t.Parallel()
 	logger := log.New(ioutil.Discard, "", 0)
 	if testing.Verbose() {
 		logger = log.New(os.Stdout, "[TestAgent_HTTPCheck] ", log.Lshortfile)
@@ -455,6 +405,7 @@ func TestAgent_HTTPCheck(t *testing.T) {
 }
 
 func TestAgent_ConsulSupportsTLSSkipVerify(t *testing.T) {
+	t.Parallel()
 	assertSupport := func(expected bool, blob string) {
 		self := map[string]map[string]interface{}{}
 		if err := json.Unmarshal([]byte("{"+blob+"}"), &self); err != nil {
@@ -561,6 +512,7 @@ func TestAgent_ConsulSupportsTLSSkipVerify(t *testing.T) {
 // TestAgent_HTTPCheckPath asserts clients and servers use different endpoints
 // for healthchecks.
 func TestAgent_HTTPCheckPath(t *testing.T) {
+	t.Parallel()
 	// Agent.agentHTTPCheck only needs a config and logger
 	a := &Agent{
 		config: DevConfig(),

command/agent/alloc_endpoint_test.go

@@ -13,7 +13,8 @@ import (
 )
 
 func TestHTTP_AllocsList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		alloc1 := mock.Alloc()
@@ -59,7 +60,8 @@ func TestHTTP_AllocsList(t *testing.T) {
 }
 
 func TestHTTP_AllocsPrefixList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 
@@ -118,7 +120,8 @@ func TestHTTP_AllocsPrefixList(t *testing.T) {
 }
 
 func TestHTTP_AllocQuery(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		alloc := mock.Alloc()
@@ -164,7 +167,8 @@ func TestHTTP_AllocQuery(t *testing.T) {
 }
 
 func TestHTTP_AllocQuery_Payload(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		alloc := mock.Alloc()
@@ -220,7 +224,8 @@ func TestHTTP_AllocQuery_Payload(t *testing.T) {
 }
 
 func TestHTTP_AllocStats(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/client/allocation/123/foo", nil)
 		if err != nil {
@@ -237,7 +242,8 @@ func TestHTTP_AllocStats(t *testing.T) {
 }
 
 func TestHTTP_AllocSnapshot(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/client/allocation/123/snapshot", nil)
 		if err != nil {
@@ -254,7 +260,8 @@ func TestHTTP_AllocSnapshot(t *testing.T) {
 }
 
 func TestHTTP_AllocGC(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/client/allocation/123/gc", nil)
 		if err != nil {
@@ -271,7 +278,8 @@ func TestHTTP_AllocGC(t *testing.T) {
 }
 
 func TestHTTP_AllocAllGC(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/client/gc", nil)
 		if err != nil {

command/agent/command_test.go

@@ -12,10 +12,12 @@ import (
 )
 
 func TestCommand_Implements(t *testing.T) {
+	t.Parallel()
 	var _ cli.Command = &Command{}
 }
 
 func TestCommand_Args(t *testing.T) {
+	t.Parallel()
 	tmpDir, err := ioutil.TempDir("", "nomad")
 	if err != nil {
 		t.Fatalf("err: %s", err)
@@ -75,9 +77,10 @@ func TestCommand_Args(t *testing.T) {
 	}
 }
 
+// TODO Why is this failing
 func TestRetryJoin(t *testing.T) {
-	dir, agent := makeAgent(t, nil)
-	defer os.RemoveAll(dir)
+	t.Parallel()
+	agent := NewTestAgent(t.Name(), nil)
 	defer agent.Shutdown()
 
 	doneCh := make(chan struct{})
@@ -97,14 +100,11 @@ func TestRetryJoin(t *testing.T) {
 		},
 	}
 
-	serfAddr := fmt.Sprintf(
-		"%s:%d",
-		agent.config.BindAddr,
-		agent.config.Ports.Serf)
+	serfAddr := agent.Config.normalizedAddrs.Serf
 
 	args := []string{
 		"-dev",
-		"-node", fmt.Sprintf(`"Node %d"`, getPort()),
+		"-node", "foo",
 		"-retry-join", serfAddr,
 		"-retry-interval", "1s",
 	}

command/agent/config_parse_test.go

@@ -13,6 +13,7 @@ import (
 )
 
 func TestConfig_Parse(t *testing.T) {
+	t.Parallel()
 	cases := []struct {
 		File   string
 		Result *Config

command/agent/deployment_endpoint_test.go

@@ -11,8 +11,9 @@ import (
 )
 
 func TestHTTP_DeploymentList(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		d1 := mock.Deployment()
@@ -41,8 +42,9 @@ func TestHTTP_DeploymentList(t *testing.T) {
 }
 
 func TestHTTP_DeploymentPrefixList(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		d1 := mock.Deployment()
@@ -74,8 +76,9 @@ func TestHTTP_DeploymentPrefixList(t *testing.T) {
 }
 
 func TestHTTP_DeploymentAllocations(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		j := mock.Job()
@@ -112,8 +115,9 @@ func TestHTTP_DeploymentAllocations(t *testing.T) {
 }
 
 func TestHTTP_DeploymentQuery(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		d := mock.Deployment()
@@ -140,8 +144,9 @@ func TestHTTP_DeploymentQuery(t *testing.T) {
 }
 
 func TestHTTP_DeploymentPause(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		j := mock.Job()
@@ -177,8 +182,9 @@ func TestHTTP_DeploymentPause(t *testing.T) {
 }
 
 func TestHTTP_DeploymentPromote(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		j := mock.Job()
@@ -214,8 +220,9 @@ func TestHTTP_DeploymentPromote(t *testing.T) {
 }
 
 func TestHTTP_DeploymentAllocHealth(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		j := mock.Job()
@@ -255,8 +262,9 @@ func TestHTTP_DeploymentAllocHealth(t *testing.T) {
 }
 
 func TestHTTP_DeploymentFail(t *testing.T) {
+	t.Parallel()
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		j := mock.Job()

command/agent/eval_endpoint_test.go

@@ -10,7 +10,8 @@ import (
 )
 
 func TestHTTP_EvalList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		eval1 := mock.Eval()
@@ -54,7 +55,8 @@ func TestHTTP_EvalList(t *testing.T) {
 }
 
 func TestHTTP_EvalPrefixList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		eval1 := mock.Eval()
@@ -105,7 +107,8 @@ func TestHTTP_EvalPrefixList(t *testing.T) {
 }
 
 func TestHTTP_EvalAllocations(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		alloc1 := mock.Alloc()
@@ -153,7 +156,8 @@ func TestHTTP_EvalAllocations(t *testing.T) {
 }
 
 func TestHTTP_EvalQuery(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Directly manipulate the state
 		state := s.Agent.server.State()
 		eval := mock.Eval()

command/agent/fs_endpoint_test.go

@@ -25,7 +25,8 @@ import (
 )
 
 func TestAllocDirFS_List_MissingParams(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/client/fs/ls/", nil)
 		if err != nil {
 			t.Fatalf("err: %v", err)
@@ -40,7 +41,8 @@ func TestAllocDirFS_List_MissingParams(t *testing.T) {
 }
 
 func TestAllocDirFS_Stat_MissingParams(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/client/fs/stat/", nil)
 		if err != nil {
 			t.Fatalf("err: %v", err)
@@ -67,7 +69,8 @@ func TestAllocDirFS_Stat_MissingParams(t *testing.T) {
 }
 
 func TestAllocDirFS_ReadAt_MissingParams(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/client/fs/readat/", nil)
 		if err != nil {
 			t.Fatalf("err: %v", err)
@@ -500,7 +503,8 @@ func TestStreamFramer_Order_PlainText(t *testing.T) {
 }
 
 func TestHTTP_Stream_MissingParams(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/client/fs/stream/", nil)
 		if err != nil {
 			t.Fatalf("err: %v", err)
@@ -560,7 +564,8 @@ func (n nopWriteCloser) Close() error {
 }
 
 func TestHTTP_Stream_NoFile(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -576,7 +581,8 @@ func TestHTTP_Stream_NoFile(t *testing.T) {
 }
 
 func TestHTTP_Stream_Modify(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -651,7 +657,8 @@ func TestHTTP_Stream_Modify(t *testing.T) {
 }
 
 func TestHTTP_Stream_Truncate(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -760,7 +767,8 @@ func TestHTTP_Stream_Truncate(t *testing.T) {
 }
 
 func TestHTTP_Stream_Delete(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -842,7 +850,8 @@ func TestHTTP_Stream_Delete(t *testing.T) {
 }
 
 func TestHTTP_Logs_NoFollow(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir and create the log dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -923,7 +932,8 @@ func TestHTTP_Logs_NoFollow(t *testing.T) {
 }
 
 func TestHTTP_Logs_Follow(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Get a temp alloc dir and create the log dir
 		ad := tempAllocDir(t)
 		defer os.RemoveAll(ad.AllocDir)
@@ -1029,7 +1039,7 @@ func BenchmarkHTTP_Logs_Follow(t *testing.B) {
 	runtime.MemProfileRate = 1
 
 	s := makeHTTPServer(t, nil)
-	defer s.Cleanup()
+	defer s.Shutdown()
 	testutil.WaitForLeader(t, s.Agent.RPC)
 
 	// Get a temp alloc dir and create the log dir

command/agent/http.go

@@ -35,7 +35,7 @@ type HTTPServer struct {
 	mux      *http.ServeMux
 	listener net.Listener
 	logger   *log.Logger
-	addr     string
+	Addr     string
 }
 
 // NewHTTPServer starts new HTTP server over the agent
@@ -76,7 +76,7 @@ func NewHTTPServer(agent *Agent, config *Config) (*HTTPServer, error) {
 		mux:      mux,
 		listener: ln,
 		logger:   agent.logger,
-		addr:     ln.Addr().String(),
+		Addr:     ln.Addr().String(),
 	}
 	srv.registerHandlers(config.EnableDebug)
 
@@ -97,7 +97,7 @@ func newScadaHttp(agent *Agent, list net.Listener) *HTTPServer {
 		mux:      mux,
 		listener: list,
 		logger:   agent.logger,
-		addr:     scadaHTTPAddr,
+		Addr:     scadaHTTPAddr,
 	}
 	srv.registerHandlers(false) // Never allow debug for SCADA
 

command/agent/http_test.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"os"
 	"strconv"
 	"testing"
 	"time"
@@ -23,41 +22,17 @@ import (
 	"github.com/hashicorp/nomad/testutil"
 )
 
-type TestServer struct {
-	T      testing.TB
-	Dir    string
-	Agent  *Agent
-	Server *HTTPServer
-}
-
-func (s *TestServer) Cleanup() {
-	s.Server.Shutdown()
-	s.Agent.Shutdown()
-	os.RemoveAll(s.Dir)
-}
-
 // makeHTTPServer returns a test server whose logs will be written to
 // the passed writer. If the writer is nil, the logs are written to stderr.
-func makeHTTPServer(t testing.TB, cb func(c *Config)) *TestServer {
-	dir, agent := makeAgent(t, cb)
-	srv, err := NewHTTPServer(agent, agent.config)
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	s := &TestServer{
-		T:      t,
-		Dir:    dir,
-		Agent:  agent,
-		Server: srv,
-	}
-	return s
+func makeHTTPServer(t testing.TB, cb func(c *Config)) *TestAgent {
+	return NewTestAgent(t.Name(), cb)
 }
 
 func BenchmarkHTTPRequests(b *testing.B) {
 	s := makeHTTPServer(b, func(c *Config) {
 		c.Client.Enabled = false
 	})
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	job := mock.Job()
 	var allocs []*structs.Allocation
@@ -85,6 +60,7 @@ func BenchmarkHTTPRequests(b *testing.B) {
 }
 
 func TestSetIndex(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	setIndex(resp, 1000)
 	header := resp.Header().Get("X-Nomad-Index")
@@ -98,6 +74,7 @@ func TestSetIndex(t *testing.T) {
 }
 
 func TestSetKnownLeader(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	setKnownLeader(resp, true)
 	header := resp.Header().Get("X-Nomad-KnownLeader")
@@ -113,6 +90,7 @@ func TestSetKnownLeader(t *testing.T) {
 }
 
 func TestSetLastContact(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	setLastContact(resp, 123456*time.Microsecond)
 	header := resp.Header().Get("X-Nomad-LastContact")
@@ -122,6 +100,7 @@ func TestSetLastContact(t *testing.T) {
 }
 
 func TestSetMeta(t *testing.T) {
+	t.Parallel()
 	meta := structs.QueryMeta{
 		Index:       1000,
 		KnownLeader: true,
@@ -144,9 +123,10 @@ func TestSetMeta(t *testing.T) {
 }
 
 func TestSetHeaders(t *testing.T) {
+	t.Parallel()
 	s := makeHTTPServer(t, nil)
 	s.Agent.config.HTTPAPIResponseHeaders = map[string]string{"foo": "bar"}
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	resp := httptest.NewRecorder()
 	handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
@@ -164,8 +144,9 @@ func TestSetHeaders(t *testing.T) {
 }
 
 func TestContentTypeIsJSON(t *testing.T) {
+	t.Parallel()
 	s := makeHTTPServer(t, nil)
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	resp := httptest.NewRecorder()
 
@@ -184,20 +165,23 @@ func TestContentTypeIsJSON(t *testing.T) {
 }
 
 func TestPrettyPrint(t *testing.T) {
+	t.Parallel()
 	testPrettyPrint("pretty=1", true, t)
 }
 
 func TestPrettyPrintOff(t *testing.T) {
+	t.Parallel()
 	testPrettyPrint("pretty=0", false, t)
 }
 
 func TestPrettyPrintBare(t *testing.T) {
+	t.Parallel()
 	testPrettyPrint("pretty", true, t)
 }
 
 func testPrettyPrint(pretty string, prettyFmt bool, t *testing.T) {
 	s := makeHTTPServer(t, nil)
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	r := &structs.Job{Name: "foo"}
 
@@ -228,6 +212,7 @@ func testPrettyPrint(pretty string, prettyFmt bool, t *testing.T) {
 }
 
 func TestParseWait(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	var b structs.QueryOptions
 
@@ -250,6 +235,7 @@ func TestParseWait(t *testing.T) {
 }
 
 func TestParseWait_InvalidTime(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	var b structs.QueryOptions
 
@@ -269,6 +255,7 @@ func TestParseWait_InvalidTime(t *testing.T) {
 }
 
 func TestParseWait_InvalidIndex(t *testing.T) {
+	t.Parallel()
 	resp := httptest.NewRecorder()
 	var b structs.QueryOptions
 
@@ -288,6 +275,7 @@ func TestParseWait_InvalidIndex(t *testing.T) {
 }
 
 func TestParseConsistency(t *testing.T) {
+	t.Parallel()
 	var b structs.QueryOptions
 
 	req, err := http.NewRequest("GET",
@@ -315,8 +303,9 @@ func TestParseConsistency(t *testing.T) {
 }
 
 func TestParseRegion(t *testing.T) {
+	t.Parallel()
 	s := makeHTTPServer(t, nil)
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	req, err := http.NewRequest("GET",
 		"/v1/jobs?region=foo", nil)
@@ -345,6 +334,7 @@ func TestParseRegion(t *testing.T) {
 // TestHTTP_VerifyHTTPSClient asserts that a client certificate signed by the
 // appropriate CA is required when VerifyHTTPSClient=true.
 func TestHTTP_VerifyHTTPSClient(t *testing.T) {
+	t.Parallel()
 	const (
 		cafile  = "../../helper/tlsutil/testdata/ca.pem"
 		foocert = "../../helper/tlsutil/testdata/nomad-foo.pem"
@@ -360,7 +350,7 @@ func TestHTTP_VerifyHTTPSClient(t *testing.T) {
 			KeyFile:           fookey,
 		}
 	})
-	defer s.Cleanup()
+	defer s.Shutdown()
 
 	reqURL := fmt.Sprintf("https://%s/v1/agent/self", s.Agent.config.AdvertiseAddrs.HTTP)
 
@@ -492,9 +482,9 @@ func getIndex(t *testing.T, resp *httptest.ResponseRecorder) uint64 {
 	return uint64(val)
 }
 
-func httpTest(t testing.TB, cb func(c *Config), f func(srv *TestServer)) {
+func httpTest(t testing.TB, cb func(c *Config), f func(srv *TestAgent)) {
 	s := makeHTTPServer(t, cb)
-	defer s.Cleanup()
+	defer s.Shutdown()
 	testutil.WaitForLeader(t, s.Agent.RPC)
 	f(s)
 }

command/agent/job_endpoint_test.go

@@ -18,7 +18,8 @@ import (
 )
 
 func TestHTTP_JobsList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		for i := 0; i < 3; i++ {
 			// Create the job
 			job := mock.Job()
@@ -70,7 +71,8 @@ func TestHTTP_PrefixJobsList(t *testing.T) {
 		"aabbbbbb-e8f7-fd38-c855-ab94ceb89706",
 		"aabbcccc-e8f7-fd38-c855-ab94ceb89706",
 	}
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		for i := 0; i < 3; i++ {
 			// Create the job
 			job := mock.Job()
@@ -119,7 +121,8 @@ func TestHTTP_PrefixJobsList(t *testing.T) {
 }
 
 func TestHTTP_JobsRegister(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := api.MockJob()
 		args := api.JobRegisterRequest{
@@ -169,7 +172,8 @@ func TestHTTP_JobsRegister(t *testing.T) {
 }
 
 func TestHTTP_JobsRegister_Defaulting(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := api.MockJob()
 
@@ -226,7 +230,8 @@ func TestHTTP_JobsRegister_Defaulting(t *testing.T) {
 }
 
 func TestHTTP_JobQuery(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -271,7 +276,8 @@ func TestHTTP_JobQuery(t *testing.T) {
 }
 
 func TestHTTP_JobQuery_Payload(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 
@@ -324,7 +330,8 @@ func TestHTTP_JobQuery_Payload(t *testing.T) {
 }
 
 func TestHTTP_JobUpdate(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := api.MockJob()
 		args := api.JobRegisterRequest{
@@ -374,7 +381,8 @@ func TestHTTP_JobUpdate(t *testing.T) {
 }
 
 func TestHTTP_JobDelete(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -466,7 +474,8 @@ func TestHTTP_JobDelete(t *testing.T) {
 }
 
 func TestHTTP_JobForceEvaluate(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -505,7 +514,8 @@ func TestHTTP_JobForceEvaluate(t *testing.T) {
 }
 
 func TestHTTP_JobEvaluations(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -552,7 +562,8 @@ func TestHTTP_JobEvaluations(t *testing.T) {
 }
 
 func TestHTTP_JobAllocations(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		alloc1 := mock.Alloc()
 		args := structs.JobRegisterRequest{
@@ -605,7 +616,8 @@ func TestHTTP_JobAllocations(t *testing.T) {
 
 func TestHTTP_JobDeployments(t *testing.T) {
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		j := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -643,7 +655,8 @@ func TestHTTP_JobDeployments(t *testing.T) {
 
 func TestHTTP_JobDeployment(t *testing.T) {
 	assert := assert.New(t)
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		j := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -680,7 +693,8 @@ func TestHTTP_JobDeployment(t *testing.T) {
 }
 
 func TestHTTP_JobVersions(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := mock.Job()
 		args := structs.JobRegisterRequest{
@@ -751,7 +765,8 @@ func TestHTTP_JobVersions(t *testing.T) {
 }
 
 func TestHTTP_PeriodicForce(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create and register a periodic job.
 		job := mock.PeriodicJob()
 		args := structs.JobRegisterRequest{
@@ -790,7 +805,8 @@ func TestHTTP_PeriodicForce(t *testing.T) {
 }
 
 func TestHTTP_JobPlan(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		job := api.MockJob()
 		args := api.JobPlanRequest{
@@ -826,7 +842,8 @@ func TestHTTP_JobPlan(t *testing.T) {
 }
 
 func TestHTTP_JobDispatch(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the parameterized job
 		job := mock.Job()
 		job.Type = "batch"
@@ -874,7 +891,8 @@ func TestHTTP_JobDispatch(t *testing.T) {
 }
 
 func TestHTTP_JobRevert(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job and register it twice
 		job := mock.Job()
 		regReq := structs.JobRegisterRequest{
@@ -926,7 +944,8 @@ func TestHTTP_JobRevert(t *testing.T) {
 }
 
 func TestHTTP_JobStable(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job and register it twice
 		job := mock.Job()
 		regReq := structs.JobRegisterRequest{

command/agent/keyring_test.go

@@ -9,11 +9,11 @@ import (
 )
 
 func TestAgent_LoadKeyrings(t *testing.T) {
+	t.Parallel()
 	key := "tbLJg26ZJyJ9pK3qhc9jig=="
 
 	// Should be no configured keyring file by default
-	dir1, agent1 := makeAgent(t, nil)
-	defer os.RemoveAll(dir1)
+	agent1 := NewTestAgent(t.Name(), nil)
 	defer agent1.Shutdown()
 
 	c := agent1.server.GetConfig()
@@ -24,14 +24,12 @@ func TestAgent_LoadKeyrings(t *testing.T) {
 		t.Fatalf("keyring should not be loaded")
 	}
 
-	// Server should auto-load LAN and WAN keyring files
-	dir2, agent2 := makeAgent(t, func(c *Config) {
-		file := filepath.Join(c.DataDir, serfKeyring)
-		if err := initKeyring(file, key); err != nil {
-			t.Fatalf("err: %s", err)
-		}
-	})
-	defer os.RemoveAll(dir2)
+	// Server should auto-load WAN keyring files
+	agent2 := &TestAgent{
+		Name: t.Name() + "2",
+		Key:  key,
+	}
+	agent2.Start()
 	defer agent2.Shutdown()
 
 	c = agent2.server.GetConfig()
@@ -44,6 +42,7 @@ func TestAgent_LoadKeyrings(t *testing.T) {
 }
 
 func TestAgent_InitKeyring(t *testing.T) {
+	t.Parallel()
 	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
 	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
 	expected := fmt.Sprintf(`["%s"]`, key1)

command/agent/log_levels_test.go

@@ -7,6 +7,7 @@ import (
 )
 
 func TestLevelFilter(t *testing.T) {
+	t.Parallel()
 
 	filt := LevelFilter()
 	filt.Levels = []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARN", "ERR"}

command/agent/log_writer_test.go

@@ -13,6 +13,7 @@ func (m *MockLogHandler) HandleLog(l string) {
 }
 
 func TestLogWriter(t *testing.T) {
+	t.Parallel()
 	h := &MockLogHandler{}
 	w := NewLogWriter(4)
 

command/agent/node_endpoint_test.go

@@ -10,7 +10,8 @@ import (
 )
 
 func TestHTTP_NodesList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		for i := 0; i < 3; i++ {
 			// Create the node
 			node := mock.Node()
@@ -57,7 +58,8 @@ func TestHTTP_NodesList(t *testing.T) {
 }
 
 func TestHTTP_NodesPrefixList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		ids := []string{
 			"12345678-abcd-efab-cdef-123456789abc",
 			"12345678-aaaa-efab-cdef-123456789abc",
@@ -113,7 +115,8 @@ func TestHTTP_NodesPrefixList(t *testing.T) {
 }
 
 func TestHTTP_NodeForceEval(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the node
 		node := mock.Node()
 		args := structs.NodeRegisterRequest{
@@ -164,7 +167,8 @@ func TestHTTP_NodeForceEval(t *testing.T) {
 }
 
 func TestHTTP_NodeAllocations(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		node := mock.Node()
 		args := structs.NodeRegisterRequest{
@@ -221,7 +225,8 @@ func TestHTTP_NodeAllocations(t *testing.T) {
 }
 
 func TestHTTP_NodeDrain(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the node
 		node := mock.Node()
 		args := structs.NodeRegisterRequest{
@@ -272,7 +277,8 @@ func TestHTTP_NodeDrain(t *testing.T) {
 }
 
 func TestHTTP_NodeQuery(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Create the job
 		node := mock.Node()
 		args := structs.NodeRegisterRequest{

command/agent/operator_endpoint_test.go

@@ -11,7 +11,8 @@ import (
 )
 
 func TestHTTP_OperatorRaftConfiguration(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		body := bytes.NewBuffer(nil)
 		req, err := http.NewRequest("GET", "/v1/operator/raft/configuration", body)
 		if err != nil {
@@ -39,7 +40,8 @@ func TestHTTP_OperatorRaftConfiguration(t *testing.T) {
 }
 
 func TestHTTP_OperatorRaftPeer(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		body := bytes.NewBuffer(nil)
 		req, err := http.NewRequest("DELETE", "/v1/operator/raft/peer?address=nope", body)
 		if err != nil {

command/agent/region_endpoint_test.go

@@ -7,7 +7,8 @@ import (
 )
 
 func TestHTTP_RegionList(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/regions", nil)
 		if err != nil {

command/agent/stats_endpoint_test.go

@@ -7,7 +7,8 @@ import (
 )
 
 func TestClientStatsRequest(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		req, err := http.NewRequest("GET", "/v1/client/stats/?since=foo", nil)
 		if err != nil {
 			t.Fatalf("err: %v", err)

command/agent/status_endpoint_test.go

@@ -7,7 +7,8 @@ import (
 )
 
 func TestHTTP_StatusLeader(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/status/leader", nil)
 		if err != nil {
@@ -29,7 +30,8 @@ func TestHTTP_StatusLeader(t *testing.T) {
 }
 
 func TestHTTP_StatusPeers(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("GET", "/v1/status/peers", nil)
 		if err != nil {

command/agent/syslog_test.go

@@ -10,6 +10,7 @@ import (
 )
 
 func TestSyslogFilter(t *testing.T) {
+	t.Parallel()
 	if runtime.GOOS == "windows" {
 		t.Skip("Syslog not supported on Windows")
 	}

command/agent/system_endpoint_test.go

@@ -7,7 +7,8 @@ import (
 )
 
 func TestHTTP_SystemGarbageCollect(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("PUT", "/v1/system/gc", nil)
 		if err != nil {
@@ -23,7 +24,8 @@ func TestHTTP_SystemGarbageCollect(t *testing.T) {
 }
 
 func TestHTTP_ReconcileJobSummaries(t *testing.T) {
-	httpTest(t, nil, func(s *TestServer) {
+	t.Parallel()
+	httpTest(t, nil, func(s *TestAgent) {
 		// Make the HTTP request
 		req, err := http.NewRequest("PUT", "/v1/system/reconcile/summaries", nil)
 		if err != nil {

command/agent/testagent.go

@@ -0,0 +1,281 @@
+package agent
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/nomad"
+	"github.com/hashicorp/nomad/nomad/structs"
+	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/hashicorp/nomad/testutil"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano()) // seed random number generator
+}
+
+// TempDir defines the base dir for temporary directories.
+var TempDir = os.TempDir()
+
+// TestAgent encapsulates an Agent with a default configuration and
+// startup procedure suitable for testing. It panics if there are errors
+// during creation or startup instead of returning errors. It manages a
+// temporary data directory which is removed after shutdown.
+type TestAgent struct {
+	// Name is an optional name of the agent.
+	Name string
+
+	// ConfigCallback is an optional callback that allows modification of the
+	// configuration before the agent is started.
+	ConfigCallback func(*Config)
+
+	// Config is the agent configuration. If Config is nil then
+	// TestConfig() is used. If Config.DataDir is set then it is
+	// the callers responsibility to clean up the data directory.
+	// Otherwise, a temporary data directory is created and removed
+	// when Shutdown() is called.
+	Config *Config
+
+	// LogOutput is the sink for the logs. If nil, logs are written
+	// to os.Stderr.
+	LogOutput io.Writer
+
+	// DataDir is the data directory which is used when Config.DataDir
+	// is not set. It is created automatically and removed when
+	// Shutdown() is called.
+	DataDir string
+
+	// Key is the optional encryption key for the keyring.
+	Key string
+
+	// Server is a reference to the started HTTP endpoint.
+	// It is valid after Start().
+	Server *HTTPServer
+
+	// Agent is the embedded Nomad agent.
+	// It is valid after Start().
+	*Agent
+}
+
+// NewTestAgent returns a started agent with the given name and
+// configuration. It panics if the agent could not be started. The
+// caller should call Shutdown() to stop the agent and remove temporary
+// directories.
+func NewTestAgent(name string, configCallback func(*Config)) *TestAgent {
+	a := &TestAgent{Name: name, ConfigCallback: configCallback}
+	a.Start()
+	return a
+}
+
+// Start starts a test agent. It panics if the agent could not be started.
+func (a *TestAgent) Start() *TestAgent {
+	if a.Agent != nil {
+		panic("TestAgent already started")
+	}
+	if a.Config == nil {
+		a.Config = a.config()
+	}
+	if a.Config.DataDir == "" {
+		name := "agent"
+		if a.Name != "" {
+			name = a.Name + "-agent"
+		}
+		name = strings.Replace(name, "/", "_", -1)
+		d, err := ioutil.TempDir(TempDir, name)
+		if err != nil {
+			panic(fmt.Sprintf("Error creating data dir %s: %s", filepath.Join(TempDir, name), err))
+		}
+		a.DataDir = d
+		a.Config.DataDir = d
+		a.Config.NomadConfig.DataDir = d
+	}
+
+	for i := 10; i >= 0; i-- {
+		pickRandomPorts(a.Config)
+		a.Config.NodeName = fmt.Sprintf("Node %d", a.Config.Ports.RPC)
+
+		// write the keyring
+		if a.Key != "" {
+			writeKey := func(key, filename string) {
+				path := filepath.Join(a.Config.DataDir, filename)
+				if err := initKeyring(path, key); err != nil {
+					panic(fmt.Sprintf("Error creating keyring %s: %s", path, err))
+				}
+			}
+			writeKey(a.Key, serfKeyring)
+		}
+
+		// we need the err var in the next exit condition
+		if agent, err := a.start(); err == nil {
+			a.Agent = agent
+			break
+		} else if i == 0 {
+			fmt.Println(a.Name, "Error starting agent:", err)
+			runtime.Goexit()
+		} else {
+			if agent != nil {
+				agent.Shutdown()
+			}
+			wait := time.Duration(rand.Int31n(2000)) * time.Millisecond
+			fmt.Println(a.Name, "retrying in", wait)
+			time.Sleep(wait)
+		}
+
+		// Clean out the data dir if we are responsible for it before we
+		// try again, since the old ports may have gotten written to
+		// the data dir, such as in the Raft configuration.
+		if a.DataDir != "" {
+			if err := os.RemoveAll(a.DataDir); err != nil {
+				fmt.Println(a.Name, "Error resetting data dir:", err)
+				runtime.Goexit()
+			}
+		}
+	}
+
+	if a.Config.NomadConfig.Bootstrap && a.Config.Server.Enabled {
+		testutil.WaitForResult(func() (bool, error) {
+			args := &structs.GenericRequest{}
+			var leader string
+			err := a.RPC("Status.Leader", args, &leader)
+			return leader != "", err
+		}, func(err error) {
+			panic(fmt.Sprintf("failed to find leader: %v", err))
+		})
+	} else {
+		testutil.WaitForResult(func() (bool, error) {
+			req, _ := http.NewRequest("GET", "/v1/agent/self", nil)
+			resp := httptest.NewRecorder()
+			_, err := a.Server.AgentSelfRequest(resp, req)
+			return err == nil && resp.Code == 200, err
+		}, func(err error) {
+			panic(fmt.Sprintf("failed OK response: %v", err))
+		})
+	}
+	return a
+}
+
+func (a *TestAgent) start() (*Agent, error) {
+	if a.LogOutput == nil {
+		a.LogOutput = os.Stderr
+	}
+
+	agent, err := NewAgent(a.Config, a.LogOutput)
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup the HTTP server
+	http, err := NewHTTPServer(agent, a.Config)
+	if err != nil {
+		return agent, err
+	}
+
+	a.Server = http
+	return agent, nil
+}
+
+// Shutdown stops the agent and removes the data directory if it is
+// managed by the test agent.
+func (a *TestAgent) Shutdown() error {
+	defer func() {
+		if a.DataDir != "" {
+			os.RemoveAll(a.DataDir)
+		}
+	}()
+
+	// shutdown agent before endpoints
+	a.Server.Shutdown()
+	return a.Agent.Shutdown()
+}
+
+func (a *TestAgent) HTTPAddr() string {
+	if a.Server == nil {
+		return ""
+	}
+	return a.Server.Addr
+}
+
+func (a *TestAgent) Client() *api.Client {
+	conf := api.DefaultConfig()
+	conf.Address = a.HTTPAddr()
+	c, err := api.NewClient(conf)
+	if err != nil {
+		panic(fmt.Sprintf("Error creating consul API client: %s", err))
+	}
+	return c
+}
+
+// FivePorts returns the first port number of a block of
+// five random ports.
+func FivePorts() int {
+	return 1030 + int(rand.Int31n(6440))*5
+}
+
+// pickRandomPorts selects random ports from fixed size random blocks of
+// ports. This does not eliminate the chance for port conflict but
+// reduces it significanltly with little overhead. Furthermore, asking
+// the kernel for a random port by binding to port 0 prolongs the test
+// execution (in our case +20sec) while also not fully eliminating the
+// chance of port conflicts for concurrently executed test binaries.
+// Instead of relying on one set of ports to be sufficient we retry
+// starting the agent with different ports on port conflict.
+func pickRandomPorts(c *Config) {
+	port := FivePorts()
+	c.Ports.HTTP = port + 1
+	c.Ports.RPC = port + 2
+	c.Ports.Serf = port + 3
+
+	if err := c.normalizeAddrs(); err != nil {
+		panic(fmt.Sprintf("error normalizing config: %v", err))
+	}
+}
+
+// TestConfig returns a unique default configuration for testing an
+// agent.
+func (a *TestAgent) config() *Config {
+	conf := DevConfig()
+
+	// Customize the server configuration
+	config := nomad.DefaultConfig()
+	conf.NomadConfig = config
+
+	// Bind and set ports
+	conf.BindAddr = "127.0.0.1"
+
+	conf.Consul = sconfig.DefaultConsulConfig()
+	conf.Vault.Enabled = new(bool)
+
+	// Tighten the Serf timing
+	config.SerfConfig.MemberlistConfig.SuspicionMult = 2
+	config.SerfConfig.MemberlistConfig.RetransmitMult = 2
+	config.SerfConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
+	config.SerfConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
+	config.SerfConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
+
+	// Tighten the Raft timing
+	config.RaftConfig.LeaderLeaseTimeout = 20 * time.Millisecond
+	config.RaftConfig.HeartbeatTimeout = 40 * time.Millisecond
+	config.RaftConfig.ElectionTimeout = 40 * time.Millisecond
+	config.RaftConfig.StartAsLeader = true
+	config.RaftTimeout = 500 * time.Millisecond
+
+	// Bootstrap ourselves
+	config.Bootstrap = true
+	config.BootstrapExpect = 1
+
+	if a.ConfigCallback != nil {
+		a.ConfigCallback(conf)
+	}
+
+	return conf
+}

scripts/test.sh

@@ -21,5 +21,5 @@ go build -i -tags "$GOTEST_TAGS" -o $TEMPDIR/nomad || exit 1
 echo "--> Running tests"
 GOBIN="`which go`"
 sudo -E PATH=$TEMPDIR:$PATH  -E GOPATH=$GOPATH   -E NOMAD_TEST_RKT=1 \
-    $GOBIN test -tags "$GOTEST_TAGS" ${GOTEST_FLAGS:--cover -timeout=900s} $($GOBIN list ./... | grep -v /vendor/)
+    $GOBIN test -tags "$GOTEST_TAGS" ${GOTEST_FLAGS:--cover -timeout=900s -v -parallel 16} $($GOBIN list ./... | grep -v /vendor/)