Enable serf encryption (#1791)

* Added the keygen command

* Added support for gossip encryption

* Changed the URL for keyring management

* Fixed the cli

* Added some tests

* Added tests for keyring operations

* Added a test for removal of keys

* Added some docs

* Fixed some docs

* Added general options

api/agent.go

@@ -16,6 +16,19 @@ type Agent struct {
 	region     string
 }
 
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	Messages map[string]string
+	Keys     map[string]int
+	NumNodes int
+}
+
+// KeyringRequest is request objects for serf key operations.
+type KeyringRequest struct {
+	Key string
+}
+
 // Agent returns a new agent which can be used to query
 // the agent-specific endpoints.
 func (c *Client) Agent() *Agent {
@@ -157,6 +170,46 @@ func (a *Agent) SetServers(addrs []string) error {
 	return err
 }
 
+// ListKeys returns the list of installed keys
+func (a *Agent) ListKeys() (*KeyringResponse, error) {
+	var resp KeyringResponse
+	_, err := a.client.query("/v1/agent/keyring/list", &resp, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+// InstallKey installs a key in the keyrings of all the serf members
+func (a *Agent) InstallKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/install", &args, &resp, nil)
+	return &resp, err
+}
+
+// UseKey uses a key from the keyring of serf members
+func (a *Agent) UseKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/use", &args, &resp, nil)
+	return &resp, err
+}
+
+// RemoveKey removes a particular key from keyrings of serf members
+func (a *Agent) RemoveKey(key string) (*KeyringResponse, error) {
+	args := KeyringRequest{
+		Key: key,
+	}
+	var resp KeyringResponse
+	_, err := a.client.write("/v1/agent/keyring/remove", &args, &resp, nil)
+	return &resp, err
+}
+
 // joinResponse is used to decode the response we get while
 // sending a member join request.
 type joinResponse struct {

command/agent/agent.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"log"
 	"net"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -371,6 +372,11 @@ func (a *Agent) setupServer() error {
 		return fmt.Errorf("server config setup failed: %s", err)
 	}
 
+	// Sets up the keyring for gossip encryption
+	if err := a.setupKeyrings(conf); err != nil {
+		return fmt.Errorf("failed to configure keyring: %v", err)
+	}
+
 	// Create the server
 	server, err := nomad.NewServer(conf, a.consulSyncer, a.logger)
 	if err != nil {
@@ -431,6 +437,30 @@ func (a *Agent) setupServer() error {
 	return nil
 }
 
+// setupKeyrings is used to initialize and load keyrings during agent startup
+func (a *Agent) setupKeyrings(config *nomad.Config) error {
+	file := filepath.Join(a.config.DataDir, serfKeyring)
+
+	if a.config.Server.EncryptKey == "" {
+		goto LOAD
+	}
+	if _, err := os.Stat(file); err != nil {
+		if err := initKeyring(file, a.config.Server.EncryptKey); err != nil {
+			return err
+		}
+	}
+
+LOAD:
+	if _, err := os.Stat(file); err == nil {
+		config.SerfConfig.KeyringFile = file
+	}
+	if err := loadKeyringFile(config.SerfConfig); err != nil {
+		return err
+	}
+	// Success!
+	return nil
+}
+
 // setupClient is used to setup the client if enabled
 func (a *Agent) setupClient() error {
 	if !a.config.Client.Enabled {

command/agent/agent_endpoint.go

@@ -3,7 +3,9 @@ package agent
 import (
 	"net"
 	"net/http"
+	"strings"
 
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/serf/serf"
 )
 
@@ -165,6 +167,56 @@ func (s *HTTPServer) updateServers(resp http.ResponseWriter, req *http.Request)
 	return nil, nil
 }
 
+// KeyringOperationRequest allows an operator to install/delete/use keys
+func (s *HTTPServer) KeyringOperationRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	srv := s.agent.Server()
+	if srv == nil {
+		return nil, CodedError(501, ErrInvalidMethod)
+	}
+
+	kmgr := srv.KeyManager()
+	var sresp *serf.KeyResponse
+	var err error
+
+	// Get the key from the req body
+	var args structs.KeyringRequest
+
+	//Get the op
+	op := strings.TrimPrefix(req.URL.Path, "/v1/agent/keyring/")
+
+	switch op {
+	case "list":
+		sresp, err = kmgr.ListKeys()
+	case "install":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.InstallKey(args.Key)
+	case "use":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.UseKey(args.Key)
+	case "remove":
+		if err := decodeBody(req, &args); err != nil {
+			return nil, CodedError(500, err.Error())
+		}
+		sresp, err = kmgr.RemoveKey(args.Key)
+	default:
+		return nil, CodedError(404, "resource not found")
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	kresp := structs.KeyringResponse{
+		Messages: sresp.Messages,
+		Keys:     sresp.Keys,
+		NumNodes: sresp.NumNodes,
+	}
+	return kresp, nil
+}
+
 type agentSelf struct {
 	Config *Config                      `json:"config"`
 	Member Member                       `json:"member,omitempty"`

command/agent/agent_endpoint_test.go

@@ -1,11 +1,15 @@
 package agent
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 func TestHTTP_AgentSelf(t *testing.T) {
@@ -177,3 +181,111 @@ func TestHTTP_AgentSetServers(t *testing.T) {
 		}
 	})
 }
+
+func TestHTTP_AgentListKeys(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
+
+func TestHTTP_AgentInstallKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 2 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}
+
+func TestHTTP_AgentRemoveKey(t *testing.T) {
+	key1 := "HS5lJ+XuTlYKWaeGYyG+/A=="
+	key2 := "wH1Bn9hlJ0emgWB1JttVRA=="
+
+	httpTest(t, func(c *Config) {
+		c.Server.EncryptKey = key1
+	}, func(s *TestServer) {
+		b, err := json.Marshal(&structs.KeyringRequest{Key: key2})
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		req, err := http.NewRequest("GET", "/v1/agent/keyring/install", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW := httptest.NewRecorder()
+		_, err = s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/remove", bytes.NewReader(b))
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		if _, err = s.Server.KeyringOperationRequest(respW, req); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+
+		req, err = http.NewRequest("GET", "/v1/agent/keyring/list", nil)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		respW = httptest.NewRecorder()
+		out, err := s.Server.KeyringOperationRequest(respW, req)
+		if err != nil {
+			t.Fatalf("err: %s", err)
+		}
+		kresp := out.(structs.KeyringResponse)
+		if len(kresp.Keys) != 1 {
+			t.Fatalf("bad: %v", kresp)
+		}
+	})
+}

command/agent/command.go

@@ -83,6 +83,7 @@ func (c *Command) readConfig() *Config {
 	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.RetryJoin), "retry-join", "")
 	flags.IntVar(&cmdConfig.Server.RetryMaxAttempts, "retry-max", 0, "")
 	flags.StringVar(&cmdConfig.Server.RetryInterval, "retry-interval", "", "")
+	flags.StringVar(&cmdConfig.Server.EncryptKey, "encrypt", "", "gossip encryption key")
 
 	// Client-only options
 	flags.StringVar(&cmdConfig.Client.StateDir, "state-dir", "", "")
@@ -195,6 +196,17 @@ func (c *Command) readConfig() *Config {
 		return config
 	}
 
+	if config.Server.EncryptKey != "" {
+		if _, err := config.Server.EncryptBytes(); err != nil {
+			c.Ui.Error(fmt.Sprintf("Invalid encryption key: %s", err))
+			return nil
+		}
+		keyfile := filepath.Join(config.DataDir, serfKeyring)
+		if _, err := os.Stat(keyfile); err == nil {
+			c.Ui.Error("WARNING: keyring exists but -encrypt given, using keyring")
+		}
+	}
+
 	// Parse the RetryInterval.
 	dur, err := time.ParseDuration(config.Server.RetryInterval)
 	if err != nil {
@@ -818,6 +830,9 @@ Server Options:
     bootstrapping the cluster. Once <num> servers have joined eachother,
     Nomad initiates the bootstrap process.
 
+  -encrypt=<key>
+    Provides the gossip encryption key
+
   -join=<address>
     Address of an agent to join at start time. Can be specified
     multiple times.

command/agent/config-test-fixtures/basic.hcl

@@ -68,6 +68,7 @@ server {
 	retry_max = 3
 	retry_interval = "15s"
 	rejoin_after_leave = true
+    encrypt = "abc"
 }
 telemetry {
 	statsite_address = "127.0.0.1:1234"

command/agent/config.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io"
 	"net"
@@ -244,6 +245,14 @@ type ServerConfig struct {
 	// the cluster until an explicit join is received. If this is set to
 	// true, we ignore the leave, and rejoin the cluster on start.
 	RejoinAfterLeave bool `mapstructure:"rejoin_after_leave"`
+
+	// Encryption key to use for the Serf communication
+	EncryptKey string `mapstructure:"encrypt" json:"-"`
+}
+
+// EncryptBytes returns the encryption key configured.
+func (s *ServerConfig) EncryptBytes() ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s.EncryptKey)
 }
 
 // Telemetry is the telemetry configuration for the server
@@ -669,6 +678,9 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 	if b.RejoinAfterLeave {
 		result.RejoinAfterLeave = true
 	}
+	if b.EncryptKey != "" {
+		result.EncryptKey = b.EncryptKey
+	}
 
 	// Add the schedulers
 	result.EnabledSchedulers = append(result.EnabledSchedulers, b.EnabledSchedulers...)

command/agent/config_parse.go

@@ -484,6 +484,7 @@ func parseServer(result **ServerConfig, list *ast.ObjectList) error {
 		"retry_max",
 		"retry_interval",
 		"rejoin_after_leave",
+		"encrypt",
 	}
 	if err := checkHCLKeys(listVal, valid); err != nil {
 		return err

command/agent/config_parse_test.go

@@ -85,6 +85,7 @@ func TestConfig_Parse(t *testing.T) {
 					RetryInterval:     "15s",
 					RejoinAfterLeave:  true,
 					RetryMaxAttempts:  3,
+					EncryptKey:        "abc",
 				},
 				Telemetry: &Telemetry{
 					StatsiteAddr:             "127.0.0.1:1234",

command/agent/http.go

@@ -122,6 +122,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.HandleFunc("/v1/agent/members", s.wrap(s.AgentMembersRequest))
 	s.mux.HandleFunc("/v1/agent/force-leave", s.wrap(s.AgentForceLeaveRequest))
 	s.mux.HandleFunc("/v1/agent/servers", s.wrap(s.AgentServersRequest))
+	s.mux.HandleFunc("/v1/agent/keyring/", s.wrap(s.KeyringOperationRequest))
 
 	s.mux.HandleFunc("/v1/regions", s.wrap(s.RegionListRequest))
 

command/agent/keyring.go

@@ -0,0 +1,106 @@
+package agent
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/memberlist"
+	"github.com/hashicorp/serf/serf"
+)
+
+const (
+	serfKeyring = "server/serf.keyring"
+)
+
+// initKeyring will create a keyring file at a given path.
+func initKeyring(path, key string) error {
+	var keys []string
+
+	if keyBytes, err := base64.StdEncoding.DecodeString(key); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	} else if err := memberlist.ValidateKey(keyBytes); err != nil {
+		return fmt.Errorf("Invalid key: %s", err)
+	}
+
+	// Just exit if the file already exists.
+	if _, err := os.Stat(path); err == nil {
+		return nil
+	}
+
+	keys = append(keys, key)
+	keyringBytes, err := json.Marshal(keys)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return err
+	}
+
+	fh, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	if _, err := fh.Write(keyringBytes); err != nil {
+		os.Remove(path)
+		return err
+	}
+
+	return nil
+}
+
+// loadKeyringFile will load a gossip encryption keyring out of a file. The file
+// must be in JSON format and contain a list of encryption key strings.
+func loadKeyringFile(c *serf.Config) error {
+	if c.KeyringFile == "" {
+		return nil
+	}
+
+	if _, err := os.Stat(c.KeyringFile); err != nil {
+		return err
+	}
+
+	// Read in the keyring file data
+	keyringData, err := ioutil.ReadFile(c.KeyringFile)
+	if err != nil {
+		return err
+	}
+
+	// Decode keyring JSON
+	keys := make([]string, 0)
+	if err := json.Unmarshal(keyringData, &keys); err != nil {
+		return err
+	}
+
+	// Decode base64 values
+	keysDecoded := make([][]byte, len(keys))
+	for i, key := range keys {
+		keyBytes, err := base64.StdEncoding.DecodeString(key)
+		if err != nil {
+			return err
+		}
+		keysDecoded[i] = keyBytes
+	}
+
+	// Guard against empty keyring
+	if len(keysDecoded) == 0 {
+		return fmt.Errorf("no keys present in keyring file: %s", c.KeyringFile)
+	}
+
+	// Create the keyring
+	keyring, err := memberlist.NewKeyring(keysDecoded, keysDecoded[0])
+	if err != nil {
+		return err
+	}
+
+	c.MemberlistConfig.Keyring = keyring
+
+	// Success!
+	return nil
+}

command/agent/keyring_test.go

@@ -0,0 +1,85 @@
+package agent
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestAgent_LoadKeyrings(t *testing.T) {
+	key := "tbLJg26ZJyJ9pK3qhc9jig=="
+
+	// Should be no configured keyring file by default
+	dir1, agent1 := makeAgent(t, nil)
+	defer os.RemoveAll(dir1)
+	defer agent1.Shutdown()
+
+	c := agent1.server.GetConfig()
+	if c.SerfConfig.KeyringFile != "" {
+		t.Fatalf("bad: %#v", c.SerfConfig.KeyringFile)
+	}
+	if c.SerfConfig.MemberlistConfig.Keyring != nil {
+		t.Fatalf("keyring should not be loaded")
+	}
+
+	// Server should auto-load LAN and WAN keyring files
+	dir2, agent2 := makeAgent(t, func(c *Config) {
+		file := filepath.Join(c.DataDir, serfKeyring)
+		if err := initKeyring(file, key); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+	})
+	defer os.RemoveAll(dir2)
+	defer agent2.Shutdown()
+
+	c = agent2.server.GetConfig()
+	if c.SerfConfig.KeyringFile == "" {
+		t.Fatalf("should have keyring file")
+	}
+	if c.SerfConfig.MemberlistConfig.Keyring == nil {
+		t.Fatalf("keyring should be loaded")
+	}
+}
+
+func TestAgent_InitKeyring(t *testing.T) {
+	key1 := "tbLJg26ZJyJ9pK3qhc9jig=="
+	key2 := "4leC33rgtXKIVUr9Nr0snQ=="
+	expected := fmt.Sprintf(`["%s"]`, key1)
+
+	dir, err := ioutil.TempDir("", "nomad")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	file := filepath.Join(dir, "keyring")
+
+	// First initialize the keyring
+	if err := initKeyring(file, key1); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+
+	// Try initializing again with a different key
+	if err := initKeyring(file, key2); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// Content should still be the same
+	content, err = ioutil.ReadFile(file)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if string(content) != expected {
+		t.Fatalf("bad: %s", content)
+	}
+}

command/keygen.go

@@ -0,0 +1,45 @@
+package command
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"strings"
+)
+
+// KeygenCommand is a Command implementation that generates an encryption
+// key for use in `nomad agent`.
+type KeygenCommand struct {
+	Meta
+}
+
+func (c *KeygenCommand) Run(_ []string) int {
+	key := make([]byte, 16)
+	n, err := rand.Reader.Read(key)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error reading random data: %s", err))
+		return 1
+	}
+	if n != 16 {
+		c.Ui.Error(fmt.Sprintf("Couldn't read enough entropy. Generate more entropy!"))
+		return 1
+	}
+
+	c.Ui.Output(base64.StdEncoding.EncodeToString(key))
+	return 0
+}
+
+func (c *KeygenCommand) Synopsis() string {
+	return "Generates a new encryption key"
+}
+
+func (c *KeygenCommand) Help() string {
+	helpText := `
+Usage: nomad keygen
+
+  Generates a new encryption key that can be used to configure the
+  agent to encrypt traffic. The output of this command is already
+  in the proper format that the agent expects.
+`
+	return strings.TrimSpace(helpText)
+}

command/keygen_test.go

@@ -0,0 +1,27 @@
+package command
+
+import (
+	"encoding/base64"
+	"testing"
+
+	"github.com/mitchellh/cli"
+)
+
+func TestKeygenCommand(t *testing.T) {
+	ui := new(cli.MockUi)
+	c := &KeygenCommand{Meta: Meta{Ui: ui}}
+	code := c.Run(nil)
+	if code != 0 {
+		t.Fatalf("bad: %d", code)
+	}
+
+	output := ui.OutputWriter.String()
+	result, err := base64.StdEncoding.DecodeString(output)
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if len(result) != 16 {
+		t.Fatalf("bad: %#v", result)
+	}
+}

command/keyring.go

@@ -0,0 +1,157 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/mitchellh/cli"
+)
+
+// KeyringCommand is a Command implementation that handles querying, installing,
+// and removing gossip encryption keys from a keyring.
+type KeyringCommand struct {
+	Meta
+}
+
+func (c *KeyringCommand) Run(args []string) int {
+	var installKey, useKey, removeKey, token string
+	var listKeys bool
+
+	flags := c.Meta.FlagSet("keys", FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
+
+	flags.StringVar(&installKey, "install", "", "install key")
+	flags.StringVar(&useKey, "use", "", "use key")
+	flags.StringVar(&removeKey, "remove", "", "remove key")
+	flags.BoolVar(&listKeys, "list", false, "list keys")
+	flags.StringVar(&token, "token", "", "acl token")
+
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	c.Ui = &cli.PrefixedUi{
+		OutputPrefix: "",
+		InfoPrefix:   "==> ",
+		ErrorPrefix:  "",
+		Ui:           c.Ui,
+	}
+
+	// Only accept a single argument
+	found := listKeys
+	for _, arg := range []string{installKey, useKey, removeKey} {
+		if found && len(arg) > 0 {
+			c.Ui.Error("Only a single action is allowed")
+			return 1
+		}
+		found = found || len(arg) > 0
+	}
+
+	// Fail fast if no actionable args were passed
+	if !found {
+		c.Ui.Error(c.Help())
+		return 1
+	}
+
+	// All other operations will require a client connection
+	client, err := c.Meta.Client()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error creating nomad cli client: %s", err))
+		return 1
+	}
+
+	if listKeys {
+		c.Ui.Info("Gathering installed encryption keys...")
+		r, err := client.Agent().ListKeys()
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		c.handleKeyResponse(r)
+		return 0
+	}
+
+	if installKey != "" {
+		c.Ui.Info("Installing new gossip encryption key...")
+		_, err := client.Agent().InstallKey(installKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return 0
+	}
+
+	if useKey != "" {
+		c.Ui.Info("Changing primary gossip encryption key...")
+		_, err := client.Agent().UseKey(useKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return 0
+	}
+
+	if removeKey != "" {
+		c.Ui.Info("Removing gossip encryption key...")
+		_, err := client.Agent().RemoveKey(removeKey)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("error: %s", err))
+			return 1
+		}
+		return 0
+	}
+
+	// Should never make it here
+	return 0
+}
+
+func (c *KeyringCommand) handleKeyResponse(resp *api.KeyringResponse) {
+	out := make([]string, len(resp.Keys)+1)
+	out[0] = "Key"
+	i := 1
+	for k := range resp.Keys {
+		out[i] = fmt.Sprintf("%s", k)
+		i = i + 1
+	}
+	c.Ui.Output(formatList(out))
+}
+
+func (c *KeyringCommand) Help() string {
+	helpText := `
+Usage: nomad keyring [options]
+
+  Manages encryption keys used for gossip messages between Nomad servers. Gossip
+  encryption is optional. When enabled, this command may be used to examine
+  active encryption keys in the cluster, add new keys, and remove old ones. When
+  combined, this functionality provides the ability to perform key rotation
+  cluster-wide, without disrupting the cluster.
+
+  All operations performed by this command can only be run against server nodes.
+
+  All variations of the keyring command return 0 if all nodes reply and there
+  are no errors. If any node fails to reply or reports failure, the exit code
+  will be 1.
+
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Keyring Options:
+
+  -install=<key>            Install a new encryption key. This will broadcast
+                            the new key to all members in the cluster.
+  -list                     List all keys currently in use within the cluster.
+  -remove=<key>             Remove the given key from the cluster. This
+                            operation may only be performed on keys which are
+                            not currently the primary key.
+  -use=<key>                Change the primary encryption key, which is used to
+                            encrypt messages. The key must already be installed
+                            before this operation can succeed.
+`
+	return strings.TrimSpace(helpText)
+}
+
+func (c *KeyringCommand) Synopsis() string {
+	return "Manages gossip layer encryption keys"
+}

commands.go

@@ -79,6 +79,16 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"keygen": func() (cli.Command, error) {
+			return &command.KeygenCommand{
+				Meta: meta,
+			}, nil
+		},
+		"keyring": func() (cli.Command, error) {
+			return &command.KeyringCommand{
+				Meta: meta,
+			}, nil
+		},
 		"logs": func() (cli.Command, error) {
 			return &command.LogsCommand{
 				Meta: meta,

nomad/server.go

@@ -935,3 +935,18 @@ func (s *Server) Stats() map[string]map[string]string {
 	}
 	return stats
 }
+
+// Region retuns the region of the server
+func (s *Server) Region() string {
+	return s.config.Region
+}
+
+// Datacenter returns the data center of the server
+func (s *Server) Datacenter() string {
+	return s.config.Datacenter
+}
+
+// GetConfig returns the config of the server for testing purposes only
+func (s *Server) GetConfig() *Config {
+	return s.config
+}

nomad/structs/structs.go

@@ -3613,3 +3613,16 @@ func Encode(t MessageType, msg interface{}) ([]byte, error) {
 	err := codec.NewEncoder(&buf, MsgpackHandle).Encode(msg)
 	return buf.Bytes(), err
 }
+
+// KeyringResponse is a unified key response and can be used for install,
+// remove, use, as well as listing key queries.
+type KeyringResponse struct {
+	Messages map[string]string
+	Keys     map[string]int
+	NumNodes int
+}
+
+// KeyringRequest is request objects for serf key operations.
+type KeyringRequest struct {
+	Key string
+}

vendor/github.com/hashicorp/memberlist/config.go

@@ -179,6 +179,11 @@ type Config struct {
 	// behavior for using LogOutput. You cannot specify both LogOutput and Logger
 	// at the same time.
 	Logger *log.Logger
+
+	// Size of Memberlist's internal channel which handles UDP messages. The
+	// size of this determines the size of the queue which Memberlist will keep
+	// while UDP messages are handled.
+	HandoffQueueDepth int
 }
 
 // DefaultLANConfig returns a sane set of configurations for Memberlist.
@@ -216,6 +221,8 @@ func DefaultLANConfig() *Config {
 		Keyring:   nil,
 
 		DNSConfigPath: "/etc/resolv.conf",
+
+		HandoffQueueDepth: 1024,
 	}
 }
 

vendor/github.com/hashicorp/memberlist/keyring.go

@@ -58,6 +58,17 @@ func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) {
 	return keyring, nil
 }
 
+// ValidateKey will check to see if the key is valid and returns an error if not.
+//
+// key should be either 16, 24, or 32 bytes to select AES-128,
+// AES-192, or AES-256.
+func ValidateKey(key []byte) error {
+	if l := len(key); l != 16 && l != 24 && l != 32 {
+		return fmt.Errorf("key size must be 16, 24 or 32 bytes")
+	}
+	return nil
+}
+
 // AddKey will install a new key on the ring. Adding a key to the ring will make
 // it available for use in decryption. If the key already exists on the ring,
 // this function will just return noop.
@@ -65,8 +76,8 @@ func NewKeyring(keys [][]byte, primaryKey []byte) (*Keyring, error) {
 // key should be either 16, 24, or 32 bytes to select AES-128,
 // AES-192, or AES-256.
 func (k *Keyring) AddKey(key []byte) error {
-	if l := len(key); l != 16 && l != 24 && l != 32 {
-		return fmt.Errorf("key size must be 16, 24 or 32 bytes")
+	if err := ValidateKey(key); err != nil {
+		return err
 	}
 
 	// No-op if key is already installed

vendor/github.com/hashicorp/memberlist/memberlist.go

@@ -129,7 +129,7 @@ func newMemberlist(conf *Config) (*Memberlist, error) {
 		leaveBroadcast: make(chan struct{}, 1),
 		udpListener:    udpLn,
 		tcpListener:    tcpLn,
-		handoff:        make(chan msgHandoff, 1024),
+		handoff:        make(chan msgHandoff, conf.HandoffQueueDepth),
 		nodeMap:        make(map[string]*nodeState),
 		nodeTimers:     make(map[string]*suspicion),
 		awareness:      newAwareness(conf.AwarenessMaxMultiplier),

vendor/vendor.json

@@ -686,10 +686,10 @@
 			"revision": "0dc08b1671f34c4250ce212759ebd880f743d883"
 		},
 		{
-			"checksumSHA1": "8ytOx52G+38QMK4G194Kl6g6YGY=",
+			"checksumSHA1": "Ozk/S4U1x/OllNP2SsMYJjCl/gs=",
 			"path": "github.com/hashicorp/memberlist",
-			"revision": "b2053e314b4a87e5f0d2d47aeafd3e03be13da90",
-			"revisionTime": "2016-06-21T23:59:43Z"
+			"revision": "7ad712f5f34ec40aebe6ca47756d07898486a8d2",
+			"revisionTime": "2016-09-15T13:02:55Z"
 		},
 		{
 			"path": "github.com/hashicorp/net-rpc-msgpackrpc",

website/source/docs/agent/config.html.md

@@ -392,6 +392,16 @@ configured on client nodes.
     join any nodes when it starts up. Addresses can be given as an IP, a domain
     name, or an IP:Port pair. If the port isn't specified the default Serf port,
     4648, is used.  DNS names may also be used.
+  * <a id="encrypt">`encrypt`</a> Specifies the secret key to use for encryption
+    of Nomad server's gossip network traffic. This key must be 16-bytes that are
+    Base64-encoded. The easiest way to create an encryption key is to use nomad
+    keygen. All the servers within a cluster must share the same encryption key
+    to communicate. The provided key is automatically persisted to the data
+    directory and loaded automatically whenever the agent is restarted. This
+    means that to encrypt Nomad server's gossip protocol, this option only needs
+    to be provided once on each agent's initial startup sequence. If it is
+    provided after Nomad has been initialized with an encryption key, then the
+    provided key is ignored and a warning will be displayed.
 
 ## Client-specific Options
 

website/source/docs/commands/keygen.html.md.erb

@@ -0,0 +1,30 @@
+---
+layout: "docs"
+page_title: "Commands: keygen"
+sidebar_current: "docs-commands-keygen"
+description: >
+  The `keygen` command generates an encryption key that can be used for Nomad
+  server's gossip traffic encryption. The keygen command uses a
+  cryptographically strong pseudo-random number generator to generate the key.
+---
+
+
+# Command: `keygen`
+
+The `keygen` command generates an encryption key that can be used for Nomad
+server's gossip traffic encryption.  The keygen command uses a cryptographically
+strong pseudo-random number generator to generate the key.
+
+## Usage 
+
+```
+nomad keygen
+```
+
+## Example
+
+```
+nomad keygen
+YgZOXLMhC7TtZqeghMT8+w==
+```
+

website/source/docs/commands/keyring.html.md.erb

@@ -0,0 +1,58 @@
+---
+layout: "docs"
+page_title: "Commands: keyring"
+sidebar_current: "docs-commands-keyring"
+---
+
+# Command: `keyring`
+
+The `keyring` command is used to examine and modify the encryption keys used in
+Nomad server. It is capable of distributing new encryption keys to the cluster,
+retiring old encryption keys, and changing the keys used by the cluster to
+encrypt messages.
+
+Nomad allows multiple encryption keys to be in use simultaneously. This is
+intended to provide a transition state while the cluster converges. It is the
+responsibility of the operator to ensure that only the required encryption keys
+are installed on the cluster. You can review the installed keys using the
+`-list` argument, and remove unneeded keys with `-remove`.
+
+All operations performed by this command can only be run against server nodes
+and will effect the entire cluster.
+
+All variations of the `keyring` command return 0 if all nodes reply and there
+are no errors. If any node fails to reply or reports failure, the exit code
+will be 1.
+
+
+## Usage
+
+Usage: `nomad keyring [options]`
+
+Only one actionable argument may be specified per run, including `-list`,
+`-install`, `-remove`, and `-use`.
+
+The list of available flags are:
+
+* `-list` - List all keys currently in use within the cluster.
+
+* `-install` - Install a new encryption key. This will broadcast the new key to
+  all members in the cluster.
+
+* `-use` - Change the primary encryption key, which is used to encrypt messages.
+  The key must already be installed before this operation can succeed.
+
+* `-remove` - Remove the given key from the cluster. This operation may only be
+  performed on keys which are not currently the primary key.
+
+## Output
+
+The output of the `nomad keyring -list` command consolidates information from
+all the Nomad servers from all datacenters and regions to provide a simple and
+easy to understand view of the cluster. 
+
+```
+==> Gathering installed encryption keys...
+Key
+PGm64/neoebUBqYR/lZTbA==
+```