Add EvictCallbackFn to handle removing entries from go-memdb when they

are removed from the event buffer.

Wire up event buffer size config, use pointers for structs.Events
instead of copying.

command/agent/agent.go

@@ -243,8 +243,11 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) {
 	if agentConfig.Server.UpgradeVersion != "" {
 		conf.UpgradeVersion = agentConfig.Server.UpgradeVersion
 	}
-	if agentConfig.Server.EnableEventPublisher {
-		conf.EnableEventPublisher = agentConfig.Server.EnableEventPublisher
+	if agentConfig.Server.EnableEventPublisher != nil {
+		conf.EnableEventPublisher = *agentConfig.Server.EnableEventPublisher
+	}
+	if agentConfig.Server.EventBufferSize > 0 {
+		conf.EventBufferSize = int64(agentConfig.Server.EventBufferSize)
 	}
 	if agentConfig.Autopilot != nil {
 		if agentConfig.Autopilot.CleanupDeadServers != nil {

command/agent/command.go

@@ -94,7 +94,6 @@ func (c *Command) readConfig() *Config {
 	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.ServerJoin.StartJoin), "join", "")
 	flags.Var((*flaghelper.StringFlag)(&cmdConfig.Server.ServerJoin.RetryJoin), "retry-join", "")
 	flags.IntVar(&cmdConfig.Server.ServerJoin.RetryMaxAttempts, "retry-max", 0, "")
-	flags.BoolVar(&cmdConfig.Server.EnableEventPublisher, "event-publisher", false, "")
 	flags.Var((flaghelper.FuncDurationVar)(func(d time.Duration) error {
 		cmdConfig.Server.ServerJoin.RetryInterval = d
 		return nil
@@ -597,7 +596,6 @@ func (c *Command) AutocompleteFlags() complete.Flags {
 		"-vault-tls-server-name":         complete.PredictAnything,
 		"-acl-enabled":                   complete.PredictNothing,
 		"-acl-replication-token":         complete.PredictAnything,
-		"-event-publisher":               complete.PredictNothing,
 	}
 }
 
@@ -1280,9 +1278,6 @@ Server Options:
   -rejoin
     Ignore a previous leave and attempts to rejoin the cluster.
 
-  -event-publisher
-    Whether to enable or disable the servers event publisher.
-
 Client Options:
 
   -client

command/agent/config.go

@@ -486,7 +486,12 @@ type ServerConfig struct {
 
 	// EnableEventPublisher configures whether this server's state store
 	// will generate events for its event stream.
-	EnableEventPublisher bool `hcl:"enable_event_publisher"`
+	EnableEventPublisher *bool `hcl:"enable_event_publisher"`
+
+	// EventBufferSize configure the amount of events to be held in memory.
+	// If EnableEventPublisher is set to true, the minimum allowable value
+	// for the EventBufferSize is 1.
+	EventBufferSize int `hcl:"event_buffer_size"`
 
 	// DurableEventCount specifies the amount of events to persist during snapshot generation.
 	// A count of 0 signals that no events should be persisted.
@@ -883,7 +888,8 @@ func DefaultConfig() *Config {
 		},
 		Server: &ServerConfig{
 			Enabled:              false,
-			EnableEventPublisher: true,
+			EnableEventPublisher: helper.BoolToPtr(true),
+			EventBufferSize:      100,
 			DurableEventCount:    100,
 			StartJoin:            []string{},
 			ServerJoin: &ServerJoin{
@@ -1409,8 +1415,12 @@ func (a *ServerConfig) Merge(b *ServerConfig) *ServerConfig {
 		result.ServerJoin = result.ServerJoin.Merge(b.ServerJoin)
 	}
 
-	if b.EnableEventPublisher {
-		result.EnableEventPublisher = true
+	if b.EnableEventPublisher != nil {
+		result.EnableEventPublisher = b.EnableEventPublisher
+	}
+
+	if b.EventBufferSize != 0 {
+		result.EventBufferSize = b.EventBufferSize
 	}
 
 	if b.DurableEventCount != 0 {

command/agent/config_parse_test.go

@@ -122,7 +122,8 @@ var basicConfig = &Config{
 		RedundancyZone:            "foo",
 		UpgradeVersion:            "0.8.0",
 		EncryptKey:                "abc",
-		EnableEventPublisher:      true,
+		EnableEventPublisher:      helper.BoolToPtr(false),
+		EventBufferSize:           200,
 		DurableEventCount:         100,
 		ServerJoin: &ServerJoin{
 			RetryJoin:        []string{"1.1.1.1", "2.2.2.2"},

command/agent/config_test.go

@@ -138,6 +138,7 @@ func TestConfig_Merge(t *testing.T) {
 			MaxHeartbeatsPerSecond: 30.0,
 			RedundancyZone:         "foo",
 			UpgradeVersion:         "foo",
+			EnableEventPublisher:   helper.BoolToPtr(false),
 		},
 		ACL: &ACLConfig{
 			Enabled:          true,
@@ -328,6 +329,7 @@ func TestConfig_Merge(t *testing.T) {
 			NonVotingServer:        true,
 			RedundancyZone:         "bar",
 			UpgradeVersion:         "bar",
+			EnableEventPublisher:   helper.BoolToPtr(true),
 		},
 		ACL: &ACLConfig{
 			Enabled:          true,
@@ -1163,3 +1165,41 @@ func TestTelemetry_Parse(t *testing.T) {
 	require.Exactly([]string{"+nomad.raft"}, config.Telemetry.PrefixFilter)
 	require.True(config.Telemetry.DisableDispatchedJobSummaryMetrics)
 }
+
+func TestEventPublisher_Parse(t *testing.T) {
+
+	require := require.New(t)
+
+	{
+		a := &ServerConfig{
+			EnableEventPublisher: helper.BoolToPtr(false),
+		}
+		b := DefaultConfig().Server
+		b.EnableEventPublisher = nil
+
+		result := a.Merge(b)
+		require.Equal(false, *result.EnableEventPublisher)
+	}
+
+	{
+		a := &ServerConfig{
+			EnableEventPublisher: helper.BoolToPtr(true),
+		}
+		b := DefaultConfig().Server
+		b.EnableEventPublisher = nil
+
+		result := a.Merge(b)
+		require.Equal(true, *result.EnableEventPublisher)
+	}
+
+	{
+		a := &ServerConfig{
+			EnableEventPublisher: helper.BoolToPtr(false),
+		}
+		b := DefaultConfig().Server
+		b.EnableEventPublisher = helper.BoolToPtr(true)
+
+		result := a.Merge(b)
+		require.Equal(true, *result.EnableEventPublisher)
+	}
+}

command/agent/event_endpoint.go

@@ -74,7 +74,7 @@ func (s *HTTPServer) EventStream(resp http.ResponseWriter, req *http.Request) (i
 	output := ioutils.NewWriteFlusher(resp)
 
 	// create an error channel to handle errors
-	errCh := make(chan HTTPCodedError, 2)
+	errCh := make(chan HTTPCodedError, 1)
 
 	go func() {
 		defer cancel()
@@ -124,9 +124,7 @@ func (s *HTTPServer) EventStream(resp http.ResponseWriter, req *http.Request) (i
 	cancel()
 	codedErr := <-errCh
 
-	if codedErr != nil &&
-		(codedErr == io.EOF ||
-			strings.Contains(codedErr.Error(), io.ErrClosedPipe.Error())) {
+	if codedErr != nil && strings.Contains(codedErr.Error(), io.ErrClosedPipe.Error()) {
 		codedErr = nil
 	}
 

command/agent/event_endpoint_test.go

@@ -39,7 +39,7 @@ func TestEventStream(t *testing.T) {
 
 		pub, err := s.Agent.server.State().EventPublisher()
 		require.NoError(t, err)
-		pub.Publish(structs.Events{Index: 100, Events: []structs.Event{{Payload: testEvent{ID: "123"}}}})
+		pub.Publish(&structs.Events{Index: 100, Events: []structs.Event{{Payload: testEvent{ID: "123"}}}})
 
 		testutil.WaitForResult(func() (bool, error) {
 			got := resp.Body.String()
@@ -56,7 +56,6 @@ func TestEventStream(t *testing.T) {
 
 		// wait for response to close to prevent race between subscription
 		// shutdown and server shutdown returning subscription closed by server err
-		// resp.Close()
 		cancel()
 		select {
 		case err := <-respErrCh:

command/agent/testdata/basic.hcl

@@ -130,7 +130,8 @@ server {
   upgrade_version               = "0.8.0"
   encrypt                       = "abc"
   raft_multiplier               = 4
-  enable_event_publisher        = true
+  enable_event_publisher        = false
+  event_buffer_size             = 200
   durable_event_count           = 100
 
   server_join {

command/agent/testdata/basic.json

@@ -261,7 +261,8 @@
       "data_dir": "/tmp/data",
       "deployment_gc_threshold": "12h",
       "enabled": true,
-      "enable_event_publisher": true,
+      "enable_event_publisher": false,
+      "event_buffer_size": 200,
       "durable_event_count": 100,
       "enabled_schedulers": [
         "test"