agent: implement sd-notify reload correctly (#25636)

First of all, we should not send the unix time, but the monotonic time.
Second of all, RELOADING= and MONOTONIC_USEC fields should be sent in
*single* message not two separate messages.

From the man page of [systemd.service](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#Type=)

> notification message via sd_notify(3) that contains the "RELOADING=1" field in
> combination with "MONOTONIC_USEC=" set to the current monotonic time (i.e.
> CLOCK_MONOTONIC in clock_gettime(2)) in μs, formatted as decimal string.

[sd_notify](https://www.freedesktop.org/software/systemd/man/latest/sd_notify.html)
now has code samples of the protocol to clarify.

Without these changes, if you'd set
Type=notify-reload on the agen'ts systemd unit, systemd
would kill the service due to the service not responding to reload
correctly.

.changelog/25636.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+agent: Fixed a bug where reloading the agent with systemd notification enabled would cause the agent to be killed by system
+```

command/agent/command.go

@@ -978,16 +978,6 @@ func (c *Command) handleRetryJoin(config *Config) error {
 	return nil
 }
 
-// These constants are for readiness signalling via the systemd notify protocol.
-// The functions we send these messages to are no-op on non-Linux systems. See
-// also https://www.man7.org/linux/man-pages/man3/sd_notify.3.html
-const (
-	sdReady     = "READY=1"
-	sdReloading = "RELOADING=1"
-	sdStopping  = "STOPPING=1"
-	sdMonotonic = "MONOTONIC_USEC=%d"
-)
-
 // handleSignals blocks until we get an exit-causing signal
 func (c *Command) handleSignals() int {
 	signalCh := make(chan os.Signal, 4)
@@ -1026,8 +1016,7 @@ WAIT:
 
 	// Check if this is a SIGHUP
 	if sig == syscall.SIGHUP {
-		sdNotify(sdSock, sdReloading)
-		sdNotify(sdSock, fmt.Sprintf(sdMonotonic, time.Now().UnixMicro()))
+		sdNotifyReloading(sdSock)
 		c.handleReload()
 		sdNotify(sdSock, sdReady)
 		goto WAIT

command/agent/sdnotify.go

@@ -0,0 +1,12 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package agent
+
+// These constants are for readiness signalling via the systemd notify protocol.
+// The functions we send these messages to are no-op on non-Linux systems. See
+// also https://www.man7.org/linux/man-pages/man3/sd_notify.3.html
+const (
+	sdReady    = "READY=1"
+	sdStopping = "STOPPING=1"
+)

command/agent/sdnotify_default.go

@@ -14,3 +14,5 @@ func openNotify() (io.WriteCloser, error) {
 }
 
 func sdNotify(_ io.Writer, _ string) {}
+
+func sdNotifyReloading(_ io.Writer) {}

command/agent/sdnotify_linux.go

@@ -6,10 +6,13 @@
 package agent
 
 import (
+	"fmt"
 	"io"
 	"net"
 	"os"
 	"time"
+
+	"golang.org/x/sys/unix"
 )
 
 const sdNotifySocketEnvVar = "NOTIFY_SOCKET"
@@ -38,3 +41,9 @@ func sdNotify(w io.Writer, msg string) {
 	}
 	w.Write([]byte(msg))
 }
+
+func sdNotifyReloading(w io.Writer) {
+	var ts unix.Timespec
+	unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts)
+	sdNotify(w, fmt.Sprintf("RELOADING=1\nMONOTONIC_USEC=%d", ts.Nano()/1000))
+}