metrics: emit stats for vault token next_renewal & last_renewal #5222 (#12435)

nomad/vault.go

@@ -153,6 +153,14 @@ type VaultStats struct {
 
 	// TokenExpiry is the recorded expiry time of the current token
 	TokenExpiry time.Time
+
+	// LastRenewalTime is the time since the token was last renewed
+	LastRenewalTime     time.Time
+	TimeFromLastRenewal time.Duration
+
+	// NextRenewalTime is the time the token will attempt to renew
+	NextRenewalTime   time.Time
+	TimeToNextRenewal time.Duration
 }
 
 // PurgeVaultAccessorFn is called to remove VaultAccessors from the system. If
@@ -222,6 +230,9 @@ type vaultClient struct {
 	// currentExpiration is the time the current token lease expires
 	currentExpiration     time.Time
 	currentExpirationLock sync.Mutex
+	lastRenewalTime       time.Time
+	nextRenewalTime       time.Time
+	renewalTimeLock       sync.Mutex
 
 	tomb   *tomb.Tomb
 	logger log.Logger
@@ -547,6 +558,11 @@ func (v *vaultClient) renewalLoop() {
 			if err == nil {
 				// Attempt to renew the token at half the expiration time
 				durationUntilRenew := time.Until(currentExpiration) / 2
+				v.renewalTimeLock.Lock()
+				now := time.Now()
+				v.lastRenewalTime = now
+				v.nextRenewalTime = now.Add(durationUntilRenew)
+				v.renewalTimeLock.Unlock()
 
 				v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew)
 				authRenewTimer.Reset(durationUntilRenew)
@@ -577,6 +593,9 @@ func (v *vaultClient) renewalLoop() {
 			}
 
 			durationUntilRetry := time.Duration(backoff) * time.Second
+			v.renewalTimeLock.Lock()
+			v.nextRenewalTime = time.Now().Add(durationUntilRetry)
+			v.renewalTimeLock.Unlock()
 			v.logger.Info("backing off renewal", "retry", durationUntilRetry)
 
 			authRenewTimer.Reset(durationUntilRetry)
@@ -1407,15 +1426,26 @@ func (v *vaultClient) Stats() map[string]string {
 	stat := v.stats()
 
 	expireTimeStr := ""
-
 	if !stat.TokenExpiry.IsZero() {
 		expireTimeStr = stat.TokenExpiry.Format(time.RFC3339)
 	}
 
+	lastRenewTimeStr := ""
+	if !stat.LastRenewalTime.IsZero() {
+		lastRenewTimeStr = stat.LastRenewalTime.Format(time.RFC3339)
+	}
+
+	nextRenewTimeStr := ""
+	if !stat.NextRenewalTime.IsZero() {
+		nextRenewTimeStr = stat.NextRenewalTime.Format(time.RFC3339)
+	}
+
 	return map[string]string{
-		"tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke),
-		"token_ttl":           stat.TokenTTL.Round(time.Second).String(),
-		"token_expire_time":   expireTimeStr,
+		"tracked_for_revoked":     strconv.Itoa(stat.TrackedForRevoke),
+		"token_ttl":               stat.TokenTTL.Round(time.Second).String(),
+		"token_expire_time":       expireTimeStr,
+		"token_last_renewal_time": lastRenewTimeStr,
+		"token_next_renewal_time": nextRenewTimeStr,
 	}
 }
 
@@ -1431,10 +1461,22 @@ func (v *vaultClient) stats() *VaultStats {
 	stats.TokenExpiry = v.currentExpiration
 	v.currentExpirationLock.Unlock()
 
+	v.renewalTimeLock.Lock()
+	stats.NextRenewalTime = v.nextRenewalTime
+	stats.LastRenewalTime = v.lastRenewalTime
+	v.renewalTimeLock.Unlock()
+
 	if !stats.TokenExpiry.IsZero() {
 		stats.TokenTTL = time.Until(stats.TokenExpiry)
 	}
 
+	if !stats.LastRenewalTime.IsZero() {
+		stats.TimeFromLastRenewal = time.Since(stats.LastRenewalTime)
+	}
+	if !stats.NextRenewalTime.IsZero() {
+		stats.TimeToNextRenewal = time.Until(stats.NextRenewalTime)
+	}
+
 	return stats
 }
 
@@ -1451,6 +1493,8 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) {
 			stats := v.stats()
 			metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke))
 			metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond))
+			metrics.SetGauge([]string{"nomad", "vault", "token_last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond))
+			metrics.SetGauge([]string{"nomad", "vault", "token_next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond))
 
 		case <-stopCh:
 			return