diff --git a/nomad/vault.go b/nomad/vault.go index e63deb525..8f93a4f91 100644 --- a/nomad/vault.go +++ b/nomad/vault.go @@ -153,6 +153,14 @@ type VaultStats struct { // TokenExpiry is the recorded expiry time of the current token TokenExpiry time.Time + + // LastRenewalTime is the time since the token was last renewed + LastRenewalTime time.Time + TimeFromLastRenewal time.Duration + + // NextRenewalTime is the time the token will attempt to renew + NextRenewalTime time.Time + TimeToNextRenewal time.Duration } // PurgeVaultAccessorFn is called to remove VaultAccessors from the system. If @@ -222,6 +230,9 @@ type vaultClient struct { // currentExpiration is the time the current token lease expires currentExpiration time.Time currentExpirationLock sync.Mutex + lastRenewalTime time.Time + nextRenewalTime time.Time + renewalTimeLock sync.Mutex tomb *tomb.Tomb logger log.Logger @@ -547,6 +558,11 @@ func (v *vaultClient) renewalLoop() { if err == nil { // Attempt to renew the token at half the expiration time durationUntilRenew := time.Until(currentExpiration) / 2 + v.renewalTimeLock.Lock() + now := time.Now() + v.lastRenewalTime = now + v.nextRenewalTime = now.Add(durationUntilRenew) + v.renewalTimeLock.Unlock() v.logger.Info("successfully renewed token", "next_renewal", durationUntilRenew) authRenewTimer.Reset(durationUntilRenew) @@ -577,6 +593,9 @@ func (v *vaultClient) renewalLoop() { } durationUntilRetry := time.Duration(backoff) * time.Second + v.renewalTimeLock.Lock() + v.nextRenewalTime = time.Now().Add(durationUntilRetry) + v.renewalTimeLock.Unlock() v.logger.Info("backing off renewal", "retry", durationUntilRetry) authRenewTimer.Reset(durationUntilRetry) @@ -1407,15 +1426,26 @@ func (v *vaultClient) Stats() map[string]string { stat := v.stats() expireTimeStr := "" - if !stat.TokenExpiry.IsZero() { expireTimeStr = stat.TokenExpiry.Format(time.RFC3339) } + lastRenewTimeStr := "" + if !stat.LastRenewalTime.IsZero() { + lastRenewTimeStr = stat.LastRenewalTime.Format(time.RFC3339) + } + + nextRenewTimeStr := "" + if !stat.NextRenewalTime.IsZero() { + nextRenewTimeStr = stat.NextRenewalTime.Format(time.RFC3339) + } + return map[string]string{ - "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), - "token_ttl": stat.TokenTTL.Round(time.Second).String(), - "token_expire_time": expireTimeStr, + "tracked_for_revoked": strconv.Itoa(stat.TrackedForRevoke), + "token_ttl": stat.TokenTTL.Round(time.Second).String(), + "token_expire_time": expireTimeStr, + "token_last_renewal_time": lastRenewTimeStr, + "token_next_renewal_time": nextRenewTimeStr, } } @@ -1431,10 +1461,22 @@ func (v *vaultClient) stats() *VaultStats { stats.TokenExpiry = v.currentExpiration v.currentExpirationLock.Unlock() + v.renewalTimeLock.Lock() + stats.NextRenewalTime = v.nextRenewalTime + stats.LastRenewalTime = v.lastRenewalTime + v.renewalTimeLock.Unlock() + if !stats.TokenExpiry.IsZero() { stats.TokenTTL = time.Until(stats.TokenExpiry) } + if !stats.LastRenewalTime.IsZero() { + stats.TimeFromLastRenewal = time.Since(stats.LastRenewalTime) + } + if !stats.NextRenewalTime.IsZero() { + stats.TimeToNextRenewal = time.Until(stats.NextRenewalTime) + } + return stats } @@ -1451,6 +1493,8 @@ func (v *vaultClient) EmitStats(period time.Duration, stopCh <-chan struct{}) { stats := v.stats() metrics.SetGauge([]string{"nomad", "vault", "distributed_tokens_revoking"}, float32(stats.TrackedForRevoke)) metrics.SetGauge([]string{"nomad", "vault", "token_ttl"}, float32(stats.TokenTTL/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "token_last_renewal"}, float32(stats.TimeFromLastRenewal/time.Millisecond)) + metrics.SetGauge([]string{"nomad", "vault", "token_next_renewal"}, float32(stats.TimeToNextRenewal/time.Millisecond)) case <-stopCh: return