Expose issuer cert TTL via log and prometheus

Add a prometheus gauge function in the identity package that exposes
the current TTL in seconds of the issuer certificate.

When a new issuer certificate is loaded, log its NotAfter time
in unix epoch format, along with the current process wall clock time.

This addresses #11215

Signed-off-by: Nathan J. Mehl <[email protected]>

pkg/identity/service.go

@@ -17,6 +17,7 @@ import (
 
 	pb "github.com/linkerd/linkerd2-proxy-api/go/identity"
 	"github.com/linkerd/linkerd2/pkg/tls"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -50,6 +51,7 @@ type (
 		recordEvent  func(parent runtime.Object, eventType, reason, message string)
 
 		expectedName, issuerPathCrt, issuerPathKey string
+		issuerCertExpiration                       time.Time
 	}
 
 	// Validator implementors accept a bearer token, validates it, and returns a
@@ -82,6 +84,7 @@ func (svc *Service) Initialize() error {
 		return err
 	}
 	svc.updateIssuer(credentials)
+	svc.registerCertExpirationMetrics()
 	return nil
 }
 
@@ -92,6 +95,37 @@ func (svc *Service) updateIssuer(newIssuer tls.Issuer) {
 	svc.issuerMutex.Unlock()
 }
 
+func (svc *Service) getIssuerCertTTL() float64 {
+	if svc.issuer == nil {
+		log.Warn("Certificate issuer is not ready; cannot get expiration")
+		return float64(0)
+	}
+	if svc.issuerCertExpiration.IsZero() {
+		log.Warn("Certificate issuer expiration is not set; cannot get expiration")
+		return float64(0)
+	}
+	return time.Until(svc.issuerCertExpiration).Seconds()
+}
+
+func (svc *Service) getTrustAnchorCertTTL() float64 {
+	if svc.trustAnchors == nil {
+		log.Warn("Trust anchors are not loaded; cannot get expiration")
+		return float64(0)
+	}
+	var minTTL time.Time
+	for _, certBytes := range svc.trustAnchors.Subjects() {
+		cert, err := x509.ParseCertificate(certBytes)
+		if err != nil {
+			log.Errorf("could not parse trust anchor certificate: %s", err)
+			continue
+		}
+		if minTTL.IsZero() || cert.NotAfter.Before(minTTL) {
+			minTTL = cert.NotAfter
+		}
+	}
+	return time.Until(minTTL).Seconds()
+}
+
 // Run reads from the issuer and error channels and reloads the issuer certs when necessary
 func (svc *Service) Run(issuerEvent <-chan struct{}, issuerError <-chan error) {
 	for {
@@ -131,10 +165,47 @@ func (svc *Service) loadCredentials() (tls.Issuer, error) {
 		return nil, fmt.Errorf("failed to verify issuer certificate: it must be an intermediate-CA, but it is not")
 	}
 
+	svc.issuerCertExpiration = creds.Certificate.NotAfter
+
 	log.Debugf("Loaded issuer cert: %s", creds.EncodeCertificatePEM())
+	now := time.Now().Unix()
+	log.WithFields(log.Fields{
+		"invalid_after":      creds.Certificate.NotAfter.Unix(),
+		"process_clock_time": now,
+		"ttl_seconds":        creds.Certificate.NotAfter.Unix() - now,
+	}).Info("Issuer cert loaded")
 	return tls.NewCA(*creds, *svc.validity), nil
 }
 
+func (svc *Service) registerCertExpirationMetrics() {
+	// register a metric for the expiration of the issuer cert
+	issuerCertExpireGauge := prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name: "issuer_cert_ttl_seconds",
+		Help: "The remaining seconds until the issuer certificate expires",
+	}, svc.getIssuerCertTTL)
+	if err := prometheus.Register(issuerCertExpireGauge); err != nil {
+		var are prometheus.AlreadyRegisteredError
+		if errors.As(err, &are) {
+			log.Warn("issuer_cert_ttl_seconds metric already registered")
+		} else {
+			log.WithError(err).Warn("failed to register issuer_cert_ttl_seconds metric")
+		}
+	}
+	// register a metric for the expiration of the trust anchor cert with the lowest TTL
+	trustAnchorExpireGauge := prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name: "trust_anchor_ttl_seconds",
+		Help: "The remaining seconds until the trust anchor expires",
+	}, svc.getTrustAnchorCertTTL)
+	if err := prometheus.Register(trustAnchorExpireGauge); err != nil {
+		var are prometheus.AlreadyRegisteredError
+		if errors.As(err, &are) {
+			log.Warn("trust_anchor_ttl_seconds metric already registered")
+		} else {
+			log.WithError(err).Warn("failed to register trust_anchor_ttl_seconds metric")
+		}
+	}
+}
+
 // NewService creates a new identity service.
 func NewService(validator Validator, trustAnchors *x509.CertPool, validity *tls.Validity, recordEvent func(parent runtime.Object, eventType, reason, message string), expectedName, issuerPathCrt, issuerPathKey string) *Service {
 	return &Service{
@@ -148,6 +219,7 @@ func NewService(validator Validator, trustAnchors *x509.CertPool, validity *tls.
 		expectedName,
 		issuerPathCrt,
 		issuerPathKey,
+		time.Time{},
 	}
 }