Add metric for expiring certs

Add certificate validity check
Fail on certificate failures and reset candlepin label for Prometheus gauge.
HMS-10103: add metric and alert for expiring certs in patch and update content-sources SOP

Author: swadeley

base/certutil/cert_expiry.go

@@ -0,0 +1,43 @@
+package certutil
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"time"
+)
+
+// ErrNoParseableCertificate is returned when the chain is non-empty but no DER entry could be parsed.
+var ErrNoParseableCertificate = errors.New("certutil: no parseable certificate in chain")
+
+// DaysTillExpiration returns whole days until the earliest NotAfter in the TLS certificate chain.
+// If every raw certificate fails to parse, it returns an error so callers do not treat a broken
+// configuration as “0 days left”.
+func DaysTillExpiration(certs *tls.Certificate) (int, error) {
+	expires := time.Time{}.UTC()
+	found := false
+	if certs == nil {
+		return 0, nil
+	}
+	var lastParseErr error
+	for _, tlsCert := range certs.Certificate {
+		parsed, err := x509.ParseCertificate(tlsCert)
+		if err != nil {
+			lastParseErr = err
+			continue
+		}
+		if !found || parsed.NotAfter.Before(expires) {
+			expires = parsed.NotAfter
+			found = true
+		}
+	}
+	if !found {
+		if lastParseErr != nil {
+			return 0, fmt.Errorf("certutil: parse certificate: %w", lastParseErr)
+		}
+		return 0, ErrNoParseableCertificate
+	}
+	diff := time.Until(expires)
+	return int(diff.Hours() / 24), nil
+}

base/certutil/cert_expiry_test.go

@@ -0,0 +1,63 @@
+package certutil
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDaysTillExpiration(t *testing.T) {
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	assert.NoError(t, err)
+
+	notBefore := time.Now().UTC().Add(-time.Hour)
+	notAfter := notBefore.Add(100 * 24 * time.Hour)
+
+	tmpl := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "test"},
+		NotBefore:    notBefore,
+		NotAfter:     notAfter,
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key)
+	assert.NoError(t, err)
+
+	cert := tls.Certificate{
+		Certificate: [][]byte{der},
+		PrivateKey:  key,
+	}
+
+	days, err := DaysTillExpiration(&cert)
+	assert.NoError(t, err)
+	assert.GreaterOrEqual(t, days, 99)
+	assert.LessOrEqual(t, days, 100)
+}
+
+func TestDaysTillExpirationNil(t *testing.T) {
+	days, err := DaysTillExpiration(nil)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, days)
+}
+
+func TestDaysTillExpirationUnparsableChain(t *testing.T) {
+	cert := tls.Certificate{
+		Certificate: [][]byte{[]byte("not valid der")},
+	}
+	_, err := DaysTillExpiration(&cert)
+	assert.Error(t, err)
+	assert.NotErrorIs(t, err, ErrNoParseableCertificate)
+}
+
+func TestDaysTillExpirationEmptyChain(t *testing.T) {
+	cert := tls.Certificate{Certificate: [][]byte{}}
+	_, err := DaysTillExpiration(&cert)
+	assert.ErrorIs(t, err, ErrNoParseableCertificate)
+}

base/metrics/certificate_expiry.go

@@ -0,0 +1,47 @@
+package metrics
+
+import (
+	"app/base/certutil"
+	"app/base/utils"
+	"crypto/tls"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const candlepinCertLabel = "candlepin"
+
+// CertificateExpiryDays mirrors content-sources-backend certificate_expiry_days (GaugeVec by label).
+// It is registered only where metrics are pushed (e.g. vmaas_sync), not on every pod's default registry.
+var CertificateExpiryDays = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+	Help:      "Number of days until the certificate expires by the certificate label",
+	Namespace: "patchman_engine",
+	Subsystem: "core",
+	Name:      "certificate_expiry_days",
+}, []string{"certificate_label"})
+
+// UpdateCandlepinCertificateExpiry refreshes the candlepin series from CoreCfg (single shot; no background loop).
+func UpdateCandlepinCertificateExpiry() {
+	applyCandlepinCertExpiry(CertificateExpiryDays, utils.CoreCfg.CandlepinCert, utils.CoreCfg.CandlepinKey)
+}
+
+// applyCandlepinCertExpiry refreshes or removes the candlepin expiry series. On parse/calculation
+// errors it deletes the label so Prometheus does not keep a stale last-good value.
+func applyCandlepinCertExpiry(gauge *prometheus.GaugeVec, certPEM, keyPEM string) {
+	if certPEM == "" || keyPEM == "" {
+		gauge.DeleteLabelValues(candlepinCertLabel)
+		return
+	}
+	cert, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))
+	if err != nil {
+		utils.LogError("err", err, "certificate_expiry: candlepin X509KeyPair")
+		gauge.DeleteLabelValues(candlepinCertLabel)
+		return
+	}
+	days, err := certutil.DaysTillExpiration(&cert)
+	if err != nil {
+		utils.LogError("err", err, "certificate_expiry: candlepin DaysTillExpiration")
+		gauge.DeleteLabelValues(candlepinCertLabel)
+		return
+	}
+	gauge.WithLabelValues(candlepinCertLabel).Set(float64(days))
+}

base/metrics/certificate_expiry_test.go

@@ -0,0 +1,96 @@
+package metrics
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testCandlepinPEMs(t *testing.T) (certPEM, keyPEM string) {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+
+	notBefore := time.Now().UTC().Add(-time.Hour)
+	notAfter := notBefore.Add(100 * 24 * time.Hour)
+	tmpl := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "test-candlepin"},
+		NotBefore:    notBefore,
+		NotAfter:     notAfter,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+
+	var certBuf, keyBuf bytes.Buffer
+	require.NoError(t, pem.Encode(&certBuf, &pem.Block{Type: "CERTIFICATE", Bytes: der}))
+	require.NoError(t, pem.Encode(&keyBuf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}))
+	return certBuf.String(), keyBuf.String()
+}
+
+func TestApplyCandlepinCertExpiry_setsGauge(t *testing.T) {
+	gv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "patchman_engine",
+		Subsystem: "core",
+		Name:      "certificate_expiry_days_test_helper",
+		Help:      "test",
+	}, []string{"certificate_label"})
+
+	certPEM, keyPEM := testCandlepinPEMs(t)
+	applyCandlepinCertExpiry(gv, certPEM, keyPEM)
+
+	v := testutil.ToFloat64(gv.WithLabelValues(candlepinCertLabel))
+	assert.GreaterOrEqual(t, v, 99.0)
+	assert.LessOrEqual(t, v, 100.0)
+}
+
+func TestApplyCandlepinCertExpiry_badPEMDeletesSeries(t *testing.T) {
+	gv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "patchman_engine",
+		Subsystem: "core",
+		Name:      "certificate_expiry_days_test_helper_bad",
+		Help:      "test",
+	}, []string{"certificate_label"})
+
+	certPEM, keyPEM := testCandlepinPEMs(t)
+	reg := prometheus.NewPedanticRegistry()
+	reg.MustRegister(gv)
+
+	applyCandlepinCertExpiry(gv, certPEM, keyPEM)
+	require.NotZero(t, testutil.ToFloat64(gv.WithLabelValues(candlepinCertLabel)))
+
+	applyCandlepinCertExpiry(gv, "not-valid-pem", "not-valid-pem")
+	n, err := testutil.GatherAndCount(reg)
+	require.NoError(t, err)
+	assert.Zero(t, n)
+}
+
+func TestApplyCandlepinCertExpiry_missingConfigDeletesSeries(t *testing.T) {
+	gv := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "patchman_engine",
+		Subsystem: "core",
+		Name:      "certificate_expiry_days_test_helper_clear",
+		Help:      "test",
+	}, []string{"certificate_label"})
+
+	reg := prometheus.NewPedanticRegistry()
+	reg.MustRegister(gv)
+
+	certPEM, keyPEM := testCandlepinPEMs(t)
+	applyCandlepinCertExpiry(gv, certPEM, keyPEM)
+	applyCandlepinCertExpiry(gv, "", "")
+	n, err := testutil.GatherAndCount(reg)
+	require.NoError(t, err)
+	assert.Zero(t, n)
+}

deploy/clowdapp.yaml

@@ -343,6 +343,8 @@ objects:
         - {name: SSL_CERT_DIR, value: '${SSL_CERT_DIR}'}
         - {name: GOMEMLIMIT, value: '${GOMEMLIMIT_VMAAS_SYNC}'}
         - {name: POD_CONFIG, value: '${JOBS_CONFIG}'}
+        - {name: CANDLEPIN_CERT, valueFrom: {secretKeyRef: {name: candlepin, key: cert}}}
+        - {name: CANDLEPIN_KEY, valueFrom: {secretKeyRef: {name: candlepin, key: key}}}
         resources:
           limits: {cpu: '${CPU_LIMIT_VMAAS_SYNC}', memory: '${MEM_LIMIT_VMAAS_SYNC}'}
           requests: {cpu: '${CPU_REQUEST_VMAAS_SYNC}', memory: '${MEM_REQUEST_VMAAS_SYNC}'}

go.mod

@@ -87,6 +87,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/mailru/easyjson v0.9.2 // indirect

tasks/vmaas_sync/metrics.go

@@ -2,6 +2,7 @@ package vmaas_sync
 
 import (
 	"app/base/database"
+	"app/base/metrics"
 	"app/base/models"
 	"app/base/utils"
 	"app/tasks"
@@ -104,7 +105,7 @@ func Metrics() *push.Pusher {
 	registry.MustRegister(vmaasCallCnt, storeAdvisoriesCnt, storePackagesCnt,
 		systemsCnt, advisoriesCnt, systemAdvisoriesStats, syncDuration, messageSendDuration, packageCnt, packageNameCnt,
 		databaseSizeBytesGaugeVec, databaseProcessesGaugeVec, systemsCntByType, tagsCntByType,
-		advisoriesCountMismatch)
+		advisoriesCountMismatch, metrics.CertificateExpiryDays)
 
 	// update advanced metrics
 	update()
@@ -119,6 +120,7 @@ func update() {
 	updateSystemAdvisoriesStats()
 	updateDBMetrics()
 	updateSystemInventoryData()
+	metrics.UpdateCandlepinCertificateExpiry()
 }
 
 func updateSystemMetrics() {