observer: add CCADB CRL prober (#8644)

Prior work: letsencrypt/crl-monitor#88

Fixes #8618

Author: jsha

cmd/boulder-observer/README.md

@@ -31,6 +31,9 @@ Prometheus.
       * [TLS](#tls)
         * [Schema](#schema-6)
         * [Example](#example-6)
+      * [CCADB](#ccadb)
+        * [Schema](#schema-7)
+        * [Example](#example-7)
   * [Metrics](#metrics)
     * [Global Metrics](#global-metrics)
       * [obs_monitors](#obs_monitors)
@@ -255,6 +258,38 @@ monitors:
       response: valid
 ```
 
+#### CCADB
+
+##### Schema
+
+`allCertificatesCSVURL`: URL of the "V4 All Certificate Information (root and
+  intermediate) in CCADB (CSV)" report from https://www.ccadb.org/resources.
+  Default value works.
+
+`certificatePEMsURL`: Base URL of the "All Certificate PEMs" report from
+  https://www.ccadb.org/resources (i.e. without the "NotBeforeDecade"
+  parameter). Default value works.
+
+`caOwner`: The value of the "CA Owner" field to filter on in the "All
+  Certificate Information" report. Default value works for ISRG.
+
+`crlAgeLimit`: Error when a CRL is older than this.
+
+`crlRegexp`: A regexp that matches our CRL URLs. Prevents fetching arbitrary
+  URLs. At a minimum this should have strict matching on the origin part of the
+  URL. Default value works.
+
+##### Example
+
+```yaml
+monitors:
+  -
+    period: 1h
+    kind: CCADB
+    settings:
+      crlAgeLimit: 2h
+```
+
 ## Metrics
 
 Observer provides the following metrics.
@@ -442,4 +477,4 @@ prometheus --config.file=boulder/test/prometheus/prometheus.yml
 ### Viewing metrics locally
 
 When developing with a local Prometheus instance you can use this link
-to view metrics: [link](http://0.0.0.0:9090)
+to view metrics: [link](http://0.0.0.0:9090)

cmd/boulder-observer/main.go

@@ -13,6 +13,8 @@ import (
 )
 
 func main() {
+	defer cmd.AuditPanic()
+
 	debugAddr := flag.String("debug-addr", "", "Debug server address override")
 	configPath := flag.String(
 		"config", "config.yml", "Path to boulder-observer configuration file")

observer/mon_conf.go

@@ -14,7 +14,7 @@ import (
 // MonConf is exported to receive YAML configuration in `ObsConf`.
 type MonConf struct {
 	Period   config.Duration  `yaml:"period"`
-	Kind     string           `yaml:"kind" validate:"required,oneof=DNS HTTP CRL TLS AIA"`
+	Kind     string           `yaml:"kind" validate:"required,oneof=DNS HTTP CRL TLS AIA CCADB"`
 	Settings probers.Settings `yaml:"settings" validate:"min=1,dive"`
 }
 

observer/obs_conf.go

@@ -99,7 +99,6 @@ func (c *ObsConf) MakeObserver() (*Observer, error) {
 		}, []string{"name", "kind", "success"})
 	metrics.MustRegister(countMonitors)
 	metrics.MustRegister(histObservations)
-	defer cmd.AuditPanic()
 	cmd.LogStartup(logger)
 	logger.Infof("Initializing boulder-observer daemon")
 	logger.Debugf("Using config: %+v", c)

observer/observer.go

@@ -6,6 +6,7 @@ import (
 	"github.com/letsencrypt/boulder/cmd"
 	blog "github.com/letsencrypt/boulder/log"
 	_ "github.com/letsencrypt/boulder/observer/probers/aia"
+	_ "github.com/letsencrypt/boulder/observer/probers/ccadb"
 	_ "github.com/letsencrypt/boulder/observer/probers/crl"
 	_ "github.com/letsencrypt/boulder/observer/probers/dns"
 	_ "github.com/letsencrypt/boulder/observer/probers/http"