Add keyHashToSerial cleanup to boulder-janitor (letsencrypt#5194)

aarongable · web-flow · commit cb6effad9faa · 2020-12-10T16:46:07.000-08:00
This adds a new job type (and corresponding config) to the janitor to clean up old rows from the `keyHashToSerial` table. Rows in this table are no longer relevant after their corresponding certificate has expired. Fixes letsencrypt#4792
diff --git a/cmd/boulder-janitor/config.go b/cmd/boulder-janitor/config.go
@@ -71,6 +71,9 @@ type Config struct {
 		// CertificatesPerName describes a cleanup job for the certificatesPerName table.
 		CertificatesPerName CleanupConfig
 
+		// KeyHashToSerial describes a cleanup job for the keyHashToSerial table.
+		KeyHashToSerial CleanupConfig
+
 		// Orders describes a cleanup job for the orders table and related rows
 		// (requestedNames, orderToAuthz2, orderFqdnSets).
 		Orders CleanupConfig
@@ -83,7 +86,13 @@ func (c Config) Valid() error {
 	if c.Janitor.DebugAddr == "" {
 		return errEmptyMetricsAddr
 	}
-	jobConfigs := []CleanupConfig{c.Janitor.Certificates, c.Janitor.CertificateStatus, c.Janitor.CertificatesPerName}
+	jobConfigs := []CleanupConfig{
+		c.Janitor.Certificates,
+		c.Janitor.CertificateStatus,
+		c.Janitor.CertificatesPerName,
+		c.Janitor.KeyHashToSerial,
+		c.Janitor.Orders,
+	}
 	for _, cc := range jobConfigs {
 		if err := cc.Valid(); err != nil {
 			return err
diff --git a/cmd/boulder-janitor/janitor.go b/cmd/boulder-janitor/janitor.go
@@ -96,6 +96,9 @@ func newJobs(
 		}
 		jobs = append(jobs, newCertificatesPerNameJob(dbMap, logger, clk, config))
 	}
+	if config.Janitor.KeyHashToSerial.Enabled {
+		jobs = append(jobs, newKeyHashToSerialJob(dbMap, logger, clk, config))
+	}
 	if config.Janitor.Orders.Enabled {
 		jobs = append(jobs, newOrdersJob(dbMap, logger, clk, config.Janitor.Orders))
 	}
diff --git a/cmd/boulder-janitor/keyhashtoserial.go b/cmd/boulder-janitor/keyhashtoserial.go
@@ -0,0 +1,34 @@
+package main
+
+import (
+	"github.com/jmhodges/clock"
+	"github.com/letsencrypt/boulder/db"
+	blog "github.com/letsencrypt/boulder/log"
+)
+
+// newKeyHashToSerialJob returns a batchedDBJob configured to delete expired
+// rows from the keyHashToSerial table.
+func newKeyHashToSerialJob(
+	dbMap db.DatabaseMap,
+	log blog.Logger,
+	clk clock.Clock,
+	config Config) *batchedDBJob {
+	purgeBefore := config.Janitor.KeyHashToSerial.GracePeriod.Duration
+	workQuery := `SELECT id, certNotAfter AS expires FROM keyHashToSerial
+		 WHERE
+		   id > :startID
+		 LIMIT :limit`
+	log.Debugf("Creating KeyHashToSerial job from config: %#v", config.Janitor.KeyHashToSerial)
+	return &batchedDBJob{
+		db:          dbMap,
+		log:         log,
+		clk:         clk,
+		purgeBefore: purgeBefore,
+		workSleep:   config.Janitor.KeyHashToSerial.WorkSleep.Duration,
+		batchSize:   config.Janitor.KeyHashToSerial.BatchSize,
+		maxDPS:      config.Janitor.KeyHashToSerial.MaxDPS,
+		parallelism: config.Janitor.KeyHashToSerial.Parallelism,
+		table:       "keyHashToSerial",
+		workQuery:   workQuery,
+	}
+}
diff --git a/test/config-next/janitor.json b/test/config-next/janitor.json
@@ -30,6 +30,14 @@
         "parallelism": 2,
         "maxDPS": 50
     },
+    "keyHashToSerial": {
+        "enabled": true,
+        "gracePeriod": "2184h",
+        "batchSize": 100,
+        "workSleep": "500ms",
+        "parallelism": 2,
+        "maxDPS": 50
+    },
     "orders": {
         "enabled": true,
         "gracePeriod": "2184h",
diff --git a/test/config/janitor.json b/test/config/janitor.json
@@ -30,6 +30,14 @@
         "parallelism": 2,
         "maxDPS": 50
     },
+    "keyHashToSerial": {
+        "enabled": true,
+        "gracePeriod": "2184h",
+        "batchSize": 100,
+        "workSleep": "500ms",
+        "parallelism": 2,
+        "maxDPS": 50
+    },
     "orders": {
         "enabled": true,
         "gracePeriod": "2184h",
diff --git a/test/sa_db_users.sql b/test/sa_db_users.sql
@@ -63,6 +63,7 @@ GRANT SELECT,DELETE ON authz2 TO 'purger'@'localhost';
 GRANT SELECT,DELETE ON certificates TO 'janitor'@'localhost';
 GRANT SELECT,DELETE ON certificateStatus TO 'janitor'@'localhost';
 GRANT SELECT,DELETE ON certificatesPerName TO 'janitor'@'localhost';
+GRANT SELECT,DELETE ON keyHashToSerial TO 'janitor'@'localhost';
 GRANT SELECT,DELETE ON orders TO 'janitor'@'localhost';
 GRANT SELECT,DELETE ON requestedNames TO 'janitor'@'localhost';
 GRANT SELECT,DELETE ON orderFqdnSets TO 'janitor'@'localhost';

Original file line number	Diff line number	Diff line change
`@@ -96,6 +96,9 @@ func newJobs(`
`96`	`96`	`}`
`97`	`97`	`jobs = append(jobs, newCertificatesPerNameJob(dbMap, logger, clk, config))`
`98`	`98`	`}`
	`99`	`+ if config.Janitor.KeyHashToSerial.Enabled {`
	`100`	`+ jobs = append(jobs, newKeyHashToSerialJob(dbMap, logger, clk, config))`
	`101`	`+ }`
`99`	`102`	`if config.Janitor.Orders.Enabled {`
`100`	`103`	`jobs = append(jobs, newOrdersJob(dbMap, logger, clk, config.Janitor.Orders))`
`101`	`104`	`}`