Support -grace parameter for contact-exporter, update docs (letsencrypt#2044)

cpu · jsha · commit c4efb7b46057 · 2016-07-25T15:01:52.000-04:00
Presently the `contact-exporter` command only exports registration IDs that have existing certificates that [have not yet expired](https://github.com/letsencrypt/boulder/blob/217b79b8c265da2ae9177791a3ab136e6d656e2f/cmd/contact-exporter/main.go#L39). The threshold is [the current time](https://github.com/letsencrypt/boulder/blob/217b79b8c265da2ae9177791a3ab136e6d656e2f/cmd/contact-exporter/main.go#L42) when run. This PR adds a user configurable grace period that we can use to offset this expiration check. E.g. export all registration IDs that have unexpired certificates, or certificates that expired in the last N days. This resolves letsencrypt#1959 This PR also catches up the documentation strings for both the `expiration-mailer` and the `contact-exporter` to include the changes landed in letsencrypt#1958. Both have been updated to describe operating on registration IDs in place of bare emails.
diff --git a/cmd/contact-exporter/main.go b/cmd/contact-exporter/main.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"time"
 
 	"gopkg.in/gorp.v1"
 
@@ -19,6 +20,7 @@ type contactExporter struct {
 	log   blog.Logger
 	dbMap *gorp.DbMap
 	clk   clock.Clock
+	grace time.Duration
 }
 
 type contact struct {
@@ -39,7 +41,7 @@ func (c contactExporter) findContacts() ([]contact, error) {
 				WHERE expires >= :expireCutoff
 			);`,
 		map[string]interface{}{
-			"expireCutoff": c.clk.Now(),
+			"expireCutoff": c.clk.Now().Add(-c.grace),
 		})
 	if err != nil {
 		c.log.AuditErr(fmt.Sprintf("Error finding contacts: %s", err))
@@ -69,26 +71,46 @@ func writeContacts(contactsList []contact, outfile string) error {
 const usageIntro = `
 Introduction:
 
-The contact exporter exists to retrieve the email addresses of all registered
-users with currently unexpired certificates. This list of email addresses can
+The contact exporter exists to retrieve the IDs of all registered
+users with currently unexpired certificates. This list of registration IDs can
 then be given as input to the notification mailer to send bulk notifications.
 
-The email addresses are deduplicated and sorted prior to being writen to the
-outfile. E.g. if two registrations exist with the contact email
-"example@example.com", this address will only appear once. Registration contacts
-that are *not* email addresses are discarded (e.g. tel:999-999-9999)
+The -grace parameter can be used to allow registrations with certificates that
+have already expired to be included in the export. The argument is a Go duration
+obeying the usual suffix rules (e.g. 24h).
+
+Registration IDs are favoured over email addresses as the intermediate format in
+order to ensure the most up to date contact information is used at the time of
+notification. The notification mailer will resolve the ID to email(s) when the
+mailing is underway, ensuring we use the correct address if a user has updated
+their contact information between the time of export and the time of
+notification.
+
+The contact exporter's registration ID output will be JSON of the form:
+  [
+   { "id": 1 },
+   ...
+   { "id": n }
+  ]
 
 Examples:
-  Export all email addresses to "emails.txt":
+  Export all registration IDs with unexpired certificates to "regs.json":
+
+  contact-exporter -config test/config/contact-exporter.json -outfile regs.json
+
+  Export all registration IDs with certificates that are unexpired or expired
+  within the last two days to "regs.json":
 
-  contact-exporter -config test/config/contact-exporter.json -outfile emails.txt
+  contact-exporter -config test/config/contact-exporter.json -grace 48h -outfile
+    "regs.json"
 
 Required arguments:
 - config
 - outfile`
 
 func main() {
 	outFile := flag.String("outfile", "", "File to write contacts to (defaults to stdout).")
+	grace := flag.Duration("grace", 2*24*time.Hour, "Include contacts with certificates that expired in < grace ago")
 	type config struct {
 		ContactExporter struct {
 			cmd.DBConfig
@@ -126,6 +148,7 @@ func main() {
 		log:   log,
 		dbMap: dbMap,
 		clk:   cmd.Clock(),
+		grace: *grace,
 	}
 
 	contacts, err := exporter.findContacts()
diff --git a/cmd/contact-exporter/main_test.go b/cmd/contact-exporter/main_test.go
@@ -67,6 +67,18 @@ func TestFindContacts(t *testing.T) {
 	test.AssertEquals(t, contacts[0].ID, regA.ID)
 	test.AssertEquals(t, contacts[1].ID, regC.ID)
 	test.AssertEquals(t, contacts[2].ID, regD.ID)
+
+	// Allow a 1 year grace period
+	testCtx.c.grace = 360 * 24 * time.Hour
+	contacts, err = testCtx.c.findContacts()
+	test.AssertNotError(t, err, "findContacts() produced error")
+	// Now all four registration should be returned, including RegB since its
+	// certificate expired within the grace period
+	test.AssertEquals(t, len(contacts), 4)
+	test.AssertEquals(t, contacts[0].ID, regA.ID)
+	test.AssertEquals(t, contacts[1].ID, regB.ID)
+	test.AssertEquals(t, contacts[2].ID, regC.ID)
+	test.AssertEquals(t, contacts[3].ID, regD.ID)
 }
 
 func exampleContacts() []contact {
diff --git a/cmd/notify-mailer/main.go b/cmd/notify-mailer/main.go
@@ -175,57 +175,65 @@ func emailsForReg(id int, dbMap dbSelector) ([]string, error) {
 const usageIntro = `
 Introduction:
 
-The notification mailer exists to send a fixed message to a list of email
-addresses. The attributes of the message (from address, subject, and message
-content) are provided by the command line arguments. The message content is used
-verbatim and must be provided as a path to a plaintext file via the -body
-argument. The list of recipient emails should be provided via the -toFile
-argument as a path to a plaintext file containing one email per line.
+The notification mailer exists to send a fixed message to the contact associated
+with a list of registration IDs. The attributes of the message (from address,
+subject, and message content) are provided by the command line arguments. The
+message content is used verbatim and must be provided as a path to a plaintext
+file via the -body argument. A list of registration IDs should be provided via
+the -toFile argument as a path to a plaintext file containing JSON of the form:
+
+  [
+   { "id": 1 },
+   ...
+   { "id": n }
+  ]
 
 To help the operator gain confidence in the mailing run before committing fully
 three safety features are supported: dry runs, checkpointing and a sleep
 interval.
 
-The -dryRun flag will use a mock mailer that prints message content to stdout
-instead of performing an SMTP transaction with a real mailserver. This can be
-used when the initial parameters are being tweaked to ensure no real emails are
-sent.
+The -dryRun=true flag will use a mock mailer that prints message content to
+stdout instead of performing an SMTP transaction with a real mailserver. This
+can be used when the initial parameters are being tweaked to ensure no real
+emails are sent. Using -dryRun=false will send real email.
 
 Checkpointing is supported via the -start and -end arguments. The -start flag
-specifies which line of the -toFile to start processing at. Similarly, the -end
-flag specifies which line of the -toFile to end processing at. In combination
-these can be used to process only a fixed number of recipients at a time, and
-to resume mailing after early termination.
+specifies which registration ID of the -toFile to start processing at.
+Similarly, the -end flag specifies which registration ID of the -toFile to end
+processing at. In combination these can be used to process only a fixed number
+of recipients at a time, and to resume mailing after early termination.
 
 During mailing the -sleep argument is used to space out individual messages.
 This can be used to ensure that the mailing happens at a steady pace with ample
 opportunity for the operator to terminate early in the event of error. The
 -sleep flag honours durations with a unit suffix (e.g. 1m for 1 minute, 10s for
-10 seconds, etc).
+10 seconds, etc). Using -sleep=0 will disable the sleep and send at full speed.
 
 Examples:
   Send an email with subject "Hello!" from the email "hello@goodbye.com" with
-  the contents read from "test_msg_body.txt" to every email listed in
-  "test_msg_recipients.txt", sleeping 10 seconds between each message:
+  the contents read from "test_msg_body.txt" to every email associated with the
+  registration IDs listed in "test_reg_recipients.json", sleeping 10 seconds
+  between each message:
 
-  notify-mailer -config test/config/notify-mailer.json 
-    -body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com 
-    -toFile cmd/notify-mailer/testdata/test_msg_recipients.txt -subject "Hello!"
-    -sleep 10s
+  notify-mailer -config test/config/notify-mailer.json -body
+    cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com
+    -toFile cmd/notify-mailer/testdata/test_msg_recipients.json -subject "Hello!"
+    -sleep 10s -dryRun=false
 
-  Do the same, but only to the first 100 recipients:
+  Do the same, but only to the first 100 recipient IDs:
 
-  notify-mailer -config test/config/notify-mailer.json 
-    -body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com 
-    -toFile cmd/notify-mailer/testdata/test_msg_recipients.txt -subject "Hello!"
-    -sleep 10s -end 100
+  notify-mailer -config test/config/notify-mailer.json
+    -body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com
+    -toFile cmd/notify-mailer/testdata/test_msg_recipients.json -subject "Hello!"
+    -sleep 10s -end 100 -dryRun=false
+
+  Send the message, but start at the 200th ID of the recipients file, ending after
+  100 registration IDs, and as a dry-run:
 
-  Send the message, but start at line 200 of the recipients file, ending after
-  100 recipients, and as a dry-run:
   notify-mailer -config test/config/notify-mailer.json 
     -body cmd/notify-mailer/testdata/test_msg_body.txt -from hello@goodbye.com 
-    -toFile cmd/notify-mailer/testdata/test_msg_recipients.txt -subject "Hello!"
-    -sleep 10s -start 200 -end 300 -dryRun
+    -toFile cmd/notify-mailer/testdata/test_msg_recipients.json -subject "Hello!"
+    -sleep 10s -start 200 -end 300 -dryRun=true
 
 Required arguments:
 - body
@@ -237,7 +245,7 @@ Required arguments:
 func main() {
 	from := flag.String("from", "", "From header for emails. Must be a bare email address.")
 	subject := flag.String("subject", "", "Subject of emails")
-	toFile := flag.String("toFile", "", "File containing a list of email addresses to send to, one per file.")
+	toFile := flag.String("toFile", "", "File containing a JSON array of registration IDs to send to.")
 	bodyFile := flag.String("body", "", "File containing the email body in plain text format.")
 	dryRun := flag.Bool("dryRun", true, "Whether to do a dry run.")
 	sleep := flag.Duration("sleep", 60*time.Second, "How long to sleep between emails.")
