Skip to content

Commit dbfb482

Browse files
jshacpu
jsha
authored and
cpu
committed
Add parallelism to SA CountCertificatesByNames. (letsencrypt#3133)
Since we can make up to 100 SQL queries from this method (based on the 100-SAN limit), sometimes it is too slow and we get a timeout for large certificates. By running some of those queries in parallel, we can speed things up and stop getting timeouts.
1 parent 23e2c4a commit dbfb482

File tree

10 files changed

+79
-19
lines changed

10 files changed

+79
-19
lines changed

‎cmd/boulder-sa/main.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ type config struct {
2020
cmd.DBConfig
2121

2222
Features map[string]bool
23+
24+
// Max simultaneous SQL queries caused by a single RPC.
25+
ParallelismPerRPC int
2326
}
2427

2528
Syslog cmd.SyslogConfig
@@ -54,7 +57,11 @@ func main() {
5457

5558
go sa.ReportDbConnCount(dbMap, scope)
5659

57-
sai, err := sa.NewSQLStorageAuthority(dbMap, cmd.Clock(), logger, scope)
60+
parallel := saConf.ParallelismPerRPC
61+
if parallel < 1 {
62+
parallel = 1
63+
}
64+
sai, err := sa.NewSQLStorageAuthority(dbMap, cmd.Clock(), logger, scope, parallel)
5865
cmd.FailOnError(err, "Failed to create SA impl")
5966

6067
var grpcSrv *grpc.Server

‎cmd/cert-checker/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ func TestGetAndProcessCerts(t *testing.T) {
183183
fc := clock.NewFake()
184184

185185
checker := newChecker(saDbMap, fc, pa, expectedValidityPeriod)
186-
sa, err := sa.NewSQLStorageAuthority(saDbMap, fc, blog.NewMock(), metrics.NewNoopScope())
186+
sa, err := sa.NewSQLStorageAuthority(saDbMap, fc, blog.NewMock(), metrics.NewNoopScope(), 1)
187187
test.AssertNotError(t, err, "Couldn't create SA to insert certificates")
188188
saCleanUp := test.ResetSATestDatabase(t)
189189
defer func() {

‎cmd/expiration-mailer/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -812,7 +812,7 @@ func setup(t *testing.T, nagTimes []time.Duration) *testCtx {
812812
t.Fatalf("Couldn't connect the database: %s", err)
813813
}
814814
fc := newFakeClock(t)
815-
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
815+
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
816816
if err != nil {
817817
t.Fatalf("unable to create SQLStorageAuthority: %s", err)
818818
}

‎cmd/expired-authz-purger/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func TestPurgeAuthzs(t *testing.T) {
2424
log := blog.UseMock()
2525
fc := clock.NewFake()
2626
fc.Add(time.Hour)
27-
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
27+
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
2828
if err != nil {
2929
t.Fatalf("unable to create SQLStorageAuthority: %s", err)
3030
}

‎cmd/id-exporter/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ func setup(t *testing.T) testCtx {
379379
cleanUp := test.ResetSATestDatabase(t)
380380

381381
fc := newFakeClock(t)
382-
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
382+
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
383383
if err != nil {
384384
t.Fatalf("unable to create SQLStorageAuthority: %s", err)
385385
}

‎cmd/ocsp-updater/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ func setup(t *testing.T) (*OCSPUpdater, core.StorageAuthority, *gorp.DbMap, cloc
130130
fc := clock.NewFake()
131131
fc.Add(1 * time.Hour)
132132

133-
sa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
133+
sa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
134134
test.AssertNotError(t, err, "Failed to create SA")
135135

136136
cleanUp := test.ResetSATestDatabase(t)

‎cmd/weak-key-search/main_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ func TestSearch(t *testing.T) {
3131
test.AssertNotError(t, err, "sa.NewDbMap failed")
3232
fc := clock.NewFake()
3333
log := blog.UseMock()
34-
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
34+
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
3535
test.AssertNotError(t, err, "sa.NewSQLStorageAuthority failed")
3636
defer test.ResetSATestDatabase(t)
3737

‎ra/ra_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, *sa.SQLStorageAut
226226
if err != nil {
227227
t.Fatalf("Failed to create dbMap: %s", err)
228228
}
229-
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
229+
ssa, err := sa.NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
230230
if err != nil {
231231
t.Fatalf("Failed to create SA: %s", err)
232232
}

‎sa/sa.go‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"math/big"
1010
"net"
1111
"strings"
12+
"sync"
1213
"time"
1314

1415
"github.com/jmhodges/clock"
@@ -33,6 +34,11 @@ type SQLStorageAuthority struct {
3334
clk clock.Clock
3435
log blog.Logger
3536
scope metrics.Scope
37+
38+
// For RPCs that generate multiple, parallelizable SQL queries, this is the
39+
// max parallelism they will use (to avoid consuming too many MariaDB
40+
// threads).
41+
parallelismPerRPC int
3642
}
3743

3844
func digest256(data []byte) []byte {
@@ -69,14 +75,16 @@ func NewSQLStorageAuthority(
6975
clk clock.Clock,
7076
logger blog.Logger,
7177
scope metrics.Scope,
78+
parallelismPerRPC int,
7279
) (*SQLStorageAuthority, error) {
7380
SetSQLDebug(dbMap, logger)
7481

7582
ssa := &SQLStorageAuthority{
76-
dbMap: dbMap,
77-
clk: clk,
78-
log: logger,
79-
scope: scope,
83+
dbMap: dbMap,
84+
clk: clk,
85+
log: logger,
86+
scope: scope,
87+
parallelismPerRPC: parallelismPerRPC,
8088
}
8189

8290
return ssa, nil
@@ -332,15 +340,60 @@ func (t TooManyCertificatesError) Error() string {
332340
// The highest count this function can return is 10,000. If there are more
333341
// certificates than that matching one of the provided domain names, it will return
334342
// TooManyCertificatesError.
343+
// Queries will be run in parallel. If any of them error, only one error will
344+
// be returned.
335345
func (ssa *SQLStorageAuthority) CountCertificatesByNames(ctx context.Context, domains []string, earliest, latest time.Time) ([]*sapb.CountByNames_MapElement, error) {
336-
var ret []*sapb.CountByNames_MapElement
346+
work := make(chan string, len(domains))
347+
type result struct {
348+
err error
349+
count int
350+
domain string
351+
}
352+
results := make(chan result, len(domains))
337353
for _, domain := range domains {
338-
currentCount, err := ssa.countCertificatesByName(domain, earliest, latest)
339-
if err != nil {
340-
return ret, err
354+
work <- domain
355+
}
356+
close(work)
357+
var wg sync.WaitGroup
358+
ctx, cancel := context.WithCancel(ctx)
359+
defer cancel()
360+
// We may perform up to 100 queries, depending on what's in the certificate
361+
// request. Parallelize them so we don't hit our timeout, but limit the
362+
// parallelism so we don't consume too many threads on the database.
363+
for i := 0; i < ssa.parallelismPerRPC; i++ {
364+
wg.Add(1)
365+
go func() {
366+
defer wg.Done()
367+
for domain := range work {
368+
select {
369+
case <-ctx.Done():
370+
results <- result{err: ctx.Err()}
371+
return
372+
default:
373+
}
374+
currentCount, err := ssa.countCertificatesByName(domain, earliest, latest)
375+
if err != nil {
376+
results <- result{err: err}
377+
// Skip any further work
378+
cancel()
379+
return
380+
}
381+
results <- result{
382+
count: currentCount,
383+
domain: domain,
384+
}
385+
}
386+
}()
387+
}
388+
wg.Wait()
389+
close(results)
390+
var ret []*sapb.CountByNames_MapElement
391+
for r := range results {
392+
if r.err != nil {
393+
return nil, r.err
341394
}
342-
name := string(domain)
343-
pbCount := int64(currentCount)
395+
name := string(r.domain)
396+
pbCount := int64(r.count)
344397
ret = append(ret, &sapb.CountByNames_MapElement{
345398
Name: &name,
346399
Count: &pbCount,

‎sa/sa_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func initSA(t *testing.T) (*SQLStorageAuthority, clock.FakeClock, func()) {
4949
fc := clock.NewFake()
5050
fc.Set(time.Date(2015, 3, 4, 5, 0, 0, 0, time.UTC))
5151

52-
sa, err := NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope())
52+
sa, err := NewSQLStorageAuthority(dbMap, fc, log, metrics.NewNoopScope(), 1)
5353
if err != nil {
5454
t.Fatalf("Failed to create SA: %s", err)
5555
}

0 commit comments

Comments
 (0)