Skip to content

Commit 844ae26

Browse files
rolandshoemakerrolandshoemaker
authored
Allow forwarding of nonce-service Redeem RPCs from one service… (letsencrypt#4297)
Fixes letsencrypt#4295.
1 parent 352899b commit 844ae26

File tree

7 files changed

+230
-15
lines changed

7 files changed

+230
-15
lines changed

‎cmd/nonce-service/main.go‎

Lines changed: 78 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,13 @@ import (
44
"context"
55
"errors"
66
"flag"
7+
"sync"
8+
"time"
79

810
"github.com/letsencrypt/boulder/cmd"
911
corepb "github.com/letsencrypt/boulder/core/proto"
1012
bgrpc "github.com/letsencrypt/boulder/grpc"
13+
blog "github.com/letsencrypt/boulder/log"
1114
"github.com/letsencrypt/boulder/nonce"
1215
noncepb "github.com/letsencrypt/boulder/nonce/proto"
1316
)
@@ -17,18 +20,69 @@ type config struct {
1720
cmd.ServiceConfig
1821
Syslog cmd.SyslogConfig
1922
MaxUsed int
23+
24+
RemoteNonceServices []cmd.GRPCClientConfig
2025
}
2126
}
2227

2328
type nonceServer struct {
24-
inner *nonce.NonceService
29+
inner *nonce.NonceService
30+
remoteServices []noncepb.NonceServiceClient
31+
log blog.Logger
2532
}
2633

27-
func (ns *nonceServer) Redeem(_ context.Context, msg *noncepb.NonceMessage) (*noncepb.ValidMessage, error) {
34+
func (ns *nonceServer) remoteRedeem(ctx context.Context, msg *noncepb.NonceMessage) bool {
35+
deadline, ok := ctx.Deadline()
36+
if !ok {
37+
ns.log.Err("Context passed to remoteRedeem does not have a deadline")
38+
return false
39+
}
40+
subCtx, cancel := context.WithDeadline(ctx, deadline.Add(-time.Millisecond*250))
41+
defer cancel()
42+
forwarded := true
43+
msg.Forwarded = &forwarded
44+
results := make(chan bool, len(ns.remoteServices))
45+
wg := new(sync.WaitGroup)
46+
for _, remote := range ns.remoteServices {
47+
wg.Add(1)
48+
go func(r noncepb.NonceServiceClient) {
49+
defer wg.Done()
50+
resp, err := r.Redeem(subCtx, msg)
51+
if err != nil {
52+
ns.log.Errf("remote Redeem call failed: %s", err)
53+
return
54+
}
55+
results <- *resp.Valid
56+
}(remote)
57+
}
58+
go func() {
59+
wg.Wait()
60+
close(results)
61+
}()
62+
for result := range results {
63+
select {
64+
case <-subCtx.Done():
65+
return false
66+
default:
67+
if result {
68+
return true
69+
}
70+
}
71+
}
72+
return false
73+
}
74+
75+
func (ns *nonceServer) Redeem(ctx context.Context, msg *noncepb.NonceMessage) (*noncepb.ValidMessage, error) {
2876
if msg.Nonce == nil {
2977
return nil, errors.New("Incomplete gRPC request message")
3078
}
3179
valid := ns.inner.Valid(*msg.Nonce)
80+
// If the nonce was not valid, we have configured remote nonce services,
81+
// and this Redeem message wasn't forwarded, then forward it to the
82+
// remote services
83+
if !valid && len(ns.remoteServices) > 0 && msg.Forwarded != nil && !*msg.Forwarded {
84+
valid = ns.remoteRedeem(ctx, msg)
85+
}
3286
return &noncepb.ValidMessage{Valid: &valid}, nil
3387
}
3488

@@ -41,13 +95,22 @@ func (ns *nonceServer) Nonce(_ context.Context, _ *corepb.Empty) (*noncepb.Nonce
4195
}
4296

4397
func main() {
98+
grpcAddr := flag.String("addr", "", "gRPC listen address override")
99+
debugAddr := flag.String("debug-addr", "", "Debug server address override")
44100
configFile := flag.String("config", "", "File path to the configuration file for this service")
45101
flag.Parse()
46102

47103
var c config
48104
err := cmd.ReadConfigFile(*configFile, &c)
49105
cmd.FailOnError(err, "Reading JSON config file into config structure")
50106

107+
if *grpcAddr != "" {
108+
c.NonceService.GRPC.Address = *grpcAddr
109+
}
110+
if *debugAddr != "" {
111+
c.NonceService.DebugAddr = *debugAddr
112+
}
113+
51114
scope, logger := cmd.StatsAndLogging(c.NonceService.Syslog, c.NonceService.DebugAddr)
52115
defer logger.AuditPanic()
53116
logger.Info(cmd.VersionString())
@@ -57,10 +120,22 @@ func main() {
57120

58121
tlsConfig, err := c.NonceService.TLS.Load()
59122
cmd.FailOnError(err, "tlsConfig config")
123+
124+
nonceServer := &nonceServer{inner: ns, log: logger}
125+
if len(c.NonceService.RemoteNonceServices) > 0 {
126+
clientMetrics := bgrpc.NewClientMetrics(scope)
127+
clk := cmd.Clock()
128+
for _, remoteNonceConfig := range c.NonceService.RemoteNonceServices {
129+
rnsConn, err := bgrpc.ClientSetup(&remoteNonceConfig, tlsConfig, clientMetrics, clk)
130+
cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to Nonce service")
131+
nonceServer.remoteServices = append(nonceServer.remoteServices, noncepb.NewNonceServiceClient(rnsConn))
132+
}
133+
}
134+
60135
serverMetrics := bgrpc.NewServerMetrics(scope)
61136
grpcSrv, l, err := bgrpc.NewServer(c.NonceService.GRPC, tlsConfig, serverMetrics, cmd.Clock())
62137
cmd.FailOnError(err, "Unable to setup nonce service gRPC server")
63-
noncepb.RegisterNonceServiceServer(grpcSrv, &nonceServer{inner: ns})
138+
noncepb.RegisterNonceServiceServer(grpcSrv, nonceServer)
64139

65140
go cmd.CatchSignals(logger, grpcSrv.GracefulStop)
66141

‎cmd/nonce-service/main_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"errors"
6+
"testing"
7+
"time"
8+
9+
corepb "github.com/letsencrypt/boulder/core/proto"
10+
blog "github.com/letsencrypt/boulder/log"
11+
"github.com/letsencrypt/boulder/metrics"
12+
"github.com/letsencrypt/boulder/nonce"
13+
noncepb "github.com/letsencrypt/boulder/nonce/proto"
14+
"github.com/letsencrypt/boulder/test"
15+
"google.golang.org/grpc"
16+
)
17+
18+
type workingRemote struct{ resp bool }
19+
20+
func (wr *workingRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
21+
return &noncepb.ValidMessage{
22+
Valid: &wr.resp,
23+
}, nil
24+
}
25+
26+
func (wr *workingRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
27+
return nil, nil
28+
}
29+
30+
type sleepingRemote struct{}
31+
32+
func (sr *sleepingRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
33+
time.Sleep(time.Millisecond * 50)
34+
valid := true
35+
return &noncepb.ValidMessage{
36+
Valid: &valid,
37+
}, nil
38+
}
39+
40+
func (sr *sleepingRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
41+
return nil, nil
42+
}
43+
44+
type brokenRemote struct{}
45+
46+
func (br *brokenRemote) Redeem(ctx context.Context, in *noncepb.NonceMessage, opts ...grpc.CallOption) (*noncepb.ValidMessage, error) {
47+
return nil, errors.New("BROKE!")
48+
}
49+
50+
func (br *brokenRemote) Nonce(ctx context.Context, in *corepb.Empty, opts ...grpc.CallOption) (*noncepb.NonceMessage, error) {
51+
return nil, nil
52+
}
53+
54+
func TestRemoteRedeem(t *testing.T) {
55+
l := blog.NewMock()
56+
57+
innerNs, err := nonce.NewNonceService(metrics.NewNoopScope(), 1)
58+
test.AssertNotError(t, err, "NewNonceService failed")
59+
ns := nonceServer{log: l, inner: innerNs}
60+
61+
// Working remote returning valid nonce message
62+
ns.remoteServices = []noncepb.NonceServiceClient{
63+
&workingRemote{resp: false},
64+
&workingRemote{resp: true},
65+
}
66+
nonce := "asd"
67+
forwarded := false
68+
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
69+
resp, err := ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
70+
cancel()
71+
test.AssertNotError(t, err, "Redeem failed")
72+
test.Assert(t, *resp.Valid, "Redeem returned the wrong response")
73+
74+
// Working remotes returning invalid nonce message
75+
ns.remoteServices = []noncepb.NonceServiceClient{
76+
&workingRemote{resp: false},
77+
&workingRemote{resp: false},
78+
}
79+
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
80+
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
81+
cancel()
82+
test.AssertNotError(t, err, "Redeem failed")
83+
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
84+
85+
// Sleeping remotes returns valid nonce message, but after 50ms, Redeem should return false
86+
ns.remoteServices = []noncepb.NonceServiceClient{
87+
&sleepingRemote{},
88+
&sleepingRemote{},
89+
}
90+
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond))
91+
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
92+
cancel()
93+
test.AssertNotError(t, err, "Redeem failed")
94+
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
95+
96+
// Already forwarded message, Redeem should return false
97+
ns.remoteServices = []noncepb.NonceServiceClient{
98+
&workingRemote{resp: true},
99+
&workingRemote{resp: true},
100+
}
101+
forwarded = true
102+
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Hour))
103+
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
104+
cancel()
105+
test.AssertNotError(t, err, "Redeem failed")
106+
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
107+
108+
// Broken remotes, Redeem should return false
109+
ns.remoteServices = []noncepb.NonceServiceClient{
110+
&brokenRemote{},
111+
&brokenRemote{},
112+
}
113+
ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(time.Millisecond))
114+
resp, err = ns.Redeem(ctx, &noncepb.NonceMessage{Nonce: &nonce, Forwarded: &forwarded})
115+
cancel()
116+
test.AssertNotError(t, err, "Redeem failed")
117+
test.Assert(t, !*resp.Valid, "Redeem returned the wrong response")
118+
}

‎nonce/proto/nonce.pb.go‎

Lines changed: 18 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎nonce/proto/nonce.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ service NonceService {
1212

1313
message NonceMessage {
1414
optional string nonce = 1;
15+
optional bool forwarded = 2;
1516
}
1617

1718
message ValidMessage {

‎test/config-next/nonce.json‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,24 @@
88
"grpc": {
99
"address": ":9101",
1010
"clientNames": [
11-
"wfe.boulder"
11+
"wfe.boulder",
12+
"nonce.boulder"
1213
]
1314
},
1415
"tls": {
1516
"caCertFile": "test/grpc-creds/minica.pem",
1617
"certFile": "test/grpc-creds/nonce.boulder/cert.pem",
1718
"keyFile": "test/grpc-creds/nonce.boulder/key.pem"
18-
}
19+
},
20+
"remoteNonceServices": [
21+
{
22+
"serverAddress": "nonce.boulder:9102",
23+
"timeout": "15s"
24+
},
25+
{
26+
"serverAddress": "nonce.boulder:9101",
27+
"timeout": "15s"
28+
}
29+
]
1930
}
2031
}

‎test/grpc-creds/generate.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ command -v minica >/dev/null 2>&1 || {
1010
}
1111

1212
for HOSTNAME in admin-revoker.boulder expiration-mailer.boulder \
13-
ocsp-updater.boulder orphan-finder.boulder wfe.boulder akamai-purger.boulder ; do
13+
ocsp-updater.boulder orphan-finder.boulder wfe.boulder akamai-purger.boulder nonce.boulder ; do
1414
minica -domains ${HOSTNAME}
1515
done
1616

‎test/startservers.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def start(race_detection, fakeclock=None, config_dir=default_config_dir):
8888
[8002, './bin/boulder-ra --config %s --addr ra1.boulder:9094 --debug-addr :8002' % os.path.join(config_dir, "ra.json")],
8989
[8102, './bin/boulder-ra --config %s --addr ra2.boulder:9094 --debug-addr :8102' % os.path.join(config_dir, "ra.json")],
9090
[8111, './bin/nonce-service --config %s' % os.path.join(config_dir, "nonce.json")],
91+
[8112, './bin/nonce-service --config %s --addr nonce.boulder:9102 --debug-addr :8112' % os.path.join(config_dir, "nonce.json")],
9192
[4431, './bin/boulder-wfe2 --config %s' % os.path.join(config_dir, "wfe2.json")],
9293
[4000, './bin/boulder-wfe --config %s' % os.path.join(config_dir, "wfe.json")],
9394
])

0 commit comments

Comments
 (0)