Skip to content

Commit 3a205d7

Browse files
committed
Add retries for the raw result upload from the lurker sidecar
When upload fails it can be painful to retry the entire scan just to rerun the file upload. Signed-off-by: Jannik Hollenbach <jannik.hollenbach@iteratec.com>
1 parent c9c3337 commit 3a205d7

5 files changed

Lines changed: 248 additions & 9 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ jobs:
194194
go vet ./...
195195
196196
- name: Test
197-
working-directory: ./operator
197+
working-directory: ./${{ matrix.component }}
198198
run: task test
199199

200200
- name: Build Container Image

‎lurker/.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,8 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
main
6+
lurker
7+
bin
8+
9+
# Output of the go coverage tool
10+
*.out

‎lurker/Taskfile.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# SPDX-FileCopyrightText: the secureCodeBox authors
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
version: "3.48.0"
6+
7+
tasks:
8+
fmt:
9+
desc: "Run go fmt against code"
10+
dir: '{{ .TASKFILE_DIR }}'
11+
cmds:
12+
- go fmt ./...
13+
14+
vet:
15+
desc: "Run go vet against code"
16+
dir: '{{ .TASKFILE_DIR }}'
17+
cmds:
18+
- go vet ./...
19+
20+
test:
21+
desc: "Run all tests"
22+
deps: [fmt, vet]
23+
dir: '{{ .TASKFILE_DIR }}'
24+
cmds:
25+
- go test ./... -coverprofile cover.out
26+
27+
view-coverage:
28+
desc: "View test coverage in browser"
29+
dir: '{{ .TASKFILE_DIR }}'
30+
cmds:
31+
- go tool cover -html=cover.out
32+
33+
build:
34+
desc: "Build the lurker binary"
35+
deps: [fmt, vet]
36+
dir: '{{ .TASKFILE_DIR }}'
37+
cmds:
38+
- go build -o bin/lurker main.go

‎lurker/main.go‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"context"
99
"flag"
1010
"fmt"
11+
"io"
1112
"log"
1213
"net/http"
1314
"net/http/httputil"
@@ -55,13 +56,44 @@ func main() {
5556

5657
log.Printf("Uploading result files.")
5758
log.Printf("Uploading %s", filePath)
58-
err = uploadFile(filePath, uploadURL)
59+
err = uploadFileWithRetries(filePath, uploadURL)
5960
if err != nil {
6061
log.Fatal(err)
6162
}
6263
log.Printf("Uploaded file successfully")
6364
}
6465

66+
// delays waited before each retry of a failed upload. The number of entries
67+
// defines how often the upload is retried after the initial attempt.
68+
var uploadBackoffs = []time.Duration{1 * time.Second, 3 * time.Second, 5 * time.Second, 10 * time.Second, 30 * time.Second}
69+
70+
// indirection to allow tests to run without actually waiting
71+
var sleep = time.Sleep
72+
73+
// uploadFileWithRetries uploads the file and retries every failure (transport
74+
// errors as well as non 2xx responses) using the uploadBackoffs schedule.
75+
func uploadFileWithRetries(path, url string) error {
76+
attempts := len(uploadBackoffs) + 1
77+
78+
var err error
79+
for attempt := 1; attempt <= attempts; attempt++ {
80+
err = uploadFile(path, url)
81+
if err == nil {
82+
return nil
83+
}
84+
85+
if attempt == attempts {
86+
break
87+
}
88+
89+
backoff := uploadBackoffs[attempt-1]
90+
log.Printf("Upload attempt %d of %d failed: %v. Retrying in %s", attempt, attempts, err, backoff)
91+
sleep(backoff)
92+
}
93+
94+
return fmt.Errorf("lurker failed to upload scan result file after %d attempts: %w", attempts, err)
95+
}
96+
6597
func uploadFile(path, url string) error {
6698
file, err := os.Open(path)
6799
if err != nil {
@@ -81,7 +113,7 @@ func uploadFile(path, url string) error {
81113
// Create a new file upload request
82114
req, err := http.NewRequest("PUT", url, file)
83115
if err != nil {
84-
log.Fatalf("Failed to create request: %v", err)
116+
log.Printf("Failed to create request: %v", err)
85117
return err
86118
}
87119

@@ -109,14 +141,16 @@ func uploadFile(path, url string) error {
109141
log.Printf("File upload returned non 2xx status code (%d)", res.StatusCode)
110142

111143
// Dump response for debugging purposes
112-
resultBytes, err := httputil.DumpResponse(res, true)
113-
if err != nil {
114-
log.Fatal(fmt.Errorf("failed to dump out failed requests to upload scan report to the s3 bucket: %w", err))
144+
resultBytes, dumpErr := httputil.DumpResponse(res, true)
145+
if dumpErr != nil {
146+
log.Printf("failed to dump out failed requests to upload scan report to the s3 bucket: %v", dumpErr)
147+
// drain the body so that the connection can be reused by the next attempt
148+
io.Copy(io.Discard, res.Body)
149+
} else {
150+
log.Println("Response of Failed Request:")
151+
log.Println(string(resultBytes))
115152
}
116153

117-
log.Println("Response of Failed Request:")
118-
log.Println(string(resultBytes))
119-
120154
return fmt.Errorf("lurker failed to upload scan result file. File upload returned non 2xx status code (%d)", res.StatusCode)
121155
}
122156

‎lurker/main_test.go‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// SPDX-FileCopyrightText: the secureCodeBox authors
2+
//
3+
// SPDX-License-Identifier: Apache-2.0
4+
5+
package main
6+
7+
import (
8+
"io"
9+
"net/http"
10+
"net/http/httptest"
11+
"os"
12+
"path/filepath"
13+
"testing"
14+
"time"
15+
)
16+
17+
const testFileContents = `{"findings": []}`
18+
19+
// writeResultFile creates a temporary scan result file and returns its path.
20+
func writeResultFile(t *testing.T) string {
21+
t.Helper()
22+
23+
path := filepath.Join(t.TempDir(), "findings.json")
24+
if err := os.WriteFile(path, []byte(testFileContents), 0o600); err != nil {
25+
t.Fatalf("failed to write test file: %v", err)
26+
}
27+
return path
28+
}
29+
30+
// stubSleep replaces the backoff sleep with a recorder, so that tests don't
31+
// have to actually wait. Returns a pointer to the recorded delays.
32+
func stubSleep(t *testing.T) *[]time.Duration {
33+
t.Helper()
34+
35+
original := sleep
36+
delays := []time.Duration{}
37+
sleep = func(d time.Duration) {
38+
delays = append(delays, d)
39+
}
40+
t.Cleanup(func() { sleep = original })
41+
42+
return &delays
43+
}
44+
45+
func TestUploadFileWithRetriesSucceedsOnFirstAttempt(t *testing.T) {
46+
delays := stubSleep(t)
47+
48+
requests := 0
49+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
50+
requests++
51+
w.WriteHeader(http.StatusOK)
52+
}))
53+
defer server.Close()
54+
55+
if err := uploadFileWithRetries(writeResultFile(t), server.URL); err != nil {
56+
t.Fatalf("expected upload to succeed, got: %v", err)
57+
}
58+
if requests != 1 {
59+
t.Errorf("expected 1 request, got %d", requests)
60+
}
61+
if len(*delays) != 0 {
62+
t.Errorf("expected no backoff, got %v", *delays)
63+
}
64+
}
65+
66+
func TestUploadFileWithRetriesRetriesUntilSuccess(t *testing.T) {
67+
delays := stubSleep(t)
68+
69+
requests := 0
70+
receivedBodies := []string{}
71+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
72+
requests++
73+
body, err := io.ReadAll(r.Body)
74+
if err != nil {
75+
t.Errorf("failed to read request body: %v", err)
76+
}
77+
receivedBodies = append(receivedBodies, string(body))
78+
79+
if requests < 3 {
80+
w.WriteHeader(http.StatusInternalServerError)
81+
return
82+
}
83+
w.WriteHeader(http.StatusOK)
84+
}))
85+
defer server.Close()
86+
87+
if err := uploadFileWithRetries(writeResultFile(t), server.URL); err != nil {
88+
t.Fatalf("expected upload to succeed after retries, got: %v", err)
89+
}
90+
if requests != 3 {
91+
t.Errorf("expected 3 requests, got %d", requests)
92+
}
93+
94+
// every attempt, especially the last one, has to send the complete file
95+
for attempt, body := range receivedBodies {
96+
if body != testFileContents {
97+
t.Errorf("attempt %d uploaded %q, expected %q", attempt+1, body, testFileContents)
98+
}
99+
}
100+
101+
expectedDelays := uploadBackoffs[:2]
102+
if len(*delays) != len(expectedDelays) {
103+
t.Fatalf("expected %d backoffs, got %v", len(expectedDelays), *delays)
104+
}
105+
for i, delay := range *delays {
106+
if delay != expectedDelays[i] {
107+
t.Errorf("backoff %d was %s, expected %s", i+1, delay, expectedDelays[i])
108+
}
109+
}
110+
}
111+
112+
func TestUploadFileWithRetriesGivesUpAfterAllAttempts(t *testing.T) {
113+
delays := stubSleep(t)
114+
115+
requests := 0
116+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
117+
requests++
118+
w.WriteHeader(http.StatusInternalServerError)
119+
}))
120+
defer server.Close()
121+
122+
if err := uploadFileWithRetries(writeResultFile(t), server.URL); err == nil {
123+
t.Fatal("expected upload to fail")
124+
}
125+
126+
expectedRequests := len(uploadBackoffs) + 1
127+
if requests != expectedRequests {
128+
t.Errorf("expected %d requests, got %d", expectedRequests, requests)
129+
}
130+
if len(*delays) != len(uploadBackoffs) {
131+
t.Errorf("expected %d backoffs, got %v", len(uploadBackoffs), *delays)
132+
}
133+
for i, delay := range *delays {
134+
if delay != uploadBackoffs[i] {
135+
t.Errorf("backoff %d was %s, expected %s", i+1, delay, uploadBackoffs[i])
136+
}
137+
}
138+
}
139+
140+
func TestUploadFileWithRetriesRetriesTransportErrors(t *testing.T) {
141+
delays := stubSleep(t)
142+
143+
// closing the server up front makes every attempt fail on the transport level
144+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
145+
url := server.URL
146+
server.Close()
147+
148+
if err := uploadFileWithRetries(writeResultFile(t), url); err == nil {
149+
t.Fatal("expected upload to fail")
150+
}
151+
if len(*delays) != len(uploadBackoffs) {
152+
t.Errorf("expected %d backoffs, got %v", len(uploadBackoffs), *delays)
153+
}
154+
}
155+
156+
func TestUploadFileWithRetriesFailsForMissingFile(t *testing.T) {
157+
stubSleep(t)
158+
159+
if err := uploadFileWithRetries(filepath.Join(t.TempDir(), "does-not-exist.json"), "http://127.0.0.1:1"); err == nil {
160+
t.Fatal("expected upload of a missing file to fail")
161+
}
162+
}

0 commit comments

Comments
 (0)