Skip to content

Ensure Lurker Does Not Run Out of Memory When Uploading Large Scan Results #2336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
J12934 merged 2 commits into from
Mar 8, 2024
Merged

Ensure Lurker Does Not Run Out of Memory When Uploading Large Scan Results #2336

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e396794
#2327 Stream uploaded file to s3 bucket instead of loading it all upf…
J12934 Mar 6, 2024
0d1d938
#2327 Avoid error case which prevented empty scan result files from b…
J12934 Mar 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 31 additions & 14 deletions lurker/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,9 @@
package main

import (
"bytes"
"context"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
Expand Down Expand Up @@ -66,16 +64,33 @@ func main() {
}

func uploadFile(path, url string) error {
fileBytes, err := ioutil.ReadFile(path)
size := len(fileBytes)
log.Printf("File has a size of %d bytes", size)
file, err := os.Open(path)
if err != nil {
log.Println("Failed to read file")
log.Fatal(err)
log.Printf("Failed to open file: %v", err)
return err
}
req, err := http.NewRequest("PUT", url, bytes.NewReader(fileBytes))
defer file.Close()

fileInfo, err := file.Stat()
if err != nil {
log.Fatal(err)
log.Printf("Failed to get file stats: %v", err)
return err
}
size := fileInfo.Size()
log.Printf("Scan result file has a size of %d bytes", size)

// Create a new file upload request
req, err := http.NewRequest("PUT", url, file)
if err != nil {
log.Fatalf("Failed to create request: %v", err)
return err
}

req.ContentLength = size
// with the default TransferEncoding golang sends out the requests for empty files without
// the required Content-Length header this is valid, but not accepted by S3 compatible APIs
if size == 0 {
req.TransferEncoding = []string{"identity"}
}

client := &http.Client{}
Expand All @@ -86,22 +101,24 @@ func uploadFile(path, url string) error {
}
defer res.Body.Close()

if res.StatusCode < 300 {
// Check the response status code
if res.StatusCode >= 200 && res.StatusCode < 300 {
// all good
return nil
}

log.Printf("File upload returned non 2xx status code (%d)", res.StatusCode)

bytes, err := httputil.DumpResponse(res, true)
// Dump response for debugging purposes
resultBytes, err := httputil.DumpResponse(res, true)
if err != nil {
log.Fatal(errors.Wrap(err, "Failed to dump out failed requests to upload scan report to the s3 bucket"))
}

log.Println("Failed Request:")
log.Println(string(bytes))
log.Println("Response of Failed Request:")
log.Println(string(resultBytes))

return fmt.Errorf("Lurker failed to upload scan result file. File upload returned non 2xx status code (%d)", res.StatusCode)
return fmt.Errorf("lurker failed to upload scan result file. File upload returned non 2xx status code (%d)", res.StatusCode)
}

func waitForMainContainerToEnd(container, pod, namespace string) {
Expand Down