Skip to content

Generate S3 URLs from go template #1389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
J12934 merged 3 commits into from
Oct 13, 2022
Merged

Generate S3 URLs from go template #1389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
960c2f2
Move s3 bucket url generation into seperate helper function
the-simmon Sep 27, 2022
2b02ddd
Add template for s3 url
the-simmon Sep 27, 2022
c055b91
Refactor url generation code
the-simmon Oct 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions operator/controllers/execution/scans/hook_reconciler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,12 @@ func (r *ScanReconciler) processPendingHook(scan *executionv1.Scan, status *exec
}

var rawFileURL string
rawFileURL, err = r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration)
rawFileURL, err = r.PresignedGetURL(*scan, scan.Status.RawResultFile, defaultPresignDuration)
if err != nil {
return err
}
var findingsFileURL string
findingsFileURL, err = r.PresignedGetURL(scan.UID, "findings.json", defaultPresignDuration)
findingsFileURL, err = r.PresignedGetURL(*scan, "findings.json", defaultPresignDuration)
if err != nil {
return err
}
Expand All @@ -214,12 +214,12 @@ func (r *ScanReconciler) processPendingHook(scan *executionv1.Scan, status *exec
}
if hook.Spec.Type == executionv1.ReadAndWrite {
var rawFileUploadURL string
rawFileUploadURL, err = r.PresignedPutURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration)
rawFileUploadURL, err = r.PresignedPutURL(*scan, scan.Status.RawResultFile, defaultPresignDuration)
if err != nil {
return err
}
var findingsUploadURL string
findingsUploadURL, err = r.PresignedPutURL(scan.UID, "findings.json", defaultPresignDuration)
findingsUploadURL, err = r.PresignedPutURL(*scan, "findings.json", defaultPresignDuration)
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions operator/controllers/execution/scans/parse_reconciler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
}
log.Info("Matching ParseDefinition Found", "ParseDefinition", parseType)

findingsUploadURL, err := r.PresignedPutURL(scan.UID, "findings.json", defaultPresignDuration)
findingsUploadURL, err := r.PresignedPutURL(*scan, "findings.json", defaultPresignDuration)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return err
}
rawResultDownloadURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, defaultPresignDuration)
rawResultDownloadURL, err := r.PresignedGetURL(*scan, scan.Status.RawResultFile, defaultPresignDuration)
if err != nil {
return err
}
Expand Down
57 changes: 48 additions & 9 deletions operator/controllers/execution/scans/scan_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,19 @@
package scancontrollers

import (
"bytes"
"context"
"fmt"
"net/url"
"os"
"strings"
"text/template"
"time"

"github.com/go-logr/logr"
batch "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -127,11 +128,15 @@ func (r *ScanReconciler) handleFinalizer(scan *executionv1.Scan) error {
if containsString(scan.ObjectMeta.Finalizers, s3StorageFinalizer) {
bucketName := os.Getenv("S3_BUCKET")
r.Log.V(3).Info("Deleting External Files from FileStorage", "ScanUID", scan.UID)
err := r.MinioClient.RemoveObject(context.Background(), bucketName, fmt.Sprintf("scan-%s/%s", scan.UID, scan.Status.RawResultFile), minio.RemoveObjectOptions{})

rawResultUrl := getPresignedUrlPath(*scan, scan.Status.RawResultFile)
err := r.MinioClient.RemoveObject(context.Background(), bucketName, rawResultUrl, minio.RemoveObjectOptions{})
if err != nil && err.Error() != errNotFound {
return err
}
err = r.MinioClient.RemoveObject(context.Background(), bucketName, fmt.Sprintf("scan-%s/findings.json", scan.UID), minio.RemoveObjectOptions{})

findingsJsonUrl := getPresignedUrlPath(*scan, "findings.json")
err = r.MinioClient.RemoveObject(context.Background(), bucketName, findingsJsonUrl, minio.RemoveObjectOptions{})

if err != nil && err.Error() != errNotFound {
return err
Expand All @@ -146,11 +151,12 @@ func (r *ScanReconciler) handleFinalizer(scan *executionv1.Scan) error {
}

// PresignedGetURL returns a presigned URL from the s3 (or compatible) serice.
func (r *ScanReconciler) PresignedGetURL(scanID types.UID, filename string, duration time.Duration) (string, error) {
func (r *ScanReconciler) PresignedGetURL(scan executionv1.Scan, filename string, duration time.Duration) (string, error) {
bucketName := os.Getenv("S3_BUCKET")

fileUrl := getPresignedUrlPath(scan, filename)
reqParams := make(url.Values)
rawResultDownloadURL, err := r.MinioClient.PresignedGetObject(context.Background(), bucketName, fmt.Sprintf("scan-%s/%s", string(scanID), filename), duration, reqParams)
rawResultDownloadURL, err := r.MinioClient.PresignedGetObject(context.Background(), bucketName, fileUrl, duration, reqParams)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return "", err
Expand All @@ -159,10 +165,11 @@ func (r *ScanReconciler) PresignedGetURL(scanID types.UID, filename string, dura
}

// PresignedPutURL returns a presigned URL from the s3 (or compatible) serice.
func (r *ScanReconciler) PresignedPutURL(scanID types.UID, filename string, duration time.Duration) (string, error) {
func (r *ScanReconciler) PresignedPutURL(scan executionv1.Scan, filename string, duration time.Duration) (string, error) {
bucketName := os.Getenv("S3_BUCKET")
fileUrl := getPresignedUrlPath(scan, filename)

rawResultDownloadURL, err := r.MinioClient.PresignedPutObject(context.Background(), bucketName, fmt.Sprintf("scan-%s/%s", string(scanID), filename), duration)
rawResultDownloadURL, err := r.MinioClient.PresignedPutObject(context.Background(), bucketName, fileUrl, duration)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return "", err
Expand All @@ -171,10 +178,11 @@ func (r *ScanReconciler) PresignedPutURL(scanID types.UID, filename string, dura
}

// PresignedHeadURL returns a presigned URL from the s3 (or compatible) serice.
func (r *ScanReconciler) PresignedHeadURL(scanID types.UID, filename string, duration time.Duration) (string, error) {
func (r *ScanReconciler) PresignedHeadURL(scan executionv1.Scan, filename string, duration time.Duration) (string, error) {
bucketName := os.Getenv("S3_BUCKET")
fileUrl := getPresignedUrlPath(scan, filename)

rawResultHeadURL, err := r.MinioClient.PresignedHeadObject(context.Background(), bucketName, fmt.Sprintf("scan-%s/%s", string(scanID), filename), duration, nil)
rawResultHeadURL, err := r.MinioClient.PresignedHeadObject(context.Background(), bucketName, fileUrl, duration, nil)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return "", err
Expand Down Expand Up @@ -270,3 +278,34 @@ func containsString(slice []string, s string) bool {
}
return false
}

func getPresignedUrlPath(scan executionv1.Scan, filename string) string {
urlTemplate, ok := os.LookupEnv("S3_URL_TEMPLATE")
if !ok {
// use default when environment variable is not set
urlTemplate = "scan-{{ .Scan.UID }}/{{ .Filename }}"
}
return executeUrlTemplate(urlTemplate, scan, filename)
}

func executeUrlTemplate(urlTemplate string, scan executionv1.Scan, filename string) string {
type Template struct {
Scan executionv1.Scan
Filename string
}

tmpl, err := template.New(urlTemplate).Parse(urlTemplate)
if err != nil {
panic(err)
} else {
var rawOutput bytes.Buffer
templateArgs := Template{
Scan: scan,
Filename: filename,
}

err = tmpl.Execute(&rawOutput, templateArgs)
output := rawOutput.String()
return output
}
}
10 changes: 5 additions & 5 deletions operator/controllers/execution/scans/scan_reconciler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,26 +95,26 @@ func (r *ScanReconciler) startScan(scan *executionv1.Scan) error {
scan.Status.RawResultType = scanType.Spec.ExtractResults.Type
scan.Status.RawResultFile = filepath.Base(scanType.Spec.ExtractResults.Location)

findingsDownloadURL, err := r.PresignedGetURL(scan.UID, "findings.json", 7*24*time.Hour)
findingsDownloadURL, err := r.PresignedGetURL(*scan, "findings.json", 7*24*time.Hour)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return err
}
scan.Status.FindingDownloadLink = findingsDownloadURL
rawResultDownloadURL, err := r.PresignedGetURL(scan.UID, scan.Status.RawResultFile, 7*24*time.Hour)
rawResultDownloadURL, err := r.PresignedGetURL(*scan, scan.Status.RawResultFile, 7*24*time.Hour)
if err != nil {
return err
}
scan.Status.RawResultDownloadLink = rawResultDownloadURL

findingsHeadURL, err := r.PresignedHeadURL(scan.UID, "findings.json", 7*24*time.Hour)
findingsHeadURL, err := r.PresignedHeadURL(*scan, "findings.json", 7*24*time.Hour)
if err != nil {
r.Log.Error(err, "Could not get presigned head url from s3 or compatible storage provider")
return err
}
scan.Status.FindingHeadLink = findingsHeadURL

rawResultsHeadURL, err := r.PresignedHeadURL(scan.UID, scan.Status.RawResultFile, 7*24*time.Hour)
rawResultsHeadURL, err := r.PresignedHeadURL(*scan, scan.Status.RawResultFile, 7*24*time.Hour)
if err != nil {
r.Log.Error(err, "Could not get presigned head url from s3 or compatible storage provider")
return err
Expand Down Expand Up @@ -161,7 +161,7 @@ func (r *ScanReconciler) checkIfScanIsCompleted(scan *executionv1.Scan) error {

func (r *ScanReconciler) constructJobForScan(scan *executionv1.Scan, scanType *executionv1.ScanType) (*batch.Job, error) {
filename := filepath.Base(scanType.Spec.ExtractResults.Location)
resultUploadURL, err := r.PresignedPutURL(scan.UID, filename, defaultPresignDuration)
resultUploadURL, err := r.PresignedPutURL(*scan, filename, defaultPresignDuration)
if err != nil {
r.Log.Error(err, "Could not get presigned url from s3 or compatible storage provider")
return nil, err
Expand Down
4 changes: 4 additions & 0 deletions operator/templates/manager/manager.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ spec:
- name: CUSTOM_CA_CERTIFICATE_NAME
value: {{ .Values.customCACertificate.certificate | quote }}
{{ end }}
{{- if .Values.s3UrlTemplate }}
- name: S3_URL_TEMPLATE
value: {{ .Values.s3UrlTemplate }}
{{ end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
securityContext:
Expand Down
3 changes: 3 additions & 0 deletions operator/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,6 @@ resources:
requests:
cpu: 100m
memory: 20Mi
# s3FileUrlTemplate -- Template that generates the url to access the result files of a scan
# @default -- scan-{{ .Scan.UID }}/{{ .Filename }}
s3UrlTemplate: null