secureCodeBox · Ilyesbdlala · Jul 23, 2024 · Jul 18, 2024 · Jul 18, 2024 · Jul 18, 2024
diff --git a/documentation/docs/getting-started/installation.md b/documentation/docs/getting-started/installation.md
@@ -91,6 +91,21 @@ s3:
   keySecret: gcs-bucket-credentials
 ```
 
+### Prometheus Metrics
+
+If you want to monitor the scans managed by your _secureCodeBox Operator_ you can use the [Prometheus](https://prometheus.io/) metrics that the operator provides.
+To use them you need to configure your prometheus instance to scrape these metrics from port 8080 of the operator pod.
+
+If you are using the Prometheus operator you can enable metric collection by enabling the service monitoring included in the operators helm chart.
+To do that you need to set the `metrics.serviceMonitor.enabled` flag to `true`.
+
+```bash
+helm --namespace securecodebox-system upgrade --install --create-namespace securecodebox-operator oci://ghcr.io/securecodebox/helm/operator --set="metrics.serviceMonitor.enabled=true"
+```
+
+Custom metrics from the _secureCodeBox Operator_ are prefixed with: `securecodebox_`.
+You can list them, together with an explanation in your Prometheus/Grafana explore view.
+
 ## Install SCB Scanner
 
 The following list will give you a short overview of all supported security scanner charts and how to install them.

diff --git a/operator/README.md b/operator/README.md
@@ -83,6 +83,8 @@ helm install securecodebox-operator oci://ghcr.io/securecodebox/helm/operator
 | lurker.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images |
 | lurker.image.repository | string | `"docker.io/securecodebox/lurker"` | The operator image repository |
 | lurker.image.tag | string | defaults to the charts version | Parser image tag |
+| metrics | object | `{"serviceMonitor":{"enabled":false}}` | Configuration for the metrics the operator exports |
+| metrics.serviceMonitor.enabled | bool | `false` | Creates a prometheus operator ServiceMonitor rule to automatically scrape the operators metrics: https://github.com/prometheus-operator/prometheus-operator |
 | minio | object | `{"auth":{"rootPassword":"password","rootUser":"admin"},"defaultBuckets":"securecodebox","enabled":true,"resources":{"requests":{"memory":"256Mi"}},"tls":{"enabled":false}}` | Minio default config. More config options an info: https://github.com/minio/minio/blob/master/helm/minio/values.yaml |
 | minio.enabled | bool | `true` | Enable this to use minio as storage backend instead of a cloud bucket provider like AWS S3, Google Cloud Storage, DigitalOcean Spaces etc. |
 | nodeSelector | object | `{}` |  |

diff --git a/operator/apis/execution/v1/scan_types.go b/operator/apis/execution/v1/scan_types.go
@@ -148,9 +148,25 @@ type ScanSpec struct {
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
+type ScanState string
+
+const (
+	ScanStateInit                       ScanState = "Init"
+	ScanStateScanning                   ScanState = "Scanning"
+	ScanStateScanCompleted              ScanState = "ScanCompleted"
+	ScanStateParsing                    ScanState = "Parsing"
+	ScanStateParseCompleted             ScanState = "ParseCompleted"
+	ScanStateHookProcessing             ScanState = "HookProcessing"
+	ScanStateReadAndWriteHookProcessing ScanState = "ReadAndWriteHookProcessing"
+	ScanStateReadAndWriteHookCompleted  ScanState = "ReadAndWriteHookCompleted"
+	ScanStateReadOnlyHookProcessing     ScanState = "ReadOnlyHookProcessing"
+	ScanStateErrored                    ScanState = "Errored"
+	ScanStateDone                       ScanState = "Done"
+)
+
 // ScanStatus defines the observed state of Scan
 type ScanStatus struct {
-	State string `json:"state,omitempty"`
+	State ScanState `json:"state,omitempty"`
 
 	// FinishedAt contains the time where the scan (including parser & hooks) has been marked as "Done"
 	FinishedAt       *metav1.Time `json:"finishedAt,omitempty"`

diff --git a/operator/controllers/execution/scans/hook_reconciler.go b/operator/controllers/execution/scans/hook_reconciler.go
@@ -11,8 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 
 	executionv1 "github.com/secureCodeBox/secureCodeBox/operator/apis/execution/v1"
-	"github.com/secureCodeBox/secureCodeBox/operator/utils"
-	util "github.com/secureCodeBox/secureCodeBox/operator/utils"
+	utils "github.com/secureCodeBox/secureCodeBox/operator/utils"
 	batch "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -43,7 +42,7 @@ func (r *ScanReconciler) setHookStatus(scan *executionv1.Scan) error {
 			return err
 		}
 
-		hookStatuses = util.MapHooksToHookStatus(scanCompletionHooks.Items)
+		hookStatuses = utils.MapHooksToHookStatus(scanCompletionHooks.Items)
 	} else {
 		var clusterScanCompletionHooks executionv1.ClusterScanCompletionHookList
 		if err := r.List(ctx, &clusterScanCompletionHooks,
@@ -53,16 +52,16 @@ func (r *ScanReconciler) setHookStatus(scan *executionv1.Scan) error {
 			return err
 		}
 
-		hookStatuses = util.MapClusterHooksToHookStatus(clusterScanCompletionHooks.Items)
+		hookStatuses = utils.MapClusterHooksToHookStatus(clusterScanCompletionHooks.Items)
 	}
 
 	r.Log.Info("Found ScanCompletionHooks", "ScanCompletionHooks", len(hookStatuses))
 
-	orderedHookStatus := util.FromUnorderedList(hookStatuses)
+	orderedHookStatus := utils.FromUnorderedList(hookStatuses)
 	scan.Status.OrderedHookStatuses = orderedHookStatus
-	scan.Status.State = "HookProcessing"
+	scan.Status.State = executionv1.ScanStateHookProcessing
 
-	if err := r.Status().Update(ctx, scan); err != nil {
+	if err := r.updateScanStatus(ctx, scan); err != nil {
 		r.Log.Error(err, "unable to update Scan status")
 		return err
 	}
@@ -99,14 +98,14 @@ func (r *ScanReconciler) migrateHookStatus(scan *executionv1.Scan) error {
 				Type:     executionv1.ReadOnly,
 			}
 
-			if scan.Status.State == "ReadAndWriteHookProcessing" || scan.Status.State == "ReadAndWriteHookCompleted" {
+			if scan.Status.State == executionv1.ScanStateReadAndWriteHookProcessing || scan.Status.State == executionv1.ScanStateReadAndWriteHookCompleted {
 				// ReadOnly hooks should not have started yet, so mark them all as pending
 				hookStatus.State = executionv1.Pending
-			} else if scan.Status.State == "ReadOnlyHookProcessing" {
+			} else if scan.Status.State == executionv1.ScanStateReadOnlyHookProcessing {
 				// Had already started ReadOnly hooks and should now check status.
 				// No status for ReadOnly in old CRD, so mark everything as InProgress and let processInProgressHook update it later.
 				hookStatus.State = executionv1.InProgress
-			} else if scan.Status.State == "Done" {
+			} else if scan.Status.State == executionv1.ScanStateDone {
 				// Had completely finished
 				hookStatus.State = executionv1.Completed
 			}
@@ -117,12 +116,12 @@ func (r *ScanReconciler) migrateHookStatus(scan *executionv1.Scan) error {
 		}
 	}
 
-	scan.Status.OrderedHookStatuses = util.OrderHookStatusesInsideAPrioClass(append(readOnlyHooks, strSlice...))
-	if scan.Status.State != "Done" {
-		scan.Status.State = "HookProcessing"
+	scan.Status.OrderedHookStatuses = utils.OrderHookStatusesInsideAPrioClass(append(readOnlyHooks, strSlice...))
+	if scan.Status.State != executionv1.ScanStateDone {
+		scan.Status.State = executionv1.ScanStateHookProcessing
 	}
 
-	if err := r.Status().Update(ctx, scan); err != nil {
+	if err := r.updateScanStatus(ctx, scan); err != nil {
 		r.Log.Error(err, "unable to update Scan status")
 		return err
 	}
@@ -139,27 +138,27 @@ func (r *ScanReconciler) executeHooks(scan *executionv1.Scan) error {
 
 	err, currentHooks := utils.CurrentHookGroup(scan.Status.OrderedHookStatuses)
 
-	if err != nil && scan.Status.State == "Errored" {
+	if err != nil && scan.Status.State == executionv1.ScanStateErrored {
 		r.Log.V(8).Info("Skipping hook execution as it already contains failed hooks.")
 		return nil
 	} else if err != nil {
-		scan.Status.State = "Errored"
-		scan.Status.ErrorDescription = fmt.Sprintf("Hook execution failed for a unknown hook. Check the scan.status.hookStatus field for more details")
-	} else if err == nil && currentHooks == nil {
+		scan.Status.State = executionv1.ScanStateErrored
+		scan.Status.ErrorDescription = "hook execution failed for a unknown hook. Check the scan.status.hookStatus field for more details"
+	} else if currentHooks == nil {
 		// No hooks left to execute
-		scan.Status.State = "Done"
+		scan.Status.State = executionv1.ScanStateDone
 	} else {
 		for _, hook := range currentHooks {
 			err = r.processHook(scan, hook)
 
 			if err != nil {
-				scan.Status.State = "Errored"
+				scan.Status.State = executionv1.ScanStateErrored
 				scan.Status.ErrorDescription = fmt.Sprintf("Failed to execute Hook '%s' in job '%s'. Check the logs of the hook for more information.", hook.HookName, hook.JobName)
 			}
 		}
 	}
 
-	if sErr := r.Status().Update(ctx, scan); sErr != nil {
+	if sErr := r.updateScanStatus(ctx, scan); sErr != nil {
 		r.Log.Error(sErr, "Unable to update Scan status")
 		return sErr
 	}
@@ -227,7 +226,7 @@ func (r *ScanReconciler) processPendingHook(scan *executionv1.Scan, status *exec
 		return nil
 	}
 
-	urlExpirationDuration, err := util.GetUrlExpirationDuration(util.HookController)
+	urlExpirationDuration, err := utils.GetUrlExpirationDuration(utils.HookController)
 	if err != nil {
 		r.Log.Error(err, "Failed to parse hook url expiration")
 		panic(err)
@@ -382,7 +381,7 @@ func (r *ScanReconciler) createJobForHook(hookName string, hookSpec *executionv1
 	job := &batch.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Annotations:  make(map[string]string),
-			GenerateName: util.TruncateName(fmt.Sprintf("%s-%s", hookName, scan.Name)),
+			GenerateName: utils.TruncateName(fmt.Sprintf("%s-%s", hookName, scan.Name)),
 			Namespace:    scan.Namespace,
 			Labels:       labels,
 		},
@@ -455,7 +454,7 @@ func (r *ScanReconciler) createJobForHook(hookName string, hookSpec *executionv1
 	}
 
 	// Merge NodeSelectors from Hook & Scan into Hook Job
-	job.Spec.Template.Spec.NodeSelector = util.MergeStringMaps(job.Spec.Template.Spec.NodeSelector, hookSpec.NodeSelector, scan.Spec.NodeSelector)
+	job.Spec.Template.Spec.NodeSelector = utils.MergeStringMaps(job.Spec.Template.Spec.NodeSelector, hookSpec.NodeSelector, scan.Spec.NodeSelector)
 
 	// Replace tolerations from template with those from the scan, if specified.
 	// Otherwise, stick to those from the template

diff --git a/operator/controllers/execution/scans/init.go b/operator/controllers/execution/scans/init.go
@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: the secureCodeBox authors
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package scancontrollers
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	commonMetricLabelScanType = "scan_type"
+)
+
+var (
+	scansStartedMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "securecodebox_scans_started_count",
+			Help: "Number of secureCodeBox scans started",
+		},
+		[]string{commonMetricLabelScanType},
+	)
+	scansDoneMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "securecodebox_scans_done_count",
+			Help: "Number of secureCodeBox scans that reached state 'done'. Meaning that they have not encountered any errors.",
+		},
+		[]string{commonMetricLabelScanType},
+	)
+	scansErroredMetric = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "securecodebox_scans_errored_count",
+			Help: "Number of secureCodeBox scans that reached state 'errored'. Meaning that they have encountered errors and were aborted.",
+		},
+		[]string{commonMetricLabelScanType},
+	)
+)
+
+func init() {
+	// Register custom metrics with the global prometheus registry
+	metrics.Registry.MustRegister(scansStartedMetric, scansDoneMetric, scansErroredMetric)
+}
diff --git a/operator/controllers/execution/scans/parse_reconciler.go b/operator/controllers/execution/scans/parse_reconciler.go
@@ -44,14 +44,14 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
 		if err := r.Get(ctx, types.NamespacedName{Name: parseType, Namespace: scan.Namespace}, &parseDefinition); err != nil {
 			log.V(7).Info("Unable to fetch ParseDefinition")
 
-			scan.Status.State = "Errored"
+			scan.Status.State = executionv1.ScanStateErrored
 			scan.Status.ErrorDescription = fmt.Sprintf("No ParseDefinition for ResultType '%s' found in Scans Namespace.", parseType)
-			if err := r.Status().Update(ctx, scan); err != nil {
+			if err := r.updateScanStatus(ctx, scan); err != nil {
 				r.Log.Error(err, "unable to update Scan status")
 				return err
 			}
 
-			return fmt.Errorf("No ParseDefinition of type '%s' found", parseType)
+			return fmt.Errorf("no ParseDefinition of type '%s' found", parseType)
 		}
 		log.Info("Matching ParseDefinition Found", "ParseDefinition", parseType)
 		parseDefinitionSpec = parseDefinition.Spec
@@ -60,14 +60,14 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
 		if err := r.Get(ctx, types.NamespacedName{Name: parseType}, &clusterParseDefinition); err != nil {
 			log.V(7).Info("Unable to fetch ClusterParseDefinition")
 
-			scan.Status.State = "Errored"
+			scan.Status.State = executionv1.ScanStateErrored
 			scan.Status.ErrorDescription = fmt.Sprintf("No ClusterParseDefinition for ResultType '%s' found.", parseType)
-			if err := r.Status().Update(ctx, scan); err != nil {
+			if err := r.updateScanStatus(ctx, scan); err != nil {
 				r.Log.Error(err, "unable to update Scan status")
 				return err
 			}
 
-			return fmt.Errorf("No ClusterParseDefinition of type '%s' found", parseType)
+			return fmt.Errorf("no ClusterParseDefinition of type '%s' found", parseType)
 		}
 		log.Info("Matching ClusterParseDefinition Found", "ClusterParseDefinition", parseType)
 		parseDefinitionSpec = clusterParseDefinition.Spec
@@ -250,8 +250,8 @@ func (r *ScanReconciler) startParser(scan *executionv1.Scan) error {
 		return err
 	}
 
-	scan.Status.State = "Parsing"
-	if err := r.Status().Update(ctx, scan); err != nil {
+	scan.Status.State = executionv1.ScanStateParsing
+	if err := r.updateScanStatus(ctx, scan); err != nil {
 		log.Error(err, "unable to update Scan status")
 		return err
 	}
@@ -271,15 +271,15 @@ func (r *ScanReconciler) checkIfParsingIsCompleted(scan *executionv1.Scan) error
 	switch status {
 	case completed:
 		r.Log.V(7).Info("Parsing is completed")
-		scan.Status.State = "ParseCompleted"
-		if err := r.Status().Update(ctx, scan); err != nil {
+		scan.Status.State = executionv1.ScanStateParseCompleted
+		if err := r.updateScanStatus(ctx, scan); err != nil {
 			r.Log.Error(err, "unable to update Scan status")
 			return err
 		}
 	case failed:
-		scan.Status.State = "Errored"
+		scan.Status.State = executionv1.ScanStateErrored
 		scan.Status.ErrorDescription = "Failed to run the Parser. This is likely a Bug, we would like to know about. Please open up a Issue on GitHub."
-		if err := r.Status().Update(ctx, scan); err != nil {
+		if err := r.updateScanStatus(ctx, scan); err != nil {
 			r.Log.Error(err, "unable to update Scan status")
 			return err
 		}