stackrox · porridge · Feb 8, 2024 · Feb 6, 2024 · Feb 6, 2024 · Feb 6, 2024
@@ -2,6 +2,7 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestSuite
 testDirs:
 - ./tests/central
+- ./tests/controller
 - ./tests/securedcluster
 # central pod takes a while do become healthy, typically ~25s, but sometimes over a minute since container start.
 # We need to add CR reconciliation, pod creation and image fetch to that.

@@ -2,7 +2,8 @@
 
 ## Structure
 
-All kuttl tests are in the [tests](.) directory. It contains four tests, contained in three suites:
+All kuttl tests are in the [tests](.) directory. It contains the following tests, grouped in four suites:
+- [controller/metrics](controller/metrics) - checks authenticated access to stackrox operator controller manager metrics
 - [central/central-basic](central/central-basic) - exercises some core `Central` CR features.
 - [central/central-misc](central/central-misc) - various `Central` resource configurations and switching between them.
 - [securedcluster/sc-basic](securedcluster/sc-basic) - various `SecuredCluster` resource configurations
@@ -23,7 +24,7 @@ Note that the `deploy-via-olm` and `deploy-previous-via-olm` targets in `Makefil
 In a CI job:
 1. a previous operator version is installed first (a requirement of the upgrade test) using the `deploy-previous-via-olm` rule,
 2. the upgrade test runs next; it leaves the cluster with current operator version installed,
-3. then the `central` and `securedcluster` test suites are run _in parallel_.
+3. then the `controller`, `central` and `securedcluster` test suites are run _in parallel_.
 
 An important fact is that `kuttl` creates a uniquely-named ephemeral namespace
 (something like `kuttl-test-adjective-animal`) for every test, and deletes it afterwards.

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+- command: kubectl auth can-i get /metrics --as=system:serviceaccount:$NAMESPACE:operator-metrics-privileged --quiet
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: operator-metrics-unprivileged
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: operator-metrics-privileged
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+# We cannot simply apply a ClusterRoleBinding resource, because of the variable $NAMESPACE
+- command: kubectl create clusterrolebinding rhacs-metrics-$NAMESPACE --clusterrole=rhacs-operator-metrics-reader --serviceaccount=$NAMESPACE:operator-metrics-privileged
@@ -0,0 +1,43 @@
+# Ensure that unauthenticated requests are refused with a 401.
+# This pod definition should match its siblings, except for the name, serviceAccount and command.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-no-auth
+  labels:
+    test: metrics-access
+spec:
+  restartPolicy: Never
+  containers:
+  - name: run
+    image: registry.access.redhat.com/ubi9/ubi-minimal:latest
+    args:
+    - bash
+    - "-c"
+    # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported
+    - >-
+        operator_ns="stackrox-operator"
+        url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics";
+        set -u;
+        curl --version;
+        for attempt in $(seq 5); do
+          echo Attempt $attempt:;
+          curl --insecure --dump-header /tmp/headers "$url";
+          echo response:;
+          head -n 1 /tmp/headers;
+          echo checking response headers:;
+          if grep -Ei "^HTTP/[0-9.]+ 401" /tmp/headers; then exit 0; fi;
+          sleep 1;
+        done;
+        exit 1
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+  serviceAccount: operator-metrics-unprivileged
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
@@ -0,0 +1,43 @@
+# Ensure that the metrics server correctly refuses to talk HTTP/2 (Rapid Reset mitigation).
+# This pod definition should match its siblings, except for the name, serviceAccount and command.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-no-http2
+  labels:
+    test: metrics-access
+spec:
+  restartPolicy: Never
+  containers:
+  - name: run
+    image: registry.access.redhat.com/ubi9/ubi-minimal:latest
+    args:
+    - bash
+    - "-c"
+    # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported
+    - >-
+        operator_ns="stackrox-operator"
+        url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics";
+        set -u;
+        curl --version;
+        for attempt in $(seq 5); do
+          echo Attempt $attempt:;
+          curl --http2 --insecure --dump-header /tmp/headers "$url";
+          echo response:;
+          head -n 1 /tmp/headers;
+          echo checking response headers:;
+          if grep -Ei "^HTTP/1" /tmp/headers; then exit 0; fi;
+          sleep 1;
+        done;
+        exit 1
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+  serviceAccount: operator-metrics-unprivileged
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
@@ -0,0 +1,44 @@
+# Ensure that authenticated requests from service account with necessary permission are accepted and return metrics.
+# This pod definition should match its siblings, except for the name, serviceAccount and command.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-privileged
+  labels:
+    test: metrics-access
+spec:
+  restartPolicy: Never
+  containers:
+  - name: run
+    image: registry.access.redhat.com/ubi9/ubi-minimal:latest
+    args:
+    - bash
+    - "-c"
+    # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported
+    - >-
+        operator_ns="stackrox-operator"
+        url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics";
+        set -eu;
+        curl --version;
+        token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)";
+        for attempt in $(seq 5); do
+          echo Attempt $attempt:;
+          curl --insecure --fail -H "Authorization: Bearer $token" "$url" > /tmp/response;
+          echo beginning of response body:;
+          head /tmp/response;
+          echo checking response body:;
+          if grep -Ei "^# TYPE " /tmp/response; then exit 0; fi;
+          sleep 1;
+        done;
+        exit 1
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+  serviceAccount: operator-metrics-privileged
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
@@ -0,0 +1,44 @@
+# Ensure that authenticated requests from service account without necessary permissions are refused with a 403.
+# This pod definition should match its siblings, except for the name, serviceAccount and command.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-unprivileged
+  labels:
+    test: metrics-access
+spec:
+  restartPolicy: Never
+  containers:
+  - name: run
+    image: registry.access.redhat.com/ubi9/ubi-minimal:latest
+    args:
+    - bash
+    - "-c"
+    # TODO(ROX-22287): use $TEST_NAMESPACE from Makefile once templating is supported
+    - >-
+        operator_ns="stackrox-operator"
+        url="https://rhacs-operator-controller-manager-metrics-service.$operator_ns.svc.cluster.local:8443/metrics";
+        set -u;
+        curl --version;
+        token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)";
+        for attempt in $(seq 5); do
+          echo Attempt $attempt:;
+          curl --insecure --dump-header /tmp/headers -H "Authorization: Bearer $token" "$url" > /tmp/response;
+          echo response:;
+          head -n 1 /tmp/headers;
+          echo checking response headers:;
+          if grep -Ei "^HTTP/[0-9.]+ 403" /tmp/headers; then exit 0; fi;
+          sleep 1;
+        done;
+        exit 1
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - ALL
+  serviceAccount: operator-metrics-unprivileged
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 1000
+    seccompProfile:
+      type: RuntimeDefault
@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-no-auth
+status:
+  phase: Succeeded
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-no-http2
+status:
+  phase: Succeeded
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-privileged
+status:
+  phase: Succeeded
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: operator-metrics-unprivileged
+status:
+  phase: Succeeded
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+collectors:
+- type: pod
+  selector: test=metrics-access
+  tail: -1
+- command: kubectl describe pod -n $NAMESPACE -l test=metrics-access