stackrox · gavin-stackrox · Nov 7, 2022 · Nov 5, 2022 · Nov 5, 2022 · Nov 5, 2022
diff --git a/.openshift-ci/ci_tests.py b/.openshift-ci/ci_tests.py
@@ -49,6 +49,7 @@ def set_dirs_after_start():
             post_start_hook=set_dirs_after_start,
         )
 
+
 class PostgresUpgradeTest(BaseTest):
     TEST_TIMEOUT = 60 * 60
     TEST_OUTPUT_DIR = "/tmp/postgres-upgrade-test-logs"
@@ -66,6 +67,7 @@ def set_dirs_after_start():
             post_start_hook=set_dirs_after_start,
         )
 
+
 class OperatorE2eTest(BaseTest):
     # TODO(ROX-12348): adjust these timeouts once we know average run times
     DEPLOY_TIMEOUT_SEC = 40 * 60
@@ -93,7 +95,7 @@ def run(self):
         print("Executing operator upgrade test")
         self.run_with_graceful_kill(
             ["make", "-C", "operator", "test-upgrade"],
-            OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC
+            OperatorE2eTest.UPGRADE_TEST_TIMEOUT_SEC,
         )
 
         print("Executing operator e2e tests")

diff --git a/.openshift-ci/post_tests.py b/.openshift-ci/post_tests.py
@@ -36,7 +36,7 @@ def run(self, test_outputs=None, test_results=None):
 
 class RunWithBestEffortMixin:
     def __init__(
-            self,
+        self,
     ):
         self.exitstatus = 0
         self.failed_commands: List[List[str]] = []
@@ -68,8 +68,8 @@ class StoreArtifacts(RunWithBestEffortMixin):
     """For tests that only need to store artifacts"""
 
     def __init__(
-            self,
-            artifact_destination_prefix=None,
+        self,
+        artifact_destination_prefix=None,
     ):
         super().__init__()
         self.artifact_destination_prefix = artifact_destination_prefix
@@ -86,8 +86,12 @@ def add_test_results(self, test_results):
         print("Storing test results in JUnit format")
         for to_dir, from_dir in test_results.items():
             self.run_with_best_effort(
-                ["scripts/ci/store-artifacts.sh", "store_test_results",
-                 from_dir, to_dir],
+                [
+                    "scripts/ci/store-artifacts.sh",
+                    "store_test_results",
+                    from_dir,
+                    to_dir,
+                ],
                 timeout=PostTestsConstants.ARTIFACTS_TIMEOUT,
             )
 
@@ -113,10 +117,10 @@ class PostClusterTest(StoreArtifacts):
     """The standard cluster test suite of debug gathering and analysis"""
 
     def __init__(
-            self,
-            collect_central_artifacts=True,
-            check_stackrox_logs=False,
-            artifact_destination_prefix=None,
+        self,
+        collect_central_artifacts=True,
+        check_stackrox_logs=False,
+        artifact_destination_prefix=None,
     ):
         super().__init__(artifact_destination_prefix=artifact_destination_prefix)
         self._check_stackrox_logs = check_stackrox_logs
@@ -222,10 +226,10 @@ class CheckStackroxLogs(StoreArtifacts):
     """When only stackrox logs and checks are required"""
 
     def __init__(
-            self,
-            check_for_stackrox_restarts=False,
-            check_for_errors_in_stackrox_logs=False,
-            artifact_destination_prefix=None,
+        self,
+        check_for_stackrox_restarts=False,
+        check_for_errors_in_stackrox_logs=False,
+        artifact_destination_prefix=None,
     ):
         super().__init__(artifact_destination_prefix=artifact_destination_prefix)
         self._check_for_stackrox_restarts = check_for_stackrox_restarts
@@ -286,11 +290,11 @@ class FinalPost(StoreArtifacts):
     """Collect logs that accumulate over multiple tests and other final steps"""
 
     def __init__(
-            self,
-            store_qa_test_debug_logs=False,
-            store_qa_spock_results=False,
-            artifact_destination_prefix="final",
-            handle_e2e_progress_failures=True,
+        self,
+        store_qa_test_debug_logs=False,
+        store_qa_spock_results=False,
+        artifact_destination_prefix="final",
+        handle_e2e_progress_failures=True,
     ):
         super().__init__(artifact_destination_prefix=artifact_destination_prefix)
         self._store_qa_test_debug_logs = store_qa_test_debug_logs

diff --git a/.openshift-ci/runners.py b/.openshift-ci/runners.py
@@ -4,6 +4,7 @@
 Common test run patterns
 """
 
+import subprocess
 from datetime import datetime
 from clusters import NullCluster
 from pre_tests import NullPreTest
@@ -35,6 +36,7 @@ def run(self):
             self.log_event("About to provision")
             self.cluster.provision()
             self.log_event("provisioned")
+            self.set_provisioned_state()
         except Exception as err:
             self.log_event("ERROR: provision failed")
             hold = err
@@ -101,8 +103,10 @@ def run_test_set(self, test_set):
                 hold = err
             try:
                 self.log_event("About to run post test", test_set)
-                test_set["post_test"].run(test_outputs=test_set["test"].test_outputs,
-                                          test_results=test_set["test"].test_results)
+                test_set["post_test"].run(
+                    test_outputs=test_set["test"].test_outputs,
+                    test_results=test_set["test"].test_results,
+                )
                 self.log_event("post test completed", test_set)
             except Exception as err:
                 self.log_event("ERROR: post test failed", test_set)
@@ -122,6 +126,9 @@ def log_event(self, msg, test_set=None):
         print(f"{marker} {time}: {msg}")
         print(marker)
 
+    def set_provisioned_state(self):
+        subprocess.check_call("tests/e2e/lib.sh set_provisioned_state", shell=True)
+
 
 # pylint: disable=too-many-arguments
 class ClusterTestRunner(ClusterTestSetsRunner):

diff --git a/scripts/ci/lib.sh b/scripts/ci/lib.sh
@@ -4,6 +4,7 @@
 
 SCRIPTS_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
 source "$SCRIPTS_ROOT/scripts/lib.sh"
+source "$SCRIPTS_ROOT/scripts/ci/test_state.sh"
 
 set -euo pipefail
 
@@ -544,6 +545,8 @@ poll_for_system_test_images() {
         fi
         sleep 60
     done
+
+    touch "${STATE_IMAGES_AVAILABLE}"
 }
 
 check_rhacs_eng_image_exists() {
@@ -1348,6 +1351,27 @@ __EOM__
     }
 }
 
+save_junit_success() {
+    if [[ "$#" -ne 2 ]]; then
+        die "missing args. usage: save_junit_success <class> <description>"
+    fi
+
+    if [[ -z "${ARTIFACT_DIR}" ]]; then
+        info "Warning: save_junit_success() requires the \$ARTIFACT_DIR variable to be set"
+        return
+    fi
+
+    local class="$1"
+    local description="$2"
+
+    cat << EOF > "${ARTIFACT_DIR}/junit-${class}.xml"
+<testsuite name="${class}" tests="1" skipped="0" failures="0" errors="0">
+    <testcase name="${description}" classname="${class}">
+    </testcase>
+</testsuite>
+EOF
+}
+
 save_junit_failure() {
     if [[ "$#" -ne 3 ]]; then
         die "missing args. usage: save_junit_failure <class> <description> <details>"

diff --git a/scripts/ci/test_state.sh b/scripts/ci/test_state.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# State files used to track test progress across script invocations.
+
+export STATE_CLUSTER_PROVISIONED="/tmp/stackrox_ci_state_cluster_provisioned"
+export STATE_IMAGES_AVAILABLE="/tmp/stackrox_ci_state_images_available"
+export STATE_DEPLOYED="/tmp/stackrox_ci_state_deployed"
diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh
@@ -6,12 +6,9 @@ set -euo pipefail
 # Test utility functions
 
 TEST_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/../.. && pwd)"
-
-# State
-STATE_DEPLOYED="/tmp/state_deployed"
-
 source "$TEST_ROOT/scripts/lib.sh"
 source "$TEST_ROOT/scripts/ci/lib.sh"
+source "$TEST_ROOT/scripts/ci/test_state.sh"
 
 deploy_stackrox() {
     deploy_central
@@ -413,9 +410,42 @@ db_backup_and_restore_test() {
 handle_e2e_progress_failures() {
     info "Checking for deployment failure"
 
-    if [[ ! -f "${STATE_DEPLOYED}" ]]; then
-        save_junit_failure "Stackrox_Deployment" "Could not deploy StackRox" "Check the build log" || true
+    local cluster_provisioned=("Cluster_Provision" "Is the cluster available?")
+    local images_available=("Image_Availability" "Are the required images are available?")
+    local stackrox_deployed=("Stackrox_Deployment" "Was Stackrox was deployed to the cluster?")
+
+    local check_images=false
+    local check_deployment=false
+
+    if [[ -f "${STATE_CLUSTER_PROVISIONED}" ]]; then
+        save_junit_success "${cluster_provisioned[@]}" || true
+        check_images=true
+    else
+        save_junit_failure "${cluster_provisioned[@]}" \
+            "It appears that there is no cluster to test against."
     fi
+
+    if $check_images; then
+        if [[ -f "${STATE_IMAGES_AVAILABLE}" ]]; then
+            save_junit_success "${images_available[@]}" || true
+            check_deployment=true
+        else
+            save_junit_failure "${images_available[@]}" \
+                "Did the images build OK? If yes then the poll_for_system_test_images() timeout might need to be increased."
+        fi
+    fi
+
+    if $check_deployment; then
+        if [[ -f "${STATE_DEPLOYED}" ]]; then
+            save_junit_success "${stackrox_deployed[@]}" || true
+        else
+            save_junit_failure "${stackrox_deployed[@]}" "Check the build log" || true
+        fi
+    fi
+}
+
+set_provisioned_state() {
+    touch "${STATE_CLUSTER_PROVISIONED}"
 }
 
 setup_automation_flavor_e2e_cluster() {