stackrox · davdhacs · Jun 9, 2026 · Jun 9, 2026 · Jun 9, 2026 · Jun 9, 2026
@@ -37,9 +37,9 @@ runs:
           go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}-
           go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-
 
-    - name: Cache Go Build (restore)
+    - name: Cache Go Build (restore+save for experiment)
       if: ${{ !(inputs.save == 'true' && (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch)) }}
-      uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # ratchet:actions/cache/restore@v5
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # ratchet:actions/cache@v5
       with:
         path: ${{ steps.cache-paths.outputs.GOCACHE }}
         key: go-build-v2-${{ github.job }}${{ steps.cache-paths.outputs.KEY_SUFFIX }}-${{ steps.cache-paths.outputs.GOARCH }}-${{ steps.cache-paths.outputs.GOMOD_HASH }}-${{ steps.cache-paths.outputs.TAG }}
@@ -51,16 +51,10 @@ runs:
       run: |
         gocache="${{ steps.cache-paths.outputs.GOCACHE }}"
         if [[ -d "$gocache" ]]; then
-          # Backdate all cache entries to year 2000. During the build/test,
-          # Go's markUsed() updates mtimes of accessed entries to "now"
-          # (it always updates when mtime is >1 hour old). The post-step
-          # below trims entries still at year 2000 before cache save.
-          timeout 120 find "$gocache" -type f -exec touch -t 200001010000 {} + || true
-          # Protect trim.txt: if backdated, Go's built-in Trim() sees
-          # "last trim was in year 2000" and deletes ALL backdated entries
-          # before the build starts. Setting it to now prevents this.
-          echo "$(date +%s)" > "$gocache/trim.txt"
-          echo "Marked GOCACHE entries for stale detection"
+          # EXPERIMENT: skip backdate+trim to test if it's deleting coverage
+          # profile entries and causing test cache misses.
+          echo "Skipping GOCACHE backdate (experiment — testing trim impact)"
+          echo "GOCACHE size: $(du -sm "$gocache" 2>/dev/null | cut -f1)MB"
         fi
       shell: bash
 

@@ -18,10 +18,13 @@ concurrency:
 
 env:
   # Stable version ldflags for test caching: Go's test cache keys include
-  # link ActionID which includes -X ldflags. Fixed BUILD_TAG/SHORTCOMMIT
-  # prevent per-commit cache misses.
+  # link ActionID which includes -X ldflags. ALL version vars must be fixed
+  # or any change invalidates every test binary's cache entry.
   BUILD_TAG: 0.0.0
   SHORTCOMMIT: "0000000"
+  STABLE_COLLECTOR_VERSION: 0.0.0
+  STABLE_FACT_VERSION: 0.0.0
+  STABLE_SCANNER_VERSION: 0.0.0
 
 jobs:
   detect-changes:
@@ -37,12 +40,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        gotags: [ 'GOTAGS=""', 'GOTAGS=release' ]
-        exclude:
-          - gotags: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-release-build') && 'GOTAGS=release' }}
+        gotags: [ 'GOTAGS=""' ]
+        copy: [1]
     runs-on: ubuntu-latest
-    outputs:
-      new-jiras: ${{ steps.junit2jira.outputs.new-jiras }}
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # ratchet:actions/checkout@v6
@@ -57,32 +57,75 @@ jobs:
       with:
         key-suffix: ${{ matrix.gotags }}
 
-    - name: Go Unit Tests
-      run: ${{ matrix.gotags }} make go-unit-tests
-
-    - uses: codecov/codecov-action@ab904c41d6ece82784817410c45d8b8c02684457 # ratchet:codecov/codecov-action@v3
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        flags: go-unit-tests
+    - name: Cache debug — single package with GODEBUG
+      if: matrix.copy == 1
+      run: |
+        echo "## GODEBUG cache trace" >> "$GITHUB_STEP_SUMMARY"
+        echo '```' >> "$GITHUB_STEP_SUMMARY"
+        GODEBUG=gocachetest=1 ${{ matrix.gotags }} CGO_ENABLED=1 GOEXPERIMENT=cgocheck2 \
+          bash scripts/go-test.sh -race -cover -coverprofile /dev/null \
+          github.com/stackrox/rox/pkg/set \
+          github.com/stackrox/rox/central/cluster/util \
+          github.com/stackrox/rox/central/notifiers/slack \
+          2>&1 | grep testcache >> "$GITHUB_STEP_SUMMARY" || true
+        echo '```' >> "$GITHUB_STEP_SUMMARY"
 
-    - name: Generate junit report
-      if: always()
-      run: make generate-junit-reports
+    - name: Go Unit Tests
+      run: ${{ matrix.gotags }} make go-unit-tests 2>&1 | tee /tmp/go-test-output.txt
 
-    - name: Publish Test Report
-      uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2
+    - name: Cache and CPU diagnostics
       if: always()
-      with:
-        paths: 'junit-reports/report.xml'
-
-    - name: Go Integration Unit Tests
-      run: ${{ matrix.gotags }} make integration-unit-tests
-
-    - name: Go Operator Integration Tests
-      run: ${{ matrix.gotags }} make -C operator/ test-integration
+      run: |
+        echo "## Cache & CPU Diagnostics (copy ${{ matrix.copy }})" >> "$GITHUB_STEP_SUMMARY"
+        echo "" >> "$GITHUB_STEP_SUMMARY"
+
+        # CPU
+        cpu_model=$(grep -m1 'model name' /proc/cpuinfo 2>/dev/null | cut -d: -f2 | xargs)
+        echo "**CPU:** \`${cpu_model}\`" >> "$GITHUB_STEP_SUMMARY"
+        echo "" >> "$GITHUB_STEP_SUMMARY"
+
+        # GOCACHE size
+        gocache="$(go env GOCACHE)"
+        cache_size=$(du -sm "$gocache" 2>/dev/null | cut -f1)
+        echo "**GOCACHE size:** ${cache_size}MB" >> "$GITHUB_STEP_SUMMARY"
+        echo "" >> "$GITHUB_STEP_SUMMARY"
+
+        # Test cache hit rate
+        if [[ -f /tmp/go-test-output.txt ]]; then
+          cached=$(grep -c '(cached)' /tmp/go-test-output.txt || true)
+          total=$(grep -cE 'ok\s+github\.com/stackrox' /tmp/go-test-output.txt || true)
+          failed=$(grep -cE 'FAIL\s+github\.com/stackrox' /tmp/go-test-output.txt || true)
+          noncached=$((total - cached))
+          if [[ $total -gt 0 ]]; then
+            pct=$((cached * 100 / total))
+          else
+            pct=0
+          fi
+          echo "**Test cache:** ${cached} cached / ${total} total (${pct}% hit rate), ${failed} failed" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+        fi
 
-    - name: Go Operator Helm Tests
-      run: ${{ matrix.gotags }} make -C operator/ test-helm
+        # Sample compile actionID from a stable package to detect cross-CPU differences
+        # pkg/set is small, pure Go, no external deps — ideal canary
+        echo "**Compile action IDs (canary packages):**" >> "$GITHUB_STEP_SUMMARY"
+        echo '```' >> "$GITHUB_STEP_SUMMARY"
+        for pkg in github.com/stackrox/rox/pkg/set github.com/stackrox/rox/pkg/utils github.com/stackrox/rox/pkg/version; do
+          id=$(go test -v -list '^$' -run '^$' "$pkg" 2>&1 | head -1 || true)
+          buildid=$(go test -c -o /dev/null "$pkg" 2>&1; go tool buildid "$(go env GOCACHE)"/??/*.a 2>/dev/null | head -3 || true)
+          # Use go's cache debug to get the test binary buildID
+          test_exe=$(mktemp)
+          go test -c -o "$test_exe" "$pkg" 2>/dev/null
+          if [[ -f "$test_exe" && -s "$test_exe" ]]; then
+            bid=$(go tool buildid "$test_exe" 2>/dev/null || true)
+            echo "$pkg: $bid" >> "$GITHUB_STEP_SUMMARY"
+          fi
+          rm -f "$test_exe"
+        done
+        echo '```' >> "$GITHUB_STEP_SUMMARY"
+
+        # Restored cache key
+        echo "" >> "$GITHUB_STEP_SUMMARY"
+        echo "**Go version:** $(go version)" >> "$GITHUB_STEP_SUMMARY"
 
     - name: Generate junit report
       if: always()
@@ -94,17 +137,6 @@ jobs:
       with:
         paths: 'junit-reports/report.xml'
 
-    - name: Report test failures to Jira
-      if: (!cancelled())
-      id: junit2jira
-      uses: ./.github/actions/junit2jira
-      with:
-        create-jiras: ${{ github.event_name == 'push' }}
-        jira-user: ${{ secrets.JIRA_USER }}
-        jira-token: ${{ secrets.JIRA_TOKEN }}
-        gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
-        directory: 'junit-reports'
-
   go-postgres:
     strategy:
       fail-fast: false

diff --git a/status.sh b/status.sh
@@ -1,9 +1,10 @@
 #!/bin/sh
 
 # Note: This requires .git directory in the build context (e.g. builder container)
+# Env var overrides allow CI to stabilize ldflags for test caching.
 echo "STABLE_MAIN_VERSION $(make --quiet --no-print-directory tag)"
-echo "STABLE_COLLECTOR_VERSION $(make --quiet --no-print-directory collector-tag)"
-echo "STABLE_FACT_VERSION $(make --quiet --no-print-directory fact-tag)"
-echo "STABLE_SCANNER_VERSION $(make --quiet --no-print-directory scanner-tag)"
+echo "STABLE_COLLECTOR_VERSION ${STABLE_COLLECTOR_VERSION:-$(make --quiet --no-print-directory collector-tag)}"
+echo "STABLE_FACT_VERSION ${STABLE_FACT_VERSION:-$(make --quiet --no-print-directory fact-tag)}"
+echo "STABLE_SCANNER_VERSION ${STABLE_SCANNER_VERSION:-$(make --quiet --no-print-directory scanner-tag)}"
 echo "STABLE_SCANNER_V4_VULNERABILITY_VERSION $(cat scanner/VULNERABILITY_VERSION)"
 echo "STABLE_GIT_SHORT_SHA $(make --quiet --no-print-directory shortcommit)"