Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 2 additions & 14 deletions qa-tests-backend/scripts/run-part-1.sh
Original file line number Diff line number Diff line change
Expand Up @@ -160,20 +160,8 @@ test_part_1() {
rm -f FAIL
remove_qa_test_results

local test_target
if is_openshift_CI_rehearse_PR; then
info "On an openshift rehearse PR, running BAT tests only..."
test_target="bat-test"
elif is_in_PR_context && pr_has_label ci-all-qa-tests; then
info "ci-all-qa-tests label was specified, so running all QA tests..."
test_target="test"
elif is_in_PR_context; then
info "In a PR context without ci-all-qa-tests, running BAT tests only..."
test_target="bat-test"
else
info "Running all QA tests by default..."
test_target="test"
fi
# TODO: revert this override before merging
local test_target="test"

setup_gcp
set_ci_shared_export "test_target" "${test_target}"
Expand Down
108 changes: 27 additions & 81 deletions scripts/ci/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -696,7 +696,16 @@ image_prefetcher_start_set() {
;;
ocp)
flavor=ocp
kubelet_image_creds="" # i.e. disabled
local ocp_platform
ocp_platform="$(oc get infrastructure cluster -o jsonpath='{.status.platformStatus.type}' 2>/dev/null || true)"
case "${ocp_platform}" in
AWS)
kubelet_image_creds=OCP-ECR
;;
*)
kubelet_image_creds="" # i.e. disabled
;;
esac
;;
*)
die "unsupported KUBERNETES_PROVIDER: ${KUBERNETES_PROVIDER}"
Expand All @@ -709,7 +718,6 @@ image_prefetcher_start_set() {
--version="${image_prefetcher_version}" \
--k8s-flavor="$flavor" \
--secret=stackrox \
--collect-metrics \
--namespace="$ns" \
"$name" > "$manifest"

Expand All @@ -718,15 +726,6 @@ image_prefetcher_start_set() {
image_list=$(mktemp)
populate_prefetcher_image_list "$name" "${image_list}"

# Filter out gcr.io images on non-GKE clusters (they require GKE-specific credentials)
if [[ "${KUBERNETES_PROVIDER}" != "gke" ]]; then
local filtered_image_list
filtered_image_list=$(mktemp)
info "Filtering out *.gcr.io images for non-GKE cluster"
grep -v -E '^([^/]+\.)?gcr\.io/' "${image_list}" > "${filtered_image_list}" || true
mv "${filtered_image_list}" "${image_list}"
fi

echo "---" >> "$manifest"
kubectl create --dry-run=client -o yaml configmap "$name" --from-file="images.txt=$image_list" >> "$manifest"

Expand Down Expand Up @@ -813,27 +812,13 @@ _image_prefetcher_system_await() {
image_prefetcher_await_set() {
local ns="prefetch-images"
local name="$1"
local extra_fields='{"build_id": "'"${BUILD_ID:-}"'", "job_name": "'"${JOB_NAME:-}"'", "orchestrator": "'"${ORCHESTRATOR_FLAVOR:-}"'", "build_tag": "'"${STACKROX_BUILD_TAG:-}"'"}'

info "Waiting for image prefetcher set ${name} to complete..."
if kubectl rollout status daemonset "$name" -n "$ns" --timeout 15m; then
info "All images in the set are now pre-fetched."
else
info "WARNING: Pre-fetching failed to complete in time."
info "To investigate closer, go to https://console.cloud.google.com/bigquery and run a query such as:"
local query
query=$(mktemp)
cat > "${query}" <<- EOM

SELECT started_at, duration_ms, image, error
FROM \`acs-san-stackroxci.ci_metrics.stackrox_image_prefetches\`
WHERE error IS NOT NULL AND
$(echo "${extra_fields}" | jq -r '[to_entries | .[] | select(.value != "") | (.key + "=\"" + .value + "\"")] | join(" AND ")')
ORDER BY started_at DESC LIMIT 1000

EOM
cat "${query}"
info "Note: The data is imported into the table periodically: https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml"
info "To investigate closer, see prefetcher logs in 'Additional StackRox e2e artifacts'."

if [[ -n ${ARTIFACT_DIR:-} ]]; then
local prefetcher_help="$ARTIFACT_DIR/image-pre-fetcher-${name}-failure-summary.html"
Expand All @@ -850,66 +835,12 @@ EOM
<body>

Waiting for image prefetcher set ${name} to complete timed out.<br>
To investigate closer, go to <a target="_blank" href="https://console.cloud.google.com/bigquery">BigQuery</a> and run a query such as the following:
<br>
<pre>
EOM
cat >> "${prefetcher_help}" "${query}"
cat >> "${prefetcher_help}" <<- EOM
</pre>
Note: The data is imported into the table <a target="_blank" href="https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml">periodically</a>.
<br><br>
To investigate closer, see prefetcher logs in 'Additional StackRox e2e artifacts'.
</body>
</html>
EOM
fi
rm -f "${query}"
fi
info "Now retrieving prefetcher metrics..."
local attempt=0
local service="service/${name}-metrics"
while [[ -z $(kubectl -n "${ns}" get "${service}" -o jsonpath="{.status.loadBalancer.ingress}" 2>/dev/null) ]]; do
if [ "$attempt" -lt "60" ]; then
info "Waiting for ${service} to obtain endpoint ..."
((attempt++))
sleep 10
else
info "Something is wrong with the ${service} service. See the following 'describe' output."
kubectl -n "${ns}" describe "${service}" || true
die "Timeout waiting for ${service} to obtain endpoint!"
fi
done
local endpoint
endpoint="$(kubectl -n "${ns}" get "${service}" -o json | service_get_endpoint)"
local fetcher_metrics
fetcher_metrics="$(mktemp --suffix=.csv)"
local fetcher_metrics_json
fetcher_metrics_json="$(mktemp --suffix=.json)"
local metrics_url="http://${endpoint}:8080/metrics"
if ! curl --silent --show-error --fail --retry 3 --retry-connrefused "${metrics_url}" > "${fetcher_metrics_json}"; then
die "Failed to fetch prefetcher metrics from ${metrics_url}"
fi
# See the stackrox_image_prefetches table definition in https://github.com/stackrox/automation-iac/blob/main/resources/testing/stackrox-ci/metrics.tf
# for the order of columns.
if ! jq --raw-output \
--argjson cols '["attempt_id", "started_at", "image", "duration_ms", "node", "size_bytes", "error", "build_id", "job_name", "orchestrator", "build_tag"]' \
--argjson extra "${extra_fields}" \
'map(.started_at = (.started_at | todate) | ($extra+.) as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' \
"${fetcher_metrics_json}" > "${fetcher_metrics}"; then
info "WARNING: Failed to convert image prefetcher metrics to CSV with extra fields ${extra_fields}"
info "Dumping the input JSON file:"
jq . < "${fetcher_metrics_json}"
die "Failed to convert image prefetcher metrics to CSV, aborting."
fi
rm -f "${fetcher_metrics_json}"

setup_gcp
if save_image_prefetches_metrics "${fetcher_metrics}"; then
info "Image pre-fetcher metrics retrieved and saved."
else
info "WARNING: failed to save image pre-fetcher metrics."
fi
rm -f "${fetcher_metrics}"
}

service_get_endpoint() {
Expand All @@ -935,6 +866,12 @@ populate_prefetcher_image_list() {
;;
qa-e2e)
cp "$SCRIPTS_ROOT/qa-tests-backend/scripts/images-to-prefetch.txt" "$image_list"
# Append ECR test image for qa-tests-backend/src/test/groovy/ImageScanningTest.groovy when registry ID and region are known
if [[ -n "${AWS_ECR_REGISTRY_NAME:-}" && -n "${AWS_ECR_REGISTRY_REGION:-}" ]]; then
local ecr_image="${AWS_ECR_REGISTRY_NAME}.dkr.ecr.${AWS_ECR_REGISTRY_REGION}.amazonaws.com/stackrox-qa-ecr-test:registry-image-no-secrets"
info "Adding ECR test image to prefetch list: ${ecr_image}"
echo "${ecr_image}" >> "${image_list}"
fi
;;
qa-nongroovy-e2e)
cp "$SCRIPTS_ROOT/tests/images-to-prefetch.txt" "$image_list"
Expand All @@ -949,6 +886,15 @@ populate_prefetcher_image_list() {
die "ERROR: An unsupported image prefetcher target was requested: $name"
;;
esac

# Filter out gcr.io images on non-GKE clusters (they require GKE-specific credentials)
if [[ "${KUBERNETES_PROVIDER}" != "gke" ]]; then
local filtered_image_list
filtered_image_list=$(mktemp)
info "Filtering out *.gcr.io images for non-GKE cluster"
grep -v -E '^([^/]+\.)?gcr\.io/' "${image_list}" > "${filtered_image_list}" || true
mv "${filtered_image_list}" "${image_list}"
fi
}

populate_stackrox_image_list() {
Expand Down
2 changes: 1 addition & 1 deletion tools/test/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.25

require (
github.com/jstemmer/go-junit-report/v2 v2.1.0
github.com/stackrox/image-prefetcher/deploy v0.5.1
github.com/stackrox/image-prefetcher/deploy v0.5.2-0.20260611062959-1745c737d0dc
github.com/stackrox/prometheus-metric-parser v0.0.0-20240718111853-be492d554cd5
)

Expand Down
4 changes: 2 additions & 2 deletions tools/test/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading