if ! is_CI; then

die "A CI environment is required."

fi

if [[ "$#" -ne 2 ]]; then

die "missing args. usage: ci_export <env-name> <env-value>"

fi

local env_name="$1"

local env_value="$2"

if command -v cci-export >/dev/null; then

cci-export "$env_name" "$env_value"

else

export "$env_name"="$env_value"

fi

if [[ "$#" -ne 2 ]]; then

die "missing args. usage: set_ci_shared_export <env-name> <env-value>"

fi

ci_export "$@"

local env_name="$1"

local env_value="$2"

printf 'export %s=%q\n' "${env_name}" "${env_value}" >> "${SHARED_DIR:-/tmp}/shared_env"

printf '%s=%q\n' "${env_name}" "${env_value}" >> "${GITHUB_ENV:-/dev/null}"

local exit_code="$?"

info "Executing a general purpose exit trap for CI"

echo "Exit code is: ${exit_code}"

if [[ "${exit_code}" == "0" ]]; then

set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_PASSED}"

elif [[ "${exit_code}" == "130" ]]; then

set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_CANCELED}"

else

set_ci_shared_export JOB_DISPATCH_OUTCOME "${OUTCOME_FAILED}"

fi

post_process_test_results "${JOB_SLACK_FAILURE_ATTACHMENTS}" "${JOB_JUNIT2JIRA_SUMMARY_FILE}"

while [[ -e /tmp/hold ]]; do

info "Holding this job for debug"

sleep 60

done

handle_dangling_processes

gate_flaky_tests "${exit_code}"

info "Process state at exit:"

ps -e -O ppid

local psline pid ppid

ps -e -O ppid | while read -r pid ppid psline; do

# Example output:

# PID PPID S TTY TIME COMMAND

# 1 0 S ? 00:00:00 /tmp/entrypoint-wrapper/entrypoint-wrapper /tools/entrypoint

# [...]

# 179283 25 R ? 00:00:00 ps -e -O ppid

# trim leading whitespace

psline="$pid $ppid $psline"

if [[ "$pid" == "PID" ]]; then

# Ignoring header

continue

fi

if [[ "$pid" == "$$" ]]; then

echo "Ignoring self: $psline"

continue

fi

if [[ "$ppid" == "$$" ]]; then

echo "Ignoring child: $psline"

continue

fi

if [[ "$psline" =~ entrypoint|defunct ]]; then

echo "Ignoring ci-operator entrypoint or defunct process: $psline"

continue

fi

echo "A candidate to kill: $psline"

echo "Will kill $pid"

kill "$pid" || {

echo "Error killing $pid"

}

done

info "Setting up the deployment environment"

if [[ "$#" -ne 2 ]]; then

die "missing args. usage: setup_deployment_env <docker-login> <use-websocket>"

fi

local docker_login="$1"

local use_websocket="$2"

if [[ "$docker_login" == "true" ]]; then

registry_ro_login "quay.io/rhacs-eng"

fi

if [[ "$use_websocket" == "true" ]]; then

ci_export CLUSTER_API_ENDPOINT "wss://central.stackrox:443"

fi

ci_export REGISTRY_USERNAME "$QUAY_RHACS_ENG_RO_USERNAME"

ci_export REGISTRY_PASSWORD "$QUAY_RHACS_ENG_RO_PASSWORD"

if [[ -z "${MAIN_IMAGE_TAG:-}" ]]; then

ci_export MAIN_IMAGE_TAG "$(make --quiet --no-print-directory tag)"

fi

ci_export ROX_PRODUCT_BRANDING "RHACS_BRANDING"

info "Getting a central debug dump"

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: get_central_debug_dump <output_dir>"

fi

local output_dir="$1"

require_environment "API_ENDPOINT"

require_environment "ROX_ADMIN_PASSWORD"

roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \

central debug dump --output-dir "${output_dir}"

ls -l "${output_dir}"

info "Processing metrics from central debug dump"

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: process_central_metrics <debug_dump_dir>"

fi

local output_dir="$1"

local metrics_output

local csv_output

local debug_dump_zip

metrics_output="$(mktemp --suffix=.prom)"

csv_output="$(mktemp --suffix=.csv)"

# shellcheck disable=SC2012

debug_dump_zip="$(ls -t "${output_dir}"/*.zip | head -1)"

unzip -p "${debug_dump_zip}" metrics-1 > "${metrics_output}"

get_prometheus_metrics_parser

# We need a link to repository. In case it's not part of job spec (e.g., periodic`s)

# we will fallback to short commit

base_link="$(echo "$JOB_SPEC" | jq ".refs.base_link | select( . != null )" -r)"

calculated_base_link="https://github.com/stackrox/stackrox/commit/$(make --quiet --no-print-directory shortcommit)"

local metadata="build_tag=${STACKROX_BUILD_TAG:-none},build_id=${BUILD_ID:-none},orchestrator_flavor=${ORCHESTRATOR_FLAVOR:-PROW},job_name=${JOB_NAME:-missing},base_link=${base_link:-$calculated_base_link}"

prometheus-metric-parser single \

--format csv \

--file "${metrics_output}" \

--labels "${metadata}" \

> "${csv_output}"

setup_gcp

save_central_metrics "${csv_output}"

local parserBin

local parserDir

parserBin=$(make prometheus-metric-parser -C "$ROOT" --silent | tail -1)

parserDir=$(dirname "${parserBin}")

export PATH="$parserDir":$PATH

prometheus-metric-parser help

info "Getting central diagnostics"

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: get_central_diagnostics <output_dir>"

fi

local output_dir="$1"

require_environment "API_ENDPOINT"

require_environment "ROX_ADMIN_PASSWORD"

roxctl -e "${API_ENDPOINT}" --ca "" --insecure-skip-tls-verify \

central debug download-diagnostics --output-dir "${output_dir}"

ls -l "${output_dir}"

info "Pushing main, roxctl and central-db images as manifest lists"

if [[ "$#" -ne 3 ]]; then

die "missing arg. usage: push_image_manifest_lists <push_context> <brand> <architectures (CSV)>"

fi

local push_context="$1"

local brand="$2"

local architectures="$3"

local main_image_set=("main" "roxctl" "central-db")

local registry

registry="$(registry_from_branding "$brand")"

local tag

tag="$(make --quiet --no-print-directory tag)"

registry_rw_login "$registry"

for image in "${main_image_set[@]}"; do

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat

if [[ "$push_context" == "merge-to-master" ]]; then

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:latest" "$architectures" | cat

fi

done

# Push manifest lists for scanner and collector for amd64 only

local amd64_image_set=("scanner" "scanner-db" "scanner-slim" "scanner-db-slim" "collector")

for image in "${amd64_image_set[@]}"; do

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "amd64" | cat

done

local branding="$1"

if [[ "$branding" == "STACKROX_BRANDING" ]]; then

registry="quay.io/stackrox-io"

elif [[ "$branding" == "RHACS_BRANDING" ]]; then

registry="quay.io/rhacs-eng"

else

die "$branding is not a supported branding"

fi

echo "$registry"

info "Pushing main, roxctl and central-db images"

if [[ "$#" -ne 3 ]]; then

die "missing arg. usage: push_main_image_set <push_context> <brand> <arch>"

fi

local push_context="$1"

local brand="$2"

local arch="$3"

local main_image_set=("main" "roxctl" "central-db")

_push_main_image_set() {

local registry="$1"

local tag="$2"

for image in "${main_image_set[@]}"; do

retry 5 true \

docker push "${registry}/${image}:${tag}" | cat

done

}

_tag_main_image_set() {

local local_tag="$1"

local registry="$2"

local remote_tag="$3"

for image in "${main_image_set[@]}"; do

docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"

done

}

local registry

registry="$(registry_from_branding "$brand")"

local tag

tag="$(make --quiet --no-print-directory tag)"

registry_rw_login "$registry"

_tag_main_image_set "$tag" "$registry" "$tag-$arch"

_push_main_image_set "$registry" "$tag-$arch"

if [[ "$push_context" == "merge-to-master" ]]; then

_tag_main_image_set "$tag" "$registry" "latest-${arch}"

_push_main_image_set "$registry" "latest-${arch}"

fi

info "Pushing stackrox-operator image"

if [[ "$#" -ne 3 ]]; then

die "Missing parameter. Usage: push_operator_image <push_context> <brand> <arch>"

fi

local push_context="$1" # Allowed to be empty.

local brand="$2"

local arch="$3"

if [[ "$brand" == "" ]]; then

die "Brand must be non-empty"

fi

if [[ "$arch" == "" ]]; then

die "Arch must be non-empty"

fi

_push_operator_image() {

local registry="$1"

local tag="$2"

local arch="$3"

retry 5 true \

docker push "${registry}/stackrox-operator:${tag}-${arch}" | cat

}

local registry

registry="$(registry_from_branding "$brand")"

local tag

tag="$(make -C operator/ --quiet --no-print-directory tag)"

registry_rw_login "$registry"

docker tag "${registry}/stackrox-operator:${tag}" "${registry}/stackrox-operator:${tag}-${arch}"

_push_operator_image "$registry" "$tag" "$arch"

if [[ "$push_context" == "merge-to-master" ]]; then

docker tag "${registry}/stackrox-operator:${tag}" "${registry}/stackrox-operator:latest-${arch}"

_push_operator_image "$registry" "latest" "$arch"

fi

if [[ $tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

# For release builds, also push a major.minor tag, see operator/install/README.md

local major_minor="${tag%.*}"

docker tag "${registry}/stackrox-operator:${tag}" "${registry}/stackrox-operator:${major_minor}-${arch}"

_push_operator_image "$registry" "$major_minor" "$arch"

fi

info "Pushing scanner-v4 and scanner-v4-db images as manifest lists"

if [[ "$#" -ne 2 ]]; then

die "missing arg. usage: push_scanner_image_manifest_lists <registry> <architectures (CSV)>"

fi

local registry="$1"

local architectures="$2"

local scanner_image_set=("scanner-v4" "scanner-v4-db")

local tag

tag="$(make --quiet --no-print-directory -C scanner tag)"

registry_rw_login "$registry"

for image in "${scanner_image_set[@]}"; do

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/${image}:${tag}" "$architectures" | cat

done

info "Pushing scanner-v4 and scanner-v4-db images"

if [[ "$#" -ne 2 ]]; then

die "missing arg. usage: push_scanner_image_set <registry> <arch>"

fi

local registry="$1"

local arch="$2"

local scanner_image_set=("scanner-v4" "scanner-v4-db")

_push_scanner_image_set() {

local registry="$1"

local tag="$2"

for image in "${scanner_image_set[@]}"; do

retry 5 true \

docker push "${registry}/${image}:${tag}" | cat

done

}

_tag_scanner_image_set() {

local local_tag="$1"

local registry="$2"

local remote_tag="$3"

for image in "${scanner_image_set[@]}"; do

docker tag "stackrox/${image}:${local_tag}" "${registry}/${image}:${remote_tag}"

done

}

local tag

tag="$(make --quiet --no-print-directory -C scanner tag)"

registry_rw_login "$registry"

_tag_scanner_image_set "$tag" "$registry" "$tag-$arch"

_push_scanner_image_set "$registry" "$tag-$arch"

info "Pushing stackrox-operator images as manifest lists"

if [[ "$#" -ne 3 ]]; then

die "missing arg. usage: push_image_manifest_lists <push_context> <brand> <architectures (CSV)>"

fi

local push_context="$1"

local brand="$2"

local architectures="$3"

local registry

registry="$(registry_from_branding "$brand")"

local tag

tag="$(make -C operator --quiet --no-print-directory tag)"

registry_rw_login "$registry"

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/stackrox-operator:${tag}" "$architectures" | cat

if [[ "$push_context" == "merge-to-master" ]]; then

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/stackrox-operator:latest" "$architectures" | cat

fi

if [[ $tag =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then

# For release builds, also push a major.minor tag, see operator/install/README.md

local major_minor="${tag%.*}"

retry 5 true \

"$SCRIPTS_ROOT/scripts/ci/push-as-multiarch-manifest-list.sh" "${registry}/stackrox-operator:${major_minor}" "$architectures" | cat

fi

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: registry_rw_login <registry>"

fi

local registry="$1"

case "$registry" in

quay.io/rhacs-eng)

_login() {

# shellcheck disable=SC2317

docker login -u "$QUAY_RHACS_ENG_RW_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RW_PASSWORD" quay.io

}

retry 5 true _login

;;

quay.io/stackrox-io)

_login() {

# shellcheck disable=SC2317

docker login -u "$QUAY_STACKROX_IO_RW_USERNAME" --password-stdin <<<"$QUAY_STACKROX_IO_RW_PASSWORD" quay.io

}

retry 5 true _login

;;

*)

die "Unsupported registry login: $registry"

esac

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: registry_ro_login <registry>"

fi

local registry="$1"

case "$registry" in

quay.io/rhacs-eng)

_login() {

# shellcheck disable=SC2317

docker login -u "$QUAY_RHACS_ENG_RO_USERNAME" --password-stdin <<<"$QUAY_RHACS_ENG_RO_PASSWORD" quay.io

}

retry 5 true _login

;;

*)

die "Unsupported registry login: $registry"

esac

info "Pushing collector & scanner images tagged with main-version to quay.io/rhacs-eng"

if [[ "$#" -ne 2 ]]; then

die "missing arg. usage: push_matching_collector_scanner_images <brand> <arch>"

fi

local brand="$1"

local arch="$2"

local registry

registry="$(registry_from_branding "$brand")"

_retag() {

retry 5 true "$SCRIPTS_ROOT/scripts/ci/pull-retag-push.sh" "$1" "$2"

}

if [[ "$arch" != "amd64" ]]; then

echo "Skipping rebundling for non-amd64 arch"

exit 0

fi

local main_tag

main_tag="$(make --quiet --no-print-directory tag)"

local scanner_version

scanner_version="$(make --quiet --no-print-directory scanner-tag)"

local collector_version

collector_version="$(make --quiet --no-print-directory collector-tag)"

local fact_version

fact_version="$(make --quiet --no-print-directory fact-tag)"

registry_rw_login "${registry}"

_retag "${registry}/scanner:${scanner_version}" "${registry}/scanner:${main_tag}-${arch}"

_retag "${registry}/scanner-db:${scanner_version}" "${registry}/scanner-db:${main_tag}-${arch}"

_retag "${registry}/scanner-slim:${scanner_version}" "${registry}/scanner-slim:${main_tag}-${arch}"

_retag "${registry}/scanner-db-slim:${scanner_version}" "${registry}/scanner-db-slim:${main_tag}-${arch}"

_retag "${registry}/collector:${collector_version}" "${registry}/collector:${main_tag}-${arch}"

_retag "${registry}/fact:${fact_version}" "${registry}/fact:${main_tag}-${arch}"

info "Polling for images required for system tests"

if [[ "$#" -ne 1 ]]; then

die "missing arg. usage: poll_for_system_test_images <seconds to wait>"

fi

local time_limit="$1"

require_environment "QUAY_RHACS_ENG_BEARER_TOKEN"

local image_list

image_list="$(mktemp)"

populate_stackrox_image_list "${image_list}"

info "Will poll for: $(awk '{print $1}' "${image_list}")"

local start_time

start_time="$(date '+%s')"

local tag

local image

while read -r image tag

do

while ! check_rhacs_eng_image_exists "$image" "$tag"

do

info "$image does not exist"

if (( $(date '+%s') - start_time > time_limit )); then

check_build_workflows "$(get_commit_sha)"

die "ERROR: Timed out waiting for images after ${time_limit} seconds"

fi

sleep 60

done

done < "$image_list"

info "All images exist."

touch "${STATE_IMAGES_AVAILABLE}"

info "Starting image pre-fetcher of pre-built images (NOT built for current commit)..."

junit_wrap image-prefetcher-prebuilt-start \

"Start image pre-fetcher for pre-built images (NOT built for current commit)." \

"See log for error details." \

_image_prefetcher_prebuilt_start

info "Starting image pre-fetcher of system images (built for current commit)..."

junit_wrap image-prefetcher-system-start \

"Start image pre-fetcher for system images (built for current commit)." \

"See log for error details." \

_image_prefetcher_system_start

# NOTE: when changing this function, make corresponding changes to

# _image_prefetcher_prebuilt_await

case "$CI_JOB_NAME" in

*qa-e2e-tests)

image_prefetcher_start_set qa-e2e

# Override the default image pull policy for containers with quay.io

# images to rely on prefetched images. This helps ensure that the static

# prefect list stays up to date with additions.

ci_export "IMAGE_PULL_POLICY_FOR_QUAY_IO" "Never"

;;

*nongroovy-e2e-tests)

image_prefetcher_start_set qa-nongroovy-e2e

# Override the default image pull policy for containers with quay.io

# images to rely on prefetched images. This helps ensure that the static

# prefect list stays up to date with additions.

ci_export "IMAGE_PULL_POLICY_FOR_QUAY_IO" "Never"

;;

*-operator-e2e-tests)

image_prefetcher_start_set operator-e2e

# TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.

;;

*)

info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}."

;;

esac

# NOTE: when changing this function, make corresponding changes to

# _image_prefetcher_system_await

case "$CI_JOB_NAME" in

# ROX-24818: GKE is excluded from system image prefetch as it causes

# flakes in test.

*-operator-e2e-tests|*ocp*qa-e2e-tests)

image_prefetcher_start_set stackrox-images

;;

# Enabling scanner V4 installation tests as well, even though they also run on GKE,

# for gathering some more data points for CI reliability.

*-scanner-v4-install-tests)

image_prefetcher_start_set stackrox-images

;;

*)

info "No system image prefetching is performed for: ${CI_JOB_NAME}."

;;

esac

local ns="prefetch-images"

local name="$1"

make image-prefetcher-deploy-bin

local image_prefetcher_deploy_bin

image_prefetcher_deploy_bin="$(make print-image-prefetcher-deploy-bin)"

local image_prefetcher_version

image_prefetcher_version="$(go -C tools/test list -m -f '{{.Version}}' github.com/stackrox/image-prefetcher/deploy)"

info "Using ${image_prefetcher_deploy_bin} ${image_prefetcher_version} for image prefetch deployment"

local manifest

manifest=$(mktemp)

case "${ORCHESTRATOR_FLAVOR}" in

k8s)

flavor=vanilla

;;

openshift)

flavor=ocp

;;

*)

die "unsupported ORCHESTRATOR: ${ORCHESTRATOR_FLAVOR}"

;;

esac

# daemonset, etc

${image_prefetcher_deploy_bin} \

--version="${image_prefetcher_version}" \

--k8s-flavor="$flavor" \

--secret=stackrox \

--collect-metrics \

--namespace="$ns" \

"$name" > "$manifest"

# image list

local image_list

image_list=$(mktemp)

populate_prefetcher_image_list "$name" "${image_list}"

echo "---" >> "$manifest"

kubectl create --dry-run=client -o yaml configmap "$name" --from-file="images.txt=$image_list" >> "$manifest"

# pull secret

REGISTRY_PASSWORD="${QUAY_RHACS_ENG_RO_PASSWORD}" \

REGISTRY_USERNAME="${QUAY_RHACS_ENG_RO_USERNAME}" \

NAMESPACE=$ns \

make -C operator stackrox-image-pull-secret

# apply configmap, daemonset etc

retry 5 true kubectl apply --namespace=$ns -f "$manifest"

info "Image pre-fetcher is now running in the background. Its status will be checked later (look for message starting with ${image_prefetcher_await_msg_prefix}). Proceeding with other tasks in the meantime."

rm -f "$image_list" "$manifest"

if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then

return

fi

info "${image_prefetcher_await_msg_prefix} of pre-built images to complete..."

junit_wrap image-prefetcher-prebuilt-await \

"Waiting for pre-fetcher of pre-built images to complete." \

"See log for error details." \

_image_prefetcher_prebuilt_await

if [[ "${IMAGE_PREFETCH_DISABLED:-false}" == "true" ]]; then

return

fi

info "${image_prefetcher_await_msg_prefix} of system images to complete..."

junit_wrap image-prefetcher-system-await \

"Waiting for pre-fetcher of system images to complete." \

"See log for error details." \

_image_prefetcher_system_await

case "$CI_JOB_NAME" in

# Note: when adding a case for a new test job below, add a image_prefetcher_prebuilt_await call

# at the last moment before any of the prebuilt images is used. (See other existing examples.)

# This way we save time since prefetching can happen in parallel with whatever other setup the test job needs.

*qa-e2e-tests)

image_prefetcher_await_set qa-e2e

;;

*nongroovy-e2e-tests)

image_prefetcher_await_set qa-nongroovy-e2e

;;

*-operator-e2e-tests)

image_prefetcher_await_set operator-e2e

# TODO(ROX-20508): pre-fetch images of the release from which operator upgrade test starts as well.

;;

*)

info "No pre-built image prefetching is currently performed for: ${CI_JOB_NAME}. Nothing to wait for."

;;

esac

case "$CI_JOB_NAME" in

# ROX-24818: GKE is excluded from system image prefetch as it causes

# flakes in test.

*-operator-e2e-tests|*ocp*qa-e2e-tests)

image_prefetcher_await_set stackrox-images

;;

# Enabling scanner V4 installation tests as well, even though they also run on GKE,

# for gathering some more data points for CI reliability.

*-scanner-v4-install-tests)

image_prefetcher_await_set stackrox-images

;;

*)

info "No system image prefetching is performed for: ${CI_JOB_NAME}. Nothing to wait for."

;;

esac

local ns="prefetch-images"

local name="$1"

local extra_fields='{"build_id": "'"${BUILD_ID:-}"'", "job_name": "'"${JOB_NAME:-}"'", "orchestrator": "'"${ORCHESTRATOR_FLAVOR:-}"'", "build_tag": "'"${STACKROX_BUILD_TAG:-}"'"}'

info "Waiting for image prefetcher set ${name} to complete..."

if kubectl rollout status daemonset "$name" -n "$ns" --timeout 15m; then

info "All images in the set are now pre-fetched."

else

info "WARNING: Pre-fetching failed to complete in time."

info "To investigate closer, go to https://console.cloud.google.com/bigquery and run a query such as:"

local query

query=$(mktemp)

cat > "${query}" <<- EOM

cat "${query}"

info "Note: The data is imported into the table periodically: https://github.com/stackrox/stackrox/actions/workflows/batch-load-test-metrics.yml"

if [[ -n ${ARTIFACT_DIR:-} ]]; then

local prefetcher_help="$ARTIFACT_DIR/image-pre-fetcher-${name}-failure-summary.html"

cat > "${prefetcher_help}" <<- EOM

cat >> "${prefetcher_help}" "${query}"

cat >> "${prefetcher_help}" <<- EOM

fi

rm -f "${query}"

fi

info "Now retrieving prefetcher metrics..."

local attempt=0

local service="service/${name}-metrics"

while [[ -z $(kubectl -n "${ns}" get "${service}" -o jsonpath="{.status.loadBalancer.ingress}" 2>/dev/null) ]]; do

if [ "$attempt" -lt "60" ]; then

info "Waiting for ${service} to obtain endpoint ..."

((attempt++))

sleep 10

else

info "Something is wrong with the ${service} service. See the following 'describe' output."

kubectl -n "${ns}" describe "${service}" || true

die "Timeout waiting for ${service} to obtain endpoint!"

fi

done

local endpoint

endpoint="$(kubectl -n "${ns}" get "${service}" -o json | service_get_endpoint)"

local fetcher_metrics

fetcher_metrics="$(mktemp --suffix=.csv)"

local fetcher_metrics_json

fetcher_metrics_json="$(mktemp --suffix=.json)"

local metrics_url="http://${endpoint}:8080/metrics"

if ! curl --silent --show-error --fail --retry 3 --retry-connrefused "${metrics_url}" > "${fetcher_metrics_json}"; then

die "Failed to fetch prefetcher metrics from ${metrics_url}"

fi

# See the stackrox_image_prefetches table definition in https://github.com/stackrox/automation-iac/blob/main/resources/testing/stackrox-ci/metrics.tf

# for the order of columns.

if ! jq --raw-output \

--argjson cols '["attempt_id", "started_at", "image", "duration_ms", "node", "size_bytes", "error", "build_id", "job_name", "orchestrator", "build_tag"]' \

--argjson extra "${extra_fields}" \

'map(.started_at = (.started_at | todate) | ($extra+.) as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' \

"${fetcher_metrics_json}" > "${fetcher_metrics}"; then

info "WARNING: Failed to convert image prefetcher metrics to CSV with extra fields ${extra_fields}"

info "Dumping the input JSON file:"

jq . < "${fetcher_metrics_json}"

die "Failed to convert image prefetcher metrics to CSV, aborting."

fi

rm -f "${fetcher_metrics_json}"

setup_gcp

if save_image_prefetches_metrics "${fetcher_metrics}"; then

info "Image pre-fetcher metrics retrieved and saved."

else

info "WARNING: failed to save image pre-fetcher metrics."

fi

rm -f "${fetcher_metrics}"

local name="$1"

local image_list="$2"

case "$name" in

stackrox-images)

local image_tag_list

image_tag_list=$(mktemp)

populate_stackrox_image_list "${image_tag_list}"

# convert format from "basename tag" to "quay.io/.../basename:tag" expected by pre-fetcher

awk '{print "quay.io/rhacs-eng/" $1 ":" $2}' "$image_tag_list" > "$image_list"

rm -f "$image_tag_list"

;;

operator-e2e)

# shellcheck disable=SC2046

grep PREFETCH-THIS-IMAGE -h -A 1 $(find operator/tests/ -type f) | awk '$1 == "image:" {print $2}' | sort -u > "$image_list"

;;

qa-e2e)

cp "$SCRIPTS_ROOT/qa-tests-backend/scripts/images-to-prefetch.txt" "$image_list"

;;

qa-nongroovy-e2e)

cp "$SCRIPTS_ROOT/tests/images-to-prefetch.txt" "$image_list"

;;

*)

die "ERROR: An unsupported image prefetcher target was requested: $name"

;;

esac

local image_list="$1"

local tag

tag="$(make --quiet --no-print-directory tag)"

local operator_metadata_tag

operator_metadata_tag="$(echo "v${tag}" | sed 's,x,0,')"

local operator_controller_tag="${tag//x/0}"

# Require images based on the job

case "$CI_JOB_NAME" in

*-operator-e2e-tests)

cat >> "${image_list}" << END

;;

*-race-condition-qa-e2e-tests)

cat >> "${image_list}" << END

if is_in_PR_context && ! pr_has_label "ci-build-race-condition-debug"; then

echo "ERROR: Your PR is missing the \"ci-build-race-condition-debug\" label."

echo "ERROR: This label is required to build the images for $CI_JOB_NAME."

# Quietly continue to allow labels added after tests start.

# Otherwise this message will surface in the Prow log when

# images timeout out below.

fi

;;

*-scanner-v4-install-tests)

cat >> "${image_list}" << END

;;

*)

cat >> "${image_list}" << END

;;

esac

if [[ "${DEPLOY_STACKROX_VIA_OPERATOR:-}" == "true" ]]; then

cat >> "${image_list}" << END

fi

# Remove duplicates.