Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
316 changes: 316 additions & 0 deletions .github/workflows/_qa-e2e-tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,316 @@
name: _qa-e2e-tests

on:
workflow_call:
inputs:
ci-job-name:
description: "Unique job name, e.g. gke-qa-e2e-tests"
required: true
type: string
trigger-label:
description: "PR label that triggers this workflow, e.g. qa-e2e-tests-gke"
required: true
type: string
extra-env-vars:
description: "Flavor-specific env vars as multi-line KEY=VALUE pairs"
required: false
type: string
default: ""

# ----- Infra cluster provisioning inputs -----
infra-flavor:
description: "Infra cluster flavor, e.g. gke-default, ocp-4"
required: false
type: string
default: "gke-default"
infra-args:
description: "Infra cluster args, e.g. nodes=3,machine-type=e2-standard-8"
required: false
type: string
default: "nodes=3,machine-type=e2-standard-8"
infra-lifespan:
description: "Infra cluster lifespan (auto-deleted after expiry)"
required: false
type: string
default: "5h"

# ----- Timeout inputs -----
cluster-wait-timeout-minutes:
description: "Timeout for waiting on cluster provisioning (OCP needs longer)"
required: false
type: number
default: 25
job-timeout-minutes:
description: "Overall job timeout"
required: false
type: number
default: 360
part1-timeout-minutes:
description: "Part 1 test timeout"
required: false
type: number
default: 250
part2-timeout-minutes:
description: "Part 2 test timeout"
required: false
type: number
default: 35

defaults:
run:
shell: bash

jobs:
wait-for-images:
if: >-
!github.event.pull_request.head.repo.fork && (
github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, inputs.trigger-label)
)
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # ratchet:actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: ./.github/actions/handle-tagged-build
- name: Compute image tags
run: |
tag="${MAIN_IMAGE_TAG:-"$(make --quiet --no-print-directory tag)"}"
# stackrox-operator uses the main tag with 'x' replaced by '0';
# stackrox-operator-bundle additionally gets a 'v' prefix (applied below).
operator_tag="${tag//x/0}"
{
echo "tag=${tag}"
echo "operator_tag=${operator_tag}"
} | tee -a "$GITHUB_ENV"
- name: Wait for images
uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1
with:
token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
# collector has its own version (COLLECTOR_VERSION), but the build
# pipeline retags it with the main tag (see push_matching_collector_scanner_images).
image: |
rhacs-eng/main:${{ env.tag }}
rhacs-eng/central-db:${{ env.tag }}
rhacs-eng/scanner-v4:${{ env.tag }}
rhacs-eng/scanner-v4-db:${{ env.tag }}
rhacs-eng/collector:${{ env.tag }}
rhacs-eng/stackrox-operator:${{ env.operator_tag }}
rhacs-eng/stackrox-operator-bundle:v${{ env.operator_tag }}

qa-e2e-tests:
needs: [wait-for-images]
if: >-
!github.event.pull_request.head.repo.fork && (
github.event_name != 'pull_request' ||
contains(github.event.pull_request.labels.*.name, inputs.trigger-label)
)
runs-on: ubuntu-latest
env:
CI_JOB_NAME: ${{ inputs.ci-job-name }}
ARTIFACT_DIR: ./junit-reports/
USE_LOCAL_ROXCTL: "true"
ROX_RISK_REPROCESSING_INTERVAL: 15s
ROX_SENSOR_CONNECTION_RETRY_MAX_INTERVAL: 30s
GCP_SERVICE_ACCOUNT_STACKROX_CI: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
# 28-char limit enforced by infra service
CLUSTER_NAME: qa-${{ github.run_id }}-${{ github.run_attempt }}
GOOGLE_CREDENTIALS_GCR_SCANNER_V2: ${{ secrets.GOOGLE_CREDENTIALS_GCR_SCANNER_V2 }}
QA_TEST_DEBUG_LOGS: /tmp/qa-tests-backend-logs
timeout-minutes: ${{ inputs.job-timeout-minutes }}
steps:
# ================================================================
# Shared setup
# ================================================================

- name: Checkout repo
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # ratchet:actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- uses: ./.github/actions/handle-tagged-build

- uses: ./.github/actions/job-preamble
with:
gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}

- name: Docker login to Quay.io
env:
REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
run: docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}"

- name: Set flavor-specific env vars
env:
EXTRA_ENV_VARS: ${{ inputs.extra-env-vars }}
run: |
while IFS= read -r line; do
if [[ -n "$line" ]]; then
echo "$line" >> "$GITHUB_ENV"
fi
done <<< "$EXTRA_ENV_VARS"

# ================================================================
# Cluster provisioning via infra
# ================================================================

- name: Trigger cluster creation
uses: stackrox/actions/infra/create-cluster@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/infra/create-cluster@v1
with:
token: ${{ secrets.INFRA_TOKEN }}
flavor: ${{ inputs.infra-flavor }}
name: ${{ env.CLUSTER_NAME }}
lifespan: ${{ inputs.infra-lifespan }}
args: ${{ inputs.infra-args }}
wait: false

- name: Set up Cloud SDK
if: startsWith(inputs.infra-flavor, 'gke')
uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # ratchet:google-github-actions/setup-gcloud@v3
with:
install_components: "gke-gcloud-auth-plugin"

- uses: stackrox/actions/infra/install-infractl@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/infra/install-infractl@v1

- name: Install roxctl
uses: stackrox/roxctl-installer-action@47fb4f5b275066b8322369e6e33fa010915b0d13 # ratchet:stackrox/roxctl-installer-action@v1
with:
roxctl-release: latest

- name: Install roxie
run: |
curl -fsSL --retry 5 --retry-all-errors -o /usr/local/bin/roxie https://github.com/stackrox/roxie/releases/download/v0.4.0/roxie-linux-amd64
chmod +x /usr/local/bin/roxie

- name: Wait for cluster and get kubeconfig
timeout-minutes: ${{ inputs.cluster-wait-timeout-minutes }}
run: |
infractl wait "${CLUSTER_NAME}"
infractl artifacts "${CLUSTER_NAME}" --download-dir artifacts > /dev/null
echo "KUBECONFIG=$(pwd)/artifacts/kubeconfig" >> "$GITHUB_ENV"

- name: Verify kubectl access
run: |
kubectl cluster-info
kubectl get nodes

- name: Create artifact directory
run: mkdir -p "$ARTIFACT_DIR"

- name: Download Go module dependencies
# The qa-tests-backend Makefile copies scanner protos from the Go module
# cache (via go list -m). Unlike Prow's apollo-ci image, GHA runners
# don't have the cache pre-populated.
run: go mod download github.com/stackrox/scanner

- name: Run pre-test
shell: python
env:
QUAY_RHACS_ENG_BEARER_TOKEN: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
PYTHONUNBUFFERED: "1"
run: |
import sys
sys.path.append('.openshift-ci')
from pre_tests import PreSystemTests
PreSystemTests().run()

- name: Run Part 1 tests
env:
QUAY_RHACS_ENG_RO_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
QUAY_RHACS_ENG_RO_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
GOOGLE_GCS_BACKUP_SERVICE_ACCOUNT_V2: ${{ secrets.GOOGLE_GCS_BACKUP_SERVICE_ACCOUNT_V2 }}
GCP_GCS_BACKUP_TEST_BUCKET_NAME_V2: ${{ secrets.GCP_GCS_BACKUP_TEST_BUCKET_NAME_V2 }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_ECR_REGISTRY_NAME: ${{ secrets.AWS_ECR_REGISTRY_NAME }}
AWS_ECR_REGISTRY_REGION: ${{ secrets.AWS_ECR_REGISTRY_REGION }}
BUILD_ID: ${{ github.run_id }}
timeout-minutes: ${{ inputs.part1-timeout-minutes }}
run: qa-tests-backend/scripts/run-part-1.sh

- name: Run Part 2 tests
env:
BUILD_ID: ${{ github.run_id }}
timeout-minutes: ${{ inputs.part2-timeout-minutes }}
run: qa-tests-backend/scripts/run-part-2.sh

# ================================================================
# Shared post-test & reporting
# ================================================================

- name: Persist env vars for post-test
if: always() && env.KUBECONFIG != ''
run: |
source tests/e2e/lib.sh
wait_for_api || true
echo "API_ENDPOINT=${API_ENDPOINT:-}" >> "$GITHUB_ENV"
if [[ -f deploy/k8s/central-deploy/password ]]; then
echo "ROX_ADMIN_PASSWORD=$(cat deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV"
fi

- name: Run post-test
if: always() && env.KUBECONFIG != ''
shell: python
env:
PYTHONUNBUFFERED: "1"
ROX_USERNAME: admin
run: |
import os, sys
sys.path.append('.openshift-ci')
from post_tests import PostClusterTest
post = PostClusterTest(
check_stackrox_logs=True,
artifact_destination_prefix="part-1",
)
if os.environ.get("ORCHESTRATOR_FLAVOR") == "k8s":
post.openshift_namespaces = []
post.run(['/tmp/qa-part1-misc-logs', os.environ.get('ARTIFACT_DIR', './junit-reports/')])

- name: Run final-post
if: always()
shell: python
env:
PYTHONUNBUFFERED: "1"
run: |
import sys
sys.path.append('.openshift-ci')
from post_tests import FinalPost
FinalPost(store_qa_tests_data=True).run()

# ================================================================
# Cluster teardown
# ================================================================

- name: Delete cluster
if: always()
run: infractl delete "${CLUSTER_NAME}" || echo "infractl delete failed; relying on lifespan expiry"

# ================================================================
# Shared reporting
# ================================================================

- name: Publish test summary
uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # ratchet:test-summary/action@v2
if: always()
with:
paths: ${{ env.ARTIFACT_DIR }}/**/*.xml
show: all

- name: Report junit failures in jira
if: (!cancelled())
id: junit2jira
uses: ./.github/actions/junit2jira
with:
create-jiras: ${{ github.event_name == 'push' }}
jira-user: ${{ secrets.JIRA_USER }}
jira-token: ${{ secrets.JIRA_TOKEN }}
gcp-account: ${{ secrets.GCP_SERVICE_ACCOUNT_STACKROX_CI }}
directory: ${{ env.ARTIFACT_DIR }}
36 changes: 36 additions & 0 deletions .github/workflows/qa-e2e-tests-gke.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
name: qa-e2e-tests-gke

on:
workflow_dispatch:
push:
branches:
- master
pull_request:
types:
- opened
- reopened
- labeled
- synchronize

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
run:
uses: ./.github/workflows/_qa-e2e-tests.yaml
with:
ci-job-name: gke-qa-e2e-tests
trigger-label: qa-e2e-tests-gke
infra-flavor: gke-default
infra-args: nodes=3,machine-type=e2-standard-8
infra-lifespan: 5h
extra-env-vars: |
ORCHESTRATOR_FLAVOR=k8s
KUBERNETES_PROVIDER=gke
OUTPUT_FORMAT=helm
GCP_IMAGE_TYPE=cos_containerd
SCANNER_V4_DB_STORAGE_CLASS=stackrox-gke-ssd
LOAD_BALANCER=lb
USE_ROXIE_DEPLOY=true
secrets: inherit
4 changes: 3 additions & 1 deletion tests/e2e/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,9 @@ deploy_stackrox_with_roxie() {

# Persist and load (extended) roxie environment, mimicking the effect of ci_export in a more concise way.
extend_roxie_envrc "$roxie_envrc"
if [[ -n "${BASH_ENV:-}" ]]; then
if is_GITHUB_ACTIONS; then
sed < "$roxie_envrc" -e 's/^export //' >> "$GITHUB_ENV"
elif [[ -n "${BASH_ENV:-}" ]]; then
cat "$roxie_envrc" >> "$BASH_ENV"
fi
# shellcheck source=/dev/null
Expand Down
Loading