-
Notifications
You must be signed in to change notification settings - Fork 174
204 lines (177 loc) · 8.94 KB
/
performance-tests.yml
File metadata and controls
204 lines (177 loc) · 8.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
name: Performance tests
on:
pull_request:
branches:
- master
types:
- labeled
- synchronize
concurrency: performance-tests-${{ github.ref }}
jobs:
wait-for-images:
name: Wait for images
runs-on: ubuntu-latest
if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance-tests') }}
steps:
- name: Checkout repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Get PR image tag
run: |
MAIN_IMAGE_TAG="$(make --quiet --no-print-directory tag)"
echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
- name: Wait for images
uses: stackrox/actions/release/wait-for-image@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/release/wait-for-image@v1
with:
token: ${{ secrets.QUAY_RHACS_ENG_BEARER_TOKEN }}
image: |
rhacs-eng/main:${{ env.MAIN_IMAGE_TAG }}
rhacs-eng/roxctl:${{ env.MAIN_IMAGE_TAG }}
run-k6-performance-tests:
needs: [wait-for-images]
name: Run k6 performance tests
runs-on: ubuntu-latest
if: ${{ contains(github.event.pull_request.labels.*.name, 'ci-performance-tests') }}
env:
USE_GKE_GCLOUD_AUTH_PLUGIN: "True"
INFRA_TOKEN: ${{ secrets.INFRA_TOKEN }}
KUBECONFIG: artifacts/kubeconfig
TERM: xterm
DEFAULT_IMAGE_REGISTRY: "quay.io/rhacs-eng"
SCALE_CLUSTER_NAME: k6-test-${{ github.event.pull_request.number }}
SCALE_WORKLOAD_TYPE: default
SCALE_WORKLOAD_CLUSTERS: 10
steps:
- name: Checkout repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Checkout workflow scripts
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # ratchet:actions/checkout@v6
with:
repository: "stackrox/workflow"
path: workflow
# Create scale cluster with larger nodes.
- name: Create testing cluster
uses: stackrox/actions/infra/create-cluster@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/infra/create-cluster@v1
with:
token: ${{ secrets.INFRA_TOKEN }}
flavor: gke-default
name: ${{ env.SCALE_CLUSTER_NAME }}
lifespan: 12h
args: nodes=3,machine-type=e2-standard-32
wait: true
# Fetch infractl artifacts
- uses: stackrox/actions/infra/install-infractl@9238e423c3ae1ac4eb0f254cbb98da9daae24d86 # ratchet:stackrox/actions/infra/install-infractl@v1
- uses: 'google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093' # ratchet:google-github-actions/auth@v3
with:
credentials_json: "${{ secrets.GCP_PERFORMANCE_TESTER_CI_SA }}"
- name: "Set up Cloud SDK"
uses: 'google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db' # ratchet:google-github-actions/setup-gcloud@v3
with:
install_components: "gke-gcloud-auth-plugin"
- name: Test cluster readiness
run: |
STATUS=$(infractl get "${SCALE_CLUSTER_NAME//./-}" --json | jq -r .Status)
if [ "$STATUS" != "READY" ]; then
exit 1
fi
- name: Fetch infra artifacts
run: |
infractl artifacts "${SCALE_CLUSTER_NAME//./-}" --download-dir artifacts > /dev/null
- name: Add Docker login for Quay.io
env:
REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
run: |
docker login -u "${REGISTRY_USERNAME}" --password-stdin quay.io <<<"${REGISTRY_PASSWORD}"
- name: Clean previous installation
run: |
source ./workflow/env.sh
./scripts/k8s/cleanup.sh stackrox
# Delete all namespaces created by scale script.
for i in $(seq 1 "${SCALE_WORKLOAD_CLUSTERS}"); do
./scripts/k8s/cleanup.sh "stackrox${i}"
done
- name: Get PR image tag
run: |
MAIN_IMAGE_TAG="$(make tag)"
echo "MAIN_IMAGE_TAG=${MAIN_IMAGE_TAG}" >> "$GITHUB_ENV"
# Install Central + CentralDB
- name: Launch central services
env:
LOAD_BALANCER: lb
ROX_POSTGRES_DATASTORE: "true"
MONITORING_SUPPORT: "false"
REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
run: |
source ./workflow/env.sh
# ROX-29641: We are facing issues with workload generation on GKE when Scanner v4 is enabled.
export ROX_SCANNER_V4=false
./scale/dev/launch_central.sh
# Required for k6 tests to run.
echo "ROX_ADMIN_PASSWORD=$(cat ./deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV"
echo "ROX_PASSWORD=$(cat ./deploy/k8s/central-deploy/password)" >> "$GITHUB_ENV"
echo "::add-mask::$(cat ./deploy/k8s/central-deploy/password)"
LB_IP="$(kubectl -n stackrox get svc/central-loadbalancer -o json | jq -r '.status.loadBalancer.ingress[0] | .ip // .hostname')"
echo "HOST=https://${LB_IP}:443" >> "$GITHUB_ENV"
echo "API_ENDPOINT=${LB_IP}:443" >> "$GITHUB_ENV"
echo "::add-mask::${LB_IP}"
- name: Generate workload
env:
LOAD_BALANCER: lb
ROX_POSTGRES_DATASTORE: "true"
MONITORING_SUPPORT: "false"
REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }}
REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }}
run: |
source ./workflow/env.sh
./scale/dev/run-many.sh "${SCALE_WORKLOAD_TYPE}" "${SCALE_WORKLOAD_CLUSTERS}"
num_deployments=$(yq '.deploymentWorkload[] | select(.deploymentType == "Deployment") | .numDeployments' "./scale/workloads/${SCALE_WORKLOAD_TYPE}.yaml")
max_deployments=$(( num_deployments * SCALE_WORKLOAD_CLUSTERS - 20 )) # num_deployments * SCALE_WORKLOAD_CLUSTERS is never quite reached so stop before than
./scale/dev/wait-for-deployments.sh "${max_deployments}"
- name: Stop workload generation
run: |
source ./workflow/env.sh
./scale/dev/kill-sensors.sh
- name: Create result directory
run: |
mkdir -p performance-results
- name: Run local k6 test
uses: grafana/k6-action@e4714b734f2b0afaabeb7b4a69142745548ab9ec # ratchet:grafana/k6-action@v0.3.1
with:
filename: ./tests/performance/tests/testK6Integration.js
flags: --out json=performance-results/raw.json --out csv=performance-results/raw.csv --iterations 50 --duration 20m --tag github_run_id=${{ github.run_id }} --tag github_ref=${{ github.ref }} --tag github_sha=${{ github.sha }}
- name: Publish JUnit test report in job summary
uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # ratchet:test-summary/action@v2
with:
paths: performance-results/report.xml
- name: Upload results to GCS
uses: google-github-actions/upload-cloud-storage@6397bd7208e18d13ba2619ee21b9873edc94427a # ratchet:google-github-actions/upload-cloud-storage@v3
with:
path: performance-results
destination: acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }}
gzip: false
process_gcloudignore: false
- name: Render PR comment from text report and links
run: |
report_txt=$(cat performance-results/report.txt)
cat <<EOF > pr-comment.md
# Performance test results
## Summary
$report_txt
## Sources
- Summary: [HTML Report](https://storage.cloud.google.com/acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }}/performance-results/report.html) | [JSON](https://storage.cloud.google.com/acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }}/performance-results/report.json)
- Raw Data: [CSV](https://storage.cloud.google.com/acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }}/performance-results/raw.csv) | [JSON](https://storage.cloud.google.com/acs-performance-test-results/${{ github.ref }}/${{ github.sha }}/${{ github.run_id }}/${{ github.run_attempt }}/performance-results/raw.json)
EOF
cat pr-comment.md >> "$GITHUB_STEP_SUMMARY"
- name: Add text report and links as PR comment
uses: mshick/add-pr-comment@64b8e914979889d746c99dea15a76e77ef64580a # ratchet:mshick/add-pr-comment@v3
with:
message-id: performance-evaluation
message-path: |
pr-comment.md