forked from PostHog/posthog
-
Notifications
You must be signed in to change notification settings - Fork 0
303 lines (264 loc) · 13.6 KB
/
Copy pathci-python.yml
File metadata and controls
303 lines (264 loc) · 13.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
# This workflow runs generic Python tests global to Dagster and PostHog.
name: Python CI
on:
push:
branches:
- master
pull_request:
permissions:
contents: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
RUNS_ON_INTERNAL_PR: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}
jobs:
# Job to decide if we should run python ci
# See .github/actions/paths-filter/README.md for filter semantics
changes:
runs-on: ubuntu-latest
timeout-minutes: 5
name: Determine need to run python checks
# Set job outputs to values from filter step
outputs:
python: ${{ steps.filter.outputs.python || 'true' }}
steps:
# For pull requests it's not necessary to checkout the code, but we
# also want this to run on master so we need to checkout
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
id: app-token
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
with:
client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }}
private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }}
- uses: ./.github/actions/paths-filter
id: filter
if: github.event_name != 'push' # Run all tests on master push
with:
token: ${{ steps.app-token.outputs.token || github.token }}
filters: |
python:
# ruff/mypy/ty run repo-wide (`.`) in code-quality, so any Python
# file anywhere must trigger this workflow — otherwise an unformatted
# file lands undetected and trips the next unrelated PR that does run.
- '**/*.py'
- '*.py' # root-level scripts (e.g. manage.py); '**/' alone may not match root
- 'pyproject.toml'
- 'uv.lock'
- 'posthog/**/*'
# Make sure we run if someone is explicitly change the workflows
- .github/workflows/ci-python.yml
- .github/workflows/ci-dagster.yml
- .github/workflows/ci-backend.yml
- .flox/env/manifest.toml
# Schema changes need to trigger Python CI to validate schema.py is regenerated
- 'frontend/src/queries/schema.json'
# Products changes need to trigger Python CI to validate the json usage is still valid
- 'frontend/src/products.json'
# hogli CLI changes need validation
- 'tools/hogli/**'
- 'tools/hogli-commands/hogli_commands/**'
- 'tools/owners/**'
- 'hogli.yaml'
- 'bin/**'
# Dev tools with dedicated pytest steps below
- 'tools/pr-approval-agent/**'
- 'products/stamphog/packages/pr-approval-agent/**'
- 'tools/query-performance-ai/**'
# Stamphog policy files validated by the pr-approval-agent suite
- '.stamphog/**'
- '**/AGENT_APPROVALS.md'
code-quality:
needs: changes
if: needs.changes.outputs.python == 'true'
timeout-minutes: 30
name: Python code quality (depot-ubuntu-24.04)
runs-on: depot-ubuntu-24.04
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 1
- name: Mint setup-action GitHub token
id: setup-gh-token
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
continue-on-error: true
uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1
with:
client-id: ${{ secrets.GH_APP_POSTHOG_SETUP_ACTIONS_APP_ID }}
private-key: ${{ secrets.GH_APP_POSTHOG_SETUP_ACTIONS_PRIVATE_KEY }}
skip-token-revoke: true
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: 3.13.13
token: ${{ steps.setup-gh-token.outputs.token || github.token }}
- name: Install uv
id: setup-uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
version: '0.11.28' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit
enable-cache: true
cache-dependency-glob: uv.lock
save-cache: ${{ github.ref == 'refs/heads/master' }}
- name: Check uv and Python version compatibility
shell: bash
run: |
python3 bin/check_uv_python_compatibility.py
- name: Install SAML (python3-saml) dependencies
if: steps.setup-uv.outputs.cache-hit != 'true'
shell: bash
run: |
sudo rm -f /etc/apt/sources.list.d/*twingate*
sudo apt-get update
sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl
- name: Install Python dependencies
shell: bash
run: |
UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
- name: Download GeoIP database
shell: bash
run: |
./bin/download-mmdb
- name: Add ty Problem Matcher
shell: bash
run: |
echo "::add-matcher::.github/ty-problem-matcher.json"
- name: Check ty type checking
shell: bash
run: |
echo "📊 Feedback welcome in #team-devex"
uv run ty check
- name: Add mypy Problem Matcher
shell: bash
run: |
echo "::add-matcher::.github/mypy-problem-matcher.json"
- name: Restore mypy cache
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: .mypy_cache
key: mypy-cache-${{ runner.os }}-3.12-mypy2-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }}
restore-keys: |
mypy-cache-${{ runner.os }}-3.12-mypy2-
- name: Check static typing
shell: bash -e {0}
env:
MYPY_NUM_WORKERS: 2
run: mypy --version && mypy --cache-fine-grained .
- name: Save mypy cache
if: github.ref == 'refs/heads/master'
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: .mypy_cache
key: mypy-cache-${{ runner.os }}-3.12-mypy2-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }}
- name: Check if "schema.py" is up to date
shell: bash
run: |
npm run schema:build:python && git diff --exit-code
- parallel:
- name: Check for syntax errors, import sort, and code style violations
shell: bash
run: ruff check .
- name: Check formatting
shell: bash
run: ruff format --check --diff .
- name: Check hogli manifest completeness
shell: bash
run: ./bin/hogli meta:check
- name: Run hogli framework tests
shell: bash
run: pytest tools/hogli/tests -v --junitxml=junit-hogli.xml
- name: Run hogli extension tests
shell: bash
run: pytest tools/hogli-commands/hogli_commands/tests -v --junitxml=junit-hogli-commands.xml
- name: Run owners resolver tests
shell: bash
run: pytest tools/owners/tests -v --junitxml=junit-owners.xml
- name: Run query-performance-ai tests
shell: bash
run: pytest tools/query-performance-ai/query_performance_ai -v --junitxml=junit-query-performance-ai.xml
- name: Run pr-approval-agent (stamphog) tests
shell: bash
# Two candidate paths. The engine now lives under the stamphog product, but an
# open PR that has not rebased still carries it under tools/, and this workflow
# runs against that branch merged with master. The loop tests both trees when
# both exist, because that merge can add a file at the old path next to the new
# tree.
run: |
found=0
for dir in products/stamphog/packages/pr-approval-agent tools/pr-approval-agent; do
[ -d "$dir" ] || continue
found=1
pytest "$dir" -v --junitxml="junit-pr-approval-agent-$(echo "$dir" | tr / -).xml"
done
if [ "$found" = "0" ]; then
echo "::error::pr-approval-agent engine not found at either path"
exit 1
fi
- name: Run dependency detection script tests
shell: bash
run: pytest bin/test/test_find_python_dependencies.py -v --junitxml=junit-dependency-detection.xml
- name: Upload test results
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
if: always()
with:
name: junit-results-hogli
path: |
junit-hogli.xml
junit-hogli-commands.xml
junit-owners.xml
junit-query-performance-ai.xml
junit-pr-approval-agent-*.xml
- name: Upload test results for dependency detection
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
if: always()
with:
name: junit-results-dependency-detection
path: junit-dependency-detection.xml
# Upload results to Trunk for flaky-test detection. Non-gating (continue-on-error),
# so a Trunk outage never fails the job. Runs the token only on internal PRs
# and master, never on fork PRs or Dependabot, which have no secret.
# The TRUNK_UPLOAD_ENABLED repo variable is the kill-switch: when it is not 'true'
# this upload skips.
- name: Upload test results to Trunk
if: ${{ !cancelled() && env.RUNS_ON_INTERNAL_PR == 'true' && github.repository == 'PostHog/posthog' && github.actor != 'dependabot[bot]' && vars.TRUNK_UPLOAD_ENABLED == 'true' }}
continue-on-error: true
uses: trunk-io/analytics-uploader@385f1ccdf345b4532dc4b6c665dd432b702b8e28 # v2.1.2
with:
junit-paths: junit-*.xml
org-slug: posthog-inc
token: ${{ secrets.TRUNK_API_TOKEN }}
# - name: Check if "taxonomy.json/taxonomy.tsx" is up to date
# if: needs.changes.outputs.python == 'true'
# shell: bash
# run: |
# npm run taxonomy:build:json && git diff --exit-code
# Collate job - single required status check for branch protection.
# The code-quality job skips when no Python changes, but this job always
# runs and reports success, allowing PRs to merge.
python_tests:
needs: [changes, code-quality]
name: Python code quality checks
runs-on: ubuntu-latest
timeout-minutes: 5
if: always()
steps:
- name: Check Python CI status
run: |
# Block cancelled too: outputs are empty then, so the exit below would report green.
if [[ "${{ needs.changes.result }}" != "success" && "${{ needs.changes.result }}" != "skipped" ]]; then
echo "Change detection did not succeed (result: ${{ needs.changes.result }})."
exit 1
fi
# Pass if no Python changes detected (jobs were skipped)
if [[ "${{ needs.changes.outputs.python }}" != "true" ]]; then
echo "Python checks were skipped (no relevant changes)"
exit 0
fi
# Check actual job result
if [[ "${{ needs.code-quality.result }}" != "success" && "${{ needs.code-quality.result }}" != "skipped" ]]; then
echo "Python code quality checks failed."
exit 1
fi
echo "All Python checks passed."