-
-
Notifications
You must be signed in to change notification settings - Fork 10
219 lines (191 loc) · 10.9 KB
/
benchmark.yml
File metadata and controls
219 lines (191 loc) · 10.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
name: Benchmark PHP Debugger performance
on:
# This workflow runs on three triggers: Weekly on schedule, manually, or when pull requests are labeled
schedule:
# Every Sunday at 03:00 UTC
- cron: '0 3 * * 0'
workflow_dispatch:
pull_request:
types: [ labeled ]
jobs:
run-benchmark:
# When triggered when a pull request is labeled we check the name of the label
if: |
github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' ||
(
github.event_name == 'pull_request' &&
github.event.label.name == 'performance-check'
)
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
command: ["bench", "symfony", "rector"]
php: ["8.2", "8.3", "8.4", "8.5"]
php_debugger_mode: ["no", "off", "debug", "debug-on-demand", "debug-connected"]
steps:
- uses: actions/checkout@v6
- name: Setup PHP
uses: shivammathur/setup-php@v2
with:
php-version: "${{ matrix.php }}"
coverage: none
# We need to use a specific version of Composer because we are using a specific version of the
# Symfony Demo which is not compatible with the latest versions of Composer
tools: composer:v2.2.21
- name: Disable ASLR
# ASLR can cause a lot of noise due to missed sse opportunities for memcpy
# and other operations, so we disable it during benchmarking.
run: echo 0 | sudo tee /proc/sys/kernel/randomize_va_space
- name: Install valgrind
run: sudo apt-get update && sudo apt-get install -y valgrind
- name: Compile
run: |
phpize
./configure --enable-php-debugger
make -j$(nproc)
sudo make install
- name: create results and opcache folder
run: |
mkdir -p results
mkdir -p /tmp/opcache
- name: install Symfony demo
if: matrix.command == 'symfony'
# For versions of PHP greater than 8.4 we need to run a different version of the Symfony Demo.
# The older versions suffer from the nulllable parameter deprecation and this generated a lot of
# deprecations in the code and this skewed the tests results a lot. And we cannot use the newer version
# for older versions of PHP as it is not compatible.
run: |
if [ "${{ matrix.php }}" \< "8.4" ]; then
version="v2.0.2"
else
version="v2.7.0"
fi
git clone -q --branch "$version" --depth 1 https://github.com/symfony/demo.git ./symfony-demo
cd symfony-demo
composer install
# We do not want our tests to be skewed by any error, warning, notice or deprecation thrown by the
# code, so we set the error level to 0 on purpose. Unfortunately, Symfony overrides this setup and
# sets their own error reporting level. We fix this by changing all the places where Symfony is
# setting the error reporting level so that it continues to be set to 0
find . -type f -name "*.php" -exec sed -i.bak -E 's/error_reporting\(.*\);/error_reporting(0);/g' {} +
- name: install Rector
if: matrix.command == 'rector'
run: |
composer require rector/rector:2.1.2 phpstan/phpstan:2.1.33 --dev
cp .github/benchmark-files/rector.php .
- name: Copy ini file
run: |
sudo cp ./.github/benchmark-files/benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d
sudo cp ./.github/benchmark-files/benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d
- name: Copy php_debugger ini file
if: matrix.php_debugger_mode != 'no' && matrix.php_debugger_mode != 'debug-on-demand' && matrix.php_debugger_mode != 'debug-connected'
run: |
sudo cp ./.github/benchmark-files/php-debugger-benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d
sudo cp ./.github/benchmark-files/php-debugger-benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d
- name: Copy on-demand php_debugger ini file
if: matrix.php_debugger_mode == 'debug-on-demand'
run: |
sudo cp ./.github/benchmark-files/php-debugger-on-demand-benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d
sudo cp ./.github/benchmark-files/php-debugger-on-demand-benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d
- name: Copy connected php_debugger ini file
if: matrix.php_debugger_mode == 'debug-connected'
run: |
sudo cp ./.github/benchmark-files/php-debugger-connected-benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d
sudo cp ./.github/benchmark-files/php-debugger-connected-benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d
- name: Set PHP Debugger mode
if: matrix.php_debugger_mode != 'debug-on-demand' && matrix.php_debugger_mode != 'debug-connected'
run: echo "PHP_DEBUGGER_MODE=${{ matrix.php_debugger_mode }}" >> $GITHUB_ENV
- name: Set PHP Debugger mode for on-demand and debug-connected
if: matrix.php_debugger_mode == 'debug-on-demand' || matrix.php_debugger_mode == 'debug-connected'
run: echo "PHP_DEBUGGER_MODE=debug" >> $GITHUB_ENV
- name: Start DBGP emulator daemon
if: matrix.php_debugger_mode == 'debug-connected'
run: |
# Start the emulator in the background with the run command
php tests/debugger/dbgp/dbgp-emulator.php .github/benchmark-files/commands/run.txt &
# Give it a moment to start listening
sleep 2
- name: benchmark bench.php
if: matrix.command == 'bench'
run: valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php ./.github/benchmark-files/bench.php
- name: benchmark Symfony
if: matrix.command == 'symfony'
run: |
cd symfony-demo
# The Symfony demo is run using php-cgi to simulate a web server. Symfony needs that this env
# variable is set, it would usually be set by the web server, we need to set it manually instead
export SCRIPT_FILENAME="$(realpath public/index.php)"
# Symfony also needs this to be set to a non empty value
export APP_SECRET=APP_SECRET
valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php public/index.php
mv callgrind.out ..
cd ..
- name: benchmark rector.php
if: matrix.command == 'rector'
# Rector needs to be run with the --xdebug flag so that it does not try to disable PHP Debugger and also with
# the --debug flag so that it does not try to run several threads in parallel
run: valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php vendor/bin/rector --dry-run --xdebug --debug || true
- name: save matrix values
run: |
# We read the number of executed instructions from the callgrind.out file and save it in an artifact
# to be processed later
awk '/^summary:/ { print $2 }' callgrind.out > results/php-${{ matrix.php }}_cmd-${{ matrix.command }}_php_debugger-${{ matrix.php_debugger_mode }}.txt
# We also save the matrix variables so that in a later step we can build a list of the executed
# options
echo "${{ matrix.php }},${{ matrix.command }},${{ matrix.php_debugger_mode }}" > results/matrix-values-${{ matrix.php }}-${{ matrix.command }}-${{ matrix.php_debugger_mode }}.txt
- name: Upload result and matrix info
uses: actions/upload-artifact@v6
with:
name: results-${{ matrix.command }}-${{ matrix.php }}-${{ matrix.php_debugger_mode }}
path: results/
performance:
# After running all benchmarks for all commands, PHP versions and PHP Debugger modes, we run a new job that builds a
# summary of all execution times
needs: run-benchmark
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
comparison:
- name: "base-branch"
branch: "" # Empty means use default base branch logic
- name: "base"
branch: "base"
steps:
- name: Setup PHP
uses: shivammathur/setup-php@v2
with:
php-version: "8.5"
coverage: none
- uses: actions/checkout@v6
- name: Download all artifacts
uses: actions/download-artifact@v7
with:
path: results
- name: Generate summary table as markdown
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMPARISON_BRANCH: ${{ matrix.comparison.branch }}
run: php .github/benchmark-files/generate_summary.php
- name: print summary
# This generated summary file is copied into a special file which GitHub has defined
# in this special env variable and it will use it to print a summary for the workflow
run: cat summary.md >> $GITHUB_STEP_SUMMARY
- name: Upload summary.csv as artifact
if: matrix.comparison.name == 'base-branch'
uses: actions/upload-artifact@v6
with:
name: summary.csv
path: summary.csv
cleanup:
# Clean up intermediary artifacts after both performance jobs complete
needs: performance
runs-on: ubuntu-latest
steps:
- name: delete intermediary artifacts
# The intermediate files that we generated are not needed and can be deleted,
# helping us keep the summary page clean
uses: geekyeggo/delete-artifact@v6
with:
name: results-*