Skip to content

Commit 6a362e8

Browse files
addshoreaddshore
committed
more async, less settings, less containers
1 parent 43c7324 commit 6a362e8

File tree

8 files changed

+64
-48
lines changed

8 files changed

+64
-48
lines changed

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@ jobs:
1111
instances:
1212
- 1
1313
- 2
14-
- 3
14+
# - 3 # 3 generally seemed to be a bit slower? Could just be resource constraints though... (should try on k8s? :D)
1515
async:
16-
- 40
17-
- 80
16+
- 60
17+
- 90
1818
- 120
1919
images:
2020
# - wikibase/wikibase:1.33-base
@@ -24,11 +24,10 @@ jobs:
2424
# - mariadb:10.3 # No real difference between 10.3 and 10.5 mariadb, so just use the newest one
2525
- mariadb:10.5
2626
# - mysql:5.7 # MariaDB seems to generally be the faster choice...
27-
# - mysql:8.0 # Errors with Call to a member function getIP() on null in /var/www/html/includes/user/User.php:2153
27+
- mysql:8.0 # Errors with Call to a member function getIP() on null in /var/www/html/includes/user/User.php:2153
2828
settings:
2929
# - default
3030
- lightweight
31-
- lightweight-wbOwnIdConnection
3231
# - mwNoJobs
3332
# - wbNoChangesTable
3433
# - wbNewTerms
@@ -48,7 +47,6 @@ jobs:
4847
run: |
4948
./docker-compose-wait1.sh
5049
./docker-compose-wait2.sh
51-
./docker-compose-wait3.sh
5250
sleep 5
5351
- name: Perform test
5452
run: |

‎docker-compose.yml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,18 @@ services:
2525
- "8182:80"
2626
depends_on:
2727
- mysql
28-
wikibase3:
29-
image: ${PROFILE_IMAGE}
30-
entrypoint: /code-repo/docker-compose-entry2.sh
31-
volumes:
32-
- ./:/code-repo
33-
- ./settings/${PROFILE_SETTINGS}.sh:/code-settings.sh
34-
links:
35-
- mysql
36-
ports:
37-
- "8183:80"
38-
depends_on:
39-
- mysql
28+
# wikibase3:
29+
# image: ${PROFILE_IMAGE}
30+
# entrypoint: /code-repo/docker-compose-entry2.sh
31+
# volumes:
32+
# - ./:/code-repo
33+
# - ./settings/${PROFILE_SETTINGS}.sh:/code-settings.sh
34+
# links:
35+
# - mysql
36+
# ports:
37+
# - "8183:80"
38+
# depends_on:
39+
# - mysql
4040
mysql:
4141
image: ${PROFILE_SQL}
4242
environment:

‎loads/2000EmptyItems.sh‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,23 @@ curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F
33
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=property' -F "data={\"datatype\":\"string\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
44

55
# Then do the bulk of the stuff
6-
for i in {1..2000}
6+
7+
# Divice ASYNC by 2, as we do 2 loops at the same time..
8+
ASYNC=$(expr $ASYNC / 2)
9+
10+
(for i in {1..1000}
11+
do
12+
(
13+
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F 'data={}' http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
14+
)&
15+
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
16+
done)&
17+
(for i in {1..1000}
718
do
819
(
920
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F 'data={}' http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
1021
)&
1122
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
12-
done
23+
done)&
1324

1425
wait

‎loads/2000FullishItems.sh‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F
44

55
# Then do the bulk of the stuff
66

7+
# Divice ASYNC by 2, as we do 2 loops at the same time..
8+
ASYNC=$(expr $ASYNC / 2)
9+
710
# Create some properties
811
# P2 = wikibase-item
912
# P3 = wikibase-property
@@ -18,21 +21,21 @@ done
1821
# Create some items including statements
1922

2023
# 1000 with strings
21-
for i in {1..1000}
24+
(for i in {1..1000}
2225
do
2326
(
2427
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]},\"claims\":{\"P4\":[{\"mainsnak\":{\"snaktype\":\"value\",\"property\":\"P4\",\"datavalue\":{\"value\":\"statement-string-value\",\"type\":\"string\"},\"datatype\":\"string\"},\"type\":\"statement\",\"rank\":\"normal\"}]}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
2528
)&
2629
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
27-
done
30+
done)&
2831

2932
# 1000 linking to other items
30-
for i in {1..1000}
33+
(for i in {1..1000}
3134
do
3235
(
3336
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]},\"claims\":{\"P2\":[{\"mainsnak\":{\"snaktype\":\"value\",\"property\":\"P2\",\"datavalue\":{\"value\":{\"entity-type\":\"item\",\"numeric-id\":1,\"id\":\"Q1\"},\"type\":\"wikibase-entityid\"},\"datatype\":\"wikibase-item\"},\"type\":\"statement\",\"rank\":\"normal\"}]}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
3437
)&
3538
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
36-
done
39+
done)&
3740

3841
wait

‎loads/2000MultiTermItems.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,22 @@ curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F
44

55
# Then do the bulk of the stuff
66

7-
for i in {1..2000}
7+
# Divice ASYNC by 2, as we do 2 loops at the same time..
8+
ASYNC=$(expr $ASYNC / 2)
9+
10+
(for i in {1..1000}
11+
do
12+
(
13+
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
14+
)&
15+
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
16+
done)&
17+
(for i in {1..1000}
818
do
919
(
1020
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
1121
)&
1222
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
13-
done
23+
done)&
1424

1525
wait

‎loads/2000SingleTermItems.sh‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,22 @@ curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F
44

55
# Then do the bulk of the stuff
66

7-
for i in {1..2000}
7+
# Divice ASYNC by 2, as we do 2 loops at the same time..
8+
ASYNC=$(expr $ASYNC / 2)
9+
10+
(for i in {1..1000}
11+
do
12+
(
13+
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
14+
)&
15+
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
16+
done)&
17+
(for i in {1..1000}
818
do
919
(
1020
curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=item' -F "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php
1121
)&
1222
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then wait; fi
13-
done
23+
done)&
1424

1525
wait

‎settings/lightweight-wbOwnIdConnection.sh‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎settings/lightweight.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,7 @@ echo "\$wgWBRepoSettings['tmpPropertySearchMigrationStage'] = MIGRATION_NEW;" >>
1414
# No recording of data in wb_changes table (for dispatching to clients)
1515
# https://doc.wikimedia.org/Wikibase/master/php/md_docs_topics_options.html#autotoc_md261
1616
echo "\$wgWBRepoSettings['useChangesTable'] = false;" >> LocalSettings.php
17+
18+
# Use a seperate DB connection for ID allocation to reduce contention
19+
# https://doc.wikimedia.org/Wikibase/master/php/md_docs_topics_options.html#repo_idGeneratorSeparateDbConnection
20+
echo "\$wgWBRepoSettings['idGeneratorSeparateDbConnection'] = true;" >> LocalSettings.php

0 commit comments

Comments
 (0)