Skip to content

Commit b3661e9

Browse files
addshoreaddshore
committed
More reworking, ready to go through MW versions too...
1 parent 440fb26 commit b3661e9

15 files changed

+277
-302
lines changed

‎docker-compose.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3'
2-
31
services:
42
wikibase1:
53
image: ${PROFILE_IMAGE}

‎loads/init.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
# Single entity request for initialization
4+
make_single_request() {
5+
local response
6+
local url="http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php"
7+
response=$(curl -s -m 20 -X POST -F "action=wbfastimport" -F "format=json" -F "data=[$1]" "$url")
8+
echo "$response" >> process.out
9+
if [[ "$response" != *"success"* ]]; then
10+
echo "success missing, retrying"
11+
make_single_request "$1"
12+
fi
13+
}
14+
15+
make_request() {
16+
local response
17+
local url="http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php"
18+
response=$(curl -s -m 20 -X POST -F "action=wbeditentity" -F "format=json" -F "token=+\\" -F "$1" -F "$2" "$url")
19+
echo "$response" >> process.out
20+
if [[ "$response" != *"success"* ]]; then
21+
echo "success missing, retrying"
22+
make_request "$1" "$2"
23+
fi
24+
}
25+
26+
# Get the ball rolling with a simple entity
27+
make_single_request "{\"type\":\"item\"}"
28+
make_request "new=property" "data={\"datatype\":\"string\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}"
29+
30+
# For properties, we need to use wbeditentity as wbfastimport doesn't seem to handle property creation
31+
(echo $(curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=property' -F "data={\"datatype\":\"string\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php)) >> process.out
32+
33+
# Create some properties
34+
# P2 = wikibase-item
35+
# P3 = wikibase-property
36+
# P4 = string
37+
# P5 = time
38+
properties=( wikibase-item wikibase-property string time )
39+
for p in "${properties[@]}"
40+
do
41+
(echo $(curl -s -m 20 -X POST -F 'action=wbeditentity' -F 'format=json' -F 'token=+\' -F 'new=property' -F "data={\"datatype\":\"${p}\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}" http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php)) >> process.out
42+
done

‎loads/wbeditentity-2000EmptyItems.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎loads/wbeditentity-2000FullishItems.sh‎

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎loads/wbeditentity-2000MultiTermItems.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎loads/wbeditentity-2000SingleTermItems.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

loads/wbeditentity-5000FullishItems.sh renamed to loads/wbeditentity-FullishItems.sh

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,36 +10,33 @@ make_request() {
1010
fi
1111
}
1212

13-
# Get the ball (and id counters rolling) with some simple entities..
14-
make_request "new=item" "data={}"
15-
make_request "new=property" "data={\"datatype\":\"string\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}"
13+
ENTITY_GROUP=${ENTITY_GROUP:-1}
14+
# die if it isnt 1
15+
if [ $ENTITY_GROUP -ne 1 ]; then
16+
echo "ENTITY_GROUP must be 1"
17+
exit 1
18+
fi
1619

17-
# Then do the bulk of the stuff
20+
# Get entity count, default to 2000 if not set
21+
ENTITY_COUNT=${ENTITY_COUNT:-2000}
22+
# Calculate half for each batch (string items and item references)
23+
HALF_COUNT=$((ENTITY_COUNT / 2))
1824

19-
# Create some properties
20-
# P2 = wikibase-item
21-
# P3 = wikibase-property
22-
# P4 = string
23-
# P5 = time
24-
properties=( wikibase-item wikibase-property string time )
25-
for p in "${properties[@]}"
26-
do
27-
make_request "new=property" "data={\"datatype\":\"${p}\",\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}}}"
28-
done
25+
echo "Creating $ENTITY_COUNT entities ($HALF_COUNT each type)"
2926

3027
# Create some items including statements
3128

32-
# 2500 with strings
33-
for i in {1..2500}
29+
# Half with strings
30+
for i in $(seq 1 $HALF_COUNT)
3431
do
3532
(
3633
make_request "new=item" "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]},\"claims\":{\"P4\":[{\"mainsnak\":{\"snaktype\":\"value\",\"property\":\"P4\",\"datavalue\":{\"value\":\"statement-string-value\",\"type\":\"string\"},\"datatype\":\"string\"},\"type\":\"statement\",\"rank\":\"normal\"}]}}"
3734
)&
3835
if (( $(wc -w <<<$(jobs -p)) % $ASYNC == 0 )); then sleep 0.5; fi
3936
done
4037

41-
# 2500 linking to other items
42-
for i in {1..2500}
38+
# Half linking to other items
39+
for i in $(seq 1 $HALF_COUNT)
4340
do
4441
(
4542
make_request "new=item" "data={\"labels\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"descriptions\":{\"en\":{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}},\"aliases\":{\"en\":[{\"language\":\"en\",\"value\":\"$(sed "s/[^a-zA-Z0-9]//g" <<< $(openssl rand -base64 30))\"}]},\"claims\":{\"P2\":[{\"mainsnak\":{\"snaktype\":\"value\",\"property\":\"P2\",\"datavalue\":{\"value\":{\"entity-type\":\"item\",\"numeric-id\":1,\"id\":\"Q1\"},\"type\":\"wikibase-entityid\"},\"datatype\":\"wikibase-item\"},\"type\":\"statement\",\"rank\":\"normal\"}]}}"

‎loads/wbfastimport-2000full-batch100-before.sh‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎loads/wbfastimport-2000full-batch100.sh‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎loads/wbfastimport-5000full-batch250-before.sh‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

0 commit comments

Comments
 (0)