Finaly updates pre blog post

addshore · addshore · commit 7784cb15ce75 · 2025-03-29T09:14:54.000Z
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # wikibase-profile
 
-The general idea of this repo is to allow some basic profiling of different settings using docker and possible Github actions?
+The general idea of this repo is to allow some basic profiling of different setups of Wikibase, using Docker, in a mildly automated way.
 
 Steps:
 
@@ -11,46 +11,15 @@ Steps:
 - Do some amount of known work
 - Stop timer
 - Output result...
-- Send result to Google Sheet via IFTTT
 
 ## Local profiling
 
-You can also try running them locally... For example...
+You can also try running them locally...
 
-```sh
-export PROFILE_IMAGE=wikibase/wikibase:mw1.43.0
-export PROFILE_SETTINGS=default
-export PROFILE_SQL=mariadb:11.7
-
-docker-compose up -d mysql wikibase1
-./docker-compose-wait1.sh
-docker-compose up -d wikibase2
-./docker-compose-wait2.sh
-docker-compose up -d --force-recreate wikibase1
-./docker-compose-wait1.sh
-
-echo "Sleeping for 5 seconds..."
-sleep 5
-echo "Starting profiling..."
-TIMING=$(time ASYNC=30 INSTANCES=2 ./loads/wbeditentity-2000EmptyItems.sh)
-LAST_LINE=$(tail -n 1 ./process.out)
-STATS=$(curl 'http://localhost:8181/w/api.php?action=query&format=json&meta=siteinfo&siprop=statistics')
-```
-
-And to delete everything, just...
+Setup matrix.yml to your liking, and then run the following command:
 
 ```sh
-docker-compose down --volumes
-```
-
-## Running Matrix Jobs Locally
-
-To run the matrix jobs locally, use the `run_matrix.sh` script. This script reads the matrix configuration from `matrix.yml` and runs the jobs locally, storing the outputs in the `.data` directory.
-
-### Usage
-
-```bash
 ./run_matrix.sh
 ```
 
-The outputs will be stored in the `.data` directory with a directory schema reflecting the matrix configuration. Each run will also be summarized in a CSV file located at `.data/summary.csv`, and the statistics from the API will be included in the output.
+You'll find some useful stuff output, but also in the `.data` directory, which will contain the results of the profiling.
diff --git a/loads/wbfastimport-FullishItems.sh b/loads/wbfastimport-FullishItems.sh
@@ -44,7 +44,7 @@ make_group_request() {
   local response
   local url="http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php"
   
-  response=$(curl -s -m 20 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
+  response=$(curl -s -m 300 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
   echo "$response" >> process.out
   
   if [[ "$response" != *"success"* ]]; then
@@ -82,7 +82,7 @@ do
     # Send the request with the temp file data
     data=$(cat "$temp_file")
     url="http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php"
-    response=$(curl -s -m 20 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
+    response=$(curl -s -m 300 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
     echo "$response" >> process.out
     
     # Clean up
@@ -125,7 +125,7 @@ do
     # Send the request with the temp file data
     data=$(cat "$temp_file")
     url="http://localhost:818$((1 + RANDOM % $INSTANCES))/w/api.php"
-    response=$(curl -s -m 20 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
+    response=$(curl -s -m 300 -X POST -F "action=wbfastimport" -F "format=json" -F "data=${data}" "$url")
     echo "$response" >> process.out
     
     # Clean up
diff --git a/matrix.yml b/matrix.yml
@@ -4,10 +4,16 @@ instances:
 async:
   # - 1
   # - 5
-  - 10
+  # - 10
+  - 16
+  # - 20
+  # - 25
   # - 30
   # - 60
-  # - 100
+  # - 90
+  # - 120
+  # - 150
+  # - 200
 images:
   # - wikibase/wikibase:1.39.1-wmde.11
   # - wikibase/wikibase:1.40.3-wmde.19
@@ -19,19 +25,33 @@ sql:
   - mariadb:11.7
 settings:
   # - default
-  - wbOwnIdConnection
+  # - wbOwnIdConnection
+  - wbOwnIdConnectionUpsert
   # - lightweight
 loads:
   # - wbeditentity-FullishItems
   - wbfastimport-FullishItems
+  # - rest-FullishItems
 entity_groups:
   # - 1
   # - 2
   # - 5
   # - 10
   # - 25
-  - 50
+  # - 50
+  # - 100
+  # - 150
+  # - 200
+  - 250
 entity_counts:
   # - 2000
-  - 5000
-  # - 10000
+  # - 5000
+  # - 10000
+  # - 25000
+  # - 35000
+  # - 50000
+  # - 100000
+  # - 250000
+  - 500000
+  # - 1000000
+  # - 2000000
diff --git a/run_matrix.sh b/run_matrix.sh
@@ -56,7 +56,7 @@ for instance in $instances; do
                 docker-compose up -d --force-recreate wikibase1
                 ./docker-compose-wait1.sh
                 sleep 2
-                
+
                 echo "Running init.."
                 start_time=$(date +%s)
                 INSTANCES=$instance ./loads/init.sh
@@ -71,8 +71,11 @@ for instance in $instances; do
                 stop_time=$(date +%s)
                 total_time=$((stop_time - start_time))
                 echo "Stop time: $stop_time" > $job_dir/stop.out
-                echo "Total time: $total_time" > $job_dir/time.out
+                echo "Total time: $total_time" > $job_dir/time.o
                 success_line=$(grep -o "success" process.out | wc -l)
+                if [[ $load == *"rest"* ]]; then
+                  success_line=$((success_line + $(grep -o '"type":"item"' process.out | wc -l)))
+                fi
                 echo "Success line: $success_line" > $job_dir/success_line.out
                 cp process.out $job_dir/process.out
                 # Get statistics, but do it twice to make sure things have finished processing, and sleep in the middle?
diff --git a/settings/wbOwnIdConnectionUpsert.sh b/settings/wbOwnIdConnectionUpsert.sh
@@ -0,0 +1,4 @@
+# Use a separate DB connection for ID allocation to reduce contention
+# And try upserting...
+echo "\$wgWBRepoSettings['idGeneratorSeparateDbConnection'] = true;" >> LocalSettings.php
+echo "\$wgWBRepoSettings['idGenerator'] = 'mysql-upsert';" >> LocalSettings.php
