add mediawiki update job verification script (#2371)

deer-wmde · rosalieper · wmdebot · web-flow · commit eb53a96b7ba2 · 2025-12-04T13:53:01.000+01:00
* add mediawiki update job verification script * decrease caching TTL in nginx conf and add a delay before removing the read-only mode of a wiki (#2370) * Fix broken curl command (#2372) * (chore): Update api-queue CPU requests (#2342) * (chore): Update api-queue CPU requests * Update api.values.yaml.gotmpl * argo values * Staging+Local: Deploy new MediaWiki image ghcr.io/wbstack/mediawiki:sha-a72fadf (#2373) * Production: Deploy new MediaWiki image ghcr.io/wbstack/mediawiki:sha-a72fadf (#2374) * improve output * update README * clean up output * fix typo --------- Co-authored-by: Perside Rosalie <perside.rosalie@wikimedia.de> Co-authored-by: wmdebot <121924840+wmdebot@users.noreply.github.com>
diff --git a/k8s/jobs/README.md b/k8s/jobs/README.md
@@ -81,4 +81,21 @@ It uses the primary replica to write to.
 use the ENV `WBS_DOMAIN
 
 This script loops through all wikis and run `addPlatformReservedUser.sh` on each wiki to put `PlatformReservedUser` to bot group.
-Run `list-wiki-domains` first, save it to a file (for example: `domains.txt`), then run `addPlatformReservedUserToBotGroupForAllWikis.sh`.
+Run `list-wiki-domains` first, save it to a file (for example: `domains.txt`), then run `addPlatformReservedUserToBotGroupForAllWikis.sh`.
+
+## mediawikiUpdate.sh
+Example of ENV variables to set
+```sh
+WBS_DOMAIN=coffeebase.wikibase.cloud
+MEDIAWIKI_BACKEND_INSTANCE_LABEL=mediawiki-143
+MW_VERSION=mw1.43-wbs1
+```
+
+Creates a Job that spins up it's own mediawiki pod to run the mediawiki update maintenance script (`run.php update --quick`)
+
+- Before that script is ran the wiki gets set to read-only mode
+- After it has run the dbVersion gets updated in the API
+- and read-only mode gets disabled again after two minutes (nginx cache TTL)
+
+## verifyMediaWikiUpdates.sh
+Reports status for k8s Jobs kicked off via `mediawikiUpdate.sh`. Filters for them via namespace `adhoc-jobs` and the metadata label `wikiDomain`.
diff --git a/k8s/jobs/mediawikiUpdate.sh b/k8s/jobs/mediawikiUpdate.sh
@@ -36,5 +36,7 @@ kubectl create -f mediawikiUpdate.yaml -o=json --dry-run=client | \
   jq --arg wbs "$WBS_DOMAIN" --arg mwver "$MW_VERSION" '.[0].spec.template.spec.containers[0].env += [{"name": "WBS_DOMAIN", "value": $wbs}, {"name": "MW_VERSION", "value": $mwver}]' | \
   # return just the Job spec (dropping the existing Pod one)
   jq '.[0]' | \
+  # add a k8s label to the job object with the wiki domain
+  jq --arg wbs "$WBS_DOMAIN" '.metadata.labels += {"wikiDomain": $wbs}' | \
   # create the now fully formed Job from the spec
   kubectl create -f -
diff --git a/k8s/jobs/mediawikiUpdate.yaml b/k8s/jobs/mediawikiUpdate.yaml
@@ -45,13 +45,12 @@ spec:
                   curl --fail --location --silent --request PUT \
                     "http://${PLATFORM_API_BACKEND_HOST}/backend/setWikiDbVersion?domain=${WBS_DOMAIN}&dbVersion=${MW_VERSION}"
                   log_info "database version updated to ${MW_VERSION}"
-                  log_info "Pause for 2 minutes before disabling readOnly mood"
+                  log_info "Pause for 2 minutes before disabling readOnly mode"
                   sleep 120
                   curl --fail --location --silent --request PUT \
                     "http://${PLATFORM_API_BACKEND_HOST}/backend/setWikiReadOnly?domain=${WBS_DOMAIN}&readOnly=0"
                   log_info "readOnly mode will be disabled for ${WBS_DOMAIN}"
                 else
-                  log_error "update.php failed to run and exit with code $status"
                   log_error "update.php failed to run and exit with code $status"
                   exit 1
                 fi
diff --git a/k8s/jobs/verifyMediawikiUpdates.sh b/k8s/jobs/verifyMediawikiUpdates.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+JOBS=$(kubectl get jobs -n adhoc-jobs -o json)
+
+COMPLETED=$(echo $JOBS | jq -r '.items[] | select((.status.succeeded // 0) >= 1 and (.metadata.labels | has("wikiDomain"))) | "\(.metadata.name)\t\(.metadata.labels.wikiDomain)"')
+
+INCOMPLETE=$(echo $JOBS | jq -r '.items[] | select((.status.succeeded // 0) < 1 and (.metadata.labels | has("wikiDomain"))) | "\(.metadata.name)\t\(.metadata.labels.wikiDomain)"')
+# alternative query for incomplete jobs without wikiDomain label
+#INCOMPLETE=$(echo $JOBS | jq -r '.items[] | select((.status.succeeded // 0) < 1) | "\(.metadata.name)"')
+
+COMPLETED_COUNT=$(printf "%s" "$COMPLETED" | wc -l)
+INCOMPLETE_COUNT=$(printf "%s" "$INCOMPLETE" | wc -l)
+
+echo "Completed jobs:"
+echo "$COMPLETED"
+echo 
+echo "Number of completed jobs: $COMPLETED_COUNT"
+echo
+echo '---'
+echo
+echo "Incomplete jobs:"
+echo "$INCOMPLETE"
+echo
+echo "Number of completed jobs: $INCOMPLETE_COUNT"
