Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 55 additions & 25 deletions .github/scripts/cycle_config.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
"""Read a cycle TOML and emit its build configuration as GITHUB_OUTPUT lines.

python .github/scripts/cycle_config.py cycles/2026_03.toml 3.14
python .github/scripts/cycle_config.py cycles/2026_04.toml 3.14

Writes to $GITHUB_OUTPUT when set, otherwise stdout, so it can be run locally
to see exactly what a workflow run would get. tomllib is stdlib from 3.11, and
GitHub runners are newer than that, so this needs no dependency.

The emitted matrix holds only the flavors this Python can actually build: the
right architecture, and a pylock file present in the cycle directory. A flavor
the cycle declares but has no lockfile for -- `whl` today -- costs nothing, so
it can stay declared until it comes back. Paths are relative to the working
directory, which in CI is the checkout root.
Everything the build job needs is decided here rather than re-derived per
matrix leg in PowerShell:

* which flavors this Python can build -- right architecture, and a pylock
present in the cycle directory. A flavor the cycle declares but has no
lockfile for costs nothing, so it can stay declared until it comes back.
* the file names those flavors use, so a leg never has to look one up.
* a cycle with no lockfiles at all fails here, loudly, instead of starting
runners whose every step is then skipped.

Paths are relative to the working directory, which in CI is the checkout root.
"""
import json
import os
Expand All @@ -19,25 +25,34 @@
from pathlib import Path


def buildable_flavors(cfg: dict, requested: str, ver2: str) -> list[dict]:
"""Flavors with the right architecture and a lockfile on disk.
def flavor_entry(cfg: dict, flavor: dict, ver2: str, python_version: str) -> dict | None:
"""One matrix leg, or None when this flavor has no lockfile to build.

Same two conditions the build job used to re-test in PowerShell, one per
matrix leg. Deciding here instead means a leg with nothing to do never
starts a runner, and a cycle with no lockfiles at all fails loudly rather
than going green having built nothing.
Names follow the layout the publish step writes: pylock.64-<ver2 with
underscores><flavor><release level>.toml, plus the _wheels variants the
wheelhouse flavor adds.
"""
arch = "64F" if requested.endswith("F") else "64"
cycle_dir = Path(cfg["cycle_dir"])
level = cfg.get("release_level", "")
tag = ver2.replace(".", "_")
kept = []
for flavor in cfg["flavors"]:
if str(flavor.get("WINPYARCHDET", "")) != arch:
continue
if (cycle_dir / f"pylock.64-{tag}{flavor['name']}{level}.toml").is_file():
kept.append(flavor)
return kept
stem = f"64-{ver2.replace('.', '_')}{flavor['name']}{level}"

lockfile = cycle_dir / f"pylock.{stem}.toml"
if not lockfile.is_file():
return None

def optional(path: Path) -> str:
return path.as_posix() if path.is_file() else ""

return {
"name": flavor["name"],
"PANDOC": flavor["PANDOC"],
"formats": flavor["formats"],
"lockfile": lockfile.as_posix(),
"lockfile_wheels": optional(cycle_dir / f"pylock.{stem}_wheels.toml"),
"requirements_wheels": optional(cycle_dir / f"requir.{stem}_wheels.txt"),
"winpyver": f"{ver2}{flavor['name']}{level}",
"artifact_name": f"publish_{python_version}{flavor['name']}",
}


def build_config(cfg: dict, requested: str) -> dict:
Expand All @@ -50,24 +65,39 @@ def build_config(cfg: dict, requested: str) -> dict:

# the check the PowerShell version did: ver2's first 3 parts must appear in
# the tarball URL, so a copy-paste slip between the two cannot go unnoticed
short = ".".join(entry["ver2"].split(".")[:3])
ver2 = entry["ver2"]
short = ".".join(ver2.split(".")[:3])
if short not in entry["src"]:
raise SystemExit(f"{requested}: '{short}' not found in src {entry['src']}")

flavors = buildable_flavors(cfg, requested, entry["ver2"])
# a trailing F marks the free-threaded build; it is not part of the version
python_version = requested[:-1] if requested.endswith("F") else requested
arch = "64F" if requested.endswith("F") else "64"

flavors = []
for flavor in cfg["flavors"]:
if str(flavor.get("WINPYARCHDET", "")) != arch:
continue
leg = flavor_entry(cfg, flavor, ver2, python_version)
if leg is not None:
flavors.append(leg)
if not flavors:
raise SystemExit(
f"{requested}: no pylock.64-{entry['ver2'].replace('.', '_')}<flavor>"
f"{requested}: no pylock.64-{ver2.replace('.', '_')}<flavor>"
f"{cfg.get('release_level', '')}.toml under {cfg['cycle_dir']}; "
"commit the lockfiles for this cycle before dispatching"
)

build_location = f"WPy64-{ver2.replace('.', '')}"
return {
"ver2": entry["ver2"],
"ver2": ver2,
"python_version": python_version,
"src": entry["src"],
"sha": entry["sha"],
"cycle_dir": cfg["cycle_dir"],
"release_level": cfg.get("release_level", ""),
"build_location": build_location,
"destwheelhouse": f"{build_location}\\wheelhouse\\included.wheels",
"pandoc_source": cfg["pandoc"]["source"],
"pandoc_sha256": cfg["pandoc"]["sha256"],
# consumed by the build job as strategy.matrix via fromJSON
Expand Down
105 changes: 11 additions & 94 deletions .github/workflows/build_winpython_cycle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ jobs:
runs-on: ubuntu-latest
outputs:
ver2: ${{ steps.cfg.outputs.ver2 }}
build_location: ${{ steps.cfg.outputs.build_location }}
destwheelhouse: ${{ steps.cfg.outputs.destwheelhouse }}
src: ${{ steps.cfg.outputs.src }}
sha: ${{ steps.cfg.outputs.sha }}
cycle_dir: ${{ steps.cfg.outputs.cycle_dir }}
release_level: ${{ steps.cfg.outputs.release_level }}
pandoc_source: ${{ steps.cfg.outputs.pandoc_source }}
pandoc_sha256: ${{ steps.cfg.outputs.pandoc_sha256 }}
matrix: ${{ steps.cfg.outputs.matrix }}
Expand All @@ -58,120 +58,40 @@ jobs:
matrix: ${{ fromJSON(needs.config.outputs.matrix) }}

env:
PYTHON_VERSIONF: ${{ inputs.python_versionf }}
WINPYFLAVOR: ${{ matrix.flavor.name }}
PANDOC: ${{ matrix.flavor.PANDOC }}
WINPYARCHDET: ${{ matrix.flavor.WINPYARCHDET }}
my_cycle: ${{ needs.config.outputs.cycle_dir }}
my_release_level: ${{ needs.config.outputs.release_level }}
WINPYVER: ${{ matrix.flavor.winpyver }}
WINPYVER2: ${{ needs.config.outputs.ver2 }}
build_location: ${{ needs.config.outputs.build_location }}
destwheelhouse: ${{ needs.config.outputs.destwheelhouse }}
WINPYLOCKFILE: ${{ matrix.flavor.lockfile }}
WINPYLOCKFILEwhl: ${{ matrix.flavor.lockfile_wheels }}

steps:
- name: Checkout repository
uses: actions/checkout@v6

- name: Set static and matrix variables based on selected Python version
shell: pwsh
env:
PYTHON_VERSIONF: ${{ env.PYTHON_VERSIONF }}
WINPYARCHDET: ${{ env.WINPYARCHDET }}
WINPYVER2: ${{ needs.config.outputs.ver2 }}
my_cycle: ${{ env.my_cycle }}
my_release_level: ${{ env.my_release_level }}
FLAVOR_NAME: ${{ matrix.flavor.name }}
run: |
# Normalize PYTHON_VERSION by removing trailing 'F' if present
$PYTHON_VERSION = $env:PYTHON_VERSIONF -replace 'F$',''
Add-Content -Path $env:GITHUB_ENV -Value "PYTHON_VERSION=$PYTHON_VERSION"

# Detect architecture (64 or 64F)
$detected_arch = if ($env:PYTHON_VERSIONF -like '*F') { '64F' } else { '64' }

$WINPYVER2 = $env:WINPYVER2
$BUILD_LOCATION = "WPy64-" + ($WINPYVER2 -replace '[.]', "")
Add-Content -Path $env:GITHUB_ENV -Value "build_location=$BUILD_LOCATION"

# ver2-vs-source consistency is checked in cycle_config.py

$WINPYREQUIREMENTS = ''
$WINPYREQUIREMENTSwhl = ''

# Generate requirement files expected names dynamically
$V_TAG = $env:WINPYVER2 -replace '\.', '_'
$testreq = "$($env:my_cycle)/requir.64-$($V_TAG)$($env:FLAVOR_NAME)$($env:my_release_level).txt"
$testwhl = "$($env:my_cycle)/requir.64-$($V_TAG)$($env:FLAVOR_NAME)$($env:my_release_level)_wheels.txt"

Write-Host "Checking for requirements files: $testreq and $testwhl (expected arch $detected_arch)"

if ($env:WINPYARCHDET -eq $detected_arch -and (Test-Path $testreq)) {
$WINPYREQUIREMENTS = $testreq
Write-Host "Found $WINPYREQUIREMENTS"
if (Test-Path $testwhl) {
$WINPYREQUIREMENTSwhl = $testwhl
Write-Host "Found also $WINPYREQUIREMENTSwhl"
}
}

Add-Content -Path $env:GITHUB_ENV -Value "WINPYREQUIREMENTS=$WINPYREQUIREMENTS"
Add-Content -Path $env:GITHUB_ENV -Value "WINPYREQUIREMENTSwhl=$WINPYREQUIREMENTSwhl"

$WINPYLOCKFILE = ''
$WINPYLOCKFILEwhl = ''

# Generate pylock files expected names dynamically
$testreq = "$($env:my_cycle)/pylock.64-$($V_TAG)$($env:FLAVOR_NAME)$($env:my_release_level).toml"
$testwhl = "$($env:my_cycle)/pylock.64-$($V_TAG)$($env:FLAVOR_NAME)$($env:my_release_level)_wheels.toml"

Write-Host "Checking for pylock files: $testreq and $testwhl (expected arch $detected_arch)"

if ($env:WINPYARCHDET -eq $detected_arch -and (Test-Path $testreq)) {
$WINPYLOCKFILE = $testreq
Write-Host "Found $WINPYLOCKFILE"
if (Test-Path $testwhl) {
$WINPYLOCKFILEwhl = $testwhl
Write-Host "Found also $WINPYLOCKFILEwhl"
}
}

Add-Content -Path $env:GITHUB_ENV -Value "WINPYLOCKFILE=$WINPYLOCKFILE"
Add-Content -Path $env:GITHUB_ENV -Value "WINPYLOCKFILEwhl=$WINPYLOCKFILEwhl"

$ARTIFACT_NAME = "publish_${PYTHON_VERSION}$($env:FLAVOR_NAME)"
Add-Content -Path $env:GITHUB_ENV -Value "ARTIFACT_NAME=$ARTIFACT_NAME"

$destwheelhouse = "$BUILD_LOCATION\wheelhouse\included.wheels"
Add-Content -Path $env:GITHUB_ENV -Value "destwheelhouse=$destwheelhouse"

$WINPYVER = "${WINPYVER2}$($env:FLAVOR_NAME)$($env:my_release_level)"
Add-Content -Path $env:GITHUB_ENV -Value "WINPYVER=$WINPYVER"

# Store WINPYVER2 in env for later steps
Add-Content -Path $env:GITHUB_ENV -Value "WINPYVER2=$WINPYVER2"

- name: Download, verify and extract python standalone
if: env.WINPYLOCKFILE != ''
uses: ./.github/actions/python-setup
with:
python_source: ${{ needs.config.outputs.src }}
python_sha256: ${{ needs.config.outputs.sha }}
build_location: ${{ env.build_location }}

- name: Download, checking hash and integrating pandoc binary
if: env.WINPYLOCKFILE != '' && env.PANDOC == '1'
if: matrix.flavor.PANDOC == '1'
uses: ./.github/actions/pandoc-setup
with:
pandoc_source: ${{ needs.config.outputs.pandoc_source }}
pandoc_sha256: ${{ needs.config.outputs.pandoc_sha256 }}
build_location: ${{ env.build_location }}

- name: Upgrade pip and patch launchers
if: env.WINPYLOCKFILE != ''
shell: pwsh
run: |
& "$env:build_location\python\python.exe" -m pip install --upgrade --force-reinstall pip --no-warn-script-location
& "$env:build_location\python\python.exe" -c "from wppm import wppm;dist=wppm.Distribution();dist.patch_standard_packages('pip', to_movable=True)"

- name: Download all requirements
if: ${{ env.WINPYLOCKFILE != '' }}
shell: pwsh
run: |
$py = "$env:build_location\python\python.exe"
Expand All @@ -181,13 +101,11 @@ jobs:
}

- name: Install lockfile
if: env.WINPYLOCKFILE != ''
shell: pwsh
run: |
& "$env:build_location\python\python.exe" -m pip install --no-deps --require-hashes -r $env:WINPYLOCKFILE --no-warn-script-location

- name: Generate Assets and Hashes
if: env.WINPYLOCKFILE != ''
uses: ./.github/actions/publish-winpython
with:
build_location: ${{ env.build_location }}
Expand All @@ -196,15 +114,14 @@ jobs:
winpy_ver: ${{ env.WINPYVER }}
winpy_ver2: ${{ env.WINPYVER2 }}
dotwheelhouse: ${{ env.dotwheelhouse }}
winpy_requirements_whl: ${{ env.WINPYREQUIREMENTSwhl }}
winpy_requirements_whl: ${{ matrix.flavor.requirements_wheels }}
format_zip: ${{ matrix.flavor.formats.zip }}
format_7z: ${{ matrix.flavor.formats['7z'] }}
format_exe: ${{ matrix.flavor.formats.exe }}

- name: Upload artifacts
if: env.WINPYLOCKFILE != ''
uses: actions/upload-artifact@v6
with:
name: ${{ env.ARTIFACT_NAME }}
name: ${{ matrix.flavor.artifact_name }}
path: publish_output
retention-days: 66 # keeps artifact for 66 days