Skip to content

Commit d75ade9

Browse files
stonebigstonebig
committed
add competing pythonic replacement for generate_a_winpython.bat
from 200 lines up to 350 lines, so not yet lean, but goal was to be "on par" first
1 parent 1a14534 commit d75ade9

File tree

2 files changed

+344
-0
lines changed

2 files changed

+344
-0
lines changed

‎generate_a_winpython_distropy.bat‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
rem generate_a_winpython_distropy.bat: to be launched with a winpython sub-directory
2+
rem where 'build_winpython.py' and 'make.py' are
3+
@echo on
4+
5+
REM === Initialize default values ===
6+
if not defined my_release_level set "my_release_level=b1"
7+
if not defined my_create_installer set "my_create_installer=True"
8+
if not defined my_constraints set "my_constraints=C:\WinP\constraints.txt"
9+
if not defined target_python_exe set "target_python_exe=python.exe"
10+
if not defined mandatory_requirements set "mandatory_requirements=%~dp0mandatory_requirements.txt"
11+
12+
set "my_archive_dir=%~dp0WinPython_build_logs"
13+
if not exist "%my_archive_dir%" mkdir "%my_archive_dir%"
14+
15+
REM === Format log timestamp ===
16+
set "my_time=%time:~0,5%"
17+
set "my_time=%my_time::=_%"
18+
set "my_time=%my_time: =0%"
19+
20+
set "my_archive_log=%my_archive_dir%\build_%my_pyver%_%my_release%%my_flavor%_%my_release_level%_of_%date:/=-%at_%my_time%.txt"
21+
22+
REM === Define base build and distribution paths ===
23+
set "my_basedir=%my_root_dir_for_builds%\bd%my_python_target%"
24+
set "my_WINPYDIRBASE=%my_basedir%\bu%my_flavor%\WPy%my_arch%-%my_python_target_release%%my_release%%my_release_level%"
25+
26+
rem a building env need is a Python with packages: WinPython + build + flit + packaging + mkshim400.py
27+
set "my_buildenv=C:\WinPdev\WPy64-310111"
28+
set "my_buildenvi=C:\WinPdev\WPy64-310111\python-3.10.11.amd64"
29+
set "my_python_exe=C:\WinPdev\WPy64-310111\python-3.10.11.amd64\python.exe"
30+
31+
if "%my_requirements_pre%" == "" set "my_requirements_pre=%mandatory_requirements%"
32+
set "my_requirements_pre=%mandatory_requirements%"
33+
34+
cd/d %~dp0
35+
echo %my_python_exe% -m winpython.build_winpython --buildenv %my_buildenvi% --python-target %my_python_target% --release %my_release% --release-level %my_release_level% --winpydirbase %my_WINPYDIRBASE% --flavor %my_flavor% --source_dirs %my_source_dirs% --tools_dirs %my_toolsdirs% --log-dir %~dp0WinPython_build_logs --mandatory-req %mandatory_requirements% --pre-req %my_requirements_pre% --requirements %my_requirements% --constraints %my_constraints% --find-links %my_find_links% --wheelhousereq "%wheelhousereq%" --create-installer "%my_create_installer%"
36+
%my_python_exe% -m winpython.build_winpython --buildenv %my_buildenvi% --python-target %my_python_target% --release %my_release% --release-level %my_release_level% --winpydirbase %my_WINPYDIRBASE% --flavor %my_flavor% --source_dirs %my_source_dirs% --tools_dirs %my_toolsdirs% --log-dir %~dp0WinPython_build_logs --mandatory-req %mandatory_requirements% --pre-req %my_requirements_pre% --requirements %my_requirements% --constraints %my_constraints% --find-links %my_find_links% --wheelhousereq "%wheelhousereq%" --create-installer "%my_create_installer%"
37+
pause
38+
exit

‎winpython/build_winpython.py‎

Lines changed: 306 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,306 @@
1+
# build_winpython.py
2+
import os, sys, argparse, datetime, subprocess, shutil
3+
from pathlib import Path
4+
5+
# --- logging, helpers (same as Step 2) ---
6+
def log_section(logfile, message):
7+
ts = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
8+
section = f"\n{'-'*40}\n({ts}) {message}\n{'-'*40}\n"
9+
print(section); open(logfile, 'a', encoding='utf-8').write(section)
10+
11+
# --- Helpers ---
12+
def get_target_release(python_target):
13+
mapping = {
14+
'311': ('3119', '2'),
15+
'312': ('31210', '2'),
16+
'313': ('3135', '1'),
17+
'314': ('3140', '1')
18+
}
19+
return mapping.get(python_target, (None, None))
20+
21+
def delete_folder_if_exists(folder: Path,check_flavor: str=""):
22+
check_last = folder.parent.name if not folder.is_dir() else folder.name
23+
if folder.exists() and folder.is_dir() and check_last=="bu" + check_flavor:
24+
print ("hello", folder)
25+
folder_old = Path(str(folder)+'.old')
26+
if folder_old.exists():
27+
shutil.rmtree(Path(str(folder)+'.old'))
28+
os.rename(folder, folder_old)
29+
shutil.rmtree(Path(str(folder)+'.old'))
30+
return
31+
for item in folder_old:
32+
if item.is_dir():
33+
pass
34+
# delete_folder_if_exists(item)
35+
else:
36+
pass
37+
# item.unlink()
38+
# folder.rmdir()
39+
40+
def activate_env(env_path):
41+
# Windows-specific virtual env activation
42+
env_script = Path(env_path) / "scripts" / "env.bat"
43+
if not env_script.exists():
44+
raise FileNotFoundError(f"Cannot find env.bat at {env_script}")
45+
# Note: This step is simplified here. Full environment activation logic will be added later.
46+
47+
48+
def run_make_py(build_python, winpydirbase, args, logfile):
49+
cmd = [
50+
build_python, "-c",
51+
(
52+
"from wppm import make; "
53+
f"make.make_all({args.release}, '{args.release_level}', basedir_wpy=r'{winpydirbase}', "
54+
f"verbose=True, flavor='{args.flavor}', source_dirs=r'{args.source_dirs}', "
55+
f"toolsdirs=r'{args.tools_dirs}')" #, portable_dir=r'{args.portable_dir}')"
56+
)
57+
]
58+
print(cmd)
59+
from . import make
60+
make.make_all( args.release , args.release_level , basedir_wpy= winpydirbase ,
61+
verbose=True, flavor= args.flavor , source_dirs= args.source_dirs ,
62+
toolsdirs= args.tools_dirs) #, portable_dir= args.portable_dir )
63+
#subprocess.run(cmd, stdout=open(logfile, 'a'), stderr=subprocess.STDOUT, check=True)
64+
65+
def pip_install(python_exe: Path, req_file: str, constraints: str, find_links: str, logfile: Path, label: str):
66+
if req_file and Path(req_file).exists():
67+
cmd = [
68+
str(python_exe), "-m", "pip", "install",
69+
"-r", req_file, "-c", constraints,
70+
"--pre", "--no-index", f"--find-links={find_links}"
71+
]
72+
log_section(logfile, f"Pip‑install {label}")
73+
subprocess.run(cmd, stdout=open(logfile, 'a'), stderr=subprocess.STDOUT, check=True)
74+
else:
75+
log_section(logfile, f"No {label} specified/skipped")
76+
77+
def run_command(cmd, env=None, log_file=None):
78+
print(f"[RUN] {cmd}")
79+
with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, universal_newlines=True) as proc:
80+
for line in proc.stdout:
81+
print(line, end="")
82+
if log_file:
83+
with open(log_file, "a", encoding="utf-8") as logf:
84+
logf.write(line)
85+
86+
def patch_winpython(python_exe: Path, logfile: Path):
87+
cmd = [str(python_exe), "-c" ,
88+
(
89+
"from wppm import wppm;"
90+
"wppm.Distribution().patch_standard_packages('', to_movable=True)"
91+
)]
92+
print(cmd)
93+
subprocess.run(cmd, stdout=open(logfile, 'a'), stderr=subprocess.STDOUT, check=True)
94+
#run_command(f'{activate_env(WINPYDIRBASE)} python -c "from wppm import wppm; wppm.Distribution(r\'{WINPYDIRBASE}\').patch_standard_packages(\'\', to_movable=True)"')
95+
96+
97+
def check_env_bat(winpydirbase: Path):
98+
envbat = winpydirbase / "scripts" / "env.bat"
99+
if not envbat.exists():
100+
raise FileNotFoundError(f"Missing env.bat at {envbat}")
101+
102+
def generate_lockfiles(target_python: Path, winpydirbase: Path, constraints: str, find_links: str, logfile: Path, file_postfix: str):
103+
pip_req = winpydirbase.parent / "requirement_temp.txt"
104+
with subprocess.Popen([str(target_python), "-m", "pip", "freeze"], stdout=subprocess.PIPE) as proc:
105+
packages = [l for l in proc.stdout if b"winpython" not in l]
106+
pip_req.write_bytes(b"".join(packages))
107+
# Lock to web and local (scaffolding)
108+
# if local --no-index --trusted-host=None --find-links="%my_find_links%"
109+
for kind in ("", "local"):
110+
out = winpydirbase.parent / f"pylock.{file_postfix}_{kind}.toml"
111+
outreq = winpydirbase.parent / f"requir.{file_postfix}_{kind}.txt"
112+
print(
113+
[str(target_python), "-m", "pip", "lock", "--no-deps", "-c", constraints] +
114+
(["--find-links",find_links] if kind =="local" else []) +
115+
["-r", str(pip_req), "-o", str(out)]
116+
)
117+
subprocess.run(
118+
[str(target_python), "-m", "pip", "lock", "--no-deps", "-c", constraints] +
119+
(["--find-links",find_links] if kind =="local" else []) +
120+
["-r", str(pip_req), "-o", str(out)],
121+
stdout=open(logfile, 'a'), stderr=subprocess.STDOUT, check=True
122+
)
123+
# Convert both locks to requirement.txt with hash256
124+
cmd = f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{out}', r'{outreq}')"
125+
print(
126+
[str(target_python), "-c", cmd]
127+
)
128+
subprocess.run(
129+
[str(target_python), "-c", cmd],
130+
stdout=open(logfile, 'a'), stderr=subprocess.STDOUT, check=False
131+
)
132+
# check equality
133+
from filecmp import cmp
134+
web, local = "", "local"
135+
if not cmp(winpydirbase.parent / f"requir.{file_postfix}_{web}.txt", winpydirbase.parent / f"requir.{file_postfix}_{local}.txt"):
136+
print("ALARM differences in ", winpydirbase.parent / f"requir.{file_postfix}_{web}.txt", winpydirbase.parent / f"requir.{file_postfix}_{local}.txt")
137+
raise os.error
138+
else:
139+
print ("match ok ",winpydirbase.parent / f"requir.{file_postfix}_{web}.txt", winpydirbase.parent / f"requir.{file_postfix}_{local}.txt")
140+
141+
# --- main ---
142+
def main():
143+
parser = argparse.ArgumentParser()
144+
145+
parser.add_argument('--python-target', required=True, help='Target Python version, e.g. 311')
146+
parser.add_argument('--release', default='', help='Release')
147+
parser.add_argument('--flavor', default='', help='Build flavor')
148+
parser.add_argument('--arch', default='64', help='Architecture')
149+
150+
parser.add_argument('--release-level', default='b1', help='Release level (e.g., b1, rc)')
151+
parser.add_argument('--winpydirbase', required=True, help='Path to put environment')
152+
parser.add_argument('--source_dirs', required=True, help='Path to directory with python zip')
153+
154+
parser.add_argument('--tools_dirs', required=True, help='Path to directory with python zip')
155+
#parser.add_argument('--portable_dir', required=True, help='Path to normal make.py')
156+
157+
parser.add_argument('--buildenv', required=True, help='Path to build environment')
158+
parser.add_argument('--constraints', default='constraints.txt', help='Constraints file')
159+
parser.add_argument('--requirements', help='Main requirements.txt file')
160+
parser.add_argument('--find-links', default='wheelhouse', help='Path to local wheelhouse')
161+
parser.add_argument('--log-dir', default='WinPython_build_logs', help='Directory for logs')
162+
parser.add_argument('--mandatory-req', help='Mandatory requirements file')
163+
parser.add_argument('--pre-req', help='Pre requirements file')
164+
parser.add_argument('--wheelhousereq', help='Wheelhouse requirements file')
165+
parser.add_argument('--create-installer', default='', help='default installer to create')
166+
args = parser.parse_args()
167+
168+
# compute paths (same as Step2)...
169+
build_python = Path(args.buildenv) / "python.exe"
170+
winpydirbase = Path(args.winpydirbase) # from Step2 logic
171+
target_python = winpydirbase / "python" / "python.exe"
172+
173+
# Setup paths and logs
174+
now = datetime.datetime.now()
175+
log_dir = Path(args.log_dir)
176+
log_dir.mkdir(exist_ok=True)
177+
time_str = now.strftime("%Y-%m-%d_at_%H%M")
178+
log_file = log_dir / f"build_{args.python_target}_{args.flavor}_{args.release_level}_{time_str}.txt"
179+
180+
#logs termination
181+
z = Path(winpydirbase).name[(4+len(args.arch)):-len(args.release_level)]
182+
tada = f"{z[:1]}_{z[1:3]}_{z[3]}_{args.release}"
183+
winpyver2 = tada.replace('_','.')
184+
file_postfix=f"{args.arch}-{args.python_target[:1]}_{args.python_target[1:]}_{args.release}{args.flavor}{args.release_level}"
185+
file_postfix=f"{args.arch}-{tada}{args.flavor}{args.release_level}"
186+
187+
log_section(log_file, f"Preparing build for Python {args.python_target} ({args.arch}-bit)")
188+
189+
log_section(log_file, f"🙏 Step 0: displace old {Path(winpydirbase)}")
190+
# O) Pre-clear
191+
192+
#delete_folder_if_exists(Path(winpydirbase))
193+
delete_folder_if_exists(winpydirbase.parent, check_flavor=args.flavor) #bu{flavor]}
194+
195+
log_section(log_file, f"🙏 Step 1: make.py Python with {str(build_python)} at ({winpydirbase}")
196+
# 1) run make.py
197+
run_make_py(str(build_python), winpydirbase, args, log_file)
198+
199+
# 2) env.bat exists
200+
check_env_bat(winpydirbase)
201+
202+
# 3) pip install in built environment
203+
log_section(log_file, "🙏 Step 3: install requirements")
204+
205+
for label, req in [
206+
("Mandatory", args.mandatory_req),
207+
("Pre", args.pre_req),
208+
("Main", args.requirements),
209+
]:
210+
pip_install(target_python, req, args.constraints, args.find_links, log_file, label)
211+
212+
# 4) patch Winpython
213+
log_section(log_file, "🙏 Step 4: Patch Winpython")
214+
patch_winpython(target_python, log_file)
215+
216+
# 5) Install Wheelhouse
217+
if args.wheelhousereq:
218+
log_section(log_file, f"🙏 Step 5: install wheelhouse requirements {args.wheelhousereq}")
219+
wheelhousereq = Path(args.wheelhousereq)
220+
kind = "local"
221+
out = winpydirbase.parent / f"pylock.{file_postfix}_wheels{kind}.toml"
222+
outreq = winpydirbase.parent / f"requir.{file_postfix}_wheels{kind}.txt"
223+
if wheelhousereq.is_file():
224+
# Generate pylock from wheelhousereq
225+
cmd = [str(target_python), "-m" , "pip", "lock","--no-index", "--trusted-host=None",
226+
"--find-links", args.find_links, "-c", args.constraints, "-r", wheelhousereq,
227+
"-o", out ]
228+
subprocess.run(cmd, stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=True)
229+
# Convert pylock to requirements with hash
230+
cmd = f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{out}', r'{outreq}')"
231+
print( [str(target_python), "-c", cmd] )
232+
subprocess.run([str(target_python), "-c", cmd],
233+
stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=False
234+
)
235+
236+
kind = ""
237+
outw = winpydirbase.parent / f"pylock.{file_postfix}_wheels{kind}.toml"
238+
outreqw = winpydirbase.parent / f"requir.{file_postfix}_wheels{kind}.txt"
239+
# Generate web pylock from local frozen hashes
240+
cmd = [str(target_python), "-m" , "pip", "lock","--no-deps", "--require-hashes",
241+
"-r", str(outreq), "-o", str(outw) ]
242+
subprocess.run(cmd, stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=True)
243+
cmd = f"from wppm import wheelhouse as wh; wh.pylock_to_req(r'{outw}', r'{outreqw}')"
244+
print( [str(target_python), "-c", cmd] )
245+
subprocess.run([str(target_python), "-c", cmd],
246+
stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=False
247+
)
248+
249+
# Use wppm to download local from req made with web hashes
250+
wheelhouse = winpydirbase / "wheelhouse" / "included.wheels"
251+
cmd = [str(target_python), "-X", "utf8", "-m", "wppm", str(out), "-ws", args.find_links,
252+
"-wd", str(wheelhouse)
253+
]
254+
print(cmd)
255+
subprocess.run(cmd, stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=False)
256+
257+
258+
259+
# 6) lock files
260+
log_section(log_file, "🙏 Step 6: install lockfiles")
261+
print(target_python, winpydirbase, args.constraints, args.find_links, log_file)
262+
print(' - - -')
263+
generate_lockfiles(target_python, winpydirbase, args.constraints, args.find_links, log_file, file_postfix)
264+
265+
# 5b) Archive lock files
266+
267+
# 6) generate changelog
268+
mdn = f"WinPython{args.flavor}-{args.arch}bit-{winpyver2}.md"
269+
out = f"WinPython{args.flavor}-{args.arch}bit-{winpyver2}_History.md"
270+
changelog_dir = log_dir.parent/ "changelogs"
271+
272+
log_section(log_file, f"🙏 Step 6: generate changelog {mdn}")
273+
274+
cmd = ["set", f"WINPYVER2={winpyver2}&", "set", f"WINPYFLAVOR={args.flavor}&",
275+
"set", f"WINPYVER={winpyver2}{args.flavor}{args.release_level}&",
276+
str(target_python), "-c" ,
277+
(
278+
"from wppm import wppm;"
279+
"result = wppm.Distribution().generate_package_index_markdown();"
280+
f"open(r'{winpydirbase.parent / mdn}', 'w', encoding='utf-8').write(result)"
281+
)]
282+
print(cmd)
283+
subprocess.run(cmd, stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=True, shell=True)
284+
shutil.copyfile (winpydirbase.parent / mdn, changelog_dir / mdn)
285+
286+
cmd = [str(target_python), "-c",
287+
(
288+
"from wppm import diff;"
289+
f"result = diff.compare_package_indexes('{winpyver2}', searchdir=r'{log_dir}', flavor=r'{args.flavor}', architecture={args.arch});"
290+
f"open(r'{winpydirbase.parent / out}', 'w', encoding='utf-8').write(result)"
291+
)]
292+
subprocess.run(cmd, stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=True)
293+
shutil.copyfile (winpydirbase.parent / out, changelog_dir / out)
294+
log_section(log_file, "✅ Step 6 complete")
295+
296+
# 7) Create Installers
297+
if args.create_installer != "":
298+
log_section(log_file, "🙏 Step 7 Create Installer")
299+
stem = f"WinPython{args.arch}-{winpyver2}{args.flavor}{args.release_level}"
300+
cmd = f"from wppm import utils; utils.command_installer_7zip(r'{winpydirbase}', r'{winpydirbase.parent}', r'{stem}', r'{args.create_installer}')"
301+
print( [str(target_python), "-c", cmd] )
302+
subprocess.run([str(target_python), "-c", cmd],
303+
stdout=open(log_file, 'a'), stderr=subprocess.STDOUT, check=False)
304+
305+
if __name__ == '__main__':
306+
main()

0 commit comments

Comments
 (0)