Skip to content

Commit e7b27fc

Browse files
JulienPalardJulienPalard
committed
Backport MAkefile and merge.py
1 parent d025454 commit e7b27fc

File tree

2 files changed

+244
-78
lines changed

2 files changed

+244
-78
lines changed

‎Makefile‎

Lines changed: 94 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@
33
# Here is what you can do:
44
#
55
# - make # Automatically build an HTML local version
6-
# - make todo # To list remaining tasks
6+
# - make todo # To list remaining tasks and show current progression
77
# - make verifs # To check for correctness: wrapping, spelling
8-
# - make wrap # To check for wrapping
8+
# - make wrap # To rewrap modified files
99
# - make spell # To check for spelling
10-
# - make merge # To merge pot from upstream
10+
# - make clean # To remove build artifacts
1111
# - make fuzzy # To find fuzzy strings
12-
# - make progress # To compute current progression
1312
#
1413
# Modes are: autobuild-stable, autobuild-dev, and autobuild-html,
1514
# documented in gen/src/3.6/Doc/Makefile as we're only delegating the
@@ -22,13 +21,41 @@
2221
# test build, we're building with the .rst files that generated our
2322
# .po files.
2423
CPYTHON_CURRENT_COMMIT := cb0b009cfb4ae2a87a8737250c526b8f95d0551f
25-
26-
CPYTHON_PATH := ../cpython/
27-
2824
LANGUAGE := fr
2925
BRANCH := 3.9
3026

31-
EXCLUDED := whatsnew/ c-api/
27+
EXCLUDED := \
28+
whatsnew/2.?.po \
29+
whatsnew/3.[0-8].po \
30+
c-api/ \
31+
distutils/ \
32+
install/ \
33+
library/2to3.po \
34+
library/distutils.po \
35+
library/imp.po \
36+
library/tkinter.tix.po \
37+
library/test.po \
38+
library/aifc.po \
39+
library/asynchat.po \
40+
library/asyncore.po \
41+
library/audioop.po \
42+
library/cgi.po \
43+
library/cgitb.po \
44+
library/chunk.po \
45+
library/crypt.po \
46+
library/imghdr.po \
47+
library/msilib.po \
48+
library/nntplib.po \
49+
library/nis.po \
50+
library/ossaudiodev.po \
51+
library/pipes.po \
52+
library/smtpd.po \
53+
library/sndhdr.po \
54+
library/spwd.po \
55+
library/sunau.po \
56+
library/telnetlib.po \
57+
library/uu.po \
58+
library/xdrlib.po
3259

3360
# Internal variables
3461

@@ -38,6 +65,7 @@ PYTHON := $(shell which python3)
3865
MODE := html
3966
POSPELL_TMP_DIR := .pospell/
4067
JOBS := auto
68+
SPHINXERRORHANDLING = -W
4169

4270
# Detect OS
4371

@@ -58,69 +86,50 @@ endif
5886

5987
.PHONY: all
6088
all: ensure_prerequisites
61-
git -C $(CPYTHON_PATH) checkout $(CPYTHON_CURRENT_COMMIT)
89+
git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT) || (git -C venv/cpython fetch && git -C venv/cpython checkout $(CPYTHON_CURRENT_COMMIT))
6290
mkdir -p locales/$(LANGUAGE)/LC_MESSAGES/
6391
$(CP_CMD) -u --parents *.po */*.po locales/$(LANGUAGE)/LC_MESSAGES/
64-
$(MAKE) -C $(CPYTHON_PATH)/Doc/ \
65-
SPHINXOPTS='-W -j$(JOBS) \
66-
-D locale_dirs=$(abspath locales) \
92+
$(MAKE) -C venv/cpython/Doc/ \
93+
JOBS='$(JOBS)' \
94+
SPHINXOPTS='-D locale_dirs=$(abspath locales) \
6795
-D language=$(LANGUAGE) \
68-
-D gettext_compact=0 \
96+
-D gettext_compact=0 \
6997
-D latex_engine=xelatex \
7098
-D latex_elements.inputenc= \
7199
-D latex_elements.fontenc=' \
100+
SPHINXERRORHANDLING=$(SPHINXERRORHANDLING) \
72101
$(MODE)
73-
@echo "Build success, open file://$(abspath $(CPYTHON_PATH))/Doc/build/html/index.html or run 'make serve' to see them."
102+
@echo "Build success, open file://$(abspath venv/cpython/)/Doc/build/html/index.html or run 'make htmlview' to see them."
103+
104+
105+
# We clone cpython/ inside venv/ because venv/ is the only directory
106+
# excluded by cpython' Sphinx configuration.
107+
venv/cpython/.git/HEAD:
108+
git clone https://github.com/python/cpython venv/cpython
74109

75110

76111
.PHONY: ensure_prerequisites
77-
ensure_prerequisites:
78-
@if [ -z $(CPYTHON_PATH) ]; then \
79-
echo "Your CPYTHON_PATH is empty, please provide one."; \
80-
exit 1; \
81-
fi
82-
@if ! [ -d $(CPYTHON_PATH) ]; then \
83-
echo "Building the translation requires a cpython clone."; \
84-
echo "Please provide the path to a clone using the CPYTHON_PATH variable."; \
85-
echo "(Currently CPYTHON_PATH is $(CPYTHON_PATH)."; \
86-
echo "So you may want to run:"; \
87-
echo ""; \
88-
echo " git clone $(UPSTREAM) $(CPYTHON_PATH)"; \
89-
exit 1; \
90-
fi
91-
@if [ -n "$$(git -C $(CPYTHON_PATH) status --porcelain)" ]; then \
92-
echo "Your cpython clone at $(CPYTHON_PATH) is not clean."; \
93-
echo "In order to avoid breaking things, please clean it first."; \
94-
exit 1; \
95-
fi
112+
ensure_prerequisites: venv/cpython/.git/HEAD
96113
@if ! (blurb help >/dev/null 2>&1 && sphinx-build --version >/dev/null 2>&1); then \
97-
git -C $(CPYTHON_PATH) checkout $(BRANCH); \
98-
echo "You're missing dependencies, please enable a venv and install:"; \
114+
git -C venv/cpython/ checkout $(BRANCH); \
115+
echo "You're missing dependencies please install:"; \
99116
echo ""; \
100-
echo " python -m pip install -r requirements.txt -r $(CPYTHON_PATH)/Doc/requirements.txt"; \
117+
echo " python -m pip install -r requirements.txt -r venv/cpython/Doc/requirements.txt"; \
101118
exit 1; \
102119
fi
103120

104-
.PHONY: serve
105-
serve:
106-
$(MAKE) -C $(CPYTHON_PATH)/Doc/ serve
107-
108-
109-
.PHONY: progress
110-
progress:
111-
@$(PYTHON) -c 'import sys; print("{:.1%}".format(int(sys.argv[1]) / int(sys.argv[2])))' \
112-
$(shell msgcat *.po */*.po | msgattrib --translated | grep -c '^msgid') \
113-
$(shell msgcat *.po */*.po | grep -c '^msgid')
114-
121+
.PHONY: htmlview
122+
htmlview: MODE=htmlview
123+
htmlview: all
115124

116125
.PHONY: todo
117126
todo: ensure_prerequisites
118-
potodo --exclude venv .venv $(EXCLUDED)
127+
potodo --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED)
119128

120129
.PHONY: wrap
121130
wrap: ensure_prerequisites
122-
@echo "Verify wrapping"
123-
powrap --check --quiet *.po **/*.po
131+
@echo "Re wrapping modified files"
132+
powrap -m
124133

125134
SRCS = $(shell git diff --name-only $(BRANCH) | grep '.po$$')
126135
# foo/bar.po => $(POSPELL_TMP_DIR)/foo/bar.po.out
@@ -129,44 +138,51 @@ DESTS = $(addprefix $(POSPELL_TMP_DIR)/,$(addsuffix .out,$(SRCS)))
129138
.PHONY: spell
130139
spell: ensure_prerequisites $(DESTS)
131140

141+
.PHONY: line-length
142+
line-length:
143+
@echo "Searching for long lines..."
144+
@awk '{if (length(gensub(/శ్రీనివాస్/, ".", "g", $$0)) > 80 && length(gensub(/[^ ]/, "", "g")) > 1) {print FILENAME ":" FNR, "line too long:", $$0; ERRORS+=1}} END {if (ERRORS>0) {exit 1}}' *.po */*.po
145+
146+
.PHONY: sphinx-lint
147+
sphinx-lint:
148+
@echo "Checking all files using sphinx-lint..."
149+
@sphinx-lint --enable all --disable line-too-long *.po */*.po
150+
132151
$(POSPELL_TMP_DIR)/%.po.out: %.po dict
133152
@echo "Pospell checking $<..."
134-
mkdir -p $(@D)
153+
@mkdir -p $(@D)
135154
pospell -p dict -l fr_FR $< && touch $@
136155

137156
.PHONY: fuzzy
138157
fuzzy: ensure_prerequisites
139-
potodo -f --exclude venv .venv $(EXCLUDED)
158+
potodo --only-fuzzy --api-url 'https://git.afpy.org/api/v1/repos/AFPy/python-docs-fr/issues?state=open&type=issues' --exclude venv .venv $(EXCLUDED)
159+
160+
.PHONY: check-headers
161+
check-headers:
162+
@grep -L '^# Copyright (C) [0-9-]*, Python Software Foundation' *.po */*.po | while read -r file;\
163+
do \
164+
echo "Please update the po comment in $$file"; \
165+
done
166+
@grep -L '^"Project-Id-Version: Python 3\\n"$$' *.po */*.po | while read -r file;\
167+
do \
168+
echo "Please update the 'Project-Id-Version' header in $$file"; \
169+
done
170+
@grep -L '^"Language: fr\\n"$$' *.po */*.po | while read -r file;\
171+
do \
172+
echo "Please update the 'Language' header in $$file"; \
173+
done
174+
@grep -L '^"Language-Team: FRENCH <traductions@lists.afpy.org>\\n"' *.po */*.po | while read -r file;\
175+
do \
176+
echo "Please update the 'Language-Team' header in $$file"; \
177+
done
140178

141179
.PHONY: verifs
142-
verifs: wrap spell
143-
144-
.PHONY: merge
145-
merge: ensure_prerequisites
146-
@echo "Merge from $(UPSTREAM)"
147-
git -C $(CPYTHON_PATH) checkout $(BRANCH)
148-
git -C $(CPYTHON_PATH) pull --ff-only
149-
(cd $(CPYTHON_PATH)/Doc; sphinx-build -Q -b gettext -D gettext_compact=0 . ../pot)
150-
find $(CPYTHON_PATH)/pot/ -name '*.pot' |\
151-
while read -r POT; \
152-
do \
153-
PO="./$$(echo "$$POT" | sed "s#$(CPYTHON_PATH)/pot/##; s#\.pot\$$#.po#")"; \
154-
mkdir -p "$$(dirname "$$PO")"; \
155-
if [ -f "$$PO" ]; \
156-
then \
157-
msgmerge --backup=off --force-po -U "$$PO" "$$POT"; \
158-
else \
159-
msgcat -o "$$PO" "$$POT"; \
160-
fi \
161-
done
162-
rm -fr $(CPYTHON_PATH)/pot/
163-
sed -i 's|^#: .*Doc/|#: |' *.po */*.po
164-
powrap -m
165-
@printf "\n%s %s\n" "Replace CPYTHON_CURRENT_COMMIT in Makefile by: " $(shell git -C $(CPYTHON_PATH) rev-parse HEAD)
166-
@printf 'To add, you can use:\n git status -s | grep "^ M .*\.po" | cut -d" " -f3 | while read -r file; do if [ $$(git diff "$$file" | wc -l) -gt 13 ]; then git add "$$file"; fi ; done\n'
180+
verifs: spell line-length sphinx-lint check-headers
167181

168182
.PHONY: clean
169183
clean:
170184
@echo "Cleaning *.mo and $(POSPELL_TMP_DIR)"
171-
rm -fr $(POSPELL_TMP_DIR)
185+
rm -fr $(POSPELL_TMP_DIR) locales/$(LANGUAGE)/LC_MESSAGES/
172186
find -name '*.mo' -delete
187+
@echo "Cleaning build directory"
188+
$(MAKE) -C venv/cpython/Doc/ clean

‎merge.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
"""Tool to merge cpython pot files to python-docs-fr po files for a
2+
given branch.
3+
"""
4+
5+
import argparse
6+
import re
7+
import shutil
8+
import subprocess
9+
from pathlib import Path
10+
from subprocess import PIPE
11+
12+
from tqdm import tqdm
13+
14+
NOT_TO_TRANSLATE = {Path("whatsnew/changelog.po")}
15+
16+
17+
def run(*args: str | Path, **kwargs) -> subprocess.CompletedProcess:
18+
"""Run a shell command with subprocess.run() with check=True and
19+
encoding="UTF-8".
20+
"""
21+
return subprocess.run(list(args), encoding="UTF-8", check=True, **kwargs)
22+
23+
24+
def parse_args():
25+
parser = argparse.ArgumentParser(description=__doc__)
26+
parser.add_argument(
27+
"--cpython_repo",
28+
default=Path("venv/cpython"),
29+
type=Path,
30+
help="Use this given cpython clone.",
31+
)
32+
parser.add_argument(
33+
"branch",
34+
help="Merge from this branch or from this commit",
35+
)
36+
return parser.parse_args()
37+
38+
39+
def setup_repo(repo_path: Path, branch: str):
40+
"""Ensure we're up-to-date."""
41+
if branch.find('.') == 1:
42+
run("git", "-C", repo_path, "checkout", branch)
43+
run("git", "-C", repo_path, "pull", "--ff-only")
44+
else: # it's a commit
45+
run("git", "-C", repo_path, "checkout", branch)
46+
47+
48+
def copy_new_files(new_files: set[Path], pot_path: Path) -> None:
49+
"""Just copy new po files to our hierarchy."""
50+
print(f"{len(new_files)} new files.")
51+
for file in new_files:
52+
file.parent.mkdir(parents=True, exist_ok=True)
53+
src = (pot_path / file).with_suffix(".pot")
54+
run("msgcat", "-o", file, src)
55+
run("git", "add", file)
56+
57+
58+
def update_known_files(known_files: set[Path], pot_path: Path) -> None:
59+
"""msgmerge updated pot files in our po files."""
60+
print(f"{len(known_files)} files to update.")
61+
for file in tqdm(known_files, desc="merging pot files"):
62+
src = (pot_path / file).with_suffix(".pot")
63+
run("msgmerge", "-q", "--backup=off", "--force-po", "-U", file, src)
64+
65+
66+
def remove_old_files(old_files: set[Path]) -> None:
67+
"""Remove files removed upstream."""
68+
print(f"{len(old_files)} removed files.")
69+
70+
for file in old_files:
71+
run("git", "rm", file)
72+
73+
74+
def clean_paths(files: set[Path]) -> None:
75+
"""Ensure the path present in po files are always relative.
76+
77+
This avoid having diffs on those paths when we change something in
78+
a script.
79+
"""
80+
for file in tqdm(files, desc="Cleaning rst path in pot files"):
81+
contents = file.read_text(encoding="UTF-8")
82+
contents = re.sub("^#: .*Doc/", "#: ", contents, flags=re.M)
83+
file.write_text(contents, encoding="UTF-8")
84+
85+
86+
def update_makefile(cpython_repo: Path) -> None:
87+
"""Update CPYTHON_CURRENT_COMMIT in the Makefile.
88+
89+
So that when we run `make` it use the same commit than the one
90+
used to generate the `po` files.
91+
"""
92+
makefile = Path("Makefile").read_text(encoding="UTF-8")
93+
head = run(
94+
"git", "-C", cpython_repo, "rev-parse", "HEAD", stdout=PIPE
95+
).stdout.strip()
96+
makefile = re.sub(
97+
"^CPYTHON_CURRENT_COMMIT :=.*$",
98+
f"CPYTHON_CURRENT_COMMIT := {head}",
99+
makefile,
100+
flags=re.M,
101+
)
102+
Path("Makefile").write_text(makefile, encoding="UTF-8")
103+
run("git", "add", "Makefile")
104+
105+
106+
def git_add_relevant_files():
107+
"""Add only files with relevant modifications.
108+
109+
This only add files with actual modifications, not just metadata
110+
modifications, to avoid noise in history.
111+
"""
112+
modified_files = run("git", "ls-files", "-m", stdout=PIPE).stdout.split("\n")
113+
modified_po_files = [line for line in modified_files if line.endswith(".po")]
114+
for file in modified_po_files:
115+
diff = run("git", "diff", "-U0", file, stdout=PIPE).stdout
116+
if len(diff.split("\n")) > 8:
117+
run("git", "add", file)
118+
else:
119+
run("git", "checkout", "--", file)
120+
121+
122+
def main():
123+
args = parse_args()
124+
setup_repo(args.cpython_repo, args.branch)
125+
run(
126+
*["sphinx-build", "-jauto", "-QDgettext_compact=0", "-bgettext", ".", "../pot"],
127+
cwd=args.cpython_repo / "Doc",
128+
)
129+
pot_path = args.cpython_repo / "pot"
130+
upstream = {
131+
file.relative_to(pot_path).with_suffix(".po")
132+
for file in pot_path.glob("**/*.pot")
133+
} - NOT_TO_TRANSLATE
134+
downstream = {
135+
Path(po)
136+
for po in run("git", "ls-files", "*.po", stdout=PIPE).stdout.splitlines()
137+
}
138+
copy_new_files(upstream - downstream, pot_path=pot_path)
139+
update_known_files(upstream & downstream, pot_path=pot_path)
140+
remove_old_files(downstream - upstream)
141+
clean_paths(upstream)
142+
shutil.rmtree(pot_path)
143+
run("powrap", "-m")
144+
update_makefile(args.cpython_repo)
145+
git_add_relevant_files()
146+
run("git", "commit", "-m", "Make merge")
147+
148+
149+
if __name__ == "__main__":
150+
main()

0 commit comments

Comments
 (0)