1+ #!/usr/bin/env python3
2+ """
3+ scripts/po_sync.py
4+
5+ Shared logic for syncing this repo's .po files against a CPython docs
6+ checkout. Used by both:
7+
8+ - scripts/update_python_version.py (manual, deliberate version bumps,
9+ full clone, creates new .po files for brand-new pages)
10+ - .github/workflows/sync-with-cpython.yml (nightly automated msgid sync,
11+ sparse checkout, merge-only, opens an issue for fuzzy strings)
12+
13+ Keeping this logic in one place means both paths build .pot files and run
14+ msgmerge/msgfmt identically -- no more silent flag drift (e.g. one path
15+ passing --no-location --no-wrap and the other not), which otherwise shows
16+ up as spurious rewrap-only diffs on whichever path runs next.
17+
18+ This module is a library first, CLI second. As a CLI it exposes just the
19+ "mechanical middle" of the sync -- build .pot templates, merge them into
20+ existing .po files, flag new upstream pages with no .po yet, validate --
21+ so the GitHub Actions workflow can shell out to one command instead of
22+ reimplementing the loop in bash/awk.
23+ """
24+ from __future__ import annotations
25+
26+ import argparse
27+ import shutil
28+ import subprocess
29+ import sys
30+ from dataclasses import dataclass , field
31+ from pathlib import Path
32+
33+ REPO_ROOT = Path (__file__ ).resolve ().parent .parent
34+
35+ # Flags msgmerge is run with everywhere. Keeping this in one constant is the
36+ # whole point: previously the script omitted --no-location --no-wrap while
37+ # the workflow included them, so whichever ran second would produce a huge
38+ # rewrap-only diff on top of (and obscuring) any real content changes.
39+ MSGMERGE_FLAGS = ["--update" , "--backup=off" , "--no-location" , "--no-wrap" ]
40+
41+ IGNORED_DIR_NAMES = {".git" , ".cpython-src" , ".pot-templates" }
42+
43+
44+ def run (cmd : list , cwd : Path | None = None , check : bool = True ) -> subprocess .CompletedProcess :
45+ print (f"$ { ' ' .join (str (c ) for c in cmd )} )
46+ return subprocess .run (cmd , cwd = cwd , check = check )
47+
48+
49+ def _is_ignored (path : Path ) -> bool :
50+ return any (part in IGNORED_DIR_NAMES for part in path .parts )
51+
52+
53+ def iter_po_files (repo_root : Path = REPO_ROOT ):
54+ for po_path in sorted (repo_root .rglob ("*.po" )):
55+ if not _is_ignored (po_path .relative_to (repo_root )):
56+ yield po_path
57+
58+
59+ # ---------------------------------------------------------------------------
60+ # Fetch + build .pot templates
61+ # ---------------------------------------------------------------------------
62+
63+ def fetch_cpython_full (tag : str , workdir : Path ) -> None :
64+ """Full clone of CPython at `tag`. Used by the version-bump script,
65+ which needs the full doc tree to build every .pot (including ones for
66+ brand-new pages that a sparse checkout might not anticipate)."""
67+ if workdir .exists ():
68+ shutil .rmtree (workdir )
69+ run (["git" , "clone" , "--depth" , "1" , "--branch" , tag ,
70+ "https://github.com/python/cpython.git" , str (workdir )])
71+
72+
73+ def fetch_cpython_sparse (tag : str , workdir : Path ) -> None :
74+ """Sparse, blobless clone of just Doc/ + Include/. Used by the nightly
75+ workflow, where we only ever merge into .po files that already exist,
76+ so we don't need the rest of the tree."""
77+ if workdir .exists ():
78+ shutil .rmtree (workdir )
79+ run (["git" , "clone" , "--depth" , "1" , "--filter=blob:none" , "--sparse" ,
80+ "--branch" , tag , "https://github.com/python/cpython.git" , str (workdir )])
81+ run (["git" , "sparse-checkout" , "set" , "Doc" , "Include" ], cwd = workdir )
82+
83+
84+ def build_gettext (doc_dir : Path ) -> Path :
85+ """Build .pot templates from a CPython Doc/ checkout, return their root dir."""
86+ venv_dir = doc_dir / "venv"
87+ run ([sys .executable , "-m" , "venv" , str (venv_dir )])
88+ pip = venv_dir / "bin" / "pip"
89+ sphinx_build = venv_dir / "bin" / "sphinx-build"
90+ run ([str (pip ), "install" , "-r" , "requirements.txt" ], cwd = doc_dir )
91+ pot_root = doc_dir / "build" / "gettext"
92+ run ([str (sphinx_build ), "-b" , "gettext" , "." , str (pot_root )], cwd = doc_dir )
93+ return pot_root
94+
95+
96+ # ---------------------------------------------------------------------------
97+ # Merge
98+ # ---------------------------------------------------------------------------
99+
100+ @dataclass
101+ class MergeReport :
102+ updated : list = field (default_factory = list )
103+ new_po_created : list = field (default_factory = list )
104+ missing_pot : list = field (default_factory = list ) # .po with no matching .pot upstream
105+ new_pot_no_po : list = field (default_factory = list ) # .pot with no .po yet (new upstream page)
106+
107+ def summary (self ) -> str :
108+ lines = [
109+ f"{ len (self .updated )} ,
110+ f"{ len (self .new_po_created )} ,
111+ f"{ len (self .missing_pot )}
112+ f"(page may have been removed/renamed upstream)" ,
113+ f"{ len (self .new_pot_no_po )} ,
114+ ]
115+ return "Summary: " + ", " .join (lines )
116+
117+
118+ def merge_existing (pot_root : Path , repo_root : Path = REPO_ROOT ) -> MergeReport :
119+ """Merge new .pot content into every existing .po file. This is the
120+ core operation both the nightly workflow and the version-bump script
121+ need, and previously the only one the workflow performed."""
122+ report = MergeReport ()
123+ for po_path in iter_po_files (repo_root ):
124+ rel = po_path .relative_to (repo_root )
125+ pot_path = pot_root / rel .with_suffix (".pot" )
126+ if not pot_path .exists ():
127+ print (f" ! no matching .pot for { rel }
128+ f"(page may have been removed/renamed upstream -- review manually)" )
129+ report .missing_pot .append (rel )
130+ continue
131+ run (["msgmerge" , * MSGMERGE_FLAGS , str (po_path ), str (pot_path )])
132+ report .updated .append (rel )
133+ return report
134+
135+
136+ def detect_new_pot_files (pot_root : Path , repo_root : Path = REPO_ROOT ) -> list :
137+ """Find .pot files with no corresponding .po file yet -- i.e. pages
138+ added upstream since the last sync. Both entry points can call this;
139+ only the version-bump script actually creates the .po (see
140+ create_po_for_new_pot), but the nightly workflow can now at least
141+ *report* these instead of silently dropping them (item 1)."""
142+ new_pot = []
143+ for pot_path in sorted (pot_root .rglob ("*.pot" )):
144+ rel = pot_path .relative_to (pot_root )
145+ po_path = repo_root / rel .with_suffix (".po" )
146+ if not po_path .exists ():
147+ new_pot .append (rel )
148+ return new_pot
149+
150+
151+ def create_po_for_new_pot (pot_root : Path , rel_pot_paths : list , locale : str = "fa" ,
152+ repo_root : Path = REPO_ROOT ) -> list :
153+ """Create a fresh .po (via msginit) for each given new .pot. Only called
154+ from the version-bump script -- the nightly workflow reports these via
155+ detect_new_pot_files() but leaves creation to a human-reviewed run."""
156+ created = []
157+ for rel in rel_pot_paths :
158+ pot_path = pot_root / rel
159+ po_path = repo_root / rel .with_suffix (".po" )
160+ po_path .parent .mkdir (parents = True , exist_ok = True )
161+ run (["msginit" , "--no-translator" , "-l" , locale ,
162+ "-i" , str (pot_path ), "-o" , str (po_path )])
163+ created .append (rel )
164+ return created
165+
166+
167+ def merge_all (pot_root : Path , create_new : bool , locale : str = "fa" ,
168+ repo_root : Path = REPO_ROOT ) -> MergeReport :
169+ report = merge_existing (pot_root , repo_root )
170+ new_pot = detect_new_pot_files (pot_root , repo_root )
171+ if create_new :
172+ report .new_po_created = create_po_for_new_pot (pot_root , new_pot , locale , repo_root )
173+ else :
174+ report .new_pot_no_po = new_pot
175+ return report
176+
177+
178+ # ---------------------------------------------------------------------------
179+ # Validate
180+ # ---------------------------------------------------------------------------
181+
182+ def check_po_files (repo_root : Path = REPO_ROOT ) -> list :
183+ """Run msgfmt --check on every .po file. Returns a list of (path, stderr)
184+ for any that fail; empty list means all good."""
185+ bad = []
186+ for po_path in iter_po_files (repo_root ):
187+ result = subprocess .run (
188+ ["msgfmt" , "--check" , "-o" , "/dev/null" , str (po_path )],
189+ capture_output = True , text = True ,
190+ )
191+ if result .returncode != 0 :
192+ bad .append ((po_path , result .stderr .strip ()))
193+ return bad
194+
195+
196+ # ---------------------------------------------------------------------------
197+ # CLI -- the "sync-only" mode the workflow shells out to (item 4)
198+ # ---------------------------------------------------------------------------
199+
200+ def _cli_sync_only (args : argparse .Namespace ) -> int :
201+ """Sparse clone + build gettext + merge into existing .po files +
202+ validate. This is everything the nightly workflow needs, in one call,
203+ instead of inline bash/awk. Report-only for new upstream pages (does
204+ NOT create new .po files -- that stays a deliberate, human-run action
205+ via update_python_version.py)."""
206+ workdir = REPO_ROOT / ".cpython-src"
207+ tag = args .tag
208+
209+ print (f"== Sparse-fetching CPython { tag } )
210+ fetch_cpython_sparse (tag , workdir )
211+
212+ print ("\n == Building gettext templates ==" )
213+ pot_root = build_gettext (workdir / "Doc" )
214+
215+ print ("\n == Merging into existing .po files ==" )
216+ report = merge_all (pot_root , create_new = False )
217+ print (f"\n { report .summary ()} )
218+ if report .new_pot_no_po :
219+ print ("\n New upstream pages with no .po yet (run update_python_version.py to create):" )
220+ for rel in report .new_pot_no_po :
221+ print (f" - { rel } )
222+
223+ print ("\n == Validating .po files ==" )
224+ bad = check_po_files ()
225+ if bad :
226+ print ("\n Broken .po files (fix before committing):" )
227+ for path , err in bad :
228+ print (f" { path } \n { err } )
229+
230+ if not args .keep_src :
231+ shutil .rmtree (workdir , ignore_errors = True )
232+ shutil .rmtree (REPO_ROOT / ".pot-templates" , ignore_errors = True )
233+
234+ if bad :
235+ return 1
236+ return 0
237+
238+
239+ def main () -> None :
240+ parser = argparse .ArgumentParser (description = __doc__ )
241+ sub = parser .add_subparsers (dest = "command" , required = True )
242+
243+ sync = sub .add_parser (
244+ "sync-only" ,
245+ help = "Sparse-checkout sync used by the nightly workflow: fetch, "
246+ "build gettext, merge into existing .po files, report new "
247+ "upstream pages, validate. Does not create new .po files." ,
248+ )
249+ sync .add_argument ("tag" , help = "CPython git tag to sync against, e.g. v3.14.7" )
250+ sync .add_argument ("--keep-src" , action = "store_true" ,
251+ help = "keep the scratch CPython checkout instead of deleting it" )
252+ sync .set_defaults (func = _cli_sync_only )
253+
254+ args = parser .parse_args ()
255+ sys .exit (args .func (args ))
256+
257+
258+ if __name__ == "__main__" :
259+ main ()
0 commit comments