Skip to content

Commit 44213bc

Browse files
tomasr8tomasr8
authored
gh-130453: pygettext: Extend support for specifying custom keywords (GH-130463)
1 parent 31ef8fd commit 44213bc

File tree

5 files changed

+226
-18
lines changed

5 files changed

+226
-18
lines changed

‎Lib/test/test_tools/i18n_data/custom_keywords.pot‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# SOME DESCRIPTIVE TITLE.
2+
# Copyright (C) YEAR ORGANIZATION
3+
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
4+
#
5+
msgid ""
6+
msgstr ""
7+
"Project-Id-Version: PACKAGE VERSION\n"
8+
"POT-Creation-Date: 2000-01-01 00:00+0000\n"
9+
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
10+
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
11+
"Language-Team: LANGUAGE <LL@li.org>\n"
12+
"MIME-Version: 1.0\n"
13+
"Content-Type: text/plain; charset=UTF-8\n"
14+
"Content-Transfer-Encoding: 8bit\n"
15+
"Generated-By: pygettext.py 1.5\n"
16+
17+
18+
#: custom_keywords.py:9 custom_keywords.py:10
19+
msgid "bar"
20+
msgstr ""
21+
22+
#: custom_keywords.py:12
23+
msgid "cat"
24+
msgid_plural "cats"
25+
msgstr[0] ""
26+
msgstr[1] ""
27+
28+
#: custom_keywords.py:13
29+
msgid "dog"
30+
msgid_plural "dogs"
31+
msgstr[0] ""
32+
msgstr[1] ""
33+
34+
#: custom_keywords.py:15
35+
msgctxt "context"
36+
msgid "bar"
37+
msgstr ""
38+
39+
#: custom_keywords.py:17
40+
msgctxt "context"
41+
msgid "cat"
42+
msgid_plural "cats"
43+
msgstr[0] ""
44+
msgstr[1] ""
45+

‎Lib/test/test_tools/i18n_data/custom_keywords.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from gettext import (
2+
gettext as foo,
3+
ngettext as nfoo,
4+
pgettext as pfoo,
5+
npgettext as npfoo,
6+
gettext as bar,
7+
)
8+
9+
foo('bar')
10+
foo('bar', 'baz')
11+
12+
nfoo('cat', 'cats', 1)
13+
nfoo('dog', 'dogs')
14+
15+
pfoo('context', 'bar')
16+
17+
npfoo('context', 'cat', 'cats', 1)
18+
19+
# This is an unknown keyword and should be ignored
20+
bar('baz')
21+
22+
# 'nfoo' requires at least 2 arguments
23+
nfoo('dog')
24+
25+
# 'pfoo' requires at least 2 arguments
26+
pfoo('context')
27+
28+
# 'npfoo' requires at least 3 arguments
29+
npfoo('context')
30+
npfoo('context', 'cat')

‎Lib/test/test_tools/test_i18n.py‎

Lines changed: 63 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from pathlib import Path
99

1010
from test.support.script_helper import assert_python_ok
11-
from test.test_tools import skip_if_missing, toolsdir
11+
from test.test_tools import imports_under_tool, skip_if_missing, toolsdir
1212
from test.support.os_helper import temp_cwd, temp_dir
1313

1414

@@ -17,6 +17,10 @@
1717
DATA_DIR = Path(__file__).resolve().parent / 'i18n_data'
1818

1919

20+
with imports_under_tool("i18n"):
21+
from pygettext import parse_spec
22+
23+
2024
def normalize_POT_file(pot):
2125
"""Normalize the POT creation timestamp, charset and
2226
file locations to make the POT file easier to compare.
@@ -377,16 +381,8 @@ class _(object):
377381

378382
def test_pygettext_output(self):
379383
"""Test that the pygettext output exactly matches snapshots."""
380-
for input_file in DATA_DIR.glob('*.py'):
381-
output_file = input_file.with_suffix('.pot')
382-
with self.subTest(input_file=f'i18n_data/{input_file}'):
383-
contents = input_file.read_text(encoding='utf-8')
384-
with temp_cwd(None):
385-
Path(input_file.name).write_text(contents)
386-
assert_python_ok('-Xutf8', self.script, '--docstrings',
387-
'--add-comments=i18n:', input_file.name)
388-
output = Path('messages.pot').read_text(encoding='utf-8')
389-
384+
for input_file, output_file, output in extract_from_snapshots():
385+
with self.subTest(input_file=input_file):
390386
expected = output_file.read_text(encoding='utf-8')
391387
self.assert_POT_equal(expected, output)
392388

@@ -485,17 +481,67 @@ def test_comments_not_extracted_without_tags(self):
485481
'''), raw=True)
486482
self.assertNotIn('#.', data)
487483

488-
489-
def update_POT_snapshots():
490-
for input_file in DATA_DIR.glob('*.py'):
484+
def test_parse_keyword_spec(self):
485+
valid = (
486+
('foo', ('foo', {0: 'msgid'})),
487+
('foo:1', ('foo', {0: 'msgid'})),
488+
('foo:1,2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
489+
('foo:1, 2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
490+
('foo:1,2c', ('foo', {0: 'msgid', 1: 'msgctxt'})),
491+
('foo:2c,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
492+
('foo:2c ,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
493+
('foo:1,2,3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
494+
('foo:1, 2, 3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
495+
('foo:3c,1,2', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
496+
)
497+
for spec, expected in valid:
498+
with self.subTest(spec=spec):
499+
self.assertEqual(parse_spec(spec), expected)
500+
501+
invalid = (
502+
('foo:', "Invalid keyword spec 'foo:': missing argument positions"),
503+
('foo:bar', "Invalid keyword spec 'foo:bar': position is not an integer"),
504+
('foo:0', "Invalid keyword spec 'foo:0': argument positions must be strictly positive"),
505+
('foo:-2', "Invalid keyword spec 'foo:-2': argument positions must be strictly positive"),
506+
('foo:1,1', "Invalid keyword spec 'foo:1,1': duplicate positions"),
507+
('foo:1,2,1', "Invalid keyword spec 'foo:1,2,1': duplicate positions"),
508+
('foo:1c,2,1c', "Invalid keyword spec 'foo:1c,2,1c': duplicate positions"),
509+
('foo:1c,2,3c', "Invalid keyword spec 'foo:1c,2,3c': msgctxt can only appear once"),
510+
('foo:1,2,3', "Invalid keyword spec 'foo:1,2,3': too many positions"),
511+
('foo:1c', "Invalid keyword spec 'foo:1c': msgctxt cannot appear without msgid"),
512+
)
513+
for spec, message in invalid:
514+
with self.subTest(spec=spec):
515+
with self.assertRaises(ValueError) as cm:
516+
parse_spec(spec)
517+
self.assertEqual(str(cm.exception), message)
518+
519+
520+
def extract_from_snapshots():
521+
snapshots = {
522+
'messages.py': (),
523+
'fileloc.py': ('--docstrings',),
524+
'docstrings.py': ('--docstrings',),
525+
'comments.py': ('--add-comments=i18n:',),
526+
'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
527+
'--keyword=pfoo:1c,2',
528+
'--keyword=npfoo:1c,2,3'),
529+
}
530+
531+
for filename, args in snapshots.items():
532+
input_file = DATA_DIR / filename
491533
output_file = input_file.with_suffix('.pot')
492534
contents = input_file.read_bytes()
493535
with temp_cwd(None):
494536
Path(input_file.name).write_bytes(contents)
495-
assert_python_ok('-Xutf8', Test_pygettext.script, '--docstrings',
496-
'--add-comments=i18n:', input_file.name)
497-
output = Path('messages.pot').read_text(encoding='utf-8')
537+
assert_python_ok('-Xutf8', Test_pygettext.script, *args,
538+
input_file.name)
539+
yield (input_file, output_file,
540+
Path('messages.pot').read_text(encoding='utf-8'))
498541

542+
543+
def update_POT_snapshots():
544+
for _, output_file, output in extract_from_snapshots():
499545
output = normalize_POT_file(output)
500546
output_file.write_text(output, encoding='utf-8')
501547

‎Misc/NEWS.d/next/Tools-Demos/2025-02-22-18-08-35.gh-issue-130453.njRXG8.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Extend support for specifying custom keywords in :program:`pygettext`.

‎Tools/i18n/pygettext.py‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,88 @@ def getFilesForName(name):
294294
}
295295

296296

297+
def parse_spec(spec):
298+
"""Parse a keyword spec string into a dictionary.
299+
300+
The keyword spec format defines the name of the gettext function and the
301+
positions of the arguments that correspond to msgid, msgid_plural, and
302+
msgctxt. The format is as follows:
303+
304+
name - the name of the gettext function, assumed to
305+
have a single argument that is the msgid.
306+
name:pos1 - the name of the gettext function and the position
307+
of the msgid argument.
308+
name:pos1,pos2 - the name of the gettext function and the positions
309+
of the msgid and msgid_plural arguments.
310+
name:pos1,pos2c - the name of the gettext function and the positions
311+
of the msgid and msgctxt arguments.
312+
name:pos1,pos2,pos3c - the name of the gettext function and the
313+
positions of the msgid, msgid_plural, and
314+
msgctxt arguments.
315+
316+
As an example, the spec 'foo:1,2,3c' means that the function foo has three
317+
arguments, the first one is the msgid, the second one is the msgid_plural,
318+
and the third one is the msgctxt. The positions are 1-based.
319+
320+
The msgctxt argument can appear in any position, but it can only appear
321+
once. For example, the keyword specs 'foo:3c,1,2' and 'foo:1,2,3c' are
322+
equivalent.
323+
324+
See https://www.gnu.org/software/gettext/manual/gettext.html
325+
for more information.
326+
"""
327+
parts = spec.strip().split(':', 1)
328+
if len(parts) == 1:
329+
name = parts[0]
330+
return name, {0: 'msgid'}
331+
332+
name, args = parts
333+
if not args:
334+
raise ValueError(f'Invalid keyword spec {spec!r}: '
335+
'missing argument positions')
336+
337+
result = {}
338+
for arg in args.split(','):
339+
arg = arg.strip()
340+
is_context = False
341+
if arg.endswith('c'):
342+
is_context = True
343+
arg = arg[:-1]
344+
345+
try:
346+
pos = int(arg) - 1
347+
except ValueError as e:
348+
raise ValueError(f'Invalid keyword spec {spec!r}: '
349+
'position is not an integer') from e
350+
351+
if pos < 0:
352+
raise ValueError(f'Invalid keyword spec {spec!r}: '
353+
'argument positions must be strictly positive')
354+
355+
if pos in result.values():
356+
raise ValueError(f'Invalid keyword spec {spec!r}: '
357+
'duplicate positions')
358+
359+
if is_context:
360+
if 'msgctxt' in result:
361+
raise ValueError(f'Invalid keyword spec {spec!r}: '
362+
'msgctxt can only appear once')
363+
result['msgctxt'] = pos
364+
elif 'msgid' not in result:
365+
result['msgid'] = pos
366+
elif 'msgid_plural' not in result:
367+
result['msgid_plural'] = pos
368+
else:
369+
raise ValueError(f'Invalid keyword spec {spec!r}: '
370+
'too many positions')
371+
372+
if 'msgid' not in result and 'msgctxt' in result:
373+
raise ValueError(f'Invalid keyword spec {spec!r}: '
374+
'msgctxt cannot appear without msgid')
375+
376+
return name, {v: k for k, v in result.items()}
377+
378+
297379
@dataclass(frozen=True)
298380
class Location:
299381
filename: str
@@ -646,7 +728,11 @@ class Options:
646728
make_escapes(not options.escape)
647729

648730
# calculate all keywords
649-
options.keywords = {kw: {0: 'msgid'} for kw in options.keywords}
731+
try:
732+
options.keywords = dict(parse_spec(spec) for spec in options.keywords)
733+
except ValueError as e:
734+
print(e, file=sys.stderr)
735+
sys.exit(1)
650736
if not no_default_keywords:
651737
options.keywords |= DEFAULTKEYWORDS
652738

0 commit comments

Comments
 (0)