Skip to content

Commit ec5569b

Browse files
miss-islingtonTobotimus
miss-islington
and
Tobotimus
authored
bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745)
(cherry picked from commit eee72d4) Co-authored-by: Tobotimus <Tobotimus@users.noreply.github.com>
1 parent e650fd3 commit ec5569b

3 files changed

Lines changed: 102 additions & 4 deletions

File tree

‎Lib/test/test_tools/test_i18n.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import os
44
import unittest
5+
import textwrap
56

67
from test.support.script_helper import assert_python_ok
78
from test.test_tools import skip_if_missing, toolsdir
@@ -27,6 +28,41 @@ def get_header(self, data):
2728
headers[key] = val.strip()
2829
return headers
2930

31+
def get_msgids(self, data):
32+
""" utility: return all msgids in .po file as a list of strings """
33+
msgids = []
34+
reading_msgid = False
35+
cur_msgid = []
36+
for line in data.split('\n'):
37+
if reading_msgid:
38+
if line.startswith('"'):
39+
cur_msgid.append(line.strip('"'))
40+
else:
41+
msgids.append('\n'.join(cur_msgid))
42+
cur_msgid = []
43+
reading_msgid = False
44+
continue
45+
if line.startswith('msgid '):
46+
line = line[len('msgid '):]
47+
cur_msgid.append(line.strip('"'))
48+
reading_msgid = True
49+
else:
50+
if reading_msgid:
51+
msgids.append('\n'.join(cur_msgid))
52+
53+
return msgids
54+
55+
def extract_docstrings_from_str(self, module_content):
56+
""" utility: return all msgids extracted from module_content """
57+
filename = 'test_docstrings.py'
58+
with temp_cwd(None) as cwd:
59+
with open(filename, 'w') as fp:
60+
fp.write(module_content)
61+
assert_python_ok(self.script, '-D', filename)
62+
with open('messages.pot') as fp:
63+
data = fp.read()
64+
return self.get_msgids(data)
65+
3066
def test_header(self):
3167
"""Make sure the required fields are in the header, according to:
3268
http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry
@@ -70,3 +106,55 @@ def test_POT_Creation_Date(self):
70106

71107
# This will raise if the date format does not exactly match.
72108
datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z')
109+
110+
def test_funcdocstring_annotated_args(self):
111+
""" Test docstrings for functions with annotated args """
112+
msgids = self.extract_docstrings_from_str(textwrap.dedent('''\
113+
def foo(bar: str):
114+
"""doc"""
115+
'''))
116+
self.assertIn('doc', msgids)
117+
118+
def test_funcdocstring_annotated_return(self):
119+
""" Test docstrings for functions with annotated return type """
120+
msgids = self.extract_docstrings_from_str(textwrap.dedent('''\
121+
def foo(bar) -> str:
122+
"""doc"""
123+
'''))
124+
self.assertIn('doc', msgids)
125+
126+
def test_funcdocstring_defvalue_args(self):
127+
""" Test docstring for functions with default arg values """
128+
msgids = self.extract_docstrings_from_str(textwrap.dedent('''\
129+
def foo(bar=()):
130+
"""doc"""
131+
'''))
132+
self.assertIn('doc', msgids)
133+
134+
def test_funcdocstring_multiple_funcs(self):
135+
""" Test docstring extraction for multiple functions combining
136+
annotated args, annotated return types and default arg values
137+
"""
138+
msgids = self.extract_docstrings_from_str(textwrap.dedent('''\
139+
def foo1(bar: tuple=()) -> str:
140+
"""doc1"""
141+
142+
def foo2(bar: List[1:2]) -> (lambda x: x):
143+
"""doc2"""
144+
145+
def foo3(bar: 'func'=lambda x: x) -> {1: 2}:
146+
"""doc3"""
147+
'''))
148+
self.assertIn('doc1', msgids)
149+
self.assertIn('doc2', msgids)
150+
self.assertIn('doc3', msgids)
151+
152+
def test_classdocstring_early_colon(self):
153+
""" Test docstring extraction for a class with colons occuring within
154+
the parentheses.
155+
"""
156+
msgids = self.extract_docstrings_from_str(textwrap.dedent('''\
157+
class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)):
158+
"""doc"""
159+
'''))
160+
self.assertIn('doc', msgids)

‎Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Fix pygettext not extracting docstrings for functions with type annotated
2+
arguments.
3+
Patch by Toby Harradine.

‎Tools/i18n/pygettext.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -320,6 +320,7 @@ def __init__(self, options):
320320
self.__lineno = -1
321321
self.__freshmodule = 1
322322
self.__curfile = None
323+
self.__enclosurecount = 0
323324

324325
def __call__(self, ttype, tstring, stup, etup, line):
325326
# dispatch
@@ -340,17 +341,23 @@ def __waiting(self, ttype, tstring, lineno):
340341
elif ttype not in (tokenize.COMMENT, tokenize.NL):
341342
self.__freshmodule = 0
342343
return
343-
# class docstring?
344+
# class or func/method docstring?
344345
if ttype == tokenize.NAME and tstring in ('class', 'def'):
345346
self.__state = self.__suiteseen
346347
return
347348
if ttype == tokenize.NAME and tstring in opts.keywords:
348349
self.__state = self.__keywordseen
349350

350351
def __suiteseen(self, ttype, tstring, lineno):
351-
# ignore anything until we see the colon
352-
if ttype == tokenize.OP and tstring == ':':
353-
self.__state = self.__suitedocstring
352+
# skip over any enclosure pairs until we see the colon
353+
if ttype == tokenize.OP:
354+
if tstring == ':' and self.__enclosurecount == 0:
355+
# we see a colon and we're not in an enclosure: end of def
356+
self.__state = self.__suitedocstring
357+
elif tstring in '([{':
358+
self.__enclosurecount += 1
359+
elif tstring in ')]}':
360+
self.__enclosurecount -= 1
354361

355362
def __suitedocstring(self, ttype, tstring, lineno):
356363
# ignore any intervening noise

0 commit comments

Comments
 (0)