Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 51 additions & 9 deletions Lib/email/_header_value_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -2911,12 +2911,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
last_ew = None # if there is an encoded word in the last line of lines,
# points to the encoded word's first character
last_charset = None
last_comment_end = None # (line count, column) of the last emitted comment
wrap_as_ew_blocked = 0
want_encoding = False # This is set to True if we need to encode this part
end_ew_not_allowed = Terminal('', 'wrap_as_ew_blocked')
parts = list(parse_tree)
while parts:
part = parts.pop(0)
if part.token_type == 'cfws':
# Visit comments themselves before changing encoded-word state.
parts = list(part) + parts
continue
if part.token_type == 'comment':
# Encoded words must not combine across comment boundaries.
if (last_word_is_ew and parse_tree.token_type != 'comment' and
not lines[-1].endswith((' ', '\t'))):
lines[-1] += ' '
last_ew = None
last_word_is_ew = False
if part is end_ew_not_allowed:
wrap_as_ew_blocked -= 1
continue
Expand Down Expand Up @@ -2946,6 +2958,24 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
charset = 'utf-8'
want_encoding = True

if (part.token_type == 'comment' and not wrap_as_ew_blocked and
(want_encoding or
(parse_tree.token_type == 'comment' and
len(tstr) > maxlen - len(lines[-1])))):
# Comment.__str__ supplies the parentheses. Keep them outside
# encoded words, including those in nested comments.
if len(lines[-1]) >= maxlen:
newline = _steal_trailing_WSP_if_exists(lines)
lines.append(newline or ' ')
lines[-1] += '('
_refold_with_ew(part, lines, maxlen, encoding, policy=policy)
if len(lines[-1]) >= maxlen:
lines.append(' ')
lines[-1] += ')'
last_comment_end = (len(lines), len(lines[-1]))
want_encoding = False
continue

if want_encoding and not wrap_as_ew_blocked:
if any(
not x.as_ew_allowed for x in part
Expand All @@ -2955,7 +2985,10 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
last_ew = None
if part.syntactic_break:
encoded_part = part.fold(policy=policy)[:-len(policy.linesep)]
if policy.linesep not in encoded_part:
if (policy.linesep not in encoded_part and
(not part.contains_token_type('comment') or
len(encoded_part) <= maxlen - len(lines[-1]) or
len(encoded_part) + 1 <= maxlen)):
# It fits on a single line
if len(encoded_part) > maxlen - len(lines[-1]):
# But not on this one, so start a new one.
Expand All @@ -2977,6 +3010,9 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
elif part.as_ew_allowed:
# It's a terminal, wrap it as an encoded word, possibly
# combining it with previously encoded words if allowed.
if (parse_tree.token_type != 'comment' and
last_comment_end == (len(lines), len(lines[-1]))):
lines[-1] += ' '
if (last_ew is not None and
charset != last_charset and
(last_charset == 'unknown-8bit' or
Expand Down Expand Up @@ -3004,6 +3040,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):

if len(tstr) <= maxlen - len(lines[-1]):
lines[-1] += tstr
if part.token_type == 'comment':
last_comment_end = (len(lines), len(lines[-1]))
last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP))
continue

Expand All @@ -3015,6 +3053,8 @@ def _refold_with_ew(parse_tree, lines, maxlen, encoding, *, policy):
newline = _steal_trailing_WSP_if_exists(lines)
if newline or part.startswith_fws():
lines.append(newline + tstr)
if part.token_type == 'comment':
last_comment_end = (len(lines), len(lines[-1]))
last_word_is_ew = (last_word_is_ew
and not bool(lines[-1].strip(_WSP)))
last_ew = None
Expand Down Expand Up @@ -3116,21 +3156,23 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset,
while to_encode:
remaining_space = maxlen - len(lines[-1])
text_space = remaining_space - chrome_len
if text_space <= 0:
newline = _steal_trailing_WSP_if_exists(lines)
lines.append(newline or ' ')
new_last_ew = len(lines[-1])
continue

to_encode_word = to_encode[:text_space]
to_encode_word = to_encode[:max(1, text_space)]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
while excess > 0:
while excess > 0 and len(to_encode_word) > 1:
# Since the chunk to encode is guaranteed to fit into less than 100 characters,
# shrinking it by one at a time shouldn't take long.
to_encode_word = to_encode_word[:-1]
encoded_word = _ew.encode(to_encode_word, charset=encode_as)
excess = len(encoded_word) - remaining_space
if excess > 0:
if not lines[-1].strip(_WSP):
raise errors.HeaderParseError(
"max_line_length is too small to fit an encoded word")
newline = _steal_trailing_WSP_if_exists(lines)
lines.append(newline or ' ')
new_last_ew = len(lines[-1])
continue
lines[-1] += encoded_word
to_encode = to_encode[len(to_encode_word):]

Expand Down
8 changes: 5 additions & 3 deletions Lib/test/test_email/test__header_value_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -3409,9 +3409,11 @@ def test_address_list_with_long_unwrapable_comment(self):
with self.subTest(to=to):
self._test(parser.get_address_list(to)[0], folded, policy=policy)

# XXX Need tests with comments on various sides of a unicode token,
# and with unicode tokens in the comments. Spaces inside the quotes
# currently don't do the right thing.
def test_address_list_with_unicode_comment(self):
# Regression for gh-162125: parentheses are syntax, not part of
# the encoded word containing the comment text.
self._test(parser.get_address_list('Ana (ñ) <a@example.com>')[0],
'Ana (=?utf-8?q?=C3=B1?=) <a@example.com>\n')

def test_split_at_whitespace_after_header_before_long_token(self):
body = parser.get_unstructured(' ' + 'x'*77)
Expand Down
84 changes: 84 additions & 0 deletions Lib/test/test_email/test_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import sys
from email import message_from_string, message_from_bytes
from email.message import EmailMessage
from email.header import decode_header, make_header
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.generator import Generator, BytesGenerator
Expand Down Expand Up @@ -296,6 +297,89 @@ def test_keep_long_encoded_newlines(self):
g.flatten(msg)
self.assertEqual(s.getvalue(), self.typ(expected))

def test_non_ascii_address_comments(self):
cases = [
# (value, max_line_length, number of comment delimiters)
('Ana (ñ) <a@example.com>', 20, 1),
('Ana (ñ) <a@example.com>', 27, 1),
('Ana (((é))) <a@example.com>', 20, 3),
]
for value, maxlen, comment_count in cases:
p = self.policy.clone(max_line_length=maxlen)
with self.subTest(value=value, maxlen=maxlen):
msg = EmailMessage(policy=p)
msg['To'] = value
s = self.ioclass()
self.genclass(s, policy=p).flatten(msg)
folded = s.getvalue()
self.assertEqual(folded.count(self.typ('(')), comment_count)
self.assertEqual(folded.count(self.typ(')')), comment_count)
self.assertNotIn(self.typ('=?utf-8?q??='), folded)
for line in folded.splitlines():
self.assertLessEqual(len(line), maxlen)
reparsed = self.msgmaker(folded, policy=p)
self.assertEqual(reparsed['To'].defects, ())
self.assertEqual(reparsed['To'].addresses,
(Address('Ana', 'a', 'example.com'),))

def test_generated_non_ascii_address_comments(self):
for case in range(32):
words = [''.join(random.choices('abcdeéñΩЖ日本',
k=random.randint(1, 8)))
for _ in range(random.randint(1, 3))]
comment = random.choice('éñΩ日') + ''.join(
random.choices('abcéñΩ日()\\ ', k=random.randrange(24)))
comment = ' '.join(comment.split())
escaped = (comment.replace('\\', '\\\\')
.replace('(', r'\(').replace(')', r'\)'))
position = random.randrange(len(words) + 1)
left = ' '.join(words[:position])
right = ' '.join(words[position:])
gap = random.choice(('', ' ', '\t'))
value = f'{left}{gap}({escaped}){gap}{right} <a@example.com>'
maxlen = random.choice((26, 40, 78))
p = self.policy.clone(max_line_length=maxlen)

with self.subTest(case=case, value=value, maxlen=maxlen):
msg = EmailMessage(policy=p)
msg['To'] = value
s = self.ioclass()
self.genclass(s, policy=p).flatten(msg)
folded = s.getvalue()
self.assertNotRegex(folded, self.typ(r'=\?[^?]+\?[bqBQ]\?\?='))
for line in folded.splitlines():
self.assertLessEqual(len(line), maxlen)

header = self.msgmaker(folded, policy=p)['To']
self.assertEqual(header.defects, ())
self.assertEqual(header.addresses, (
Address(' '.join(words), 'a', 'example.com'),))
comments = header._parse_tree.comments
self.assertEqual(len(comments), 1)
decoded = str(make_header(decode_header(comments[0])))
self.assertEqual(' '.join(decoded.split()), comment)

def test_encoded_word_after_literal_parenthesis(self):
msg = EmailMessage(policy=self.policy)
msg['Subject'] = ')=?utf-8?q?=C3=A9?='
s = self.ioclass()
self.genclass(s, policy=self.policy).flatten(msg)
reparsed = self.msgmaker(s.getvalue())
self.assertEqual(str(reparsed['Subject']), ')é')

def test_non_ascii_content_disposition_comment(self):
msg = EmailMessage(policy=self.policy)
msg['Content-Disposition'] = 'attachment (é)'
s = self.ioclass()
self.genclass(s, policy=self.policy).flatten(msg)
header = self.msgmaker(s.getvalue(), policy=self.policy)[
'Content-Disposition']
self.assertEqual(header.defects, ())
self.assertEqual(header.content_disposition, 'attachment')
comments = header._parse_tree.comments
self.assertEqual(len(comments), 1)
self.assertEqual(str(make_header(decode_header(comments[0]))), 'é')

def test_non_ascii_addr_spec_raises(self):
# non-ascii is not permitted in any part of an addr-spec. If the
# programmer generated it, it's an error. (See also
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Preserve parentheses around non-ASCII comments when folding :mod:`email`
headers with encoded words.
Loading