File bpo24211-RFC6532-supp-email.patch of Package python3.36643
commit 224ef3ec3b0758956789c4c98e6cc93704304419
Author: R David Murray <rdmurray@bitdance.com>
Date: Sun May 17 11:29:21 2015 -0400
#24211: Add RFC6532 support to the email library.
This could use more edge case tests, but the basic functionality is tested.
(Note that this changeset does not add tailored support for the RFC 6532
message/global MIME type, but the email package generic facilities will handle
it.)
Reviewed by Maciej Szulik.
---
Doc/library/email.policy.rst | 8 ++++++++
Lib/email/_header_value_parser.py | 11 +++++++----
Lib/email/policy.py | 15 ++++++++++++++-
Lib/test/test_email/test_generator.py | 22 ++++++++++++++++++++++
Lib/test/test_email/test_policy.py | 4 ++++
Misc/NEWS | 3 +++
6 files changed, 58 insertions(+), 5 deletions(-)
--- a/Doc/library/email.policy.rst
+++ b/Doc/library/email.policy.rst
@@ -378,6 +378,14 @@ added matters. To illustrate::
In addition to the settable attributes listed above that apply to all
policies, this policy adds the following additional attributes:
+ .. attribute:: utf8
+
+ If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in
+ headers by encoding them as "encoded words". If ``True``, follow
+ :rfc:`6532` and use ``utf-8`` encoding for headers. Messages
+ formatted in this way may be passed to SMTP servers that support
+ the ``SMTPUTF8`` extension (:rfc:`6531`).
+
.. attribute:: refold_source
If the value for a header in the ``Message`` object originated from a
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -320,17 +320,18 @@ class TokenList(list):
return ''.join(res)
def _fold(self, folded):
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
tlen = len(tstr)
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
charset = 'unknown-8bit'
else:
- # XXX: this should be a policy setting
+ # XXX: this should be a policy setting when utf8 is False.
charset = 'utf-8'
tstr = part.cte_encode(charset, folded.policy)
tlen = len(tstr)
@@ -394,11 +395,12 @@ class UnstructuredTokenList(TokenList):
def _fold(self, folded):
last_ew = None
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
is_ew = False
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
@@ -475,12 +477,13 @@ class Phrase(TokenList):
# comment that becomes a barrier across which we can't compose encoded
# words.
last_ew = None
+ encoding = 'utf-8' if folded.policy.utf8 else 'ascii'
for part in self.parts:
tstr = str(part)
tlen = len(tstr)
has_ew = False
try:
- str(part).encode('us-ascii')
+ str(part).encode(encoding)
except UnicodeEncodeError:
if any(isinstance(x, errors.UndecodableBytesDefect)
for x in part.all_defects):
--- a/Lib/email/policy.py
+++ b/Lib/email/policy.py
@@ -35,6 +35,13 @@ class EmailPolicy(Policy):
In addition to the settable attributes listed above that apply to
all Policies, this policy adds the following additional attributes:
+ utf8 -- if False (the default) message headers will be
+ serialized as ASCII, using encoded words to encode
+ any non-ASCII characters in the source strings. If
+ True, the message headers will be serialized using
+ utf8 and will not contain encoded words (see RFC
+ 6532 for more on this serialization format).
+
refold_source -- if the value for a header in the Message object
came from the parsing of some source, this attribute
indicates whether or not a generator should refold
@@ -72,6 +79,7 @@ class EmailPolicy(Policy):
"""
+ utf8 = False
refold_source = 'long'
header_factory = HeaderRegistry()
content_manager = raw_data_manager
@@ -175,9 +183,13 @@ class EmailPolicy(Policy):
refold_header setting, since there is no way to know whether the binary
data consists of single byte characters or multibyte characters.
+ If utf8 is true, headers are encoded to utf8, otherwise to ascii with
+ non-ASCII unicode rendered as encoded words.
+
"""
folded = self._fold(name, value, refold_binary=self.cte_type=='7bit')
- return folded.encode('ascii', 'surrogateescape')
+ charset = 'utf8' if self.utf8 else 'ascii'
+ return folded.encode(charset, 'surrogateescape')
def _fold(self, name, value, refold_binary=False):
if hasattr(value, 'name'):
@@ -199,3 +211,4 @@ del default.header_factory
strict = default.clone(raise_on_defect=True)
SMTP = default.clone(linesep='\r\n')
HTTP = default.clone(linesep='\r\n', max_line_length=None)
+SMTPUTF8 = SMTP.clone(utf8=True)
--- a/Lib/test/test_email/test_generator.py
+++ b/Lib/test/test_email/test_generator.py
@@ -2,6 +2,7 @@ import io
import textwrap
import unittest
from email import message_from_string, message_from_bytes
+from email.message import EmailMessage
from email.generator import Generator, BytesGenerator
from email import policy
from test.test_email import TestEmailBase, parameterize
@@ -194,6 +195,27 @@ class TestBytesGenerator(TestGeneratorBa
g.flatten(msg)
self.assertEqual(s.getvalue(), expected)
+ def test_smtputf8_policy(self):
+ msg = EmailMessage()
+ msg['From'] = "Páolo <főo@bar.com>"
+ msg['To'] = 'Dinsdale'
+ msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
+ msg.set_content("oh là là, know what I mean, know what I mean?")
+ expected = textwrap.dedent("""\
+ From: Páolo <főo@bar.com>
+ To: Dinsdale
+ Subject: Nudge nudge, wink, wink \u1F609
+ Content-Type: text/plain; charset="utf-8"
+ Content-Transfer-Encoding: 8bit
+ MIME-Version: 1.0
+
+ oh là là, know what I mean, know what I mean?
+ """).encode('utf-8').replace(b'\n', b'\r\n')
+ s = io.BytesIO()
+ g = BytesGenerator(s, policy=policy.SMTPUTF8)
+ g.flatten(msg)
+ self.assertEqual(s.getvalue(), expected)
+
if __name__ == '__main__':
unittest.main()
--- a/Lib/test/test_email/test_policy.py
+++ b/Lib/test/test_email/test_policy.py
@@ -27,6 +27,7 @@ class PolicyAPITests(unittest.TestCase):
# If any of these defaults change, the docs must be updated.
policy_defaults = compat32_defaults.copy()
policy_defaults.update({
+ 'utf8': False,
'raise_on_defect': False,
'header_factory': email.policy.EmailPolicy.header_factory,
'refold_source': 'long',
@@ -42,6 +43,9 @@ class PolicyAPITests(unittest.TestCase):
email.policy.default: make_defaults(policy_defaults, {}),
email.policy.SMTP: make_defaults(policy_defaults,
{'linesep': '\r\n'}),
+ email.policy.SMTPUTF8: make_defaults(policy_defaults,
+ {'linesep': '\r\n',
+ 'utf8': True}),
email.policy.HTTP: make_defaults(policy_defaults,
{'linesep': '\r\n',
'max_line_length': None}),
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -22,6 +22,9 @@ Security
Library
-------
+- Issue #24211: The email library now supports RFC 6532: it can generate
+ headers using utf-8 instead of encoded words.
+
- bpo-35121: Don't set cookie for a request when the request path is a
prefix match of the cookie's path attribute but doesn't end with "/".
Patch by Karthikeyan Singaravelan.
