Mercurial > p > roundup > code
changeset 5493:725266c03eab
updated mailgw to no longer use mimetools based on jerrykan's patch
| author | Christof Meerwald <cmeerw@cmeerw.org> |
|---|---|
| date | Sun, 12 Aug 2018 16:15:10 +0100 |
| parents | 6b0c542642be |
| children | b7fa56ced601 |
| files | roundup/cgi/client.py roundup/mailer.py roundup/mailgw.py roundup/roundupdb.py test/db_test_base.py test/test_mailgw.py test/test_mailgw_roundupmessage.py test/test_multipart.py |
| diffstat | 8 files changed, 740 insertions(+), 400 deletions(-) [+] |
line wrap: on
line diff
--- a/roundup/cgi/client.py Sun Aug 12 16:05:42 2018 +0100 +++ b/roundup/cgi/client.py Sun Aug 12 16:15:10 2018 +0100 @@ -32,7 +32,7 @@ FormError, NotFound, NotModified, Redirect, SendFile, SendStaticFile, DetectorError, SeriousError) from roundup.cgi.form_parser import FormParser -from roundup.mailer import Mailer, MessageSendError, encode_quopri +from roundup.mailer import Mailer, MessageSendError from roundup.cgi import accept_language from roundup import xmlrpc @@ -1538,12 +1538,11 @@ to = [self.mailer.config.ADMIN_EMAIL] message = MIMEMultipart('alternative') self.mailer.set_message_attributes(message, to, subject) - part = MIMEBase('text', 'html') - part.set_charset('utf-8') - part.set_payload(html) - encode_quopri(part) + part = self.mail.get_text_message('utf-8', 'html') + part.set_payload(html, part.get_charset()) message.attach(part) - part = MIMEText(txt) + part = self.mail.get_text_message() + part.set_payload(text, part.get_charset()) message.attach(part) self.mailer.smtp_send(to, message.as_string())
--- a/roundup/mailer.py Sun Aug 12 16:05:42 2018 +0100 +++ b/roundup/mailer.py Sun Aug 12 16:15:10 2018 +0100 @@ -7,12 +7,15 @@ from roundup import __version__ from roundup.date import get_timezone, Date +from email import charset from email.utils import formatdate, formataddr, specialsre, escapesre +from email.charset import Charset from email.message import Message from email.header import Header from email.mime.base import MIMEBase from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart from roundup.anypy import email_ from roundup.anypy.strings import b2s, s2b, s2u @@ -26,13 +29,6 @@ class MessageSendError(RuntimeError): pass -def encode_quopri(msg): - orig = s2b(msg.get_payload()) - encdata = quopri.encodestring(orig) - msg.set_payload(b2s(encdata)) - del msg['Content-Transfer-Encoding'] - msg['Content-Transfer-Encoding'] = 'quoted-printable' - def nice_sender_header(name, address, charset): # construct an address header so it's as human-readable as possible # even in the presence of a non-ASCII name part @@ -115,16 +111,23 @@ # finally, an aid to debugging problems message['X-Roundup-Version'] = __version__ + def get_text_message(self, _charset='utf-8', _subtype='plain'): + message = MIMENonMultipart('text', _subtype) + cs = Charset(_charset) + if cs.body_encoding == charset.BASE64: + cs.body_encoding = charset.QP + message.set_charset(cs) + del message['Content-Transfer-Encoding'] + return message + def get_standard_message(self, multipart=False): '''Form a standard email message from Roundup. Returns a Message object. ''' - charset = getattr(self.config, 'EMAIL_CHARSET', 'utf-8') if multipart: message = MIMEMultipart() else: - message = MIMEText("") - message.set_charset(charset) + message = self.get_text_message(getattr(self.config, 'EMAIL_CHARSET', 'utf-8')) return message @@ -132,7 +135,7 @@ """Send a standard message. Arguments: - - to: a list of addresses usable by rfc822.parseaddr(). + - to: a list of addresses usable by email.utils.parseaddr(). - subject: the subject as a string. - content: the body of the message as a string. - author: the sender as a (name, address) tuple @@ -141,8 +144,7 @@ """ message = self.get_standard_message() self.set_message_attributes(message, to, subject, author) - message.set_payload(content) - encode_quopri(message) + message.set_payload(s2u(content)) self.smtp_send(to, message.as_string()) def bounce_message(self, bounced_message, to, error, @@ -150,8 +152,8 @@ """Bounce a message, attaching the failed submission. Arguments: - - bounced_message: an RFC822 Message object. - - to: a list of addresses usable by rfc822.parseaddr(). Might be + - bounced_message: an mailgw.RoundupMessage object. + - to: a list of addresses usable by email.utils.parseaddr(). Might be extended or overridden according to the config ERROR_MESSAGES_TO setting. - error: the reason of failure as a string. @@ -185,18 +187,7 @@ message.attach(part) # attach the original message to the returned message - body = [] - for header in bounced_message.headers: - body.append(header) - try: - bounced_message.rewindbody() - except IOError as errmessage: - body.append("*** couldn't include message body: %s ***" % - errmessage) - else: - body.append('\n') - body.append(bounced_message.fp.read()) - part = MIMEText(''.join(body)) + part = MIMEText(bounced_message.flatten()) message.attach(part) self.logger.debug("bounce_message: to=%s, crypt_to=%s", to, crypt_to)
--- a/roundup/mailgw.py Sun Aug 12 16:05:42 2018 +0100 +++ b/roundup/mailgw.py Sun Aug 12 16:15:10 2018 +0100 @@ -27,7 +27,7 @@ and given "file" class nodes that are linked to the "msg" node. . In a multipart/alternative message or part, we look for a text/plain subpart and ignore the other parts. - . A message/rfc822 is treated similar tomultipart/mixed (except for + . A message/rfc822 is treated similar to multipart/mixed (except for special handling of the first text part) if unpack_rfc822 is set in the mailgw config section. @@ -95,11 +95,13 @@ from __future__ import print_function __docformat__ = 'restructuredtext' -import base64, re, os, smtplib, socket, binascii, quopri +import base64, re, os, smtplib, socket, binascii import time, sys, logging import codecs import traceback +import email import email.utils +from email.generator import Generator from .anypy.email_ import decode_header from roundup.anypy.my_input import my_input @@ -108,7 +110,7 @@ from roundup.mailer import Mailer, MessageSendError from roundup.i18n import _ from roundup.hyperdb import iter_roles -from roundup.anypy.strings import b2s, StringIO +from roundup.anypy.strings import StringIO, b2s, u2s import roundup.anypy.random_ as random_ try: @@ -116,13 +118,6 @@ except ImportError: pyme = None -try: - import mimetools -except ImportError: - class mimetools: - class Message: - pass - SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '') class MailGWError(ValueError): @@ -143,11 +138,11 @@ """ A general class of message that we should ignore. """ pass class IgnoreBulk(IgnoreMessage): - """ This is email from a mailing list or from a vacation program. """ - pass + """ This is email from a mailing list or from a vacation program. """ + pass class IgnoreLoop(IgnoreMessage): - """ We've seen this message before... """ - pass + """ We've seen this message before... """ + pass def initialiseSecurity(security): ''' Create some Permissions and Roles on the security object @@ -159,26 +154,6 @@ description="User may use the email interface") security.addPermissionToRole('Admin', p) -def getparam(str, param): - ''' From the rfc822 "header" string, extract "param" if it appears. - ''' - if ';' not in str: - return None - str = str[str.index(';'):] - while str[:1] == ';': - str = str[1:] - if ';' in str: - # XXX Should parse quotes! - end = str.index(';') - else: - end = len(str) - f = str[:end] - if '=' in f: - i = f.index('=') - if f[:i].strip().lower() == param: - return email.utils.unquote(f[i+1:].strip()) - return None - def gpgh_key_getall(key, attr): ''' return list of given attribute for all uids in a key @@ -222,57 +197,9 @@ elif not may_be_unsigned: raise MailUsageError(_("Unsigned Message")) -class Message(mimetools.Message): - ''' subclass mimetools.Message so we can retrieve the parts of the - message... - ''' - def getpart(self): - ''' Get a single part of a multipart message and return it as a new - Message instance. - ''' - boundary = self.getparam('boundary') - mid, end = '--'+boundary, '--'+boundary+'--' - s = StringIO() - while 1: - line = self.fp.readline() - if not line: - break - if line.strip() in (mid, end): - # according to rfc 1431 the preceding line ending is part of - # the boundary so we need to strip that - length = s.tell() - s.seek(-2, 1) - lineending = s.read(2) - if lineending == '\r\n': - s.truncate(length - 2) - elif lineending[1] in ('\r', '\n'): - s.truncate(length - 1) - else: - raise ValueError('Unknown line ending in message.') - break - s.write(line) - if not s.getvalue().strip(): - return None - s.seek(0) - return Message(s) - - def getparts(self): - """Get all parts of this multipart message.""" - # skip over the intro to the first boundary - self.fp.seek(0) - self.getpart() - - # accumulate the other parts +class RoundupMessage(email.message.Message): + def _decode_header(self, hdr): parts = [] - while 1: - part = self.getpart() - if part is None: - break - parts.append(part) - return parts - - def _decode_header_to_utf8(self, hdr): - l = [] for part, encoding in decode_header(hdr): if encoding: part = part.decode(encoding) @@ -281,86 +208,41 @@ # or vice-versa we must preserve a space. Multiple adjacent # non-encoded parts should not occur. This is now # implemented in our patched decode_header method in anypy - l.append(part) - return ''.join([s.encode('utf-8') for s in l]) + parts.append(part) + + return ''.join([u2s(p) for p in parts]) - def getheader(self, name, default=None): - hdr = mimetools.Message.getheader(self, name, default) - # TODO are there any other False values possible? - # TODO if not hdr: return hdr - if hdr is None: - return None - if not hdr: - return '' - if hdr: - hdr = hdr.replace('\n','') # Inserted by rfc822.readheaders - return self._decode_header_to_utf8(hdr) + def flatten(self): + fp = StringIO() + generator = Generator(fp, mangle_from_=False) + generator.flatten(self) + return fp.getvalue() - def getaddrlist(self, name): - # overload to decode the name part of the address - l = [] - for (name, addr) in mimetools.Message.getaddrlist(self, name): - name = self._decode_header_to_utf8(name) - l.append((name, addr)) - return l + def get_header(self, header, default=None): + value = self.get(header, default) + + if value: + return self._decode_header(value.replace('\n', '')) + + return value - def getname(self): - """Find an appropriate name for this message.""" - name = None - if self.gettype() == 'message/rfc822': - # handle message/rfc822 specially - the name should be - # the subject of the actual e-mail embedded here - # we add a '.eml' extension like other email software does it - self.fp.seek(0) - s = StringIO(self.getbody()) - name = Message(s).getheader('subject') - if name: - name = name + '.eml' - if not name: - # try name on Content-Type - name = self.getparam('name') - if not name: - disp = self.getheader('content-disposition', None) - if disp: - name = getparam(disp, 'filename') + def get_address_list(self, header): + addresses = [] - if name: - return name.strip() + for name, addr in email.utils.getaddresses(self.get_all(header, [])): + addresses.append((self._decode_header(name), addr)) - def getbody(self): + return addresses + + def get_body(self): """Get the decoded message body.""" - self.rewindbody() - encoding = self.getencoding() - data = None - if encoding == 'base64': - # BUG: is base64 really used for text encoding or - # are we inserting zip files here. - data = binascii.a2b_base64(self.fp.read()) - elif encoding == 'quoted-printable': - # the quopri module wants to work with files - decoded = StringIO() - quopri.decode(self.fp, decoded) - data = decoded.getvalue() - elif encoding == 'uuencoded': - data = binascii.a2b_uu(self.fp.read()) - else: - # take it as text - data = self.fp.read() + content = self.get_payload(decode=True) - # Encode message to unicode - charset = self.getparam("charset") - if charset: - charset = charset.lower().replace("windows-", 'cp') - # Do conversion only if charset specified - handle - # badly-specified charsets - edata = codecs.decode(data, charset, 'replace').encode('utf-8') - # Convert from dos eol to unix - edata = edata.replace('\r\n', '\n') - else: - # Leave message content as is - edata = data + if content is not None: + charset = self.get_content_charset() + content = u2s(content.decode(charset or 'iso8859-1', 'replace')) - return edata + return content # General multipart handling: # Take the first text/plain part, anything else is considered an @@ -392,32 +274,38 @@ # For web forms only. # message/rfc822: # Only if configured in [mailgw] unpack_rfc822 - def extract_content(self, parent_type=None, ignore_alternatives=False, - unpack_rfc822=False, html2text=None): - """Extract the body and the attachments recursively. + unpack_rfc822=False, html2text=None): + """ + Extract the body and the attachments recursively. - If the content is hidden inside a multipart/alternative part, - we use the *last* text/plain part of the *first* - multipart/alternative in the whole message. + If the content is hidden inside a multipart/alternative part, we use + the *last* text/plain part of the *first* multipart/alternative in + the whole message. + + If ignore_alteratives is True then only the alternative parts in the + same multipart/alternative part as where the content is found are + ignored. """ - content_type = self.gettype() + content_type = self.get_content_type() content = None attachments = [] html_part = False if content_type == 'text/plain': - content = self.getbody() + content = self.get_body() elif content_type == 'text/html' and html2text: # if user allows html conversion run this. - content = html2text(self.getbody()) + content = html2text(self.get_body()) attachments.append(self.as_attachment()) html_part = True - elif content_type[:10] == 'multipart/': + elif content_type == 'message/rfc822' and not unpack_rfc822: + attachments.append(self.as_attachment()) + elif self.is_multipart(): content_found = False ig = ignore_alternatives html_part_found = False - for part in self.getparts(): + for part in self.get_payload(): new_content, new_attach, html_part = part.extract_content( content_type, not content and ig, unpack_rfc822, html2text) @@ -457,67 +345,78 @@ if ig and content_type == 'multipart/alternative' and content: attachments = [] html_part = False - elif unpack_rfc822 and content_type == 'message/rfc822': - s = StringIO(self.getbody()) - m = Message(s) - ig = ignore_alternatives and not content - new_content, attachments, html_part = m.extract_content(m.gettype(), ig, - unpack_rfc822, html2text) - attachments.insert(0, m.text_as_attachment()) elif (parent_type == 'multipart/signed' and - content_type == 'application/pgp-signature'): - # ignore it so it won't be saved as an attachment + content_type == 'application/pgp-signature'): + # Don't save signatures for signed messages as attachments pass else: attachments.append(self.as_attachment()) + return content, attachments, html_part def text_as_attachment(self): """Return first text/plain part as Message""" - if not self.gettype().startswith ('multipart/'): + if not self.is_multipart(): return self.as_attachment() - for part in self.getparts(): - content_type = part.gettype() - if content_type == 'text/plain': - return part.as_attachment() - elif content_type.startswith ('multipart/'): + for part in self.get_payload(): + if part.is_multipart(): p = part.text_as_attachment() if p: return p + elif part.get_content_type() == 'text/plain': + return part.as_attachment() return None def as_attachment(self): """Return this message as an attachment.""" - return (self.getname(), self.gettype(), self.getbody()) + filename = self.get_filename() + content_type = self.get_content_type() + content = self.get_body() + + if content is None and self.get_content_type() == 'message/rfc822': + # handle message/rfc822 specially - the name should be + # the subject of the actual e-mail embedded here + # we add a '.eml' extension like other email software does it + subject = self.get_payload(0).get('subject') + if subject: + filename = '{0}.eml'.format(subject) + + content = self.get_payload(0).flatten() + + return (filename, content_type, content) def pgp_signed(self): - ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter - ''' - return self.gettype() == 'multipart/signed' \ - and self.typeheader.find('protocol="application/pgp-signature"') != -1 + """ + RFC 3156 requires OpenPGP MIME mail to have the protocol parameter + """ + return (self.get_content_type() == 'multipart/signed' and + self.get_param('protocol') == 'application/pgp-signature') def pgp_encrypted(self): - ''' RFC 3156 requires OpenPGP MIME mail to have the protocol parameter - ''' - return self.gettype() == 'multipart/encrypted' \ - and self.typeheader.find('protocol="application/pgp-encrypted"') != -1 + """ + RFC 3156 requires OpenPGP MIME mail to have the protocol parameter + """ + return (self.get_content_type() == 'multipart/encrypted' and + self.get_param('protocol') == 'application/pgp-encrypted') def decrypt(self, author, may_be_unsigned=False): - ''' decrypt an OpenPGP MIME message - This message must be signed as well as encrypted using the - "combined" method if incoming signatures are configured. - The decrypted contents are returned as a new message. ''' - (hdr, msg) = self.getparts() + Decrypt an OpenPGP MIME message + + This message must be signed as well as encrypted using the "combined" + method if incoming signatures are configured. The decrypted contents + are returned as a new message. + ''' + (hdr, msg) = self.get_payload() # According to the RFC 3156 encrypted mail must have exactly two parts. # The first part contains the control information. Let's verify that # the message meets the RFC before we try to decrypt it. - if hdr.getbody().strip() != 'Version: 1' \ - or hdr.gettype() != 'application/pgp-encrypted': + if (hdr.get_payload().strip() != 'Version: 1' or + hdr.get_content_type() != 'application/pgp-encrypted'): raise MailUsageError(_("Unknown multipart/encrypted version.")) context = pyme.core.Context() - ciphertext = pyme.core.Data(msg.getbody()) + ciphertext = pyme.core.Data(msg.get_payload()) plaintext = pyme.core.Data() result = context.op_decrypt_verify(ciphertext, plaintext) @@ -530,43 +429,37 @@ # was signed by someone we trust result = context.op_verify_result() check_pgp_sigs(result.signatures, context, author, - may_be_unsigned = may_be_unsigned) + may_be_unsigned=may_be_unsigned) - plaintext.seek(0,0) + plaintext.seek(0, 0) # pyme.core.Data implements a seek method with a different signature # than roundup can handle. So we'll put the data in a container that # the Message class can work with. - c = StringIO() - c.write(plaintext.read()) - c.seek(0) - return Message(c) + return email.message_from_string(plaintext.read(), RoundupMessage) def verify_signature(self, author): - ''' verify the signature of an OpenPGP MIME message - This only handles detached signatures. Old style - PGP mail (i.e. '-----BEGIN PGP SIGNED MESSAGE----') - is archaic and not supported :) - ''' + """ + Verify the signature of an OpenPGP MIME message + + This only handles detached signatures. Old style PGP mail (i.e. + '-----BEGIN PGP SIGNED MESSAGE----') is archaic and not supported :) + """ # we don't check the micalg parameter...gpgme seems to # figure things out on its own - (msg, sig) = self.getparts() + (msg, sig) = self.get_payload() - if sig.gettype() != 'application/pgp-signature': + if sig.get_content_type() != 'application/pgp-signature': raise MailUsageError(_("No PGP signature found in message.")) - # msg.getbody() is skipping over some headers that are - # required to be present for verification to succeed so - # we'll do this by hand - msg.fp.seek(0) # according to rfc 3156 the data "MUST first be converted # to its content-type specific canonical form. For # text/plain this means conversion to an appropriate # character set and conversion of line endings to the # canonical <CR><LF> sequence." # TODO: what about character set conversion? - canonical_msg = re.sub('(?<!\r)\n', '\r\n', msg.fp.read()) + canonical_msg = re.sub('(?<!\r)\n', '\r\n', msg.flatten()) msg_data = pyme.core.Data(canonical_msg) - sig_data = pyme.core.Data(sig.getbody()) + sig_data = pyme.core.Data(sig.get_payload()) context = pyme.core.Context() context.op_verify(sig_data, msg_data, None) @@ -582,16 +475,16 @@ self.config = mailgw.instance.config self.db = mailgw.db self.message = message - self.subject = message.getheader('subject', '') + self.subject = message.get_header('subject', '') self.has_prefix = False self.matches = dict.fromkeys(['refwd', 'quote', 'classname', 'nodeid', 'title', 'args', 'argswhole']) self.keep_real_from = self.config['EMAIL_KEEP_REAL_FROM'] if self.keep_real_from: - self.from_list = message.getaddrlist('from') + self.from_list = message.get_address_list('from') else: - self.from_list = message.getaddrlist('resent-from') \ - or message.getaddrlist('from') + self.from_list = (message.get_address_list('resent-from') or + message.get_address_list('from')) self.pfxmode = self.config['MAILGW_SUBJECT_PREFIX_PARSING'] self.sfxmode = self.config['MAILGW_SUBJECT_SUFFIX_PARSING'] # these are filled in by subsequent parsing steps @@ -612,9 +505,9 @@ detect loops and Precedence: Bulk, or Microsoft Outlook autoreplies ''' - if self.message.getheader('x-roundup-loop', ''): + if self.message.get_header('x-roundup-loop', ''): raise IgnoreLoop - if (self.message.getheader('precedence', '') == 'bulk' + if (self.message.get_header('precedence', '') == 'bulk' or self.subject.lower().find("autoreply") > 0): raise IgnoreBulk @@ -812,7 +705,7 @@ nodeid = self.matches['nodeid'] # try in-reply-to to match the message if there's no nodeid - inreplyto = self.message.getheader('in-reply-to') or '' + inreplyto = self.message.get_header('in-reply-to') or '' if nodeid is None and inreplyto: l = self.db.getclass('msg').stringFind(messageid=inreplyto) if l: @@ -958,8 +851,8 @@ # now update the recipients list recipients = [] tracker_email = self.config['TRACKER_EMAIL'].lower() - msg_to = self.message.getaddrlist('to') - msg_cc = self.message.getaddrlist('cc') + msg_to = self.message.get_address_list('to') + msg_cc = self.message.get_address_list('cc') for recipient in msg_to + msg_cc: r = recipient[1].strip().lower() if r == tracker_email or not r: @@ -1110,7 +1003,6 @@ ignore_alternatives=ig, unpack_rfc822=self.config.MAILGW_UNPACK_RFC822, html2text=html2text ) - def create_files(self): ''' Create a file for each attachment in the message @@ -1119,7 +1011,7 @@ return files = [] file_props = self.mailgw.get_class_arguments('file') - + if self.attachments: for (name, mime_type, data) in self.attachments: if not self.db.security.hasPermission('Create', self.author, @@ -1160,8 +1052,8 @@ self.msg_props.update (msg_props) # Get the message ids - inreplyto = self.message.getheader('in-reply-to') or '' - messageid = self.message.getheader('message-id') + inreplyto = self.message.get_header('in-reply-to') or '' + messageid = self.message.get_header('message-id') # generate a messageid if there isn't one if not messageid: messageid = "<%s.%s.%s%s@%s>"%(time.time(), @@ -1247,7 +1139,7 @@ # issue_re = config['MAILGW_ISSUE_ADDRESS_RE'] # if issue_re: # for header in ['to', 'cc', 'bcc']: -# addresses = message.getheader(header, '') +# addresses = message.get_header(header, '') # if addresses: # # FIXME, this only finds the first match in the addresses. # issue = re.search(issue_re, addresses, 'i') @@ -1348,33 +1240,29 @@ """ Read a series of messages from the specified unix mailbox file and pass each to the mail handler. """ - # open the spool file and lock it - import fcntl - # FCNTL is deprecated in py2.3 and fcntl takes over all the symbols - if hasattr(fcntl, 'LOCK_EX'): - FCNTL = fcntl - else: - import FCNTL - f = open(filename, 'r+') - fcntl.flock(f.fileno(), FCNTL.LOCK_EX) + import mailbox - # handle and clear the mailbox + class mboxRoundupMessage(mailbox.mboxMessage, RoundupMessage): + pass + try: - from mailbox import UnixMailbox - mailbox = UnixMailbox(f, factory=Message) - # grab one message - message = next(mailbox) - while message: - # handle this message - self.handle_Message(message) - message = next(mailbox) - # nuke the file contents - os.ftruncate(f.fileno(), 0) - except: - import traceback + mbox = mailbox.mbox(filename, factory=mboxRoundupMessage, + create=False) + mbox.lock() + except (mailbox.NoSuchMailboxError, mailbox.ExternalClashError) as e: + if isinstance(e, mailbox.ExternalClashError): + mbox.close() traceback.print_exc() return 1 - fcntl.flock(f.fileno(), FCNTL.LOCK_UN) + + try: + for key in mbox.keys(): + self.handle_Message(mbox.get(key)) + mbox.remove(key) + finally: + mbox.unlock() + mbox.close() + return 0 def do_imap(self, server, user='', password='', mailbox='', ssl=0, @@ -1503,9 +1391,8 @@ # [ array of message lines ], # number of octets ] lines = server.retr(i)[1] - s = StringIO('\n'.join(lines)) - s.seek(0) - self.handle_Message(Message(s)) + self.handle_Message( + email.message_from_string('\n'.join(lines), RoundupMessage)) # delete the message server.dele(i) @@ -1516,7 +1403,7 @@ def main(self, fp): ''' fp - the file from which to read the Message. ''' - return self.handle_Message(Message(fp)) + return self.handle_Message(email.message_from_file(fp, RoundupMessage)) def handle_Message(self, message): """Handle an RFC822 Message @@ -1532,20 +1419,20 @@ self.parsed_message = None crypt = False - sendto = message.getaddrlist('resent-from') + sendto = message.get_address_list('resent-from') if not sendto or self.instance.config['EMAIL_KEEP_REAL_FROM']: - sendto = message.getaddrlist('from') + sendto = message.get_address_list('from') if not sendto: # very bad-looking message - we don't even know who sent it msg = ['Badly formed message from mail gateway. Headers:'] - msg.extend(message.headers) + msg.extend([': '.join(args) for args in message.items()]) msg = '\n'.join(map(str, msg)) self.logger.error(msg) return msg = 'Handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') + if message.get_header('message-id'): + msg += ' (Message-id=%r)'%message.get_header('message-id') self.logger.info(msg) # try normal message-handling @@ -1595,14 +1482,14 @@ # do not take any action # this exception is thrown when email should be ignored msg = 'IgnoreMessage raised' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') + if message.get_header('message-id'): + msg += ' (Message-id=%r)'%message.get_header('message-id') self.logger.info(msg) return except: msg = 'Exception handling message' - if message.getheader('message-id'): - msg += ' (Message-id=%r)'%message.getheader('message-id') + if message.get_header('message-id'): + msg += ' (Message-id=%r)'%message.get_header('message-id') self.logger.exception(msg) # bounce the message back to the sender with the error message
--- a/roundup/roundupdb.py Sun Aug 12 16:05:42 2018 +0100 +++ b/roundup/roundupdb.py Sun Aug 12 16:15:10 2018 +0100 @@ -36,8 +36,7 @@ from roundup.i18n import _ from roundup.hyperdb import iter_roles -from roundup.mailer import Mailer, MessageSendError, encode_quopri, \ - nice_sender_header +from roundup.mailer import Mailer, MessageSendError, nice_sender_header from roundup.anypy.strings import b2s, s2u import roundup.anypy.random_ as random_ @@ -498,8 +497,6 @@ # construct the content and convert to unicode object body = s2u('\n'.join(m)) - if type(body) != type(''): - body = body.encode(charset) # make sure the To line is always the same (for testing mostly) sendto.sort() @@ -610,28 +607,31 @@ # attach files if message_files: # first up the text as a part - part = MIMEText(body) - part.set_charset(charset) - encode_quopri(part) + part = mailer.get_standard_message() + part.set_payload(body, part.get_charset()) message.attach(part) for fileid in message_files: name = files.get(fileid, 'name') - mime_type = files.get(fileid, 'type') - content = files.get(fileid, 'content') + mime_type = (files.get(fileid, 'type') or + mimetypes.guess_type(name)[0] or + 'application/octet-stream') if mime_type == 'text/plain': + content = files.get(fileid, 'content') + part = MIMEText('') + del part['Content-Transfer-Encoding'] try: - content.decode('ascii') + enc = content.encode('ascii') + part = mailer.get_text_message('us-ascii') + part.set_payload(enc) except UnicodeError: # the content cannot be 7bit-encoded. # use quoted printable # XXX stuffed if we know the charset though :( - part = MIMEText(content) - encode_quopri(part) - else: - part = MIMEText(content) - part['Content-Transfer-Encoding'] = '7bit' + part = mailer.get_text_message('utf-8') + part.set_payload(content, part.get_charset()) elif mime_type == 'message/rfc822': + content = files.get(fileid, 'content') main, sub = mime_type.split('/') p = FeedParser() p.feed(content) @@ -639,11 +639,7 @@ part.set_payload([p.close()]) else: # some other type, so encode it - if not mime_type: - # this should have been done when the file was saved - mime_type = mimetypes.guess_type(name)[0] - if mime_type is None: - mime_type = 'application/octet-stream' + content = files.get(fileid, 'binary_content') main, sub = mime_type.split('/') part = MIMEBase(main, sub) part.set_payload(content) @@ -653,8 +649,7 @@ message.attach(part) else: - message.set_payload(body) - encode_quopri(message) + message.set_payload(body, message.get_charset()) if crypt: send_msg = self.encrypt_to (message, sendto)
--- a/test/db_test_base.py Sun Aug 12 16:05:42 2018 +0100 +++ b/test/db_test_base.py Sun Aug 12 16:15:10 2018 +0100 @@ -2579,8 +2579,8 @@ res["mail_to"], res["mail_msg"] = to, msg backup, Mailer.smtp_send = Mailer.smtp_send, dummy_snd try : - f1 = db.file.create(name="test1.txt", content="x" * 20) - f2 = db.file.create(name="test2.txt", content="y" * 5000) + f1 = db.file.create(name="test1.txt", content="x" * 20, type="application/octet-stream") + f2 = db.file.create(name="test2.txt", content="y" * 5000, type="application/octet-stream") m = db.msg.create(content="one two", author="admin", files = [f1, f2]) i = db.issue.create(title='spam', files = [f1, f2], @@ -2601,6 +2601,37 @@ roundupdb._ = old_translate_ Mailer.smtp_send = backup + def testNosyMailTextAndBinary(self) : + """Creates one issue with two attachments, one as text and one as binary. + """ + old_translate_ = roundupdb._ + roundupdb._ = i18n.get_translation(language='C').gettext + db = self.db + res = dict(mail_to = None, mail_msg = None) + def dummy_snd(s, to, msg, res=res) : + res["mail_to"], res["mail_msg"] = to, msg + backup, Mailer.smtp_send = Mailer.smtp_send, dummy_snd + try : + f1 = db.file.create(name="test1.txt", content="Hello world", type="text/plain") + f2 = db.file.create(name="test2.bin", content=b"\x01\x02\x03\xfe\xff", type="application/octet-stream") + m = db.msg.create(content="one two", author="admin", + files = [f1, f2]) + i = db.issue.create(title='spam', files = [f1, f2], + messages = [m], nosy = [db.user.lookup("fred")]) + + db.issue.nosymessage(i, m, {}) + mail_msg = str(res["mail_msg"]) + self.assertEqual(res["mail_to"], ["fred@example.com"]) + self.assert_("From: admin" in mail_msg) + self.assert_("Subject: [issue1] spam" in mail_msg) + self.assert_("New submission from admin" in mail_msg) + self.assert_("one two" in mail_msg) + self.assert_("Hello world" in mail_msg) + self.assert_(b2s(base64.encodestring(b"\x01\x02\x03\xfe\xff")).rstrip() in mail_msg) + finally : + roundupdb._ = old_translate_ + Mailer.smtp_send = backup + @pytest.mark.skipif(gpgmelib.pyme is None, reason='Skipping PGPNosy test') def testPGPNosyMail(self) : """Creates one issue with two attachments, one smaller and one larger @@ -2623,8 +2654,8 @@ try : john = db.user.create(username="john", roles='User,pgp', address='john@test.test', realname='John Doe') - f1 = db.file.create(name="test1.txt", content="x" * 20) - f2 = db.file.create(name="test2.txt", content="y" * 5000) + f1 = db.file.create(name="test1.txt", content="x" * 20, type="application/octet-stream") + f2 = db.file.create(name="test2.txt", content="y" * 5000, type="application/octet-stream") m = db.msg.create(content="one two", author="admin", files = [f1, f2]) i = db.issue.create(title='spam', files = [f1, f2],
--- a/test/test_mailgw.py Sun Aug 12 16:05:42 2018 +0100 +++ b/test/test_mailgw.py Sun Aug 12 16:15:10 2018 +0100 @@ -28,7 +28,7 @@ reason="Skipping PGP tests: 'pyme' not installed")) -from roundup.anypy.strings import StringIO, u2s +from roundup.anypy.strings import StringIO, b2s, u2s if 'SENDMAILDEBUG' not in os.environ: os.environ['SENDMAILDEBUG'] = 'mail-test.log' @@ -894,7 +894,7 @@ self.assertEqual(msg.content, 'test attachment second text/plain') def testMultipartCharsetUTF8NoAttach(self): - c = 'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f' + c = b2s(b'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f') self.doNewIssue() self.db.config.NOSY_MAX_ATTACHMENT_SIZE = 0 self._handle_mail(self.multipart_msg_latin1) @@ -944,7 +944,7 @@ ''') def testMultipartCharsetLatin1NoAttach(self): - c = 'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f' + c = b2s(b'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f') self.doNewIssue() self.db.config.NOSY_MAX_ATTACHMENT_SIZE = 0 self.db.config.MAIL_CHARSET = 'iso-8859-1' @@ -995,7 +995,7 @@ ''') def testMultipartCharsetUTF8AttachFile(self): - c = 'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f' + c = b2s(b'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f') self.doNewIssue() self._handle_mail(self.multipart_msg_latin1) messages = self.db.issue.get('1', 'messages') @@ -1057,7 +1057,7 @@ ''') def testMultipartCharsetLatin1AttachFile(self): - c = 'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f' + c = b2s(b'umlaut \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f') self.doNewIssue() self.db.config.MAIL_CHARSET = 'iso-8859-1' self._handle_mail(self.multipart_msg_latin1) @@ -1287,7 +1287,7 @@ messages.sort() msg = self.db.msg.getnode(messages[-1]) # html converted to utf-8 text - self.assertEqual(msg.content, mycontent+" \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f") + self.assertEqual(msg.content, mycontent+b2s(b" \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f")) self.assertEqual(msg.type, None) self.assertEqual(len(msg.files), 2) name = "unnamed" # no name for any files @@ -1295,7 +1295,7 @@ # replace quoted printable string at end of html document # with it's utf-8 encoded equivalent so comparison # works. - content = { 0: "75,23,16,18\n", 1: self.html_doc.replace(" =E4=F6=FC=C4=D6=DC=DF"," \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f")} + content = { 0: "75,23,16,18\n", 1: self.html_doc.replace(" =E4=F6=FC=C4=D6=DC=DF",b2s(b" \xc3\xa4\xc3\xb6\xc3\xbc\xc3\x84\xc3\x96\xc3\x9c\xc3\x9f"))} for n, id in enumerate (msg.files): f = self.db.file.getnode (id) self.assertEqual(f.name, name) @@ -2648,11 +2648,11 @@ self._allowAnonymousSubmit() self._handle_mail(message) title = self.db.issue.get('1', 'title') - self.assertEquals(title, 'Test \xc3\x84\xc3\x96\xc3\x9c umlauts X1 X2') + self.assertEquals(title, b2s(b'Test \xc3\x84\xc3\x96\xc3\x9c umlauts X1 X2')) m = set(self.db.user.list()) new = list(m - l)[0] name = self.db.user.get(new, 'realname') - self.assertEquals(name, 'Firstname \xc3\xa4\xc3\xb6\xc3\x9f Last') + self.assertEquals(name, b2s(b'Firstname \xc3\xa4\xc3\xb6\xc3\x9f Last')) def testNewUserAuthorMixedEncodedNameSpacing(self): l = set(self.db.user.list()) @@ -2670,12 +2670,12 @@ self._allowAnonymousSubmit() self._handle_mail(message) title = self.db.issue.get('1', 'title') - self.assertEquals(title, 'Test (\xc3\x84\xc3\x96\xc3\x9c) umlauts X1') + self.assertEquals(title, b2s(b'Test (\xc3\x84\xc3\x96\xc3\x9c) umlauts X1')) m = set(self.db.user.list()) new = list(m - l)[0] name = self.db.user.get(new, 'realname') self.assertEquals(name, - '(\xc3\xa4\xc3\xb6\xc3\x9f\xc3\xa4\xc3\xb6\xc3\x9f)') + b2s(b'(\xc3\xa4\xc3\xb6\xc3\x9f\xc3\xa4\xc3\xb6\xc3\x9f)')) def testUnknownUser(self): l = set(self.db.user.list()) @@ -2839,9 +2839,7 @@ def testMultipartEnc01(self): self.doNewIssue() - self._handle_mail('''Content-Type: text/plain; - charset="iso-8859-1" -From: mary <mary@test.test> + self._handle_mail('''From: mary <mary@test.test> To: issue_tracker@your.tracker.email.domain.example Message-Id: <followup_dummy_id> In-Reply-To: <dummy_test_message_id> @@ -2893,9 +2891,7 @@ def testContentDisposition(self): self.doNewIssue() - self._handle_mail('''Content-Type: text/plain; - charset="iso-8859-1" -From: mary <mary@test.test> + self._handle_mail('''From: mary <mary@test.test> To: issue_tracker@your.tracker.email.domain.example Message-Id: <followup_dummy_id> In-Reply-To: <dummy_test_message_id> @@ -4187,7 +4183,7 @@ def testForwardedMessageAttachment(self): message = '''Return-Path: <rgg@test.test> Received: from localhost(127.0.0.1), claiming to be "[115.130.26.69]" -via SMTP by localhost, id smtpdAAApLaWrq; Tue Apr 13 23:10:05 2010 + via SMTP by localhost, id smtpdAAApLaWrq; Tue Apr 13 23:10:05 2010 Message-ID: <4BC4F9C7.50409@test.test> Date: Wed, 14 Apr 2010 09:09:59 +1000 From: Rupert Goldie <rgg@test.test>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/test_mailgw_roundupmessage.py Sun Aug 12 16:15:10 2018 +0100 @@ -0,0 +1,442 @@ +# -*- coding: utf-8 -*- +import email +import textwrap +from unittest import TestCase + +from roundup.mailgw import RoundupMessage + +PART_TYPES = { + 'multipart/signed': ' boundary="boundary-{indent}";\n', + 'multipart/mixed': ' boundary="boundary-{indent}";\n', + 'multipart/alternative': ' boundary="boundary-{indent}";\n', + 'text/plain': ' name="foo.txt"\n\nfoo\n', + 'text/plain_2': ' name="foo2.txt"\n\nfoo2\n', + 'text/plain_3': ' name="foo3.txt"\n\nfoo3\n', + 'text/html': ' name="foo.html"\n\n<html>foo</html>\n', + 'application/pgp-signature': ' name="foo.gpg"\nfoo\n', + 'application/pdf': ' name="foo.pdf"\nfoo\n', + 'application/pdf_2': ' name="foo2.pdf"\nfoo2\n', + 'message/rfc822': '\nSubject: foo\n\nfoo\n', +} + + +def message_from_string(msg): + return email.message_from_string( + textwrap.dedent(msg).lstrip(), + RoundupMessage) + + +def construct_message(spec, depth=0): + parts = [] + for content_type in spec: + if isinstance(content_type, list): + parts.extend(construct_message(content_type, depth=(depth + 1))) + parts.append('\n--boundary-{0}--\n'.format(depth + 1)) + else: + if depth > 0: + parts.append('\n--boundary-{0}\n'.format(depth)) + + parts.append( + 'Content-Type: {0};\n'.format(content_type.split('_')[0])) + parts.append(PART_TYPES[content_type].format(indent=(depth + 1))) + + if depth == 0: + return email.message_from_string(''.join(parts), RoundupMessage) + else: + return parts + + +class FlattenRoundupMessageTests(TestCase): + def test_flatten_with_from(self): + msg_string = textwrap.dedent(""" + From: Some User <some.user@example.com> + To: issue_tracker@example.com + Message-Id: <dummy_test_message_id> + Subject: Test line start with from + + From here to there! + """).lstrip() + + msg = email.message_from_string(msg_string, RoundupMessage) + self.assertEqual(msg.flatten(), msg_string) + + +class HeaderRoundupMessageTests(TestCase): + msg = message_from_string(""" + Content-Type: text/plain; + charset="iso-8859-1" + From: =?utf8?b?SOKCrGxsbw==?= <hello@example.com> + To: Issue Tracker <issue_tracker@example.com> + Cc: =?utf8?b?SOKCrGxsbw==?= <hello@example.com>, + Some User <some.user@example.com> + Message-Id: <dummy_test_message_id> + Subject: [issue] Testing... + + This is a test submission of a new issue. + """) + + def test_get_plain_header(self): + self.assertEqual( + self.msg.get_header('to'), + 'Issue Tracker <issue_tracker@example.com>') + + def test_get_encoded_header(self): + self.assertEqual( + self.msg.get_header('from'), + 'H€llo <hello@example.com>') + + def test_get_address_list(self): + self.assertEqual(self.msg.get_address_list('cc'), [ + ('H€llo', 'hello@example.com'), + ('Some User', 'some.user@example.com'), + ]) + + +class BodyRoundupMessageTests(TestCase): + def test_get_body_iso_8859_1(self): + msg = message_from_string(""" + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + A message with encoding (encoded oe =F6) + """) + + self.assertEqual( + msg.get_body(), + 'A message with encoding (encoded oe ö)\n') + + def test_get_body_utf_8(self): + msg = message_from_string(""" + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: quoted-printable + + A message with encoding (encoded oe =C3=B6) + """) + + self.assertEqual( + msg.get_body(), + 'A message with encoding (encoded oe ö)\n') + + def test_get_body_base64(self): + msg = message_from_string(""" + Content-Type: application/octet-stream + Content-Disposition: attachment; filename="message.dat" + Content-Transfer-Encoding: base64 + + dGVzdCBlbmNvZGVkIG1lc3NhZ2U= + """) + + self.assertEqual(msg.get_body(), 'test encoded message') + + +class AsAttachmentRoundupMessageTests(TestCase): + def test_text_plain(self): + msg = message_from_string(""" + Content-Type: text/plain; charset="iso-8859-1 + + Plain text message + """) + + self.assertEqual( + msg.as_attachment(), + (None, 'text/plain', 'Plain text message\n')) + + def test_octet_stream(self): + msg = message_from_string(""" + Content-Type: application/octet-stream + Content-Disposition: attachment; filename="message.dat" + Content-Transfer-Encoding: base64 + + dGVzdCBlbmNvZGVkIG1lc3NhZ2U= + """) + + self.assertEqual( + msg.as_attachment(), + ('message.dat', 'application/octet-stream', + 'test encoded message')) + + def test_rfc822(self): + msg = message_from_string(""" + Content-Type: message/rfc822 + + Subject: foo + + foo + """) + + self.assertEqual( + msg.as_attachment(), + ('foo.eml', 'message/rfc822', 'Subject: foo\n\nfoo\n')) + + def test_rfc822_no_subject(self): + msg = message_from_string(""" + Content-Type: message/rfc822 + + X-No-Headers: nope + + foo + """) + + self.assertEqual( + msg.as_attachment(), + (None, 'message/rfc822', 'X-No-Headers: nope\n\nfoo\n')) + + def test_rfc822_no_payload(self): + msg = message_from_string("""\ + Content-Type: message/rfc822 + """) + + self.assertEqual( + msg.as_attachment(), + (None, 'message/rfc822', '\n')) + + +class ExtractContentRoundupMessageTests(TestCase): + def test_text_plain(self): + msg = construct_message(['text/plain']) + + self.assertEqual(msg.extract_content(), ('foo\n', [], False)) + + def test_attached_text_plain(self): + msg = construct_message([ + 'multipart/mixed', [ + 'text/plain', + 'text/plain', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo\n', + [('foo.txt', 'text/plain', 'foo\n')], + False + )) + + def test_multipart_mixed(self): + msg = construct_message([ + 'multipart/mixed', [ + 'text/plain', + 'application/pdf', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo\n', + [('foo.pdf', 'application/pdf', 'foo\n')], + False + )) + + def test_multipart_alternative(self): + msg = construct_message([ + 'multipart/alternative', [ + 'text/plain', + 'text/html', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo\n', + [('foo.html', 'text/html', '<html>foo</html>\n')], + False + )) + + def test_deep_multipart_alternative(self): + msg = construct_message([ + 'multipart/mixed', [ + 'multipart/alternative', [ + 'text/plain', + 'application/pdf', + 'text/plain_2', + 'text/html', + ], + 'multipart/alternative', [ + 'text/plain_3', + 'application/pdf_2', + ], + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo2\n', [ + ('foo.pdf', 'application/pdf', 'foo\n'), + ('foo.txt', 'text/plain', 'foo\n'), + ('foo.html', 'text/html', '<html>foo</html>\n'), + ('foo3.txt', 'text/plain', 'foo3\n'), + ('foo2.pdf', 'application/pdf', 'foo2\n'), + ], + False + )) + + def test_deep_multipart_alternative_ignore(self): + msg = construct_message([ + 'multipart/mixed', [ + 'multipart/alternative', [ + 'text/plain', + 'application/pdf', + 'text/plain_2', + 'text/html', + ], + 'multipart/alternative', [ + 'text/plain_3', + 'application/pdf_2', + ], + ], + ]) + + msg.extract_content(ignore_alternatives=True) + self.assertEqual(msg.extract_content(ignore_alternatives=True), ( + 'foo2\n', [ + ('foo3.txt', 'text/plain', 'foo3\n'), + ('foo2.pdf', 'application/pdf', 'foo2\n'), + ], + False + )) + + def test_signed_text(self): + msg = construct_message([ + 'multipart/signed', [ + 'text/plain', + 'application/pgp-signature', + ], + ]) + + self.assertEqual(msg.extract_content(), ('foo\n', [], False)) + + def test_signed_attachemts(self): + msg = construct_message([ + 'multipart/signed', [ + 'multipart/mixed', [ + 'text/plain', + 'application/pdf', + ], + 'application/pgp-signature', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo\n', + [('foo.pdf', 'application/pdf', 'foo\n')], + False + )) + + def test_attached_signature(self): + msg = construct_message([ + 'multipart/mixed', [ + 'text/plain', + 'application/pgp-signature', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + 'foo\n', + [('foo.gpg', 'application/pgp-signature', 'foo\n')], + False + )) + + def test_rfc822_message(self): + msg = construct_message([ + 'multipart/mixed', [ + 'message/rfc822', + ], + ]) + + self.assertEqual(msg.extract_content(), ( + None, + [('foo.eml', 'message/rfc822', 'Subject: foo\n\nfoo\n')], + False + )) + + def test_rfc822_message_unpack(self): + msg = construct_message([ + 'multipart/mixed', [ + 'text/plain', + 'message/rfc822', + ], + ]) + + self.assertEqual(msg.extract_content(unpack_rfc822=True), ( + 'foo\n', + [(None, 'text/plain', 'foo\n')], + False + )) + + +class PgpDetectRoundupMessageTests(TestCase): + def test_pgp_message_signed(self): + msg = message_from_string(""" + Content-Type: multipart/signed; micalg=pgp-sha1; + protocol="application/pgp-signature" + + Fake Body + """) + + self.assertTrue(msg.pgp_signed()) + + def test_pgp_message_not_signed(self): + msg = message_from_string(""" + Content-Type: text/plain + + Fake Body + """) + + self.assertFalse(msg.pgp_signed()) + + def test_pgp_message_signed_protocol_missing(self): + msg = message_from_string(""" + Content-Type: multipart/signed; micalg=pgp-sha1 + + Fake Body + """) + + self.assertFalse(msg.pgp_signed()) + + def test_pgp_message_signed_protocol_invalid(self): + msg = message_from_string(""" + Content-Type: multipart/signed; + protocol="application/not-pgp-signature" + + Fake Body + """) + + self.assertFalse(msg.pgp_signed()) + + def test_pgp_message_encrypted(self): + msg = message_from_string(""" + Content-Type: multipart/encrypted; + protocol="application/pgp-encrypted" + + Fake Body + """) + + self.assertTrue(msg.pgp_encrypted()) + + def test_pgp_message_not_encrypted(self): + msg = message_from_string(""" + Content-Type: text/plain + + Fake Body + """) + + self.assertFalse(msg.pgp_encrypted()) + + def test_pgp_message_encrypted_protocol_missing(self): + msg = message_from_string(""" + Content-Type: multipart/encrypted + + Fake Body + """) + + self.assertFalse(msg.pgp_encrypted()) + + def test_pgp_message_encrypted_protocol_invalid(self): + msg = message_from_string(""" + Content-Type: multipart/encrypted; + protocol="application/not-pgp-encrypted" + + Fake Body + """) + + self.assertFalse(msg.pgp_encrypted()) + +# TODO: testing of the verify_signature() and decrypt() RoundupMessage methods. +# The whole PGP testing stuff seems a bit messy, so we will rely on the tests +# in test_mailgw for the time being
--- a/test/test_multipart.py Sun Aug 12 16:05:42 2018 +0100 +++ b/test/test_multipart.py Sun Aug 12 16:15:10 2018 +0100 @@ -15,12 +15,30 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. +import email import unittest from roundup.anypy.strings import StringIO -from roundup.mailgw import Message +from roundup.mailgw import RoundupMessage + +def gen_message(spec): + """Create a basic MIME message according to 'spec'. + + Each line of a spec has one content-type, which is optionally indented. + The indentation signifies how deep in the MIME hierarchy the + content-type is. -class ExampleMessage(Message): + """ + + def getIndent(line): + """Get the current line's indentation, using four-space indents.""" + count = 0 + for char in line: + if char != ' ': + break + count += 1 + return count // 4 + # A note on message/rfc822: The content of such an attachment is an # email with at least one header line. RFC2046 tells us: """ A # media type of "message/rfc822" indicates that the body contains an @@ -42,36 +60,22 @@ 'application/pdf': ' name="foo.pdf"\nfoo\n', 'message/rfc822': '\nSubject: foo\n\nfoo\n'} - def __init__(self, spec): - """Create a basic MIME message according to 'spec'. - - Each line of a spec has one content-type, which is optionally indented. - The indentation signifies how deep in the MIME hierarchy the - content-type is. - - """ - parts = [] - for line in spec.splitlines(): - content_type = line.strip() - if not content_type: - continue + parts = [] + for line in spec.splitlines(): + content_type = line.strip() + if not content_type: + continue - indent = self.getIndent(line) - if indent: - parts.append('\n--boundary-%s\n' % indent) - parts.append('Content-type: %s;\n' % content_type) - parts.append(self.table[content_type] % {'indent': indent + 1}) - - Message.__init__(self, StringIO(''.join(parts))) + indent = getIndent(line) + if indent: + parts.append('\n--boundary-%s\n' % indent) + parts.append('Content-type: %s;\n' % content_type) + parts.append(table[content_type] % {'indent': indent + 1}) - def getIndent(self, line): - """Get the current line's indentation, using four-space indents.""" - count = 0 - for char in line: - if char != ' ': - break - count += 1 - return count // 4 + for i in range(indent, 0, -1): + parts.append('\n--boundary-%s--\n' % i) + + return email.message_from_file(StringIO(''.join(parts)), RoundupMessage) class MultipartTestCase(unittest.TestCase): def setUp(self): @@ -110,54 +114,49 @@ self.fp.seek(0) def testMultipart(self): - m = Message(self.fp) + m = email.message_from_file(self.fp, RoundupMessage) self.assert_(m is not None) - # skip the first bit - p = m.getpart() - self.assert_(p is not None) - self.assertEqual(p.fp.read(), - 'This is a multipart message. Ignore this bit.\r\n') + it = iter(m.get_payload()) # first text/plain - p = m.getpart() + p = next(it, None) self.assert_(p is not None) - self.assertEqual(p.gettype(), 'text/plain') - self.assertEqual(p.fp.read(), + self.assertEqual(p.get_content_type(), 'text/plain') + self.assertEqual(p.get_payload(), 'Hello, world!\r\n\r\nBlah blah\r\nfoo\r\n-foo\r\n') # sub-multipart - p = m.getpart() + p = next(it, None) self.assert_(p is not None) - self.assertEqual(p.gettype(), 'multipart/alternative') + self.assertEqual(p.get_content_type(), 'multipart/alternative') # sub-multipart text/plain - q = p.getpart() + qit = iter(p.get_payload()) + q = next(qit, None) self.assert_(q is not None) - q = p.getpart() - self.assert_(q is not None) - self.assertEqual(q.gettype(), 'text/plain') - self.assertEqual(q.fp.read(), 'Hello, world!\r\n\r\nBlah blah\r\n') + self.assertEqual(q.get_content_type(), 'text/plain') + self.assertEqual(q.get_payload(), 'Hello, world!\r\n\r\nBlah blah\r\n') # sub-multipart text/html - q = p.getpart() + q = next(qit, None) self.assert_(q is not None) - self.assertEqual(q.gettype(), 'text/html') - self.assertEqual(q.fp.read(), '<b>Hello, world!</b>\r\n') + self.assertEqual(q.get_content_type(), 'text/html') + self.assertEqual(q.get_payload(), '<b>Hello, world!</b>\r\n') # sub-multipart end - q = p.getpart() + q = next(qit, None) self.assert_(q is None) # final text/plain - p = m.getpart() + p = next(it, None) self.assert_(p is not None) - self.assertEqual(p.gettype(), 'text/plain') - self.assertEqual(p.fp.read(), + self.assertEqual(p.get_content_type(), 'text/plain') + self.assertEqual(p.get_payload(), 'Last bit\n') # end - p = m.getpart() + p = next(it, None) self.assert_(p is None) def TestExtraction(self, spec, expected, convert_html_with=False): @@ -167,7 +166,7 @@ else: html2text=None - self.assertEqual(ExampleMessage(spec).extract_content( + self.assertEqual(gen_message(spec).extract_content( html2text=html2text), expected) def testTextPlain(self):