Mercurial > p > roundup > code

comparison roundup/mailgw.py @ 7064:3359dc1dabb0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add OAuth authentication to the mailgw script Now IMAPS can be used with OAuth as required by several large cloud providers. Move command line processing of the mailgw script to ``argparse``. Note that the command line options of the mailgw have changed, see upgrading.txt for details.
author Ralf Schlatterbeck <rsc@runtux.com>
date Wed, 23 Nov 2022 10:13:48 +0100
parents bd2c3b2010c3
children f918351a0fe6
comparison
equal deleted inserted replaced
7063:aca710a3b687 7064:3359dc1dabb0
105 import re 105 import re
106 import sys 106 import sys
107 import time 107 import time
108 import traceback 108 import traceback
109 109
110 import mailbox
111 import socket
112 import getpass
113 import poplib
114 import imaplib
115 try:
116 import requests
117 except ImportError:
118 requests = None
110 from email.generator import Generator 119 from email.generator import Generator
111 120
112 import roundup.anypy.random_ as random_ 121 import roundup.anypy.random_ as random_
113 import roundup.anypy.ssl_ as ssl_ 122 import roundup.anypy.ssl_ as ssl_
114 123
118 from roundup.anypy.my_input import my_input 127 from roundup.anypy.my_input import my_input
119 from roundup.anypy.strings import StringIO, b2s, u2s 128 from roundup.anypy.strings import StringIO, b2s, u2s
120 from roundup.hyperdb import iter_roles 129 from roundup.hyperdb import iter_roles
121 from roundup.i18n import _ 130 from roundup.i18n import _
122 from roundup.mailer import Mailer 131 from roundup.mailer import Mailer
132 from roundup.dehtml import dehtml
123 133
124 try: 134 try:
125 import gpg, gpg.core, gpg.constants, gpg.constants.sigsum # noqa: E401 135 import gpg, gpg.core, gpg.constants, gpg.constants.sigsum # noqa: E401
126 except ImportError: 136 except ImportError:
127 gpg = None 137 gpg = None
717 # back on the default 727 # back on the default
718 attempts = [] 728 attempts = []
719 if classname: 729 if classname:
720 attempts.append(classname) 730 attempts.append(classname)
721 731
722 if self.mailgw.default_class: 732 attempts.append(self.mailgw.default_class)
723 attempts.append(self.mailgw.default_class)
724 else:
725 attempts.append(self.config['MAILGW_DEFAULT_CLASS'])
726 733
727 # first valid class name wins 734 # first valid class name wins
728 self.cl = None 735 self.cl = None
729 for trycl in attempts: 736 for trycl in attempts:
730 try: 737 try:
800 # match. Not sure if this would work better in 807 # match. Not sure if this would work better in
801 # production, so not implementing now. 808 # production, so not implementing now.
802 nodeid = None 809 nodeid = None
803 # trigger Subject match 810 # trigger Subject match
804 self.matches['refwd'] = True 811 self.matches['refwd'] = True
805 812
806 # but we do need either a title or a nodeid... 813 # but we do need either a title or a nodeid...
807 if nodeid is None and not title: 814 if nodeid is None and not title:
808 raise MailUsageError(_(""" 815 raise MailUsageError(_("""
809 I cannot match your message to a node in the database - you need to either 816 I cannot match your message to a node in the database - you need to either
810 supply a full designator (with number, eg "[issue123]") or keep the 817 supply a full designator (with number, eg "[issue123]") or keep the
1087 encrypted.""")) 1094 encrypted."""))
1088 1095
1089 def get_content_and_attachments(self): 1096 def get_content_and_attachments(self):
1090 ''' get the attachments and first text part from the message 1097 ''' get the attachments and first text part from the message
1091 ''' 1098 '''
1092 from roundup.dehtml import dehtml
1093 html2text = dehtml(self.config['MAILGW_CONVERT_HTMLTOTEXT']).html2text 1099 html2text = dehtml(self.config['MAILGW_CONVERT_HTMLTOTEXT']).html2text
1094 1100
1095 ig = self.config.MAILGW_IGNORE_ALTERNATIVES 1101 ig = self.config.MAILGW_IGNORE_ALTERNATIVES
1096 self.message.instance = self.mailgw.instance 1102 self.message.instance = self.mailgw.instance
1097 self.content, self.attachments, html_part = \ 1103 self.content, self.attachments, html_part = \
1316 # To override the message parsing, derive your own class from 1322 # To override the message parsing, derive your own class from
1317 # parsedMessage and assign to parsed_message_class in a derived 1323 # parsedMessage and assign to parsed_message_class in a derived
1318 # class of MailGW 1324 # class of MailGW
1319 parsed_message_class = parsedMessage 1325 parsed_message_class = parsedMessage
1320 1326
1321 def __init__(self, instance, arguments=()): 1327 def __init__(self, instance, arguments):
1322 self.instance = instance 1328 self.instance = instance
1323 self.arguments = arguments 1329 self.arguments = arguments
1324 self.default_class = None 1330 self.default_class = self.arguments.default_class.strip()
1325 for option, value in self.arguments: 1331 if not self.default_class:
1326 if option == '-c': 1332 self.default_class = instance.config ['MAILGW_DEFAULT_CLASS']
1327 self.default_class = value.strip() 1333 self.props_by_class = {}
1334 self.parse_set_value()
1328 1335
1329 self.mailer = Mailer(instance.config) 1336 self.mailer = Mailer(instance.config)
1330 self.logger = logging.getLogger('roundup.mailgw') 1337 self.logger = logging.getLogger('roundup.mailgw')
1331 1338
1332 # should we trap exceptions (normal usage) or pass them through 1339 # should we trap exceptions (normal usage) or pass them through
1333 # (for testing) 1340 # (for testing)
1334 self.trapExceptions = 1 1341 self.trapExceptions = 1
1342
1343 def parse_set_value(self):
1344 """ Parse properties given with '-S' or --set-value option on
1345 command line
1346 """
1347 errors = []
1348 for v in self.arguments.set_value:
1349 try:
1350 n, r = v.split('=', 1)
1351 except ValueError:
1352 errors.append ('"%s" is not of the form "property=value"' % v)
1353 break
1354 try:
1355 classname, rv = n.split('.', 1)
1356 r = '='.join((rv, r))
1357 except ValueError:
1358 classname = 'msg'
1359 r = v
1360 if classname not in self.props_by_class:
1361 # We can only check later if this is a valid class
1362 self.props_by_class [classname] = []
1363 self.props_by_class [classname].append (r)
1364 if errors:
1365 mailadmin = self.instance.config['ADMIN_EMAIL']
1366 raise MailUsageError(_("""
1367 The mail gateway is not properly set up. Please contact
1368 %(mailadmin)s and have them fix the incorrect properties:
1369 %(errors)s
1370 """) % locals())
1335 1371
1336 def do_pipe(self): 1372 def do_pipe(self):
1337 """ Read a message from standard input and pass it to the mail handler. 1373 """ Read a message from standard input and pass it to the mail handler.
1338 1374
1339 Read into an internal structure that we can seek on (in case 1375 Read into an internal structure that we can seek on (in case
1353 1389
1354 def do_mailbox(self, filename): 1390 def do_mailbox(self, filename):
1355 """ Read a series of messages from the specified unix mailbox file and 1391 """ Read a series of messages from the specified unix mailbox file and
1356 pass each to the mail handler. 1392 pass each to the mail handler.
1357 """ 1393 """
1358 import mailbox
1359
1360 class mboxRoundupMessage(mailbox.mboxMessage, RoundupMessage): 1394 class mboxRoundupMessage(mailbox.mboxMessage, RoundupMessage):
1361 pass 1395 pass
1362 1396
1363 # The mailbox class constructs email.message.Message objects 1397 # The mailbox class constructs email.message.Message objects
1364 # using various email.message_from_* methods, without allowing 1398 # using various email.message_from_* methods, without allowing
1398 for method in orig_methods: 1432 for method in orig_methods:
1399 setattr(email, method, orig) 1433 setattr(email, method, orig)
1400 1434
1401 return 0 1435 return 0
1402 1436
1403 def do_imap(self, server, user='', password='', mailbox='', ssl=0, cram=0): 1437 def get_oauth_tokens(self, oauth_path):
1438 if not oauth_path:
1439 oauth_path = self.instance.config ['TRACKER_HOME']
1440 oauth_path = os.path.join(oauth_path, 'oauth')
1441 self.oauth_path = oauth_path
1442 with open(os.path.join(oauth_path, 'access_token'), 'r') as f:
1443 self.access_token = f.read().strip()
1444 with open(os.path.join(oauth_path, 'refresh_token'), 'r') as f:
1445 self.refresh_token = f.read().strip()
1446
1447 def write_token (self, tokenname):
1448 n = os.path.join(self.oauth_path, tokenname)
1449 tmp = n + '.tmp'
1450 old = n + '.old'
1451 with open(tmp, 'w') as f:
1452 f.write(getattr(self, tokenname))
1453 try:
1454 os.remove(old)
1455 except OSError:
1456 pass
1457 os.rename(n, old)
1458 os.rename(tmp, n)
1459
1460 def renew_oauth_tokens(self):
1461 """ Get new token(s) via refresh token
1462 """
1463 with open(os.path.join(self.oauth_path, 'client_secret'), 'r') as f:
1464 client_secret = f.read().strip()
1465 data = dict \
1466 ( client_id = self.oauth_client_id
1467 , client_secret = client_secret
1468 , refresh_token = self.refresh_token
1469 , grant_type = 'refresh_token'
1470 )
1471 session = requests.session()
1472 r = session.post(self.token_endpoint, data = data)
1473 if not 200 <= r.status_code <= 299:
1474 raise RuntimeError('Invalid get result: %s: %s\n %s'
1475 %(r.status_code, r.reason, r.text))
1476 d = r.json()
1477 if d ['refresh_token'] != self.refresh_token:
1478 self.refresh_token = d ['refresh_token']
1479 self.write_token ('refresh_token')
1480 if d ['access_token'] != self.access_token:
1481 self.access_token = d ['access_token']
1482 self.write_token ('access_token')
1483
1484 def do_imap(self, server, user='', password='', mailbox='', **kw):
1404 ''' Do an IMAP connection 1485 ''' Do an IMAP connection
1405 ''' 1486 '''
1406 import getpass, imaplib, socket # noqa: E401
1407 try: 1487 try:
1408 if not user: 1488 if not user:
1409 user = my_input('User: ') 1489 user = my_input('User: ')
1410 if not password: 1490 if not password and not kw.get('oauth'):
1411 password = getpass.getpass() 1491 password = getpass.getpass()
1412 except (KeyboardInterrupt, EOFError): 1492 except (KeyboardInterrupt, EOFError):
1413 # Ctrl C or D maybe also Ctrl Z under Windows. 1493 # Ctrl C or D maybe also Ctrl Z under Windows.
1414 print("\nAborted by user.") 1494 print("\nAborted by user.")
1415 return 1 1495 return 1
1416 # open a connection to the server and retrieve all messages 1496 # open a connection to the server and retrieve all messages
1417 try: 1497 try:
1418 if ssl: 1498 if kw.get('ssl'):
1419 self.logger.debug('Trying server %r with ssl' % server) 1499 self.logger.debug('Trying server %r with ssl' % server)
1420 server = imaplib.IMAP4_SSL(server) 1500 server = imaplib.IMAP4_SSL(server)
1421 else: 1501 else:
1422 self.logger.debug('Trying server %r without ssl' % server) 1502 self.logger.debug('Trying server %r without ssl' % server)
1423 server = imaplib.IMAP4(server) 1503 server = imaplib.IMAP4(server)
1424 except (imaplib.IMAP4.error, socket.error, ssl_.SSLError): 1504 except (imaplib.IMAP4.error, socket.error, ssl_.SSLError):
1425 self.logger.exception('IMAP server error') 1505 self.logger.exception('IMAP server error')
1426 return 1 1506 return 1
1427 1507
1428 try: 1508 if kw.get('oauth'):
1429 if cram: 1509 if requests is None:
1430 server.login_cram_md5(user, password) 1510 self.logger.error('For OAUTH, the requests library '
1431 else: 1511 'must be installed')
1432 server.login(user, password) 1512 return 1
1433 except imaplib.IMAP4.error as e: 1513 self.get_oauth_tokens(kw.get('oauth_path'))
1434 self.logger.exception('IMAP login failure: %s' % e) 1514 # The following are mandatory for oauth and are passed by
1435 return 1 1515 # the command-line handler:
1516 self.token_endpoint = kw ['token_endpoint']
1517 self.oauth_client_id = kw ['oauth_client_id']
1518 for k in range(2):
1519 t = self.access_token
1520 s = 'user=%s\1auth=Bearer %s\1\1' % (user, t)
1521 # Try twice, access token may be too old
1522 try:
1523 server.authenticate('XOAUTH2', lambda x: s)
1524 break
1525 except imaplib.IMAP4.error:
1526 if k:
1527 self.logger.exception('OAUTH authentication failed')
1528 return 1
1529 try:
1530 self.renew_oauth_tokens()
1531 except RuntimeError:
1532 self.logger.exception('OAUTH token renew failed')
1533 return 1
1534 else:
1535 try:
1536 if kw.get('cram'):
1537 server.login_cram_md5(user, password)
1538 else:
1539 server.login(user, password)
1540 except imaplib.IMAP4.error as e:
1541 self.logger.exception('IMAP login failure: %s' % e)
1542 return 1
1436 1543
1437 try: 1544 try:
1438 if not mailbox: 1545 if not mailbox:
1439 (typ, data) = server.select() 1546 (typ, data) = server.select()
1440 else: 1547 else:
1480 self._do_pop(server, user, password, False, ssl) 1587 self._do_pop(server, user, password, False, ssl)
1481 1588
1482 def _do_pop(self, server, user, password, apop, ssl): 1589 def _do_pop(self, server, user, password, apop, ssl):
1483 '''Read a series of messages from the specified POP server. 1590 '''Read a series of messages from the specified POP server.
1484 ''' 1591 '''
1485 import getpass, poplib, socket # noqa: E401
1486 # Monkey-patch poplib to have a large line-limit 1592 # Monkey-patch poplib to have a large line-limit
1487 # Seems that in python2.7 poplib applies a line-length limit not 1593 # Seems that in python2.7 poplib applies a line-length limit not
1488 # just to the lines that take care of the pop3 protocol but also 1594 # just to the lines that take care of the pop3 protocol but also
1489 # to all email content 1595 # to all email content
1490 # See, e.g., 1596 # See, e.g.,
1698 - 'file' specifies a file-type class 1804 - 'file' specifies a file-type class
1699 - 'msg' is the message-class 1805 - 'msg' is the message-class
1700 classname - the name of the current issue-type class 1806 classname - the name of the current issue-type class
1701 1807
1702 Parse the commandline arguments and retrieve the properties that 1808 Parse the commandline arguments and retrieve the properties that
1703 are relevant to the class_type. We now allow multiple -S options 1809 are relevant to the class_type.
1704 per class_type (-C option).
1705 ''' 1810 '''
1706 allprops = {} 1811 allprops = {}
1707 1812
1708 classname = classname or class_type 1813 classname = classname or class_type
1709 cls_lookup = {'issue': classname} 1814
1710 1815 # check if the classname is valid
1711 # Allow other issue-type classes -- take the real classname from
1712 # previous parsing-steps of the message:
1713 clsname = cls_lookup.get(class_type, class_type)
1714
1715 # check if the clsname is valid
1716 try: 1816 try:
1717 self.db.getclass(clsname) 1817 cls = self.db.getclass(classname)
1718 except KeyError: 1818 except KeyError:
1719 mailadmin = self.instance.config['ADMIN_EMAIL'] 1819 mailadmin = self.instance.config['ADMIN_EMAIL']
1720 raise MailUsageError(_(""" 1820 raise MailUsageError(_("""
1721 The mail gateway is not properly set up. Please contact 1821 The mail gateway is not properly set up. Please contact
1722 %(mailadmin)s and have them fix the incorrect class specified as: 1822 %(mailadmin)s and have them fix the incorrect class specified as:
1723 %(clsname)s 1823 %(classname)s
1724 """) % locals()) 1824 """) % locals())
1725 1825
1726 if self.arguments: 1826 if classname not in self.props_by_class:
1727 # The default type on the commandline is msg 1827 return {}
1728 if class_type == 'msg': 1828 for propstring in self.props_by_class[classname]:
1729 current_type = class_type 1829 errors, props = setPropArrayFromString(self, cls,
1730 else: 1830 propstring.strip())
1731 current_type = None 1831
1732 1832
1733 # Handle the arguments specified by the email gateway command line. 1833 if errors:
1734 # We do this by looping over the list of self.arguments looking for 1834 mailadmin = self.instance.config['ADMIN_EMAIL']
1735 # a -C to match the class we want, then use the -S setting string. 1835 raise MailUsageError(_("""
1736 for option, propstring in self.arguments:
1737 if option in ('-C', '--class'):
1738 current_type = propstring.strip()
1739
1740 if current_type != class_type:
1741 current_type = None
1742
1743 elif current_type and option in ('-S', '--set'):
1744 cls = cls_lookup.get(current_type, current_type)
1745 temp_cl = self.db.getclass(cls)
1746 errors, props = setPropArrayFromString(self,
1747 temp_cl,
1748 propstring.strip())
1749
1750 if errors:
1751 mailadmin = self.instance.config['ADMIN_EMAIL']
1752 raise MailUsageError(_("""
1753 The mail gateway is not properly set up. Please contact 1836 The mail gateway is not properly set up. Please contact
1754 %(mailadmin)s and have them fix the incorrect properties: 1837 %(mailadmin)s and have them fix the incorrect properties:
1755 %(errors)s 1838 %(errors)s
1756 """) % locals()) 1839 """) % locals())
1757 allprops.update(props) 1840 allprops.update(props)
1758 1841
1759 return allprops 1842 return allprops
1760 1843
1761 1844
1762 def setPropArrayFromString(self, cl, propString, nodeid=None): 1845 def setPropArrayFromString(self, cl, propString, nodeid=None):
1999 if not keep_body: 2082 if not keep_body:
2000 content = '\n\n'.join(kept_lines) 2083 content = '\n\n'.join(kept_lines)
2001 2084
2002 return summary, content 2085 return summary, content
2003 2086
2004 # vim: set filetype=python sts=4 sw=4 et si : 2087 # vim: set filetype=python sts=4 sw=4 et :
Roundup Issue Tracker: http://roundup-tracker.org/