# HG changeset patch # User John Rouillard # Date 1765770046 18000 # Node ID 520075b29474aa5dd2586b2e9393ae95b47b0911 # Parent 918792e35e0c8f2ef1529810a854f898c335a650 feat: support justhtml parsing library to convert email to plain text justhtml is an pure python, fast, HTML5 compliant parser. It is now an option for converting html only emails to plain text. Its output format differs slightly from dehtml or beautifulsoup. Mostly by removing extra blank lines. dehtml.py: Using the stream parser of justhtml. Unable to get the full document parser to successfully strip script and style blocks. If I can fix this and use the standard parser, I can in theory generate markdown from the DOM tree generated by justhtml. Updated test case to include inline elements that should not cause a line break when they are encountered. Running dehtml as: `python roundup/dehtml.py foo.html` will load foo.html and parse it using all available parsers. configuration.py: justhtml is available as an option. docs: updated CHANGES.txt, doc/tracker_config.txt added beautifulsoup and justhtml to the optional software section of doc/installtion.txt. test_mailgw.py, .github/workflows/ci-test Updated tests and install justhtml as part of CI. diff -r 918792e35e0c -r 520075b29474 .github/workflows/ci-test.yml --- a/.github/workflows/ci-test.yml Sat Dec 13 23:02:53 2025 -0500 +++ b/.github/workflows/ci-test.yml Sun Dec 14 22:40:46 2025 -0500 @@ -240,7 +240,7 @@ # pygments for markdown2 to highlight code blocks pip install markdown2 pygments # docutils for ReStructuredText - pip install beautifulsoup4 brotli docutils jinja2 \ + pip install beautifulsoup4 justhtml brotli docutils jinja2 \ mistune==0.8.4 pyjwt pytz whoosh # gpg on PyPi is currently broken with newer OS platform # ubuntu 24.04 diff -r 918792e35e0c -r 520075b29474 CHANGES.txt --- a/CHANGES.txt Sat Dec 13 23:02:53 2025 -0500 +++ b/CHANGES.txt Sun Dec 14 22:40:46 2025 -0500 @@ -64,6 +64,10 @@ config.ini. (John Rouillard) - issue2551152 - added basic PGP setup/use info to admin_guide. (John Rouillard) +- add support for the 'justhtml' html 5 parser library. It is written + in pure Python. Used to convert html emails into plain text. Faster + then beautifulsoup4 and it passes the html 5 standard browser test + suite. Beautifulsoup is still supported. (John Rouillard) 2025-07-13 2.5.0 diff -r 918792e35e0c -r 520075b29474 doc/installation.txt --- a/doc/installation.txt Sat Dec 13 23:02:53 2025 -0500 +++ b/doc/installation.txt Sun Dec 14 22:40:46 2025 -0500 @@ -311,6 +311,14 @@ roundup-gettext, you must install polib_. See the `developer's guide`_ for details on translating your tracker. +beautifulsoup, justhtml + When HTML only email is received, Roundup can convert it into + plain text using the native dehtml parser. To convert HTML + email into plain text, beautifulsoup4_ or justhtml_ can also be + used. You can choose the converter in the tracker's + config. Note that justhtml is pure Python, fast and conforms to + HTML 5 standards. + pywin32 - Windows Service You can run Roundup as a Windows service if pywin32_ is installed. Otherwise it must be started manually. @@ -2423,6 +2431,7 @@ .. _`adding MySQL users`: https://dev.mysql.com/doc/refman/8.0/en/creating-accounts.html .. _apache: https://httpd.apache.org/ +.. _beautifulsoup4: https://pypi.org/project/beautifulsoup4/ .. _brotli: https://pypi.org/project/Brotli/ .. _`developer's guide`: developers.html .. _defusedxml: https://pypi.org/project/defusedxml/ @@ -2430,6 +2439,7 @@ .. _flup: https://pypi.org/project/flup/ .. _gpg: https://www.gnupg.org/software/gpgme/index.html .. _jinja2: https://palletsprojects.com/projects/jinja/ +.. _justhtml: https://pypi.org/project/justhtml/ .. _markdown: https://python-markdown.github.io/ .. _markdown2: https://github.com/trentm/python-markdown2 .. _mistune: https://pypi.org/project/mistune/ diff -r 918792e35e0c -r 520075b29474 doc/tracker_config.txt --- a/doc/tracker_config.txt Sat Dec 13 23:02:53 2025 -0500 +++ b/doc/tracker_config.txt Sun Dec 14 22:40:46 2025 -0500 @@ -1112,12 +1112,12 @@ # If an email has only text/html parts, use this module # to convert the html to text. Choose from beautifulsoup 4, - # dehtml - (internal code), or none to disable conversion. - # If 'none' is selected, email without a text/plain part - # will be returned to the user with a message. If + # justhtml, dehtml - (internal code), or none to disable + # conversion. If 'none' is selected, email without a text/plain + # part will be returned to the user with a message. If # beautifulsoup is selected but not installed dehtml will # be used instead. - # Allowed values: beautifulsoup, dehtml, none + # Allowed values: beautifulsoup, justhtml, dehtml, none # Default: none convert_htmltotext = none diff -r 918792e35e0c -r 520075b29474 roundup/configuration.py --- a/roundup/configuration.py Sat Dec 13 23:02:53 2025 -0500 +++ b/roundup/configuration.py Sun Dec 14 22:40:46 2025 -0500 @@ -384,17 +384,17 @@ """What module should be used to convert emails with only text/html parts into text for display in roundup. Choose from beautifulsoup - 4, dehtml - the internal code or none to disable html to text - conversion. If beautifulsoup chosen but not available, dehtml will - be used. + 4, justhtml, dehtml - the internal code or none to disable html to + text conversion. If beautifulsoup or justhtml is chosen but not + available, dehtml will be used. """ - class_description = "Allowed values: beautifulsoup, dehtml, none" + class_description = "Allowed values: beautifulsoup, justhtml, dehtml, none" def str2value(self, value): _val = value.lower() - if _val in ("beautifulsoup", "dehtml", "none"): + if _val in ("beautifulsoup", "justhtml", "dehtml", "none"): return _val else: raise OptionValueError(self, value, self.class_description) @@ -1811,11 +1811,11 @@ (HtmlToTextOption, "convert_htmltotext", "none", "If an email has only text/html parts, use this module\n" "to convert the html to text. Choose from beautifulsoup 4,\n" - "dehtml - (internal code), or none to disable conversion.\n" - "If 'none' is selected, email without a text/plain part\n" - "will be returned to the user with a message. If\n" - "beautifulsoup is selected but not installed dehtml will\n" - "be used instead."), + "justhtml, dehtml - (internal code), or none to disable\n" + "conversion. If 'none' is selected, email without a text/plain\n" + "part will be returned to the user with a message. If\n" + "beautifulsoup or justhtml is selected but not installed\n" + "dehtml will be used instead."), (BooleanOption, "keep_real_from", "no", "When handling emails ignore the Resent-From:-header\n" "and use the original senders From:-header instead.\n" diff -r 918792e35e0c -r 520075b29474 roundup/dehtml.py --- a/roundup/dehtml.py Sat Dec 13 23:02:53 2025 -0500 +++ b/roundup/dehtml.py Sun Dec 14 22:40:46 2025 -0500 @@ -5,6 +5,10 @@ from roundup.anypy.strings import u2s, uchr +# ruff PLC0415 ignore imports not at top of file +# ruff RET505 ignore else after return +# ruff: noqa: PLC0415 RET505 + _pyver = sys.version_info[0] @@ -29,6 +33,108 @@ return u2s(soup.get_text("\n", strip=True)) self.html2text = html2text + elif converter == "justhtml": + from justhtml import stream + + def html2text(html): + # The below does not work. + # Using stream parser since I couldn't seem to strip + # 'script' and 'style' blocks. But stream doesn't + # have error reporting or stripping of text nodes + # and dropping empty nodes. Also I would like to try + # its GFM markdown output too even though it keeps + # tables as html and doesn't completely covert as + # this would work well for those supporting markdown. + # + # ctx used for for testing since I have a truncated + # test doc. It eliminates error from missing DOCTYPE + # and head. + # + #from justhtml import JustHTML + # from justhtml.context import FragmentContext + # + #ctx = FragmentContext('html') + #justhtml = JustHTML(html,collect_errors=True, + # fragment_context=ctx) + # I still have the text output inside style/script tags. + # with :not(style, script). I do get text contents + # with query("style, script"). + # + #return u2s("\n".join( + # [elem.to_text(separator="\n", strip=True) + # for elem in justhtml.query(":not(style, script)")]) + # ) + + # define inline elements so I can accumulate all unbroken + # text in a single line with embedded inline elements. + # 'br' is inline but should be treated it as a line break + # and element before/after should not be accumulated + # together. + inline_elements = ( + "a", + "address", + "b", + "cite", + "code", + "em", + "i", + "img", + "mark", + "q", + "s", + "small", + "span", + "strong", + "sub", + "sup", + "time") + + # each line is appended and joined at the end + text = [] + # the accumulator for all text in inline elements + text_accumulator = "" + # if set skip all lines till matching end tag found + # used to skip script/style blocks + skip_till_endtag = None + # used to force text_accumulator into text with added + # newline so we have a blank line between paragraphs. + _need_parabreak = False + + for event, data in stream(html): + if event == "end" and skip_till_endtag == data: + skip_till_endtag = None + continue + if skip_till_endtag: + continue + if (event == "start" and + data[0] in ('script', 'style')): + skip_till_endtag = data[0] + continue + if (event == "start" and + text_accumulator and + data[0] not in inline_elements): + # add accumulator to "text" + text.append(text_accumulator) + text_accumulator = "" + _need_parabreak = False + elif event == "text": + if not data.isspace(): + text_accumulator = text_accumulator + data + _need_parabreak = True + elif (_need_parabreak and + event == "start" and + data[0] == "p"): + text.append(text_accumulator + "\n") + text_accumulator = "" + _need_parabreak = False + + # save anything left in the accumulator at end of document + if text_accumulator: + # add newline to match dehtml and beautifulsoup + text.append(text_accumulator + "\n") + return u2s("\n".join(text)) + + self.html2text = html2text else: raise ImportError except ImportError: @@ -96,6 +202,16 @@ if __name__ == "__main__": + # ruff: noqa: B011 S101 + + try: + assert False + except AssertionError: + pass + else: + print("Error, assertions turned off. Test fails") + sys.exit(1) + html = """