Mercurial > p > roundup > code
changeset 6099:55c56ceacb8e
escape HTML tags in markdown content
enabled fenced code blocks for markdown
allow mistune to be used as a markdown parser
test all installed markdown parsers and fallback code
| author | Christof Meerwald <cmeerw@cmeerw.org> |
|---|---|
| date | Mon, 24 Feb 2020 22:20:19 +0000 |
| parents | 72a281a55a17 |
| children | d4ce26b14cf5 |
| files | .travis.yml doc/installation.txt roundup/cgi/templating.py test/test_templating.py |
| diffstat | 4 files changed, 172 insertions(+), 60 deletions(-) [+] |
line wrap: on
line diff
--- a/.travis.yml Thu Feb 20 21:38:32 2020 -0500 +++ b/.travis.yml Mon Feb 24 22:20:19 2020 +0000 @@ -94,6 +94,8 @@ - pip install gpg pytz whoosh pyjwt - pip install pytest-cov codecov - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install docutils; fi + - if [[ $TRAVIS_PYTHON_VERSION != "3.4"* ]]; then pip install mistune; fi + - if [[ $TRAVIS_PYTHON_VERSION == "3.[5-9]"* ]]; then pip install Markdown; fi - pip install markdown2 before_script:
--- a/doc/installation.txt Thu Feb 20 21:38:32 2020 -0500 +++ b/doc/installation.txt Mon Feb 24 22:20:19 2020 +0000 @@ -120,9 +120,9 @@ To use ReStructuredText rendering you need to have the docutils package installed. -markdown or markdown2 - To use markdown rendering you need to either have the markdown or - markdown2 package installed. +markdown, markdown2 or mistune + To use markdown rendering you need to have the markdown, markdown2 + or mistune package installed. Windows Service You can run Roundup as a Windows service if pywin32_ is installed. @@ -139,6 +139,7 @@ .. _docutils: https://docutils.sourceforge.io/ .. _markdown: https://python-markdown.github.io/ .. _markdown2: https://github.com/trentm/python-markdown2 +.. _mistune: https://github.com/lepture/mistune Getting Roundup
--- a/roundup/cgi/templating.py Thu Feb 20 21:38:32 2020 -0500 +++ b/roundup/cgi/templating.py Mon Feb 24 22:20:19 2020 +0000 @@ -54,17 +54,50 @@ except ImportError: ReStructuredText = None try: - from markdown2 import markdown -except ImportError: - try: - from markdown import markdown - except ImportError: - markdown = None -try: from itertools import zip_longest except ImportError: from itertools import izip_longest as zip_longest +def _import_markdown2(): + try: + import markdown2, re + class Markdown(markdown2.Markdown): + # don't restrict protocols in links + _safe_protocols = re.compile('', re.IGNORECASE) + + markdown = lambda s: Markdown(safe_mode='escape', extras={ 'fenced-code-blocks' : True }).convert(s) + except ImportError: + markdown = None + + return markdown + +def _import_markdown(): + try: + from markdown import markdown as markdown_impl + from markdown.extensions import Extension as MarkdownExtension + + # make sure any HTML tags get escaped + class EscapeHtml(MarkdownExtension): + def extendMarkdown(self, md): + md.preprocessors.deregister('html_block') + md.inlinePatterns.deregister('html') + + markdown = lambda s: markdown_impl(s, extensions=[EscapeHtml(), 'fenced_code']) + except ImportError: + markdown = None + + return markdown + +def _import_mistune(): + try: + from mistune import markdown + except ImportError: + markdown = None + + return markdown + +markdown = _import_markdown2() or _import_markdown() or _import_mistune() + # bring in the templating support from roundup.cgi import TranslationService, ZTUtils
--- a/test/test_templating.py Thu Feb 20 21:38:32 2020 -0500 +++ b/test/test_templating.py Mon Feb 24 22:20:19 2020 +0000 @@ -9,36 +9,36 @@ import pytest from .pytest_patcher import mark_class -try: - from docutils.core import publish_parts as ReStructuredText +if ReStructuredText: skip_rst = lambda func, *args, **kwargs: func -except ImportError: - ReStructuredText = None +else: skip_rst = mark_class(pytest.mark.skip( reason='ReStructuredText not available')) -try: - from StructuredText.StructuredText import HTML as StructuredText +if StructuredText: skip_stext = lambda func, *args, **kwargs: func -except ImportError: - try: # older version - import StructuredText - skip_stext = lambda func, *args, **kwargs: func - except ImportError: - StructuredText = None - skip_stext = mark_class(pytest.mark.skip( - reason='StructuredText not available')) +else: + skip_stext = mark_class(pytest.mark.skip( + reason='StructuredText not available')) + +import roundup.cgi.templating +if roundup.cgi.templating._import_mistune(): + skip_mistune = lambda func, *args, **kwargs: func +else: + skip_mistune = mark_class(pytest.mark.skip( + reason='mistune not available')) -try: +if roundup.cgi.templating._import_markdown2(): + skip_markdown2 = lambda func, *args, **kwargs: func +else: + skip_markdown2 = mark_class(pytest.mark.skip( + reason='markdown2 not available')) + +if roundup.cgi.templating._import_markdown(): skip_markdown = lambda func, *args, **kwargs: func - from markdown2 import markdown -except ImportError: - try: - from markdown import markdown - except ImportError: - markdown = None - skip_markdown = mark_class(pytest.mark.skip( - reason='markdown not available')) +else: + skip_markdown = mark_class(pytest.mark.skip( + reason='markdown not available')) from roundup.anypy.strings import u2s, s2u @@ -247,21 +247,7 @@ self.assertEqual(p.hyperlinked(), 'A string <b> with <a href="mailto:rouilj@example.com">rouilj@example.com</a> embedded &lt; html</b>') - @skip_markdown - def test_string_markdown_installed(self): - pass # just so we have a record of a skipped test - - def test_string_markdown(self): - p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with cmeerw@example.com *embedded* \u00df')) - if markdown: - self.assertEqual(p.markdown().strip(), u2s(u'<p>A string <a href="http://localhost">http://localhost</a> with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> <em>embedded</em> \u00df</p>')) - else: - self.assertEqual(p.markdown(), u2s(u'A string <a href="http://localhost" rel="nofollow noopener">http://localhost</a> with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> *embedded* \u00df')) - @skip_rst - def test_string_rst_installed(self): - pass # just so we have a record of a skipped test - def test_string_rst(self): p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with cmeerw@example.com *embedded* \u00df')) @@ -295,23 +281,14 @@ </div> </div> ''' - if ReStructuredText: - self.assertEqual(p.rst(), u2s(u'<div class="document">\n<p>A string with <a class="reference external" href="mailto:cmeerw@example.com">cmeerw@example.com</a> <em>embedded</em> \u00df</p>\n</div>\n')) - self.assertEqual(q.rst(), u2s(q_result)) - self.assertEqual(r.rst(), u2s(r_result)) - else: - self.assertEqual(p.rst(), u2s(u'A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> *embedded* \u00df')) + self.assertEqual(p.rst(), u2s(u'<div class="document">\n<p>A string with <a class="reference external" href="mailto:cmeerw@example.com">cmeerw@example.com</a> <em>embedded</em> \u00df</p>\n</div>\n')) + self.assertEqual(q.rst(), u2s(q_result)) + self.assertEqual(r.rst(), u2s(r_result)) @skip_stext - def test_string_stext_installed(self): - pass # just so we have a record of a skipped test - def test_string_stext(self): p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with cmeerw@example.com *embedded* \u00df')) - if StructuredText: - self.assertEqual(p.stext(), u2s(u'<p>A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> <em>embedded</em> \u00df</p>\n')) - else: - self.assertEqual(p.stext(), u2s(u'A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> *embedded* \u00df')) + self.assertEqual(p.stext(), u2s(u'<p>A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> <em>embedded</em> \u00df</p>\n')) def test_string_field(self): p = StringHTMLProperty(self.client, 'test', '1', None, 'test', 'A string <b> with rouilj@example.com embedded < html</b>') @@ -436,6 +413,105 @@ input=input_xhtml(**attrs) self.assertEqual(input, '<input class="required" disabled="disabled" size="30" type="text"/>') +# common markdown test cases +class MarkdownTests: + def test_string_markdown(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with cmeerw@example.com <br> *embedded* \u00df')) + self.assertEqual(p.markdown().strip(), u2s(u'<p>A string <a href="http://localhost">http://localhost</a> with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> <br> <em>embedded</em> \u00df</p>')) + + def test_string_markdown_code_block(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'embedded code block\n\n```\nline 1\nline 2\n```\n\nnew paragraph')) + self.assertEqual(p.markdown().strip().replace('\n\n', '\n'), u2s(u'<p>embedded code block</p>\n<pre><code>line 1\nline 2\n</code></pre>\n<p>new paragraph</p>')) + +@skip_mistune +class MistuneTestCase(TemplatingTestCase, MarkdownTests) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__markdown = templating.markdown + templating.markdown = templating._import_mistune() + + def tearDown(self): + from roundup.cgi import templating + templating.markdown = self.__markdown + +@skip_markdown2 +class Markdown2TestCase(TemplatingTestCase, MarkdownTests) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__markdown = templating.markdown + templating.markdown = templating._import_markdown2() + + def tearDown(self): + from roundup.cgi import templating + templating.markdown = self.__markdown + +@skip_markdown +class MarkdownTestCase(TemplatingTestCase, MarkdownTests) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__markdown = templating.markdown + templating.markdown = templating._import_markdown() + + def tearDown(self): + from roundup.cgi import templating + templating.markdown = self.__markdown + + +class NoMarkdownTestCase(TemplatingTestCase) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__markdown = templating.markdown + templating.markdown = None + + def tearDown(self): + from roundup.cgi import templating + templating.markdown = self.__markdown + + def test_string_markdown(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string http://localhost with cmeerw@example.com <br> *embedded* \u00df')) + self.assertEqual(p.markdown(), u2s(u'A string <a href="http://localhost" rel="nofollow noopener">http://localhost</a> with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> <br> *embedded* \u00df')) + +class NoRstTestCase(TemplatingTestCase) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__ReStructuredText = templating.ReStructuredText + templating.ReStructuredText = None + + def tearDown(self): + from roundup.cgi import templating + templating.ReStructuredText = self.__ReStructuredText + + def test_string_rst(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with cmeerw@example.com *embedded* \u00df')) + self.assertEqual(p.rst(), u2s(u'A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> *embedded* \u00df')) + +class NoStextTestCase(TemplatingTestCase) : + def setUp(self): + TemplatingTestCase.setUp(self) + + from roundup.cgi import templating + self.__StructuredText = templating.StructuredText + templating.StructuredText = None + + def tearDown(self): + from roundup.cgi import templating + templating.StructuredText = self.__StructuredText + + def test_string_stext(self): + p = StringHTMLProperty(self.client, 'test', '1', None, 'test', u2s(u'A string with cmeerw@example.com *embedded* \u00df')) + self.assertEqual(p.stext(), u2s(u'A string with <a href="mailto:cmeerw@example.com">cmeerw@example.com</a> *embedded* \u00df')) + + r''' class HTMLPermissions: def is_edit_ok(self):