--- a/CHANGES.txt	Mon Mar 25 22:33:24 2024 -0400
+++ b/CHANGES.txt	Tue Mar 26 14:15:46 2024 -0400
@@ -146,6 +146,13 @@
 - issue2551212 - wsgi performance improvement feature added in 2.2.0
   is active by default. Can be turned off if needed. See upgrading.txt
   for info. (John Rouillard)
+- issue2551270 - Better templating support for JavaScript. Add
+  utils.readfile(file, optional=False) and utils.expandfile(file,
+  token_dict=None, optional=False). Allows reading an external file
+  (e.g. JavaScript) and inserting it using tal:contents or equivalent
+  jinja function. expandfile allows setting a dictionary and tokens in
+  the file of the form "%(token_name)s" will be replaced in the file
+  with the values from the dict. (John Rouillard)
 
 2023-07-13 2.3.0
 

--- a/doc/reference.txt	Mon Mar 25 22:33:24 2024 -0400
+++ b/doc/reference.txt	Tue Mar 26 14:15:46 2024 -0400
@@ -3596,14 +3596,37 @@
   Method          Description
   =============== ========================================================
   Batch           return a batch object using the supplied list
-  url_quote       quote some text as safe for a URL (ie. space, %, ...)
+  anti_csrf_nonce returns the random nonce generated for this session
+  expandfile      load a file into a template and expand
+                  '%(tokenname)s' in the file using
+                  values from the supplied dictionary.
   html_quote      quote some text as safe in HTML (ie. <, >, ...)
   html_calendar   renders an HTML calendar used by the
 		  ``_generic.calendar.html`` template (itself invoked by
 		  the popupCalendar DateHTMLProperty method
-  anti_csrf_nonce returns the random nonce generated for this session
+  readfile        read Javascript or other content in an external
+                  file into the template.
+  url_quote       quote some text as safe for a URL (ie. space, %, ...)
   =============== ========================================================
 
+Additional info can be obtained by starting ``python`` with the
+``roundup`` subdirectory on your PYTHONPATH and using the Python help
+function like::
+
+   >>> from roundup.cgi.templating import TemplatingUtils
+   >>> help(TemplatingUtils.readfile)
+   Help on function readfile in module roundup.cgi.templating:
+
+   readfile(self, name, optional=False)
+       Read an file in the template directory.
+
+       Used to inline file content into a template.  If file
+       is not found in template directory, reports an error
+       to the user unless optional=True. Then it returns an
+       empty string. Useful inlining JavaScript kept in an
+       external file where you can use linters/minifiers and
+
+(Note: ``>>>``` is the Python REPL prompt. Don't type the ``>>>```.)
 
 Batching
 ::::::::

--- a/roundup/cgi/templating.py	Mon Mar 25 22:33:24 2024 -0400
+++ b/roundup/cgi/templating.py	Tue Mar 26 14:15:46 2024 -0400
@@ -21,6 +21,7 @@
 
 import calendar
 import csv
+import logging
 import os.path
 import re
 import textwrap
@@ -52,6 +53,7 @@
 except ImportError:
     from itertools import izip_longest as zip_longest
 
+logger = logging.getLogger('roundup.template')
 
 # List of schemes that are not rendered as links in rst and markdown.
 _disable_url_schemes = ['javascript', 'data']
@@ -329,9 +331,10 @@
             src = os.path.join(realsrc, f)
             realpath = os.path.realpath(src)
             if not realpath.startswith(realsrc):
-                return  # will raise invalid template
+                return None # will raise invalid template
             if os.path.exists(src):
                 return (src, f)
+        return None
 
     def check(self, name):
         return bool(self._find(name))
@@ -3569,6 +3572,7 @@
     """
     def __init__(self, client):
         self.client = client
+        self._ = self.client._
 
     def Batch(self, sequence, size, start, end=0, orphan=0, overlap=0):
         return Batch(self.client, sequence, size, start, end, orphan,
@@ -3619,7 +3623,7 @@
         display = request.form.getfirst("display", date_str)
         template = request.form.getfirst("@template", "calendar")
         form = request.form.getfirst("form")
-        property = request.form.getfirst("property")
+        aproperty = request.form.getfirst("property")
         curr_date = ""
         try:
             # date_str and display can be set to an invalid value
@@ -3656,7 +3660,7 @@
         res = []
 
         base_link = "%s?@template=%s&property=%s&form=%s&date=%s" % \
-                    (request.classname, template, property, form, curr_date)
+                    (request.classname, template, aproperty, form, curr_date)
 
         # navigation
         # month
@@ -3718,6 +3722,92 @@
         res.append('</table></td></tr></table>')
         return "\n".join(res)
 
+    def readfile(self, name, optional=False):
+        """Used to inline a file from the template directory.
+
+           Used to inline file content into a template. If file
+           is not found in the template directory and
+           optional=False, it reports an error to the user via a
+           NoTemplate exception. If optional=True it returns an
+           empty string when it can't find the file.
+
+           Useful for inlining JavaScript kept in an external
+           file where you can use linters/minifiers and other
+           tools on it.
+
+           A TAL example:
+
+             <script tal:attributes="nonce request/client/client_nonce"
+             tal:content="python:utils.readfile('mylibrary.js')"></script>
+
+           This method does not expands any tokens in the file.
+           See expandfile() for replacing tokens in the file.
+        """
+        file_result = self.client.instance.templates._find(name)
+
+        if file_result is None:
+            if optional:
+                return ""
+            template_name = self.client.selectTemplate(
+                self.client.classname, self.client.template)
+            raise NoTemplate(self._(
+                "Unable to read or expand file '%(name)s' "
+                "in template '%(template)s'.") % {
+                    "name": name, 'template': template_name})
+
+        fullpath, name = file_result
+        with open(fullpath) as f:
+            contents = f.read()
+        return contents
+
+    def expandfile(self, name, values=None, optional=False):
+        """Read a file and replace token placeholders.
+
+           Given a file name and a dict of tokens and
+           replacements, read the file from the tracker template
+           directory. Then replace all tokens of the form
+           '%(token_name)s' with the values in the dict. If the
+           values dict is set to None, it acts like
+           readfile(). In addition to values passed into the
+           method, the value for the tracker base directory taken
+           from TRACKER_WEB is available as the 'base' token. The
+           client_nonce used for Content Security Policy (CSP) is
+           available as 'client_nonce'.  If a token is not in the
+           dict, an empty string is returned and an error log
+           message is logged. See readfile for an usage example.
+        """
+        # readfile() raises NoTemplate if optional = false and
+        # the file is not found. Returns empty string if file not
+        # found and optional = true. File contents otherwise.
+        contents = self.readfile(name, optional=optional)
+
+        if values is None or not contents: # nothing to expand
+            return contents
+        tokens = {'base': self.client.db.config.TRACKER_WEB,
+                  'client_nonce': self.client.client_nonce}
+        tokens.update(values)
+        try:
+            return contents % tokens
+        except KeyError as e:
+            template_name = self.client.selectTemplate(
+                self.client.classname, self.client.template)
+            fullpath, name = self.client.instance.templates._find(name)
+            logger.error(
+                "When running expandfile('%(fullpath)s') in "
+                "'%(template)s' there was no value for token: '%(token)s'.",
+                {'fullpath': fullpath, 'token': e.args[0],
+                 'template': template_name})
+            return ""
+        except ValueError as e:
+            fullpath, name = self.client.instance.templates._find(name)
+            logger.error(self._(
+                "Found an incorrect token when expandfile applied "
+                "string subsitution on '%(fullpath)s'. "
+                "ValueError('%(issue)s') was raised. Check the format "
+                "of your named conversion specifiers."),
+                {'fullpath': fullpath, 'issue': e.args[0]})
+            return ""
+
 
 class MissingValue(object):
     def __init__(self, description, **kwargs):

--- a/test/test_cgi.py	Mon Mar 25 22:33:24 2024 -0400
+++ b/test/test_cgi.py	Tue Mar 26 14:15:46 2024 -0400
@@ -20,6 +20,7 @@
 from roundup.exceptions import UsageError, Reject
 from roundup.cgi.templating import HTMLItem, HTMLRequest, NoTemplate
 from roundup.cgi.templating import HTMLProperty, _HTMLItem, anti_csrf_nonce
+from roundup.cgi.templating import TemplatingUtils
 from roundup.cgi.form_parser import FormParser
 from roundup import init, instance, password, hyperdb, date
 from roundup.anypy.strings import u2s, b2s, s2b
@@ -2995,6 +2996,269 @@
         r = t.selectTemplate("user", "subdir/item")
         self.assertEqual("subdir/user.item", r)
 
+class TemplateUtilsTestCase(unittest.TestCase):
+    ''' Test various TemplateUtils
+    '''
+    def setUp(self):
+        self.dirname = '_test_template'
+        # set up and open a tracker
+        self.instance = setupTracker(self.dirname)
+
+        # open the database
+        self.db = self.instance.open('admin')
+        self.db.tx_Source = "web"
+        self.db.user.create(username='Chef', address='chef@bork.bork.bork',
+            realname='Bork, Chef', roles='User')
+        self.db.user.create(username='mary', address='mary@test.test',
+            roles='User', realname='Contrary, Mary')
+        self.db.post_init()
+
+    def tearDown(self):
+        self.db.close()
+        try:
+            shutil.rmtree(self.dirname)
+        except OSError as error:
+            if error.errno not in (errno.ENOENT, errno.ESRCH): raise
+
+    @pytest.fixture(autouse=True)
+    def inject_fixtures(self, caplog):
+        self._caplog = caplog
+
+    def testReadfile(self):
+        # create new files in html dir
+        testfiles = [
+            { "name": "file_to_read.js",
+              "content": ('hello world'),
+            },
+            { # for future test expanding TAL
+              "name": "_generic.readfile_success.html",
+              "content": (
+
+                  '''<span tal:content="python:utils.readfile('''
+                  """'example.js')"></span>""" ),
+            },
+        ]
+
+        for file_spec in testfiles:
+            file_path = "%s/html/%s" % (self.dirname, file_spec['name'])
+            with open(file_path, "w") as f:
+                f.write(file_spec['content'])
+
+        # get the client instance The form is needed to initialize,
+        # but not used since I call selectTemplate directly.
+        t = client.Client(self.instance, "user",
+                {'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
+         form=db_test_base.makeForm({"@template": "readfile_success"}))
+
+        tu = TemplatingUtils(t)
+        # testcase 1 - file exists
+        r = tu.readfile("file_to_read.js")
+        self.assertEqual(r, 'hello world')
+        r = None
+
+        # testcase 2 - file does not exist
+        with self.assertRaises(NoTemplate) as e:
+            r = tu.readfile("no_file_to_read.js")
+
+        self.assertEqual(
+            e.exception.args[0],
+            "Unable to read or expand file 'no_file_to_read.js' "
+            "in template 'home'.")
+        r = None
+
+        # testcase 3 - file does not exist - optional = True
+        r = tu.readfile("no_file_to_read.js", optional=True)
+        self.assertEqual(r, '')
+
+        # make sure a created template is found
+        # note that the extension is not included just the basename
+        self.assertEqual("_generic.readfile_success",
+                         t.selectTemplate("", "readfile_success"))
+
+
+        
+    def testExpandfile(self):
+        # test for templates in subdirectories
+
+        # make the directory
+        subdir = self.dirname + "/html/subdir"
+        os.mkdir(subdir)
+
+        # create new files in html dir
+        testfiles = [
+            { "name": "file_to_read.js",
+              "content": ('hello world'),
+            },
+            { "name": "file_no_content.js",
+              "content": '',
+            },
+            { "name": "file_to_expand.js",
+              "content": ('hello world %(base)s'),
+            },
+            { "name": "file_with_broken_expand_type.js",
+              "content": ('hello world %(base)'),
+            },
+            { "name": "file_with_odd_token.js",
+              "content": ('hello world %(base)s, %(No,token)s'),
+            },
+            { "name": "file_with_missing.js",
+              "content": ('hello world %(base)s, %(idontexist)s'),
+            },
+            { "name": "subdir/file_to_read.js",
+              "content": ('hello world from subdir'),
+            },
+            { # for future test expanding TAL
+              "name": "_generic.expandfile_success.html",
+              "content": (
+
+                  '''<span tal:content="python:utils.expandfile('''
+                  """'example.js', { 'No Token': "NT",
+                  "dict_token': 'DT'})"></span>""" ),
+            },
+        ]
+
+        for file_spec in testfiles:
+            file_path = "%s/html/%s" % (self.dirname, file_spec['name'])
+            with open(file_path, "w") as f:
+                f.write(file_spec['content'])
+
+        # get the client instance The form is needed to initialize,
+        # but not used since I call selectTemplate directly.
+        t = client.Client(self.instance, "user",
+                {'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
+         form=db_test_base.makeForm({"@template": "readfile_success"}))
+
+        t.db = MockNull()
+        t.db.config = MockNull()
+        t.db.config.TRACKER_WEB = '_tracker_template'
+        tu = TemplatingUtils(t)
+
+        # testcase 1 - file exists
+        r = tu.expandfile("file_to_read.js")
+        self.assertEqual(r, 'hello world')
+        r = None
+
+        # testcase 2 - file does not exist
+        with self.assertRaises(NoTemplate) as e:
+            r = tu.expandfile("no_file_to_read.js")
+
+        self.assertEqual(
+            e.exception.args[0],
+            "Unable to read or expand file 'no_file_to_read.js' "
+            "in template 'home'.")
+        r = None
+
+        # testcase 3 - file does not exist - optional = True
+        r = tu.expandfile("no_file_to_read.js", optional=True)
+        self.assertEqual(r, '')
+        r = None
+
+        # testcase 4 - file is empty
+        r = tu.expandfile("file_no_content.js")
+        self.assertEqual(r, '')
+        r = None
+
+        # testcase 5 - behave like readfile (values = None)
+        r = tu.expandfile("file_to_expand.js")
+        self.assertEqual(r, "hello world %(base)s")
+        r = None
+
+        # testcase 6 - expand predefined
+        r = tu.expandfile("file_to_expand.js", {})
+        self.assertEqual(r, "hello world _tracker_template")
+        r = None
+
+        # testcase 7 - missing trailing type specifier
+        r = tu.expandfile("file_with_broken_expand_type.js", {})
+
+        self.assertEqual(r, "")
+
+        # self._caplog.record_tuples[0] - without line breaks
+        # ('roundup.template', 40, "Found an incorrect token when
+        # expandfile applied string subsitution on
+        # '/home/roundup/_test_template/html/file_with_broken_expand_type.js.
+        # ValueError('incomplete format') was raised. Check the format 
+        # of your named conversion specifiers."
+
+        # name used for logging
+        self.assertEqual(self._caplog.record_tuples[0][0], 'roundup.template')
+        # severity ERROR = 40
+        self.assertEqual(self._caplog.record_tuples[0][1], 40,
+                        msg="logging level != 40 (ERROR)")
+        # message. It includes a full path to the problem file, so Regex
+        # match the changable filename directory
+        self.assertRegex(self._caplog.record_tuples[0][2], (
+            r"^Found an incorrect token when expandfile applied "
+            r"string subsitution on "
+            r"'[^']*/_test_template/html/file_with_broken_expand_type.js'. "
+            r"ValueError\('incomplete format'\) was raised. Check the format "
+            r"of your named conversion specifiers."))
+        self._caplog.clear()
+        r = None
+
+        # testcase 8 - odd token. Apparently names are not identifiers
+        r = tu.expandfile("file_with_odd_token.js", {'No,token': 'NT'})
+
+        self.assertEqual(r, "hello world _tracker_template, NT")
+
+        # self._caplog.record_tuples[0] - without line breaks
+        # ('roundup.template', 40, "Found an incorrect token when
+        # expandfile applied string subsitution on
+        # '/home/roundup/_test_template/html/file_with_broken_expand_type.js.
+        # ValueError('incomplete format') was raised. Check the format 
+        # of your named conversion specifiers."
+
+        # no logs should be present
+        self.assertEqual(self._caplog.text, '')
+        r = None
+
+        # testcase 9 - key missing from values
+        r = tu.expandfile("file_with_missing.js", {})
+
+        self.assertEqual(r, "")
+
+        # self._caplog.record_tuples[0] - without line breaks
+        # ('roundup.template', 40, "Found an incorrect token when
+        # expandfile applied string subsitution on
+        # '/home/roundup/_test_template/html/file_with_broken_expand_type.js.
+        # ValueError('incomplete format') was raised. Check the format 
+        # of your named conversion specifiers."
+
+        # name used for logging
+        self.assertEqual(self._caplog.record_tuples[0][0], 'roundup.template')
+        # severity ERROR = 40
+        self.assertEqual(self._caplog.record_tuples[0][1], 40,
+                        msg="logging level != 40 (ERROR)")
+        # message. It includes a full path to the problem file, so Regex
+        # match the changable filename directory
+        self.assertRegex(self._caplog.record_tuples[0][2], (
+            r"When running "
+            r"expandfile\('[^']*/_test_template/html/file_with_missing.js'\) "
+            r"in 'home' there was no value for token: 'idontexist'."))
+        self._caplog.clear()
+        r = None
+
+        # testcase 10 - set key missing from values in test 8
+        r = tu.expandfile("file_with_missing.js", {'idontexist': 'I do exist'})
+
+        self.assertEqual(r, "hello world _tracker_template, I do exist")
+
+        # no logging
+        self.assertEqual(self._caplog.text, '')
+        self._caplog.clear()
+
+        # testcase 11 - file exists in subdir
+        r = tu.expandfile("subdir/file_to_read.js")
+        self.assertEqual(r, 'hello world from subdir')
+        r = None
+
+
+        # make sure a created template is found
+        # note that the extension is not included just the basename
+        self.assertEqual("_generic.expandfile_success",
+                         t.selectTemplate("", "expandfile_success"))        
+
+
 class SqliteNativeFtsCgiTest(unittest.TestCase, testFtsQuery, testCsvExport):
     """All of the rest of the tests use anydbm as the backend.
        In addtion to normal fts test, this class tests renderError