Mercurial > p > roundup > code
changeset 2348:8c2402a78bb0
beginning getting ZPT up to date: TAL first
| author | Richard Jones <richard@users.sourceforge.net> |
|---|---|
| date | Fri, 21 May 2004 05:36:30 +0000 |
| parents | fbbda3b1816d |
| children | b43efe461b3e |
| files | roundup/cgi/TAL/DummyEngine.py roundup/cgi/TAL/HTMLParser.py roundup/cgi/TAL/HTMLTALParser.py roundup/cgi/TAL/TALDefs.py roundup/cgi/TAL/TALGenerator.py roundup/cgi/TAL/TALInterpreter.py roundup/cgi/TAL/TALParser.py roundup/cgi/TAL/TranslationContext.py roundup/cgi/TAL/XMLParser.py roundup/cgi/TAL/__init__.py roundup/cgi/TAL/markupbase.py roundup/cgi/TAL/talgettext.py |
| diffstat | 12 files changed, 1385 insertions(+), 253 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/roundup/cgi/TAL/DummyEngine.py Fri May 21 05:36:30 2004 +0000 @@ -0,0 +1,274 @@ +############################################################################## +# +# Copyright (c) 2001, 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +# Modifications for Roundup: +# 1. commented out ITALES references +# 2. implemented ustr as str +""" +Dummy TALES engine so that I can test out the TAL implementation. +""" + +import re +import sys + +from TALDefs import NAME_RE, TALESError, ErrorInfo +#from ITALES import ITALESCompiler, ITALESEngine +#from DocumentTemplate.DT_Util import ustr +ustr = str + +IDomain = None +if sys.modules.has_key('Zope'): + try: + from Zope.I18n.ITranslationService import ITranslationService + from Zope.I18n.IDomain import IDomain + except ImportError: + pass +if IDomain is None: + # Before 2.7, or not in Zope + class ITranslationService: pass + class IDomain: pass + +class _Default: + pass +Default = _Default() + +name_match = re.compile(r"(?s)(%s):(.*)\Z" % NAME_RE).match + +class CompilerError(Exception): + pass + +class DummyEngine: + + position = None + source_file = None + + #__implements__ = ITALESCompiler, ITALESEngine + + def __init__(self, macros=None): + if macros is None: + macros = {} + self.macros = macros + dict = {'nothing': None, 'default': Default} + self.locals = self.globals = dict + self.stack = [dict] + self.translationService = DummyTranslationService() + + def getCompilerError(self): + return CompilerError + + def getCompiler(self): + return self + + def setSourceFile(self, source_file): + self.source_file = source_file + + def setPosition(self, position): + self.position = position + + def compile(self, expr): + return "$%s$" % expr + + def uncompile(self, expression): + assert (expression.startswith("$") and expression.endswith("$"), + expression) + return expression[1:-1] + + def beginScope(self): + self.stack.append(self.locals) + + def endScope(self): + assert len(self.stack) > 1, "more endScope() than beginScope() calls" + self.locals = self.stack.pop() + + def setLocal(self, name, value): + if self.locals is self.stack[-1]: + # Unmerge this scope's locals from previous scope of first set + self.locals = self.locals.copy() + self.locals[name] = value + + def setGlobal(self, name, value): + self.globals[name] = value + + def evaluate(self, expression): + assert (expression.startswith("$") and expression.endswith("$"), + expression) + expression = expression[1:-1] + m = name_match(expression) + if m: + type, expr = m.group(1, 2) + else: + type = "path" + expr = expression + if type in ("string", "str"): + return expr + if type in ("path", "var", "global", "local"): + return self.evaluatePathOrVar(expr) + if type == "not": + return not self.evaluate(expr) + if type == "exists": + return self.locals.has_key(expr) or self.globals.has_key(expr) + if type == "python": + try: + return eval(expr, self.globals, self.locals) + except: + raise TALESError("evaluation error in %s" % `expr`) + if type == "position": + # Insert the current source file name, line number, + # and column offset. + if self.position: + lineno, offset = self.position + else: + lineno, offset = None, None + return '%s (%s,%s)' % (self.source_file, lineno, offset) + raise TALESError("unrecognized expression: " + `expression`) + + def evaluatePathOrVar(self, expr): + expr = expr.strip() + if self.locals.has_key(expr): + return self.locals[expr] + elif self.globals.has_key(expr): + return self.globals[expr] + else: + raise TALESError("unknown variable: %s" % `expr`) + + def evaluateValue(self, expr): + return self.evaluate(expr) + + def evaluateBoolean(self, expr): + return self.evaluate(expr) + + def evaluateText(self, expr): + text = self.evaluate(expr) + if text is not None and text is not Default: + text = ustr(text) + return text + + def evaluateStructure(self, expr): + # XXX Should return None or a DOM tree + return self.evaluate(expr) + + def evaluateSequence(self, expr): + # XXX Should return a sequence + return self.evaluate(expr) + + def evaluateMacro(self, macroName): + assert (macroName.startswith("$") and macroName.endswith("$"), + macroName) + macroName = macroName[1:-1] + file, localName = self.findMacroFile(macroName) + if not file: + # Local macro + macro = self.macros[localName] + else: + # External macro + import driver + program, macros = driver.compilefile(file) + macro = macros.get(localName) + if not macro: + raise TALESError("macro %s not found in file %s" % + (localName, file)) + return macro + + def findMacroDocument(self, macroName): + file, localName = self.findMacroFile(macroName) + if not file: + return file, localName + import driver + doc = driver.parsefile(file) + return doc, localName + + def findMacroFile(self, macroName): + if not macroName: + raise TALESError("empty macro name") + i = macroName.rfind('/') + if i < 0: + # No slash -- must be a locally defined macro + return None, macroName + else: + # Up to last slash is the filename + fileName = macroName[:i] + localName = macroName[i+1:] + return fileName, localName + + def setRepeat(self, name, expr): + seq = self.evaluateSequence(expr) + return Iterator(name, seq, self) + + def createErrorInfo(self, err, position): + return ErrorInfo(err, position) + + def getDefault(self): + return Default + + def translate(self, domain, msgid, mapping, default=None): + return self.translationService.translate(domain, msgid, mapping, + default=default) + + +class Iterator: + + # This is not an implementation of a Python iterator. The next() + # method returns true or false to indicate whether another item is + # available; if there is another item, the iterator instance calls + # setLocal() on the evaluation engine passed to the constructor. + + def __init__(self, name, seq, engine): + self.name = name + self.seq = seq + self.engine = engine + self.nextIndex = 0 + + def next(self): + i = self.nextIndex + try: + item = self.seq[i] + except IndexError: + return 0 + self.nextIndex = i+1 + self.engine.setLocal(self.name, item) + return 1 + +class DummyDomain: + __implements__ = IDomain + + def translate(self, msgid, mapping=None, context=None, + target_language=None, default=None): + # This is a fake translation service which simply uppercases non + # ${name} placeholder text in the message id. + # + # First, transform a string with ${name} placeholders into a list of + # substrings. Then upcase everything but the placeholders, then glue + # things back together. + + # simulate an unknown msgid by returning None + if msgid == "don't translate me": + text = default + else: + text = msgid.upper() + + def repl(m, mapping=mapping): + return ustr(mapping[m.group(m.lastindex).lower()]) + cre = re.compile(r'\$(?:(%s)|\{(%s)\})' % (NAME_RE, NAME_RE)) + return cre.sub(repl, text) + +class DummyTranslationService: + __implements__ = ITranslationService + + def translate(self, domain, msgid, mapping=None, context=None, + target_language=None, default=None): + return self.getDomain(domain).translate(msgid, mapping, context, + target_language, + default=default) + + def getDomain(self, domain): + return DummyDomain()
--- a/roundup/cgi/TAL/HTMLParser.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/HTMLParser.py Fri May 21 05:36:30 2004 +0000 @@ -1,5 +1,4 @@ """A parser for HTML and XHTML.""" -__docformat__ = 'restructuredtext' # This file is based on sgmllib.py, but the API is slightly different. @@ -11,7 +10,6 @@ import markupbase import re -import string # Regular expressions used for parsing @@ -261,7 +259,7 @@ match = tagfind.match(rawdata, i+1) assert match, 'unexpected call to parse_starttag()' k = match.end() - self.lasttag = tag = string.lower(rawdata[i+1:k]) + self.lasttag = tag = rawdata[i+1:k].lower() while k < endpos: m = attrfind.match(rawdata, k) @@ -274,16 +272,16 @@ attrvalue[:1] == '"' == attrvalue[-1:]: attrvalue = attrvalue[1:-1] attrvalue = self.unescape(attrvalue) - attrs.append((string.lower(attrname), attrvalue)) + attrs.append((attrname.lower(), attrvalue)) k = m.end() - end = string.strip(rawdata[k:endpos]) + end = rawdata[k:endpos].strip() if end not in (">", "/>"): lineno, offset = self.getpos() if "\n" in self.__starttag_text: - lineno = lineno + string.count(self.__starttag_text, "\n") + lineno = lineno + self.__starttag_text.count("\n") offset = len(self.__starttag_text) \ - - string.rfind(self.__starttag_text, "\n") + - self.__starttag_text.rfind("\n") else: offset = offset + len(self.__starttag_text) self.error("junk characters in start tag: %s" @@ -340,7 +338,7 @@ match = endtagfind.match(rawdata, i) # </ + tag + > if not match: self.error("bad end tag: %s" % `rawdata[i:j]`) - tag = string.lower(match.group(1)) + tag = match.group(1).lower() if ( self.cdata_endtag is not None and tag != self.cdata_endtag): # Should be a mismatched end tag, but we'll treat it @@ -396,9 +394,9 @@ def unescape(self, s): if '&' not in s: return s - s = string.replace(s, "<", "<") - s = string.replace(s, ">", ">") - s = string.replace(s, "'", "'") - s = string.replace(s, """, '"') - s = string.replace(s, "&", "&") # Must be last + s = s.replace("<", "<") + s = s.replace(">", ">") + s = s.replace("'", "'") + s = s.replace(""", '"') + s = s.replace("&", "&") # Must be last return s
--- a/roundup/cgi/TAL/HTMLTALParser.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/HTMLTALParser.py Fri May 21 05:36:30 2004 +0000 @@ -2,25 +2,25 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Parse HTML and compile to TALInterpreter intermediate code. """ -__docformat__ = 'restructuredtext' +Parse HTML and compile to TALInterpreter intermediate code. +""" import sys -import string from TALGenerator import TALGenerator -from TALDefs import ZOPE_METAL_NS, ZOPE_TAL_NS, METALError, TALError from HTMLParser import HTMLParser, HTMLParseError +from TALDefs import \ + ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS, METALError, TALError, I18NError BOOLEAN_HTML_ATTRS = [ # List of Boolean attributes in HTML that may be given in @@ -75,7 +75,7 @@ % (tagstack[0], endtag)) else: msg = ('Open tags <%s> do not match close tag </%s>' - % (string.join(tagstack, '>, <'), endtag)) + % ('>, <'.join(tagstack), endtag)) else: msg = 'No tags are open to match </%s>' % endtag HTMLParseError.__init__(self, msg, position) @@ -107,13 +107,20 @@ self.gen = gen self.tagstack = [] self.nsstack = [] - self.nsdict = {'tal': ZOPE_TAL_NS, 'metal': ZOPE_METAL_NS} + self.nsdict = {'tal': ZOPE_TAL_NS, + 'metal': ZOPE_METAL_NS, + 'i18n': ZOPE_I18N_NS, + } def parseFile(self, file): f = open(file) data = f.read() f.close() - self.parseString(data) + try: + self.parseString(data) + except TALError, e: + e.setFile(file) + raise def parseString(self, data): self.feed(data) @@ -133,9 +140,14 @@ def handle_starttag(self, tag, attrs): self.close_para_tags(tag) self.scan_xmlns(attrs) - tag, attrlist, taldict, metaldict = self.process_ns(tag, attrs) + tag, attrlist, taldict, metaldict, i18ndict \ + = self.process_ns(tag, attrs) + if tag in EMPTY_HTML_TAGS and taldict.get("content"): + raise TALError( + "empty HTML tags cannot use tal:content: %s" % `tag`, + self.getpos()) self.tagstack.append(tag) - self.gen.emitStartElement(tag, attrlist, taldict, metaldict, + self.gen.emitStartElement(tag, attrlist, taldict, metaldict, i18ndict, self.getpos()) if tag in EMPTY_HTML_TAGS: self.implied_endtag(tag, -1) @@ -143,14 +155,19 @@ def handle_startendtag(self, tag, attrs): self.close_para_tags(tag) self.scan_xmlns(attrs) - tag, attrlist, taldict, metaldict = self.process_ns(tag, attrs) + tag, attrlist, taldict, metaldict, i18ndict \ + = self.process_ns(tag, attrs) if taldict.get("content"): + if tag in EMPTY_HTML_TAGS: + raise TALError( + "empty HTML tags cannot use tal:content: %s" % `tag`, + self.getpos()) self.gen.emitStartElement(tag, attrlist, taldict, metaldict, - self.getpos()) + i18ndict, self.getpos()) self.gen.emitEndElement(tag, implied=-1) else: self.gen.emitStartElement(tag, attrlist, taldict, metaldict, - self.getpos(), isend=1) + i18ndict, self.getpos(), isend=1) self.pop_xmlns() def handle_endtag(self, tag): @@ -236,7 +253,7 @@ def scan_xmlns(self, attrs): nsnew = {} for key, value in attrs: - if key[:6] == "xmlns:": + if key.startswith("xmlns:"): nsnew[key[6:]] = value if nsnew: self.nsstack.append(self.nsdict) @@ -250,10 +267,10 @@ def fixname(self, name): if ':' in name: - prefix, suffix = string.split(name, ':', 1) + prefix, suffix = name.split(':', 1) if prefix == 'xmlns': nsuri = self.nsdict.get(suffix) - if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS): + if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS, ZOPE_I18N_NS): return name, name, prefix else: nsuri = self.nsdict.get(prefix) @@ -261,12 +278,15 @@ return name, suffix, 'tal' elif nsuri == ZOPE_METAL_NS: return name, suffix, 'metal' + elif nsuri == ZOPE_I18N_NS: + return name, suffix, 'i18n' return name, name, 0 def process_ns(self, name, attrs): attrlist = [] taldict = {} metaldict = {} + i18ndict = {} name, namebase, namens = self.fixname(name) for item in attrs: key, value = item @@ -284,7 +304,12 @@ raise METALError("duplicate METAL attribute " + `keybase`, self.getpos()) metaldict[keybase] = value + elif ns == 'i18n': + if i18ndict.has_key(keybase): + raise I18NError("duplicate i18n attribute " + + `keybase`, self.getpos()) + i18ndict[keybase] = value attrlist.append(item) if namens in ('metal', 'tal'): taldict['tal tag'] = namens - return name, attrlist, taldict, metaldict + return name, attrlist, taldict, metaldict, i18ndict
--- a/roundup/cgi/TAL/TALDefs.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/TALDefs.py Fri May 21 05:36:30 2004 +0000 @@ -2,37 +2,44 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Common definitions used by TAL and METAL compilation an transformation. +# Modifications for Roundup: +# 1. commented out ITALES references """ -__docformat__ = 'restructuredtext' +Common definitions used by TAL and METAL compilation an transformation. +""" from types import ListType, TupleType -TAL_VERSION = "1.3.2" +#from ITALES import ITALESErrorInfo + +TAL_VERSION = "1.4" XML_NS = "http://www.w3.org/XML/1998/namespace" # URI for XML namespace XMLNS_NS = "http://www.w3.org/2000/xmlns/" # URI for XML NS declarations ZOPE_TAL_NS = "http://xml.zope.org/namespaces/tal" ZOPE_METAL_NS = "http://xml.zope.org/namespaces/metal" +ZOPE_I18N_NS = "http://xml.zope.org/namespaces/i18n" -NAME_RE = "[a-zA-Z_][a-zA-Z0-9_]*" +# This RE must exactly match the expression of the same name in the +# zope.i18n.simpletranslationservice module: +NAME_RE = "[a-zA-Z_][-a-zA-Z0-9_]*" KNOWN_METAL_ATTRIBUTES = [ "define-macro", "use-macro", "define-slot", "fill-slot", - "slot" + "slot", ] KNOWN_TAL_ATTRIBUTES = [ @@ -47,6 +54,16 @@ "tal tag", ] +KNOWN_I18N_ATTRIBUTES = [ + "translate", + "domain", + "target", + "source", + "attributes", + "data", + "name", + ] + class TALError(Exception): def __init__(self, msg, position=(None, None)): @@ -54,6 +71,10 @@ self.msg = msg self.lineno = position[0] self.offset = position[1] + self.filename = None + + def setFile(self, filename): + self.filename = filename def __str__(self): result = self.msg @@ -61,6 +82,8 @@ result = result + ", at line %d" % self.lineno if self.offset is not None: result = result + ", column %d" % (self.offset + 1) + if self.filename is not None: + result = result + ', in file %s' % self.filename return result class METALError(TALError): @@ -69,8 +92,14 @@ class TALESError(TALError): pass +class I18NError(TALError): + pass + + class ErrorInfo: + #__implements__ = ITALESErrorInfo + def __init__(self, err, position=(None, None)): if isinstance(err, Exception): self.type = err.__class__ @@ -81,20 +110,24 @@ self.lineno = position[0] self.offset = position[1] + + import re _attr_re = re.compile(r"\s*([^\s]+)\s+([^\s].*)\Z", re.S) _subst_re = re.compile(r"\s*(?:(text|structure)\s+)?(.*)\Z", re.S) del re -def parseAttributeReplacements(arg): +def parseAttributeReplacements(arg, xml): dict = {} for part in splitParts(arg): m = _attr_re.match(part) if not m: - raise TALError("Bad syntax in attributes:" + `part`) + raise TALError("Bad syntax in attributes: " + `part`) name, expr = m.group(1, 2) + if not xml: + name = name.lower() if dict.has_key(name): - raise TALError("Duplicate attribute name in attributes:" + `part`) + raise TALError("Duplicate attribute name in attributes: " + `part`) dict[name] = expr return dict @@ -110,11 +143,10 @@ def splitParts(arg): # Break in pieces at undoubled semicolons and # change double semicolons to singles: - import string - arg = string.replace(arg, ";;", "\0") - parts = string.split(arg, ';') - parts = map(lambda s, repl=string.replace: repl(s, "\0", ";"), parts) - if len(parts) > 1 and not string.strip(parts[-1]): + arg = arg.replace(";;", "\0") + parts = arg.split(';') + parts = [p.replace("\0", ";") for p in parts] + if len(parts) > 1 and not parts[-1].strip(): del parts[-1] # It ended in a semicolon return parts @@ -139,7 +171,23 @@ return version return None -import cgi -def quote(s, escape=cgi.escape): - return '"%s"' % escape(s, 1) -del cgi +import re +_ent1_re = re.compile('&(?![A-Z#])', re.I) +_entch_re = re.compile('&([A-Z][A-Z0-9]*)(?![A-Z0-9;])', re.I) +_entn1_re = re.compile('&#(?![0-9X])', re.I) +_entnx_re = re.compile('&(#X[A-F0-9]*)(?![A-F0-9;])', re.I) +_entnd_re = re.compile('&(#[0-9][0-9]*)(?![0-9;])') +del re + +def attrEscape(s): + """Replace special characters '&<>' by character entities, + except when '&' already begins a syntactically valid entity.""" + s = _ent1_re.sub('&', s) + s = _entch_re.sub(r'&\1', s) + s = _entn1_re.sub('&#', s) + s = _entnx_re.sub(r'&\1', s) + s = _entnd_re.sub(r'&\1', s) + s = s.replace('<', '<') + s = s.replace('>', '>') + s = s.replace('"', '"') + return s
--- a/roundup/cgi/TAL/TALGenerator.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/TALGenerator.py Fri May 21 05:36:30 2004 +0000 @@ -2,39 +2,57 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Code generator for TALInterpreter intermediate code. """ -__docformat__ = 'restructuredtext' +Code generator for TALInterpreter intermediate code. +""" -import string import re import cgi -from TALDefs import * +import TALDefs + +from TALDefs import NAME_RE, TAL_VERSION +from TALDefs import I18NError, METALError, TALError +from TALDefs import parseSubstitution +from TranslationContext import TranslationContext, DEFAULT_DOMAIN + +I18N_REPLACE = 1 +I18N_CONTENT = 2 +I18N_EXPRESSION = 3 + +_name_rx = re.compile(NAME_RE) + class TALGenerator: inMacroUse = 0 inMacroDef = 0 source_file = None - + def __init__(self, expressionCompiler=None, xml=1, source_file=None): if not expressionCompiler: from DummyEngine import DummyEngine expressionCompiler = DummyEngine() self.expressionCompiler = expressionCompiler self.CompilerError = expressionCompiler.getCompilerError() + # This holds the emitted opcodes representing the input self.program = [] + # The program stack for when we need to do some sub-evaluation for an + # intermediate result. E.g. in an i18n:name tag for which the + # contents describe the ${name} value. self.stack = [] + # Another stack of postponed actions. Elements on this stack are a + # dictionary; key/values contain useful information that + # emitEndElement needs to finish its calculations self.todoStack = [] self.macros = {} self.slots = {} @@ -45,6 +63,8 @@ if source_file is not None: self.source_file = source_file self.emit("setSourceFile", source_file) + self.i18nContext = TranslationContext() + self.i18nLevel = 0 def getCode(self): assert not self.stack @@ -54,7 +74,7 @@ def optimize(self, program): output = [] collect = [] - rawseen = cursor = 0 + cursor = 0 if self.xml: endsep = "/>" else: @@ -83,9 +103,15 @@ # instructions to be joined together. output.append(self.optimizeArgsList(item)) continue - text = string.join(collect, "") + if opcode == 'noop': + # This is a spacer for end tags in the face of i18n:name + # attributes. We can't let the optimizer collect immediately + # following end tags into the same rawtextOffset. + opcode = None + pass + text = "".join(collect) if text: - i = string.rfind(text, "\n") + i = text.rfind("\n") if i >= 0: i = len(text) - (i + 1) output.append(("rawtextColumn", (text, i))) @@ -93,7 +119,6 @@ output.append(("rawtextOffset", (text, len(text)))) if opcode != None: output.append(self.optimizeArgsList(item)) - rawseen = cursor+1 collect = [] return self.optimizeCommonTriple(output) @@ -103,9 +128,28 @@ else: return item[0], tuple(item[1:]) - actionIndex = {"replace":0, "insert":1, "metal":2, "tal":3, "xmlns":4, - 0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + # These codes are used to indicate what sort of special actions + # are needed for each special attribute. (Simple attributes don't + # get action codes.) + # + # The special actions (which are modal) are handled by + # TALInterpreter.attrAction() and .attrAction_tal(). + # + # Each attribute is represented by a tuple: + # + # (name, value) -- a simple name/value pair, with + # no special processing + # + # (name, value, action, *extra) -- attribute with special + # processing needs, action is a + # code that indicates which + # branch to take, and *extra + # contains additional, + # action-specific information + # needed by the processing + # def optimizeStartTag(self, collect, name, attrlist, end): + # return true if the tag can be converted to plain text if not attrlist: collect.append("<%s%s" % (name, end)) return 1 @@ -116,18 +160,15 @@ if len(item) > 2: opt = 0 name, value, action = item[:3] - action = self.actionIndex[action] attrlist[i] = (name, value, action) + item[3:] else: if item[1] is None: s = item[0] else: - s = "%s=%s" % (item[0], quote(item[1])) + s = '%s="%s"' % (item[0], TALDefs.attrEscape(item[1])) attrlist[i] = item[0], s - if item[1] is None: - new.append(" " + item[0]) - else: - new.append(" %s=%s" % (item[0], quote(item[1]))) + new.append(" " + s) + # if no non-optimizable attributes were found, convert to plain text if opt: new.append(end) collect.extend(new) @@ -139,9 +180,9 @@ output = program[:2] prev2, prev1 = output for item in program[2:]: - if ( item[0] == "beginScope" - and prev1[0] == "setPosition" - and prev2[0] == "rawtextColumn"): + if ( item[0] == "beginScope" + and prev1[0] == "setPosition" + and prev2[0] == "rawtextColumn"): position = output.pop()[1] text, column = output.pop()[1] prev1 = None, None @@ -215,7 +256,7 @@ if cexpr: cexpr = self.compileExpression(optTag[0]) self.emit("optTag", name, cexpr, optTag[1], isend, start, program) - + def emitRawText(self, text): self.emit("rawtext", text) @@ -223,7 +264,7 @@ self.emitRawText(cgi.escape(text)) def emitDefines(self, defines): - for part in splitParts(defines): + for part in TALDefs.splitParts(defines): m = re.match( r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part) if not m: @@ -278,9 +319,56 @@ assert key == "structure" self.emit("insertStructure", cexpr, attrDict, program) + def emitI18nVariable(self, stuff): + # Used for i18n:name attributes. arg is extra information describing + # how the contents of the variable should get filled in, and it will + # either be a 1-tuple or a 2-tuple. If arg[0] is None, then the + # i18n:name value is taken implicitly from the contents of the tag, + # e.g. "I live in <span i18n:name="country">the USA</span>". In this + # case, arg[1] is the opcode sub-program describing the contents of + # the tag. + # + # When arg[0] is not None, it contains the tal expression used to + # calculate the contents of the variable, e.g. + # "I live in <span i18n:name="country" + # tal:replace="here/countryOfOrigin" />" + varname, action, expression = stuff + m = _name_rx.match(varname) + if m is None or m.group() != varname: + raise TALError("illegal i18n:name: %r" % varname, self.position) + key = cexpr = None + program = self.popProgram() + if action == I18N_REPLACE: + # This is a tag with an i18n:name and a tal:replace (implicit or + # explicit). Get rid of the first and last elements of the + # program, which are the start and end tag opcodes of the tag. + program = program[1:-1] + elif action == I18N_CONTENT: + # This is a tag with an i18n:name and a tal:content + # (explicit-only). Keep the first and last elements of the + # program, so we keep the start and end tag output. + pass + else: + assert action == I18N_EXPRESSION + key, expr = parseSubstitution(expression) + cexpr = self.compileExpression(expr) + # XXX Would key be anything but 'text' or None? + assert key in ('text', None) + self.emit('i18nVariable', varname, program, cexpr) + + def emitTranslation(self, msgid, i18ndata): + program = self.popProgram() + if i18ndata is None: + self.emit('insertTranslation', msgid, program) + else: + key, expr = parseSubstitution(i18ndata) + cexpr = self.compileExpression(expr) + assert key == 'text' + self.emit('insertTranslation', msgid, program, cexpr) + def emitDefineMacro(self, macroName): program = self.popProgram() - macroName = string.strip(macroName) + macroName = macroName.strip() if self.macros.has_key(macroName): raise METALError("duplicate macro definition: %s" % `macroName`, self.position) @@ -299,7 +387,7 @@ def emitDefineSlot(self, slotName): program = self.popProgram() - slotName = string.strip(slotName) + slotName = slotName.strip() if not re.match('%s$' % NAME_RE, slotName): raise METALError("invalid slot name: %s" % `slotName`, self.position) @@ -307,7 +395,7 @@ def emitFillSlot(self, slotName): program = self.popProgram() - slotName = string.strip(slotName) + slotName = slotName.strip() if self.slots.has_key(slotName): raise METALError("duplicate fill-slot name: %s" % `slotName`, self.position) @@ -338,7 +426,7 @@ self.program[i] = ("rawtext", text[:m.start()]) collect.append(m.group()) collect.reverse() - return string.join(collect, "") + return "".join(collect) def unEmitNewlineWhitespace(self): collect = [] @@ -357,7 +445,7 @@ break text, rest = m.group(1, 2) collect.reverse() - rest = rest + string.join(collect, "") + rest = rest + "".join(collect) del self.program[i:] if text: self.emit("rawtext", text) @@ -365,23 +453,30 @@ return None def replaceAttrs(self, attrlist, repldict): + # Each entry in attrlist starts like (name, value). + # Result is (name, value, action, expr, xlat) if there is a + # tal:attributes entry for that attribute. Additional attrs + # defined only by tal:attributes are added here. + # + # (name, value, action, expr, xlat) if not repldict: return attrlist newlist = [] for item in attrlist: key = item[0] if repldict.has_key(key): - item = item[:2] + ("replace", repldict[key]) + expr, xlat, msgid = repldict[key] + item = item[:2] + ("replace", expr, xlat, msgid) del repldict[key] newlist.append(item) - for key, value in repldict.items(): # Add dynamic-only attributes - item = (key, None, "insert", value) - newlist.append(item) + # Add dynamic-only attributes + for key, (expr, xlat, msgid) in repldict.items(): + newlist.append((key, None, "insert", expr, xlat, msgid)) return newlist - def emitStartElement(self, name, attrlist, taldict, metaldict, + def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict, position=(None, None), isend=0): - if not taldict and not metaldict: + if not taldict and not metaldict and not i18ndict: # Handle the simple, common case self.emitStartTag(name, attrlist, isend) self.todoPush({}) @@ -391,18 +486,24 @@ self.position = position for key, value in taldict.items(): - if key not in KNOWN_TAL_ATTRIBUTES: + if key not in TALDefs.KNOWN_TAL_ATTRIBUTES: raise TALError("bad TAL attribute: " + `key`, position) if not (value or key == 'omit-tag'): raise TALError("missing value for TAL attribute: " + `key`, position) for key, value in metaldict.items(): - if key not in KNOWN_METAL_ATTRIBUTES: + if key not in TALDefs.KNOWN_METAL_ATTRIBUTES: raise METALError("bad METAL attribute: " + `key`, - position) + position) if not value: raise TALError("missing value for METAL attribute: " + `key`, position) + for key, value in i18ndict.items(): + if key not in TALDefs.KNOWN_I18N_ATTRIBUTES: + raise I18NError("bad i18n attribute: " + `key`, position) + if not value and key in ("attributes", "data", "id"): + raise I18NError("missing value for i18n attribute: " + + `key`, position) todo = {} defineMacro = metaldict.get("define-macro") useMacro = metaldict.get("use-macro") @@ -417,13 +518,36 @@ onError = taldict.get("on-error") omitTag = taldict.get("omit-tag") TALtag = taldict.get("tal tag") + i18nattrs = i18ndict.get("attributes") + # Preserve empty string if implicit msgids are used. We'll generate + # code with the msgid='' and calculate the right implicit msgid during + # interpretation phase. + msgid = i18ndict.get("translate") + varname = i18ndict.get('name') + i18ndata = i18ndict.get('data') + + if varname and not self.i18nLevel: + raise I18NError( + "i18n:name can only occur inside a translation unit", + position) + + if i18ndata and not msgid: + raise I18NError("i18n:data must be accompanied by i18n:translate", + position) + if len(metaldict) > 1 and (defineMacro or useMacro): raise METALError("define-macro and use-macro cannot be used " "together or with define-slot or fill-slot", position) - if content and replace: - raise TALError("content and replace are mutually exclusive", - position) + if replace: + if content: + raise TALError( + "tal:content and tal:replace are mutually exclusive", + position) + if msgid is not None: + raise I18NError( + "i18n:translate and tal:replace are mutually exclusive", + position) repeatWhitespace = None if repeat: @@ -441,8 +565,8 @@ self.inMacroUse = 0 else: if fillSlot: - raise METALError, ("fill-slot must be within a use-macro", - position) + raise METALError("fill-slot must be within a use-macro", + position) if not self.inMacroUse: if defineMacro: self.pushProgram() @@ -459,13 +583,29 @@ self.inMacroUse = 1 if defineSlot: if not self.inMacroDef: - raise METALError, ( + raise METALError( "define-slot must be within a define-macro", position) self.pushProgram() todo["defineSlot"] = defineSlot - if taldict: + if defineSlot or i18ndict: + + domain = i18ndict.get("domain") or self.i18nContext.domain + source = i18ndict.get("source") or self.i18nContext.source + target = i18ndict.get("target") or self.i18nContext.target + if ( domain != DEFAULT_DOMAIN + or source is not None + or target is not None): + self.i18nContext = TranslationContext(self.i18nContext, + domain=domain, + source=source, + target=target) + self.emit("beginI18nContext", + {"domain": domain, "source": source, + "target": target}) + todo["i18ncontext"] = 1 + if taldict or i18ndict: dict = {} for item in attrlist: key, value = item[:2] @@ -493,18 +633,60 @@ if repeatWhitespace: self.emitText(repeatWhitespace) if content: - todo["content"] = content - if replace: - todo["replace"] = replace + if varname: + todo['i18nvar'] = (varname, I18N_CONTENT, None) + todo["content"] = content + self.pushProgram() + else: + todo["content"] = content + elif replace: + # tal:replace w/ i18n:name has slightly different semantics. What + # we're actually replacing then is the contents of the ${name} + # placeholder. + if varname: + todo['i18nvar'] = (varname, I18N_EXPRESSION, replace) + else: + todo["replace"] = replace self.pushProgram() + # i18n:name w/o tal:replace uses the content as the interpolation + # dictionary values + elif varname: + todo['i18nvar'] = (varname, I18N_REPLACE, None) + self.pushProgram() + if msgid is not None: + self.i18nLevel += 1 + todo['msgid'] = msgid + if i18ndata: + todo['i18ndata'] = i18ndata optTag = omitTag is not None or TALtag if optTag: todo["optional tag"] = omitTag, TALtag self.pushProgram() - if attrsubst: - repldict = parseAttributeReplacements(attrsubst) + if attrsubst or i18nattrs: + if attrsubst: + repldict = TALDefs.parseAttributeReplacements(attrsubst, + self.xml) + else: + repldict = {} + if i18nattrs: + i18nattrs = _parseI18nAttributes(i18nattrs, attrlist, repldict, + self.position, self.xml, + self.source_file) + else: + i18nattrs = {} + # Convert repldict's name-->expr mapping to a + # name-->(compiled_expr, translate) mapping for key, value in repldict.items(): - repldict[key] = self.compileExpression(value) + if i18nattrs.get(key, None): + raise I18NError( + ("attribute [%s] cannot both be part of tal:attributes" + + " and have a msgid in i18n:attributes") % key, + position) + ce = self.compileExpression(value) + repldict[key] = ce, key in i18nattrs, i18nattrs.get(key) + for key in i18nattrs: + if not repldict.has_key(key): + repldict[key] = None, 1, i18nattrs.get(key) else: repldict = {} if replace: @@ -513,7 +695,11 @@ self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend) if optTag: self.pushProgram() - if content: + if content and not varname: + self.pushProgram() + if msgid is not None: + self.pushProgram() + if content and varname: self.pushProgram() if todo and position != (None, None): todo["position"] = position @@ -543,6 +729,10 @@ repldict = todo.get("repldict", {}) scope = todo.get("scope") optTag = todo.get("optional tag") + msgid = todo.get('msgid') + i18ncontext = todo.get("i18ncontext") + varname = todo.get('i18nvar') + i18ndata = todo.get('i18ndata') if implied > 0: if defineMacro or useMacro or defineSlot or fillSlot: @@ -554,14 +744,58 @@ raise exc("%s attributes on <%s> require explicit </%s>" % (what, name, name), position) + # If there's no tal:content or tal:replace in the tag with the + # i18n:name, tal:replace is the default. if content: self.emitSubstitution(content, {}) + # If we're looking at an implicit msgid, emit the insertTranslation + # opcode now, so that the end tag doesn't become part of the implicit + # msgid. If we're looking at an explicit msgid, it's better to emit + # the opcode after the i18nVariable opcode so we can better handle + # tags with both of them in them (and in the latter case, the contents + # would be thrown away for msgid purposes). + # + # Still, we should emit insertTranslation opcode before i18nVariable + # in case tal:content, i18n:translate and i18n:name in the same tag + if msgid is not None: + if (not varname) or ( + varname and (varname[1] == I18N_CONTENT)): + self.emitTranslation(msgid, i18ndata) + self.i18nLevel -= 1 if optTag: self.emitOptTag(name, optTag, isend) elif not isend: + # If we're processing the end tag for a tag that contained + # i18n:name, we need to make sure that optimize() won't collect + # immediately following end tags into the same rawtextOffset, so + # put a spacer here that the optimizer will recognize. + if varname: + self.emit('noop') self.emitEndTag(name) + # If i18n:name appeared in the same tag as tal:replace then we're + # going to do the substitution a little bit differently. The results + # of the expression go into the i18n substitution dictionary. if replace: self.emitSubstitution(replace, repldict) + elif varname: + # o varname[0] is the variable name + # o varname[1] is either + # - I18N_REPLACE for implicit tal:replace + # - I18N_CONTENT for tal:content + # - I18N_EXPRESSION for explicit tal:replace + # o varname[2] will be None for the first two actions and the + # replacement tal expression for the third action. + assert (varname[1] + in [I18N_REPLACE, I18N_CONTENT, I18N_EXPRESSION]) + self.emitI18nVariable(varname) + # Do not test for "msgid is not None", i.e. we only want to test for + # explicit msgids here. See comment above. + if msgid is not None: + # in case tal:content, i18n:translate and i18n:name in the + # same tag insertTranslation opcode has already been + # emitted + if varname and (varname[1] <> I18N_CONTENT): + self.emitTranslation(msgid, i18ndata) if repeat: self.emitRepeat(repeat) if condition: @@ -570,6 +804,10 @@ self.emitOnError(name, onError, optTag and optTag[1], isend) if scope: self.emit("endScope") + if i18ncontext: + self.emit("endI18nContext") + assert self.i18nContext.parent is not None + self.i18nContext = self.i18nContext.parent if defineSlot: self.emitDefineSlot(defineSlot) if fillSlot: @@ -579,6 +817,54 @@ if defineMacro: self.emitDefineMacro(defineMacro) + +def _parseI18nAttributes(i18nattrs, attrlist, repldict, position, + xml, source_file): + + def addAttribute(dic, attr, msgid, position, xml): + if not xml: + attr = attr.lower() + if attr in dic: + raise TALError( + "attribute may only be specified once in i18n:attributes: " + + attr, + position) + dic[attr] = msgid + + d = {} + if ';' in i18nattrs: + i18nattrlist = i18nattrs.split(';') + i18nattrlist = [attr.strip().split() + for attr in i18nattrlist if attr.strip()] + for parts in i18nattrlist: + if len(parts) > 2: + raise TALError("illegal i18n:attributes specification: %r" + % parts, position) + if len(parts) == 2: + attr, msgid = parts + else: + # len(parts) == 1 + attr = parts[0] + msgid = None + addAttribute(d, attr, msgid, position, xml) + else: + i18nattrlist = i18nattrs.split() + if len(i18nattrlist) == 2: + staticattrs = [attr[0] for attr in attrlist if len(attr) == 2] + if (not i18nattrlist[1] in staticattrs) and ( + not i18nattrlist[1] in repldict): + attr, msgid = i18nattrlist + addAttribute(d, attr, msgid, position, xml) + else: + msgid = None + for attr in i18nattrlist: + addAttribute(d, attr, msgid, position, xml) + else: + msgid = None + for attr in i18nattrlist: + addAttribute(d, attr, msgid, position, xml) + return d + def test(): t = TALGenerator() t.pushProgram()
--- a/roundup/cgi/TAL/TALInterpreter.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/TALInterpreter.py Fri May 21 05:36:30 2004 +0000 @@ -2,32 +2,35 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Interpreter for a pre-compiled TAL program. +# Modifications for Roundup: +# 1. implemented ustr as str """ -__docformat__ = 'restructuredtext' +Interpreter for a pre-compiled TAL program. +""" import sys import getopt - +import re +from types import ListType from cgi import escape +# Do not use cStringIO here! It's not unicode aware. :( +from StringIO import StringIO +#from DocumentTemplate.DT_Util import ustr +ustr = str -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -from TALDefs import quote, TAL_VERSION, TALError, METALError +from TALDefs import TAL_VERSION, TALError, METALError, attrEscape from TALDefs import isCurrentVersion, getProgramVersion, getProgramMode from TALGenerator import TALGenerator +from TranslationContext import TranslationContext BOOLEAN_HTML_ATTRS = [ # List of Boolean attributes in HTML that should be rendered in @@ -40,13 +43,43 @@ "defer" ] -EMPTY_HTML_TAGS = [ - # List of HTML tags with an empty content model; these are - # rendered in minimized form, e.g. <img />. - # From http://www.w3.org/TR/xhtml1/#dtds - "base", "meta", "link", "hr", "br", "param", "img", "area", - "input", "col", "basefont", "isindex", "frame", -] +def normalize(text): + # Now we need to normalize the whitespace in implicit message ids and + # implicit $name substitution values by stripping leading and trailing + # whitespace, and folding all internal whitespace to a single space. + return ' '.join(text.split()) + + +NAME_RE = r"[a-zA-Z][a-zA-Z0-9_]*" +_interp_regex = re.compile(r'(?<!\$)(\$(?:%(n)s|{%(n)s}))' %({'n': NAME_RE})) +_get_var_regex = re.compile(r'%(n)s' %({'n': NAME_RE})) + +def interpolate(text, mapping): + """Interpolate ${keyword} substitutions. + + This is called when no translation is provided by the translation + service. + """ + if not mapping: + return text + # Find all the spots we want to substitute. + to_replace = _interp_regex.findall(text) + # Now substitute with the variables in mapping. + for string in to_replace: + var = _get_var_regex.findall(string)[0] + if mapping.has_key(var): + # Call ustr because we may have an integer for instance. + subst = ustr(mapping[var]) + try: + text = text.replace(string, subst) + except UnicodeError: + # subst contains high-bit chars... + # As we have no way of knowing the correct encoding, + # substitue something instead of raising an exception. + subst = `subst`[1:-1] + text = text.replace(string, subst) + return text + class AltTALGenerator(TALGenerator): @@ -60,16 +93,18 @@ def emit(self, *args): if self.enabled: - apply(TALGenerator.emit, (self,) + args) + TALGenerator.emit(self, *args) - def emitStartElement(self, name, attrlist, taldict, metaldict, + def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict, position=(None, None), isend=0): metaldict = {} taldict = {} + i18ndict = {} if self.enabled and self.repldict: taldict["attributes"] = "x x" TALGenerator.emitStartElement(self, name, attrlist, - taldict, metaldict, position, isend) + taldict, metaldict, i18ndict, + position, isend) def replaceAttrs(self, attrlist, repldict): if self.enabled and self.repldict: @@ -82,10 +117,10 @@ def __init__(self, program, macros, engine, stream=None, debug=0, wrap=60, metal=1, tal=1, showtal=-1, - strictinsert=1, stackLimit=100): + strictinsert=1, stackLimit=100, i18nInterpolate=1): self.program = program self.macros = macros - self.engine = engine + self.engine = engine # Execution engine (aka context) self.Default = engine.getDefault() self.stream = stream or sys.stdout self._stream_write = self.stream.write @@ -107,28 +142,36 @@ self.endsep = "/>" self.endlen = len(self.endsep) self.macroStack = [] - self.popMacro = self.macroStack.pop self.position = None, None # (lineno, offset) self.col = 0 self.level = 0 self.scopeLevel = 0 self.sourceFile = None + self.i18nStack = [] + self.i18nInterpolate = i18nInterpolate + self.i18nContext = TranslationContext() + + def StringIO(self): + # Third-party products wishing to provide a full Unicode-aware + # StringIO can do so by monkey-patching this method. + return FasterStringIO() def saveState(self): return (self.position, self.col, self.stream, - self.scopeLevel, self.level) + self.scopeLevel, self.level, self.i18nContext) def restoreState(self, state): - (self.position, self.col, self.stream, scopeLevel, level) = state + (self.position, self.col, self.stream, scopeLevel, level, i18n) = state self._stream_write = self.stream.write assert self.level == level while self.scopeLevel > scopeLevel: self.engine.endScope() self.scopeLevel = self.scopeLevel - 1 self.engine.setPosition(self.position) + self.i18nContext = i18n def restoreOutputState(self, state): - (dummy, self.col, self.stream, scopeLevel, level) = state + (dummy, self.col, self.stream, scopeLevel, level, i18n) = state self._stream_write = self.stream.write assert self.level == level assert self.scopeLevel == scopeLevel @@ -137,28 +180,25 @@ if len(self.macroStack) >= self.stackLimit: raise METALError("macro nesting limit (%d) exceeded " "by %s" % (self.stackLimit, `macroName`)) - self.macroStack.append([macroName, slots, entering]) + self.macroStack.append([macroName, slots, entering, self.i18nContext]) - def macroContext(self, what): - macroStack = self.macroStack - i = len(macroStack) - while i > 0: - i = i-1 - if macroStack[i][0] == what: - return i - return -1 + def popMacro(self): + return self.macroStack.pop() def __call__(self): assert self.level == 0 assert self.scopeLevel == 0 + assert self.i18nContext.parent is None self.interpret(self.program) assert self.level == 0 assert self.scopeLevel == 0 + assert self.i18nContext.parent is None if self.col > 0: self._stream_write("\n") self.col = 0 - def stream_write(self, s, len=len): + def stream_write(self, s, + len=len): self._stream_write(s) i = s.rfind('\n') if i < 0: @@ -168,6 +208,16 @@ bytecode_handlers = {} + def interpretWithStream(self, program, stream): + oldstream = self.stream + self.stream = stream + self._stream_write = stream.write + try: + self.interpret(program) + finally: + self.stream = oldstream + self._stream_write = oldstream.write + def interpret(self, program): oldlevel = self.level self.level = oldlevel + 1 @@ -175,8 +225,8 @@ try: if self.debug: for (opcode, args) in program: - s = "%sdo_%s%s\n" % (" "*self.level, opcode, - repr(args)) + s = "%sdo_%s(%s)\n" % (" "*self.level, opcode, + repr(args)) if len(s) > 80: s = s[:76] + "...\n" sys.stderr.write(s) @@ -221,10 +271,9 @@ # for start tags with no attributes; those are optimized down # to rawtext events. Hence, there is no special "fast path" # for that case. - _stream_write = self._stream_write - _stream_write("<" + name) - namelen = _len(name) - col = self.col + namelen + 1 + L = ["<", name] + append = L.append + col = self.col + _len(name) + 1 wrap = self.wrap align = col + 1 if align >= wrap/2: @@ -235,20 +284,28 @@ if _len(item) == 2: name, s = item else: - ok, name, s = attrAction(self, item) + # item[2] is the 'action' field: + if item[2] in ('metal', 'tal', 'xmlns', 'i18n'): + if not self.showtal: + continue + ok, name, s = self.attrAction(item) + else: + ok, name, s = attrAction(self, item) if not ok: continue slen = _len(s) if (wrap and col >= align and col + 1 + slen > wrap): - _stream_write("\n" + " "*align) + append("\n") + append(" "*align) col = align + slen else: - s = " " + s + append(" ") col = col + 1 + slen - _stream_write(s) - _stream_write(end) + append(s) + append(end) + self._stream_write("".join(L)) col = col + endlen finally: self.col = col @@ -256,10 +313,10 @@ def attrAction(self, item): name, value, action = item[:3] - if action == 1 or (action > 1 and not self.showtal): + if action == 'insert': return 0, name, value macs = self.macroStack - if action == 2 and self.metal and macs: + if action == 'metal' and self.metal and macs: if len(macs) > 1 or not macs[-1][2]: # Drop all METAL attributes at a use-depth above one. return 0, name, value @@ -273,7 +330,7 @@ name = prefix + "use-macro" value = macs[-1][0] # Macro name elif suffix == "define-slot": - name = prefix + "slot" + name = prefix + "fill-slot" elif suffix == "fill-slot": pass else: @@ -282,43 +339,52 @@ if value is None: value = name else: - value = "%s=%s" % (name, quote(value)) + value = '%s="%s"' % (name, attrEscape(value)) return 1, name, value def attrAction_tal(self, item): name, value, action = item[:3] - if action > 1: - return self.attrAction(item) ok = 1 + expr, xlat, msgid = item[3:] if self.html and name.lower() in BOOLEAN_HTML_ATTRS: evalue = self.engine.evaluateBoolean(item[3]) if evalue is self.Default: - if action == 1: # Cancelled insert + if action == 'insert': # Cancelled insert ok = 0 elif evalue: value = None else: ok = 0 - else: + elif expr is not None: evalue = self.engine.evaluateText(item[3]) if evalue is self.Default: - if action == 1: # Cancelled insert + if action == 'insert': # Cancelled insert ok = 0 else: if evalue is None: ok = 0 value = evalue + else: + evalue = None + if ok: + if xlat: + translated = self.translate(msgid or value, value, {}) + if translated is not None: + value = translated if value is None: value = name - value = "%s=%s" % (name, quote(value)) + elif evalue is self.Default: + value = attrEscape(value) + else: + value = escape(value, quote=1) + value = '%s="%s"' % (name, value) return ok, name, value - bytecode_handlers["<attrAction>"] = attrAction def no_tag(self, start, program): state = self.saveState() - self.stream = stream = StringIO() + self.stream = stream = self.StringIO() self._stream_write = stream.write self.interpret(start) self.restoreOutputState(state) @@ -328,7 +394,7 @@ omit=0): if tag_ns and not self.showtal: return self.no_tag(start, program) - + self.interpret(start) if not isend: self.interpret(program) @@ -345,18 +411,11 @@ self.do_optTag(stuff) bytecode_handlers["optTag"] = do_optTag - def dumpMacroStack(self, prefix, suffix, value): - sys.stderr.write("+---- %s%s = %s\n" % (prefix, suffix, value)) - for i in range(len(self.macroStack)): - what, macroName, slots = self.macroStack[i] - sys.stderr.write("| %2d. %-12s %-12s %s\n" % - (i, what, macroName, slots and slots.keys())) - sys.stderr.write("+--------------------------------------\n") - def do_rawtextBeginScope(self, (s, col, position, closeprev, dict)): self._stream_write(s) self.col = col - self.do_setPosition(position) + self.position = position + self.engine.setPosition(position) if closeprev: engine = self.engine engine.endScope() @@ -368,8 +427,9 @@ def do_rawtextBeginScope_tal(self, (s, col, position, closeprev, dict)): self._stream_write(s) self.col = col - self.do_setPosition(position) engine = self.engine + self.position = position + engine.setPosition(position) if closeprev: engine.endScope() engine.beginScope() @@ -406,6 +466,19 @@ self.engine.setGlobal(name, self.engine.evaluateValue(expr)) bytecode_handlers["setGlobal"] = do_setLocal + def do_beginI18nContext(self, settings): + get = settings.get + self.i18nContext = TranslationContext(self.i18nContext, + domain=get("domain"), + source=get("source"), + target=get("target")) + bytecode_handlers["beginI18nContext"] = do_beginI18nContext + + def do_endI18nContext(self, notused=None): + self.i18nContext = self.i18nContext.parent + assert self.i18nContext is not None + bytecode_handlers["endI18nContext"] = do_endI18nContext + def do_insertText(self, stuff): self.interpret(stuff[1]) @@ -425,6 +498,65 @@ self.col = len(s) - (i + 1) bytecode_handlers["insertText"] = do_insertText + def do_i18nVariable(self, stuff): + varname, program, expression = stuff + if expression is None: + # The value is implicitly the contents of this tag, so we have to + # evaluate the mini-program to get the value of the variable. + state = self.saveState() + try: + tmpstream = self.StringIO() + self.interpretWithStream(program, tmpstream) + value = normalize(tmpstream.getvalue()) + finally: + self.restoreState(state) + else: + # Evaluate the value to be associated with the variable in the + # i18n interpolation dictionary. + value = self.engine.evaluate(expression) + # Either the i18n:name tag is nested inside an i18n:translate in which + # case the last item on the stack has the i18n dictionary and string + # representation, or the i18n:name and i18n:translate attributes are + # in the same tag, in which case the i18nStack will be empty. In that + # case we can just output the ${name} to the stream + i18ndict, srepr = self.i18nStack[-1] + i18ndict[varname] = value + placeholder = '${%s}' % varname + srepr.append(placeholder) + self._stream_write(placeholder) + bytecode_handlers['i18nVariable'] = do_i18nVariable + + def do_insertTranslation(self, stuff): + i18ndict = {} + srepr = [] + obj = None + self.i18nStack.append((i18ndict, srepr)) + msgid = stuff[0] + # We need to evaluate the content of the tag because that will give us + # several useful pieces of information. First, the contents will + # include an implicit message id, if no explicit one was given. + # Second, it will evaluate any i18nVariable definitions in the body of + # the translation (necessary for $varname substitutions). + # + # Use a temporary stream to capture the interpretation of the + # subnodes, which should /not/ go to the output stream. + tmpstream = self.StringIO() + self.interpretWithStream(stuff[1], tmpstream) + default = tmpstream.getvalue() + # We only care about the evaluated contents if we need an implicit + # message id. All other useful information will be in the i18ndict on + # the top of the i18nStack. + if msgid == '': + msgid = normalize(default) + self.i18nStack.pop() + # See if there is was an i18n:data for msgid + if len(stuff) > 2: + obj = self.engine.evaluate(stuff[2]) + xlated_msgid = self.translate(msgid, default, i18ndict, obj) + assert xlated_msgid is not None, self.position + self._stream_write(xlated_msgid) + bytecode_handlers['insertTranslation'] = do_insertTranslation + def do_insertStructure(self, stuff): self.interpret(stuff[2]) @@ -435,7 +567,7 @@ if structure is self.Default: self.interpret(block) return - text = str(structure) + text = ustr(structure) if not (repldict or self.strictinsert): # Take a shortcut, no error checking self.stream_write(text) @@ -448,7 +580,7 @@ def insertHTMLStructure(self, text, repldict): from HTMLTALParser import HTMLTALParser - gen = AltTALGenerator(repldict, self.engine, 0) + gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0) p = HTMLTALParser(gen) # Raises an exception if text is invalid p.parseString(text) program, macros = p.getCode() @@ -456,7 +588,7 @@ def insertXMLStructure(self, text, repldict): from TALParser import TALParser - gen = AltTALGenerator(repldict, self.engine, 0) + gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0) p = TALParser(gen) gen.enable(0) p.parseFragment('<!DOCTYPE foo PUBLIC "foo" "bar"><foo>') @@ -476,6 +608,15 @@ self.interpret(block) bytecode_handlers["loop"] = do_loop + def translate(self, msgid, default, i18ndict, obj=None): + if obj: + i18ndict.update(obj) + if not self.i18nInterpolate: + return msgid + # XXX We need to pass in one of context or target_language + return self.engine.translate(self.i18nContext.domain, + msgid, i18ndict, default=default) + def do_rawtextColumn(self, (s, col)): self._stream_write(s) self.col = col @@ -498,6 +639,7 @@ if not entering: macs.append(None) self.interpret(macro) + assert macs[-1] is None macs.pop() return self.interpret(macro) @@ -520,12 +662,11 @@ raise METALError("macro %s has incompatible mode %s" % (`macroName`, `mode`), self.position) self.pushMacro(macroName, compiledSlots) - saved_source = self.sourceFile - saved_position = self.position # Used by Boa Constructor + prev_source = self.sourceFile self.interpret(macro) - if self.sourceFile != saved_source: - self.engine.setSourceFile(saved_source) - self.sourceFile = saved_source + if self.sourceFile != prev_source: + self.engine.setSourceFile(prev_source) + self.sourceFile = prev_source self.popMacro() bytecode_handlers["useMacro"] = do_useMacro @@ -541,21 +682,18 @@ return macs = self.macroStack if macs and macs[-1] is not None: - saved_source = self.sourceFile - saved_position = self.position # Used by Boa Constructor macroName, slots = self.popMacro()[:2] slot = slots.get(slotName) if slot is not None: + prev_source = self.sourceFile self.interpret(slot) - if self.sourceFile != saved_source: - self.engine.setSourceFile(saved_source) - self.sourceFile = saved_source + if self.sourceFile != prev_source: + self.engine.setSourceFile(prev_source) + self.sourceFile = prev_source self.pushMacro(macroName, slots, entering=0) return self.pushMacro(macroName, slots) - if len(macs) == 1: - self.interpret(block) - return + # Falling out of the 'if' allows the macro to be interpreted. self.interpret(block) bytecode_handlers["defineSlot"] = do_defineSlot @@ -564,7 +702,7 @@ def do_onError_tal(self, (block, handler)): state = self.saveState() - self.stream = stream = StringIO() + self.stream = stream = self.StringIO() self._stream_write = stream.write try: self.interpret(block) @@ -597,24 +735,24 @@ bytecode_handlers_tal["optTag"] = do_optTag_tal -def test(): - from driver import FILE, parsefile - from DummyEngine import DummyEngine - try: - opts, args = getopt.getopt(sys.argv[1:], "") - except getopt.error, msg: - print msg - sys.exit(2) - if args: - file = args[0] - else: - file = FILE - doc = parsefile(file) - compiler = TALCompiler(doc) - program, macros = compiler() - engine = DummyEngine() - interpreter = TALInterpreter(program, macros, engine) - interpreter() +class FasterStringIO(StringIO): + """Append-only version of StringIO. + + This let's us have a much faster write() method. + """ + def close(self): + if not self.closed: + self.write = _write_ValueError + StringIO.close(self) -if __name__ == "__main__": - test() + def seek(self, pos, mode=0): + raise RuntimeError("FasterStringIO.seek() not allowed") + + def write(self, s): + #assert self.pos == self.len + self.buflist.append(s) + self.len = self.pos = self.pos + len(s) + + +def _write_ValueError(s): + raise ValueError, "I/O operation on closed file"
--- a/roundup/cgi/TAL/TALParser.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/TALParser.py Fri May 21 05:36:30 2004 +0000 @@ -2,22 +2,21 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Parse XML and compile to TALInterpreter intermediate code. """ -__docformat__ = 'restructuredtext' +Parse XML and compile to TALInterpreter intermediate code. +""" -import string from XMLParser import XMLParser -from TALDefs import * +from TALDefs import XML_NS, ZOPE_I18N_NS, ZOPE_METAL_NS, ZOPE_TAL_NS from TALGenerator import TALGenerator class TALParser(XMLParser): @@ -59,13 +58,15 @@ # attrs is a dict of {name: value} attrlist = attrs.items() attrlist.sort() # For definiteness - name, attrlist, taldict, metaldict = self.process_ns(name, attrlist) + name, attrlist, taldict, metaldict, i18ndict \ + = self.process_ns(name, attrlist) attrlist = self.xmlnsattrs() + attrlist - self.gen.emitStartElement(name, attrlist, taldict, metaldict) + self.gen.emitStartElement(name, attrlist, taldict, metaldict, i18ndict) def process_ns(self, name, attrlist): taldict = {} metaldict = {} + i18ndict = {} fixedattrlist = [] name, namebase, namens = self.fixname(name) for key, value in attrlist: @@ -78,10 +79,14 @@ elif ns == 'tal': taldict[keybase] = value item = item + ("tal",) + elif ns == 'i18n': + assert 0, "dealing with i18n: " + `(keybase, value)` + i18ndict[keybase] = value + item = item + ('i18n',) fixedattrlist.append(item) - if namens in ('metal', 'tal'): + if namens in ('metal', 'tal', 'i18n'): taldict['tal tag'] = namens - return name, fixedattrlist, taldict, metaldict + return name, fixedattrlist, taldict, metaldict, i18ndict def xmlnsattrs(self): newlist = [] @@ -90,7 +95,7 @@ key = "xmlns:" + prefix else: key = "xmlns" - if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS): + if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS): item = (key, uri, "xmlns") else: item = (key, uri) @@ -100,7 +105,7 @@ def fixname(self, name): if ' ' in name: - uri, name = string.split(name, ' ') + uri, name = name.split(' ') prefix = self.nsDict[uri] prefixed = name if prefix: @@ -110,6 +115,8 @@ ns = 'tal' elif uri == ZOPE_METAL_NS: ns = 'metal' + elif uri == ZOPE_I18N_NS: + ns = 'i18n' return (prefixed, name, ns) return (name, name, None)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/roundup/cgi/TAL/TranslationContext.py Fri May 21 05:36:30 2004 +0000 @@ -0,0 +1,41 @@ +############################################################################## +# +# Copyright (c) 2001, 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Translation context object for the TALInterpreter's I18N support. + +The translation context provides a container for the information +needed to perform translation of a marked string from a page template. + +$Id: TranslationContext.py,v 1.1 2004-05-21 05:36:30 richard Exp $ +""" + +DEFAULT_DOMAIN = "default" + +class TranslationContext: + """Information about the I18N settings of a TAL processor.""" + + def __init__(self, parent=None, domain=None, target=None, source=None): + if parent: + if not domain: + domain = parent.domain + if not target: + target = parent.target + if not source: + source = parent.source + elif domain is None: + domain = DEFAULT_DOMAIN + + self.parent = parent + self.domain = domain + self.target = target + self.source = source
--- a/roundup/cgi/TAL/XMLParser.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/XMLParser.py Fri May 21 05:36:30 2004 +0000 @@ -2,23 +2,22 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE -# +# ############################################################################## -"""Generic expat-based XML parser base class. +# Modifications for Roundup: +# 1. commented out zLOG references +""" +Generic expat-based XML parser base class. +""" -Modified for Roundup 0.5 release: - -- removed dependency on zLOG - -""" -__docformat__ = 'restructuredtext' +#import zLOG class XMLParser:
--- a/roundup/cgi/TAL/__init__.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/__init__.py Fri May 21 05:36:30 2004 +0000 @@ -2,14 +2,13 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE -# +# ############################################################################## -"""Template Attribute Language package """ -__docformat__ = 'restructuredtext' +""" Template Attribute Language package """
--- a/roundup/cgi/TAL/markupbase.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/markupbase.py Fri May 21 05:36:30 2004 +0000 @@ -1,9 +1,6 @@ -"""Shared support for scanning document type declarations in HTML and XHTML. -""" -__docformat__ = 'restructuredtext' +"""Shared support for scanning document type declarations in HTML and XHTML.""" -import re -import string +import re, string _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match _declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match @@ -23,6 +20,13 @@ """Return current line number and offset.""" return self.lineno, self.offset + def error(self, message): + """Return an error, showing current line number and offset. + + Concrete subclasses *must* override this method. + """ + raise NotImplementedError + # Internal -- update line number and offset. This should be # called for each piece of data exactly once, in order -- in other # words the concatenation of all the input strings to this @@ -31,10 +35,10 @@ if i >= j: return j rawdata = self.rawdata - nlines = string.count(rawdata, "\n", i, j) + nlines = rawdata.count("\n", i, j) if nlines: self.lineno = self.lineno + nlines - pos = string.rindex(rawdata, "\n", i, j) # Should not fail + pos = rawdata.rindex("\n", i, j) # Should not fail self.offset = j-(pos+1) else: self.offset = self.offset + j-i @@ -171,7 +175,7 @@ return -1 # style content model; just skip until '>' if '>' in rawdata[j:]: - return string.find(rawdata, ">", j) + 1 + return rawdata.find(">", j) + 1 return -1 # Internal -- scan past <!ATTLIST declarations @@ -195,10 +199,10 @@ if c == "(": # an enumerated type; look for ')' if ")" in rawdata[j:]: - j = string.find(rawdata, ")", j) + 1 + j = rawdata.find(")", j) + 1 else: return -1 - while rawdata[j:j+1] in string.whitespace: + while rawdata[j:j+1].isspace(): j = j + 1 if not rawdata[j:]: # end of buffer, incomplete @@ -299,10 +303,10 @@ m = _declname_match(rawdata, i) if m: s = m.group() - name = string.strip(s) + name = s.strip() if (i + len(s)) == n: return None, -1 # end of buffer - return string.lower(name), m.end() + return name.lower(), m.end() else: self.updatepos(declstartpos, i) - self.error("expected name token", self.getpos()) + self.error("expected name token")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/roundup/cgi/TAL/talgettext.py Fri May 21 05:36:30 2004 +0000 @@ -0,0 +1,313 @@ +#!/usr/bin/env python +############################################################################## +# +# Copyright (c) 2002 Zope Corporation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +# Modifications for Roundup: +# 1. commented out ITALES references + +"""Program to extract internationalization markup from Page Templates. + +Once you have marked up a Page Template file with i18n: namespace tags, use +this program to extract GNU gettext .po file entries. + +Usage: talgettext.py [options] files +Options: + -h / --help + Print this message and exit. + -o / --output <file> + Output the translation .po file to <file>. + -u / --update <file> + Update the existing translation <file> with any new translation strings + found. +""" + +import sys +import time +import getopt +import traceback + +from TAL.HTMLTALParser import HTMLTALParser +from TAL.TALInterpreter import TALInterpreter +from TAL.DummyEngine import DummyEngine +#from ITALES import ITALESEngine +from TAL.TALDefs import TALESError + +__version__ = '$Revision: 1.1 $' + +pot_header = '''\ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR ORGANIZATION +# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\\n" +"POT-Creation-Date: %(time)s\\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n" +"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n" +"Language-Team: LANGUAGE <LL@li.org>\\n" +"MIME-Version: 1.0\\n" +"Content-Type: text/plain; charset=CHARSET\\n" +"Content-Transfer-Encoding: ENCODING\\n" +"Generated-By: talgettext.py %(version)s\\n" +''' + +NLSTR = '"\n"' + +try: + True +except NameError: + True=1 + False=0 + +def usage(code, msg=''): + # Python 2.1 required + print >> sys.stderr, __doc__ + if msg: + print >> sys.stderr, msg + sys.exit(code) + + +class POTALInterpreter(TALInterpreter): + def translate(self, msgid, default, i18ndict=None, obj=None): + # XXX is this right? + if i18ndict is None: + i18ndict = {} + if obj: + i18ndict.update(obj) + # XXX Mmmh, it seems that sometimes the msgid is None; is that really + # possible? + if msgid is None: + return None + # XXX We need to pass in one of context or target_language + return self.engine.translate(msgid, self.i18nContext.domain, i18ndict, + position=self.position, default=default) + + +class POEngine(DummyEngine): + #__implements__ = ITALESEngine + + def __init__(self, macros=None): + self.catalog = {} + DummyEngine.__init__(self, macros) + + def evaluate(*args): + return '' # who cares + + def evaluatePathOrVar(*args): + return '' # who cares + + def evaluateSequence(self, expr): + return (0,) # dummy + + def evaluateBoolean(self, expr): + return True # dummy + + def translate(self, msgid, domain=None, mapping=None, default=None, + # XXX position is not part of the ITALESEngine + # interface + position=None): + + if domain not in self.catalog: + self.catalog[domain] = {} + domain = self.catalog[domain] + + if msgid not in domain: + domain[msgid] = [] + domain[msgid].append((self.file, position)) + return 'x' + + +class UpdatePOEngine(POEngine): + """A slightly-less braindead POEngine which supports loading an existing + .po file first.""" + + def __init__ (self, macros=None, filename=None): + POEngine.__init__(self, macros) + + self._filename = filename + self._loadFile() + self.base = self.catalog + self.catalog = {} + + def __add(self, id, s, fuzzy): + "Add a non-fuzzy translation to the dictionary." + if not fuzzy and str: + # check for multi-line values and munge them appropriately + if '\n' in s: + lines = s.rstrip().split('\n') + s = NLSTR.join(lines) + self.catalog[id] = s + + def _loadFile(self): + # shamelessly cribbed from Python's Tools/i18n/msgfmt.py + # 25-Mar-2003 Nathan R. Yergler (nathan@zope.org) + # 14-Apr-2003 Hacked by Barry Warsaw (barry@zope.com) + + ID = 1 + STR = 2 + + try: + lines = open(self._filename).readlines() + except IOError, msg: + print >> sys.stderr, msg + sys.exit(1) + + section = None + fuzzy = False + + # Parse the catalog + lno = 0 + for l in lines: + lno += True + # If we get a comment line after a msgstr, this is a new entry + if l[0] == '#' and section == STR: + self.__add(msgid, msgstr, fuzzy) + section = None + fuzzy = False + # Record a fuzzy mark + if l[:2] == '#,' and l.find('fuzzy'): + fuzzy = True + # Skip comments + if l[0] == '#': + continue + # Now we are in a msgid section, output previous section + if l.startswith('msgid'): + if section == STR: + self.__add(msgid, msgstr, fuzzy) + section = ID + l = l[5:] + msgid = msgstr = '' + # Now we are in a msgstr section + elif l.startswith('msgstr'): + section = STR + l = l[6:] + # Skip empty lines + if not l.strip(): + continue + # XXX: Does this always follow Python escape semantics? + l = eval(l) + if section == ID: + msgid += l + elif section == STR: + msgstr += '%s\n' % l + else: + print >> sys.stderr, 'Syntax error on %s:%d' % (infile, lno), \ + 'before:' + print >> sys.stderr, l + sys.exit(1) + # Add last entry + if section == STR: + self.__add(msgid, msgstr, fuzzy) + + def evaluate(self, expression): + try: + return POEngine.evaluate(self, expression) + except TALESError: + pass + + def evaluatePathOrVar(self, expr): + return 'who cares' + + def translate(self, msgid, domain=None, mapping=None, default=None, + position=None): + if msgid not in self.base: + POEngine.translate(self, msgid, domain, mapping, default, position) + return 'x' + + +def main(): + try: + opts, args = getopt.getopt( + sys.argv[1:], + 'ho:u:', + ['help', 'output=', 'update=']) + except getopt.error, msg: + usage(1, msg) + + outfile = None + engine = None + update_mode = False + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-o', '--output'): + outfile = arg + elif opt in ('-u', '--update'): + update_mode = True + if outfile is None: + outfile = arg + engine = UpdatePOEngine(filename=arg) + + if not args: + print 'nothing to do' + return + + # We don't care about the rendered output of the .pt file + class Devnull: + def write(self, s): + pass + + # check if we've already instantiated an engine; + # if not, use the stupidest one available + if not engine: + engine = POEngine() + + # process each file specified + for filename in args: + try: + engine.file = filename + p = HTMLTALParser() + p.parseFile(filename) + program, macros = p.getCode() + POTALInterpreter(program, macros, engine, stream=Devnull(), + metal=False)() + except: # Hee hee, I love bare excepts! + print 'There was an error processing', filename + traceback.print_exc() + + # Now output the keys in the engine. Write them to a file if --output or + # --update was specified; otherwise use standard out. + if (outfile is None): + outfile = sys.stdout + else: + outfile = file(outfile, update_mode and "a" or "w") + + catalog = {} + for domain in engine.catalog.keys(): + catalog.update(engine.catalog[domain]) + + messages = catalog.copy() + try: + messages.update(engine.base) + except AttributeError: + pass + if '' not in messages: + print >> outfile, pot_header % {'time': time.ctime(), + 'version': __version__} + + msgids = catalog.keys() + # XXX: You should not sort by msgid, but by filename and position. (SR) + msgids.sort() + for msgid in msgids: + positions = catalog[msgid] + for filename, position in positions: + outfile.write('#: %s:%s\n' % (filename, position[0])) + + outfile.write('msgid "%s"\n' % msgid) + outfile.write('msgstr ""\n') + outfile.write('\n') + + +if __name__ == '__main__': + main()