Mercurial > p > roundup > code
diff roundup/cgi/TAL/TALGenerator.py @ 2348:8c2402a78bb0
beginning getting ZPT up to date: TAL first
| author | Richard Jones <richard@users.sourceforge.net> |
|---|---|
| date | Fri, 21 May 2004 05:36:30 +0000 |
| parents | fc52d57c6c3e |
| children | 198b6e810c67 |
line wrap: on
line diff
--- a/roundup/cgi/TAL/TALGenerator.py Thu May 20 23:16:58 2004 +0000 +++ b/roundup/cgi/TAL/TALGenerator.py Fri May 21 05:36:30 2004 +0000 @@ -2,39 +2,57 @@ # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. -# +# # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE -# +# FOR A PARTICULAR PURPOSE. +# ############################################################################## -"""Code generator for TALInterpreter intermediate code. """ -__docformat__ = 'restructuredtext' +Code generator for TALInterpreter intermediate code. +""" -import string import re import cgi -from TALDefs import * +import TALDefs + +from TALDefs import NAME_RE, TAL_VERSION +from TALDefs import I18NError, METALError, TALError +from TALDefs import parseSubstitution +from TranslationContext import TranslationContext, DEFAULT_DOMAIN + +I18N_REPLACE = 1 +I18N_CONTENT = 2 +I18N_EXPRESSION = 3 + +_name_rx = re.compile(NAME_RE) + class TALGenerator: inMacroUse = 0 inMacroDef = 0 source_file = None - + def __init__(self, expressionCompiler=None, xml=1, source_file=None): if not expressionCompiler: from DummyEngine import DummyEngine expressionCompiler = DummyEngine() self.expressionCompiler = expressionCompiler self.CompilerError = expressionCompiler.getCompilerError() + # This holds the emitted opcodes representing the input self.program = [] + # The program stack for when we need to do some sub-evaluation for an + # intermediate result. E.g. in an i18n:name tag for which the + # contents describe the ${name} value. self.stack = [] + # Another stack of postponed actions. Elements on this stack are a + # dictionary; key/values contain useful information that + # emitEndElement needs to finish its calculations self.todoStack = [] self.macros = {} self.slots = {} @@ -45,6 +63,8 @@ if source_file is not None: self.source_file = source_file self.emit("setSourceFile", source_file) + self.i18nContext = TranslationContext() + self.i18nLevel = 0 def getCode(self): assert not self.stack @@ -54,7 +74,7 @@ def optimize(self, program): output = [] collect = [] - rawseen = cursor = 0 + cursor = 0 if self.xml: endsep = "/>" else: @@ -83,9 +103,15 @@ # instructions to be joined together. output.append(self.optimizeArgsList(item)) continue - text = string.join(collect, "") + if opcode == 'noop': + # This is a spacer for end tags in the face of i18n:name + # attributes. We can't let the optimizer collect immediately + # following end tags into the same rawtextOffset. + opcode = None + pass + text = "".join(collect) if text: - i = string.rfind(text, "\n") + i = text.rfind("\n") if i >= 0: i = len(text) - (i + 1) output.append(("rawtextColumn", (text, i))) @@ -93,7 +119,6 @@ output.append(("rawtextOffset", (text, len(text)))) if opcode != None: output.append(self.optimizeArgsList(item)) - rawseen = cursor+1 collect = [] return self.optimizeCommonTriple(output) @@ -103,9 +128,28 @@ else: return item[0], tuple(item[1:]) - actionIndex = {"replace":0, "insert":1, "metal":2, "tal":3, "xmlns":4, - 0: 0, 1: 1, 2: 2, 3: 3, 4: 4} + # These codes are used to indicate what sort of special actions + # are needed for each special attribute. (Simple attributes don't + # get action codes.) + # + # The special actions (which are modal) are handled by + # TALInterpreter.attrAction() and .attrAction_tal(). + # + # Each attribute is represented by a tuple: + # + # (name, value) -- a simple name/value pair, with + # no special processing + # + # (name, value, action, *extra) -- attribute with special + # processing needs, action is a + # code that indicates which + # branch to take, and *extra + # contains additional, + # action-specific information + # needed by the processing + # def optimizeStartTag(self, collect, name, attrlist, end): + # return true if the tag can be converted to plain text if not attrlist: collect.append("<%s%s" % (name, end)) return 1 @@ -116,18 +160,15 @@ if len(item) > 2: opt = 0 name, value, action = item[:3] - action = self.actionIndex[action] attrlist[i] = (name, value, action) + item[3:] else: if item[1] is None: s = item[0] else: - s = "%s=%s" % (item[0], quote(item[1])) + s = '%s="%s"' % (item[0], TALDefs.attrEscape(item[1])) attrlist[i] = item[0], s - if item[1] is None: - new.append(" " + item[0]) - else: - new.append(" %s=%s" % (item[0], quote(item[1]))) + new.append(" " + s) + # if no non-optimizable attributes were found, convert to plain text if opt: new.append(end) collect.extend(new) @@ -139,9 +180,9 @@ output = program[:2] prev2, prev1 = output for item in program[2:]: - if ( item[0] == "beginScope" - and prev1[0] == "setPosition" - and prev2[0] == "rawtextColumn"): + if ( item[0] == "beginScope" + and prev1[0] == "setPosition" + and prev2[0] == "rawtextColumn"): position = output.pop()[1] text, column = output.pop()[1] prev1 = None, None @@ -215,7 +256,7 @@ if cexpr: cexpr = self.compileExpression(optTag[0]) self.emit("optTag", name, cexpr, optTag[1], isend, start, program) - + def emitRawText(self, text): self.emit("rawtext", text) @@ -223,7 +264,7 @@ self.emitRawText(cgi.escape(text)) def emitDefines(self, defines): - for part in splitParts(defines): + for part in TALDefs.splitParts(defines): m = re.match( r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part) if not m: @@ -278,9 +319,56 @@ assert key == "structure" self.emit("insertStructure", cexpr, attrDict, program) + def emitI18nVariable(self, stuff): + # Used for i18n:name attributes. arg is extra information describing + # how the contents of the variable should get filled in, and it will + # either be a 1-tuple or a 2-tuple. If arg[0] is None, then the + # i18n:name value is taken implicitly from the contents of the tag, + # e.g. "I live in <span i18n:name="country">the USA</span>". In this + # case, arg[1] is the opcode sub-program describing the contents of + # the tag. + # + # When arg[0] is not None, it contains the tal expression used to + # calculate the contents of the variable, e.g. + # "I live in <span i18n:name="country" + # tal:replace="here/countryOfOrigin" />" + varname, action, expression = stuff + m = _name_rx.match(varname) + if m is None or m.group() != varname: + raise TALError("illegal i18n:name: %r" % varname, self.position) + key = cexpr = None + program = self.popProgram() + if action == I18N_REPLACE: + # This is a tag with an i18n:name and a tal:replace (implicit or + # explicit). Get rid of the first and last elements of the + # program, which are the start and end tag opcodes of the tag. + program = program[1:-1] + elif action == I18N_CONTENT: + # This is a tag with an i18n:name and a tal:content + # (explicit-only). Keep the first and last elements of the + # program, so we keep the start and end tag output. + pass + else: + assert action == I18N_EXPRESSION + key, expr = parseSubstitution(expression) + cexpr = self.compileExpression(expr) + # XXX Would key be anything but 'text' or None? + assert key in ('text', None) + self.emit('i18nVariable', varname, program, cexpr) + + def emitTranslation(self, msgid, i18ndata): + program = self.popProgram() + if i18ndata is None: + self.emit('insertTranslation', msgid, program) + else: + key, expr = parseSubstitution(i18ndata) + cexpr = self.compileExpression(expr) + assert key == 'text' + self.emit('insertTranslation', msgid, program, cexpr) + def emitDefineMacro(self, macroName): program = self.popProgram() - macroName = string.strip(macroName) + macroName = macroName.strip() if self.macros.has_key(macroName): raise METALError("duplicate macro definition: %s" % `macroName`, self.position) @@ -299,7 +387,7 @@ def emitDefineSlot(self, slotName): program = self.popProgram() - slotName = string.strip(slotName) + slotName = slotName.strip() if not re.match('%s$' % NAME_RE, slotName): raise METALError("invalid slot name: %s" % `slotName`, self.position) @@ -307,7 +395,7 @@ def emitFillSlot(self, slotName): program = self.popProgram() - slotName = string.strip(slotName) + slotName = slotName.strip() if self.slots.has_key(slotName): raise METALError("duplicate fill-slot name: %s" % `slotName`, self.position) @@ -338,7 +426,7 @@ self.program[i] = ("rawtext", text[:m.start()]) collect.append(m.group()) collect.reverse() - return string.join(collect, "") + return "".join(collect) def unEmitNewlineWhitespace(self): collect = [] @@ -357,7 +445,7 @@ break text, rest = m.group(1, 2) collect.reverse() - rest = rest + string.join(collect, "") + rest = rest + "".join(collect) del self.program[i:] if text: self.emit("rawtext", text) @@ -365,23 +453,30 @@ return None def replaceAttrs(self, attrlist, repldict): + # Each entry in attrlist starts like (name, value). + # Result is (name, value, action, expr, xlat) if there is a + # tal:attributes entry for that attribute. Additional attrs + # defined only by tal:attributes are added here. + # + # (name, value, action, expr, xlat) if not repldict: return attrlist newlist = [] for item in attrlist: key = item[0] if repldict.has_key(key): - item = item[:2] + ("replace", repldict[key]) + expr, xlat, msgid = repldict[key] + item = item[:2] + ("replace", expr, xlat, msgid) del repldict[key] newlist.append(item) - for key, value in repldict.items(): # Add dynamic-only attributes - item = (key, None, "insert", value) - newlist.append(item) + # Add dynamic-only attributes + for key, (expr, xlat, msgid) in repldict.items(): + newlist.append((key, None, "insert", expr, xlat, msgid)) return newlist - def emitStartElement(self, name, attrlist, taldict, metaldict, + def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict, position=(None, None), isend=0): - if not taldict and not metaldict: + if not taldict and not metaldict and not i18ndict: # Handle the simple, common case self.emitStartTag(name, attrlist, isend) self.todoPush({}) @@ -391,18 +486,24 @@ self.position = position for key, value in taldict.items(): - if key not in KNOWN_TAL_ATTRIBUTES: + if key not in TALDefs.KNOWN_TAL_ATTRIBUTES: raise TALError("bad TAL attribute: " + `key`, position) if not (value or key == 'omit-tag'): raise TALError("missing value for TAL attribute: " + `key`, position) for key, value in metaldict.items(): - if key not in KNOWN_METAL_ATTRIBUTES: + if key not in TALDefs.KNOWN_METAL_ATTRIBUTES: raise METALError("bad METAL attribute: " + `key`, - position) + position) if not value: raise TALError("missing value for METAL attribute: " + `key`, position) + for key, value in i18ndict.items(): + if key not in TALDefs.KNOWN_I18N_ATTRIBUTES: + raise I18NError("bad i18n attribute: " + `key`, position) + if not value and key in ("attributes", "data", "id"): + raise I18NError("missing value for i18n attribute: " + + `key`, position) todo = {} defineMacro = metaldict.get("define-macro") useMacro = metaldict.get("use-macro") @@ -417,13 +518,36 @@ onError = taldict.get("on-error") omitTag = taldict.get("omit-tag") TALtag = taldict.get("tal tag") + i18nattrs = i18ndict.get("attributes") + # Preserve empty string if implicit msgids are used. We'll generate + # code with the msgid='' and calculate the right implicit msgid during + # interpretation phase. + msgid = i18ndict.get("translate") + varname = i18ndict.get('name') + i18ndata = i18ndict.get('data') + + if varname and not self.i18nLevel: + raise I18NError( + "i18n:name can only occur inside a translation unit", + position) + + if i18ndata and not msgid: + raise I18NError("i18n:data must be accompanied by i18n:translate", + position) + if len(metaldict) > 1 and (defineMacro or useMacro): raise METALError("define-macro and use-macro cannot be used " "together or with define-slot or fill-slot", position) - if content and replace: - raise TALError("content and replace are mutually exclusive", - position) + if replace: + if content: + raise TALError( + "tal:content and tal:replace are mutually exclusive", + position) + if msgid is not None: + raise I18NError( + "i18n:translate and tal:replace are mutually exclusive", + position) repeatWhitespace = None if repeat: @@ -441,8 +565,8 @@ self.inMacroUse = 0 else: if fillSlot: - raise METALError, ("fill-slot must be within a use-macro", - position) + raise METALError("fill-slot must be within a use-macro", + position) if not self.inMacroUse: if defineMacro: self.pushProgram() @@ -459,13 +583,29 @@ self.inMacroUse = 1 if defineSlot: if not self.inMacroDef: - raise METALError, ( + raise METALError( "define-slot must be within a define-macro", position) self.pushProgram() todo["defineSlot"] = defineSlot - if taldict: + if defineSlot or i18ndict: + + domain = i18ndict.get("domain") or self.i18nContext.domain + source = i18ndict.get("source") or self.i18nContext.source + target = i18ndict.get("target") or self.i18nContext.target + if ( domain != DEFAULT_DOMAIN + or source is not None + or target is not None): + self.i18nContext = TranslationContext(self.i18nContext, + domain=domain, + source=source, + target=target) + self.emit("beginI18nContext", + {"domain": domain, "source": source, + "target": target}) + todo["i18ncontext"] = 1 + if taldict or i18ndict: dict = {} for item in attrlist: key, value = item[:2] @@ -493,18 +633,60 @@ if repeatWhitespace: self.emitText(repeatWhitespace) if content: - todo["content"] = content - if replace: - todo["replace"] = replace + if varname: + todo['i18nvar'] = (varname, I18N_CONTENT, None) + todo["content"] = content + self.pushProgram() + else: + todo["content"] = content + elif replace: + # tal:replace w/ i18n:name has slightly different semantics. What + # we're actually replacing then is the contents of the ${name} + # placeholder. + if varname: + todo['i18nvar'] = (varname, I18N_EXPRESSION, replace) + else: + todo["replace"] = replace self.pushProgram() + # i18n:name w/o tal:replace uses the content as the interpolation + # dictionary values + elif varname: + todo['i18nvar'] = (varname, I18N_REPLACE, None) + self.pushProgram() + if msgid is not None: + self.i18nLevel += 1 + todo['msgid'] = msgid + if i18ndata: + todo['i18ndata'] = i18ndata optTag = omitTag is not None or TALtag if optTag: todo["optional tag"] = omitTag, TALtag self.pushProgram() - if attrsubst: - repldict = parseAttributeReplacements(attrsubst) + if attrsubst or i18nattrs: + if attrsubst: + repldict = TALDefs.parseAttributeReplacements(attrsubst, + self.xml) + else: + repldict = {} + if i18nattrs: + i18nattrs = _parseI18nAttributes(i18nattrs, attrlist, repldict, + self.position, self.xml, + self.source_file) + else: + i18nattrs = {} + # Convert repldict's name-->expr mapping to a + # name-->(compiled_expr, translate) mapping for key, value in repldict.items(): - repldict[key] = self.compileExpression(value) + if i18nattrs.get(key, None): + raise I18NError( + ("attribute [%s] cannot both be part of tal:attributes" + + " and have a msgid in i18n:attributes") % key, + position) + ce = self.compileExpression(value) + repldict[key] = ce, key in i18nattrs, i18nattrs.get(key) + for key in i18nattrs: + if not repldict.has_key(key): + repldict[key] = None, 1, i18nattrs.get(key) else: repldict = {} if replace: @@ -513,7 +695,11 @@ self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend) if optTag: self.pushProgram() - if content: + if content and not varname: + self.pushProgram() + if msgid is not None: + self.pushProgram() + if content and varname: self.pushProgram() if todo and position != (None, None): todo["position"] = position @@ -543,6 +729,10 @@ repldict = todo.get("repldict", {}) scope = todo.get("scope") optTag = todo.get("optional tag") + msgid = todo.get('msgid') + i18ncontext = todo.get("i18ncontext") + varname = todo.get('i18nvar') + i18ndata = todo.get('i18ndata') if implied > 0: if defineMacro or useMacro or defineSlot or fillSlot: @@ -554,14 +744,58 @@ raise exc("%s attributes on <%s> require explicit </%s>" % (what, name, name), position) + # If there's no tal:content or tal:replace in the tag with the + # i18n:name, tal:replace is the default. if content: self.emitSubstitution(content, {}) + # If we're looking at an implicit msgid, emit the insertTranslation + # opcode now, so that the end tag doesn't become part of the implicit + # msgid. If we're looking at an explicit msgid, it's better to emit + # the opcode after the i18nVariable opcode so we can better handle + # tags with both of them in them (and in the latter case, the contents + # would be thrown away for msgid purposes). + # + # Still, we should emit insertTranslation opcode before i18nVariable + # in case tal:content, i18n:translate and i18n:name in the same tag + if msgid is not None: + if (not varname) or ( + varname and (varname[1] == I18N_CONTENT)): + self.emitTranslation(msgid, i18ndata) + self.i18nLevel -= 1 if optTag: self.emitOptTag(name, optTag, isend) elif not isend: + # If we're processing the end tag for a tag that contained + # i18n:name, we need to make sure that optimize() won't collect + # immediately following end tags into the same rawtextOffset, so + # put a spacer here that the optimizer will recognize. + if varname: + self.emit('noop') self.emitEndTag(name) + # If i18n:name appeared in the same tag as tal:replace then we're + # going to do the substitution a little bit differently. The results + # of the expression go into the i18n substitution dictionary. if replace: self.emitSubstitution(replace, repldict) + elif varname: + # o varname[0] is the variable name + # o varname[1] is either + # - I18N_REPLACE for implicit tal:replace + # - I18N_CONTENT for tal:content + # - I18N_EXPRESSION for explicit tal:replace + # o varname[2] will be None for the first two actions and the + # replacement tal expression for the third action. + assert (varname[1] + in [I18N_REPLACE, I18N_CONTENT, I18N_EXPRESSION]) + self.emitI18nVariable(varname) + # Do not test for "msgid is not None", i.e. we only want to test for + # explicit msgids here. See comment above. + if msgid is not None: + # in case tal:content, i18n:translate and i18n:name in the + # same tag insertTranslation opcode has already been + # emitted + if varname and (varname[1] <> I18N_CONTENT): + self.emitTranslation(msgid, i18ndata) if repeat: self.emitRepeat(repeat) if condition: @@ -570,6 +804,10 @@ self.emitOnError(name, onError, optTag and optTag[1], isend) if scope: self.emit("endScope") + if i18ncontext: + self.emit("endI18nContext") + assert self.i18nContext.parent is not None + self.i18nContext = self.i18nContext.parent if defineSlot: self.emitDefineSlot(defineSlot) if fillSlot: @@ -579,6 +817,54 @@ if defineMacro: self.emitDefineMacro(defineMacro) + +def _parseI18nAttributes(i18nattrs, attrlist, repldict, position, + xml, source_file): + + def addAttribute(dic, attr, msgid, position, xml): + if not xml: + attr = attr.lower() + if attr in dic: + raise TALError( + "attribute may only be specified once in i18n:attributes: " + + attr, + position) + dic[attr] = msgid + + d = {} + if ';' in i18nattrs: + i18nattrlist = i18nattrs.split(';') + i18nattrlist = [attr.strip().split() + for attr in i18nattrlist if attr.strip()] + for parts in i18nattrlist: + if len(parts) > 2: + raise TALError("illegal i18n:attributes specification: %r" + % parts, position) + if len(parts) == 2: + attr, msgid = parts + else: + # len(parts) == 1 + attr = parts[0] + msgid = None + addAttribute(d, attr, msgid, position, xml) + else: + i18nattrlist = i18nattrs.split() + if len(i18nattrlist) == 2: + staticattrs = [attr[0] for attr in attrlist if len(attr) == 2] + if (not i18nattrlist[1] in staticattrs) and ( + not i18nattrlist[1] in repldict): + attr, msgid = i18nattrlist + addAttribute(d, attr, msgid, position, xml) + else: + msgid = None + for attr in i18nattrlist: + addAttribute(d, attr, msgid, position, xml) + else: + msgid = None + for attr in i18nattrlist: + addAttribute(d, attr, msgid, position, xml) + return d + def test(): t = TALGenerator() t.pushProgram()