view roundup/cgi/PageTemplates/Expressions.py @ 5525:bb7865241f8a

Make CSV import/export compatible across Python versions (also RDBMS journals) (issue 2550976, issue 2550975). The roundup-admin export and import commands are used for migrating between different database backends. It is desirable that they should be usable also for migrations between Python 2 and Python 3, and in some cases (e.g. with the anydbm backend) this may be required. To be usable for such migrations, the format of the generated CSV files needs to be stable, meaning the same as currently used with Python 2. The export process uses repr() to produce the fields in the CSV files and eval() to convert them back to Python data structures. repr() of strings with non-ASCII characters produces different results for Python 2 and Python 3. This patch adds repr_export and eval_import functions to roundup/anypy/strings.py which provide the required operations that are just repr() and eval() in Python 2, but are more complicated in Python 3 to use data representations compatible with Python 2. These functions are then used in the required places for export and import. repr() and eval() are also used in storing the dict of changed values in the journal for the RDBMS backends. It is similarly desirable that the database be compatible between Python 2 and Python 3, so that export and import do not need to be used for a migration between Python versions for non-anydbm back ends. Thus, this patch changes rdbms_common.py in the places involved in storing journals in the database, not just in those involved in import/export. Given this patch, import/export with non-ASCII characters appear based on some limited testing to work across Python versions, and an instance using the sqlite backend appears to be compatible between Python versions without needing import/export, *if* the sessions/otks databases (which use anydbm) are deleted when changing Python version.
author Joseph Myers <jsm@polyomino.org.uk>
date Sun, 02 Sep 2018 23:48:04 +0000
parents 5a871a250670
children 33b25e51e127
line wrap: on
line source

##############################################################################
#
# Copyright (c) 2001 Zope Corporation and Contributors. All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
# Modified for Roundup:
# 
# 1. removed all Zope-specific code (doesn't even try to import that stuff now)
# 2. removed all Acquisition
# 3. removed blocking of leading-underscore URL components

"""Page Template Expression Engine

Page Template-specific implementation of TALES, with handlers
for Python expressions, string literals, and paths.
"""

import collections, re, sys
from .TALES import Engine, CompilerError, _valid_name, NAME_RE, \
     Undefined, Default, _parse_expr


_engine = None
def getEngine():
    global _engine
    if _engine is None:
        from .PathIterator import Iterator
        _engine = Engine(Iterator)
        installHandlers(_engine)
    return _engine

def installHandlers(engine):
    reg = engine.registerType
    pe = PathExpr
    for pt in ('standard', 'path', 'exists', 'nocall'):
        reg(pt, pe)
    reg('string', StringExpr)
    reg('python', PythonExpr)
    reg('not', NotExpr)
    reg('defer', DeferExpr)

from .PythonExpr import getSecurityManager, PythonExpr
guarded_getattr = getattr
try:
    from zExceptions import Unauthorized
except ImportError:
    class Unauthorized(BaseException):
        pass

def acquisition_security_filter(orig, inst, name, v, real_validate):
    if real_validate(orig, inst, name, v):
        return 1
    raise Unauthorized(name)

def call_with_ns(f, ns, arg=1):
    if arg==2:
        return f(None, ns)
    else:
        return f(ns)

class _SecureModuleImporter:
    """Simple version of the importer for use with trusted code."""
    __allow_access_to_unprotected_subobjects__ = 1
    def __getitem__(self, module):
        __import__(module)
        return sys.modules[module]

SecureModuleImporter = _SecureModuleImporter()

Undefs = (Undefined, AttributeError, KeyError,
          TypeError, IndexError, Unauthorized)

def render(ob, ns):
    """
    Calls the object, possibly a document template, or just returns it if
    not callable.  (From DT_Util.py)
    """
    if hasattr(ob, '__render_with_namespace__'):
        ob = call_with_ns(ob.__render_with_namespace__, ns)
    else:
        base = ob
        if isinstance(base, collections.Callable):
            try:
                if getattr(base, 'isDocTemp', 0):
                    ob = call_with_ns(ob, ns, 2)
                else:
                    ob = ob()
            except AttributeError as n:
                if str(n) != '__call__':
                    raise
    return ob

class SubPathExpr:
    def __init__(self, path):
        self._path = path = path.strip().split('/')
        self._base = base = path.pop(0)
        if base and not _valid_name(base):
            raise CompilerError('Invalid variable name "%s"' % base)
        # Parse path
        self._dp = dp = []
        for i in range(len(path)):
            e = path[i]
            if e[:1] == '?' and _valid_name(e[1:]):
                dp.append((i, e[1:]))
        dp.reverse()

    def _eval(self, econtext,
              list=list, isinstance=isinstance, StringType=type('')):
        vars = econtext.vars
        path = self._path
        if self._dp:
            path = list(path) # Copy!
            for i, varname in self._dp:
                val = vars[varname]
                if isinstance(val, StringType):
                    path[i] = val
                else:
                    # If the value isn't a string, assume it's a sequence
                    # of path names.
                    path[i:i+1] = list(val)
        base = self._base
        __traceback_info__ = 'path expression "%s"'%('/'.join(self._path))
        if base == 'CONTEXTS' or not base:
            ob = econtext.contexts
        else:
            ob = vars[base]
        if isinstance(ob, DeferWrapper):
            ob = ob()
        if path:
            ob = restrictedTraverse(ob, path, getSecurityManager())
        return ob

class PathExpr:
    def __init__(self, name, expr, engine):
        self._s = expr
        self._name = name
        self._hybrid = 0
        paths = expr.split('|')
        self._subexprs = []
        add = self._subexprs.append
        for i in range(len(paths)):
            path = paths[i].lstrip()
            if _parse_expr(path):
                # This part is the start of another expression type,
                # so glue it back together and compile it.
                add(engine.compile(('|'.join(paths[i:]).lstrip())))
                self._hybrid = 1
                break
            add(SubPathExpr(path)._eval)

    def _exists(self, econtext):
        for expr in self._subexprs:
            try:
                expr(econtext)
            except Undefs:
                pass
            else:
                return 1
        return 0

    def _eval(self, econtext,
              isinstance=isinstance, StringType=type(''), render=render):
        for expr in self._subexprs[:-1]:
            # Try all but the last subexpression, skipping undefined ones.
            try:
                ob = expr(econtext)
            except Undefs:
                pass
            else:
                break
        else:
            # On the last subexpression allow exceptions through, and
            # don't autocall if the expression was not a subpath.
            ob = self._subexprs[-1](econtext)
            if self._hybrid:
                return ob

        if self._name == 'nocall' or isinstance(ob, StringType):
            return ob
        # Return the rendered object
        return render(ob, econtext.vars)

    def __call__(self, econtext):
        if self._name == 'exists':
            return self._exists(econtext)
        return self._eval(econtext)

    def __str__(self):
        return '%s expression %s' % (self._name, repr(self._s))

    def __repr__(self):
        return '%s:%s' % (self._name, repr(self._s))


_interp = re.compile(r'\$(%(n)s)|\${(%(n)s(?:/[^}]*)*)}' % {'n': NAME_RE})

class StringExpr:
    def __init__(self, name, expr, engine):
        self._s = expr
        if '%' in expr:
            expr = expr.replace('%', '%%')
        self._vars = vars = []
        if '$' in expr:
            parts = []
            for exp in expr.split('$$'):
                if parts: parts.append('$')
                m = _interp.search(exp)
                while m is not None:
                    parts.append(exp[:m.start()])
                    parts.append('%s')
                    vars.append(PathExpr('path', m.group(1) or m.group(2),
                                         engine))
                    exp = exp[m.end():]
                    m = _interp.search(exp)
                if '$' in exp:
                    raise CompilerError(
                        '$ must be doubled or followed by a simple path')
                parts.append(exp)
            expr = ''.join(parts)
        self._expr = expr

    def __call__(self, econtext):
        vvals = []
        for var in self._vars:
            v = var(econtext)
            # I hope this isn't in use anymore.
            ## if isinstance(v, Exception):
            ##     raise v
            vvals.append(v)
        return self._expr % tuple(vvals)

    def __str__(self):
        return 'string expression %s' % repr(self._s)

    def __repr__(self):
        return 'string:%s' % repr(self._s)

class NotExpr:
    def __init__(self, name, expr, compiler):
        self._s = expr = expr.lstrip()
        self._c = compiler.compile(expr)

    def __call__(self, econtext):
        # We use the (not x) and 1 or 0 formulation to avoid changing
        # the representation of the result in Python 2.3, where the
        # result of "not" becomes an instance of bool.
        return (not econtext.evaluateBoolean(self._c)) and 1 or 0

    def __repr__(self):
        return 'not:%s' % repr(self._s)

class DeferWrapper:
    def __init__(self, expr, econtext):
        self._expr = expr
        self._econtext = econtext

    def __str__(self):
        return str(self())

    def __call__(self):
        return self._expr(self._econtext)

class DeferExpr:
    def __init__(self, name, expr, compiler):
        self._s = expr = expr.lstrip()
        self._c = compiler.compile(expr)

    def __call__(self, econtext):
        return DeferWrapper(self._c, econtext)

    def __repr__(self):
        return 'defer:%s' % repr(self._s)

class TraversalError:
    def __init__(self, path, name):
        self.path = path
        self.name = name



def restrictedTraverse(object, path, securityManager,
                       get=getattr, has=hasattr, N=None, M=[],
                       TupleType=type(()) ):

    REQUEST = {'path': path}
    REQUEST['TraversalRequestNameStack'] = path = path[:] # Copy!
    path.reverse()
    validate = securityManager.validate
    __traceback_info__ = REQUEST
    done = []
    while path:
        name = path.pop()
        __traceback_info__ = TraversalError(done, name)

        if isinstance(name, TupleType):
            object = object(*name)
            continue

        if not name:
            # Skip directly to item access
            o = object[name]
            # Check access to the item.
            if not validate(object, object, name, o):
                raise Unauthorized(name)
            object = o
            continue

        # Try an attribute.
        o = guarded_getattr(object, name, M)
        if o is M:
            # Try an item.
            try:
                # XXX maybe in Python 2.2 we can just check whether
                # the object has the attribute "__getitem__"
                # instead of blindly catching exceptions.
                o = object[name]
            except AttributeError as exc:
                if str(exc).find('__getitem__') >= 0:
                    # The object does not support the item interface.
                    # Try to re-raise the original attribute error.
                    # XXX I think this only happens with
                    # ExtensionClass instances.
                    guarded_getattr(object, name)
                raise
            except TypeError as exc:
                if str(exc).find('unsubscriptable') >= 0:
                    # The object does not support the item interface.
                    # Try to re-raise the original attribute error.
                    # XXX This is sooooo ugly.
                    guarded_getattr(object, name)
                raise
        done.append((name, o))
        object = o

    return object

Roundup Issue Tracker: http://roundup-tracker.org/