view roundup/backends/sessions_redis.py @ 6823:fe0091279f50

Refactor session db logging and key generation for sessions/otks While I was working on the redis sessiondb stuff, I noticed that log_wanrning, get_logger ... was duplicated. Also there was code to generate a unique key for otks that was duplicated. Changes: creating new sessions_common.py and SessionsCommon class to provide methods: log_warning, log_info, log_debug, get_logger, getUniqueKey getUniqueKey method is closer to the method used to make session keys in client.py. sessions_common.py now report when random_.py chooses a weak random number generator. Removed same from rest.py. get_logger reconciles all logging under roundup.hyperdb.backends.<name of BasicDatabase class> some backends used to log to root logger. have BasicDatabase in other sessions_*.py modules inherit from SessionCommon. change logging to use log_* methods. In addition: remove unused imports reported by flake8 and other formatting changes modify actions.py, rest.py, templating.py to use getUniqueKey method. add tests for new methods test_redis_session.py swap out ModuleNotFoundError for ImportError to prevent crash in python2 when redis is not present. allow injection of username:password or just password into redis connection URL. set pytest_redis_pw envirnment variable to password or user:password when running test.
author John Rouillard <rouilj@ieee.org>
date Sun, 07 Aug 2022 01:51:11 -0400
parents 3f60a71b0812
children dd15c307c409
line wrap: on
line source

"""This module defines a redis based store that's used by
the CGI interface to store session and one-time-key
information.

Yes, it's called "sessions" - because originally it only
defined a session class. It's now also used for One Time Key
handling too.

It uses simple strings rather than redis hash structure
because the hash the values are always strings. We need to
be able to represent the same data types available to rdbms
and dbm session stores.

session_dbm uses marshal.dumps and marshal.loads. This seems
4 or 18 times faster than the repr()/eval() used by
session_rdbms. So use marshal even though it is impossible
to read when viewing (using redis-cli).
"""
__docformat__ = 'restructuredtext'

import marshal, redis, time

from roundup.anypy.html import html_escape as escape

from roundup.i18n import _

from roundup.backends.sessions_common import SessionCommon


class BasicDatabase(SessionCommon):
    ''' Provide a nice encapsulation of a redis store.

        Keys are id strings, values are automatically marshalled data.
    '''
    name = None
    default_lifetime = 60*60*24*7  # 1 week

    # FIXME: figure out how to allow admin to change this
    # to repr/eval using interfaces.py or other method.
    # marshalled data is not readable when debugging.
    tostr = marshal.dumps
    todict = marshal.loads

    def __init__(self, db):
        self.config = db.config
        url = self.config.SESSIONDB_REDIS_URL

        # Example at default port without auth.
        #    redis://localhost:6379/0?health_check_interval=2
        #
        # Do not allow decode_responses=True in url, data is
        # marshal'ed binary data that will get broken by decoding.
        # Enforce this in configuration.
        self.redis = redis.Redis.from_url(url=url, decode_responses=False)

    def makekey(self, key):
        '''method to namespace all keys using self.name:....'''
        return "%s:%s" % (self.name, key)

    def exists(self, infoid):
        return self.redis.exists(self.makekey(infoid))

    def clear(self):
        '''Delete all keys from the database.'''
        self.redis.flushdb()

    _marker = []

    def get(self, infoid, value, default=_marker):
        '''get a specific value from the data associated with a key'''
        infoid = self.makekey(infoid)
        v = self.redis.get(infoid)
        if not v:
            if default != self._marker:
                return default
            raise KeyError(_('Key %(key)s not found in %(name)s '
                             'database.' % {"name": self.name,
                                            "key": escape(infoid)}))
        return self.todict(v)[value]

    def getall(self, infoid):
        '''return all values associated with a key'''
        try:
            d = self.redis.get(self.makekey(infoid))
            if d is not None:
                d = self.todict(d)
            else:
                d = {}
            del d['__timestamp']
            return d
        except KeyError:
            # It is possible for d to be malformed missing __timestamp.
            # If so, we get a misleading error, but anydbm does the
            # same so....
            raise KeyError(_('Key %(key)s not found in %(name)s '
                             'database.' % {"name": self.name,
                                            "key": escape(infoid)}))

        ''' def set_no_tranaction(self, infoid, **newvalues):
        """ this is missing transaction and may be affected by
            a race condition on update. This will work for
            redis-like embedded databases that don't support
            watch/multi/exec
        """
        infoid = self.makekey(infoid)
        timestamp=None
        values = self.redis.get(infoid)
        if values is not None:
            values = self.todict(values)
        else:
            values={}
        try:
            timestamp = float(values['__timestamp'])
        except KeyError:
            pass  # stay at None

        if '__timestamp' in newvalues:
            try:
                float(newvalues['__timestamp'])
            except ValueError:
                # keep original timestamp if present
                newvalues['__timestamp'] = timestamp or \
                                        (time.time() + self.default_lifetime)
        else:
            newvalues['__timestamp'] = time.time() + self.default_lifetime

        values.update(newvalues)

        self.redis.set(infoid, self.tostr(values))
        self.redis.expireat(infoid, int(values['__timestamp']))
        '''

    def set(self, infoid, **newvalues):
        """ Implement set using watch/multi/exec to get some
            protection against a change committing between
            getting the data and setting new fields and
            saving.
        """
        infoid = self.makekey(infoid)
        timestamp = None

        with self.redis.pipeline() as transaction:
            # Give up and log after three tries.
            # Do not loop forever.
            for _retry in [1, 2, 3]:
                # I am ignoring transaction return values.
                # Assuming all errors will be via exceptions.
                # Not clear that return values that useful.
                transaction.watch(infoid)
                values = transaction.get(infoid)
                if values is not None:
                    values = self.todict(values)
                else:
                    values = {}

                try:
                    timestamp = float(values['__timestamp'])
                except KeyError:
                    pass  # stay at None

                if '__timestamp' in newvalues:
                    try:
                        float(newvalues['__timestamp'])
                    except ValueError:
                        # keep original timestamp if present
                        newvalues['__timestamp'] = timestamp or \
                                (time.time() + self.default_lifetime)
                else:
                    newvalues['__timestamp'] = time.time() + \
                                               self.default_lifetime

                values.update(newvalues)

                transaction.multi()
                transaction.set(infoid, self.tostr(values))
                transaction.expireat(infoid, int(values['__timestamp']))
                try:
                    # assume this works or raises an WatchError
                    # exception indicating I need to retry.
                    # Since this is not a transaction, an error
                    # in one step doesn't roll back other changes.
                    # so I again ignore the return codes as it is not
                    # clear that I can do the rollback myself.
                    # Format and other errors (e.g. expireat('d', 'd'))
                    # raise exceptions tht bubble up and result in mail
                    # to admin.
                    transaction.execute()
                    break
                except redis.Exceptions.WatchError:
                    self.log_info(
                        _('Key %(key)s changed in %(name)s db' %
                          {"key": escape(infoid), "name": self.name})
                    )
            else:
                raise Exception(_("Redis set failed afer 3 retries"))

    def list(self):
        return list(self.redis.keys(self.makekey('*')))

    def destroy(self, infoid=None):
        '''use unlink rather than delete as unlink is async and doesn't
           wait for memory to be freed server-side
        '''
        self.redis.unlink(self.makekey(infoid))

    def commit(self):
        ''' no-op '''
        pass

    def lifetime(self, key_lifetime=None):
        """Return the proper timestamp to expire a key with key_lifetime
           specified in seconds. Default lifetime is self.default_lifetime.
        """
        return time.time() + (key_lifetime or self.default_lifetime)

    def updateTimestamp(self, sessid):
        ''' Other backends update only if timestamp would change by more
            than 60 seconds. To do this in redis requires:
               get data _timestamp
                 calculate if update needed
               if needed,
               set new timestamp
            why bother. Just set and forget.
        '''
        # no need to do timestamp calculations
        lifetime = self.lifetime()
        # note set also updates the expireat on the key in redis
        self.set(sessid, __timestamp=lifetime)

    def clean(self):
        ''' redis handles key expiration, so nothing to do here.
        '''
        pass

    def close(self):
        ''' redis uses a connection pool that self manages, so nothing
            to do on close.'''
        pass


class Sessions(BasicDatabase):
    name = 'sessions'


class OneTimeKeys(BasicDatabase):
    name = 'otks'

# vim: set sts ts=4 sw=4 et si :

Roundup Issue Tracker: http://roundup-tracker.org/