view roundup/install_util.py @ 5710:0b79bfcb3312

Add support for making an idempotent POST. This allows retrying a POST that was interrupted. It involves creating a post once only (poe) url /rest/data/<class>/@poe/<random_token>. This url acts the same as a post to /rest/data/<class>. However once the @poe url is used, it can't be used for a second POST. To make these changes: 1) Take the body of post_collection into a new post_collection_inner function. Have post_collection call post_collection_inner. 2) Add a handler for POST to rest/data/class/@poe. This will return a unique POE url. By default the url expires after 30 minutes. The POE random token is only good for a specific user and is stored in the session db. 3) Add a handler for POST to rest/data/<class>/@poe/<random token>. The random token generated in 2 is validated for proper class (if token is not generic) and proper user and must not have expired. If everything is valid, call post_collection_inner to process the input and generate the new entry. To make recognition of 2 stable (so it's not confused with rest/data/<:class_name>/<:item_id>), removed @ from Routing::url_to_regex. The current Routing.execute method stops on the first regular expression to match the URL. Since item_id doesn't accept a POST, I was getting 405 bad method sometimes. My guess is the order of the regular expressions is not stable, so sometime I would get the right regexp for /data/<class>/@poe and sometime I would get the one for /data/<class>/<item_id>. By removing the @ from the url_to_regexp, there was no way for the item_id case to match @poe. There are alternate fixes we may need to look at. If a regexp matches but the method does not, return to the regexp matching loop in execute() looking for another match. Only once every possible match has failed should the code return a 405 method failure. Another fix is to implement a more sophisticated mechanism so that @Routing.route("/data/<:class_name>/<:item_id>/<:attr_name>", 'PATCH') has different regexps for matching <:class_name> <:item_id> and <:attr_name>. Currently the regexp specified by url_to_regex is used for every component. Other fixes: Made failure to find any props in props_from_args return an empty dict rather than throwing an unhandled error. Make __init__ for SimulateFieldStorageFromJson handle an empty json doc. Useful for POSTing to rest/data/class/@poe with an empty document. Testing: added testPostPOE to test/rest_common.py that I think covers all the code that was added. Documentation: Add doc to rest.txt in the "Client API" section titled: Safely Re-sending POST". Move existing section "Adding new rest endpoints" in "Client API" to a new second level section called "Programming the REST API". Also a minor change to the simple rest client moving the header setting to continuation lines rather than showing one long line.
author John Rouillard <rouilj@ieee.org>
date Sun, 14 Apr 2019 21:07:11 -0400
parents 88cf5614e0f4
children 7ada5d44b21d
line wrap: on
line source

#
# Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
# This module is free software, and you may redistribute it and/or modify
# under the same terms as Python, so long as this copyright message and
# disclaimer are retained in their original form.
#
# IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
# OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
# BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE.  THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
# BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
# 

"""Support module to generate and check fingerprints of installed files.
"""
__docformat__ = 'restructuredtext'

import os, shutil
from hashlib import sha1

from roundup.anypy.strings import s2b

sgml_file_types = [".xml", ".ent", ".html"]
hash_file_types = [".py", ".sh", ".conf", ".cgi"]
slast_file_types = [".css"]

digested_file_types = sgml_file_types + hash_file_types + slast_file_types

def extractFingerprint(lines):
    # get fingerprint from last line
    if lines[-1].startswith(b"#SHA: "):
        # handle .py/.sh comment
        return lines[-1][6:].strip()
    elif lines[-1].startswith(b"<!-- SHA: "):
        # handle xml/html files
        fingerprint = lines[-1][10:]
        fingerprint = fingerprint.replace(b'-->', b'')
        return fingerprint.strip()
    elif lines[-1].startswith(b"/* SHA: "):
        # handle css files
        fingerprint = lines[-1][8:]
        fingerprint = fingerprint.replace(b'*/', b'')
        return fingerprint.strip()
    return None

def checkDigest(filename):
    """Read file, check for valid fingerprint, return TRUE if ok"""
    # open and read file
    inp = open(filename, "rb")
    lines = inp.readlines()
    inp.close()

    fingerprint = extractFingerprint(lines)
    if fingerprint is None:
        return 0
    del lines[-1]

    # calculate current digest
    digest = sha1()
    for line in lines:
        digest.update(line)

    # compare current to stored digest
    return fingerprint == s2b(digest.hexdigest())


class DigestFile:
    """ A class that you can use like open() and that calculates
        and writes a SHA digest to the target file.
    """

    def __init__(self, filename):
        self.filename = filename
        self.digest = sha1()
        self.file = open(self.filename, "wb")

    def write(self, data):
        lines = data.splitlines()
        # if the file is coming from an installed tracker being used as a
        # template, then we will want to re-calculate the SHA
        fingerprint = extractFingerprint(lines)
        if fingerprint is not None:
            data = b'\n'.join(lines[:-1]) + b'\n'
        self.file.write(data)
        self.digest.update(data)

    def close(self):
        file, ext = os.path.splitext(self.filename)

        if ext in sgml_file_types:
            self.file.write(s2b("<!-- SHA: %s -->\n" % (self.digest.hexdigest(),)))
        elif ext in hash_file_types:
            self.file.write(s2b("#SHA: %s\n" % (self.digest.hexdigest(),)))
        elif ext in slast_file_types:
            self.file.write(s2b("/* SHA: %s */\n" % (self.digest.hexdigest(),)))

        self.file.close()


def copyDigestedFile(src, dst, copystat=1):
    """ Copy data from `src` to `dst`, adding a fingerprint to `dst`.
        If `copystat` is true, the file status is copied, too
        (like shutil.copy2).
    """
    if os.path.isdir(dst):
        dst = os.path.join(dst, os.path.basename(src))

    dummy, ext = os.path.splitext(src)
    if ext not in digested_file_types:
        if copystat:
            return shutil.copy2(src, dst)
        else:
            return shutil.copyfile(src, dst)

    fsrc = None
    fdst = None
    try:
        fsrc = open(src, 'rb')
        fdst = DigestFile(dst)
        shutil.copyfileobj(fsrc, fdst)
    finally:
        if fdst: fdst.close()
        if fsrc: fsrc.close()

    if copystat: shutil.copystat(src, dst)


def test():
    import sys

    testdata = open(sys.argv[0], 'rb').read()

    for ext in digested_file_types:
        testfile = "__digest_test" + ext

        out = DigestFile(testfile)
        out.write(testdata)
        out.close()

        assert checkDigest(testfile), "digest ok w/o modification"

        mod = open(testfile, 'r+b')
        mod.seek(0)
        mod.write('# changed!')
        mod.close()

        assert not checkDigest(testfile), "digest fails after modification"

        os.remove(testfile)


if __name__ == '__main__':
    test()

# vim: set filetype=python ts=4 sw=4 et si

Roundup Issue Tracker: http://roundup-tracker.org/