#
# Copyright (c) 2003 Richard Jones, rjones@ekit-inc.com
# This module is free software, and you may redistribute it and/or modify
# under the same terms as Python, so long as this copyright message and
# disclaimer are retained in their original form.
#
# This module is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

import unittest, os, shutil, errno, sys, difflib, cgi, re, StringIO

from roundup.cgi import client, actions, exceptions
from roundup.cgi.exceptions import FormError
from roundup.cgi.templating import HTMLItem, HTMLRequest, NoTemplate, anti_csrf_nonce
from roundup.cgi.templating import HTMLProperty, _HTMLItem
from roundup.cgi.form_parser import FormParser
from roundup import init, instance, password, hyperdb, date

# For testing very simple rendering
from roundup.cgi.engine_zopetal import RoundupPageTemplate

from mocknull import MockNull

import db_test_base

class FileUpload:
    def __init__(self, content, filename):
        self.content = content
        self.filename = filename

class FileList:
    def __init__(self, name, *files):
        self.name  = name
        self.files = files
    def items (self):
        for f in self.files:
            yield (self.name, f)

def makeForm(args):
    form = cgi.FieldStorage()
    for k,v in args.items():
        if type(v) is type([]):
            [form.list.append(cgi.MiniFieldStorage(k, x)) for x in v]
        elif isinstance(v, FileUpload):
            x = cgi.MiniFieldStorage(k, v.content)
            x.filename = v.filename
            form.list.append(x)
        else:
            form.list.append(cgi.MiniFieldStorage(k, v))
    return form

cm = client.add_message
class MessageTestCase(unittest.TestCase):
    # Note: Escaping is now handled on a message-by-message basis at a
    # point where we still know what generates a message. In this way we
    # can decide when to escape and when not. We test the add_message
    # routine here.
    # Of course we won't catch errors in judgement when to escape here
    # -- but at the time of this change only one message is not escaped.
    def testAddMessageOK(self):
        self.assertEqual(cm([],'a\nb'), ['a<br />\nb'])
        self.assertEqual(cm([],'a\nb\nc\n'), ['a<br />\nb<br />\nc<br />\n'])

    def testAddMessageBAD(self):
        self.assertEqual(cm([],'<script>x</script>'),
            ['&lt;script&gt;x&lt;/script&gt;'])
        self.assertEqual(cm([],'<iframe>x</iframe>'),
            ['&lt;iframe&gt;x&lt;/iframe&gt;'])
        self.assertEqual(cm([],'<<script >>alert(42);5<</script >>'),
            ['&lt;&lt;script &gt;&gt;alert(42);5&lt;&lt;/script &gt;&gt;'])
        self.assertEqual(cm([],'<a  target="_top" href="/newspapers?url=y">x</a>'),
            ['&lt;a href="y"&gt;x&lt;/a&gt;'])
        self.assertEqual(cm([],'<A HREF="y">x</A>'),
            ['&lt;A HREF="y"&gt;x&lt;/A&gt;'])
        self.assertEqual(cm([],'<br>x<br />'), ['&lt;br&gt;x&lt;br /&gt;'])
        self.assertEqual(cm([],'<i>x</i>'), ['&lt;i&gt;x&lt;/i&gt;'])
        self.assertEqual(cm([],'<b>x</b>'), ['&lt;b&gt;x&lt;/b&gt;'])
        self.assertEqual(cm([],'<BR>x<BR />'), ['&lt;BR&gt;x&lt;BR /&gt;'])
        self.assertEqual(cm([],'<I>x</I>'), ['&lt;I&gt;x&lt;/I&gt;'])
        self.assertEqual(cm([],'<B>x</B>'), ['&lt;B&gt;x&lt;/B&gt;'])

    def testAddMessageNoEscape(self):
        self.assertEqual(cm([],'<i>x</i>',False), ['<i>x</i>'])
        self.assertEqual(cm([],'<i>x</i>\n<b>x</b>',False),
            ['<i>x</i><br />\n<b>x</b>'])

class FormTestCase(unittest.TestCase):
    def setUp(self):
        self.dirname = '_test_cgi_form'
        # set up and open a tracker
        self.instance = db_test_base.setupTracker(self.dirname)

        # open the database
        self.db = self.instance.open('admin')
        self.db.tx_Source = "web"
        self.db.user.create(username='Chef', address='chef@bork.bork.bork',
            realname='Bork, Chef', roles='User')
        self.db.user.create(username='mary', address='mary@test.test',
            roles='User', realname='Contrary, Mary')

        self.db.issue.addprop(tx_Source=hyperdb.String())
        self.db.msg.addprop(tx_Source=hyperdb.String())

        self.db.post_init()

        vars = {}
        thisdir = os.path.dirname(__file__)
        execfile(os.path.join(thisdir, "tx_Source_detector.py"), vars)
        vars['init'](self.db)

        test = self.instance.backend.Class(self.db, "test",
            string=hyperdb.String(), number=hyperdb.Number(),
            intval=hyperdb.Integer(), boolean=hyperdb.Boolean(),
            link=hyperdb.Link('test'), multilink=hyperdb.Multilink('test'),
            date=hyperdb.Date(), messages=hyperdb.Multilink('msg'),
            interval=hyperdb.Interval())

        # compile the labels re
        classes = '|'.join(self.db.classes.keys())
        self.FV_SPECIAL = re.compile(FormParser.FV_LABELS%classes,
            re.VERBOSE)

    def setupClient(self, form, classname, nodeid=None, template='item', env_addon=None):
        cl = client.Client(self.instance, None, {'PATH_INFO':'/',
            'REQUEST_METHOD':'POST'}, makeForm(form))
        cl.classname = classname
        cl.base = 'http://whoami.com/path/'
        cl.nodeid = nodeid
        cl.language = ('en',)
        cl.userid = '1'
        cl.db = self.db
        cl.user = 'admin'
        cl.template = template
        if env_addon is not None:
            cl.env.update(env_addon)
        return cl

    def parseForm(self, form, classname='test', nodeid=None):
        cl = self.setupClient(form, classname, nodeid)
        return cl.parsePropsFromForm(create=1)

    def tearDown(self):
        self.db.close()
        try:
            shutil.rmtree(self.dirname)
        except OSError, error:
            if error.errno not in (errno.ENOENT, errno.ESRCH): raise

    #
    # form label extraction
    #
    def tl(self, s, c, i, a, p):
        m = self.FV_SPECIAL.match(s)
        self.assertNotEqual(m, None)
        d = m.groupdict()
        self.assertEqual(d['classname'], c)
        self.assertEqual(d['id'], i)
        for action in 'required add remove link note file'.split():
            if a == action:
                self.assertNotEqual(d[action], None)
            else:
                self.assertEqual(d[action], None)
        self.assertEqual(d['propname'], p)

    def testLabelMatching(self):
        self.tl('<propname>', None, None, None, '<propname>')
        self.tl(':required', None, None, 'required', None)
        self.tl(':confirm:<propname>', None, None, 'confirm', '<propname>')
        self.tl(':add:<propname>', None, None, 'add', '<propname>')
        self.tl(':remove:<propname>', None, None, 'remove', '<propname>')
        self.tl(':link:<propname>', None, None, 'link', '<propname>')
        self.tl('test1:<prop>', 'test', '1', None, '<prop>')
        self.tl('test1:required', 'test', '1', 'required', None)
        self.tl('test1:add:<prop>', 'test', '1', 'add', '<prop>')
        self.tl('test1:remove:<prop>', 'test', '1', 'remove', '<prop>')
        self.tl('test1:link:<prop>', 'test', '1', 'link', '<prop>')
        self.tl('test1:confirm:<prop>', 'test', '1', 'confirm', '<prop>')
        self.tl('test-1:<prop>', 'test', '-1', None, '<prop>')
        self.tl('test-1:required', 'test', '-1', 'required', None)
        self.tl('test-1:add:<prop>', 'test', '-1', 'add', '<prop>')
        self.tl('test-1:remove:<prop>', 'test', '-1', 'remove', '<prop>')
        self.tl('test-1:link:<prop>', 'test', '-1', 'link', '<prop>')
        self.tl('test-1:confirm:<prop>', 'test', '-1', 'confirm', '<prop>')
        self.tl(':note', None, None, 'note', None)
        self.tl(':file', None, None, 'file', None)

    #
    # Empty form
    #
    def testNothing(self):
        self.assertEqual(self.parseForm({}), ({('test', None): {}}, []))

    def testNothingWithRequired(self):
        self.assertRaises(FormError, self.parseForm, {':required': 'string'})
        self.assertRaises(FormError, self.parseForm,
            {':required': 'title,status', 'status':'1'}, 'issue')
        self.assertRaises(FormError, self.parseForm,
            {':required': ['title','status'], 'status':'1'}, 'issue')
        self.assertRaises(FormError, self.parseForm,
            {':required': 'status', 'status':''}, 'issue')
        self.assertRaises(FormError, self.parseForm,
            {':required': 'nosy', 'nosy':''}, 'issue')
        self.assertRaises(FormError, self.parseForm,
            {':required': 'msg-1@content', 'msg-1@content':''}, 'issue')
        self.assertRaises(FormError, self.parseForm,
            {':required': 'msg-1@content'}, 'issue')

    #
    # Nonexistant edit
    #
    def testEditNonexistant(self):
        self.assertRaises(FormError, self.parseForm, {'boolean': ''},
            'test', '1')

    #
    # String
    #
    def testEmptyString(self):
        self.assertEqual(self.parseForm({'string': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'string': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'string': ['', '']})

    def testSetString(self):
        self.assertEqual(self.parseForm({'string': 'foo'}),
            ({('test', None): {'string': 'foo'}}, []))
        self.assertEqual(self.parseForm({'string': 'a\r\nb\r\n'}),
            ({('test', None): {'string': 'a\nb'}}, []))
        nodeid = self.db.issue.create(title='foo')
        self.assertEqual(self.parseForm({'title': 'foo'}, 'issue', nodeid),
            ({('issue', nodeid): {}}, []))

    def testEmptyStringSet(self):
        nodeid = self.db.issue.create(title='foo')
        self.assertEqual(self.parseForm({'title': ''}, 'issue', nodeid),
            ({('issue', nodeid): {'title': None}}, []))
        nodeid = self.db.issue.create(title='foo')
        self.assertEqual(self.parseForm({'title': ' '}, 'issue', nodeid),
            ({('issue', nodeid): {'title': None}}, []))

    def testStringLinkId(self):
        self.db.status.set('1', name='2')
        self.db.status.set('2', name='1')
        issue = self.db.issue.create(title='i1-status1', status='1')
        self.assertEqual(self.db.issue.get(issue,'status'),'1')
        self.assertEqual(self.db.status.lookup('1'),'2')
        self.assertEqual(self.db.status.lookup('2'),'1')
        self.assertEqual(self.db.issue.get('1','tx_Source'),'web')
        form = cgi.FieldStorage()
        cl = client.Client(self.instance, None, {'PATH_INFO':'/'}, form)
        cl.classname = 'issue'
        cl.nodeid = issue
        cl.db = self.db
        cl.language = ('en',)
        item = HTMLItem(cl, 'issue', issue)
        self.assertEqual(item.status.id, '1')
        self.assertEqual(item.status.name, '2')

    def testStringMultilinkId(self):
        id = self.db.keyword.create(name='2')
        self.assertEqual(id,'1')
        id = self.db.keyword.create(name='1')
        self.assertEqual(id,'2')
        issue = self.db.issue.create(title='i1-status1', keyword=['1'])
        self.assertEqual(self.db.issue.get(issue,'keyword'),['1'])
        self.assertEqual(self.db.keyword.lookup('1'),'2')
        self.assertEqual(self.db.keyword.lookup('2'),'1')
        self.assertEqual(self.db.issue.get(issue,'tx_Source'),'web')
        form = cgi.FieldStorage()
        cl = client.Client(self.instance, None, {'PATH_INFO':'/'}, form)
        cl.classname = 'issue'
        cl.nodeid = issue
        cl.db = self.db
        cl.language = ('en',)
        cl.userid = '1'
        item = HTMLItem(cl, 'issue', issue)
        for keyword in item.keyword:
            self.assertEqual(keyword.id, '1')
            self.assertEqual(keyword.name, '2')

    def testFileUpload(self):
        file = FileUpload('foo', 'foo.txt')
        self.assertEqual(self.parseForm({'content': file}, 'file'),
            ({('file', None): {'content': 'foo', 'name': 'foo.txt',
            'type': 'text/plain'}}, []))

    def testSingleFileUpload(self):
        file = FileUpload('foo', 'foo.txt')
        self.assertEqual(self.parseForm({'@file': file}, 'issue'),
            ({('file', '-1'): {'content': 'foo', 'name': 'foo.txt',
            'type': 'text/plain'},
              ('issue', None): {}},
             [('issue', None, 'files', [('file', '-1')])]))

    def testMultipleFileUpload(self):
        f1 = FileUpload('foo', 'foo.txt')
        f2 = FileUpload('bar', 'bar.txt')
        f3 = FileUpload('baz', 'baz.txt')
        files = FileList('@file', f1, f2, f3)

        self.assertEqual(self.parseForm(files, 'issue'),
            ({('file', '-1'): {'content': 'foo', 'name': 'foo.txt',
               'type': 'text/plain'},
              ('file', '-2'): {'content': 'bar', 'name': 'bar.txt',
               'type': 'text/plain'},
              ('file', '-3'): {'content': 'baz', 'name': 'baz.txt',
               'type': 'text/plain'},
              ('issue', None): {}},
             [ ('issue', None, 'files', [('file', '-1')])
             , ('issue', None, 'files', [('file', '-2')])
             , ('issue', None, 'files', [('file', '-3')])
             ]))

    def testEditFileClassAttributes(self):
        self.assertEqual(self.parseForm({'name': 'foo.txt',
                                         'type': 'application/octet-stream'},
                                        'file'),
                         ({('file', None): {'name': 'foo.txt',
                                            'type': 'application/octet-stream'}},[]))

    #
    # Link
    #
    def testEmptyLink(self):
        self.assertEqual(self.parseForm({'link': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'link': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'link': ['', '']})
        self.assertEqual(self.parseForm({'link': '-1'}),
            ({('test', None): {}}, []))

    def testSetLink(self):
        self.assertEqual(self.parseForm({'status': 'unread'}, 'issue'),
            ({('issue', None): {'status': '1'}}, []))
        self.assertEqual(self.parseForm({'status': '1'}, 'issue'),
            ({('issue', None): {'status': '1'}}, []))
        nodeid = self.db.issue.create(status='unread')
        self.assertEqual(self.parseForm({'status': 'unread'}, 'issue', nodeid),
            ({('issue', nodeid): {}}, []))
        self.assertEqual(self.db.issue.get(nodeid,'tx_Source'),'web')

    def testUnsetLink(self):
        nodeid = self.db.issue.create(status='unread')
        self.assertEqual(self.parseForm({'status': '-1'}, 'issue', nodeid),
            ({('issue', nodeid): {'status': None}}, []))
        self.assertEqual(self.db.issue.get(nodeid,'tx_Source'),'web')

    def testInvalidLinkValue(self):
# XXX This is not the current behaviour - should we enforce this?
#        self.assertRaises(IndexError, self.parseForm,
#            {'status': '4'}))
        self.assertRaises(FormError, self.parseForm, {'link': 'frozzle'})
        self.assertRaises(FormError, self.parseForm, {'status': 'frozzle'},
            'issue')

    #
    # Multilink
    #
    def testEmptyMultilink(self):
        self.assertEqual(self.parseForm({'nosy': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'nosy': ' '}),
            ({('test', None): {}}, []))

    def testSetMultilink(self):
        self.assertEqual(self.parseForm({'nosy': '1'}, 'issue'),
            ({('issue', None): {'nosy': ['1']}}, []))
        self.assertEqual(self.parseForm({'nosy': 'admin'}, 'issue'),
            ({('issue', None): {'nosy': ['1']}}, []))
        self.assertEqual(self.parseForm({'nosy': ['1','2']}, 'issue'),
            ({('issue', None): {'nosy': ['1','2']}}, []))
        self.assertEqual(self.parseForm({'nosy': '1,2'}, 'issue'),
            ({('issue', None): {'nosy': ['1','2']}}, []))
        self.assertEqual(self.parseForm({'nosy': 'admin,2'}, 'issue'),
            ({('issue', None): {'nosy': ['1','2']}}, []))

    def testMixedMultilink(self):
        form = cgi.FieldStorage()
        form.list.append(cgi.MiniFieldStorage('nosy', '1,2'))
        form.list.append(cgi.MiniFieldStorage('nosy', '3'))
        cl = client.Client(self.instance, None, {'PATH_INFO':'/'}, form)
        cl.classname = 'issue'
        cl.nodeid = None
        cl.db = self.db
        cl.language = ('en',)
        self.assertEqual(cl.parsePropsFromForm(create=1),
            ({('issue', None): {'nosy': ['1','2', '3']}}, []))

    def testEmptyMultilinkSet(self):
        nodeid = self.db.issue.create(nosy=['1','2'])
        self.assertEqual(self.parseForm({'nosy': ''}, 'issue', nodeid),
            ({('issue', nodeid): {'nosy': []}}, []))
        nodeid = self.db.issue.create(nosy=['1','2'])
        self.assertEqual(self.parseForm({'nosy': ' '}, 'issue', nodeid),
            ({('issue', nodeid): {'nosy': []}}, []))
        self.assertEqual(self.parseForm({'nosy': '1,2'}, 'issue', nodeid),
            ({('issue', nodeid): {}}, []))

    def testInvalidMultilinkValue(self):
# XXX This is not the current behaviour - should we enforce this?
#        self.assertRaises(IndexError, self.parseForm,
#            {'nosy': '4'}))
        self.assertRaises(FormError, self.parseForm, {'nosy': 'frozzle'},
            'issue')
        self.assertRaises(FormError, self.parseForm, {'nosy': '1,frozzle'},
            'issue')
        self.assertRaises(FormError, self.parseForm, {'multilink': 'frozzle'})

    def testMultilinkAdd(self):
        nodeid = self.db.issue.create(nosy=['1'])
        # do nothing
        self.assertEqual(self.parseForm({':add:nosy': ''}, 'issue', nodeid),
            ({('issue', nodeid): {}}, []))

        # do something ;)
        self.assertEqual(self.parseForm({':add:nosy': '2'}, 'issue', nodeid),
            ({('issue', nodeid): {'nosy': ['1','2']}}, []))
        self.assertEqual(self.parseForm({':add:nosy': '2,mary'}, 'issue',
            nodeid), ({('issue', nodeid): {'nosy': ['1','2','4']}}, []))
        self.assertEqual(self.parseForm({':add:nosy': ['2','3']}, 'issue',
            nodeid), ({('issue', nodeid): {'nosy': ['1','2','3']}}, []))

    def testMultilinkAddNew(self):
        self.assertEqual(self.parseForm({':add:nosy': ['2','3']}, 'issue'),
            ({('issue', None): {'nosy': ['2','3']}}, []))

    def testMultilinkRemove(self):
        nodeid = self.db.issue.create(nosy=['1','2'])
        # do nothing
        self.assertEqual(self.parseForm({':remove:nosy': ''}, 'issue', nodeid),
            ({('issue', nodeid): {}}, []))

        # do something ;)
        self.assertEqual(self.parseForm({':remove:nosy': '1'}, 'issue',
            nodeid), ({('issue', nodeid): {'nosy': ['2']}}, []))
        self.assertEqual(self.parseForm({':remove:nosy': 'admin,2'},
            'issue', nodeid), ({('issue', nodeid): {'nosy': []}}, []))
        self.assertEqual(self.parseForm({':remove:nosy': ['1','2']},
            'issue', nodeid), ({('issue', nodeid): {'nosy': []}}, []))

        # add and remove
        self.assertEqual(self.parseForm({':add:nosy': ['3'],
            ':remove:nosy': ['1','2']},
            'issue', nodeid), ({('issue', nodeid): {'nosy': ['3']}}, []))

        # remove one that doesn't exist?
        self.assertRaises(FormError, self.parseForm, {':remove:nosy': '4'},
            'issue', nodeid)

    def testMultilinkRetired(self):
        self.db.user.retire('2')
        self.assertEqual(self.parseForm({'nosy': ['2','3']}, 'issue'),
            ({('issue', None): {'nosy': ['2','3']}}, []))
        nodeid = self.db.issue.create(nosy=['1','2'])
        self.assertEqual(self.parseForm({':remove:nosy': '2'}, 'issue',
            nodeid), ({('issue', nodeid): {'nosy': ['1']}}, []))
        self.assertEqual(self.parseForm({':add:nosy': '3'}, 'issue', nodeid),
            ({('issue', nodeid): {'nosy': ['1','2','3']}}, []))

    def testAddRemoveNonexistant(self):
        self.assertRaises(FormError, self.parseForm, {':remove:foo': '2'},
            'issue')
        self.assertRaises(FormError, self.parseForm, {':add:foo': '2'},
            'issue')

    #
    # Password
    #
    def testEmptyPassword(self):
        self.assertEqual(self.parseForm({'password': ''}, 'user'),
            ({('user', None): {}}, []))
        self.assertEqual(self.parseForm({'password': ''}, 'user'),
            ({('user', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'password': ['', '']},
            'user')
        self.assertRaises(FormError, self.parseForm, {'password': 'foo',
            ':confirm:password': ['', '']}, 'user')

    def testSetPassword(self):
        self.assertEqual(self.parseForm({'password': 'foo',
            ':confirm:password': 'foo'}, 'user'),
            ({('user', None): {'password': 'foo'}}, []))

    def testSetPasswordConfirmBad(self):
        self.assertRaises(FormError, self.parseForm, {'password': 'foo'},
            'user')
        self.assertRaises(FormError, self.parseForm, {'password': 'foo',
            ':confirm:password': 'bar'}, 'user')

    def testEmptyPasswordNotSet(self):
        nodeid = self.db.user.create(username='1',
            password=password.Password('foo'))
        self.assertEqual(self.parseForm({'password': ''}, 'user', nodeid),
            ({('user', nodeid): {}}, []))
        nodeid = self.db.user.create(username='2',
            password=password.Password('foo'))
        self.assertEqual(self.parseForm({'password': '',
            ':confirm:password': ''}, 'user', nodeid),
            ({('user', nodeid): {}}, []))

    def testPasswordMigration(self):
        chef = self.db.user.lookup('Chef')
        form = dict(__login_name='Chef', __login_password='foo')
        cl = self._make_client(form)
        # assume that the "best" algorithm is the first one and doesn't
        # need migration, all others should be migrated.
        for scheme in password.Password.deprecated_schemes:
            if scheme == 'crypt' and os.name == 'nt':
                continue  # crypt is not available on Windows
            pw1 = password.Password('foo', scheme=scheme)
            self.assertEqual(pw1.needs_migration(), True)
            self.db.user.set(chef, password=pw1)
            self.db.commit()
            actions.LoginAction(cl).handle()
            pw = self.db.user.get(chef, 'password')
            self.assertEqual(pw, 'foo')
            self.assertEqual(pw.needs_migration(), False)
        pw1 = pw
        self.assertEqual(pw1.needs_migration(), False)
        scheme = password.Password.known_schemes[0]
        self.assertEqual(scheme, pw1.scheme)
        actions.LoginAction(cl).handle()
        pw = self.db.user.get(chef, 'password')
        self.assertEqual(pw, 'foo')
        self.assertEqual(pw, pw1)
        cl.db.close()

    def testPasswordConfigOption(self):
        chef = self.db.user.lookup('Chef')
        form = dict(__login_name='Chef', __login_password='foo')
        cl = self._make_client(form)
        self.db.config.PASSWORD_PBKDF2_DEFAULT_ROUNDS = 1000
        pw1 = password.Password('foo', scheme='MD5')
        self.assertEqual(pw1.needs_migration(), True)
        self.db.user.set(chef, password=pw1)
        self.db.commit()
        actions.LoginAction(cl).handle()
        pw = self.db.user.get(chef, 'password')
        self.assertEqual('PBKDF2', pw.scheme)
        self.assertEqual(1000, password.pbkdf2_unpack(pw.password)[0])
        cl.db.close()

    #
    # Boolean
    #
    def testEmptyBoolean(self):
        self.assertEqual(self.parseForm({'boolean': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'boolean': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'boolean': ['', '']})

    def testSetBoolean(self):
        self.assertEqual(self.parseForm({'boolean': 'yes'}),
            ({('test', None): {'boolean': 1}}, []))
        self.assertEqual(self.parseForm({'boolean': 'a\r\nb\r\n'}),
            ({('test', None): {'boolean': 0}}, []))
        nodeid = self.db.test.create(boolean=1)
        self.assertEqual(self.parseForm({'boolean': 'yes'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))
        nodeid = self.db.test.create(boolean=0)
        self.assertEqual(self.parseForm({'boolean': 'no'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))

    def testEmptyBooleanSet(self):
        nodeid = self.db.test.create(boolean=0)
        self.assertEqual(self.parseForm({'boolean': ''}, 'test', nodeid),
            ({('test', nodeid): {'boolean': None}}, []))
        nodeid = self.db.test.create(boolean=1)
        self.assertEqual(self.parseForm({'boolean': ' '}, 'test', nodeid),
            ({('test', nodeid): {'boolean': None}}, []))

    def testRequiredBoolean(self):
        self.assertRaises(FormError, self.parseForm, {'boolean': '',
            ':required': 'boolean'})
        try:
            self.parseForm({'boolean': 'no', ':required': 'boolean'})
        except FormError:
            self.fail('boolean "no" raised "required missing"')

    #
    # Number
    #
    def testEmptyNumber(self):
        self.assertEqual(self.parseForm({'number': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'number': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'number': ['', '']})

    def testInvalidNumber(self):
        self.assertRaises(FormError, self.parseForm, {'number': 'hi, mum!'})

    def testSetNumber(self):
        self.assertEqual(self.parseForm({'number': '1'}),
            ({('test', None): {'number': 1}}, []))
        self.assertEqual(self.parseForm({'number': '0'}),
            ({('test', None): {'number': 0}}, []))
        self.assertEqual(self.parseForm({'number': '\n0\n'}),
            ({('test', None): {'number': 0}}, []))

    def testSetNumberReplaceOne(self):
        nodeid = self.db.test.create(number=1)
        self.assertEqual(self.parseForm({'number': '1'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))
        self.assertEqual(self.parseForm({'number': '0'}, 'test', nodeid),
            ({('test', nodeid): {'number': 0}}, []))

    def testSetNumberReplaceZero(self):
        nodeid = self.db.test.create(number=0)
        self.assertEqual(self.parseForm({'number': '0'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))

    def testSetNumberReplaceNone(self):
        nodeid = self.db.test.create()
        self.assertEqual(self.parseForm({'number': '0'}, 'test', nodeid),
            ({('test', nodeid): {'number': 0}}, []))
        self.assertEqual(self.parseForm({'number': '1'}, 'test', nodeid),
            ({('test', nodeid): {'number': 1}}, []))

    def testEmptyNumberSet(self):
        nodeid = self.db.test.create(number=0)
        self.assertEqual(self.parseForm({'number': ''}, 'test', nodeid),
            ({('test', nodeid): {'number': None}}, []))
        nodeid = self.db.test.create(number=1)
        self.assertEqual(self.parseForm({'number': ' '}, 'test', nodeid),
            ({('test', nodeid): {'number': None}}, []))

    def testRequiredNumber(self):
        self.assertRaises(FormError, self.parseForm, {'number': '',
            ':required': 'number'})
        try:
            self.parseForm({'number': '0', ':required': 'number'})
        except FormError:
            self.fail('number "no" raised "required missing"')

    #
    # Integer
    #
    def testEmptyInteger(self):
        self.assertEqual(self.parseForm({'intval': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'intval': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'intval': ['', '']})

    def testInvalidInteger(self):
        self.assertRaises(FormError, self.parseForm, {'intval': 'hi, mum!'})

    def testSetInteger(self):
        self.assertEqual(self.parseForm({'intval': '1'}),
            ({('test', None): {'intval': 1}}, []))
        self.assertEqual(self.parseForm({'intval': '0'}),
            ({('test', None): {'intval': 0}}, []))
        self.assertEqual(self.parseForm({'intval': '\n0\n'}),
            ({('test', None): {'intval': 0}}, []))

    def testSetIntegerReplaceOne(self):
        nodeid = self.db.test.create(intval=1)
        self.assertEqual(self.parseForm({'intval': '1'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))
        self.assertEqual(self.parseForm({'intval': '0'}, 'test', nodeid),
            ({('test', nodeid): {'intval': 0}}, []))

    def testSetIntegerReplaceZero(self):
        nodeid = self.db.test.create(intval=0)
        self.assertEqual(self.parseForm({'intval': '0'}, 'test', nodeid),
            ({('test', nodeid): {}}, []))

    def testSetIntegerReplaceNone(self):
        nodeid = self.db.test.create()
        self.assertEqual(self.parseForm({'intval': '0'}, 'test', nodeid),
            ({('test', nodeid): {'intval': 0}}, []))
        self.assertEqual(self.parseForm({'intval': '1'}, 'test', nodeid),
            ({('test', nodeid): {'intval': 1}}, []))

    def testEmptyIntegerSet(self):
        nodeid = self.db.test.create(intval=0)
        self.assertEqual(self.parseForm({'intval': ''}, 'test', nodeid),
            ({('test', nodeid): {'intval': None}}, []))
        nodeid = self.db.test.create(intval=1)
        self.assertEqual(self.parseForm({'intval': ' '}, 'test', nodeid),
            ({('test', nodeid): {'intval': None}}, []))

    def testRequiredInteger(self):
        self.assertRaises(FormError, self.parseForm, {'intval': '',
            ':required': 'intval'})
        try:
            self.parseForm({'intval': '0', ':required': 'intval'})
        except FormError:
            self.fail('intval "no" raised "required missing"')

    #
    # Date
    #
    def testEmptyDate(self):
        self.assertEqual(self.parseForm({'date': ''}),
            ({('test', None): {}}, []))
        self.assertEqual(self.parseForm({'date': ' '}),
            ({('test', None): {}}, []))
        self.assertRaises(FormError, self.parseForm, {'date': ['', '']})

    def testInvalidDate(self):
        self.assertRaises(FormError, self.parseForm, {'date': '12'})

    def testSetDate(self):
        self.assertEqual(self.parseForm({'date': '2003-01-01'}),
            ({('test', None): {'date': date.Date('2003-01-01')}}, []))
        nodeid = self.db.test.create(date=date.Date('2003-01-01'))
        self.assertEqual(self.parseForm({'date': '2003-01-01'}, 'test',
            nodeid), ({('test', nodeid): {}}, []))

    def testEmptyDateSet(self):
        nodeid = self.db.test.create(date=date.Date('.'))
        self.assertEqual(self.parseForm({'date': ''}, 'test', nodeid),
            ({('test', nodeid): {'date': None}}, []))
        nodeid = self.db.test.create(date=date.Date('1970-01-01.00:00:00'))
        self.assertEqual(self.parseForm({'date': ' '}, 'test', nodeid),
            ({('test', nodeid): {'date': None}}, []))

    #
    # Test multiple items in form
    #
    def testMultiple(self):
        self.assertEqual(self.parseForm({'string': 'a', 'issue-1@title': 'b'}),
            ({('test', None): {'string': 'a'},
              ('issue', '-1'): {'title': 'b'}
             }, []))

    def testMultipleExistingContext(self):
        nodeid = self.db.test.create()
        self.assertEqual(self.parseForm({'string': 'a', 'issue-1@title': 'b'},
            'test', nodeid),({('test', nodeid): {'string': 'a'},
            ('issue', '-1'): {'title': 'b'}}, []))

    def testLinking(self):
        self.assertEqual(self.parseForm({
            'string': 'a',
            'issue-1@add@nosy': '1',
            'issue-2@link@superseder': 'issue-1',
            }),
            ({('test', None): {'string': 'a'},
              ('issue', '-1'): {'nosy': ['1']},
             },
             [('issue', '-2', 'superseder', [('issue', '-1')])
             ]
            )
        )

    def testMessages(self):
        self.assertEqual(self.parseForm({
            'msg-1@content': 'asdf',
            'msg-2@content': 'qwer',
            '@link@messages': 'msg-1, msg-2'}),
            ({('test', None): {},
              ('msg', '-2'): {'content': 'qwer'},
              ('msg', '-1'): {'content': 'asdf'}},
             [('test', None, 'messages', [('msg', '-1'), ('msg', '-2')])]
            )
        )

    def testLinkBadDesignator(self):
        self.assertRaises(FormError, self.parseForm,
            {'test-1@link@link': 'blah'})
        self.assertRaises(FormError, self.parseForm,
            {'test-1@link@link': 'issue'})

    def testLinkNotLink(self):
        self.assertRaises(FormError, self.parseForm,
            {'test-1@link@boolean': 'issue-1'})
        self.assertRaises(FormError, self.parseForm,
            {'test-1@link@string': 'issue-1'})

    def testBackwardsCompat(self):
        res = self.parseForm({':note': 'spam'}, 'issue')
        date = res[0][('msg', '-1')]['date']
        self.assertEqual(res, ({('issue', None): {}, ('msg', '-1'):
            {'content': 'spam', 'author': '1', 'date': date}},
            [('issue', None, 'messages', [('msg', '-1')])]))
        file = FileUpload('foo', 'foo.txt')
        self.assertEqual(self.parseForm({':file': file}, 'issue'),
            ({('issue', None): {}, ('file', '-1'): {'content': 'foo',
            'name': 'foo.txt', 'type': 'text/plain'}},
            [('issue', None, 'files', [('file', '-1')])]))

    def testFormValuePreserveOnError(self):
        page_template = """
        <html>
         <body>
          <p tal:condition="options/error_message|nothing"
             tal:repeat="m options/error_message"
             tal:content="structure m"/>
          <p tal:content="context/title/plain"/>
          <p tal:content="context/priority/plain"/>
          <p tal:content="context/status/plain"/>
          <p tal:content="context/nosy/plain"/>
          <p tal:content="context/keyword/plain"/>
          <p tal:content="structure context/superseder/field"/>
         </body>
        </html>
        """.strip ()
        self.db.keyword.create (name = 'key1')
        self.db.keyword.create (name = 'key2')
        nodeid = self.db.issue.create (title = 'Title', priority = '1',
            status = '1', nosy = ['1'], keyword = ['1'])
        self.db.commit ()
        form = {':note': 'msg-content', 'title': 'New title',
            'priority': '2', 'status': '2', 'nosy': '1,2', 'keyword': '',
            'superseder': '5000', ':action': 'edit'}
        cl = self.setupClient(form, 'issue', '1',
                env_addon = {'HTTP_REFERER': 'http://whoami.com/path/'})
        pt = RoundupPageTemplate()
        pt.pt_edit(page_template, 'text/html')
        out = []
        def wh(s):
            out.append(s)
        cl.write_html = wh
        # Enable the following if we get a templating error:
        #def send_error (*args, **kw):
        #    import pdb; pdb.set_trace()
        #cl.send_error_to_admin = send_error
        # Need to rollback the database on error -- this usually happens
        # in web-interface (and for other databases) anyway, need it for
        # testing that the form values are really used, not the database!
        # We do this together with the setup of the easy template above
        def load_template(x):
            cl.db.rollback()
            return pt
        cl.instance.templates.load = load_template
        cl.selectTemplate = MockNull()
        cl.determine_context = MockNull ()
        def hasPermission(s, p, classname=None, d=None, e=None, **kw):
            return True
        actions.Action.hasPermission = hasPermission
        e1 = _HTMLItem.is_edit_ok
        _HTMLItem.is_edit_ok = lambda x : True
        e2 = HTMLProperty.is_edit_ok
        HTMLProperty.is_edit_ok = lambda x : True
        cl.inner_main()
        _HTMLItem.is_edit_ok = e1
        HTMLProperty.is_edit_ok = e2
        self.assertEqual(len(out), 1)
        self.assertEqual(out [0].strip (), """
        <html>
         <body>
          <p>Edit Error: issue has no node 5000</p>
          <p>New title</p>
          <p>urgent</p>
          <p>deferred</p>
          <p>admin, anonymous</p>
          <p></p>
          <p><input type="text" name="superseder" value="5000" size="30"></p>
         </body>
        </html>
        """.strip ())

    def testCsrfProtection(self):
        # need to set SENDMAILDEBUG to prevent
        # downstream issue when email is sent on successful
        # issue creation. Also delete the file afterwards
        # just tomake sure that someother test looking for
        # SENDMAILDEBUG won't trip over ours.
        if not os.environ.has_key('SENDMAILDEBUG'):
            os.environ['SENDMAILDEBUG'] = 'mail-test1.log'
        SENDMAILDEBUG = os.environ['SENDMAILDEBUG']

        page_template = """
        <html>
         <body>
          <p tal:condition="options/error_message|nothing"
             tal:repeat="m options/error_message"
             tal:content="structure m"/>
          <p tal:content="context/title/plain"/>
          <p tal:content="context/priority/plain"/>
          <p tal:content="context/status/plain"/>
          <p tal:content="context/nosy/plain"/>
          <p tal:content="context/keyword/plain"/>
          <p tal:content="structure context/superseder/field"/>
         </body>
        </html>
        """.strip ()
        self.db.keyword.create (name = 'key1')
        self.db.keyword.create (name = 'key2')
        nodeid = self.db.issue.create (title = 'Title', priority = '1',
            status = '1', nosy = ['1'], keyword = ['1'])
        self.db.commit ()
        form = {':note': 'msg-content', 'title': 'New title',
            'priority': '2', 'status': '2', 'nosy': '1,2', 'keyword': '',
            ':action': 'edit'}
        cl = self.setupClient(form, 'issue', '1')
        pt = RoundupPageTemplate()
        pt.pt_edit(page_template, 'text/html')
        out = []
        def wh(s):
            out.append(s)
        cl.write_html = wh
        # Enable the following if we get a templating error:
        #def send_error (*args, **kw):
        #    import pdb; pdb.set_trace()
        #cl.send_error_to_admin = send_error
        # Need to rollback the database on error -- this usually happens
        # in web-interface (and for other databases) anyway, need it for
        # testing that the form values are really used, not the database!
        # We do this together with the setup of the easy template above
        def load_template(x):
            cl.db.rollback()
            return pt
        cl.instance.templates.load = load_template
        cl.selectTemplate = MockNull()
        cl.determine_context = MockNull ()
        def hasPermission(s, p, classname=None, d=None, e=None, **kw):
            return True
        actions.Action.hasPermission = hasPermission
        e1 = _HTMLItem.is_edit_ok
        _HTMLItem.is_edit_ok = lambda x : True
        e2 = HTMLProperty.is_edit_ok
        HTMLProperty.is_edit_ok = lambda x : True

        # test with no headers and config by default requires 1 
        cl.inner_main()
        match_at=out[0].find('Unable to verify sufficient headers')
        self.assertNotEqual(match_at, -1)
        del(out[0])

        # all the rest of these allow at least one header to pass
        # and the edit happens with a redirect back to issue 1
        cl.env['HTTP_REFERER'] = 'http://whoami.com/path/'
        cl.inner_main()
        match_at=out[0].find('Redirecting to <a  target="_top" href="/newspapers?url=http://whoami.com/path/issue1?@ok_message')
        self.assertEqual(match_at, 0)
        del(cl.env['HTTP_REFERER'])
        del(out[0])

        cl.env['HTTP_ORIGIN'] = 'http://whoami.com'
        cl.inner_main()
        match_at=out[0].find('Redirecting to <a  target="_top" href="/newspapers?url=http://whoami.com/path/issue1?@ok_message')
        self.assertEqual(match_at, 0)
        del(cl.env['HTTP_ORIGIN'])
        del(out[0])

        cl.env['HTTP_X-FORWARDED-HOST'] = 'whoami.com'
        # if there is an X-FORWARDED-HOST header it is used and
        # HOST header is ignored. X-FORWARDED-HOST should only be
        # passed/set by a proxy. In this case the HOST header is
        # the proxy's name for the web server and not the name
        # thatis exposed to the world.
        cl.env['HTTP_HOST'] = 'frontend1.whoami.net'
        cl.inner_main()
        match_at=out[0].find('Redirecting to <a  target="_top" href="/newspapers?url=http://whoami.com/path/issue1?@ok_message')
        self.assertNotEqual(match_at, -1)
        del(cl.env['HTTP_X-FORWARDED-HOST'])
        del(cl.env['HTTP_HOST'])
        del(out[0])

        cl.env['HTTP_HOST'] = 'whoami.com'
        cl.inner_main()
        match_at=out[0].find('Redirecting to <a  target="_top" href="/newspapers?url=http://whoami.com/path/issue1?@ok_message')
        self.assertEqual(match_at, 0)
        del(cl.env['HTTP_HOST'])
        del(out[0])

        # try failing headers
        cl.env['HTTP_X-FORWARDED-HOST'] = 'whoami.net'
        # this raises an error as the header check passes and 
        # it did the edit and tries to send mail.
        cl.inner_main()
        match_at=out[0].find('Invalid X-FORWARDED-HOST whoami.net')
        self.assertNotEqual(match_at, -1)
        del(cl.env['HTTP_X-FORWARDED-HOST'])
        del(out[0])

        import copy

        form2 = copy.copy(form)
        form2.update({'@csrf': 'booogus'})
        # add a bogus csrf field to the form and rerun the inner_main
        cl.form = makeForm(form2)

        cl.env['HTTP_REFERER'] = 'http://whoami.com/path/'
        cl.inner_main()
        match_at=out[0].find('Invalid csrf token found: booogus')
        self.assertEqual(match_at, 36)
        del(out[0])

        form2 = copy.copy(form)
        nonce = anti_csrf_nonce(cl, cl)
        form2.update({'@csrf': nonce})
        # add a real csrf field to the form and rerun the inner_main
        cl.form = makeForm(form2)
        cl.inner_main()
        # csrf passes and redirects to the new issue.
        match_at=out[0].find('Redirecting to <a  target="_top" href="/newspapers?url=http://whoami.com/path/issue1?@ok_message')
        self.assertEqual(match_at, 0)
        del(cl.env['HTTP_REFERER'])
        del(out[0])

        # clean up from email log
        if os.path.exists(SENDMAILDEBUG):
            os.remove(SENDMAILDEBUG)
        #raise ValueError

    #
    # SECURITY
    #
    # XXX test all default permissions
    def _make_client(self, form, classname='user', nodeid='1',
           userid='2', template='item'):
        cl = client.Client(self.instance, None, {'PATH_INFO':'/',
            'REQUEST_METHOD':'POST'}, makeForm(form))
        cl.classname = classname
        if nodeid is not None:
            cl.nodeid = nodeid
        cl.db = self.db
        cl.userid = userid
        cl.language = ('en',)
        cl._error_message = []
        cl._ok_message = []
        cl.template = template
        return cl

    def testClassPermission(self):
        cl = self._make_client(dict(username='bob'))
        self.failUnlessRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl.nodeid = '1'
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)

    def testCheckAndPropertyPermission(self):
        self.db.security.permissions = {}
        def own_record(db, userid, itemid):
            return userid == itemid
        p = self.db.security.addPermission(name='Edit', klass='user',
            check=own_record, properties=("password", ))
        self.db.security.addPermissionToRole('User', p)

        cl = self._make_client(dict(username='bob'))
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(roles='User,Admin'), userid='4', nodeid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(roles='User,Admin'), userid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(roles='User,Admin'))
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        # working example, mary may change her pw
        cl = self._make_client({'password':'ob', '@confirm@password':'ob'},
            nodeid='4', userid='4')
        self.assertRaises(exceptions.Redirect,
            actions.EditItemAction(cl).handle)
        cl = self._make_client({'password':'bob', '@confirm@password':'bob'})
        self.failUnlessRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)

    def testCreatePermission(self):
        # this checks if we properly differentiate between create and
        # edit permissions
        self.db.security.permissions = {}
        self.db.security.addRole(name='UserAdd')
        # Don't allow roles
        p = self.db.security.addPermission(name='Create', klass='user',
            properties=("username", "password", "address",
            "alternate_address", "realname", "phone", "organisation",
            "timezone"))
        self.db.security.addPermissionToRole('UserAdd', p)
        # Don't allow roles *and* don't allow username
        p = self.db.security.addPermission(name='Edit', klass='user',
            properties=("password", "address", "alternate_address",
            "realname", "phone", "organisation", "timezone"))
        self.db.security.addPermissionToRole('UserAdd', p)
        self.db.user.set('4', roles='UserAdd')

        # anonymous may not
        cl = self._make_client({'username':'new_user', 'password':'secret',
            '@confirm@password':'secret', 'address':'new_user@bork.bork',
            'roles':'Admin'}, nodeid=None, userid='2')
        self.assertRaises(exceptions.Unauthorised,
            actions.NewItemAction(cl).handle)
        # Don't allow creating new user with roles
        cl = self._make_client({'username':'new_user', 'password':'secret',
            '@confirm@password':'secret', 'address':'new_user@bork.bork',
            'roles':'Admin'}, nodeid=None, userid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.NewItemAction(cl).handle)
        self.assertEqual(cl._error_message,[])
        # this should work
        cl = self._make_client({'username':'new_user', 'password':'secret',
            '@confirm@password':'secret', 'address':'new_user@bork.bork'},
            nodeid=None, userid='4')
        self.assertRaises(exceptions.Redirect,
            actions.NewItemAction(cl).handle)
        self.assertEqual(cl._error_message,[])
        # don't allow changing (my own) username (in this example)
        cl = self._make_client(dict(username='new_user42'), userid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(username='new_user42'), userid='4',
            nodeid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        # don't allow changing (my own) roles
        cl = self._make_client(dict(roles='User,Admin'), userid='4',
            nodeid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(roles='User,Admin'), userid='4')
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)
        cl = self._make_client(dict(roles='User,Admin'))
        self.assertRaises(exceptions.Unauthorised,
            actions.EditItemAction(cl).handle)

    def testSearchPermission(self):
        # this checks if we properly check for search permissions
        self.db.security.permissions = {}
        self.db.security.addRole(name='User')
        self.db.security.addRole(name='Project')
        self.db.security.addPermissionToRole('User', 'Web Access')
        self.db.security.addPermissionToRole('Project', 'Web Access')
        # Allow viewing department
        p = self.db.security.addPermission(name='View', klass='department')
        self.db.security.addPermissionToRole('User', p)
        # Allow viewing interesting things (but not department) on iss
        # But users might only view issues where they are on nosy
        # (so in the real world the check method would be better)
        p = self.db.security.addPermission(name='View', klass='iss',
            properties=("title", "status"), check=lambda x,y,z: True)
        self.db.security.addPermissionToRole('User', p)
        # Allow all relevant roles access to stat
        p = self.db.security.addPermission(name='View', klass='stat')
        self.db.security.addPermissionToRole('User', p)
        self.db.security.addPermissionToRole('Project', p)
        # Allow role "Project" access to whole iss
        p = self.db.security.addPermission(name='View', klass='iss')
        self.db.security.addPermissionToRole('Project', p)

        department = self.instance.backend.Class(self.db, "department",
            name=hyperdb.String())
        status = self.instance.backend.Class(self.db, "stat",
            name=hyperdb.String())
        issue = self.instance.backend.Class(self.db, "iss",
            title=hyperdb.String(), status=hyperdb.Link('stat'),
            department=hyperdb.Link('department'))

        d1 = department.create(name='d1')
        d2 = department.create(name='d2')
        open = status.create(name='open')
        closed = status.create(name='closed')
        issue.create(title='i1', status=open, department=d2)
        issue.create(title='i2', status=open, department=d1)
        issue.create(title='i2', status=closed, department=d1)

        chef = self.db.user.lookup('Chef')
        mary = self.db.user.lookup('mary')
        self.db.user.set(chef, roles = 'User, Project')

        perm = self.db.security.hasPermission
        search = self.db.security.hasSearchPermission
        self.assert_(perm('View', chef, 'iss', 'department', '1'))
        self.assert_(perm('View', chef, 'iss', 'department', '2'))
        self.assert_(perm('View', chef, 'iss', 'department', '3'))
        self.assert_(search(chef, 'iss', 'department'))

        self.assert_(not perm('View', mary, 'iss', 'department'))
        self.assert_(perm('View', mary, 'iss', 'status'))
        # Conditionally allow view of whole iss (check is False here,
        # this might check for department owner in the real world)
        p = self.db.security.addPermission(name='View', klass='iss',
            check=lambda x,y,z: False)
        self.db.security.addPermissionToRole('User', p)
        self.assert_(perm('View', mary, 'iss', 'department'))
        self.assert_(not perm('View', mary, 'iss', 'department', '1'))
        self.assert_(not search(mary, 'iss', 'department'))

        self.assert_(perm('View', mary, 'iss', 'status'))
        self.assert_(not search(mary, 'iss', 'status'))
        # Allow user to search for iss.status
        p = self.db.security.addPermission(name='Search', klass='iss',
            properties=("status",))
        self.db.security.addPermissionToRole('User', p)
        self.assert_(search(mary, 'iss', 'status'))

        dep = {'@action':'search','columns':'id','@filter':'department',
            'department':'1'}
        stat = {'@action':'search','columns':'id','@filter':'status',
            'status':'1'}
        depsort = {'@action':'search','columns':'id','@sort':'department'}
        depgrp = {'@action':'search','columns':'id','@group':'department'}

        # Filter on department ignored for role 'User':
        cl = self._make_client(dep, classname='iss', nodeid=None, userid=mary,
            template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['1', '2', '3'])
        # Filter on department works for role 'Project':
        cl = self._make_client(dep, classname='iss', nodeid=None, userid=chef,
            template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['2', '3'])
        # Filter on status works for all:
        cl = self._make_client(stat, classname='iss', nodeid=None, userid=mary,
            template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['1', '2'])
        cl = self._make_client(stat, classname='iss', nodeid=None, userid=chef,
            template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['1', '2'])
        # Sorting and grouping for class Project works:
        cl = self._make_client(depsort, classname='iss', nodeid=None,
            userid=chef, template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['2', '3', '1'])
        self.assertEqual(cl._error_message, []) # test for empty _error_message when sort is valid
        self.assertEqual(cl._ok_message, []) # test for empty _ok_message when sort is valid

        # Test for correct _error_message for invalid sort/group properties
        baddepsort = {'@action':'search','columns':'id','@sort':'dep'}
        baddepgrp = {'@action':'search','columns':'id','@group':'dep'}
        cl = self._make_client(baddepsort, classname='iss', nodeid=None,
            userid=chef, template='index')
        h = HTMLRequest(cl)
        self.assertEqual(cl._error_message, ['Unknown sort property dep'])
        cl = self._make_client(baddepgrp, classname='iss', nodeid=None,
            userid=chef, template='index')
        h = HTMLRequest(cl)
        self.assertEqual(cl._error_message, ['Unknown group property dep'])

        cl = self._make_client(depgrp, classname='iss', nodeid=None,
            userid=chef, template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['2', '3', '1'])
        # Sorting and grouping for class User fails:
        cl = self._make_client(depsort, classname='iss', nodeid=None,
            userid=mary, template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['1', '2', '3'])
        cl = self._make_client(depgrp, classname='iss', nodeid=None,
            userid=mary, template='index')
        h = HTMLRequest(cl)
        self.assertEqual([x.id for x in h.batch()],['1', '2', '3'])

    def testEditCSV(self):
        form = dict(rows='id,name\n1,newkey')
        cl = self._make_client(form, userid='1', classname='keyword')
        cl._ok_message = []
        actions.EditCSVAction(cl).handle()
        self.assertEqual(cl._ok_message, ['Items edited OK'])
        k = self.db.keyword.getnode('1')
        self.assertEqual(k.name, 'newkey')
        form = dict(rows=u'id,name\n1,\xe4\xf6\xfc'.encode('utf-8'))
        cl = self._make_client(form, userid='1', classname='keyword')
        cl._ok_message = []
        actions.EditCSVAction(cl).handle()
        self.assertEqual(cl._ok_message, ['Items edited OK'])
        k = self.db.keyword.getnode('1')
        self.assertEqual(k.name, u'\xe4\xf6\xfc'.encode('utf-8'))

    def testRoles(self):
        cl = self._make_client({})
        self.db.user.set('1', roles='aDmin,    uSer')
        item = HTMLItem(cl, 'user', '1')
        self.assert_(item.hasRole('Admin'))
        self.assert_(item.hasRole('User'))
        self.assert_(item.hasRole('AdmiN'))
        self.assert_(item.hasRole('UseR'))
        self.assert_(item.hasRole('UseR','Admin'))
        self.assert_(item.hasRole('UseR','somethingelse'))
        self.assert_(item.hasRole('somethingelse','Admin'))
        self.assert_(not item.hasRole('userr'))
        self.assert_(not item.hasRole('adminn'))
        self.assert_(not item.hasRole(''))
        self.assert_(not item.hasRole(' '))
        self.db.user.set('1', roles='')
        self.assert_(not item.hasRole(''))

    def testCSVExport(self):
        cl = self._make_client({'@columns': 'id,name'}, nodeid=None,
            userid='1')
        cl.classname = 'status'
        output = StringIO.StringIO()
        cl.request = MockNull()
        cl.request.wfile = output
        actions.ExportCSVAction(cl).handle()
        self.assertEquals('id,name\r\n1,unread\r\n2,deferred\r\n3,chatting\r\n'
            '4,need-eg\r\n5,in-progress\r\n6,testing\r\n7,done-cbb\r\n'
            '8,resolved\r\n',
            output.getvalue())

    def testCSVExportBadColumnName(self):
        cl = self._make_client({'@columns': 'falseid,name'}, nodeid=None,
            userid='1')
        cl.classname = 'status'
        output = StringIO.StringIO()
        cl.request = MockNull()
        cl.request.wfile = output
        self.assertRaises(exceptions.NotFound,
            actions.ExportCSVAction(cl).handle)

    def testCSVExportFailPermissionBadColumn(self):
        cl = self._make_client({'@columns': 'id,email,password'}, nodeid=None,
            userid='2')
        cl.classname = 'user'
        output = StringIO.StringIO()
        cl.request = MockNull()
        cl.request.wfile = output
        # used to be self.assertRaises(exceptions.Unauthorised,
        # but not acting like the column name is not found
        # see issue2550755 - should this return Unauthorised?
        # The unauthorised user should never get to the point where
        # they can determine if the column name is valid or not.
        self.assertRaises(exceptions.NotFound,
            actions.ExportCSVAction(cl).handle)

    def testCSVExportFailPermissionValidColumn(self):
        cl = self._make_client({'@columns': 'id,address,password'}, nodeid=None,
            userid='2')
        cl.classname = 'user'
        output = StringIO.StringIO()
        cl.request = MockNull()
        cl.request.wfile = output
        # used to be self.assertRaises(exceptions.Unauthorised,
        # but not acting like the column name is not found
        self.assertRaises(exceptions.Unauthorised,
            actions.ExportCSVAction(cl).handle)

class TemplateHtmlRendering(unittest.TestCase):
    ''' try to test the rendering code for tal '''
    def setUp(self):
        self.dirname = '_test_template'
        # set up and open a tracker
        self.instance = db_test_base.setupTracker(self.dirname)

        # open the database
        self.db = self.instance.open('admin')
        self.db.tx_Source = "web"
        self.db.user.create(username='Chef', address='chef@bork.bork.bork',
            realname='Bork, Chef', roles='User')
        self.db.user.create(username='mary', address='mary@test.test',
            roles='User', realname='Contrary, Mary')
        self.db.post_init()

        # create a client instance and hijack write_html
        self.client = client.Client(self.instance, "user",
                {'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
                form=makeForm({"@template": "item"}))

        self.client._error_message = []
        self.client._ok_message = []
        self.client.db = self.db
        self.client.userid = '1'
        self.client.language = ('en',)
        self.client.session_api = MockNull(_sid="1234567890")

        self.output = []
        # ugly hack to get html_write to return data here.
        def html_write(s):
            self.output.append(s)

        # hijack html_write
        self.client.write_html = html_write

        self.db.issue.create(title='foo')

    def tearDown(self):
        self.db.close()
        try:
            shutil.rmtree(self.dirname)
        except OSError, error:
            if error.errno not in (errno.ENOENT, errno.ESRCH): raise

    def testrenderFrontPage(self):
        self.client.renderFrontPage("hello world RaNdOmJunk")
        # make sure we can find the "hello world RaNdOmJunk"
        # message in the output.
        self.assertNotEqual(-1,
           self.output[0].index('<p class="error-message">hello world RaNdOmJunk <br/ > </p>'))
        # make sure we can find issue 1 title foo in the output
        self.assertNotEqual(-1,
           self.output[0].index('<a  target="_top" href="/newspapers?url=issue1">foo</a>'))

        # make sure we can find the last SHA1 sum line at the end of the
        # page
        self.assertNotEqual(-1,
           self.output[0].index('<!-- SHA: c87a4e18d59a527331f1d367c0c6cc67ee123e63 -->'))

    def testrenderContext(self):
        # set up the client;
        # run determine_context to set the required client attributes
        # run renderContext(); check result for proper page

        # this will generate the default home page like
        # testrenderFrontPage
        self.client.form=makeForm({})
        self.client.path = ''
        self.client.determine_context()
        self.assertEqual((self.client.classname, self.client.template, self.client.nodeid), (None, '', None))
        self.assertEqual(self.client._ok_message, [])

        result = self.client.renderContext()
        self.assertNotEqual(-1,
           result.index('<!-- SHA: c87a4e18d59a527331f1d367c0c6cc67ee123e63 -->'))

        # now look at the user index page
        self.client.form=makeForm({ "@ok_message": "ok message", "@template": "index"})
        self.client.path = 'user'
        self.client.determine_context()
        self.assertEqual((self.client.classname, self.client.template, self.client.nodeid), ('user', 'index', None))
        self.assertEqual(self.client._ok_message, ['ok message'])

        result = self.client.renderContext()
        self.assertNotEqual(-1, result.index('<title>User listing - Roundup issue tracker</title>'))
        self.assertNotEqual(-1, result.index('ok message'))
        # print result

    def testRenderAltTemplates(self):
        # check that right page is returned when rendering
        #  @template=oktempl|errortmpl

        # set up the client;
        # run determine_context to set the required client attributes
        # run renderContext(); check result for proper page

        # Test ok state template that uses user.forgotten.html
        self.client.form=makeForm({"@template": "forgotten|item"})
        self.client.path = 'user'
        self.client.determine_context()
        self.client.session_api = MockNull(_sid="1234567890")
        self.assertEqual((self.client.classname, self.client.template, self.client.nodeid), ('user', 'forgotten|item', None))
        self.assertEqual(self.client._ok_message, [])
        
        result = self.client.renderContext()
        print result
        self.assertNotEqual(-1,
                            result.index('<!-- SHA: eb5dd0bec7a57d58cb7edbeb939fb0390ed1bf74 -->'))

        # now set an error in the form to get error template user.item.html
        self.client.form=makeForm({"@template": "forgotten|item",
                                   "@error_message": "this is an error"})
        self.client.path = 'user'
        self.client.determine_context()
        self.assertEqual((self.client.classname, self.client.template, self.client.nodeid), ('user', 'forgotten|item', None))
        self.assertEqual(self.client._ok_message, [])
        self.assertEqual(self.client._error_message, ["this is an error"])
        
        result = self.client.renderContext()
        print result
        self.assertNotEqual(-1,
                            result.index('<!-- SHA: 3b7ce7cbf24f77733c9b9f64a569d6429390cc3f -->'))


    def testexamine_url(self):
        ''' test the examine_url function '''

        def te(url, exception, raises=ValueError):
            with self.assertRaises(raises) as cm:
                examine_url(url)
            self.assertEqual(cm.exception.message, exception)


        action = actions.Action(self.client)
        examine_url = action.examine_url

        # Christmas tree url: test of every component that passes
        self.assertEqual(
            examine_url("http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue"),
            'http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue')

        # allow replacing http with https if base is http
        self.assertEqual(
            examine_url("https://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue"),
            'https://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue')


        # change base to use https and make sure we don't redirect to http
        saved_base = action.base
        action.base = "https://tracker.example/cgi-bin/roundup.cgi/bugs/"
        te("http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue",
           'Base url https://tracker.example/cgi-bin/roundup.cgi/bugs/ requires https. Redirect url http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue uses http.')
        action.base = saved_base

        # url doesn't have to be valid to roundup, just has to be contained
        # inside of roundup. No zoik class is defined
        self.assertEqual(examine_url("http://tracker.example/cgi-bin/roundup.cgi/bugs/zoik7;parm=bar?@template=foo&parm=(zot)#issue"), "http://tracker.example/cgi-bin/roundup.cgi/bugs/zoik7;parm=bar?@template=foo&parm=(zot)#issue")

        # test with wonky schemes
        te("email://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue",
        'Unrecognized scheme in email://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue')

        te("http%3a//tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue", 'Unrecognized scheme in http%3a//tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue')

        # test different netloc/path prefix
        # assert port
        te("http://tracker.example:1025/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue",'Net location in http://tracker.example:1025/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue does not match base: tracker.example')

        #assert user
        te("http://user@tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue", 'Net location in http://user@tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue does not match base: tracker.example')

        #assert user:password
        te("http://user:pass@tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue", 'Net location in http://user:pass@tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue does not match base: tracker.example')

        # try localhost http scheme
        te("http://localhost/cgi-bin/roundup.cgi/bugs/user3", 'Net location in http://localhost/cgi-bin/roundup.cgi/bugs/user3 does not match base: tracker.example')

        # try localhost https scheme
        te("https://localhost/cgi-bin/roundup.cgi/bugs/user3", 'Net location in https://localhost/cgi-bin/roundup.cgi/bugs/user3 does not match base: tracker.example')

        # try different host
        te("http://bad.guys.are.us/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue", 'Net location in http://bad.guys.are.us/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#issue does not match base: tracker.example')

        # change the base path to .../bug from .../bugs
        te("http://tracker.example/cgi-bin/roundup.cgi/bug/user3;parm=bar?@template=foo&parm=(zot)#issue", 'Base path /cgi-bin/roundup.cgi/bugs/ is not a prefix for url http://tracker.example/cgi-bin/roundup.cgi/bug/user3;parm=bar?@template=foo&parm=(zot)#issue')

        # change the base path eliminate - in cgi-bin
        te("http://tracker.example/cgibin/roundup.cgi/bug/user3;parm=bar?@template=foo&parm=(zot)#issue",'Base path /cgi-bin/roundup.cgi/bugs/ is not a prefix for url http://tracker.example/cgibin/roundup.cgi/bug/user3;parm=bar?@template=foo&parm=(zot)#issue')


        # scan for unencoded characters
        # we skip schema and net location since unencoded character
        # are allowed only by an explicit match to a reference.
        #
        # break components with unescaped character '<'
        # path component
        te("http://tracker.example/cgi-bin/roundup.cgi/bugs/<user3;parm=bar?@template=foo&parm=(zot)#issue", 'Path component (/cgi-bin/roundup.cgi/bugs/<user3) in http://tracker.example/cgi-bin/roundup.cgi/bugs/<user3;parm=bar?@template=foo&parm=(zot)#issue is not properly escaped')

        # params component
        te("http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=b<ar?@template=foo&parm=(zot)#issue", 'Params component (parm=b<ar) in http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=b<ar?@template=foo&parm=(zot)#issue is not properly escaped')

        # query component
        te("http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=<foo>&parm=(zot)#issue", 'Query component (@template=<foo>&parm=(zot)) in http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=<foo>&parm=(zot)#issue is not properly escaped')

        # fragment component
        te("http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#iss<ue", 'Fragment component (iss<ue) in http://tracker.example/cgi-bin/roundup.cgi/bugs/user3;parm=bar?@template=foo&parm=(zot)#iss<ue is not properly escaped')

class TemplateTestCase(unittest.TestCase):
    ''' Test the template resolving code, i.e. what can be given to @template
    '''
    def setUp(self):
        self.dirname = '_test_template'
        # set up and open a tracker
        self.instance = db_test_base.setupTracker(self.dirname)

        # open the database
        self.db = self.instance.open('admin')
        self.db.tx_Source = "web"
        self.db.user.create(username='Chef', address='chef@bork.bork.bork',
            realname='Bork, Chef', roles='User')
        self.db.user.create(username='mary', address='mary@test.test',
            roles='User', realname='Contrary, Mary')
        self.db.post_init()

    def tearDown(self):
        self.db.close()
        try:
            shutil.rmtree(self.dirname)
        except OSError, error:
            if error.errno not in (errno.ENOENT, errno.ESRCH): raise

    def testTemplateSubdirectory(self):
        # test for templates in subdirectories

        # make the directory
        subdir = self.dirname + "/html/subdir"
        os.mkdir(subdir)

        # get the client instance The form is needed to initialize,
        # but not used since I call selectTemplate directly.
        t = client.Client(self.instance, "user",
                {'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'},
         form=makeForm({"@template": "item"}))

        # create new file in subdir and a dummy file outside of
        # the tracker's html subdirectory
        shutil.copyfile(self.dirname + "/html/issue.item.html",
                        subdir + "/issue.item.html")
        shutil.copyfile(self.dirname + "/html/user.item.html",
                        self.dirname + "/user.item.html")

        # create link outside the html subdir. This should fail due to
        # path traversal check.
        os.symlink("../../user.item.html", subdir + "/user.item.html")
        # it will be removed and replaced by a later test

        # make sure a simple non-subdir template works.
        # user.item.html exists so this works.
        # note that the extension is not included just the basename
        self.assertEqual("user.item", t.selectTemplate("user", "item"))


        # make sure home templates work
        self.assertEqual("home", t.selectTemplate(None, ""))
        self.assertEqual("home.classlist", t.selectTemplate(None, "classlist"))

        # home.item doesn't exist should return _generic.item.
        self.assertEqual("_generic.item", t.selectTemplate(None, "item"))

        # test case where there is no view so generic template can't
        # be determined.
        with self.assertRaises(NoTemplate) as cm:
            t.selectTemplate("user", "")
        self.assertEqual(cm.exception.message,
                         '''Template "user" doesn't exist''')

        # there is no html/subdir/user.item.{,xml,html} so it will
        # raise NoTemplate.
        self.assertRaises(NoTemplate,
                          t.selectTemplate, "user", "subdir/item")

        # there is an html/subdir/issue.item.html so this succeeeds
        r = t.selectTemplate("issue", "subdir/item")
        self.assertEqual("subdir/issue.item", r)

        # there is a self.directory + /html/subdir/user.item.html file,
        # but it is a link to self.dir /user.item.html which is outside
        # the html subdir so is rejected by the path traversal check.
        # Prefer NoTemplate here, or should the code be changed to
        # report a new PathTraversal exception? Could the PathTraversal
        # exception leak useful info to an attacker??
        self.assertRaises(NoTemplate,
                          t.selectTemplate, "user", "subdir/item")

        # clear out the link and create a new one to self.dirname +
        # html/user.item.html which is inside the html subdir
        # so the template check returns the symbolic link path.
        os.remove(subdir + "/user.item.html")
        os.symlink("../user.item.html", subdir + "/user.item.xml")

        # template check works
        r = t.selectTemplate("user", "subdir/item")
        self.assertEquals("subdir/user.item", r)

# vim: set filetype=python sts=4 sw=4 et si :