# $Id: client.py,v 1.130.2.14 2004-02-26 22:20:43 richard Exp $
__doc__ = """
WWW request handler (also used in the stand-alone server).
"""
import os, os.path, cgi, StringIO, urlparse, re, traceback, mimetypes, urllib
import binascii, Cookie, time, random, MimeWriter, smtplib, socket, quopri
import stat, rfc822, string
from roundup import roundupdb, date, hyperdb, password, token, rcsv
from roundup.i18n import _
from roundup.cgi.templating import Templates, HTMLRequest, NoTemplate
from roundup.cgi import cgitb
from roundup.cgi.PageTemplates import PageTemplate
from roundup.rfc2822 import encode_header
from roundup.mailgw import uidFromAddress, openSMTPConnection
class HTTPException(Exception):
pass
class Unauthorised(HTTPException):
pass
class NotFound(HTTPException):
pass
class Redirect(HTTPException):
pass
class NotModified(HTTPException):
pass
class SeriousError(Exception):
''' Raised when we can't reasonably display an error message on a
templated page.
The exception value will be displayed in the error page, HTML
escaped.
'''
def __str__(self):
return '''
Roundup issue tracker: An error has occurred
%s
'''%cgi.escape(self.args[0])
# set to indicate to roundup not to actually _send_ email
# this var must contain a file to write the mail to
SENDMAILDEBUG = os.environ.get('SENDMAILDEBUG', '')
# used by a couple of routines
chars = string.letters+string.digits
# XXX actually _use_ FormError
class FormError(ValueError):
''' An "expected" exception occurred during form parsing.
- ie. something we know can go wrong, and don't want to alarm the
user with
We trap this at the user interface level and feed back a nice error
to the user.
'''
pass
class SendFile(Exception):
''' Send a file from the database '''
class SendStaticFile(Exception):
''' Send a static file from the instance html directory '''
def initialiseSecurity(security):
''' Create some Permissions and Roles on the security object
This function is directly invoked by security.Security.__init__()
as a part of the Security object instantiation.
'''
security.addPermission(name="Web Registration",
description="User may register through the web")
p = security.addPermission(name="Web Access",
description="User may access the web interface")
security.addPermissionToRole('Admin', p)
# doing Role stuff through the web - make sure Admin can
p = security.addPermission(name="Web Roles",
description="User may manipulate user Roles through the web")
security.addPermissionToRole('Admin', p)
# used to clean messages passed through CGI variables - HTML-escape any tag
# that isn't , , and values are variable, "@" may be
either ":" or "@", and other text "required" is fixed.
Most properties are specified as form variables:
- property on the current context item
"@"
- property on the indicated item (for editing related
information)
Designators name a specific item of a class.
Name an existing item of class .
"-"
Name the th new item of class . If the form
submission is successful, a new item of is
created. Within the submitted form, a particular
designator of this form always refers to the same new
item.
Once we have determined the "propname", we look at it to see
if it's special:
@required
The associated form value is a comma-separated list of
property names that must be specified when the form is
submitted for the edit operation to succeed.
When the is missing, the properties are
for the current context item. When is
present, they are for the item specified by
.
The "@required" specifier must come before any of the
properties it refers to are assigned in the form.
@remove@=id(s) or @add@=id(s)
The "@add@" and "@remove@" edit actions apply only to
Multilink properties. The form value must be a
comma-separate list of keys for the class specified by
the simple form variable. The listed items are added
to (respectively, removed from) the specified
property.
@link@=
If the edit action is "@link@", the simple form
variable must specify a Link or Multilink property.
The form value is a comma-separated list of
designators. The item corresponding to each
designator is linked to the property given by simple
form variable. These are collected up and returned in
all_links.
None of the above (ie. just a simple form value)
The value of the form variable is converted
appropriately, depending on the type of the property.
For a Link('klass') property, the form value is a
single key for 'klass', where the key field is
specified in dbinit.py.
For a Multilink('klass') property, the form value is a
comma-separated list of keys for 'klass', where the
key field is specified in dbinit.py.
Note that for simple-form-variables specifiying Link
and Multilink properties, the linked-to class must
have a key field.
For a String() property specifying a filename, the
file named by the form value is uploaded. This means we
try to set additional properties "filename" and "type" (if
they are valid for the class). Otherwise, the property
is set to the form value.
For Date(), Interval(), Boolean(), and Number()
properties, the form value is converted to the
appropriate
Any of the form variables may be prefixed with a classname or
designator.
Two special form values are supported for backwards
compatibility:
@note
This is equivalent to::
@link@messages=msg-1
@msg-1@content=value
except that in addition, the "author" and "date"
properties of "msg-1" are set to the userid of the
submitter, and the current time, respectively.
@file
This is equivalent to::
@link@files=file-1
@file-1@content=value
The String content value is handled as described above for
file uploads.
If both the "@note" and "@file" form variables are
specified, the action::
@link@msg-1@files=file-1
is also performed.
We also check that FileClass items have a "content" property with
actual content, otherwise we remove them from all_props before
returning.
The return from this method is a dict of
(classname, id): properties
... this dict _always_ has an entry for the current context,
even if it's empty (ie. a submission for an existing issue that
doesn't result in any changes would return {('issue','123'): {}})
The id may be None, which indicates that an item should be
created.
'''
# some very useful variables
db = self.db
form = self.form
if not hasattr(self, 'FV_SPECIAL'):
# generate the regexp for handling special form values
classes = '|'.join(db.classes.keys())
# specials for parsePropsFromForm
# handle the various forms (see unit tests)
self.FV_SPECIAL = re.compile(self.FV_LABELS%classes, re.VERBOSE)
self.FV_DESIGNATOR = re.compile(r'(%s)([-\d]+)'%classes)
# these indicate the default class / item
default_cn = self.classname
default_cl = self.db.classes[default_cn]
default_nodeid = self.nodeid
# we'll store info about the individual class/item edit in these
all_required = {} # required props per class/item
all_props = {} # props to set per class/item
got_props = {} # props received per class/item
all_propdef = {} # note - only one entry per class
all_links = [] # as many as are required
# we should always return something, even empty, for the context
all_props[(default_cn, default_nodeid)] = {}
keys = form.keys()
timezone = db.getUserTimezone()
# sentinels for the :note and :file props
have_note = have_file = 0
# extract the usable form labels from the form
matches = []
for key in keys:
m = self.FV_SPECIAL.match(key)
if m:
matches.append((key, m.groupdict()))
# now handle the matches
for key, d in matches:
if d['classname']:
# we got a designator
cn = d['classname']
cl = self.db.classes[cn]
nodeid = d['id']
propname = d['propname']
elif d['note']:
# the special note field
cn = 'msg'
cl = self.db.classes[cn]
nodeid = '-1'
propname = 'content'
all_links.append((default_cn, default_nodeid, 'messages',
[('msg', '-1')]))
have_note = 1
elif d['file']:
# the special file field
cn = 'file'
cl = self.db.classes[cn]
nodeid = '-1'
propname = 'content'
all_links.append((default_cn, default_nodeid, 'files',
[('file', '-1')]))
have_file = 1
else:
# default
cn = default_cn
cl = default_cl
nodeid = default_nodeid
propname = d['propname']
# the thing this value relates to is...
this = (cn, nodeid)
# skip implicit create if this isn't a create action
if not create and nodeid is None:
continue
# get more info about the class, and the current set of
# form props for it
if not all_propdef.has_key(cn):
all_propdef[cn] = cl.getprops()
propdef = all_propdef[cn]
if not all_props.has_key(this):
all_props[this] = {}
props = all_props[this]
if not got_props.has_key(this):
got_props[this] = {}
# is this a link command?
if d['link']:
value = []
for entry in extractFormList(form[key]):
m = self.FV_DESIGNATOR.match(entry)
if not m:
raise ValueError, \
'link "%s" value "%s" not a designator'%(key, entry)
value.append((m.group(1), m.group(2)))
# make sure the link property is valid
if (not isinstance(propdef[propname], hyperdb.Multilink) and
not isinstance(propdef[propname], hyperdb.Link)):
raise ValueError, '%s %s is not a link or '\
'multilink property'%(cn, propname)
all_links.append((cn, nodeid, propname, value))
continue
# detect the special ":required" variable
if d['required']:
all_required[this] = extractFormList(form[key])
continue
# see if we're performing a special multilink action
mlaction = 'set'
if d['remove']:
mlaction = 'remove'
elif d['add']:
mlaction = 'add'
# does the property exist?
if not propdef.has_key(propname):
if mlaction != 'set':
raise ValueError, 'You have submitted a %s action for'\
' the property "%s" which doesn\'t exist'%(mlaction,
propname)
# the form element is probably just something we don't care
# about - ignore it
continue
proptype = propdef[propname]
# Get the form value. This value may be a MiniFieldStorage or a list
# of MiniFieldStorages.
value = form[key]
# handle unpacking of the MiniFieldStorage / list form value
if isinstance(proptype, hyperdb.Multilink):
value = extractFormList(value)
else:
# multiple values are not OK
if isinstance(value, type([])):
raise ValueError, 'You have submitted more than one value'\
' for the %s property'%propname
# value might be a file upload...
if not hasattr(value, 'filename') or value.filename is None:
# nope, pull out the value and strip it
value = value.value.strip()
# now that we have the props field, we need a teensy little
# extra bit of help for the old :note field...
if d['note'] and value:
props['author'] = self.db.getuid()
props['date'] = date.Date()
# handle by type now
if isinstance(proptype, hyperdb.Password):
if not value:
# ignore empty password values
continue
for key, d in matches:
if d['confirm'] and d['propname'] == propname:
confirm = form[key]
break
else:
raise ValueError, 'Password and confirmation text do '\
'not match'
if isinstance(confirm, type([])):
raise ValueError, 'You have submitted more than one value'\
' for the %s property'%propname
if value != confirm.value:
raise ValueError, 'Password and confirmation text do '\
'not match'
value = password.Password(value)
elif isinstance(proptype, hyperdb.Link):
# see if it's the "no selection" choice
if value == '-1' or not value:
# if we're creating, just don't include this property
if not nodeid or nodeid.startswith('-'):
continue
value = None
else:
# handle key values
link = proptype.classname
if not num_re.match(value):
try:
value = db.classes[link].lookup(value)
except KeyError:
raise ValueError, _('property "%(propname)s": '
'%(value)s not a %(classname)s')%{
'propname': propname, 'value': value,
'classname': link}
except TypeError, message:
raise ValueError, _('you may only enter ID values '
'for property "%(propname)s": %(message)s')%{
'propname': propname, 'message': message}
elif isinstance(proptype, hyperdb.Multilink):
# perform link class key value lookup if necessary
link = proptype.classname
link_cl = db.classes[link]
l = []
for entry in value:
if not entry: continue
if not num_re.match(entry):
try:
entry = link_cl.lookup(entry)
except KeyError:
raise ValueError, _('property "%(propname)s": '
'"%(value)s" not an entry of %(classname)s')%{
'propname': propname, 'value': entry,
'classname': link}
except TypeError, message:
raise ValueError, _('you may only enter ID values '
'for property "%(propname)s": %(message)s')%{
'propname': propname, 'message': message}
l.append(entry)
l.sort()
# now use that list of ids to modify the multilink
if mlaction == 'set':
value = l
else:
# we're modifying the list - get the current list of ids
if props.has_key(propname):
existing = props[propname]
elif nodeid and not nodeid.startswith('-'):
existing = cl.get(nodeid, propname, [])
else:
existing = []
# now either remove or add
if mlaction == 'remove':
# remove - handle situation where the id isn't in
# the list
for entry in l:
try:
existing.remove(entry)
except ValueError:
raise ValueError, _('property "%(propname)s": '
'"%(value)s" not currently in list')%{
'propname': propname, 'value': entry}
else:
# add - easy, just don't dupe
for entry in l:
if entry not in existing:
existing.append(entry)
value = existing
value.sort()
elif value == '':
# if we're creating, just don't include this property
if not nodeid or nodeid.startswith('-'):
continue
# other types should be None'd if there's no value
value = None
else:
# handle ValueErrors for all these in a similar fashion
try:
if isinstance(proptype, hyperdb.String):
if (hasattr(value, 'filename') and
value.filename is not None):
# skip if the upload is empty
if not value.filename:
continue
# this String is actually a _file_
# try to determine the file content-type
fn = value.filename.split('\\')[-1]
if propdef.has_key('name'):
props['name'] = fn
# use this info as the type/filename properties
if propdef.has_key('type'):
if hasattr(value, 'type') and value.type:
props['type'] = value.type
elif mimetypes.guess_type(fn)[0]:
props['type'] = mimetypes.guess_type(fn)[0]
else:
props['type'] = "application/octet-stream"
# finally, read the content
value = value.value
else:
# normal String fix the CRLF/CR -> LF stuff
value = fixNewlines(value)
elif isinstance(proptype, hyperdb.Date):
value = date.Date(value, offset=timezone)
elif isinstance(proptype, hyperdb.Interval):
value = date.Interval(value)
elif isinstance(proptype, hyperdb.Boolean):
value = value.lower() in ('yes', 'true', 'on', '1')
elif isinstance(proptype, hyperdb.Number):
value = float(value)
except ValueError, msg:
raise ValueError, _('Error with %s property: %s')%(
propname, msg)
# register that we got this property
if value:
got_props[this][propname] = 1
# get the old value
if nodeid and not nodeid.startswith('-'):
try:
existing = cl.get(nodeid, propname)
except KeyError:
# this might be a new property for which there is
# no existing value
if not propdef.has_key(propname):
raise
# make sure the existing multilink is sorted
if isinstance(proptype, hyperdb.Multilink):
existing.sort()
# "missing" existing values may not be None
if not existing:
if isinstance(proptype, hyperdb.String) and not existing:
# some backends store "missing" Strings as empty strings
existing = None
elif isinstance(proptype, hyperdb.Number) and not existing:
# some backends store "missing" Numbers as 0 :(
existing = 0
elif isinstance(proptype, hyperdb.Boolean) and not existing:
# likewise Booleans
existing = 0
# if changed, set it
if value != existing:
props[propname] = value
else:
# don't bother setting empty/unset values
if value is None:
continue
elif isinstance(proptype, hyperdb.Multilink) and value == []:
continue
elif isinstance(proptype, hyperdb.String) and value == '':
continue
props[propname] = value
# check to see if we need to specially link a file to the note
if have_note and have_file:
all_links.append(('msg', '-1', 'files', [('file', '-1')]))
# see if all the required properties have been supplied
s = []
for thing, required in all_required.items():
# register the values we got
got = got_props.get(thing, {})
for entry in required[:]:
if got.has_key(entry):
required.remove(entry)
# any required values not present?
if not required:
continue
# tell the user to entry the values required
if len(required) > 1:
p = 'properties'
else:
p = 'property'
s.append('Required %s %s %s not supplied'%(thing[0], p,
', '.join(required)))
if s:
raise ValueError, '\n'.join(s)
# When creating a FileClass node, it should have a non-empty content
# property to be created. When editing a FileClass node, it should
# either have a non-empty content property or no property at all. In
# the latter case, nothing will change.
for (cn, id), props in all_props.items():
if isinstance(self.db.classes[cn], hyperdb.FileClass):
if id == '-1':
if not props.get('content', ''):
del all_props[(cn, id)]
elif props.has_key('content') and not props['content']:
raise ValueError, _('File is empty')
return all_props, all_links
def fixNewlines(text):
''' Homogenise line endings.
Different web clients send different line ending values, but
other systems (eg. email) don't necessarily handle those line
endings. Our solution is to convert all line endings to LF.
'''
text = text.replace('\r\n', '\n')
return text.replace('\r', '\n')
def extractFormList(value):
''' Extract a list of values from the form value.
It may be one of:
[MiniFieldStorage('value'), MiniFieldStorage('value','value',...), ...]
MiniFieldStorage('value,value,...')
MiniFieldStorage('value')
'''
# multiple values are OK
if isinstance(value, type([])):
# it's a list of MiniFieldStorages - join then into
values = ','.join([i.value.strip() for i in value])
else:
# it's a MiniFieldStorage, but may be a comma-separated list
# of values
values = value.value
value = [i.strip() for i in values.split(',')]
# filter out the empty bits
return filter(None, value)