code-c8f3ec942e29/scripts/Dockerfile0000644000000000000000000000744614152033331015435 0ustar 00000000000000# build in root dir using: # # docker build -t roundup-app --rm -f scripts/Dockerfile . # # run using: # # docker run --rm -v /.../issue.tracker:/usr/src/app/tracker \ # -p 9017:8080 roundup-app:latest # Global vars for all build stages # application directory ARG appdir=/usr/src/app # support roundup install from 'local' directory, # 'local_pip' local directory using pip to install or # latest release from 'pypi' ARG source=local FROM python:3-alpine as build # Inherit global values https://github.com/moby/moby/issues/37345 ARG appdir WORKDIR $appdir # Add packages needed to compile mysql, pgsql and other python modules. # Can't use apk to add them as that installs a 3.9 python version. # g++ installs cc1plus needed by pip install RUN apk add \ g++ \ gcc \ gpgme-dev \ linux-headers \ musl-dev \ mysql-dev \ postgresql-dev \ swig # add requirements for pip here, e.g. Whoosh, gpg, zstd or other # modules not installed in the base library. # ignore warnings from pip to use virtualenv COPY scripts/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # copy the elements of the release directory to the docker image COPY setup.py install/ COPY doc install/doc/ COPY frontends install/frontends/ COPY locale install/locale/ COPY roundup install/roundup/ COPY share install/share/ # verify source has one of two valid values then # install in python3 standard directories from local copy # or install in python3 standard directories from pypi using pip # import from global/command line ARG source RUN set -xv && if [ "$source" = "local" ] || \ [ "$source" = "pypi" ] || \ [ "$source" = "local_pip" ]; then :; \ else echo "invalid value for source: $source"; \ echo "must be local or pypi"; exit 1; fi; \ if [ "$source" = "local" ]; then cd install && ./setup.py install; fi; \ if [ "$source" = "local_pip" ]; then cd install && pip install \ --use-feature=in-tree-build . ; fi; \ if [ "$source" = "pypi" ]; then pip install roundup; \ cp -ril /usr/local/lib/python3.10/site-packages/usr/local/share/* \ /usr/local/share; fi # build a new smaller docker image for execution. Build image above # is 1G in size. FROM python:3-alpine # import from global ARG appdir WORKDIR $appdir # add libraries needed to run gpg/mysql/pgsql/brotli RUN apk add \ gpgme \ mariadb-connector-c \ libpq \ libstdc++ ARG source LABEL "org.roundup-tracker.vendor"="Roundup Issue Tracker Team" \ "org.roundup-tracker.description"="Roundup Issue Tracker using sqlite" \ "version"="2.1.0 $source" # pull over built assets COPY --from=build /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages/ COPY --from=build /usr/local/bin/roundup* /usr/local/bin/ COPY --from=build /usr/local/share /usr/local/share/ # map port 8080 to your local port EXPOSE 8080/tcp # mount a trackerdir on tracker location RUN mkdir tracker VOLUME $appdir/tracker HEALTHCHECK --start-period=1m \ CMD wget -q -O /dev/null --no-verbose http://localhost:8080/issues/ # do not run roundup as root RUN adduser -D -h /usr/src/app roundup USER roundup # run the server, disable output buffering so we can see logs. ENV PYTHONUNBUFFERED=1 ENTRYPOINT [ "roundup-server", "-n", "0.0.0.0" ] # allow the invoker to override cmd with multiple trackers # in each subdirectory under $appdir/tracker. E.G. # docker run .... \ # issues=tracker/issues foo=tracker/foo # # note using "issue=$appdir/tracker" results in error: # # No valid configuration files found in directory /usr/src/app/$appdir/tracker # # so $appdir not expanded and $PWD prefixed onto the (relative path) # $appdir/tracker. Hence use relative path for spec. CMD [ "issues=tracker" ] code-c8f3ec942e29/scripts/README.txt0000644000000000000000000000322114152033331015124 0ustar 00000000000000Scripts in this directory: add-issue Add a single issue, as specified on the command line, to your tracker. The initial message for the issue is taken from standard input. copy-user.py Copy one or more Roundup users from one tracker instance to another. imapServer.py This IMAP server script that runs in the background and checks for new email from a variety of mailboxes. import_sf.py Import tracker data from Sourceforge.NET into a new roundup instance. roundup-reminder Generate an email that lists outstanding issues. Send in both plain text and HTML formats. schema_diagram.py Generate a schema diagram for a roundup tracker. It generates a 'dot file' that is then fed into the 'dot' tool (http://www.graphviz.org) to generate a graph. schema-dump.py Use recently documented XML-RPC API to dump Roundup data schema in human readable form. spam-remover Remove file attachment spam from a tracker. (Warning destructive, read script well.) weekly-report Generate a simple report outlining the activity in one tracker for the most recent week. ---- server-ctl Control the roundup-server daemon from the command line with start, stop, restart, condstart (conditional start - only if server is stopped) and status commands. roundup.rc-debian An control script that may be installed in /etc/init.d on Debian systems. Offers start, stop and restart commands and integrates with the Debian init process. systemd.gunicorn A systemd unit file for running roundup under gnuicorn WSGI server. ---- contributors.py Analyzes Mercurial log, filters and compiles list of committers with years of contribution. Can be useful for updating COPYING.txt code-c8f3ec942e29/scripts/add-issue0000755000000000000000000000277514152033331015247 0ustar 00000000000000#! /usr/bin/env python ''' Usage: %s Create a new issue in the given tracker. Input is taken from STDIN to create the initial issue message (which may be empty). Issues will be created as the current user (%s) if they exist as a Roundup user, or "admin" otherwise. ''' from __future__ import print_function import sys, os, pwd from roundup import instance, mailgw, date # open the instance username = pwd.getpwuid(os.getuid())[0] if len(sys.argv) < 3: print("Error: Not enough arguments") print(__doc__.strip()%(sys.argv[0], username)) sys.exit(1) tracker_home = sys.argv[1] issue_priority = sys.argv[2] issue_title = ' '.join(sys.argv[3:]) # get the message, if any message_text = sys.stdin.read().strip() # open the tracker tracker = instance.open(tracker_home) db = tracker.open('admin') db.tx_Source = "cli" uid = db.user.lookup('admin') try: # try to open the tracker as the current user uid = db.user.lookup(username) db.close() db = tracker.open(username) except KeyError: pass try: # handle the message messages = [] if message_text: summary, x = mailgw.parseContent(message_text, 0, 0) msg = db.msg.create(content=message_text, summary=summary, author=uid, date=date.Date()) messages = [msg] # now create the issue db.issue.create(title=issue_title, priority=issue_priority, messages=messages) db.commit() finally: db.close() # vim: set filetype=python ts=4 sw=4 et si code-c8f3ec942e29/scripts/contributors.py0000644000000000000000000001016414152033331016541 0ustar 00000000000000""" Get Mercurial history data and output list of contributors with years. Public domain work by: anatoly techtonik """ from __future__ import print_function from subprocess import check_output # --- output settings contributors_by_year = True years_for_contributors = True verbose = True # /-- # --- project specific configuration ALIASES = { 'Richard Jones ': ['richard', 'Richard Jones '], 'Bernhard Reiter ': ['Bernhard Reiter ', 'Bernhard Reiter '], 'Ralf Schlatterbeck ': ['Ralf Schlatterbeck '], 'Stefan Seefeld ': ['Stefan Seefeld '], 'John P. Rouillard ': ['rouilj'], } ROBOTS = ['No Author '] # /-- def compress(years): """ Given a list of years like [2003, 2004, 2007], compress it into string like '2003-2004, 2007' >>> compress([2002]) '2002' >>> compress([2003, 2002]) '2002-2003' >>> compress([2009, 2004, 2005, 2006, 2007]) '2004-2007, 2009' >>> compress([2001, 2003, 2004, 2005]) '2001, 2003-2005' >>> compress([2009, 2011]) '2009, 2011' >>> compress([2009, 2010, 2011, 2006, 2007]) '2006-2007, 2009-2011' >>> compress([2002, 2003, 2004, 2005, 2006, 2009, 2012]) '2002-2006, 2009, 2012' """ years = sorted(years) # compress years into string comma = ', ' yearstr = '' for i in range(0,len(years)-1): if years[i+1]-years[i] == 1: if not yearstr or yearstr.endswith(comma): yearstr += '%s' % years[i] if yearstr.endswith('-'): pass else: yearstr += '-' else: yearstr += '%s, ' % years[i] if len(years) == 1: yearstr += str(years[0]) else: yearstr += '%s' % years[-1] return yearstr if __name__ == '__main__': if verbose: print("Getting HG log...") authorship = check_output('hg log --template "{date(date,\\"%Y\\")},{author}\n"') # authorship are strings like # 2003,Richard Jones # ... if verbose: print("Splitting...") # transform to a list of tuples authorship = [line.split(',', 1) for line in authorship.splitlines()] if verbose: print("Sorting...") years = {} # year -> set(author1, author2, ...) names = {} # author -> set(years) for year, author in authorship: if author in ROBOTS: continue # process aliases for name, aliases in ALIASES.items(): if author in aliases: author = name break author = author.replace('<', '(') author = author.replace('>', ')') # years if not year in years: years[year] = set() years[year].add(author) # names if not author in names: names[author] = set() names[author].add(int(year)) if contributors_by_year: if verbose: print("Contributors by year...") print('') for year in sorted(years, reverse=True): print(year) for author in sorted(years[year]): print(" " + author) print('') if years_for_contributors: if verbose: print("Years for each contributor...") print('') def last_year(name): """Return year of the latest contribution for a given name""" return sorted(list(names[name]))[-1] def first_year(name): """Return year of the first contribution""" return sorted(list(names[name]))[0] def year_key(name): """ Year key function. First sort by latest contribution year (desc). If it matches, compare first contribution year (asc). This ensures that the most recent and long-term contributors are at the top. """ return (last_year(name), -first_year(name)) print("Copyright (c)") for author in sorted(list(names), key=year_key, reverse=True): years = list(names[author]) yearstr = compress(years) if 0: #DEBUG print(years, yearstr, author) else: print(" %s %s" % (yearstr, author)) print('') code-c8f3ec942e29/scripts/copy-user.py0000755000000000000000000000475114152033331015742 0ustar 00000000000000#!/usr/bin/env python # Copyright (C) 2003 by Intevation GmbH # Author: # Thomas Arendsen Hein # # This program is free software dual licensed under the GPL (>=v2) # and the Roundup Licensing (see COPYING.txt in the roundup distribution). """ copy-user [...] Copy one or more Roundup users from one tracker instance to another. Example: copy-user /roundup/tracker1 /roundup/tracker2 `seq 3 10` 14 16 (copies users 3, 4, 5, 6, 7, 8, 9, 10, 14 and 16) """ from __future__ import print_function import sys import roundup.instance def copy_user(home1, home2, *userids): """Copy users which are listed by userids from home1 to home2""" copyattribs = ['username', 'password', 'address', 'realname', 'phone', 'organisation', 'alternate_addresses', 'roles', 'timezone'] try: instance1 = roundup.instance.open(home1) print("Opened source instance: %s" % home1) except: print("Can't open source instance: %s" % home1) sys.exit(1) try: instance2 = roundup.instance.open(home2) print("Opened target instance: %s" % home2) except: print("Can't open target instance: %s" % home2) sys.exit(1) db1 = instance1.open('admin') db2 = instance2.open('admin') db1.tx_Source = "cli" db2.tx_Source = "cli" userlist = db1.user.list() for userid in userids: try: userid = str(int(userid)) except ValueError as why: print("Not a numeric user id: %s Skipping ..." % (userid,)) continue if userid not in userlist: print("User %s not in source instance. Skipping ..." % userid) continue user = {} for attrib in copyattribs: value = db1.user.get(userid, attrib) if value: user[attrib] = value try: db2.user.lookup(user['username']) print("User %s: Username '%s' exists in target instance. Skipping ..." % (userid, user['username'])) continue except KeyError as why: pass print("Copying user %s (%s) ..." % (userid, user['username'])) db2.user.create(**user) db2.commit() db2.close() print("Closed target instance.") db1.close() print("Closed source instance.") if __name__ == "__main__": if len(sys.argv) < 4: print(__doc__) sys.exit(1) else: copy_user(*sys.argv[1:]) code-c8f3ec942e29/scripts/imapServer.py0000644000000000000000000003373114152033331016126 0ustar 00000000000000#!/usr/bin/env python """\ This script is a wrapper around the mailgw.py script that exists in roundup. It runs as service instead of running as a one-time shot. It also connects to a secure IMAP server. The main reasons for this script are: 1) The roundup-mailgw script isn't designed to run as a server. It expects that you either run it by hand, and enter the password each time, or you supply the password on the command line. I prefer to run a server that I initialize with the password, and then it just runs. I don't want to have to pass it on the command line, so running through crontab isn't a possibility. (This wouldn't be a problem on a local machine running through a mailspool.) 2) mailgw.py somehow screws up SSL support so IMAP4_SSL doesn't work. So hopefully running that work outside of the mailgw will allow it to work. 3) I wanted to be able to check multiple projects at the same time. roundup-mailgw is only for 1 mailbox and 1 project. *TODO*: For the first round, the program spawns a new roundup-mailgw for each imap message that it finds and pipes the result in. In the future it might be more practical to actually include the roundup files and run the appropriate commands using python. *TODO*: Look into supporting a logfile instead of using 2>/logfile *TODO*: Add an option for changing the uid/gid of the running process. """ from __future__ import print_function import getpass import logging import imaplib import optparse import os import re import time from roundup.anypy.my_input import my_input logging.basicConfig() log = logging.getLogger('roundup.IMAPServer') version = '0.1.2' class RoundupMailbox: """This contains all the info about each mailbox. Username, Password, server, security, roundup database """ def __init__(self, dbhome='', username=None, password=None, mailbox=None , server=None, protocol='imaps'): self.username = username self.password = password self.mailbox = mailbox self.server = server self.protocol = protocol self.dbhome = dbhome try: if not self.dbhome: self.dbhome = my_input('Tracker home: ') if not os.path.exists(self.dbhome): raise ValueError('Invalid home address: ' \ 'directory "%s" does not exist.' % self.dbhome) if not self.server: self.server = my_input('Server: ') if not self.server: raise ValueError('No Servername supplied') protocol = my_input('protocol [imaps]? ') self.protocol = protocol if not self.username: self.username = my_input('Username: ') if not self.username: raise ValueError('Invalid Username') if not self.password: print('For server %s, user %s' % (self.server, self.username)) self.password = getpass.getpass() # password can be empty because it could be superceeded # by a later entry #if self.mailbox is None: # self.mailbox = my_input('Mailbox [INBOX]: ') # # We allow an empty mailbox because that will # # select the INBOX, whatever it is called except (KeyboardInterrupt, EOFError): raise ValueError('Canceled by User') def __str__(self): return 'Mailbox{ server:%(server)s, protocol:%(protocol)s, ' \ 'username:%(username)s, mailbox:%(mailbox)s, ' \ 'dbhome:%(dbhome)s }' % self.__dict__ # [als] class name is misleading. this is imap client, not imap server class IMAPServer: """IMAP mail gatherer. This class runs as a server process. It is configured with a list of mailboxes to connect to, along with the roundup database directories that correspond with each email address. It then connects to each mailbox at a specified interval, and if there are new messages it reads them, and sends the result to the roundup.mailgw. *TODO*: Try to be smart about how you access the mailboxes so that you can connect once, and access multiple mailboxes and possibly multiple usernames. *NOTE*: This assumes that if you are using the same user on the same server, you are using the same password. (the last one supplied is used.) Empty passwords are ignored. Only the last protocol supplied is used. """ def __init__(self, pidfile=None, delay=5, daemon=False): #This is sorted by servername, then username, then mailboxes self.mailboxes = {} self.delay = float(delay) self.pidfile = pidfile self.daemon = daemon def setDelay(self, delay): self.delay = delay def addMailbox(self, mailbox): """ The linkage is as follows: servers -- users - mailbox:dbhome So there can be multiple servers, each with multiple users. Each username can be associated with multiple mailboxes. each mailbox is associated with 1 database home """ log.info('Adding mailbox %s', mailbox) if mailbox.server not in self.mailboxes: self.mailboxes[mailbox.server] = {'protocol':'imaps', 'users':{}} server = self.mailboxes[mailbox.server] if mailbox.protocol: server['protocol'] = mailbox.protocol if mailbox.username not in server['users']: server['users'][mailbox.username] = {'password':'', 'mailboxes':{}} user = server['users'][mailbox.username] if mailbox.password: user['password'] = mailbox.password if mailbox.mailbox in user['mailboxes']: raise ValueError('Mailbox is already defined') user['mailboxes'][mailbox.mailbox] = mailbox.dbhome def _process(self, message, dbhome): """Actually process one of the email messages""" child = os.popen('roundup-mailgw %s' % dbhome, 'wb') child.write(message) child.close() #print message def _getMessages(self, serv, count, dbhome): """This assumes that you currently have a mailbox open, and want to process all messages that are inside. """ for n in range(1, count+1): (t, data) = serv.fetch(n, '(RFC822)') if t == 'OK': self._process(data[0][1], dbhome) serv.store(n, '+FLAGS', r'(\Deleted)') def checkBoxes(self): """This actually goes out and does all the checking. Returns False if there were any errors, otherwise returns true. """ noErrors = True for server in self.mailboxes: log.info('Connecting to server: %s', server) s_vals = self.mailboxes[server] try: for user in s_vals['users']: u_vals = s_vals['users'][user] # TODO: As near as I can tell, you can only # login with 1 username for each connection to a server. protocol = s_vals['protocol'].lower() if protocol == 'imaps': serv = imaplib.IMAP4_SSL(server) elif protocol == 'imap': serv = imaplib.IMAP4(server) else: raise ValueError('Unknown protocol %s' % protocol) password = u_vals['password'] try: log.info('Connecting as user: %s', user) serv.login(user, password) for mbox in u_vals['mailboxes']: dbhome = u_vals['mailboxes'][mbox] log.info('Using mailbox: %s, home: %s', mbox, dbhome) #access a specific mailbox if mbox: (t, data) = serv.select(mbox) else: # Select the default mailbox (INBOX) (t, data) = serv.select() try: nMessages = int(data[0]) except ValueError: nMessages = 0 log.info('Found %s messages', nMessages) if nMessages: self._getMessages(serv, nMessages, dbhome) serv.expunge() # We are done with this mailbox serv.close() except: log.exception('Exception with server %s user %s', server, user) noErrors = False serv.logout() serv.shutdown() del serv except: log.exception('Exception while connecting to %s', server) noErrors = False return noErrors def makeDaemon(self): """Turn this process into a daemon. - make our parent PID 1 Write our new PID to the pidfile. From A.M. Kuuchling (possibly originally Greg Ward) with modification from Oren Tirosh, and finally a small mod from me. Originally taken from roundup.scripts.roundup_server.py """ log.info('Running as Daemon') # Fork once if os.fork() != 0: os._exit(0) # Create new session os.setsid() # Second fork to force PPID=1 pid = os.fork() if pid: if self.pidfile: pidfile = open(self.pidfile, 'w') pidfile.write(str(pid)) pidfile.close() os._exit(0) def run(self): """Run email gathering daemon. This spawns itself as a daemon, and then runs continually, just sleeping inbetween checks. It is recommended that you run checkBoxes once first before you select run. That way you can know if there were any failures. """ if self.daemon: self.makeDaemon() while True: time.sleep(self.delay * 60.0) log.info('Time: %s', time.strftime('%Y-%m-%d %H:%M:%S')) self.checkBoxes() def getItems(s): """Parse a string looking for userame@server""" myRE = re.compile( r'((?P[^:]+)://)?'#You can supply a protocol if you like r'(' #The username part is optional r'(?P[^:]+)' #You can supply the password as r'(:(?P.+))?' #username:password@server r'@)?' r'(?P[^/]+)' r'(/(?P.+))?$' ) m = myRE.match(s) if m: return m.groupdict() else: return None def main(): """This is what is called if run at the prompt""" parser = optparse.OptionParser( version=('%prog ' + version), usage="""usage: %prog [options] (home server)... So each entry has a home, and then the server configuration. Home is just a path to the roundup issue tracker. The server is something of the form: imaps://user:password@server/mailbox If you don't supply the protocol, imaps is assumed. Without user or password, you will be prompted for them. The server must be supplied. Without mailbox the INBOX is used. Examples: %prog /home/roundup/trackers/test imaps://test@imap.example.com/test %prog /home/roundup/trackers/test imap.example.com \ /home/roundup/trackers/test2 imap.example.com/test2 """ ) parser.add_option('-d', '--delay', dest='delay', type='float', metavar='', default=5, help="Set the delay between checks in minutes. (default 5)" ) parser.add_option('-p', '--pid-file', dest='pidfile', metavar='', default=None, help="The pid of the server process will be written to " ) parser.add_option('-n', '--no-daemon', dest='daemon', action='store_false', default=True, help="Do not fork into the background after running the first check." ) parser.add_option('-v', '--verbose', dest='verbose', action='store_const', const=logging.INFO, help="Be more verbose in letting you know what is going on." " Enables informational messages." ) parser.add_option('-V', '--very-verbose', dest='verbose', action='store_const', const=logging.DEBUG, help="Be very verbose in letting you know what is going on." " Enables debugging messages." ) parser.add_option('-q', '--quiet', dest='verbose', action='store_const', const=logging.ERROR, help="Be less verbose. Ignores warnings, only prints errors." ) parser.add_option('-Q', '--very-quiet', dest='verbose', action='store_const', const=logging.CRITICAL, help="Be much less verbose. Ignores warnings and errors." " Only print CRITICAL messages." ) (opts, args) = parser.parse_args() if (len(args) == 0) or (len(args) % 2 == 1): parser.error('Invalid number of arguments. ' 'Each site needs a home and a server.') if opts.verbose == None: opts.verbose = logging.WARNING log.setLevel(opts.verbose) myServer = IMAPServer(delay=opts.delay, pidfile=opts.pidfile, daemon=opts.daemon) for i in range(0,len(args),2): home = args[i] server = args[i+1] if not os.path.exists(home): parser.error('Home: "%s" does not exist' % home) info = getItems(server) if not info: parser.error('Invalid server string: "%s"' % server) myServer.addMailbox( RoundupMailbox(dbhome=home, mailbox=info['mailbox'] , username=info['username'], password=info['password'] , server=info['server'], protocol=info['protocol'] ) ) if myServer.checkBoxes(): myServer.run() if __name__ == '__main__': main() # vim: et ft=python si sts=4 sw=4 code-c8f3ec942e29/scripts/import_sf.py0000644000000000000000000003424614152033331016015 0ustar 00000000000000""" Import tracker data from Sourceforge.NET This script needs four steps to work: 1. Export the project XML data using the admin web interface at sf.net 2. Run the file fetching (these are not included in the XML): import_sf.py files this will place all the downloaded files in the files dir by file id. 3. Convert the sf.net XML to Roundup "export" format: import_sf.py import this will generate a directory "/tmp/imported" which contains the data to be imported into a Roundup tracker. 4. Import the data: roundup-admin -i import /tmp/imported And you're done! """ import sys, os, csv, time, mimetypes try: import cElementTree as ElementTree except ImportError: from elementtree import ElementTree from roundup import instance, hyperdb, date, support, password from roundup.anypy import http_, urllib_ from roundup.anypy.strings import s2b, us2s today = date.Date('.') DL_URL = 'http://sourceforge.net/tracker/download.php?group_id=%(group_id)s&atid=%(atid)s&aid=%(aid)s' def get_url(aid): """ so basically we have to jump through hoops, given an artifact id, to figure what the URL should be to access that artifact, and hence any attached files.""" # first we hit this URL... conn = http_.client.HTTPConnection("sourceforge.net") conn.request("GET", "/support/tracker.php?aid=%s"%aid) response = conn.getresponse() # which should respond with a redirect to the correct url which has the # magic "group_id" and "atid" values in it that we need assert response.status == 302, 'response code was %s'%response.status location = response.getheader('location') query = urllib_.urlparse(response.getheader('location'))[-2] info = dict([param.split('=') for param in query.split('&')]) return DL_URL%info def fetch_files(xml_file, file_dir): """ Fetch files referenced in the xml_file into the dir file_dir. """ root = ElementTree.parse(xml_file).getroot() to_fetch = set() deleted = set() for artifact in root.find('artifacts'): for field in artifact.findall('field'): if field.get('name') == 'artifact_id': aid = field.text for field in artifact.findall('field'): if field.get('name') != 'artifact_history': continue for event in field.findall('history'): d = {} for field in event.findall('field'): d[field.get('name')] = field.text if d['field_name'] == 'File Added': fid = d['old_value'].split(':')[0] to_fetch.add((aid, fid)) if d['field_name'] == 'File Deleted': fid = d['old_value'].split(':')[0] deleted.add((aid, fid)) to_fetch = to_fetch - deleted got = set(os.listdir(file_dir)) to_fetch = to_fetch - got # load cached urls (sigh) urls = {} if os.path.exists(os.path.join(file_dir, 'urls.txt')): for line in open(os.path.join(file_dir, 'urls.txt')): aid, url = line.strip().split() urls[aid] = url for aid, fid in support.Progress('Fetching files', list(to_fetch)): if fid in got: continue if aid not in urls: urls[aid] = get_url(aid) f = open(os.path.join(file_dir, 'urls.txt'), 'a') f.write('%s %s\n'%(aid, urls[aid])) f.close() url = urls[aid] + '&file_id=' + fid f = urllib_.urlopen(url) data = f.read() n = open(os.path.join(file_dir, fid), 'w') n.write(data) f.close() n.close() def import_xml(tracker_home, xml_file, file_dir): """ Generate Roundup tracker import files based on the tracker schema, sf.net xml export and downloaded files from sf.net. """ tracker = instance.open(tracker_home) db = tracker.open('admin') resolved = db.status.lookup('resolved') unread = db.status.lookup('unread') chatting = db.status.lookup('unread') critical = db.priority.lookup('critical') urgent = db.priority.lookup('urgent') bug = db.priority.lookup('bug') feature = db.priority.lookup('feature') wish = db.priority.lookup('wish') adminuid = db.user.lookup('admin') anonuid = db.user.lookup('anonymous') root = ElementTree.parse(xml_file).getroot() def to_date(ts): return date.Date(time.gmtime(float(ts))) # parse out the XML artifacts = [] categories = set() users = set() add_files = set() remove_files = set() for artifact in root.find('artifacts'): d = {} op = {} artifacts.append(d) for field in artifact.findall('field'): name = field.get('name') if name == 'artifact_messages': for message in field.findall('message'): l = d.setdefault('messages', []) m = {} l.append(m) for field in message.findall('field'): name = field.get('name') if name == 'adddate': m[name] = to_date(field.text) else: m[name] = field.text if name == 'user_name': users.add(field.text) elif name == 'artifact_history': for event in field.findall('history'): l = d.setdefault('history', []) e = {} l.append(e) for field in event.findall('field'): name = field.get('name') if name == 'entrydate': e[name] = to_date(field.text) else: e[name] = field.text if name == 'mod_by': users.add(field.text) if e['field_name'] == 'File Added': add_files.add(e['old_value'].split(':')[0]) elif e['field_name'] == 'File Deleted': remove_files.add(e['old_value'].split(':')[0]) elif name == 'details': op['body'] = field.text elif name == 'submitted_by': op['user_name'] = field.text d[name] = field.text users.add(field.text) elif name == 'open_date': thedate = to_date(field.text) op['adddate'] = thedate d[name] = thedate else: d[name] = field.text categories.add(d['category']) if 'body' in op: l = d.setdefault('messages', []) l.insert(0, op) add_files -= remove_files # create users userd = {'nobody': '2'} users.remove('nobody') data = [ {'id': '1', 'username': 'admin', 'password': password.Password('admin'), 'roles': 'Admin', 'address': 'richard@python.org'}, {'id': '2', 'username': 'anonymous', 'roles': 'Anonymous'}, ] for n, user in enumerate(list(users)): userd[user] = n+3 data.append({'id': str(n+3), 'username': user, 'roles': 'User', 'address': '%s@users.sourceforge.net'%user}) write_csv(db.user, data) users=userd # create categories categoryd = {'None': None} categories.remove('None') data = [] for n, category in enumerate(list(categories)): categoryd[category] = n data.append({'id': str(n), 'name': category}) write_csv(db.keyword, data) categories = categoryd # create issues issue_data = [] file_data = [] message_data = [] issue_journal = [] message_id = 0 for artifact in artifacts: d = {} d['id'] = artifact['artifact_id'] d['title'] = artifact['summary'] d['assignedto'] = users[artifact['assigned_to']] if d['assignedto'] == '2': d['assignedto'] = None d['creation'] = artifact['open_date'] activity = artifact['open_date'] d['creator'] = users[artifact['submitted_by']] actor = d['creator'] if categories[artifact['category']]: d['keyword'] = [categories[artifact['category']]] issue_journal.append(( d['id'], d['creation'].get_tuple(), d['creator'], "'create'", {} )) p = int(artifact['priority']) if artifact['artifact_type'] == 'Feature Requests': if p > 3: d['priority'] = feature else: d['priority'] = wish else: if p > 7: d['priority'] = critical elif p > 5: d['priority'] = urgent elif p > 3: d['priority'] = bug else: d['priority'] = feature s = artifact['status'] if s == 'Closed': d['status'] = resolved elif s == 'Deleted': d['status'] = resolved d['is retired'] = True else: d['status'] = unread nosy = set() for message in artifact.get('messages', []): authid = users[message['user_name']] if not message['body']: continue body = convert_message(message['body'], message_id) if not body: continue m = {'content': body, 'author': authid, 'date': message['adddate'], 'creation': message['adddate'], } message_data.append(m) if authid not in (None, '2'): nosy.add(authid) activity = message['adddate'] actor = authid if d['status'] == unread: d['status'] = chatting # add import message m = {'content': 'IMPORT FROM SOURCEFORGE', 'author': '1', 'date': today, 'creation': today} message_data.append(m) # sort messages and assign ids d['messages'] = [] message_data.sort(key=lambda a:a['date']) for message in message_data: message_id += 1 message['id'] = str(message_id) d['messages'].append(message_id) d['nosy'] = list(nosy) files = [] for event in artifact.get('history', []): if event['field_name'] == 'File Added': fid, name = event['old_value'].split(':', 1) if fid in add_files: files.append(fid) name = name.strip() try: f = open(os.path.join(file_dir, fid), 'rb') content = f.read() f.close() except: content = 'content missing' file_data.append({ 'id': fid, 'creation': event['entrydate'], 'creator': users[event['mod_by']], 'name': name, 'type': mimetypes.guess_type(name)[0], 'content': content, }) continue elif event['field_name'] == 'close_date': action = "'set'" info = { 'status': unread } elif event['field_name'] == 'summary': action = "'set'" info = { 'title': event['old_value'] } else: # not an interesting / translatable event continue row = [ d['id'], event['entrydate'].get_tuple(), users[event['mod_by']], action, info ] if event['entrydate'] > activity: activity = event['entrydate'] issue_journal.append(row) d['files'] = files d['activity'] = activity d['actor'] = actor issue_data.append(d) write_csv(db.issue, issue_data) write_csv(db.msg, message_data) write_csv(db.file, file_data) f = open('/tmp/imported/issue-journals.csv', 'w') writer = csv.writer(f, colon_separated) writer.writerows(issue_journal) f.close() def convert_message(content, id): """ Strip off the useless sf message header crap """ if content[:14] == 'Logged In: YES': return '\n'.join(content.splitlines()[3:]).strip() return content class colon_separated(csv.excel): delimiter = ':' def write_csv(klass, data): props = klass.getprops() if not os.path.exists('/tmp/imported'): os.mkdir('/tmp/imported') f = open('/tmp/imported/%s.csv'%klass.classname, 'w') writer = csv.writer(f, colon_separated) propnames = klass.export_propnames() propnames.append('is retired') writer.writerow(propnames) for entry in data: row = [] for name in propnames: if name == 'is retired': continue prop = props[name] if name in entry: if isinstance(prop, hyperdb.Date) or \ isinstance(prop, hyperdb.Interval): row.append(repr(entry[name].get_tuple())) elif isinstance(prop, hyperdb.Password): row.append(repr(str(entry[name]))) else: row.append(repr(entry[name])) elif isinstance(prop, hyperdb.Multilink): row.append('[]') elif name in ('creator', 'actor'): row.append("'1'") elif name in ('created', 'activity'): row.append(repr(today.get_tuple())) else: row.append('None') row.append(entry.get('is retired', False)) writer.writerow(row) if isinstance(klass, hyperdb.FileClass) and entry.get('content'): fname = klass.exportFilename('/tmp/imported/', entry['id']) support.ensureParentsExist(fname) c = open(fname, 'wb') if isinstance(entry['content'], bytes): c.write(entry['content']) else: c.write(s2b(us2s(entry['content']))) c.close() f.close() f = open('/tmp/imported/%s-journals.csv'%klass.classname, 'w') f.close() if __name__ == '__main__': if sys.argv[1] == 'import': import_xml(*sys.argv[2:]) elif sys.argv[1] == 'files': fetch_files(*sys.argv[2:]) code-c8f3ec942e29/scripts/requirements.txt0000644000000000000000000000025414152033331016715 0ustar 00000000000000# human timezones pytz # indexer Whoosh # extra database support psycopg2 mysqlclient # encryption gpg # java web tokens PyJWT # extra HTTP compression methods Brotli zstd code-c8f3ec942e29/scripts/roundup-reminder0000755000000000000000000001452614152033331016665 0ustar 00000000000000#! /usr/bin/env python # Copyright (c) 2002 ekit.com Inc (http://www.ekit-inc.com/) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ''' Simple script that emails all users of a tracker with the issues that are currently assigned to them. TODO: introduce some structure ;) TODO: possibly make this more general and configurable... ''' from __future__ import print_function import sys from email.mime.multipart import MIMEMultipart from email.utils import make_msgid from roundup import instance, date from roundup.mailer import Mailer # open the instance if len(sys.argv) != 2: print('You need to specify an instance home dir') sys.exit() instance_home = sys.argv[1] instance = instance.open(instance_home) db = instance.open('admin') resolved_id = db.status.lookup('resolved') class Reverse: """Class reversing sort order.""" def __init__(self, val): self.val = val def __lt__(self, other): return other.val < self.val def __le__(self, other): return other.val <= self.val def __eq__(self, other): return other.val == self.val def __ne__(self, other): return other.val != self.val def __gt__(self, other): return other.val > self.val def __ge__(self, other): return other.val >= self.val def listKey(x): "key for tuples such that order is positive on [0] and negative on [1]" return (x[0], Reverse(x[1])) return 0 # loop through all the users for user_id in db.user.list(): # make sure we care about this user name = db.user.get(user_id, 'realname') if name is None: name = db.user.get(user_id, 'username') address = db.user.get(user_id, 'address') if address is None: continue # extract this user's issues l = [] for issue_id in db.issue.find(assignedto=user_id): if db.issue.get(issue_id, 'status') == resolved_id: continue order = db.priority.get(db.issue.get(issue_id, 'priority'), 'order') l.append((order, db.issue.get(issue_id, 'activity'), db.issue.get(issue_id, 'creation'), issue_id)) # sort the issues by timeliness and creation date l.sort(key=listKey) if not l: continue # generate the email message mailer = Mailer(db.config) message = MIMEMultipart('alternative') mailer.set_message_attributes( message, [address], 'Your active %s issues'%db.config.TRACKER_NAME) message['Reply-To'] = '%s <%s>'%(db.config.TRACKER_NAME, db.config.ADMIN_EMAIL) message['Message-Id'] = make_msgid() # do the plain text bit text_lines = [] text_lines.append('Created ID Activity Title') text_lines.append('='*75) # '2 months 213 immediate cc_daemon barfage old_priority = None for priority_order, activity_date, creation_date, issue_id in l: priority = db.issue.get(issue_id, 'priority') if (priority != old_priority): old_priority = priority text_lines.append(' ' + db.priority.get(priority,'name')) # pretty creation creation = (creation_date - date.Date('.')).pretty() activity = (activity_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') if len(title) > 42: title = title[:38] + ' ...' text_lines.append('%-11s %-4s %-9s %-42s'%(creation, issue_id, activity, title)) # some help to finish off text_lines.append(''' To view or respond to any of the issues listed above, visit the URL %s and click on "My Issues". Do NOT respond to this message. '''%db.config.TRACKER_WEB) text = '\n'.join(text_lines) + '\n' part = mailer.get_text_message() part.set_payload(text, part.get_charset()) message.attach(part) # now the HTML one html_lines = [] colours = { 'immediate': ' bgcolor="#ffcdcd"', 'day': ' bgcolor="#ffdecd"', 'week': ' bgcolor="#ffeecd"', 'month': ' bgcolor="#ffffcd"', 'whenever': ' bgcolor="#ffffff"', } html_lines.append(''' ''') old_priority = None for priority_order, activity_date, creation_date, issue_id in l: priority = db.issue.get(issue_id,'priority') if (priority != old_priority): old_priority = priority html_lines.append(''%db.priority.get(priority,'name')) creation = (creation_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') issue_id = '%s'%(db.config.TRACKER_WEB, issue_id, issue_id) activity = (activity_date - date.Date('.')).pretty() html_lines.append(''''''%(creation, issue_id, activity, title)) html_lines.append('
Created ID Activity Title
->->->%s
%s%s%s %s
') html_lines.append('''

To view or respond to any of the issues listed above, simply click on the issue ID. Do not respond to this message.

''') html = '\n'.join(html_lines) + '\n' part = mailer.get_text_message('utf-8', 'html') part.set_payload(html, part.get_charset()) message.attach(part) # all done, send! mailer.smtp_send([address], message.as_string()) # vim: set filetype=python ts=4 sw=4 et si code-c8f3ec942e29/scripts/roundup.rc-debian0000644000000000000000000000177514152033331016704 0ustar 00000000000000#!/bin/sh -e # # roundup Startup script for the roundup http server. DESC='Roundup HTTP-Server' BINFILE=roundup-server EXECUTABLE=/usr/local/bin/$BINFILE PIDFILE=/var/run/roundup/server.pid LOGFILE=/var/log/roundup/roundup.log TRACKERS=tttech=/tttech/org/software/roundup/tttech/ OPTIONS="-- -p 8080 -u roundup -d $PIDFILE -l $LOGFILE $TRACKERS" test -x $EXECUTABLE || exit 0 start_stop() { case "$1" in start) printf "Starting $DESC:" start-stop-daemon --start --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; stop) printf "Stopping $DESC:" start-stop-daemon --stop --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; restart | force-reload) start_stop stop sleep 1 start_stop start ;; *) printf "Usage: $0 {start|stop|restart|force-reload}\n" >&2 exit 1 ;; esac } start_stop "$@" exit 0 code-c8f3ec942e29/scripts/schema-dump.py0000755000000000000000000000527614152033331016222 0ustar 00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- """ Use recently documented XML-RPC API to dump Roundup data schema in human readable form. Future development may cover: [ ] unreadable dump formats [ ] access to local database [ ] lossless dump/restore cycle [ ] data dump and filtering with preserved """ from __future__ import print_function __license__ = "Public Domain" __version__ = "1.0" __authors__ = [ "anatoly techtonik " ] import os import sys from roundup.anypy import xmlrpc_ import pprint import textwrap from optparse import OptionParser sname = os.path.basename(sys.argv[0]) usage = """\ usage: %s [options] URL URL is XML-RPC endpoint for your tracker, such as: http://localhost:8917/demo/xmlrpc options: --pprint (default) --json --yaml --raw -h --help --version """ % sname def format_pprint(var): return pprint.pformat(var) def format_json(var): jout = pprint.pformat(var) jout = jout.replace('"', "\\'") # " to \' jout = jout.replace("'", '"') # ' to " jout = jout.replace('\\"', "'") # \" to ' return jout def format_yaml(var): out = pprint.pformat(var) out = out.replace('{', ' ') out = out.replace('}', '') out = textwrap.dedent(out) out = out.replace("'", '') out = out.replace(' [[', '\n [') out = out.replace(']]', ']') out = out.replace('],', '') out = out.replace(']', '') out2 = [] for line in out.splitlines(): if '[' in line: line = ' ' + line.lstrip(' [') line = line.replace('>', '') line = line.replace('roundup.hyperdb.', '') # expandtabs(16) with limit=1 n, v = line.split(', <') if len(n) > 14: indent = 0 else: indent = 14 - len(n) line = line.replace(', <', ': '+' '*indent) line.split(",") out2.append(line) out = '\n'.join(out2) return out if __name__ == "__main__": if len(sys.argv) < 2 or "-h" in sys.argv or "--help" in sys.argv: sys.exit(usage) if "--version" in sys.argv: sys.exit(sname + " " + __version__) parser = OptionParser() parser.add_option("--raw", action='store_true') parser.add_option("--yaml", action='store_true') parser.add_option("--json", action='store_true') (options, args) = parser.parse_args() url = args[0] roundup_server = xmlrpc_.client.ServerProxy(url, allow_none=True) schema = roundup_server.schema() if options.raw: print(str(schema)) elif options.yaml: print(format_yaml(schema)) elif options.json: print(format_json(schema)) else: print(format_pprint(schema)) print("") code-c8f3ec942e29/scripts/schema_diagram.py0000644000000000000000000000307114152033331016727 0ustar 00000000000000#! /usr/bin/python # # Schema diagram generator contributed by Stefan Seefeld of the fresco # project http://www.fresco.org/. # # It generates a 'dot file' that is then fed into the 'dot' # tool (http://www.graphviz.org) to generate a graph: # # %> ./schema.py # %> dot -Tps schema.dot -o schema.ps # %> gv schema.ps # from __future__ import print_function import sys import roundup.instance # open the instance instance = roundup.instance.open(sys.argv[1]) db = instance.open() # diagram preamble print('digraph schema {') print('size="8,6"') print('node [shape="record" bgcolor="#ffe4c4" style=filled]') print('edge [taillabel="1" headlabel="1" dir=back arrowtail=ediamond]') # get all the classes types = list(db.classes.keys()) # one record node per class for i in range(len(types)): print('node%d [label=\"{%s|}"]'%(i, types[i])) # now draw in the relations for name in db.classes.keys(): type = db.classes[name] attributes = type.getprops() for a in attributes.keys(): attribute = attributes[a] if isinstance(attribute, roundup.hyperdb.Link): print('node%d -> node%d [label=%s]'%(types.index(name), types.index(attribute.classname), a)) elif isinstance(attribute, roundup.hyperdb.Multilink): print('node%d -> node%d [taillabel="*" label=%s]'%(types.index(name), types.index(attribute.classname), a)) # all done print('}') code-c8f3ec942e29/scripts/server-ctl0000755000000000000000000000352214152033331015446 0ustar 00000000000000#!/bin/sh # # Configuration # CONFFILE="/var/roundup/server-config.ini" # this will end up with extra space, but it should be ignored in the script PIDFILE=`grep '^pidfile' ${CONFFILE} | awk -F = '{print $2}' ` SERVER="/usr/local/bin/roundup-server -C ${CONFFILE}" ERROR=0 ARGV="$@" if [ "x$ARGV" = "x" ] ; then ARGS="help" fi if [ -z "${PIDFILE}" ] ; then echo "pidfile option must be set in configuration file" exit 1 fi for ARG in $@ $ARGS do # check for pidfile if [ -f $PIDFILE ] ; then PID=`cat $PIDFILE` if [ "x$PID" != "x" ] && kill -0 $PID 2>/dev/null ; then STATUS="roundup-server (pid $PID) running" RUNNING=1 else STATUS="roundup-server (pid $PID?) not running" RUNNING=0 fi else STATUS="roundup-server (no pid file) not running" RUNNING=0 fi case $ARG in start) if [ $RUNNING -eq 1 ] ; then echo "$0 $ARG: roundup-server (pid $PID) already running" continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; condstart) if [ $RUNNING -eq 1 ] ; then continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; stop) if [ $RUNNING -eq 0 ] ; then echo "$0 $ARG: $STATUS" continue fi if kill $PID ; then echo "$0 $ARG: roundup-server stopped" else echo "$0 $ARG: roundup-server could not be stopped" ERROR=2 fi ;; status) echo $STATUS ;; *) echo "usage: $0 (start|condstart|stop|status)" cat <