code-4d7977d51a4e/scripts/Docker/Dockerfile0000644000000000000000000001425314341561674016575 0ustar 00000000000000# build in root dir using: # # docker build -t roundup-app --rm -f scripts/Dockerfile . # # run using: # # docker run --rm -v /.../issue.tracker:/usr/src/app/tracker \ # -p 9017:8080 roundup-app:latest # Global vars for all build stages # application directory ARG appdir=/usr/src/app # support roundup install from 'local' directory, # 'local_pip' local directory using pip to install or # latest release from 'pypi' ARG source=local # Python version as a.b Used for installation directory and # COPY from install dir in second stage. ARG pythonversion=3.11 FROM python:3-alpine as build # Inherit global values https://github.com/moby/moby/issues/37345 ARG appdir WORKDIR $appdir # Update to get security and other improvements; RUN apk --update-cache upgrade # Add packages needed to compile mysql, pgsql and other python modules. # Can't use apk to add them as that installs a 3.9 python version. # g++ installs cc1plus needed by pip install RUN apk add \ g++ \ gcc \ gpgme-dev \ libxapian \ linux-headers \ make \ musl-dev \ mysql-dev \ postgresql-dev \ swig \ xapian-core-dev ARG pythonversion # verify that pythonversion matches the one in the image. RUN image_python_version=$(python -c 'import sys; print("%s.%s"%sys.version_info[0:2])'); \ if [ "${pythonversion}" != "${image_python_version}" ]; then \ printf "\n\n*****\npythonversion does not match.\n" ; \ printf "Add:\n --build-arg=\"pythonversion=${image_python_version}\"\nto docker build\n******\n\n"; \ exit 1; \ fi # build xapian bindings: # file with sphinx build dependencies to remove after build # they are over 70MB of space. COPY scripts/Docker/sphinxdeps.txt . # suppress warning when running pip as root ENV PIP_ROOT_USER_ACTION=ignore RUN set -xv && CWD=$PWD && \ VER=$(apk list -I 'xapian-core-dev' | \ sed 's/^xapian-core-dev-\([0-9.]*\)-.*/\1/') && \ cd /tmp && \ wget https://oligarchy.co.uk/xapian/$VER/xapian-bindings-$VER.tar.xz && \ tar -Jxvf xapian-bindings-$VER.tar.xz && \ cd xapian-bindings-$VER/ && \ pip --no-cache-dir install sphinx && \ sed -i -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' configure && \ ./configure --prefix=/usr/local --with-python3 --disable-documentation && \ make && make install && \ pip uninstall --no-cache-dir -y sphinx && \ pip uninstall --no-cache-dir -y -r $CWD/sphinxdeps.txt # add requirements for pip here, e.g. Whoosh, gpg, zstd or other # modules not installed in the base library. # ignore warnings from pip to use virtualenv COPY scripts/Docker/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # copy the elements of the release directory to the docker image COPY setup.py install/ COPY doc install/doc/ COPY frontends install/frontends/ COPY locale install/locale/ COPY roundup install/roundup/ COPY share install/share/ # verify source has one of two valid values then # install in python3 standard directories from local copy # or install in python3 standard directories from pypi using pip # import from global/command line ARG source RUN set -xv && if [ "$source" = "local" ] || \ [ "$source" = "pypi" ] || \ [ "$source" = "local_pip" ]; then :; \ else echo "invalid value for source: $source"; \ echo "must be local or pypi"; exit 1; fi; \ if [ "$source" = "local" ]; then cd install && ./setup.py install; fi; \ if [ "$source" = "local_pip" ]; then cd install && pip install \ --use-feature=in-tree-build . ; fi; \ if [ "$source" = "pypi" ]; then pip install roundup; \ cp -ril /usr/local/lib/python${pythonversion}/site-packages/usr/local/share/* \ /usr/local/share; fi # Allow user to add more modules during build ARG pip_mod RUN if [ -n "$pip_mod" ]; then pip install --no-cache-dir ${pip_mod}; fi # build a new smaller docker image for execution. Build image above # is 1G in size. FROM python:3-alpine # import from global ARG appdir WORKDIR $appdir # suppress warning when running pip as root ENV PIP_ROOT_USER_ACTION=ignore # upgrade to get any security updates; bundle with # rest of apk actions to reduce layers/wasted space # add libraries needed to run gpg/mysql/pgsql/brotli # clean out any caches to save space RUN apk --update-cache upgrade; \ apk add \ brotli-libs \ gpgme \ mariadb-connector-c \ libpq \ libstdc++ \ libxapian \ zstd-libs; \ rm -f /var/cache/apk/* ARG source LABEL "org.roundup-tracker.vendor"="Roundup Issue Tracker Team" \ "org.roundup-tracker.description"="Roundup Issue Tracker multi-backend" \ "version"="2.2.0 $source" \ "org.opencontainers.image.authors"="roundup-devel@lists.sourceforge.net" ARG pythonversion # pull over built assets COPY --from=build /usr/local/lib/python${pythonversion}/site-packages /usr/local/lib/python${pythonversion}/site-packages/ COPY --from=build /usr/local/bin/roundup* /usr/local/bin/ COPY --from=build /usr/local/share /usr/local/share/ COPY scripts/Docker/roundup_start . COPY scripts/Docker/roundup_healthcheck . # make roundup scripts execuable and mount a trackerdir on tracker location RUN chmod +x roundup_start roundup_healthcheck; mkdir tracker VOLUME $appdir/tracker # map port 8080 to your local port EXPOSE 8080/tcp HEALTHCHECK --start-period=1m \ CMD ./roundup_healthcheck # do not run roundup as root. This creates roundup user and group. ARG roundup_uid RUN adduser -D -h ${appdir} -u ${roundup_uid:-1000} roundup USER roundup # run the server, disable output buffering so we can see logs. ENV PYTHONUNBUFFERED=1 #ENTRYPOINT [ "roundup-server", "-n", "0.0.0.0" ] ENTRYPOINT [ "./roundup_start" ] # allow the invoker to override cmd with multiple trackers # in each subdirectory under $appdir/tracker. E.G. # docker run .... \ # issues=tracker/issues foo=tracker/foo # # note using "issue=$appdir/tracker" results in error: # # No valid configuration files found in directory /usr/src/app/$appdir/tracker # # so $appdir not expanded and $PWD prefixed onto the (relative path) # $appdir/tracker. Hence use relative path for spec. CMD [ "issues=tracker" ] code-4d7977d51a4e/scripts/Docker/docker-compose.yml0000644000000000000000000000406214341561674020235 0ustar 00000000000000# docker-roundup # roundup issue tracker application with mariadb running as docker container # # docker-compose.yml # # Build components: # docker-compose -f scripts/Docker/docker-compose.yml build # # Install tracker template: # $ docker-compose -f scripts/Docker/docker-compose.yml run \ # --rm --entrypoint roundup-admin --no-deps roundup-app \ # -i tracker install # # Edit scripts/Docker/tracker/config.ini configure database settings # and any NO DEFAULT settings. # # Initialize the database, wait 1 minute so # # $ docker-compose -f scripts/Docker/docker-compose.yml run \ # --rm --entrypoint roundup-admin roundup-app \ # -i tracker # # wait 1 minute for mariadb to initialize # init tracker at roundup prompt # # roundup> init # roundup> exit # # may need ^\ to get roundup-admin to exit. # # run # docker-compose -f scripts/Docker/docker-compose.yml up # # tracker should be running at port 9017. # Note: mysql volume and tracker directories will be put in the # scripts/Docker subdir. # Paths for volumes are relative to docker-compose.yml location not # docker-compose cwd or build context directory. version: '3' services: mariadb: image: lscr.io/linuxserver/mariadb container_name: mariadb restart: unless-stopped environment: - PUID=1000 - PGID=1000 - TZ=America/New_York - MYSQL_ROOT_PASSWORD=myPassword - MYSQL_DATABASE=roundup - MYSQL_USER=roundup_user - MYSQL_PASSWORD=roundup_pass # ports: # - 3306:3306 volumes: - ./dbData:/config roundup-app: container_name: roundup-app build: context: ../.. dockerfile: scripts/Docker/Dockerfile args: source: local_pip #source: local #source: pypi command: "issues=tracker" restart: unless-stopped environment: - TZ=America/New_York ports: - 9017:8080 links: - mariadb depends_on: - mariadb volumes: # will be placed in Docker subdir next to this file - ./tracker:/usr/src/app/tracker code-4d7977d51a4e/scripts/Docker/requirements.txt0000644000000000000000000000031414341561674020060 0ustar 00000000000000# human timezones pytz # indexer Whoosh # extra database support psycopg2 mysqlclient # encryption gpg # java web tokens PyJWT # extra HTTP compression methods Brotli zstd # jinja2 template engine Jinja2 code-4d7977d51a4e/scripts/Docker/roundup_healthcheck0000755000000000000000000000022514341561674020542 0ustar 00000000000000#! /bin/sh tracker=$(ps -ef | sed -ne '/roundup-server/s/^.*\s\(\w*\)=.*$/\1/p') wget -q -O /dev/null --no-verbose http://localhost:8080/$tracker/ code-4d7977d51a4e/scripts/Docker/roundup_start0000755000000000000000000000326614341561674017444 0ustar 00000000000000#! /bin/sh # When container starts verify that the trackers are configured. # If they are, start the server otherwise run roundup-admin # for installation and initialization. # "$@" should be a set of tracker=directory pairs. if ! [ -z "$SHELL_DEBUG" ]; then set -xv fi trap exit INT do_exit=0 for tracker_spec in "$@"; do # IFS== set a=b doesn't assign $1 and $2 in busybox ash # it also clobbers '$@'. 'echo mumble | read' starts read in a # subshell so vars are not available in parent. IFS="=" read tracker directory <<- EOE $tracker_spec EOE # ^ is a tab for use with <<- # was $tracker_spec in the form of a=b, if not ignore it. # allows setting CMD to -i index_template issue=tracker for example. if [ -z "$directory" ]; then continue; fi # something is specified or built wrong. Don't start. if [ ! -d "$directory" ]; then printf "Unable to find directory %s. Exiting\n" "$directory" exit 1 fi # user must define web in config.ini. if ! grep '^\s*web\s\s*=\s\s*' "$directory/config.ini" > /dev/null; then roundup-admin -i "$directory" install do_exit=1 fi # we have a valid config.ini so init database if not done if [ $do_exit == 0 -a ! -e "$directory/.init_done" ]; then if roundup-admin -i "$directory" init; then cat > "$directory/.init_done" <<- EOD Don't delete this file. The docker startup needs it so it won't re-initialze the database destroying all the data. EOD else do_exit=1 fi fi done # if any config.ini needs editing don't start up. if [ $do_exit == 0 ]; then # make roundup-server process 1 with exec exec roundup-server -n 0.0.0.0 "$@" fi exit 0 code-4d7977d51a4e/scripts/Docker/sphinxdeps.txt0000644000000000000000000000031714341561674017525 0ustar 00000000000000Babel Jinja2 MarkupSafe Pygments alabaster certifi charset-normalizer docutils idna imagesize packaging pyparsing requests six snowballstemmer sphinxcontrib-serializinghtml sphinxcontrib-websupport urllib3 code-4d7977d51a4e/scripts/README.txt0000644000000000000000000000472514341561674015075 0ustar 00000000000000Scripts in this directory: add-issue Add a single issue, as specified on the command line, to your tracker. The initial message for the issue is taken from standard input. copy-user.py Copy one or more Roundup users from one tracker instance to another. dump_dbm_sessions_db.py Simple script to dump a session style dbm database e.g. db/otks or db/sessions in readable form. imapServer.py This IMAP server script that runs in the background and checks for new email from a variety of mailboxes. import_sf.py Import tracker data from Sourceforge.NET into a new roundup instance. roundup-reminder Generate an email that lists outstanding issues. Send in both plain text and HTML formats. schema_diagram.py Generate a schema diagram for a roundup tracker. It generates a 'dot file' that is then fed into the 'dot' tool (http://www.graphviz.org) to generate a graph. schema-dump.py Use recently documented XML-RPC API to dump Roundup data schema in human readable form. spam-remover Remove file attachment spam from a tracker. (Warning destructive, read script well.) stats.xmlrpc.py Script using the xmlrpc interface to retrieve info. Generates report on what values are used by the various issues on bugs.python.org. weekly-report Generate a simple report outlining the activity in one tracker for the most recent week. ---- server-ctl Control the roundup-server daemon from the command line with start, stop, restart, condstart (conditional start - only if server is stopped) and status commands. roundup.rc-debian An control script that may be installed in /etc/init.d on Debian systems. Offers start, stop and restart commands and integrates with the Debian init process. systemd.gunicorn A systemd unit file for running roundup under gnuicorn WSGI server. ---- contributors.py Analyzes Mercurial log, filters and compiles list of committers with years of contribution. Can be useful for updating COPYING.txt ---- Docker Directory for docker setup. More info on how to use it is in doc/installation.txt. Docker/Dockerfile - Create roundup docker. Docker/requirements.txt - Python requirements built into docker. Docker/roundup-start - Startup script for roundup in docker. Docker/docker-compose.yml - Manage two docker containers for roundup and mysql. ---- oauth-get-token.py Retrieve necessary credentials from MS cloud for OAuth authentication with IMAP. This has an extensive help text so you want to call this script with --help first. code-4d7977d51a4e/scripts/add-issue0000755000000000000000000000277514341561674015206 0ustar 00000000000000#! /usr/bin/env python ''' Usage: %s Create a new issue in the given tracker. Input is taken from STDIN to create the initial issue message (which may be empty). Issues will be created as the current user (%s) if they exist as a Roundup user, or "admin" otherwise. ''' from __future__ import print_function import sys, os, pwd from roundup import instance, mailgw, date # open the instance username = pwd.getpwuid(os.getuid())[0] if len(sys.argv) < 3: print("Error: Not enough arguments") print(__doc__.strip()%(sys.argv[0], username)) sys.exit(1) tracker_home = sys.argv[1] issue_priority = sys.argv[2] issue_title = ' '.join(sys.argv[3:]) # get the message, if any message_text = sys.stdin.read().strip() # open the tracker tracker = instance.open(tracker_home) db = tracker.open('admin') db.tx_Source = "cli" uid = db.user.lookup('admin') try: # try to open the tracker as the current user uid = db.user.lookup(username) db.close() db = tracker.open(username) except KeyError: pass try: # handle the message messages = [] if message_text: summary, x = mailgw.parseContent(message_text, 0, 0) msg = db.msg.create(content=message_text, summary=summary, author=uid, date=date.Date()) messages = [msg] # now create the issue db.issue.create(title=issue_title, priority=issue_priority, messages=messages) db.commit() finally: db.close() # vim: set filetype=python ts=4 sw=4 et si code-4d7977d51a4e/scripts/contributors.py0000644000000000000000000001030314341561674016473 0ustar 00000000000000""" Get Mercurial history data and output list of contributors with years. Public domain work by: anatoly techtonik """ from __future__ import print_function from subprocess import check_output # --- output settings contributors_by_year = True years_for_contributors = True verbose = True # /-- # --- project specific configuration ALIASES = { 'Richard Jones ': ['richard', 'Richard Jones '], 'Bernhard Reiter ': ['Bernhard Reiter ', 'Bernhard Reiter '], 'Ralf Schlatterbeck ': ['Ralf Schlatterbeck '], 'Stefan Seefeld ': ['Stefan Seefeld '], 'John Rouillard ': ['rouilj@uland', 'rouilj'] } ROBOTS = ['No Author '] # /-- def compress(years): """ Given a list of years like [2003, 2004, 2007], compress it into string like '2003-2004, 2007' >>> compress([2002]) '2002' >>> compress([2003, 2002]) '2002-2003' >>> compress([2009, 2004, 2005, 2006, 2007]) '2004-2007, 2009' >>> compress([2001, 2003, 2004, 2005]) '2001, 2003-2005' >>> compress([2009, 2011]) '2009, 2011' >>> compress([2009, 2010, 2011, 2006, 2007]) '2006-2007, 2009-2011' >>> compress([2002, 2003, 2004, 2005, 2006, 2009, 2012]) '2002-2006, 2009, 2012' """ years = sorted(years) # compress years into string comma = ', ' yearstr = '' for i in range(0,len(years)-1): if years[i+1]-years[i] == 1: if not yearstr or yearstr.endswith(comma): yearstr += '%s' % years[i] if yearstr.endswith('-'): pass else: yearstr += '-' else: yearstr += '%s, ' % years[i] if len(years) == 1: yearstr += str(years[0]) else: yearstr += '%s' % years[-1] return yearstr if __name__ == '__main__': command = 'hg log --template "{date(date,\\"%Y\\")},{author}\\n"' if verbose: print("Getting HG log...") print("Using: ", command) authorship = check_output(command, shell=True) # authorship are strings like # 2003,Richard Jones # ... if verbose: print("Splitting...") # transform to a list of tuples authorship = [line.split(',', 1) for line in authorship.splitlines()] if verbose: print("Sorting...") years = {} # year -> set(author1, author2, ...) names = {} # author -> set(years) for year, author in authorship: if author in ROBOTS: continue # process aliases for name, aliases in ALIASES.items(): if author in aliases: author = name break author = author.replace('<', '(') author = author.replace('>', ')') # years if not year in years: years[year] = set() years[year].add(author) # names if not author in names: names[author] = set() names[author].add(int(year)) if contributors_by_year: if verbose: print("Contributors by year...") print('') for year in sorted(years, reverse=True): print(year) for author in sorted(years[year]): print(" " + author) print('') if years_for_contributors: if verbose: print("Years for each contributor...") print('') def last_year(name): """Return year of the latest contribution for a given name""" return sorted(list(names[name]))[-1] def first_year(name): """Return year of the first contribution""" return sorted(list(names[name]))[0] def year_key(name): """ Year key function. First sort by latest contribution year (desc). If it matches, compare first contribution year (asc). This ensures that the most recent and long-term contributors are at the top. """ return (last_year(name), -first_year(name)) print("Copyright (c)") for author in sorted(list(names), key=year_key, reverse=True): years = list(names[author]) yearstr = compress(years) if 0: #DEBUG print(years, yearstr, author) else: print(" %s %s" % (yearstr, author)) print('') code-4d7977d51a4e/scripts/copy-user.py0000755000000000000000000000475114341561674015701 0ustar 00000000000000#!/usr/bin/env python # Copyright (C) 2003 by Intevation GmbH # Author: # Thomas Arendsen Hein # # This program is free software dual licensed under the GPL (>=v2) # and the Roundup Licensing (see COPYING.txt in the roundup distribution). """ copy-user [...] Copy one or more Roundup users from one tracker instance to another. Example: copy-user /roundup/tracker1 /roundup/tracker2 `seq 3 10` 14 16 (copies users 3, 4, 5, 6, 7, 8, 9, 10, 14 and 16) """ from __future__ import print_function import sys import roundup.instance def copy_user(home1, home2, *userids): """Copy users which are listed by userids from home1 to home2""" copyattribs = ['username', 'password', 'address', 'realname', 'phone', 'organisation', 'alternate_addresses', 'roles', 'timezone'] try: instance1 = roundup.instance.open(home1) print("Opened source instance: %s" % home1) except: print("Can't open source instance: %s" % home1) sys.exit(1) try: instance2 = roundup.instance.open(home2) print("Opened target instance: %s" % home2) except: print("Can't open target instance: %s" % home2) sys.exit(1) db1 = instance1.open('admin') db2 = instance2.open('admin') db1.tx_Source = "cli" db2.tx_Source = "cli" userlist = db1.user.list() for userid in userids: try: userid = str(int(userid)) except ValueError as why: print("Not a numeric user id: %s Skipping ..." % (userid,)) continue if userid not in userlist: print("User %s not in source instance. Skipping ..." % userid) continue user = {} for attrib in copyattribs: value = db1.user.get(userid, attrib) if value: user[attrib] = value try: db2.user.lookup(user['username']) print("User %s: Username '%s' exists in target instance. Skipping ..." % (userid, user['username'])) continue except KeyError as why: pass print("Copying user %s (%s) ..." % (userid, user['username'])) db2.user.create(**user) db2.commit() db2.close() print("Closed target instance.") db1.close() print("Closed source instance.") if __name__ == "__main__": if len(sys.argv) < 4: print(__doc__) sys.exit(1) else: copy_user(*sys.argv[1:]) code-4d7977d51a4e/scripts/dump_dbm_sessions_db.py0000755000000000000000000000511014341561674020123 0ustar 00000000000000#! /usr/bin/env python3 """Usage: dump_dbm_sessions_db.py [filename] Simple script to dump the otks and sessions dbm databases. Dumps sessions db in current directory if no argument is given. Dump format: key: data where is the human readable __timestamp decoded from the data object. Data object is dumped in json format. With pretty print key: { key: val, ... } if data is not a python object, print will be key: data or key: data if pretty printed. """ import argparse, dbm, json, marshal, os, sys from datetime import datetime def indent(text, amount, ch=" "): """ Found at: https://stackoverflow.com/a/8348914 """ padding = amount * ch return ''.join(padding+line for line in text.splitlines(True)) def print_marshal(k): d = marshal.loads(db[k]) try: t = datetime.fromtimestamp(d['__timestamp']) except (KeyError, TypeError): # TypeError raised if marshalled data is not a dict (list, tuple etc) t = "no_timestamp" if args.pretty: print("%s:\n %s\n%s"%(k, t, indent(json.dumps( d, sort_keys=True, indent=4), 4))) else: print("%s: %s %s"%(k, t, d)) def print_raw(k): if args.pretty: print("%s:\n %s"%(k, db[k])) else: print("%s: %s"%(k, db[k])) parser = argparse.ArgumentParser( description='Dump DBM files used by Roundup in storage order.') parser.add_argument('-k', '--key', action="append", help='dump the entry for a key, can be used multiple times.') parser.add_argument('-K', '--keysonly', action='store_true', help='print the database keys, sorted in byte order.') parser.add_argument('-p', '--pretty', action='store_true', help='pretty print the output rather than printing on one line.') parser.add_argument('file', nargs='?', help='file to be dumped ("sessions" if not provided)') args = parser.parse_args() if args.file: file = args.file else: file="sessions" try: db = dbm.open(file) except Exception as e: print("Unable to open database for %s: %s"%(file, e)) try: os.stat(file) print(" perhaps file is invalid or was created with a different version of Python?") except OSError: # the file does exist on disk. pass exit(1) if args.keysonly: for k in sorted(db.keys()): print("%s"%k) exit(0) if args.key: for k in args.key: try: print_marshal(k) except (ValueError): print_raw(k) exit(0) k = db.firstkey() while k is not None: try: print_marshal(k) except (ValueError): # ValueError marshal.loads failed print_raw(k) k = db.nextkey(k) code-4d7977d51a4e/scripts/imapServer.py0000644000000000000000000003444514341561674016070 0ustar 00000000000000#!/usr/bin/env python3 """\ This script is a wrapper around the mailgw.py script that exists in roundup. It runs as service instead of running as a one-time shot. It also connects to a secure IMAP server. The main reasons for this script are: 1) The roundup-mailgw script isn't designed to run as a server. It expects that you either run it by hand, and enter the password each time, or you supply the password on the command line. I prefer to run a server that I initialize with the password, and then it just runs. I don't want to have to pass it on the command line, so running through crontab isn't a possibility. (This wouldn't be a problem on a local machine running through a mailspool.) 2) mailgw.py somehow screws up SSL support so IMAP4_SSL doesn't work. So hopefully running that work outside of the mailgw will allow it to work. 3) I wanted to be able to check multiple projects at the same time. roundup-mailgw is only for 1 mailbox and 1 project. *TODO*: For the first round, the program spawns a new roundup-mailgw for each imap message that it finds and pipes the result in. In the future it might be more practical to actually include the roundup files and run the appropriate commands using python. *TODO*: Look into supporting a logfile instead of using 2>/logfile *TODO*: Add an option for changing the uid/gid of the running process. """ from __future__ import print_function import getpass import logging import imaplib import argparse import os import re import time import sys from roundup.anypy.my_input import my_input logging.basicConfig() log = logging.getLogger('roundup.IMAPServer') version = '0.1.3' class RoundupMailbox: """This contains all the info about each mailbox. Username, Password, server, security, roundup database """ def __init__(self, dbhome='', username=None, password=None, mailbox=None , server=None, protocol='imaps'): self.username = username self.password = password self.mailbox = mailbox self.server = server self.protocol = protocol self.dbhome = dbhome try: if not self.dbhome: self.dbhome = my_input('Tracker home: ') if not os.path.exists(self.dbhome): raise ValueError('Invalid home address: ' \ 'directory "%s" does not exist.' % self.dbhome) if not self.server: self.server = my_input('Server: ') if not self.server: raise ValueError('No Servername supplied') protocol = my_input('protocol [imaps]? ') self.protocol = protocol if not self.username: self.username = my_input('Username: ') if not self.username: raise ValueError('Invalid Username') if not self.password: print('For server %s, user %s' % (self.server, self.username)) self.password = getpass.getpass() # password can be empty because it could be superceeded # by a later entry #if self.mailbox is None: # self.mailbox = my_input('Mailbox [INBOX]: ') # # We allow an empty mailbox because that will # # select the INBOX, whatever it is called except (KeyboardInterrupt, EOFError): raise ValueError('Canceled by User') def __str__(self): return 'Mailbox{ server:%(server)s, protocol:%(protocol)s, ' \ 'username:%(username)s, mailbox:%(mailbox)s, ' \ 'dbhome:%(dbhome)s }' % self.__dict__ # [als] class name is misleading. this is imap client, not imap server class IMAPServer: """IMAP mail gatherer. This class runs as a server process. It is configured with a list of mailboxes to connect to, along with the roundup database directories that correspond with each email address. It then connects to each mailbox at a specified interval, and if there are new messages it reads them, and sends the result to the roundup.mailgw. *TODO*: Try to be smart about how you access the mailboxes so that you can connect once, and access multiple mailboxes and possibly multiple usernames. *NOTE*: This assumes that if you are using the same user on the same server, you are using the same password. (the last one supplied is used.) Empty passwords are ignored. Only the last protocol supplied is used. """ def __init__(self, pidfile=None, delay=5, daemon=False): #This is sorted by servername, then username, then mailboxes self.mailboxes = {} self.delay = float(delay) self.pidfile = pidfile self.daemon = daemon def setDelay(self, delay): self.delay = delay def addMailbox(self, mailbox): """ The linkage is as follows: servers -- users - mailbox:dbhome So there can be multiple servers, each with multiple users. Each username can be associated with multiple mailboxes. each mailbox is associated with 1 database home """ log.info('Adding mailbox %s', mailbox) if mailbox.server not in self.mailboxes: self.mailboxes[mailbox.server] = {'protocol':'imaps', 'users':{}} server = self.mailboxes[mailbox.server] if mailbox.protocol: server['protocol'] = mailbox.protocol if mailbox.username not in server['users']: server['users'][mailbox.username] = {'password':'', 'mailboxes':{}} user = server['users'][mailbox.username] if mailbox.password: user['password'] = mailbox.password if mailbox.mailbox in user['mailboxes']: raise ValueError('Mailbox is already defined') user['mailboxes'][mailbox.mailbox] = mailbox.dbhome def _process(self, message, dbhome): """Actually process one of the email messages""" child = os.popen('roundup-mailgw %s' % dbhome, 'wb') child.write(message) child.close() #print message def _getMessages(self, serv, count, dbhome): """This assumes that you currently have a mailbox open, and want to process all messages that are inside. """ for n in range(1, count+1): (t, data) = serv.fetch(n, '(RFC822)') if t == 'OK': self._process(data[0][1], dbhome) serv.store(n, '+FLAGS', r'(\Deleted)') def checkBoxes(self): """This actually goes out and does all the checking. Returns False if there were any errors, otherwise returns true. """ noErrors = True for server in self.mailboxes: log.info('Connecting to server: %s', server) s_vals = self.mailboxes[server] try: for user in s_vals['users']: u_vals = s_vals['users'][user] # TODO: As near as I can tell, you can only # login with 1 username for each connection to a server. protocol = s_vals['protocol'].lower() if protocol == 'imaps': serv = imaplib.IMAP4_SSL(server) elif protocol == 'imap': serv = imaplib.IMAP4(server) else: raise ValueError('Unknown protocol %s' % protocol) password = u_vals['password'] try: log.info('Connecting as user: %s', user) serv.login(user, password) for mbox in u_vals['mailboxes']: dbhome = u_vals['mailboxes'][mbox] log.info('Using mailbox: %s, home: %s', mbox, dbhome) #access a specific mailbox if mbox: (t, data) = serv.select(mbox) else: # Select the default mailbox (INBOX) (t, data) = serv.select() try: nMessages = int(data[0]) except ValueError: nMessages = 0 log.info('Found %s messages', nMessages) if nMessages: self._getMessages(serv, nMessages, dbhome) serv.expunge() # We are done with this mailbox serv.close() except: log.exception('Exception with server %s user %s', server, user) noErrors = False serv.logout() serv.shutdown() del serv except: log.exception('Exception while connecting to %s', server) noErrors = False return noErrors def makeDaemon(self): """Turn this process into a daemon. - make our parent PID 1 Write our new PID to the pidfile. From A.M. Kuuchling (possibly originally Greg Ward) with modification from Oren Tirosh, and finally a small mod from me. Originally taken from roundup.scripts.roundup_server.py """ log.info('Running as Daemon') # Fork once if os.fork() != 0: os._exit(0) # Create new session os.setsid() # Second fork to force PPID=1 pid = os.fork() if pid: if self.pidfile: pidfile = open(self.pidfile, 'w') pidfile.write(str(pid)) pidfile.close() os._exit(0) def run(self): """Run email gathering daemon. This spawns itself as a daemon, and then runs continually, just sleeping inbetween checks. It is recommended that you run checkBoxes once first before you select run. That way you can know if there were any failures. """ if self.daemon: self.makeDaemon() while True: time.sleep(self.delay * 60.0) log.info('Time: %s', time.strftime('%Y-%m-%d %H:%M:%S')) self.checkBoxes() def getItems(s): """Parse a string looking for userame@server""" myRE = re.compile( r'((?P[^:]+)://)?'#You can supply a protocol if you like r'(' #The username part is optional r'(?P[^:]+)' #You can supply the password as r'(:(?P.+))?' #username:password@server r'@)?' r'(?P[^/]+)' r'(/(?P.+))?$' ) m = myRE.match(s) if m: return m.groupdict() else: return None def main(): """This is what is called if run at the prompt""" parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" The server takes pairs of home/server. So each entry has a home, and then the server configuration. Home is just a path to the roundup issue tracker. The server is something of the form: imaps://user:password@server/mailbox If you don't supply the protocol, imaps is assumed. Without user or password, you will be prompted for them. The server must be supplied. Without mailbox the INBOX is used. Examples: %(prog)s /home/roundup/trackers/test imaps://test@imap.example.com/test %(prog)s /home/roundup/trackers/test imap.example.com \\ /home/roundup/trackers/test2 imap.example.com/test2 """ % dict(prog = sys.argv [0]) ) parser.add_argument('args', nargs='*') parser.add_argument('-d', '--delay', dest='delay', type=float, metavar='', default=5, help="Set the delay between checks in minutes. (default 5)" ) parser.add_argument('-p', '--pid-file', dest='pidfile', metavar='', default=None, help="The pid of the server process will be written to " ) parser.add_argument('-n', '--no-daemon', dest='daemon', action='store_false', default=True, help="Do not fork into the background after running the first check." ) parser.add_argument('--version', action="store_true", help="Print version and exit") parser.add_argument('-v', '--verbose', dest='verbose', action='store_const', const=logging.INFO, help="Be more verbose in letting you know what is going on." " Enables informational messages." ) parser.add_argument('-V', '--very-verbose', dest='verbose', action='store_const', const=logging.DEBUG, help="Be very verbose in letting you know what is going on." " Enables debugging messages." ) parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=logging.ERROR, help="Be less verbose. Ignores warnings, only prints errors." ) parser.add_argument('-Q', '--very-quiet', dest='verbose', action='store_const', const=logging.CRITICAL, help="Be much less verbose. Ignores warnings and errors." " Only print CRITICAL messages." ) args = parser.parse_args() if args.version: print('%s %s' % (sys.argv [0], version)) sys.exit(0) if not len(args.args) or len(args.args) % 2 == 1: parser.error('Invalid number of arguments. ' 'Each site needs a home and a server.') if args.verbose == None: args.verbose = logging.WARNING log.setLevel(args.verbose) myServer = IMAPServer(delay=args.delay, pidfile=args.pidfile, daemon=args.daemon) for i in range(0,len(args.args),2): home = args.args[i] server = args.args[i+1] if not os.path.exists(home): parser.error('Home: "%s" does not exist' % home) info = getItems(server) if not info: parser.error('Invalid server string: "%s"' % server) myServer.addMailbox( RoundupMailbox(dbhome=home, mailbox=info['mailbox'] , username=info['username'], password=info['password'] , server=info['server'], protocol=info['protocol'] ) ) if myServer.checkBoxes(): myServer.run() if __name__ == '__main__': main() # vim: et ft=python si sts=4 sw=4 code-4d7977d51a4e/scripts/import_sf.py0000644000000000000000000003424614341561674015754 0ustar 00000000000000""" Import tracker data from Sourceforge.NET This script needs four steps to work: 1. Export the project XML data using the admin web interface at sf.net 2. Run the file fetching (these are not included in the XML): import_sf.py files this will place all the downloaded files in the files dir by file id. 3. Convert the sf.net XML to Roundup "export" format: import_sf.py import this will generate a directory "/tmp/imported" which contains the data to be imported into a Roundup tracker. 4. Import the data: roundup-admin -i import /tmp/imported And you're done! """ import sys, os, csv, time, mimetypes try: import cElementTree as ElementTree except ImportError: from elementtree import ElementTree from roundup import instance, hyperdb, date, support, password from roundup.anypy import http_, urllib_ from roundup.anypy.strings import s2b, us2s today = date.Date('.') DL_URL = 'http://sourceforge.net/tracker/download.php?group_id=%(group_id)s&atid=%(atid)s&aid=%(aid)s' def get_url(aid): """ so basically we have to jump through hoops, given an artifact id, to figure what the URL should be to access that artifact, and hence any attached files.""" # first we hit this URL... conn = http_.client.HTTPConnection("sourceforge.net") conn.request("GET", "/support/tracker.php?aid=%s"%aid) response = conn.getresponse() # which should respond with a redirect to the correct url which has the # magic "group_id" and "atid" values in it that we need assert response.status == 302, 'response code was %s'%response.status location = response.getheader('location') query = urllib_.urlparse(response.getheader('location'))[-2] info = dict([param.split('=') for param in query.split('&')]) return DL_URL%info def fetch_files(xml_file, file_dir): """ Fetch files referenced in the xml_file into the dir file_dir. """ root = ElementTree.parse(xml_file).getroot() to_fetch = set() deleted = set() for artifact in root.find('artifacts'): for field in artifact.findall('field'): if field.get('name') == 'artifact_id': aid = field.text for field in artifact.findall('field'): if field.get('name') != 'artifact_history': continue for event in field.findall('history'): d = {} for field in event.findall('field'): d[field.get('name')] = field.text if d['field_name'] == 'File Added': fid = d['old_value'].split(':')[0] to_fetch.add((aid, fid)) if d['field_name'] == 'File Deleted': fid = d['old_value'].split(':')[0] deleted.add((aid, fid)) to_fetch = to_fetch - deleted got = set(os.listdir(file_dir)) to_fetch = to_fetch - got # load cached urls (sigh) urls = {} if os.path.exists(os.path.join(file_dir, 'urls.txt')): for line in open(os.path.join(file_dir, 'urls.txt')): aid, url = line.strip().split() urls[aid] = url for aid, fid in support.Progress('Fetching files', list(to_fetch)): if fid in got: continue if aid not in urls: urls[aid] = get_url(aid) f = open(os.path.join(file_dir, 'urls.txt'), 'a') f.write('%s %s\n'%(aid, urls[aid])) f.close() url = urls[aid] + '&file_id=' + fid f = urllib_.urlopen(url) data = f.read() n = open(os.path.join(file_dir, fid), 'w') n.write(data) f.close() n.close() def import_xml(tracker_home, xml_file, file_dir): """ Generate Roundup tracker import files based on the tracker schema, sf.net xml export and downloaded files from sf.net. """ tracker = instance.open(tracker_home) db = tracker.open('admin') resolved = db.status.lookup('resolved') unread = db.status.lookup('unread') chatting = db.status.lookup('unread') critical = db.priority.lookup('critical') urgent = db.priority.lookup('urgent') bug = db.priority.lookup('bug') feature = db.priority.lookup('feature') wish = db.priority.lookup('wish') adminuid = db.user.lookup('admin') anonuid = db.user.lookup('anonymous') root = ElementTree.parse(xml_file).getroot() def to_date(ts): return date.Date(time.gmtime(float(ts))) # parse out the XML artifacts = [] categories = set() users = set() add_files = set() remove_files = set() for artifact in root.find('artifacts'): d = {} op = {} artifacts.append(d) for field in artifact.findall('field'): name = field.get('name') if name == 'artifact_messages': for message in field.findall('message'): l = d.setdefault('messages', []) m = {} l.append(m) for field in message.findall('field'): name = field.get('name') if name == 'adddate': m[name] = to_date(field.text) else: m[name] = field.text if name == 'user_name': users.add(field.text) elif name == 'artifact_history': for event in field.findall('history'): l = d.setdefault('history', []) e = {} l.append(e) for field in event.findall('field'): name = field.get('name') if name == 'entrydate': e[name] = to_date(field.text) else: e[name] = field.text if name == 'mod_by': users.add(field.text) if e['field_name'] == 'File Added': add_files.add(e['old_value'].split(':')[0]) elif e['field_name'] == 'File Deleted': remove_files.add(e['old_value'].split(':')[0]) elif name == 'details': op['body'] = field.text elif name == 'submitted_by': op['user_name'] = field.text d[name] = field.text users.add(field.text) elif name == 'open_date': thedate = to_date(field.text) op['adddate'] = thedate d[name] = thedate else: d[name] = field.text categories.add(d['category']) if 'body' in op: l = d.setdefault('messages', []) l.insert(0, op) add_files -= remove_files # create users userd = {'nobody': '2'} users.remove('nobody') data = [ {'id': '1', 'username': 'admin', 'password': password.Password('admin'), 'roles': 'Admin', 'address': 'richard@python.org'}, {'id': '2', 'username': 'anonymous', 'roles': 'Anonymous'}, ] for n, user in enumerate(list(users)): userd[user] = n+3 data.append({'id': str(n+3), 'username': user, 'roles': 'User', 'address': '%s@users.sourceforge.net'%user}) write_csv(db.user, data) users=userd # create categories categoryd = {'None': None} categories.remove('None') data = [] for n, category in enumerate(list(categories)): categoryd[category] = n data.append({'id': str(n), 'name': category}) write_csv(db.keyword, data) categories = categoryd # create issues issue_data = [] file_data = [] message_data = [] issue_journal = [] message_id = 0 for artifact in artifacts: d = {} d['id'] = artifact['artifact_id'] d['title'] = artifact['summary'] d['assignedto'] = users[artifact['assigned_to']] if d['assignedto'] == '2': d['assignedto'] = None d['creation'] = artifact['open_date'] activity = artifact['open_date'] d['creator'] = users[artifact['submitted_by']] actor = d['creator'] if categories[artifact['category']]: d['keyword'] = [categories[artifact['category']]] issue_journal.append(( d['id'], d['creation'].get_tuple(), d['creator'], "'create'", {} )) p = int(artifact['priority']) if artifact['artifact_type'] == 'Feature Requests': if p > 3: d['priority'] = feature else: d['priority'] = wish else: if p > 7: d['priority'] = critical elif p > 5: d['priority'] = urgent elif p > 3: d['priority'] = bug else: d['priority'] = feature s = artifact['status'] if s == 'Closed': d['status'] = resolved elif s == 'Deleted': d['status'] = resolved d['is retired'] = True else: d['status'] = unread nosy = set() for message in artifact.get('messages', []): authid = users[message['user_name']] if not message['body']: continue body = convert_message(message['body'], message_id) if not body: continue m = {'content': body, 'author': authid, 'date': message['adddate'], 'creation': message['adddate'], } message_data.append(m) if authid not in (None, '2'): nosy.add(authid) activity = message['adddate'] actor = authid if d['status'] == unread: d['status'] = chatting # add import message m = {'content': 'IMPORT FROM SOURCEFORGE', 'author': '1', 'date': today, 'creation': today} message_data.append(m) # sort messages and assign ids d['messages'] = [] message_data.sort(key=lambda a:a['date']) for message in message_data: message_id += 1 message['id'] = str(message_id) d['messages'].append(message_id) d['nosy'] = list(nosy) files = [] for event in artifact.get('history', []): if event['field_name'] == 'File Added': fid, name = event['old_value'].split(':', 1) if fid in add_files: files.append(fid) name = name.strip() try: f = open(os.path.join(file_dir, fid), 'rb') content = f.read() f.close() except: content = 'content missing' file_data.append({ 'id': fid, 'creation': event['entrydate'], 'creator': users[event['mod_by']], 'name': name, 'type': mimetypes.guess_type(name)[0], 'content': content, }) continue elif event['field_name'] == 'close_date': action = "'set'" info = { 'status': unread } elif event['field_name'] == 'summary': action = "'set'" info = { 'title': event['old_value'] } else: # not an interesting / translatable event continue row = [ d['id'], event['entrydate'].get_tuple(), users[event['mod_by']], action, info ] if event['entrydate'] > activity: activity = event['entrydate'] issue_journal.append(row) d['files'] = files d['activity'] = activity d['actor'] = actor issue_data.append(d) write_csv(db.issue, issue_data) write_csv(db.msg, message_data) write_csv(db.file, file_data) f = open('/tmp/imported/issue-journals.csv', 'w') writer = csv.writer(f, colon_separated) writer.writerows(issue_journal) f.close() def convert_message(content, id): """ Strip off the useless sf message header crap """ if content[:14] == 'Logged In: YES': return '\n'.join(content.splitlines()[3:]).strip() return content class colon_separated(csv.excel): delimiter = ':' def write_csv(klass, data): props = klass.getprops() if not os.path.exists('/tmp/imported'): os.mkdir('/tmp/imported') f = open('/tmp/imported/%s.csv'%klass.classname, 'w') writer = csv.writer(f, colon_separated) propnames = klass.export_propnames() propnames.append('is retired') writer.writerow(propnames) for entry in data: row = [] for name in propnames: if name == 'is retired': continue prop = props[name] if name in entry: if isinstance(prop, hyperdb.Date) or \ isinstance(prop, hyperdb.Interval): row.append(repr(entry[name].get_tuple())) elif isinstance(prop, hyperdb.Password): row.append(repr(str(entry[name]))) else: row.append(repr(entry[name])) elif isinstance(prop, hyperdb.Multilink): row.append('[]') elif name in ('creator', 'actor'): row.append("'1'") elif name in ('created', 'activity'): row.append(repr(today.get_tuple())) else: row.append('None') row.append(entry.get('is retired', False)) writer.writerow(row) if isinstance(klass, hyperdb.FileClass) and entry.get('content'): fname = klass.exportFilename('/tmp/imported/', entry['id']) support.ensureParentsExist(fname) c = open(fname, 'wb') if isinstance(entry['content'], bytes): c.write(entry['content']) else: c.write(s2b(us2s(entry['content']))) c.close() f.close() f = open('/tmp/imported/%s-journals.csv'%klass.classname, 'w') f.close() if __name__ == '__main__': if sys.argv[1] == 'import': import_xml(*sys.argv[2:]) elif sys.argv[1] == 'files': fetch_files(*sys.argv[2:]) code-4d7977d51a4e/scripts/oauth-get-token.py0000755000000000000000000002536114341561674016766 0ustar 00000000000000#!/usr/bin/python3 import requests import time import sys import webbrowser import ssl from urllib.parse import urlparse, urlencode, parse_qs from argparse import ArgumentParser, RawDescriptionHelpFormatter from http.server import HTTPServer, BaseHTTPRequestHandler class Request_Token: def __init__ (self, args): self.args = args self.session = requests.session () self.url = '/'.join ((args.url.rstrip ('/'), args.tenant)) self.url = '/'.join ((self.url, 'oauth2/v2.0')) self.state = None # end def __init__ def check_err (self, r): if not 200 <= r.status_code <= 299: raise RuntimeError \ ( 'Invalid result: %s: %s\n %s' % (r.status_code, r.reason, r.text) ) # end def check_err def get_url (self, path, params): url = ('/'.join ((self.url, path))) url = url + '?' + urlencode (params) return url # end def get_url def post_or_put (self, method, path, data = None, json = None): d = {} if data: d.update (data = data) if json: d.update (json = json) url = ('/'.join ((self.url, path))) r = method (url, **d) self.check_err (r) return r.json () # end def post_or_put def post (self, path, data = None, json = None): return self.post_or_put (self.session.post, path, data, json) # end def post def authcode_callback (self, handler): msg = [''] self.request_received = False r = urlparse (handler.path) if r.query: q = parse_qs (r.query) if 'state' in q: state = q ['state'][0] if state != self.state: msg.append \ ( 'State did not match: expect "%s" got "%s"' % (self.state, state) ) elif 'code' not in q: msg.append ('Got no code') else: with open ('oauth/authcode', 'w') as f: f.write (q ['code'][0]) msg.append ('Wrote code to oauth/authcode') self.request_received = True else: msg.append ('No state and no code') return 200, '\n'.join (msg).encode ('utf-8') # end def authcode_callback def request_authcode (self): with open ('oauth/client_id', 'r') as f: client_id = f.read () self.state = 'authcode' + str (time.time ()) params = dict \ ( client_id = client_id , response_type = 'code' , response_mode = 'query' , state = self.state , redirect_uri = self.args.redirect_uri , scope = ' '.join (( 'https://outlook.office.com/IMAP.AccessAsUser.All' , 'https://outlook.office.com/User.Read' , 'offline_access' )) ) url = self.get_url ('authorize', params) print (url) if self.args.webbrowser: browser = webbrowser.get (self.args.browser) browser.open_new_tab (url) if self.args.run_https_server: self.https_server () if self.args.request_tokens: self.request_token () # end def request_authcode def request_token (self): with open ('oauth/client_id', 'r') as f: client_id = f.read () with open ('oauth/client_secret', 'r') as f: client_secret = f.read ().strip () with open ('oauth/authcode', 'r') as f: authcode = f.read ().strip () params = dict \ ( client_id = client_id , code = authcode , client_secret = client_secret , redirect_uri = self.args.redirect_uri , grant_type = 'authorization_code' # Only a single scope parameter is allowed here , scope = ' '.join (( 'https://outlook.office.com/User.Read' , )) ) result = self.post ('token', data = params) with open ('oauth/refresh_token', 'w') as f: f.write (result ['refresh_token']) with open ('oauth/access_token', 'w') as f: f.write (result ['access_token']) # end def request_token def https_server (self): self.request_received = False class RQ_Handler (BaseHTTPRequestHandler): token_handler = self def do_GET (self): self.close_connection = True code, msg = self.token_handler.authcode_callback (self) self.send_response (code) self.send_header ('Content-Type', 'text/plain') self.end_headers () self.wfile.write (msg) self.wfile.flush () port = self.args.https_server_port httpd = HTTPServer (('localhost', port), RQ_Handler) httpd.socket = ssl.wrap_socket \ ( httpd.socket , keyfile = "/etc/ssl/private/ssl-cert-snakeoil.key" , certfile = "/etc/ssl/certs/ssl-cert-snakeoil.pem" , server_side = True ) while not self.request_received: httpd.handle_request () # end def https_server # end class Request_Token epilog = """\ Retrieving the necessary refresh_token and access_token credentials using this script. This asumes you have an email account (plus the password) to be used for mail retrieval. And you have registered an application in the cloud for this process. The registering of an application will give you an application id (also called client id) and a tenant in UUID format. First define the necessary TENANT variable: TENANT=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX You need to create a directory named 'oauth' (if not yet existing) and put the client id (also called application id) into the file 'oauth/client_id' and the corresponding secret into the file 'oauth/client_secret'. By default calling the script with no arguments, the whole process is automatic, but you may want to specify the tenant explicitly using: ./oauth-get-token.py -t $TENANT Specifying the tenant explicitly will select the customized company login form directly. The automatic process works as follows: - First the authorization URL is constructed and pushed to a local browser. By default the default browser on that machine is used, you can specify a different browser with the -b/--browser option. This will show a login form where you should be able to select the user to log in with. Log in with the username (the email address) and password for that user. - A web-server is started on the given port. When you fill out the authentication form pushed to the browser, the last step is a redirect to an URL that calls back to this webserver. The necessary authentication code is transmitted in a query parameter. The code is stored into the file 'oauth/authcode'. Using the authcode, the refresh_token and access_token are requested and stored in the oauth directory. These steps can be broken down into individual steps by options disabling one of the steps: - The push to the webserver can be disabled with the option -w/--dont-push-to-webbrowser -- in that case the URL is printed on standard output and must be pasted into the URL input field of a browser. It is typically a good idea to use a browser that is currently not logged into the company network. - The start of the webserver can be disabled with the option -s/--dont-run-https-server -- when called with that option no webserver is started. You get a redirect to a non-existing page. The error-message is something like: This site can’t be reached Copy the URL from the browser into the file 'oauth/authcode'. The URL has paramters. We're interested in the 'code' parameter, a very long string. Edit the file so that only that string (without the 'code=' part) is in the file. - Requesting the tokens can be disabled with the option -n/--dont-request-tokens -- if this option is given, after receiving the redirect from the webserver the authentication code is written to the file 'oauth/authcode' but no token request is started. If you have either disabled the webserver or the token request, the token can be requested (using the file 'oauth/authcode' constructed by hand as described above or written by the webserver) with the -T/--request-token option: ./oauth-get-token.py [-t $TENANT] -T If successful this will create the 'oauth/access_token' and 'oauth/refresh_token' files. Note that the authentication code has a limited lifetime. """ def main (): cmd = ArgumentParser \ (epilog=epilog, formatter_class=RawDescriptionHelpFormatter) cmd.add_argument \ ( '-T', '--request-token' , help = "Run only the token-request step" , action = 'store_true' ) cmd.add_argument \ ( '-b', '--browser' , help = "Use non-default browser" ) cmd.add_argument \ ( '-n', '--dont-request-tokens' , dest = 'request_tokens' , help = "Do not request tokens, just write authcode" , action = 'store_false' , default = True ) cmd.add_argument \ ( '-p', '--https-server-port' , type = int , help = "Port for https server to listen, default=%(default)s" " see also -r option, ports must (usually) match." , default = 8181 ) cmd.add_argument \ ( '-r', '--redirect-uri' , help = "Redirect URI, default=%(default)s" , default = 'https://localhost:8181' ) cmd.add_argument \ ( '-s', '--dont-run-https-server' , dest = 'run_https_server' , help = "Run https server to wait for connection of browser " "to transmit auth code via GET request" , action = 'store_false' , default = True ) cmd.add_argument \ ( '-t', '--tenant' , help = "Tenant part of url, default=%(default)s" , default = 'organizations' ) cmd.add_argument \ ( '-u', '--url' , help = "Base url for requests, default=%(default)s" , default = 'https://login.microsoftonline.com' ) cmd.add_argument \ ( '-w', '--dont-push-to-webbrowser' , dest = 'webbrowser' , help = "Do not push authcode url into the browser" , action = 'store_false' , default = True ) args = cmd.parse_args () rt = Request_Token (args) if args.request_token: rt.request_token () else: rt.request_authcode () # end def main if __name__ == '__main__': main () code-4d7977d51a4e/scripts/roundup-reminder0000755000000000000000000001452614341561674016624 0ustar 00000000000000#! /usr/bin/env python # Copyright (c) 2002 ekit.com Inc (http://www.ekit-inc.com/) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ''' Simple script that emails all users of a tracker with the issues that are currently assigned to them. TODO: introduce some structure ;) TODO: possibly make this more general and configurable... ''' from __future__ import print_function import sys from email.mime.multipart import MIMEMultipart from email.utils import make_msgid from roundup import instance, date from roundup.mailer import Mailer # open the instance if len(sys.argv) != 2: print('You need to specify an instance home dir') sys.exit() instance_home = sys.argv[1] instance = instance.open(instance_home) db = instance.open('admin') resolved_id = db.status.lookup('resolved') class Reverse: """Class reversing sort order.""" def __init__(self, val): self.val = val def __lt__(self, other): return other.val < self.val def __le__(self, other): return other.val <= self.val def __eq__(self, other): return other.val == self.val def __ne__(self, other): return other.val != self.val def __gt__(self, other): return other.val > self.val def __ge__(self, other): return other.val >= self.val def listKey(x): "key for tuples such that order is positive on [0] and negative on [1]" return (x[0], Reverse(x[1])) return 0 # loop through all the users for user_id in db.user.list(): # make sure we care about this user name = db.user.get(user_id, 'realname') if name is None: name = db.user.get(user_id, 'username') address = db.user.get(user_id, 'address') if address is None: continue # extract this user's issues l = [] for issue_id in db.issue.find(assignedto=user_id): if db.issue.get(issue_id, 'status') == resolved_id: continue order = db.priority.get(db.issue.get(issue_id, 'priority'), 'order') l.append((order, db.issue.get(issue_id, 'activity'), db.issue.get(issue_id, 'creation'), issue_id)) # sort the issues by timeliness and creation date l.sort(key=listKey) if not l: continue # generate the email message mailer = Mailer(db.config) message = MIMEMultipart('alternative') mailer.set_message_attributes( message, [address], 'Your active %s issues'%db.config.TRACKER_NAME) message['Reply-To'] = '%s <%s>'%(db.config.TRACKER_NAME, db.config.ADMIN_EMAIL) message['Message-Id'] = make_msgid() # do the plain text bit text_lines = [] text_lines.append('Created ID Activity Title') text_lines.append('='*75) # '2 months 213 immediate cc_daemon barfage old_priority = None for priority_order, activity_date, creation_date, issue_id in l: priority = db.issue.get(issue_id, 'priority') if (priority != old_priority): old_priority = priority text_lines.append(' ' + db.priority.get(priority,'name')) # pretty creation creation = (creation_date - date.Date('.')).pretty() activity = (activity_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') if len(title) > 42: title = title[:38] + ' ...' text_lines.append('%-11s %-4s %-9s %-42s'%(creation, issue_id, activity, title)) # some help to finish off text_lines.append(''' To view or respond to any of the issues listed above, visit the URL %s and click on "My Issues". Do NOT respond to this message. '''%db.config.TRACKER_WEB) text = '\n'.join(text_lines) + '\n' part = mailer.get_text_message() part.set_payload(text, part.get_charset()) message.attach(part) # now the HTML one html_lines = [] colours = { 'immediate': ' bgcolor="#ffcdcd"', 'day': ' bgcolor="#ffdecd"', 'week': ' bgcolor="#ffeecd"', 'month': ' bgcolor="#ffffcd"', 'whenever': ' bgcolor="#ffffff"', } html_lines.append(''' ''') old_priority = None for priority_order, activity_date, creation_date, issue_id in l: priority = db.issue.get(issue_id,'priority') if (priority != old_priority): old_priority = priority html_lines.append(''%db.priority.get(priority,'name')) creation = (creation_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') issue_id = '%s'%(db.config.TRACKER_WEB, issue_id, issue_id) activity = (activity_date - date.Date('.')).pretty() html_lines.append(''''''%(creation, issue_id, activity, title)) html_lines.append('
Created ID Activity Title
->->->%s
%s%s%s %s
') html_lines.append('''

To view or respond to any of the issues listed above, simply click on the issue ID. Do not respond to this message.

''') html = '\n'.join(html_lines) + '\n' part = mailer.get_text_message('utf-8', 'html') part.set_payload(html, part.get_charset()) message.attach(part) # all done, send! mailer.smtp_send([address], message.as_string()) # vim: set filetype=python ts=4 sw=4 et si code-4d7977d51a4e/scripts/roundup.rc-debian0000644000000000000000000000177514341561674016643 0ustar 00000000000000#!/bin/sh -e # # roundup Startup script for the roundup http server. DESC='Roundup HTTP-Server' BINFILE=roundup-server EXECUTABLE=/usr/local/bin/$BINFILE PIDFILE=/var/run/roundup/server.pid LOGFILE=/var/log/roundup/roundup.log TRACKERS=tttech=/tttech/org/software/roundup/tttech/ OPTIONS="-- -p 8080 -u roundup -d $PIDFILE -l $LOGFILE $TRACKERS" test -x $EXECUTABLE || exit 0 start_stop() { case "$1" in start) printf "Starting $DESC:" start-stop-daemon --start --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; stop) printf "Stopping $DESC:" start-stop-daemon --stop --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; restart | force-reload) start_stop stop sleep 1 start_stop start ;; *) printf "Usage: $0 {start|stop|restart|force-reload}\n" >&2 exit 1 ;; esac } start_stop "$@" exit 0 code-4d7977d51a4e/scripts/schema-dump.py0000755000000000000000000001174114341561674016153 0ustar 00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Use recently documented XML-RPC API to dump Roundup data schema in human readable form. Works with demo tracker using: http://admin:admin@localhost:8917/demo/xmlrpc Future development may cover: [ ] unreadable dump formats [ ] access to local database [ ] lossless dump/restore cycle [ ] data dump and filtering with preserved Works in Python 2 as well. """ from __future__ import print_function __license__ = "Public Domain" __version__ = "1.1" __authors__ = [ "anatoly techtonik " "John Rouillard " ] import os import pprint import sys import textwrap try: import urllib.parse as url_parser # python 3 except ImportError: import urlparse as url_parser # python 2 from argparse import ArgumentParser from roundup.anypy import xmlrpc_ sname = os.path.basename(sys.argv[0]) usage = """\ usage: %s [options] URL URL is XML-RPC endpoint for your tracker, such as: http://localhost:8917/demo/xmlrpc options: --pprint (default) --json --yaml --raw -h --help --version """ % sname def format_pprint(var): return pprint.pformat(var) def format_json(var): jout = pprint.pformat(var) jout = jout.replace('"', "\\'") # " to \' jout = jout.replace("'", '"') # ' to " jout = jout.replace('\\"', "'") # \" to ' return jout def format_yaml(var): out = pprint.pformat(var) out = out.replace('{', ' ') out = out.replace('}', '') out = textwrap.dedent(out) out = out.replace("'", '') out = out.replace(' [[', '\n [') out = out.replace(']]', ']') out = out.replace('],', '') out = out.replace(']', '') out2 = [] for line in out.splitlines(): if '[' in line: line = ' ' + line.lstrip(' [') line = line.replace('>', '') line = line.replace('roundup.hyperdb.', '') # expandtabs(16) with limit=1 n, v = line.split(', <') if len(n) > 14: indent = 0 else: indent = 14 - len(n) line = line.replace(', <', ': '+' '*indent) line.split(",") out2.append(line) out = '\n'.join(out2) return out class SpecialTransport(): """Mixin for http/https transports to implement new send_content with CSRF prevention headers to both of them. """ def send_content(self, connection, request_body): connection.putheader("Referer", "%s://%s%s%s/" % ( self.components.scheme, self.components.hostname, ':' + str(self.components.port) if self.components.port else '', self.components.path)) connection.putheader("Origin", "%s://%s%s" % ( self.components.scheme, self.components.hostname, ':' + str(self.components.port) if self.components.port else '')) connection.putheader("X-Requested-With", "XMLHttpRequest") connection.putheader("Content-Type", "text/xml") connection.putheader("Content-Length", str(len(request_body))) connection.endheaders() if request_body: connection.send(request_body) class SpecialHttpTransport(SpecialTransport, xmlrpc_.client.Transport, object): """SpecialTransport must be first to use its send_content. Explicit object inheritance required for python2 apparently.""" def __init__(self, url): self.components = url_parser.urlparse(url) # works both python2 (with object inheritance) and python3 super(SpecialHttpTransport, self).__init__(self) class SpecialHttpsTransport(SpecialTransport, xmlrpc_.client.SafeTransport, object): """SpecialTransport must be first to use its send_content. Explicit object inheritance required for python2 apparently.""" def __init__(self, url): self.components = url_parser.urlparse(url) # works both python2 (with object inheritance) and python3 super(SpecialHttpsTransport, self).__init__(self) if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("url", nargs=1) parser.add_argument("--raw", action='store_true') parser.add_argument("--yaml", action='store_true') parser.add_argument("--json", action='store_true') parser.add_argument("-v", "--version", action='store_true') args = parser.parse_args() if args.version: sys.exit(sname + " " + __version__) if args.url[0].lower().startswith('https:'): transport = SpecialHttpsTransport else: transport = SpecialHttpTransport roundup_server = xmlrpc_.client.ServerProxy( args.url[0], transport=transport(args.url[0]), verbose=False, allow_none=True) schema = roundup_server.schema() if args.raw: print(str(schema)) elif args.yaml: print(format_yaml(schema)) elif args.json: print(format_json(schema)) else: print(format_pprint(schema)) print("") code-4d7977d51a4e/scripts/schema_diagram.py0000644000000000000000000000307114341561674016666 0ustar 00000000000000#! /usr/bin/python # # Schema diagram generator contributed by Stefan Seefeld of the fresco # project http://www.fresco.org/. # # It generates a 'dot file' that is then fed into the 'dot' # tool (http://www.graphviz.org) to generate a graph: # # %> ./schema.py # %> dot -Tps schema.dot -o schema.ps # %> gv schema.ps # from __future__ import print_function import sys import roundup.instance # open the instance instance = roundup.instance.open(sys.argv[1]) db = instance.open() # diagram preamble print('digraph schema {') print('size="8,6"') print('node [shape="record" bgcolor="#ffe4c4" style=filled]') print('edge [taillabel="1" headlabel="1" dir=back arrowtail=ediamond]') # get all the classes types = list(db.classes.keys()) # one record node per class for i in range(len(types)): print('node%d [label=\"{%s|}"]'%(i, types[i])) # now draw in the relations for name in db.classes.keys(): type = db.classes[name] attributes = type.getprops() for a in attributes.keys(): attribute = attributes[a] if isinstance(attribute, roundup.hyperdb.Link): print('node%d -> node%d [label=%s]'%(types.index(name), types.index(attribute.classname), a)) elif isinstance(attribute, roundup.hyperdb.Multilink): print('node%d -> node%d [taillabel="*" label=%s]'%(types.index(name), types.index(attribute.classname), a)) # all done print('}') code-4d7977d51a4e/scripts/server-ctl0000755000000000000000000000352214341561674015405 0ustar 00000000000000#!/bin/sh # # Configuration # CONFFILE="/var/roundup/server-config.ini" # this will end up with extra space, but it should be ignored in the script PIDFILE=`grep '^pidfile' ${CONFFILE} | awk -F = '{print $2}' ` SERVER="/usr/local/bin/roundup-server -C ${CONFFILE}" ERROR=0 ARGV="$@" if [ "x$ARGV" = "x" ] ; then ARGS="help" fi if [ -z "${PIDFILE}" ] ; then echo "pidfile option must be set in configuration file" exit 1 fi for ARG in $@ $ARGS do # check for pidfile if [ -f $PIDFILE ] ; then PID=`cat $PIDFILE` if [ "x$PID" != "x" ] && kill -0 $PID 2>/dev/null ; then STATUS="roundup-server (pid $PID) running" RUNNING=1 else STATUS="roundup-server (pid $PID?) not running" RUNNING=0 fi else STATUS="roundup-server (no pid file) not running" RUNNING=0 fi case $ARG in start) if [ $RUNNING -eq 1 ] ; then echo "$0 $ARG: roundup-server (pid $PID) already running" continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; condstart) if [ $RUNNING -eq 1 ] ; then continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; stop) if [ $RUNNING -eq 0 ] ; then echo "$0 $ARG: $STATUS" continue fi if kill $PID ; then echo "$0 $ARG: roundup-server stopped" else echo "$0 $ARG: roundup-server could not be stopped" ERROR=2 fi ;; status) echo $STATUS ;; *) echo "usage: $0 (start|condstart|stop|status)" cat < propname # the class is the name of the class (e.g. issue_type, keyword -- # also used in e.g. in https://bugs.python.org/keyword) # the propname is the name used as attribute on the issue class # (e.g. issue.type, issue.keywords) classes = { # 'status': 'status', # skip this 'issue_type': 'type', 'stage': 'stage', 'component': 'components', 'version': 'versions', 'resolution': 'resolution', 'priority': 'priority', 'keyword': 'keywords', } # find the id for the 'open' status open_id = roundup.lookup('status', 'open') print(f'* Counting total issues...') total_issues_num = len(roundup.filter('issue', None, {})) print(f'* Counting open issues...') # use this list to filter only the open issues open_issues = roundup.filter('issue', None, {'status': open_id}) open_issues_num = len(open_issues) # save the totals in a dict with this structure: # totals[propname][open/all][num/perc][name] # where propname is e.g. 'keyword' and name is e.g. 'easy' totals = defaultdict(lambda: {'all': {'perc': {}, 'num': {}}, 'open': {'perc': {}, 'num': {}}}) for cls, propname in classes.items(): print(f'* Counting <{cls}>...') # get the list of ids/names for the given class (e.g. 'easy' is 6) ids = roundup.list(cls, 'id') names = roundup.list(cls, 'name') for id, name in zip(ids, names): # filter and count on *all* issues with the given propname tot_all = len(roundup.filter('issue', None, {propname: id})) totals[propname]['all']['num'][name] = tot_all totals[propname]['all']['perc'][name] = tot_all / total_issues_num # filter and count on *open* issues with the given propname tot_open = len(roundup.filter('issue', open_issues, {propname: id})) totals[propname]['open']['num'][name] = tot_open totals[propname]['open']['perc'][name] = tot_open / open_issues_num print(f'Issues (open/all): {open_issues_num}/{total_issues_num}') # print a list of markdown tables for each bpo class name for propname in classes.values(): print(f'### {propname}') print('| bpo field | open | all |') print('| :--- | ---: | ---: |') # pick the dict for the given propname (e.g. keywords) proptots = totals[propname] names = proptots['open']['num'] # sort the names (e.g. 'easy') in reverse order # based on the number of open issues for name in sorted(names, key=names.get, reverse=True): # get and print num/perc for all/open issues issues_all = proptots['all']['num'][name] issues_open = proptots['open']['num'][name] perc_all = proptots['all']['perc'][name] perc_open = proptots['open']['perc'][name] print(f'| {name:20} | {issues_open:>5} ({perc_open:5.1%}) |' f' {issues_all:>5} ({perc_all:5.1%}) |') # calc and print num/perc for all/open issues tot_issues_all = sum(proptots['all']['num'].values()) tot_issues_open = sum(proptots['open']['num'].values()) tot_perc_all = sum(proptots['all']['perc'].values()) tot_perc_open = sum(proptots['open']['perc'].values()) print(f'| **Total** | {tot_issues_open:>5} ({tot_perc_open:5.1%}) |' f' {tot_issues_all:>5} ({tot_perc_all:5.1%}) |') code-4d7977d51a4e/scripts/systemd.gunicorn0000644000000000000000000000101114341561674016616 0ustar 00000000000000[Unit] Description=Roundup Server - roundup [Service] User=roundup WorkingDirectory=/path/to/roundup/trackers/roundup SyslogIdentifier=roundup-server-roundup ExecStart=/path/to/gunicorn -b unix:///var/run/roundup/roundup.sock wsgi -w 4 --preload --max-requests 128 --max-requests-jitter 32 --timeout 60 --graceful-timeout 60 --access-logfile - --error-logfile ExecReload=/bin/kill -HUP $MAINPID ExecStop = /bin/kill -s TERM $MAINPID KillMode=mixed Restart=on-failure RestartSec=15s [Install] WantedBy=multi-user.target code-4d7977d51a4e/scripts/weekly-report0000755000000000000000000000323214341561674016126 0ustar 00000000000000#! /usr/bin/env python # This script generates a simple report outlining the activity in one # tracker for the most recent week. # This script is free software, you may redistribute it # and/or modify under the same terms as Python. from __future__ import print_function import sys, math from roundup import instance, date # open the instance if len(sys.argv) != 2: print('You need to specify an instance home dir') instance_home = sys.argv[1] instance = instance.open(instance_home) db = instance.open('admin') old = date.Date('-1w') created = [] summary = {} messages = [] # loop through all the recently-active issues for issue_id in db.issue.filter(None, {'activity': '-1w;'}): num = 0 for x,ts,userid,action,data in db.issue.history(issue_id): if ts < old: continue if action == 'create': created.append(issue_id) elif action == 'set' and 'messages' in data: num += 1 summary.setdefault(db.issue.get(issue_id, 'status'), []).append(issue_id) messages.append((num, issue_id)) #print 'STATUS SUMMARY:' #for k,v in summary.items(): # print k, len(v) print('\nCREATED:') print('\n'.join(['%s: %s'%(id, db.issue.get(id, 'title')) for id in created])) print('\nRESOLVED:') resolved_id = db.status.lookup('resolved') print('\n'.join(['%s: %s'%(id, db.issue.get(id, 'title')) for id in summary.get(resolved_id, [])])) print('\nTOP TEN MOST DISCUSSED:') messages.sort() messages.reverse() nmax = messages[0][0] fmt = '%%%dd - %%s: %%s'%(int(math.log(nmax, 10)) + 1) print('\n'.join([fmt%(num, id, db.issue.get(id, 'title')) for num, id in messages[:10]])) # vim: set filetype=python ts=4 sw=4 et si