code-53afc3ffd4a2/scripts/Docker/Dockerfile0000644000000000000000000002316114725324451016771 0ustar 00000000000000# hadolint global ignore=DL3003 # build in root dir using: # # docker build -t roundup-app --rm -f scripts/Dockerfile . # # run using: # # docker run --rm -v /.../issue.tracker:/usr/src/app/tracker \ # -p 9017:8080 roundup-app:latest # arguments expected to be useful on command line. Lower case names. # Source of roundup code: # 'local' directory using setup.py, # 'pypi[...]' to install latest final or version specified release # from PyPI, E.G. pypi=2.4; pypi<2.3 etc. All pip version specifiers # are allowed. # 'pip_local' local directory using pip to install, or # 'pip_sdist' to install dist/newest_version-tarball ARG source=local # Internal settings that are for advanced users to override. # parameterize the sha256 sum to pin version of python:3-alpine # Must use the same version in both build stages. ARG SHA256=7130f75b1bb16c7c5d802782131b4024fe3d7a87ce7d936e8948c2d2e0180bc4 # Set to any non-empy value to enable shell debugging for troubleshooting ARG VERBOSE= # application directory ARG appdir=/usr/src/app # Python version as a.b Used as path component for # installation directory and COPY from install dir # in second build stage. ARG pythonversion=3.12 #FROM python:3-alpine via SHA256 sum FROM python@sha256:$SHA256 AS build # Inherit global values https://github.com/moby/moby/issues/37345 ARG appdir WORKDIR $appdir # Update to get security and other improvements; RUN apk --no-cache upgrade # Add packages needed to compile mysql, pgsql and other python modules. # Can't use apk to add python packages as it installs for base image # in /usr. # g++ installs cc1plus needed by pip install RUN apk --no-cache add \ file \ g++ \ gcc \ gpgme-dev \ libxapian \ linux-headers \ make \ musl-dev \ mysql-dev \ postgresql-dev \ swig \ xapian-core-dev ARG pythonversion # verify that pythonversion matches the one in the image. RUN image_python_version=$(python -c 'import sys; print("%s.%s"%sys.version_info[0:2])'); \ if [ "${pythonversion}" != "${image_python_version}" ]; then \ printf "\n\n*****\npythonversion does not match.\n" ; \ printf "Add:\n --build-arg=\"pythonversion=%s\"\nto docker build\n******\n\n" "${image_python_version}"; \ exit 1; \ fi # build xapian bindings: # file with sphinx build dependencies to remove after build # they are over 70MB of space. COPY scripts/Docker/sphinxdeps.txt . # suppress warning when running pip as root ENV PIP_ROOT_USER_ACTION=ignore SHELL ["/bin/ash", "-eo", "pipefail", "-c"] ARG VERBOSE ARG pythonversion RUN [ -z "${VERBOSE}" ] || set -xv; \ CWD="$PWD" && \ upgrades=$(python3 -m pip --no-cache --disable-pip-version-check \ list --outdated | awk 'NR > 2 {print $1}'); \ if [ -n "$upgrades" ]; then \ echo "Pip updating $upgrades"; \ python -m pip --no-cache --disable-pip-version-check \ install -U $upgrades < /dev/null; \ else \ echo "Nothing to pip update"; \ fi; \ ls -l /usr/local/lib/python${pythonversion}/site-packages; \ VER=$(apk list -I 'xapian-core-dev' | \ sed 's/^xapian-core-dev-\([0-9.]*\)-.*/\1/') && \ cd /tmp && \ wget -q "https://oligarchy.co.uk/xapian/$VER/xapian-bindings-$VER.tar.xz" && \ tar -Jxvf "xapian-bindings-$VER.tar.xz" && \ cd "xapian-bindings-$VER/" && \ pip --no-cache-dir install sphinx && \ sed -i -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ configure && \ ./configure --prefix=/usr/local --with-python3 --disable-documentation && \ make && make install && \ pip uninstall --no-cache-dir -y sphinx && \ pip uninstall --no-cache-dir -y -r "$CWD/sphinxdeps.txt" && \ rm -rf /usr/local/share/doc/xapian-bindings # add requirements for pip here, e.g. Whoosh, gpg, zstd or other # modules not installed in the base library. # ignore warnings from pip to use virtualenv COPY scripts/Docker/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # copy the elements of the release directory to the docker image COPY setup.py install/ COPY doc install/doc/ COPY frontends install/frontends/ COPY locale install/locale/ COPY roundup install/roundup/ COPY share install/share/ # dist/* might not exist, so include README.txt so we don't get an error #COPY README.txt dist/* install/dist/ # verify source has one of four valid values then # install in python3 standard directories from: # * local copy using setup.py # * pypi using pip # * local files using pip # * pip using a pre-existing sdist built in original source tree: disabled # * copy from dist/* fails if dist directory is missing breaking the # Distfile. COPY used to ignore missing files if at least one # existing file was copied 8-(. # import from global/command line ARG source RUN [ -z "${VERBOSE}" ] || set -xv; \ case "$source" in \ "local") \ cd install && ./setup.py install;; \ "pypi"*) \ version_spec=$( printf "%s" "$source" | \ sed -ne 's/^pypi\([~=!<>].*\)/\1/p'); \ pip install --no-cache-dir "roundup${version_spec}"; \ cp -ril /usr/local/lib/"python${pythonversion}"/site-packages/usr/local/share/* \ /usr/local/share;; \ "pip_local") \ cd install && \ pip -V && \ pip install --no-cache-dir . ;; \ "pip_sdist") \ printf "\n\npip_sdist disabled\n\n" && exit 1 ; \ dist=$(find install/dist | sed -ne '/roundup-[0-9].*\.tar\.gz$/p' | tail -n 1); \ if [ -z "$dist" ] ; then \ printf "\n\nUnable to find a source distribution file in dist\n"; \ printf "Exiting\n\n"; \ exit 1; \ fi; \ printf "Building with distribution: %s\n" "$dist"; \ pip install --no-cache-dir "install/dist/$dist";; \ *) \ printf "\n\ninvalid value for source: %s\n" "$source"; \ printf "must be local, pypi, pip_local or pip_sdist\n\n"; \ exit 1;; \ esac # Allow user to add more modules during build ARG pip_mod RUN if [ -n "$pip_mod" ]; then pip install --no-cache-dir ${pip_mod}; fi # build a new smaller docker image for execution. Build image above # is 1G in size. # FROM python:3-alpine FROM python@sha256:$SHA256 # import from global ARG appdir WORKDIR $appdir # suppress warning when running pip as root ENV PIP_ROOT_USER_ACTION=ignore # upgrade to get any security updates; bundle with # rest of apk actions to reduce layers/wasted space # add libraries needed to run gpg/mysql/pgsql/brotli # clean out any caches to save space # upgrade pip packages to get security and other updates # bundle with apk updates SHELL ["/bin/ash", "-eo", "pipefail", "-c"] RUN apk --no-cache upgrade; \ apk --no-cache add \ brotli-libs \ dumb-init \ gpgme \ mariadb-connector-c \ libpq \ libstdc++ \ libxapian \ zstd-libs; \ upgrades=$(python3 -m pip --no-cache --disable-pip-version-check \ list --outdated | awk 'NR > 2 {print $1}'); \ if [ -n "$upgrades" ]; then \ echo "Pip updating $upgrades"; \ python -m pip --no-cache --disable-pip-version-check \ install -U $upgrades < /dev/null; \ else \ echo "Nothing to pip update"; \ fi; \ test -s /usr/bin/dumb-init ARG source ARG pythonversion ARG pip_mod LABEL "org.opencontainers.image.vendor"="Roundup Issue Tracker Team" \ "org.opencontainers.image.title"="Roundup Issue Tracker" \ "org.opencontainers.image.description"="Roundup Issue Tracker with multi-backend support installed via $source with python version $pythonversion" \ "org.opencontainers.image.version"="2.4.0" \ "org.opencontainers.image.authors"="roundup-devel@lists.sourceforge.net" \ "org.opencontainers.image.licenses"="MIT AND ZPL-2.0 AND Python-2.0" \ "org.opencontainers.image.documentation"="https://www.roundup-tracker.org/docs/installation.html" \ "pip-modules"="$pip_mod" ARG pythonversion # pull over built assets COPY --from=build /usr/local/lib/python${pythonversion}/site-packages /usr/local/lib/python${pythonversion}/site-packages/ COPY --from=build /usr/local/bin/roundup* /usr/local/bin/ COPY --from=build /usr/local/share /usr/local/share/ COPY scripts/Docker/roundup_start scripts/Docker/roundup_healthcheck ./ # Do not run roundup as root. This creates roundup user and group. ARG roundup_uid RUN adduser -D -h ${appdir} -u ${roundup_uid:-1000} roundup # make roundup scripts execuable and mount a trackerdir on tracker location RUN chmod +x roundup_start roundup_healthcheck; \ mkdir tracker; chown ${roundup_uid:-1000}:${roundup_uid:-1000} tracker VOLUME $appdir/tracker # map port 8080 to your local port EXPOSE 8080/tcp HEALTHCHECK --start-period=1m \ CMD ./roundup_healthcheck # switch to using roundup user USER roundup # run the server, disable output buffering so we can see logs. ENV PYTHONUNBUFFERED=1 #ENTRYPOINT [ "roundup-server", "-n", "0.0.0.0" ] ENTRYPOINT [ "/usr/bin/dumb-init", "./roundup_start" ] # allow the invoker to override cmd with multiple trackers # in each subdirectory under $appdir/tracker. E.G. # docker run .... \ # issues=tracker/issues foo=tracker/foo # # note using "issue=$appdir/tracker" results in error: # # No valid configuration files found in directory /usr/src/app/$appdir/tracker # # so $appdir not expanded and $PWD prefixed onto the (relative path) # $appdir/tracker. Hence use relative path for spec. CMD [ "issues=tracker" ] code-53afc3ffd4a2/scripts/Docker/docker-compose.yml0000644000000000000000000000406214725324451020433 0ustar 00000000000000# docker-roundup # roundup issue tracker application with mariadb running as docker container # # docker-compose.yml # # Build components: # docker-compose -f scripts/Docker/docker-compose.yml build # # Install tracker template: # $ docker-compose -f scripts/Docker/docker-compose.yml run \ # --rm --entrypoint roundup-admin --no-deps roundup-app \ # -i tracker install # # Edit scripts/Docker/tracker/config.ini configure database settings # and any NO DEFAULT settings. # # Initialize the database, wait 1 minute so # # $ docker-compose -f scripts/Docker/docker-compose.yml run \ # --rm --entrypoint roundup-admin roundup-app \ # -i tracker # # wait 1 minute for mariadb to initialize # init tracker at roundup prompt # # roundup> init # roundup> exit # # may need ^\ to get roundup-admin to exit. # # run # docker-compose -f scripts/Docker/docker-compose.yml up # # tracker should be running at port 9017. # Note: mysql volume and tracker directories will be put in the # scripts/Docker subdir. # Paths for volumes are relative to docker-compose.yml location not # docker-compose cwd or build context directory. version: '3' services: mariadb: image: lscr.io/linuxserver/mariadb container_name: mariadb restart: unless-stopped environment: - PUID=1000 - PGID=1000 - TZ=America/New_York - MYSQL_ROOT_PASSWORD=myPassword - MYSQL_DATABASE=roundup - MYSQL_USER=roundup_user - MYSQL_PASSWORD=roundup_pass # ports: # - 3306:3306 volumes: - ./dbData:/config roundup-app: container_name: roundup-app build: context: ../.. dockerfile: scripts/Docker/Dockerfile args: source: local_pip #source: local #source: pypi command: "issues=tracker" restart: unless-stopped environment: - TZ=America/New_York ports: - 9017:8080 links: - mariadb depends_on: - mariadb volumes: # will be placed in Docker subdir next to this file - ./tracker:/usr/src/app/tracker code-53afc3ffd4a2/scripts/Docker/requirements.txt0000644000000000000000000000050014725324451020253 0ustar 00000000000000# setuptools required for installation - missing from newer base images setuptools # human timezones pytz # indexer Whoosh # xapian is also installed # extra database support psycopg2 mysqlclient redis # encryption gpg # java web tokens PyJWT # extra HTTP compression methods Brotli zstd # jinja2 template engine Jinja2 code-53afc3ffd4a2/scripts/Docker/roundup_healthcheck0000755000000000000000000000046114725324451020742 0ustar 00000000000000#! /bin/sh # if there are multiple trackers, d=demo t=tracker ... # returns last one for testing that server is up. Does not test # each tracker. tracker=$(ps -ef | sed -ne '/roundup-server/s/^.*\s\(\w*\)=.*$/\1/p') wget -q -O /dev/null --proxy off --no-verbose http://localhost:8080/"${tracker:-demo}"/ code-53afc3ffd4a2/scripts/Docker/roundup_start0000755000000000000000000001642614725324451017644 0ustar 00000000000000#! /bin/sh # When container starts verify that the trackers are configured. # If they are, start the server otherwise run roundup-admin # for installation and initialization. # "$@" should be one of the recognized keywords or arguments for # roundup-server including one or more tracker tracker=directory pairs. # exit on errors; POSIX (e.g. dash as on ubuntu) doesn't support # pipefail. So setting it will fail. Allow that to happen then # set exit on error so the script will still run. set -o pipefail # not supported by POSIX shell, but then neither is pipefail. trap "echo 'Exiting on pipefail'" ERR set -e if [ -n "$SHELL_DEBUG" ]; then set -xv fi trap exit INT do_exit=0 if test -t 0 -a -t 1; then # see if stdin/out are associated with a tty might_be_interactive="true" else might_be_interactive="false" fi printf "Starting Roundup with arguments: $@" for tracker_spec in "$@"; do # IFS== set a=b doesn't just assign $1 and $2 in busybox ash # it also clobbers '$@'. 'printf mumble | read' starts read in a # subshell so vars are not available in parent. IFS="=" read -r tracker directory <<- EOE $tracker_spec EOE # ^ is a tab for use with <<- if [ -z "$directory" ]; then # $tracker_spec was not in the form of a=b, check to see # if it's a request to start in demo/shell mode: case "$tracker" in demo) # if grep does not find 2.2.0 it exits non-zero and # because of "-eo pipefail" the script exits. So || true # so pipeline exits true. version=$(roundup-admin -v | grep '2\.2\.0' || true) if [ -n "$version" ]; then printf "\nRoundup version: %s does not support docker demo mode\n" "$version" printf "Try building with a version newer than 2.2.0.\n" printf "For example:\n" printf " docker build --build-arg=\"source=pypi~=2.3.0a1\" ...\n" exit 3 fi if [ -z "$PORT_8080" ]; then PORT_8080=8917 printf " If docker reports a bind error, you can set the\n" printf " Docker environment variable PORT_8080 to\n" printf " fix it.\n\n" printf " Add -e PORT_8080=port_number to the docker run\n" printf " command. The port_number must match the first\n" printf " value to -p which must be an unused port\n" printf " on your server.\n\n" fi template=classic backend=sqlite shift demoArgs=$# for arg in "$@"; do # all keywords are unique, so iterate over all and # assign as appropriate case "$arg" in classic|devel|jinja2|minimal|responsive) template="$arg";; anydbm|sqlite) backend="$arg";; postgres|mysql) printf "demo mode only supports sqlite or anydbm backends, not %s. Exiting." "$1" exit 1;; nuke) nuke="$arg";; *) printf "Unknown argument %s.\n" "$1" printf "Usage: demo [template] [db]\n" printf " template: one of " printf "classic devel jinja2 minimal responsive\n" printf " db: one of: sqlite anydbm\n" printf "default: classic sqlite\n" printf "Exiting\n" exit 1 esac shift done # run demo make sure to set bind address -B to 0.0.0.0 # otherwise we can never make it out of the docker container. # use -p to force port to match the exported docker port. if [ -f tracker/demo/config.ini ] && [ -z "$nuke" ]; then if [ "$demoArgs" -ne 0 ]; then printf "Error: backend and template arguments to demo " printf "are invalid if a tracker\nis configured and " printf "'nuke' is not specified.\nExiting.\n" exit 1 fi printf "Restarting existing tracker.\n" # Restart an existing demo tracker. exec roundup-demo \ -B 0.0.0.0 \ -p 8080 \ tracker/demo else # Create new or nuke/create existing tracker. # Set parameters required to create tracker. # Inherit the url port spec from the environment # variable PORT_8080 using default value or value # specified on the docker run command line. printf "Creating new tracker.\n" exec roundup-demo \ -B 0.0.0.0 \ -p 8080 \ -b "$backend" \ --urlport "$PORT_8080" \ -t "$template" \ tracker/demo \ $nuke fi ;; shell) if [ "$might_be_interactive" = "false" ]; then printf \ "Error: must use -it on docker command line to invoke shell\n" exit 3 fi exec /bin/sh;; admin) shift exec roundup-admin "$@";; help) sed -e 's/^\t\t//' < /dev/null 2>&1 then if [ -e "$directory/config.ini" ]; then printf "Please edit %s/config.ini and set the required parameters.\n" "$directory" printf "The web setting appears to be missing.\n" exit 3 fi printf "Installing %s tracker in %s\n" "$tracker" "$directory" roundup-admin -i "$directory" install do_exit=1 fi # we have a valid config.ini so init database if not done # if we get errors, the db directory should be missing # and we print an error. if [ $do_exit = 0 ] && ! [ -e "$directory/db" ]; then printf "Initializing tracker %s\n" "$tracker" if ! roundup-admin -i "$directory" init; then # something went wrong. # verify it looks like a tracker directory # then remove the database directory test -e "$directory/TEMPLATE-INFO.txt" && \ rm -rf "$directory/db" fi do_exit=1 fi done # for "$@" # if any config.ini needs editing don't start up. if [ $do_exit = 0 ]; then # make roundup-server process pid 1 with exec exec roundup-server -n 0.0.0.0 "$@" fi exit 0 code-53afc3ffd4a2/scripts/Docker/sphinxdeps.txt0000644000000000000000000000047614725324451017731 0ustar 00000000000000Babel Jinja2 MarkupSafe Pygments alabaster certifi charset-normalizer docutils idna imagesize packaging pyparsing requests six snowballstemmer sphinxcontrib-applehelp sphinxcontrib-devhelp sphinxcontrib-htmlhelp sphinxcontrib-jsmath sphinxcontrib-qthelp sphinxcontrib-serializinghtml sphinxcontrib-websupport urllib3 code-53afc3ffd4a2/scripts/README.txt0000644000000000000000000000472514725324451015273 0ustar 00000000000000Scripts in this directory: add-issue Add a single issue, as specified on the command line, to your tracker. The initial message for the issue is taken from standard input. copy-user.py Copy one or more Roundup users from one tracker instance to another. dump_dbm_sessions_db.py Simple script to dump a session style dbm database e.g. db/otks or db/sessions in readable form. imapServer.py This IMAP server script that runs in the background and checks for new email from a variety of mailboxes. import_sf.py Import tracker data from Sourceforge.NET into a new roundup instance. roundup-reminder Generate an email that lists outstanding issues. Send in both plain text and HTML formats. schema_diagram.py Generate a schema diagram for a roundup tracker. It generates a 'dot file' that is then fed into the 'dot' tool (http://www.graphviz.org) to generate a graph. schema-dump.py Use recently documented XML-RPC API to dump Roundup data schema in human readable form. spam-remover Remove file attachment spam from a tracker. (Warning destructive, read script well.) stats.xmlrpc.py Script using the xmlrpc interface to retrieve info. Generates report on what values are used by the various issues on bugs.python.org. weekly-report Generate a simple report outlining the activity in one tracker for the most recent week. ---- server-ctl Control the roundup-server daemon from the command line with start, stop, restart, condstart (conditional start - only if server is stopped) and status commands. roundup.rc-debian An control script that may be installed in /etc/init.d on Debian systems. Offers start, stop and restart commands and integrates with the Debian init process. systemd.gunicorn A systemd unit file for running roundup under gnuicorn WSGI server. ---- contributors.py Analyzes Mercurial log, filters and compiles list of committers with years of contribution. Can be useful for updating COPYING.txt ---- Docker Directory for docker setup. More info on how to use it is in doc/installation.txt. Docker/Dockerfile - Create roundup docker. Docker/requirements.txt - Python requirements built into docker. Docker/roundup-start - Startup script for roundup in docker. Docker/docker-compose.yml - Manage two docker containers for roundup and mysql. ---- oauth-get-token.py Retrieve necessary credentials from MS cloud for OAuth authentication with IMAP. This has an extensive help text so you want to call this script with --help first. code-53afc3ffd4a2/scripts/add-issue0000755000000000000000000000277514725324451015404 0ustar 00000000000000#! /usr/bin/env python ''' Usage: %s Create a new issue in the given tracker. Input is taken from STDIN to create the initial issue message (which may be empty). Issues will be created as the current user (%s) if they exist as a Roundup user, or "admin" otherwise. ''' from __future__ import print_function import sys, os, pwd from roundup import instance, mailgw, date # open the instance username = pwd.getpwuid(os.getuid())[0] if len(sys.argv) < 3: print("Error: Not enough arguments") print(__doc__.strip()%(sys.argv[0], username)) sys.exit(1) tracker_home = sys.argv[1] issue_priority = sys.argv[2] issue_title = ' '.join(sys.argv[3:]) # get the message, if any message_text = sys.stdin.read().strip() # open the tracker tracker = instance.open(tracker_home) db = tracker.open('admin') db.tx_Source = "cli" uid = db.user.lookup('admin') try: # try to open the tracker as the current user uid = db.user.lookup(username) db.close() db = tracker.open(username) except KeyError: pass try: # handle the message messages = [] if message_text: summary, x = mailgw.parseContent(message_text, 0, 0) msg = db.msg.create(content=message_text, summary=summary, author=uid, date=date.Date()) messages = [msg] # now create the issue db.issue.create(title=issue_title, priority=issue_priority, messages=messages) db.commit() finally: db.close() # vim: set filetype=python ts=4 sw=4 et si code-53afc3ffd4a2/scripts/contributors.py0000644000000000000000000001037314725324451016700 0ustar 00000000000000""" Get Mercurial history data and output list of contributors with years. Public domain work by: anatoly techtonik """ from __future__ import print_function from subprocess import check_output # --- output settings contributors_by_year = True years_for_contributors = True verbose = True # /-- # --- project specific configuration ALIASES = { 'Richard Jones ': ['richard', 'Richard Jones '], 'Bernhard Reiter ': ['Bernhard Reiter ', 'Bernhard Reiter '], 'Ralf Schlatterbeck ': ['Ralf Schlatterbeck '], 'Stefan Seefeld ': ['Stefan Seefeld '], 'John Rouillard ': ['rouilj@uland', 'rouilj'], } ROBOTS = ['No Author '] # /-- def compress(years): """ Given a list of years like [2003, 2004, 2007], compress it into string like '2003-2004, 2007' >>> compress([2002]) '2002' >>> compress([2003, 2002]) '2002-2003' >>> compress([2009, 2004, 2005, 2006, 2007]) '2004-2007, 2009' >>> compress([2001, 2003, 2004, 2005]) '2001, 2003-2005' >>> compress([2009, 2011]) '2009, 2011' >>> compress([2009, 2010, 2011, 2006, 2007]) '2006-2007, 2009-2011' >>> compress([2002, 2003, 2004, 2005, 2006, 2009, 2012]) '2002-2006, 2009, 2012' """ years = sorted(years) # compress years into string comma = ', ' yearstr = '' for i in range(0,len(years)-1): if years[i+1]-years[i] == 1: if not yearstr or yearstr.endswith(comma): yearstr += '%s' % years[i] if yearstr.endswith('-'): pass else: yearstr += '-' else: yearstr += '%s, ' % years[i] if len(years) == 1: yearstr += str(years[0]) else: yearstr += '%s' % years[-1] return yearstr if __name__ == '__main__': command = 'hg log --template "{date(date,\\"%Y\\")},{author}\\n"' if verbose: print("Getting HG log...") print("Using: ", command) authorship = check_output(command, shell=True) if not isinstance(authorship, str): authorship = authorship.decode('utf-8') # authorship are strings like # 2003,Richard Jones # ... if verbose: print("Splitting...") # transform to a list of tuples authorship = [line.split(',', 1) for line in authorship.splitlines()] if verbose: print("Sorting...") years = {} # year -> set(author1, author2, ...) names = {} # author -> set(years) for year, author in authorship: if author in ROBOTS: continue # process aliases for name, aliases in ALIASES.items(): if author in aliases: author = name break author = author.replace('<', '(') author = author.replace('>', ')') # years if year not in years: years[year] = set() years[year].add(author) # names if author not in names: names[author] = set() names[author].add(int(year)) if contributors_by_year: if verbose: print("Contributors by year...") print('') for year in sorted(years, reverse=True): print(year) for author in sorted(years[year]): print(" " + author) print('') if years_for_contributors: if verbose: print("Years for each contributor...") print('') def last_year(name): """Return year of the latest contribution for a given name""" return sorted(names[name])[-1] def first_year(name): """Return year of the first contribution""" return sorted(names[name])[0] def year_key(name): """ Year key function. First sort by latest contribution year (desc). If it matches, compare first contribution year (asc). This ensures that the most recent and long-term contributors are at the top. """ return (last_year(name), -first_year(name)) print("Copyright (c)") for author in sorted(names, key=year_key, reverse=True): years = list(names[author]) yearstr = compress(years) if 0: #DEBUG print(years, yearstr, author) else: print(" %s %s" % (yearstr, author)) print('') code-53afc3ffd4a2/scripts/copy-user.py0000755000000000000000000000473514725324451016101 0ustar 00000000000000#!/usr/bin/env python # Copyright (C) 2003 by Intevation GmbH # Author: # Thomas Arendsen Hein # # This program is free software dual licensed under the GPL (>=v2) # and the Roundup Licensing (see COPYING.txt in the roundup distribution). """ copy-user [...] Copy one or more Roundup users from one tracker instance to another. Example: copy-user /roundup/tracker1 /roundup/tracker2 `seq 3 10` 14 16 (copies users 3, 4, 5, 6, 7, 8, 9, 10, 14 and 16) """ from __future__ import print_function import sys import roundup.instance def copy_user(home1, home2, *userids): """Copy users which are listed by userids from home1 to home2""" copyattribs = ['username', 'password', 'address', 'realname', 'phone', 'organisation', 'alternate_addresses', 'roles', 'timezone'] try: instance1 = roundup.instance.open(home1) print("Opened source instance: %s" % home1) except: print("Can't open source instance: %s" % home1) sys.exit(1) try: instance2 = roundup.instance.open(home2) print("Opened target instance: %s" % home2) except: print("Can't open target instance: %s" % home2) sys.exit(1) db1 = instance1.open('admin') db2 = instance2.open('admin') db1.tx_Source = "cli" db2.tx_Source = "cli" userlist = db1.user.list() for userid in userids: try: userid = str(int(userid)) except ValueError: print("Not a numeric user id: %s Skipping ..." % (userid,)) continue if userid not in userlist: print("User %s not in source instance. Skipping ..." % userid) continue user = {} for attrib in copyattribs: value = db1.user.get(userid, attrib) if value: user[attrib] = value try: db2.user.lookup(user['username']) print("User %s: Username '%s' exists in target instance. Skipping ..." % (userid, user['username'])) continue except KeyError: pass print("Copying user %s (%s) ..." % (userid, user['username'])) db2.user.create(**user) db2.commit() db2.close() print("Closed target instance.") db1.close() print("Closed source instance.") if __name__ == "__main__": if len(sys.argv) < 4: print(__doc__) sys.exit(1) else: copy_user(*sys.argv[1:]) code-53afc3ffd4a2/scripts/dump_dbm_sessions_db.py0000755000000000000000000000516314725324451020331 0ustar 00000000000000#! /usr/bin/env python3 """Usage: dump_dbm_sessions_db.py [filename] Simple script to dump the otks and sessions dbm databases. Dumps sessions db in current directory if no argument is given. Dump format: key: data where is the human readable __timestamp decoded from the data object. Data object is dumped in json format. With pretty print key: { key: val, ... } if data is not a python object, print will be key: data or key: data if pretty printed. """ import argparse import dbm import json import marshal import os import sys from datetime import datetime def indent(text, amount, ch=" "): """ Found at: https://stackoverflow.com/a/8348914 """ padding = amount * ch return ''.join(padding+line for line in text.splitlines(True)) def print_marshal(k): d = marshal.loads(db[k]) try: t = datetime.fromtimestamp(d['__timestamp']) except (KeyError, TypeError): # TypeError raised if marshalled data is not a dict (list, tuple etc) t = "no_timestamp" if args.pretty: print("%s:\n %s\n%s"%(k, t, indent(json.dumps( d, sort_keys=True, indent=4), 4))) else: print("%s: %s %s"%(k, t, d)) def print_raw(k): if args.pretty: print("%s:\n %s"%(k, db[k])) else: print("%s: %s"%(k, db[k])) parser = argparse.ArgumentParser( description='Dump DBM files used by Roundup in storage order.') parser.add_argument('-k', '--key', action="append", help='dump the entry for a key, can be used multiple times.') parser.add_argument('-K', '--keysonly', action='store_true', help='print the database keys, sorted in byte order.') parser.add_argument('-p', '--pretty', action='store_true', help='pretty print the output rather than printing on one line.') parser.add_argument('file', nargs='?', help='file to be dumped ("sessions" if not provided)') args = parser.parse_args() if args.file: file = args.file else: file="sessions" try: db = dbm.open(file) except Exception as e: print("Unable to open database for %s: %s"%(file, e)) try: os.stat(file) print(" perhaps file is invalid or was created with a different version of Python?") except OSError: # the file does exist on disk. pass sys.exit(1) if args.keysonly: for k in sorted(db.keys()): print("%s"%k) sys.exit(0) if args.key: for k in args.key: try: print_marshal(k) except (ValueError): print_raw(k) sys.exit(0) k = db.firstkey() while k is not None: try: print_marshal(k) except (ValueError): # ValueError marshal.loads failed print_raw(k) k = db.nextkey(k) code-53afc3ffd4a2/scripts/imapServer.py0000644000000000000000000003444514725324451016266 0ustar 00000000000000#!/usr/bin/env python3 """\ This script is a wrapper around the mailgw.py script that exists in roundup. It runs as service instead of running as a one-time shot. It also connects to a secure IMAP server. The main reasons for this script are: 1) The roundup-mailgw script isn't designed to run as a server. It expects that you either run it by hand, and enter the password each time, or you supply the password on the command line. I prefer to run a server that I initialize with the password, and then it just runs. I don't want to have to pass it on the command line, so running through crontab isn't a possibility. (This wouldn't be a problem on a local machine running through a mailspool.) 2) mailgw.py somehow screws up SSL support so IMAP4_SSL doesn't work. So hopefully running that work outside of the mailgw will allow it to work. 3) I wanted to be able to check multiple projects at the same time. roundup-mailgw is only for 1 mailbox and 1 project. *TODO*: For the first round, the program spawns a new roundup-mailgw for each imap message that it finds and pipes the result in. In the future it might be more practical to actually include the roundup files and run the appropriate commands using python. *TODO*: Look into supporting a logfile instead of using 2>/logfile *TODO*: Add an option for changing the uid/gid of the running process. """ from __future__ import print_function import getpass import logging import imaplib import argparse import os import re import time import sys from roundup.anypy.my_input import my_input logging.basicConfig() log = logging.getLogger('roundup.IMAPServer') version = '0.1.3' class RoundupMailbox: """This contains all the info about each mailbox. Username, Password, server, security, roundup database """ def __init__(self, dbhome='', username=None, password=None, mailbox=None , server=None, protocol='imaps'): self.username = username self.password = password self.mailbox = mailbox self.server = server self.protocol = protocol self.dbhome = dbhome try: if not self.dbhome: self.dbhome = my_input('Tracker home: ') if not os.path.exists(self.dbhome): raise ValueError('Invalid home address: ' \ 'directory "%s" does not exist.' % self.dbhome) if not self.server: self.server = my_input('Server: ') if not self.server: raise ValueError('No Servername supplied') protocol = my_input('protocol [imaps]? ') self.protocol = protocol if not self.username: self.username = my_input('Username: ') if not self.username: raise ValueError('Invalid Username') if not self.password: print('For server %s, user %s' % (self.server, self.username)) self.password = getpass.getpass() # password can be empty because it could be superceeded # by a later entry #if self.mailbox is None: # self.mailbox = my_input('Mailbox [INBOX]: ') # # We allow an empty mailbox because that will # # select the INBOX, whatever it is called except (KeyboardInterrupt, EOFError): raise ValueError('Canceled by User') def __str__(self): return 'Mailbox{ server:%(server)s, protocol:%(protocol)s, ' \ 'username:%(username)s, mailbox:%(mailbox)s, ' \ 'dbhome:%(dbhome)s }' % self.__dict__ # [als] class name is misleading. this is imap client, not imap server class IMAPServer: """IMAP mail gatherer. This class runs as a server process. It is configured with a list of mailboxes to connect to, along with the roundup database directories that correspond with each email address. It then connects to each mailbox at a specified interval, and if there are new messages it reads them, and sends the result to the roundup.mailgw. *TODO*: Try to be smart about how you access the mailboxes so that you can connect once, and access multiple mailboxes and possibly multiple usernames. *NOTE*: This assumes that if you are using the same user on the same server, you are using the same password. (the last one supplied is used.) Empty passwords are ignored. Only the last protocol supplied is used. """ def __init__(self, pidfile=None, delay=5, daemon=False): #This is sorted by servername, then username, then mailboxes self.mailboxes = {} self.delay = float(delay) self.pidfile = pidfile self.daemon = daemon def setDelay(self, delay): self.delay = delay def addMailbox(self, mailbox): """ The linkage is as follows: servers -- users - mailbox:dbhome So there can be multiple servers, each with multiple users. Each username can be associated with multiple mailboxes. each mailbox is associated with 1 database home """ log.info('Adding mailbox %s', mailbox) if mailbox.server not in self.mailboxes: self.mailboxes[mailbox.server] = {'protocol':'imaps', 'users':{}} server = self.mailboxes[mailbox.server] if mailbox.protocol: server['protocol'] = mailbox.protocol if mailbox.username not in server['users']: server['users'][mailbox.username] = {'password':'', 'mailboxes':{}} user = server['users'][mailbox.username] if mailbox.password: user['password'] = mailbox.password if mailbox.mailbox in user['mailboxes']: raise ValueError('Mailbox is already defined') user['mailboxes'][mailbox.mailbox] = mailbox.dbhome def _process(self, message, dbhome): """Actually process one of the email messages""" child = os.popen('roundup-mailgw %s' % dbhome, 'wb') child.write(message) child.close() #print message def _getMessages(self, serv, count, dbhome): """This assumes that you currently have a mailbox open, and want to process all messages that are inside. """ for n in range(1, count+1): (t, data) = serv.fetch(n, '(RFC822)') if t == 'OK': self._process(data[0][1], dbhome) serv.store(n, '+FLAGS', r'(\Deleted)') def checkBoxes(self): """This actually goes out and does all the checking. Returns False if there were any errors, otherwise returns true. """ noErrors = True for server in self.mailboxes: log.info('Connecting to server: %s', server) s_vals = self.mailboxes[server] try: for user in s_vals['users']: u_vals = s_vals['users'][user] # TODO: As near as I can tell, you can only # login with 1 username for each connection to a server. protocol = s_vals['protocol'].lower() if protocol == 'imaps': serv = imaplib.IMAP4_SSL(server) elif protocol == 'imap': serv = imaplib.IMAP4(server) else: raise ValueError('Unknown protocol %s' % protocol) password = u_vals['password'] try: log.info('Connecting as user: %s', user) serv.login(user, password) for mbox in u_vals['mailboxes']: dbhome = u_vals['mailboxes'][mbox] log.info('Using mailbox: %s, home: %s', mbox, dbhome) #access a specific mailbox if mbox: (t, data) = serv.select(mbox) else: # Select the default mailbox (INBOX) (t, data) = serv.select() try: nMessages = int(data[0]) except ValueError: nMessages = 0 log.info('Found %s messages', nMessages) if nMessages: self._getMessages(serv, nMessages, dbhome) serv.expunge() # We are done with this mailbox serv.close() except: log.exception('Exception with server %s user %s', server, user) noErrors = False serv.logout() serv.shutdown() del serv except: log.exception('Exception while connecting to %s', server) noErrors = False return noErrors def makeDaemon(self): """Turn this process into a daemon. - make our parent PID 1 Write our new PID to the pidfile. From A.M. Kuuchling (possibly originally Greg Ward) with modification from Oren Tirosh, and finally a small mod from me. Originally taken from roundup.scripts.roundup_server.py """ log.info('Running as Daemon') # Fork once if os.fork() != 0: os._exit(0) # Create new session os.setsid() # Second fork to force PPID=1 pid = os.fork() if pid: if self.pidfile: pidfile = open(self.pidfile, 'w') pidfile.write(str(pid)) pidfile.close() os._exit(0) def run(self): """Run email gathering daemon. This spawns itself as a daemon, and then runs continually, just sleeping inbetween checks. It is recommended that you run checkBoxes once first before you select run. That way you can know if there were any failures. """ if self.daemon: self.makeDaemon() while True: time.sleep(self.delay * 60.0) log.info('Time: %s', time.strftime('%Y-%m-%d %H:%M:%S')) self.checkBoxes() def getItems(s): """Parse a string looking for userame@server""" myRE = re.compile( r'((?P[^:]+)://)?'#You can supply a protocol if you like r'(' #The username part is optional r'(?P[^:]+)' #You can supply the password as r'(:(?P.+))?' #username:password@server r'@)?' r'(?P[^/]+)' r'(/(?P.+))?$' ) m = myRE.match(s) if m: return m.groupdict() else: return None def main(): """This is what is called if run at the prompt""" parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" The server takes pairs of home/server. So each entry has a home, and then the server configuration. Home is just a path to the roundup issue tracker. The server is something of the form: imaps://user:password@server/mailbox If you don't supply the protocol, imaps is assumed. Without user or password, you will be prompted for them. The server must be supplied. Without mailbox the INBOX is used. Examples: %(prog)s /home/roundup/trackers/test imaps://test@imap.example.com/test %(prog)s /home/roundup/trackers/test imap.example.com \\ /home/roundup/trackers/test2 imap.example.com/test2 """ % dict(prog = sys.argv [0]) ) parser.add_argument('args', nargs='*') parser.add_argument('-d', '--delay', dest='delay', type=float, metavar='', default=5, help="Set the delay between checks in minutes. (default 5)" ) parser.add_argument('-p', '--pid-file', dest='pidfile', metavar='', default=None, help="The pid of the server process will be written to " ) parser.add_argument('-n', '--no-daemon', dest='daemon', action='store_false', default=True, help="Do not fork into the background after running the first check." ) parser.add_argument('--version', action="store_true", help="Print version and exit") parser.add_argument('-v', '--verbose', dest='verbose', action='store_const', const=logging.INFO, help="Be more verbose in letting you know what is going on." " Enables informational messages." ) parser.add_argument('-V', '--very-verbose', dest='verbose', action='store_const', const=logging.DEBUG, help="Be very verbose in letting you know what is going on." " Enables debugging messages." ) parser.add_argument('-q', '--quiet', dest='verbose', action='store_const', const=logging.ERROR, help="Be less verbose. Ignores warnings, only prints errors." ) parser.add_argument('-Q', '--very-quiet', dest='verbose', action='store_const', const=logging.CRITICAL, help="Be much less verbose. Ignores warnings and errors." " Only print CRITICAL messages." ) args = parser.parse_args() if args.version: print('%s %s' % (sys.argv [0], version)) sys.exit(0) if not len(args.args) or len(args.args) % 2 == 1: parser.error('Invalid number of arguments. ' 'Each site needs a home and a server.') if args.verbose == None: args.verbose = logging.WARNING log.setLevel(args.verbose) myServer = IMAPServer(delay=args.delay, pidfile=args.pidfile, daemon=args.daemon) for i in range(0,len(args.args),2): home = args.args[i] server = args.args[i+1] if not os.path.exists(home): parser.error('Home: "%s" does not exist' % home) info = getItems(server) if not info: parser.error('Invalid server string: "%s"' % server) myServer.addMailbox( RoundupMailbox(dbhome=home, mailbox=info['mailbox'] , username=info['username'], password=info['password'] , server=info['server'], protocol=info['protocol'] ) ) if myServer.checkBoxes(): myServer.run() if __name__ == '__main__': main() # vim: et ft=python si sts=4 sw=4 code-53afc3ffd4a2/scripts/import_sf.py0000644000000000000000000003424614725324451016152 0ustar 00000000000000""" Import tracker data from Sourceforge.NET This script needs four steps to work: 1. Export the project XML data using the admin web interface at sf.net 2. Run the file fetching (these are not included in the XML): import_sf.py files this will place all the downloaded files in the files dir by file id. 3. Convert the sf.net XML to Roundup "export" format: import_sf.py import this will generate a directory "/tmp/imported" which contains the data to be imported into a Roundup tracker. 4. Import the data: roundup-admin -i import /tmp/imported And you're done! """ import sys, os, csv, time, mimetypes try: import cElementTree as ElementTree except ImportError: from elementtree import ElementTree from roundup import instance, hyperdb, date, support, password from roundup.anypy import http_, urllib_ from roundup.anypy.strings import s2b, us2s today = date.Date('.') DL_URL = 'http://sourceforge.net/tracker/download.php?group_id=%(group_id)s&atid=%(atid)s&aid=%(aid)s' def get_url(aid): """ so basically we have to jump through hoops, given an artifact id, to figure what the URL should be to access that artifact, and hence any attached files.""" # first we hit this URL... conn = http_.client.HTTPConnection("sourceforge.net") conn.request("GET", "/support/tracker.php?aid=%s"%aid) response = conn.getresponse() # which should respond with a redirect to the correct url which has the # magic "group_id" and "atid" values in it that we need assert response.status == 302, 'response code was %s'%response.status location = response.getheader('location') query = urllib_.urlparse(response.getheader('location'))[-2] info = dict([param.split('=') for param in query.split('&')]) return DL_URL%info def fetch_files(xml_file, file_dir): """ Fetch files referenced in the xml_file into the dir file_dir. """ root = ElementTree.parse(xml_file).getroot() to_fetch = set() deleted = set() for artifact in root.find('artifacts'): for field in artifact.findall('field'): if field.get('name') == 'artifact_id': aid = field.text for field in artifact.findall('field'): if field.get('name') != 'artifact_history': continue for event in field.findall('history'): d = {} for field in event.findall('field'): d[field.get('name')] = field.text if d['field_name'] == 'File Added': fid = d['old_value'].split(':')[0] to_fetch.add((aid, fid)) if d['field_name'] == 'File Deleted': fid = d['old_value'].split(':')[0] deleted.add((aid, fid)) to_fetch = to_fetch - deleted got = set(os.listdir(file_dir)) to_fetch = to_fetch - got # load cached urls (sigh) urls = {} if os.path.exists(os.path.join(file_dir, 'urls.txt')): for line in open(os.path.join(file_dir, 'urls.txt')): aid, url = line.strip().split() urls[aid] = url for aid, fid in support.Progress('Fetching files', list(to_fetch)): if fid in got: continue if aid not in urls: urls[aid] = get_url(aid) f = open(os.path.join(file_dir, 'urls.txt'), 'a') f.write('%s %s\n'%(aid, urls[aid])) f.close() url = urls[aid] + '&file_id=' + fid f = urllib_.urlopen(url) data = f.read() n = open(os.path.join(file_dir, fid), 'w') n.write(data) f.close() n.close() def import_xml(tracker_home, xml_file, file_dir): """ Generate Roundup tracker import files based on the tracker schema, sf.net xml export and downloaded files from sf.net. """ tracker = instance.open(tracker_home) db = tracker.open('admin') resolved = db.status.lookup('resolved') unread = db.status.lookup('unread') chatting = db.status.lookup('unread') critical = db.priority.lookup('critical') urgent = db.priority.lookup('urgent') bug = db.priority.lookup('bug') feature = db.priority.lookup('feature') wish = db.priority.lookup('wish') adminuid = db.user.lookup('admin') anonuid = db.user.lookup('anonymous') root = ElementTree.parse(xml_file).getroot() def to_date(ts): return date.Date(time.gmtime(float(ts))) # parse out the XML artifacts = [] categories = set() users = set() add_files = set() remove_files = set() for artifact in root.find('artifacts'): d = {} op = {} artifacts.append(d) for field in artifact.findall('field'): name = field.get('name') if name == 'artifact_messages': for message in field.findall('message'): l = d.setdefault('messages', []) m = {} l.append(m) for field in message.findall('field'): name = field.get('name') if name == 'adddate': m[name] = to_date(field.text) else: m[name] = field.text if name == 'user_name': users.add(field.text) elif name == 'artifact_history': for event in field.findall('history'): l = d.setdefault('history', []) e = {} l.append(e) for field in event.findall('field'): name = field.get('name') if name == 'entrydate': e[name] = to_date(field.text) else: e[name] = field.text if name == 'mod_by': users.add(field.text) if e['field_name'] == 'File Added': add_files.add(e['old_value'].split(':')[0]) elif e['field_name'] == 'File Deleted': remove_files.add(e['old_value'].split(':')[0]) elif name == 'details': op['body'] = field.text elif name == 'submitted_by': op['user_name'] = field.text d[name] = field.text users.add(field.text) elif name == 'open_date': thedate = to_date(field.text) op['adddate'] = thedate d[name] = thedate else: d[name] = field.text categories.add(d['category']) if 'body' in op: l = d.setdefault('messages', []) l.insert(0, op) add_files -= remove_files # create users userd = {'nobody': '2'} users.remove('nobody') data = [ {'id': '1', 'username': 'admin', 'password': password.Password('admin'), 'roles': 'Admin', 'address': 'richard@python.org'}, {'id': '2', 'username': 'anonymous', 'roles': 'Anonymous'}, ] for n, user in enumerate(list(users)): userd[user] = n+3 data.append({'id': str(n+3), 'username': user, 'roles': 'User', 'address': '%s@users.sourceforge.net'%user}) write_csv(db.user, data) users=userd # create categories categoryd = {'None': None} categories.remove('None') data = [] for n, category in enumerate(list(categories)): categoryd[category] = n data.append({'id': str(n), 'name': category}) write_csv(db.keyword, data) categories = categoryd # create issues issue_data = [] file_data = [] message_data = [] issue_journal = [] message_id = 0 for artifact in artifacts: d = {} d['id'] = artifact['artifact_id'] d['title'] = artifact['summary'] d['assignedto'] = users[artifact['assigned_to']] if d['assignedto'] == '2': d['assignedto'] = None d['creation'] = artifact['open_date'] activity = artifact['open_date'] d['creator'] = users[artifact['submitted_by']] actor = d['creator'] if categories[artifact['category']]: d['keyword'] = [categories[artifact['category']]] issue_journal.append(( d['id'], d['creation'].get_tuple(), d['creator'], "'create'", {} )) p = int(artifact['priority']) if artifact['artifact_type'] == 'Feature Requests': if p > 3: d['priority'] = feature else: d['priority'] = wish else: if p > 7: d['priority'] = critical elif p > 5: d['priority'] = urgent elif p > 3: d['priority'] = bug else: d['priority'] = feature s = artifact['status'] if s == 'Closed': d['status'] = resolved elif s == 'Deleted': d['status'] = resolved d['is retired'] = True else: d['status'] = unread nosy = set() for message in artifact.get('messages', []): authid = users[message['user_name']] if not message['body']: continue body = convert_message(message['body'], message_id) if not body: continue m = {'content': body, 'author': authid, 'date': message['adddate'], 'creation': message['adddate'], } message_data.append(m) if authid not in (None, '2'): nosy.add(authid) activity = message['adddate'] actor = authid if d['status'] == unread: d['status'] = chatting # add import message m = {'content': 'IMPORT FROM SOURCEFORGE', 'author': '1', 'date': today, 'creation': today} message_data.append(m) # sort messages and assign ids d['messages'] = [] message_data.sort(key=lambda a:a['date']) for message in message_data: message_id += 1 message['id'] = str(message_id) d['messages'].append(message_id) d['nosy'] = list(nosy) files = [] for event in artifact.get('history', []): if event['field_name'] == 'File Added': fid, name = event['old_value'].split(':', 1) if fid in add_files: files.append(fid) name = name.strip() try: f = open(os.path.join(file_dir, fid), 'rb') content = f.read() f.close() except: content = 'content missing' file_data.append({ 'id': fid, 'creation': event['entrydate'], 'creator': users[event['mod_by']], 'name': name, 'type': mimetypes.guess_type(name)[0], 'content': content, }) continue elif event['field_name'] == 'close_date': action = "'set'" info = { 'status': unread } elif event['field_name'] == 'summary': action = "'set'" info = { 'title': event['old_value'] } else: # not an interesting / translatable event continue row = [ d['id'], event['entrydate'].get_tuple(), users[event['mod_by']], action, info ] if event['entrydate'] > activity: activity = event['entrydate'] issue_journal.append(row) d['files'] = files d['activity'] = activity d['actor'] = actor issue_data.append(d) write_csv(db.issue, issue_data) write_csv(db.msg, message_data) write_csv(db.file, file_data) f = open('/tmp/imported/issue-journals.csv', 'w') writer = csv.writer(f, colon_separated) writer.writerows(issue_journal) f.close() def convert_message(content, id): """ Strip off the useless sf message header crap """ if content[:14] == 'Logged In: YES': return '\n'.join(content.splitlines()[3:]).strip() return content class colon_separated(csv.excel): delimiter = ':' def write_csv(klass, data): props = klass.getprops() if not os.path.exists('/tmp/imported'): os.mkdir('/tmp/imported') f = open('/tmp/imported/%s.csv'%klass.classname, 'w') writer = csv.writer(f, colon_separated) propnames = klass.export_propnames() propnames.append('is retired') writer.writerow(propnames) for entry in data: row = [] for name in propnames: if name == 'is retired': continue prop = props[name] if name in entry: if isinstance(prop, hyperdb.Date) or \ isinstance(prop, hyperdb.Interval): row.append(repr(entry[name].get_tuple())) elif isinstance(prop, hyperdb.Password): row.append(repr(str(entry[name]))) else: row.append(repr(entry[name])) elif isinstance(prop, hyperdb.Multilink): row.append('[]') elif name in ('creator', 'actor'): row.append("'1'") elif name in ('created', 'activity'): row.append(repr(today.get_tuple())) else: row.append('None') row.append(entry.get('is retired', False)) writer.writerow(row) if isinstance(klass, hyperdb.FileClass) and entry.get('content'): fname = klass.exportFilename('/tmp/imported/', entry['id']) support.ensureParentsExist(fname) c = open(fname, 'wb') if isinstance(entry['content'], bytes): c.write(entry['content']) else: c.write(s2b(us2s(entry['content']))) c.close() f.close() f = open('/tmp/imported/%s-journals.csv'%klass.classname, 'w') f.close() if __name__ == '__main__': if sys.argv[1] == 'import': import_xml(*sys.argv[2:]) elif sys.argv[1] == 'files': fetch_files(*sys.argv[2:]) code-53afc3ffd4a2/scripts/oauth-get-token.py0000755000000000000000000003141114725324451017155 0ustar 00000000000000#!/usr/bin/python3 import requests import time import sys import webbrowser import ssl from urllib.parse import urlparse, urlencode, parse_qs from argparse import ArgumentParser, RawDescriptionHelpFormatter from http.server import HTTPServer, BaseHTTPRequestHandler class Request_Token: def __init__ (self, args): self.args = args self.session = requests.session () self.url = '/'.join ((args.url.rstrip ('/'), args.tenant)) self.url = '/'.join ((self.url, 'oauth2/v2.0')) self.state = None self.use_tls = self.args.use_tls if self.use_tls is None: self.use_tls = self.args.redirect_uri.startswith ('https') # end def __init__ def check_err (self, r): if not 200 <= r.status_code <= 299: raise RuntimeError \ ( 'Invalid result: %s: %s\n %s' % (r.status_code, r.reason, r.text) ) # end def check_err def get_url (self, path, params): url = ('/'.join ((self.url, path))) url = url + '?' + urlencode (params) return url # end def get_url def post_or_put (self, method, path, data = None, json = None): d = {} if data: d.update (data = data) if json: d.update (json = json) url = ('/'.join ((self.url, path))) r = method (url, **d) self.check_err (r) return r.json () # end def post_or_put def post (self, path, data = None, json = None): return self.post_or_put (self.session.post, path, data, json) # end def post def authcode_callback (self, handler): msg = [''] self.request_received = False r = urlparse (handler.path) if r.query: q = parse_qs (r.query) if 'state' in q: state = q ['state'][0] if state != self.state: msg.append \ ( 'State did not match: expect "%s" got "%s"' % (self.state, state) ) elif 'code' not in q: msg.append ('Got no code') else: with open ('oauth/authcode', 'w') as f: f.write (q ['code'][0]) msg.append ('Wrote code to oauth/authcode') self.request_received = True else: msg.append ('No state and no code') return 200, '\n'.join (msg).encode ('utf-8') # end def authcode_callback def request_authcode (self): with open ('oauth/client_id', 'r') as f: client_id = f.read () self.state = 'authcode' + str (time.time ()) params = dict \ ( client_id = client_id , response_type = 'code' , response_mode = 'query' , state = self.state , redirect_uri = self.args.redirect_uri , scope = ' '.join (( 'https://outlook.office.com/IMAP.AccessAsUser.All' , 'https://outlook.office.com/User.Read' , 'offline_access' )) ) url = self.get_url ('authorize', params) print (url) if self.args.webbrowser: browser = webbrowser.get (self.args.browser) browser.open_new_tab (url) if self.args.run_https_server: self.https_server () if self.args.request_tokens: self.request_token () # end def request_authcode def request_token (self): with open ('oauth/client_id', 'r') as f: client_id = f.read () with open ('oauth/client_secret', 'r') as f: client_secret = f.read ().strip () with open ('oauth/authcode', 'r') as f: authcode = f.read ().strip () params = dict \ ( client_id = client_id , code = authcode , client_secret = client_secret , redirect_uri = self.args.redirect_uri , grant_type = 'authorization_code' # Only a single scope parameter is allowed here , scope = ' '.join (( 'https://outlook.office.com/User.Read' , )) ) result = self.post ('token', data = params) with open ('oauth/refresh_token', 'w') as f: f.write (result ['refresh_token']) with open ('oauth/access_token', 'w') as f: f.write (result ['access_token']) # end def request_token def https_server (self): self.request_received = False class RQ_Handler (BaseHTTPRequestHandler): token_handler = self def do_GET (self): self.close_connection = True code, msg = self.token_handler.authcode_callback (self) self.send_response (code) self.send_header ('Content-Type', 'text/plain') self.end_headers () self.wfile.write (msg) self.wfile.flush () port = self.args.https_server_port httpd = HTTPServer (('localhost', port), RQ_Handler) if self.use_tls: # note this opens a server on localhost. Only # a process on the same host can get the credentials. # Even unencrypted (http://) url is fine as the credentials # will be saved in clear text on disk for use. So a # compromised local host will still get the credentials. context = ssl.SSLContext(ssl_version=ssl.PROTOCOL_TLS_SERVER) # This should not be needed as PROTOCOL_TLS_SERVER disables # unsafe protocols. Uses Python 3.10+ setting ssl.TLSVersion.... # context.minimum_version = ssl.TLSVersion.TLSv1_2 # for previous Python versions 3.6+ maybe: # ssl.PROTOCOL_TLSv1_2 # would work? context.load_cert_chain \ ( keyfile = self.args.keyfile , certfile = self.args.certfile ) httpd.socket = context.wrap_socket \ (httpd.socket, server_side = True) while not self.request_received: httpd.handle_request () # end def https_server # end class Request_Token epilog = """\ Retrieving the necessary refresh_token and access_token credentials using this script. This asumes you have an email account (plus the password) to be used for mail retrieval. And you have registered an application in the cloud for this process. The registering of an application will give you an application id (also called client id) and a tenant in UUID format. First define the necessary TENANT variable: TENANT=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX You need to create a directory named 'oauth' (if not yet existing) and put the client id (also called application id) into the file 'oauth/client_id' and the corresponding secret into the file 'oauth/client_secret'. By default calling the script with no arguments, the whole process is automatic. Note that the default TLS key used for the built-in server is a self-signed certificate which is automatically created on Debian-based (including Ubuntu) Linux distributions. But the key-file is not readable for everyone, you need to be in the group 'ssl-cert' or need otherwise elevated privileges. If you're using a http (as opposed to https) redirect URI, of course no TLS files are needed. You may want to specify the tenant explicitly using: ./oauth-get-token.py -t $TENANT Specifying the tenant explicitly will select the customized company login form directly. The automatic process works as follows: - First the authorization URL is constructed and pushed to a local browser. By default the default browser on that machine is used, you can specify a different browser with the -b/--browser option. This will show a login form where you should be able to select the user to log in with. Log in with the username (the email address) and password for that user. - A web-server is started on the given port. When you fill out the authentication form pushed to the browser, the last step is a redirect to an URL that calls back to this webserver. The necessary authentication code is transmitted in a query parameter. The code is stored into the file 'oauth/authcode'. Using the authcode, the refresh_token and access_token are requested and stored in the oauth directory. These steps can be broken down into individual steps by options disabling one of the steps: - The push to the webserver can be disabled with the option -w/--dont-push-to-webbrowser -- in that case the URL is printed on standard output and must be pasted into the URL input field of a browser. It is typically a good idea to use a browser that is currently not logged into the company network. - The start of the webserver can be disabled with the option -s/--dont-run-https-server -- when called with that option no webserver is started. You get a redirect to a non-existing page. The error-message is something like: This site can’t be reached Copy the URL from the browser into the file 'oauth/authcode'. The URL has paramters. We're interested in the 'code' parameter, a very long string. Edit the file so that only that string (without the 'code=' part) is in the file. - Requesting the tokens can be disabled with the option -n/--dont-request-tokens -- if this option is given, after receiving the redirect from the webserver the authentication code is written to the file 'oauth/authcode' but no token request is started. If you have either disabled the webserver or the token request, the token can be requested (using the file 'oauth/authcode' constructed by hand as described above or written by the webserver) with the -T/--request-token option: ./oauth-get-token.py [-t $TENANT] -T If successful this will create the 'oauth/access_token' and 'oauth/refresh_token' files. Note that the authentication code has a limited lifetime. """ def main (): cmd = ArgumentParser \ (epilog=epilog, formatter_class=RawDescriptionHelpFormatter) cmd.add_argument \ ( '-b', '--browser' , help = "Use non-default browser" ) cmd.add_argument \ ( '--certfile' , help = "TLS certificate file, default=%(default)s" , default = "/etc/ssl/certs/ssl-cert-snakeoil.pem" ) cmd.add_argument \ ( '--keyfile' , help = "TLS key file, default=%(default)s" , default = "/etc/ssl/private/ssl-cert-snakeoil.key" ) cmd.add_argument \ ( '-n', '--dont-request-tokens' , dest = 'request_tokens' , help = "Do not request tokens, just write authcode" , action = 'store_false' , default = True ) cmd.add_argument \ ( '-p', '--https-server-port' , type = int , help = "Port for https server to listen, default=%(default)s" " see also -r option, ports must (usually) match." , default = 8181 ) cmd.add_argument \ ( '-r', '--redirect-uri' , help = "Redirect URI, default=%(default)s" , default = 'https://localhost:8181' ) cmd.add_argument \ ( '-s', '--dont-run-https-server' , dest = 'run_https_server' , help = "Run https server to wait for connection of browser " "to transmit auth code via GET request" , action = 'store_false' , default = True ) cmd.add_argument \ ( '-T', '--request-token' , help = "Run only the token-request step" , action = 'store_true' ) cmd.add_argument \ ( '-t', '--tenant' , help = "Tenant part of url, default=%(default)s" , default = 'organizations' ) cmd.add_argument \ ( '--use-tls' , help = "Enforce use of TLS even if the redirect uri is http" , action = 'store_true' , default = None ) cmd.add_argument \ ( '--no-use-tls', '--dont-use-tls' , help = "Disable use of TLS even if the redirect uri is https" , dest = 'use_tls' , action = 'store_false' , default = None ) cmd.add_argument \ ( '-u', '--url' , help = "Base url for requests, default=%(default)s" , default = 'https://login.microsoftonline.com' ) cmd.add_argument \ ( '-w', '--dont-push-to-webbrowser' , dest = 'webbrowser' , help = "Do not push authcode url into the browser" , action = 'store_false' , default = True ) args = cmd.parse_args () rt = Request_Token (args) if args.request_token: rt.request_token () else: rt.request_authcode () # end def main if __name__ == '__main__': main () code-53afc3ffd4a2/scripts/roundup-reminder0000755000000000000000000001457014725324451017021 0ustar 00000000000000#! /usr/bin/env python # Copyright (c) 2002 ekit.com Inc (http://www.ekit-inc.com/) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ''' Simple script that emails all users of a tracker with the issues that are currently assigned to them. TODO: introduce some structure ;) TODO: possibly make this more general and configurable... ''' from __future__ import print_function import sys from email.mime.multipart import MIMEMultipart from email.utils import make_msgid from roundup import date, instance from roundup.mailer import Mailer # open the instance if len(sys.argv) != 2: print('You need to specify an instance home dir') sys.exit() instance_home = sys.argv[1] instance = instance.open(instance_home) db = instance.open('admin') resolved_id = db.status.lookup('resolved') class Reverse: """Class reversing sort order.""" def __init__(self, val): self.val = val def __lt__(self, other): return other.val < self.val def __le__(self, other): return other.val <= self.val def __eq__(self, other): return other.val == self.val def __ne__(self, other): return other.val != self.val def __gt__(self, other): return other.val > self.val def __ge__(self, other): return other.val >= self.val def listKey(x): "key for tuples such that order is positive on [0] and negative on [1]" return (x[0], Reverse(x[1])) return 0 # loop through all the users for user_id in db.user.list(): # make sure we care about this user name = db.user.get(user_id, 'realname') if name is None: name = db.user.get(user_id, 'username') address = db.user.get(user_id, 'address') if address is None: continue # extract this user's issues issues = [] for issue_id in db.issue.find(assignedto=user_id): if db.issue.get(issue_id, 'status') == resolved_id: continue order = db.priority.get(db.issue.get(issue_id, 'priority'), 'order') issues.append((order, db.issue.get(issue_id, 'activity'), db.issue.get(issue_id, 'creation'), issue_id)) # sort the issues by timeliness and creation date issues.sort(key=listKey) if not issues: continue # generate the email message mailer = Mailer(db.config) message = MIMEMultipart('alternative') mailer.set_message_attributes( message, [address], 'Your active %s issues'%db.config.TRACKER_NAME) message['Reply-To'] = '%s <%s>'%(db.config.TRACKER_NAME, db.config.ADMIN_EMAIL) message['Message-Id'] = make_msgid() # do the plain text bit text_lines = [] text_lines.append('Created ID Activity Title') text_lines.append('='*75) # '2 months 213 immediate cc_daemon barfage old_priority = None for _priority_order, activity_date, creation_date, issue_id in issues: priority = db.issue.get(issue_id, 'priority') if (priority != old_priority): old_priority = priority text_lines.append(' ' + db.priority.get(priority,'name')) # pretty creation creation = (creation_date - date.Date('.')).pretty() activity = (activity_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') if len(title) > 42: title = title[:38] + ' ...' text_lines.append('%-11s %-4s %-9s %-42s'%(creation, issue_id, activity, title)) # some help to finish off text_lines.append(''' To view or respond to any of the issues listed above, visit the URL %s and click on "My Issues". Do NOT respond to this message. '''%db.config.TRACKER_WEB) text = '\n'.join(text_lines) + '\n' part = mailer.get_text_message() part.set_payload(text, part.get_charset()) message.attach(part) # now the HTML one html_lines = [] colours = { 'immediate': ' bgcolor="#ffcdcd"', 'day': ' bgcolor="#ffdecd"', 'week': ' bgcolor="#ffeecd"', 'month': ' bgcolor="#ffffcd"', 'whenever': ' bgcolor="#ffffff"', } html_lines.append(''' ''') old_priority = None for _priority_order, activity_date, creation_date, issue_id in issues: priority = db.issue.get(issue_id,'priority') if (priority != old_priority): old_priority = priority html_lines.append(''%db.priority.get(priority,'name')) creation = (creation_date - date.Date('.')).pretty() title = db.issue.get(issue_id, 'title') issue_id = '%s'%(db.config.TRACKER_WEB, issue_id, issue_id) activity = (activity_date - date.Date('.')).pretty() html_lines.append(''''''%(creation, issue_id, activity, title)) html_lines.append('
Created ID Activity Title
->->->%s
%s%s%s %s
') html_lines.append('''

To view or respond to any of the issues listed above, simply click on the issue ID. Do not respond to this message.

''') html = '\n'.join(html_lines) + '\n' part = mailer.get_text_message('utf-8', 'html') part.set_payload(html, part.get_charset()) message.attach(part) # all done, send! mailer.smtp_send([address], message.as_string()) # vim: set filetype=python ts=4 sw=4 et si code-53afc3ffd4a2/scripts/roundup.rc-debian0000644000000000000000000000177514725324451017041 0ustar 00000000000000#!/bin/sh -e # # roundup Startup script for the roundup http server. DESC='Roundup HTTP-Server' BINFILE=roundup-server EXECUTABLE=/usr/local/bin/$BINFILE PIDFILE=/var/run/roundup/server.pid LOGFILE=/var/log/roundup/roundup.log TRACKERS=tttech=/tttech/org/software/roundup/tttech/ OPTIONS="-- -p 8080 -u roundup -d $PIDFILE -l $LOGFILE $TRACKERS" test -x $EXECUTABLE || exit 0 start_stop() { case "$1" in start) printf "Starting $DESC:" start-stop-daemon --start --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; stop) printf "Stopping $DESC:" start-stop-daemon --stop --oknodo --quiet \ --pidfile $PIDFILE \ --exec $EXECUTABLE $OPTIONS printf " $BINFILE" printf ".\n" ;; restart | force-reload) start_stop stop sleep 1 start_stop start ;; *) printf "Usage: $0 {start|stop|restart|force-reload}\n" >&2 exit 1 ;; esac } start_stop "$@" exit 0 code-53afc3ffd4a2/scripts/schema-dump.py0000755000000000000000000001174414725324451016354 0ustar 00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Use recently documented XML-RPC API to dump Roundup data schema in human readable form. Works with demo tracker using: http://admin:admin@localhost:8917/demo/xmlrpc Future development may cover: [ ] unreadable dump formats [ ] access to local database [ ] lossless dump/restore cycle [ ] data dump and filtering with preserved Works in Python 2 as well. """ from __future__ import print_function __license__ = "Public Domain" __version__ = "1.1" __authors__ = [ "anatoly techtonik " "John Rouillard ", ] import os import pprint import sys import textwrap try: import urllib.parse as url_parser # python 3 except ImportError: import urlparse as url_parser # python 2 from argparse import ArgumentParser from roundup.anypy import xmlrpc_ sname = os.path.basename(sys.argv[0]) usage = """\ usage: %s [options] URL URL is XML-RPC endpoint for your tracker, such as: http://localhost:8917/demo/xmlrpc options: --pprint (default) --json --yaml --raw -h --help --version """ % sname def format_pprint(var): return pprint.pformat(var) def format_json(var): jout = pprint.pformat(var) jout = jout.replace('"', "\\'") # " to \' jout = jout.replace("'", '"') # ' to " jout = jout.replace('\\"', "'") # \" to ' return jout def format_yaml(var): out = pprint.pformat(var) out = out.replace('{', ' ') out = out.replace('}', '') out = textwrap.dedent(out) out = out.replace("'", '') out = out.replace(' [[', '\n [') out = out.replace(']]', ']') out = out.replace('],', '') out = out.replace(']', '') out2 = [] for line in out.splitlines(): if '[' in line: line = ' ' + line.lstrip(' [') line = line.replace('>', '') line = line.replace('roundup.hyperdb.', '') # expandtabs(16) with limit=1 n, v = line.split(', <') if len(n) > 14: indent = 0 else: indent = 14 - len(n) line = line.replace(', <', ': '+' '*indent) line.split(",") out2.append(line) out = '\n'.join(out2) return out class SpecialTransport(): """Mixin for http/https transports to implement new send_content with CSRF prevention headers to both of them. """ def send_content(self, connection, request_body): connection.putheader("Referer", "%s://%s%s%s/" % ( self.components.scheme, self.components.hostname, ':' + str(self.components.port) if self.components.port else '', self.components.path)) connection.putheader("Origin", "%s://%s%s" % ( self.components.scheme, self.components.hostname, ':' + str(self.components.port) if self.components.port else '')) connection.putheader("X-Requested-With", "XMLHttpRequest") connection.putheader("Content-Type", "text/xml") connection.putheader("Content-Length", str(len(request_body))) connection.endheaders() if request_body: connection.send(request_body) class SpecialHttpTransport(SpecialTransport, xmlrpc_.client.Transport, object): """SpecialTransport must be first to use its send_content. Explicit object inheritance required for python2 apparently.""" def __init__(self, url): self.components = url_parser.urlparse(url) # works both python2 (with object inheritance) and python3 super(SpecialHttpTransport, self).__init__(self) class SpecialHttpsTransport(SpecialTransport, xmlrpc_.client.SafeTransport, object): """SpecialTransport must be first to use its send_content. Explicit object inheritance required for python2 apparently.""" def __init__(self, url): self.components = url_parser.urlparse(url) # works both python2 (with object inheritance) and python3 super(SpecialHttpsTransport, self).__init__(self) if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("url", nargs=1) parser.add_argument("--raw", action='store_true') parser.add_argument("--yaml", action='store_true') parser.add_argument("--json", action='store_true') parser.add_argument("-v", "--version", action='store_true') args = parser.parse_args() if args.version: sys.exit(sname + " " + __version__) if args.url[0].lower().startswith('https:'): transport = SpecialHttpsTransport else: transport = SpecialHttpTransport roundup_server = xmlrpc_.client.ServerProxy( args.url[0], transport=transport(args.url[0]), verbose=False, allow_none=True) schema = roundup_server.schema() if args.raw: print(str(schema)) elif args.yaml: print(format_yaml(schema)) elif args.json: print(format_json(schema)) else: print(format_pprint(schema)) print("") code-53afc3ffd4a2/scripts/schema_diagram.py0000644000000000000000000000307114725324451017064 0ustar 00000000000000#! /usr/bin/python # # Schema diagram generator contributed by Stefan Seefeld of the fresco # project http://www.fresco.org/. # # It generates a 'dot file' that is then fed into the 'dot' # tool (http://www.graphviz.org) to generate a graph: # # %> ./schema.py # %> dot -Tps schema.dot -o schema.ps # %> gv schema.ps # from __future__ import print_function import sys import roundup.instance # open the instance instance = roundup.instance.open(sys.argv[1]) db = instance.open() # diagram preamble print('digraph schema {') print('size="8,6"') print('node [shape="record" bgcolor="#ffe4c4" style=filled]') print('edge [taillabel="1" headlabel="1" dir=back arrowtail=ediamond]') # get all the classes types = list(db.classes.keys()) # one record node per class for i in range(len(types)): print('node%d [label=\"{%s|}"]'%(i, types[i])) # now draw in the relations for name in db.classes.keys(): type = db.classes[name] attributes = type.getprops() for a in attributes.keys(): attribute = attributes[a] if isinstance(attribute, roundup.hyperdb.Link): print('node%d -> node%d [label=%s]'%(types.index(name), types.index(attribute.classname), a)) elif isinstance(attribute, roundup.hyperdb.Multilink): print('node%d -> node%d [taillabel="*" label=%s]'%(types.index(name), types.index(attribute.classname), a)) # all done print('}') code-53afc3ffd4a2/scripts/server-ctl0000755000000000000000000000352214725324451015603 0ustar 00000000000000#!/bin/sh # # Configuration # CONFFILE="/var/roundup/server-config.ini" # this will end up with extra space, but it should be ignored in the script PIDFILE=`grep '^pidfile' ${CONFFILE} | awk -F = '{print $2}' ` SERVER="/usr/local/bin/roundup-server -C ${CONFFILE}" ERROR=0 ARGV="$@" if [ "x$ARGV" = "x" ] ; then ARGS="help" fi if [ -z "${PIDFILE}" ] ; then echo "pidfile option must be set in configuration file" exit 1 fi for ARG in $@ $ARGS do # check for pidfile if [ -f $PIDFILE ] ; then PID=`cat $PIDFILE` if [ "x$PID" != "x" ] && kill -0 $PID 2>/dev/null ; then STATUS="roundup-server (pid $PID) running" RUNNING=1 else STATUS="roundup-server (pid $PID?) not running" RUNNING=0 fi else STATUS="roundup-server (no pid file) not running" RUNNING=0 fi case $ARG in start) if [ $RUNNING -eq 1 ] ; then echo "$0 $ARG: roundup-server (pid $PID) already running" continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; condstart) if [ $RUNNING -eq 1 ] ; then continue fi if $SERVER ; then echo "$0 $ARG: roundup-server started" else echo "$0 $ARG: roundup-server could not be started" ERROR=1 fi ;; stop) if [ $RUNNING -eq 0 ] ; then echo "$0 $ARG: $STATUS" continue fi if kill $PID ; then echo "$0 $ARG: roundup-server stopped" else echo "$0 $ARG: roundup-server could not be stopped" ERROR=2 fi ;; status) echo $STATUS ;; *) echo "usage: $0 (start|condstart|stop|status)" cat < propname # the class is the name of the class (e.g. issue_type, keyword -- # also used in e.g. in https://bugs.python.org/keyword) # the propname is the name used as attribute on the issue class # (e.g. issue.type, issue.keywords) classes = { # 'status': 'status', # skip this 'issue_type': 'type', 'stage': 'stage', 'component': 'components', 'version': 'versions', 'resolution': 'resolution', 'priority': 'priority', 'keyword': 'keywords', } # find the id for the 'open' status open_id = roundup.lookup('status', 'open') print(f'* Counting total issues...') total_issues_num = len(roundup.filter('issue', None, {})) print(f'* Counting open issues...') # use this list to filter only the open issues open_issues = roundup.filter('issue', None, {'status': open_id}) open_issues_num = len(open_issues) # save the totals in a dict with this structure: # totals[propname][open/all][num/perc][name] # where propname is e.g. 'keyword' and name is e.g. 'easy' totals = defaultdict(lambda: {'all': {'perc': {}, 'num': {}}, 'open': {'perc': {}, 'num': {}}}) for cls, propname in classes.items(): print(f'* Counting <{cls}>...') # get the list of ids/names for the given class (e.g. 'easy' is 6) ids = roundup.list(cls, 'id') names = roundup.list(cls, 'name') for id, name in zip(ids, names): # filter and count on *all* issues with the given propname tot_all = len(roundup.filter('issue', None, {propname: id})) totals[propname]['all']['num'][name] = tot_all totals[propname]['all']['perc'][name] = tot_all / total_issues_num # filter and count on *open* issues with the given propname tot_open = len(roundup.filter('issue', open_issues, {propname: id})) totals[propname]['open']['num'][name] = tot_open totals[propname]['open']['perc'][name] = tot_open / open_issues_num print(f'Issues (open/all): {open_issues_num}/{total_issues_num}') # print a list of markdown tables for each bpo class name for propname in classes.values(): print(f'### {propname}') print('| bpo field | open | all |') print('| :--- | ---: | ---: |') # pick the dict for the given propname (e.g. keywords) proptots = totals[propname] names = proptots['open']['num'] # sort the names (e.g. 'easy') in reverse order # based on the number of open issues for name in sorted(names, key=names.get, reverse=True): # get and print num/perc for all/open issues issues_all = proptots['all']['num'][name] issues_open = proptots['open']['num'][name] perc_all = proptots['all']['perc'][name] perc_open = proptots['open']['perc'][name] print(f'| {name:20} | {issues_open:>5} ({perc_open:5.1%}) |' f' {issues_all:>5} ({perc_all:5.1%}) |') # calc and print num/perc for all/open issues tot_issues_all = sum(proptots['all']['num'].values()) tot_issues_open = sum(proptots['open']['num'].values()) tot_perc_all = sum(proptots['all']['perc'].values()) tot_perc_open = sum(proptots['open']['perc'].values()) print(f'| **Total** | {tot_issues_open:>5} ({tot_perc_open:5.1%}) |' f' {tot_issues_all:>5} ({tot_perc_all:5.1%}) |') code-53afc3ffd4a2/scripts/systemd.gunicorn0000644000000000000000000000101114725324451017014 0ustar 00000000000000[Unit] Description=Roundup Server - roundup [Service] User=roundup WorkingDirectory=/path/to/roundup/trackers/roundup SyslogIdentifier=roundup-server-roundup ExecStart=/path/to/gunicorn -b unix:///var/run/roundup/roundup.sock wsgi -w 4 --preload --max-requests 128 --max-requests-jitter 32 --timeout 60 --graceful-timeout 60 --access-logfile - --error-logfile ExecReload=/bin/kill -HUP $MAINPID ExecStop = /bin/kill -s TERM $MAINPID KillMode=mixed Restart=on-failure RestartSec=15s [Install] WantedBy=multi-user.target code-53afc3ffd4a2/scripts/weekly-report0000755000000000000000000000646314725324451016335 0ustar 00000000000000#! /usr/bin/env python # This script generates a simple report outlining the activity in one # tracker for the most recent week. # A second argument is the negative interval to change period # of time. # This script is free software, you may redistribute it # and/or modify under the same terms as Python. #Example output #CREATED: #2702: new item # #RESOLVED: #1995: Where is my Power plugs #2501: Can you help me with Sanity #459: I need Sanity # #TOP TEN MOST DISCUSSED: #2 - 491: Can you help me with Sanity #1 - 1995: Where is my Power plugs from __future__ import print_function import sys from roundup import date, instance # position for arguments tracker_home_pos = 1 optional_interval_pos = 2 # gather args arg_len = len(sys.argv) # map pos to length by adding 1. if arg_len not in [tracker_home_pos + 1, optional_interval_pos + 1]: print('Usage: %s tracker-home [interval -1w]' % sys.argv[0]) if (arg_len < tracker_home_pos + 1 ): print(' You need to specify a tracker home directory') sys.exit(1) instance_home = sys.argv[tracker_home_pos] lookback_interval = sys.argv[optional_interval_pos] if \ len(sys.argv) == optional_interval_pos + 1 else '-1w' # open the instance instance = instance.open(instance_home) db = instance.open('admin') old = date.Date(lookback_interval) created = [] # [issue_id_created_issue] summary = {} # {status_id: [issue_ids in that status]} messages = [] # [(number_of_messages, issue_id)] # loop through all the recently-active issues for issue_id in db.issue.filter(None, {'activity': '-1w;'}): message_count = 0 for _x,ts,_userid,action,data in db.issue.history(issue_id): if ts < old: # history occurred before our current window continue if action == 'create': created.append(issue_id) elif action == 'set' and 'messages' in data: message_count += 1 summary.setdefault(db.issue.get(issue_id, 'status'), []).append(issue_id) messages.append((message_count, issue_id)) #print('STATUS SUMMARY:') #for k,v in summary.items(): # print(k, len(v)) print('\nCREATED:') if created: print('\n'.join(['%s: %s'%(itemid, db.issue.get(itemid, 'title')) for itemid in created])) else: print("No issue created in interval %s" % lookback_interval) print('\nRESOLVED:') resolved_id = db.status.lookup('resolved') if summary: # deduplicate - duplicates happen when issue with resolved status # has multiple history items (e.g. message or other # change after resolution) resolved_ids = sorted(set(summary.get(resolved_id, [])), key=int) print('\n'.join(['%s: %s' % (itemid, db.issue.get(itemid, 'title')) for itemid in resolved_ids ])) else: print("No issue resolved in interval %s" % lookback_interval) print('\nTOP TEN MOST DISCUSSED:') # filter out issues with no messages messages = [ message for message in messages if message[0] > 0 ] if messages: messages.sort() messages.reverse() nmax = messages[0][0] or 1 fmt = '%%%dd - %%s: %%s'%(len(str(nmax))) print('\n'.join([fmt%(num, itemid, db.issue.get(itemid, 'title')) for num, itemid in messages[:10]])) else: print("No issues discussed in interval %s" % lookback_interval) # vim: set filetype=python ts=4 sw=4 et si