code-e44b65651012/.github/workflows/anchore.yml0000644000000000000000000000444714643240751017262 0ustar 00000000000000# This workflow uses actions that are not certified by GitHub. # They are provided by a third-party and are governed by # separate terms of service, privacy policy, and support # documentation. # This workflow checks out code, builds an image, performs a container image # vulnerability scan with Anchore's Grype tool, and integrates the results with GitHub Advanced Security # code scanning feature. For more information on the Anchore scan action usage # and parameters, see https://github.com/anchore/scan-action. For more # information on Anchore's container image scanning tool Grype, see # https://github.com/anchore/grype name: Anchore Container Scan on: push: branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above branches: [ "master" ] #schedule: #- cron: '38 21 * * 6' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: Anchore-Build-Scan: if: "!contains(github.event.head_commit.message, 'no-github-ci')" permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status runs-on: ubuntu-latest steps: - name: Checkout the code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Build the Docker image run: docker pull python:3-alpine; docker build . --file scripts/Docker/Dockerfile --tag localbuild/testimage:latest - name: List the Docker image run: docker image ls - name: Run the Anchore scan action itself with GitHub Advanced Security code scanning integration enabled uses: anchore/scan-action@c35e932c2a7c572bfdb1c3dbcdadb2d4fc62418d # 3.6.1 id: scan with: image: "localbuild/testimage:latest" fail-build: true - name: Upload Anchore Scan Report if: always() uses: github/codeql-action/upload-sarif@65c74964a9ed8c44ed9f19d4bbc5757a6a8e9ab9 # v2.16.1 with: sarif_file: ${{ steps.scan.outputs.sarif }} - name: Inspect action SARIF report if: always() run: cat ${{ steps.scan.outputs.sarif }} code-e44b65651012/.github/workflows/build-xapian.yml0000644000000000000000000000767614643240751020227 0ustar 00000000000000 name: build-xapian on: push: # skip if github.ref is 'refs/heads/maint-1.6' # aka github.ref_name of 'maint-1.6' # see https://github.com/orgs/community/discussions/26253 # for mechanism to control matrix based on branch branches: [ "*", '!maint-1.6' ] workflow_dispatch: inputs: debug_enabled: type: boolean description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' required: false default: false # GITHUB_TOKEN only has read repo context. permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: build xapian runs-on: ubuntu-22.04 env: # get colorized pytest output even without a controlling tty PYTEST_ADDOPTS: "--color=yes" # OS: ${{ matrix.os }} PYTHON_VERSION: 3.13 steps: # Checkout the latest code from the repo - name: Checkout source # example directives: # disable step # if: {{ false }} # continue running if step fails # continue-on-error: true uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 # Setup version of Python to use - name: Set Up Python 3.13 uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: 3.13 allow-prereleases: true cache: 'pip' - name: Install build tools - setuptools run: pip install setuptools # Display the Python version being used - name: Display Python and key module versions run: | python -c "import sys; print('python version: ', sys.version)" python -c "import sqlite3; print('sqlite version: ', sqlite3.sqlite_version)" python -c "import setuptools; print('setuptools version: ', setuptools.__version__);" - name: Update pip run: python -m pip install --upgrade pip # https://github.com/mxschmitt/action-tmate # allow remote ssh into the CI container. I need this to debug # some xfail cases - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} timeout-minutes: 60 with: limit-access-to-actor: true - name: Install xapian run: | set -xv sudo apt-get install libxapian-dev # Sphinx required to build the xapian python bindings. Use 1.8.5 on # older python and newest on newer. pip install sphinx XAPIAN_VER="1.4.25"; echo $XAPIAN_VER; cd /tmp curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz cd xapian-bindings-$XAPIAN_VER/ # edit the configure script. # distutils.sysconfig.get_config_vars('SO') doesn't work for # 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. # DISABLED use their script if [[ $PYTHON_VERSION == "X."* ]]; then cp configure configure.FCS; sed -i \ -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ configure; diff -u configure.FCS configure || true; fi ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation make && sudo make install python -c 'import xapian' code-e44b65651012/.github/workflows/ci-test.yml0000644000000000000000000003406214643240751017207 0ustar 00000000000000# merged in python-package.yml workflow # reference docs: # https://blog.deepjyoti30.dev/tests-github-python # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python # https://github.com/pypa/twine/blob/main/.github/workflows/main.yml name: roundup-ci on: push: # skip if github.ref is 'refs/heads/maint-1.6' # aka github.ref_name of 'maint-1.6' # see https://github.com/orgs/community/discussions/26253 # for mechanism to control matrix based on branch branches: [ "*", '!maint-1.6' ] # pull_request: # branches: [ "master" ] schedule: # monthly build/check - cron: '23 17 1 * *' workflow_dispatch: inputs: debug_enabled: type: boolean description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' required: false default: false # GITHUB_TOKEN only has read repo context. permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test: name: CI build test #runs-on: ubuntu-latest # use below if running on multiple OS's. runs-on: ${{ matrix.os }} if: "!contains(github.event.head_commit.message, 'no-github-ci')" strategy: fail-fast: false max-parallel: 4 matrix: # Run in all these versions of Python python-version: # - "2.7" - "3.12" # - 3.6 run via include on ubunto 20.04 # - "3.7" - "3.8" # - "3.9" - "3.10" - "3.11" # use for multiple os or ubuntu versions #os: [ubuntu-latest, macos-latest, windows-latest] # ubuntu latest 22.04 12/2022 os: [ubuntu-latest, ubuntu-20.04] # if the ones above fail. fail the build experimental: [ false ] include: # example: if 3.13 fails the jobs still succeeds # allow-prereleases in setup-python allows alpha/beta # releases to run - python-version: 3.13 os: ubuntu-22.04 experimental: true # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu - python-version: 3.6 os: ubuntu-20.04 exclude: # skip all python versions on 20.04 except explicitly included - os: ubuntu-20.04 # run the finalizer for coveralls even if one or more # experimental matrix runs fail. # moving it above strategy produces unexpected value false # moving it below (here) produces unexpected value ''. # continue-on-error: ${{ matrix.experimental }} env: # get colorized pytest output even without a controlling tty PYTEST_ADDOPTS: "--color=yes" # OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} steps: # Checkout the latest code from the repo - name: Checkout source # example directives: # disable step # if: {{ false }} # continue running if step fails # continue-on-error: true uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 # Setup version of Python to use - name: Set Up Python ${{ matrix.python-version }} uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: ${{ matrix.python-version }} allow-prereleases: true cache: 'pip' - name: Install build tools - setuptools run: pip install setuptools # Display the Python version being used - name: Display Python and key module versions run: | python -c "import sys; print('python version: ', sys.version)" python -c "import sqlite3; print('sqlite version: ', sqlite3.sqlite_version)" python -c "import setuptools; print('setuptools version: ', setuptools.__version__);" - name: Update pip run: python -m pip install --upgrade pip # note pytest-env is not needed for redis password as there is # no password on CI's redis. - name: Install pytest and other packages needed for running tests run: pip install flake8 hypothesis mock pytest pytest-cov requests sphinx-tabs # https://github.com/mxschmitt/action-tmate # allow remote ssh into the CI container. I need this to debug # some xfail cases - name: Setup tmate session uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} timeout-minutes: 10 with: limit-access-to-actor: true - name: run flake8 - abort for syntax error, otherwise warn only run: | # stop the build for Python syntax errors or undefined names # talgettext is a utility function ignore it. flake8 roundup --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude talgettext.py # exit-zero treats all errors as warnings. # The GitHub editor is 127 chars wide flake8 roundup --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics # Install the databases - name: Install mysql/mariadb run: | #set -xv # mysql is pre-installed and active but this is the install command # sudo apt-get install mysql-server mysql-client # set up mysql database sudo sed -i -e '/^\[mysqld\]/,/^\[mysql/s/^#* *max_allowed_packet.*/max_allowed_packet = 500M/' /etc/mysql/mysql.conf.d/mysqld.cnf; sleep 3 #tail -n +0 /etc/mysql/my.cnf /etc/mysql/mysql.conf.d/mysqld.cnf #grep max_allowed /etc/mysql/mysql.conf.d/mysqld.cnf #ls /etc/mysql/conf.d/ /etc/mysql/mysql.conf.d/ #sleep 5 # try to improve performance speed by disabling some ACID # settings and change some layout defaults. sudo sed -i -e '$a\innodb_flush_log_at_trx_commit = 2' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_file_per_table = OFF' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_doublewrite=OFF' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_fast_shutdown=2' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_log_file_size=1048576' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_flush_method=O_DIRECT' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_log_buffer_size=3M' /etc/mysql/mysql.conf.d/mysqld.cnf sudo sed -i -e '$a\innodb_buffer_pool_size=180M' /etc/mysql/mysql.conf.d/mysqld.cnf sleep 3 sudo service mysql restart #sleep 10 #ps -ef | grep mysqld #sudo netstat -anp | grep mysqld sudo mysql -u root -proot -e 'CREATE USER "rounduptest"@"localhost" IDENTIFIED WITH mysql_native_password BY "rounduptest"; GRANT ALL on rounduptest.* TO "rounduptest"@"localhost";' - name: Install postgres run: | sudo apt-get update && sudo apt-get install postgresql # Disable fsync, full page writes for speed, # don't care about data durability when testing sudo sed -i -e '$a\fsync = off' /etc/postgresql/*/*/postgresql.conf sudo sed -i -e '$a\full_page_writes = off' /etc/postgresql/*/*/postgresql.conf sudo sed -i -e '$a\synchronous_commit = off' /etc/postgresql/*/*/postgresql.conf sudo service postgresql restart; sleep 10 # set up postgresql database sudo -u postgres psql -c "CREATE ROLE rounduptest WITH CREATEDB LOGIN PASSWORD 'rounduptest';" -U postgres sudo -u postgres psql -c "CREATE ROLE rounduptest_schema LOGIN PASSWORD 'rounduptest';" -U postgres sudo -u postgres psql -c "CREATE DATABASE rounduptest_schema;" -U postgres sudo -u postgres psql -c "GRANT CREATE ON DATABASE rounduptest_schema TO rounduptest_schema;" -U postgres - name: install redis run: | sudo apt-get install redis pip install redis - name: Install python db libraries run: | pip install mysqlclient if [[ "$PYTHON_VERSION" != "3.13" ]]; then pip install psycopg2; fi - name: Install auxiliary packages run: | sudo apt-get install swig gpgsm libgpgme-dev # pygments for markdown2 to highlight code blocks pip install markdown2 pygments # docutils for ReStructuredText pip install beautifulsoup4 brotli docutils gpg jinja2 \ mistune==0.8.4 pyjwt pytz whoosh - name: Install aux packages that need versions differences # if zstd fails install, keep going with test, don't abort run: | set -xv pip install zstd || true if [[ "$PYTHON_VERSION" != "2."* ]]; then pip install Markdown; fi - name: Install xapian if: matrix.python-version != '3.13' run: | set -xv sudo apt-get install libxapian-dev # Sphinx required to build the xapian python bindings. Use 1.8.5 on # older python and newest on newer. if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi if [[ $PYTHON_VERSION != "3.13" ]]; then XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER; else XAPIAN_VER="1.4.25"; echo $XAPIAN_VER; fi cd /tmp curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz cd xapian-bindings-$XAPIAN_VER/ if [[ $PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi # edit the configure script. # distutils.sysconfig.get_config_vars('SO') doesn't work for # 3.11 or newer. # Change distutils.sysconfig... to just sysconfig and SO # to EXT_SUFFIX to get valid value. if [[ $PYTHON_VERSION == "3."* ]]; then \ cp configure configure.FCS; \ sed -i \ -e '/PYTHON3_SO=/s/distutils\.//g' \ -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \ -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \ -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \ configure; \ diff -u configure.FCS configure || true; \ ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \ fi case "$PYTHON_VERSION" in "3.13") echo skipping xapian build;; *) make && sudo make install; esac - name: Test build roundup and install locale so lang tests work. run: | sudo apt-get install gettext python setup.py build (cd locale; make local_install; ls -lR locale/de/LC_MESSAGES) # Run the tests using pytest with test files in tests directory. - name: Run tests run: | if [[ "$PYTHON_VERSION" != "2."* ]]; then pytest -r a \ --durations=20 \ -W default \ -W "ignore:SelectableGroups:DeprecationWarning" \ -W "ignore:the imp module:DeprecationWarning:gpg.gpgme:15" \ -W "ignore:'U' mode::docutils.io" \ -W "ignore:unclosed:ResourceWarning:roundup.roundup.demo" \ -W "ignore:unclosed file:ResourceWarning:enum" \ -v test/ --cov=roundup if [[ "$PYTHON_VERSION" != "3.6" ]]; then # coverage before 3.6 doesn't support lcov output coverage lcov fi else # python2 case pytest -v -r a --durations=20 test/ --cov=roundup fi - name: Build docs run: python ./setup.py build_doc - name: Upload coverage to Codecov # see: https://github.com/codecov/codecov-action#usage uses: codecov/codecov-action@4fe8c5f003fae66aa5ebb77cfd3e7bfbbda0b6b0 # v3.1.5 with: verbose: true token: ${{ secrets.CODECOV_TOKEN }} - name: Upload coverage to Coveralls # python 2.7 and 3.6 versions of coverage can't produce lcov files. if: matrix.python-version != '2.7' && matrix.python-version != '3.6' uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3 with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: coverage.lcov parallel: run-{{ matrix.python-version }}-{{ matrix.os }} #- name: test docker build current version ubuntu-latest # if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest' # run: | # docker build -t roundup-app-dev -f scripts/Docker/Dockerfile . # mkdir tracker; chmod 777 tracker # docker run -d --rm -p 9017:8080 \ # -v $PWD/tracker:/usr/src/app/tracker \ # roundup-app-dev:latest demo # expect 200 # curl --fail http://localhost:9017/demo/ > /dev/null #- name: test docker build released pip version # run: | # docker build -t roundup-app-rel --build-arg="source=pypi" \ # -f scripts/Docker/Dockerfile . - name: run benchmarks if: "contains(github.event.head_commit.message, 'benchmark')" run: INCI=1 python test/benchmark.py # in parallel build coveralls requires a finish step finish: needs: test runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'no-github-ci')" steps: - name: Coveralls Finished uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3 with: github-token: ${{ secrets.github_token }} parallel-finished: true code-e44b65651012/.github/workflows/codeql-analysis.yml0000644000000000000000000000542314643240751020726 0ustar 00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # ******** NOTE ******** name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '28 17 * * 1' permissions: contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: analyze: name: Analyze runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'no-github-ci')" permissions: contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'javascript', 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@65c74964a9ed8c44ed9f19d4bbc5757a6a8e9ab9 # v2.16.1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@65c74964a9ed8c44ed9f19d4bbc5757a6a8e9ab9 # v2.16.1 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@65c74964a9ed8c44ed9f19d4bbc5757a6a8e9ab9 # v2.16.1 code-e44b65651012/.github/workflows/ossf-scorecard.yml0000644000000000000000000000575614643240751020564 0ustar 00000000000000# This workflow uses actions that are not certified by GitHub. They are provided # by a third-party and are governed by separate terms of service, privacy # policy, and support documentation. name: Scorecard supply-chain security on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection branch_protection_rule: # To guarantee Maintained check is occasionally updated. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained schedule: - cron: '25 21 * * 5' push: branches: [ "master" ] # Declare default permissions as read only. permissions: read-all jobs: analysis: name: Scorecard analysis runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'no-github-ci')" permissions: # Needed to upload the results to code-scanning dashboard. security-events: write # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. # contents: read # actions: read steps: - name: "Checkout code" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: persist-credentials: false - name: "Run analysis" uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.10 with: results_file: results.sarif results_format: sarif # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. # repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers # - Allows the repository to include the Scorecard badge. # - See https://github.com/ossf/scorecard-action#publishing-results. # For private repositories: # - `publish_results` will always be set to `false`, regardless # of the value entered here. publish_results: true # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 with: name: SARIF file path: results.sarif retention-days: 5 # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" uses: github/codeql-action/upload-sarif@65c74964a9ed8c44ed9f19d4bbc5757a6a8e9ab9 # v2.16.1 with: sarif_file: results.sarif