chore: improve and document testing against different images

nejch · nejch · commit 98d3f770c4cc · 2020-03-21T19:12:46.000-04:00
diff --git a/.travis.yml b/.travis.yml
@@ -39,6 +39,22 @@ jobs:
       script:
         - pip3 install tox
         - tox -e py_func_v4
+    - stage: test
+      name: cli_func_nightly
+      dist: bionic
+      python: 3.8
+      env: GITLAB_TAG=nightly
+      script:
+        - pip3 install tox
+        - tox -e cli_func_v4
+    - stage: test
+      name: py_func_nightly
+      dist: bionic
+      python: 3.8
+      env: GITLAB_TAG=nightly
+      script:
+        - pip3 install tox
+        - tox -e py_func_v4
     - stage: test
       name: docs
       dist: bionic
@@ -67,3 +83,5 @@ jobs:
       script:
         - pip3 install tox
         - tox -e py38
+  allow_failures:
+    - env: IMAGE_TAG=nightly
diff --git a/README.rst b/README.rst
@@ -156,6 +156,26 @@ To run these tests:
    # run the python API tests:
    ./tools/py_functional_tests.sh
 
+By default, the tests run against the ``gitlab/gitlab-ce:latest`` image. You can
+override both the image and tag with the ``-i`` and ``-t`` options, or by providing
+either the ``GITLAB_IMAGE`` or ``GITLAB_TAG`` environment variables.
+
+This way you can run tests against different versions, such as ``nightly`` for
+features in an upcoming release, or an older release (e.g. ``12.8.0-ce.0``).
+The tag must match an exact tag on Docker Hub:
+
+.. code-block:: bash
+
+   # run tests against `nightly` or specific tag
+   ./tools/py_functional_tests.sh -t nightly
+   ./tools/py_functional_tests.sh -t 12.8.0-ce.0
+
+   # run tests against the latest gitlab EE image
+   ./tools/py_functional_tests.sh -i gitlab/gitlab-ee
+
+   # override tags with environment variables
+   GITLAB_TAG=nightly ./tools/py_functional_tests.sh
+
 You can also build a test environment using the following command:
 
 .. code-block:: bash
diff --git a/tools/build_test_env.sh b/tools/build_test_env.sh
@@ -28,14 +28,15 @@ try() { "$@" || fatal "'$@' failed"; }
 REUSE_CONTAINER=
 NOVENV=
 API_VER=4
-GITLAB_IMAGE="gitlab/gitlab-ce"
-GITLAB_TAG="latest"
+GITLAB_IMAGE="${GITLAB_IMAGE:-gitlab/gitlab-ce}"
+GITLAB_TAG="${GITLAB_TAG:-latest}"
 VENV_CMD="python3 -m venv"
-while getopts :knp:a: opt "$@"; do
+while getopts :knp:a:i:t: opt "$@"; do
     case $opt in
         k) REUSE_CONTAINER=1;;
         n) NOVENV=1;;
         a) API_VER=$OPTARG;;
+        i) GITLAB_IMAGE=$OPTARG;;
         t) GITLAB_TAG=$OPTARG;;
         :) fatal "Option -${OPTARG} requires a value";;
         '?') fatal "Unknown option: -${OPTARG}";;
