wip: trying to make functional test startup more reliable

JohnVillalovos · JohnVillalovos · commit 0a2f2a78bd94 · 2022-07-26T09:29:41.000-07:00
The functional tests have been erratic. Current theory is that we are
starting the tests before the GitLab container is fully up and
running.
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -6,11 +6,14 @@
 from subprocess import check_output
 
 import pytest
+import requests
 
 import gitlab
 import gitlab.base
 from tests.functional import helpers
 
+SLEEP_TIME = 20
+
 
 @pytest.fixture(scope="session")
 def fixture_dir(test_dir):
@@ -129,15 +132,30 @@ def check_is_alive():
     Return a healthcheck function fixture for the GitLab container spinup.
     """
 
-    def _check(container: str, start_time: float) -> bool:
+    def _check(
+        *, container: str, start_time: float, docker_ip: str, docker_port: int
+    ) -> bool:
         setup_time = time.perf_counter() - start_time
         minutes, seconds = int(setup_time / 60), int(setup_time % 60)
         logging.info(
             f"Checking if GitLab container is up. "
             f"Have been checking for {minutes} minute(s), {seconds} seconds ..."
         )
         logs = ["docker", "logs", container]
-        return "gitlab Reconfigured!" in check_output(logs).decode()
+        if "gitlab Reconfigured!" not in check_output(logs).decode():
+            return False
+        logging.info("GitLab has finished reconfiguring.")
+        gitlab_url = f"http://{docker_ip}:{docker_port}"
+        for check in ("health", "readiness", "liveness"):
+            url = f"{gitlab_url}/-/{check}"
+            logging.info(f"Checking {check!r} endpoint at: {url}")
+            result = requests.get(url)
+            if result.status_code != 200:
+                logging.info(f"{check!r} check did not return 200: {result}")
+                return False
+        logging.info(f"Sleeping for {SLEEP_TIME}")
+        time.sleep(SLEEP_TIME)
+        return True
 
     return _check
 
@@ -176,7 +194,12 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
     docker_services.wait_until_responsive(
         timeout=300,
         pause=10,
-        check=lambda: check_is_alive("gitlab-test", start_time=start_time),
+        check=lambda: check_is_alive(
+            container="gitlab-test",
+            start_time=start_time,
+            docker_ip=docker_ip,
+            docker_port=port,
+        ),
     )
     setup_time = time.perf_counter() - start_time
     minutes, seconds = int(setup_time / 60), int(setup_time % 60)
@@ -208,6 +231,7 @@ def gl(gitlab_config):
     logging.info("Instantiating python-gitlab gitlab.Gitlab instance")
     instance = gitlab.Gitlab.from_config("local", [gitlab_config])
 
+    logging.info("Reset GitLab")
     reset_gitlab(instance)
 
     return instance
diff --git a/tests/functional/fixtures/docker-compose.yml b/tests/functional/fixtures/docker-compose.yml
@@ -32,6 +32,7 @@ services:
         grafana['enable'] = false
         letsencrypt['enable'] = false
         gitlab_rails['initial_license_file'] = '/python-gitlab-ci.gitlab-license'
+        gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']
     entrypoint:
       - /bin/sh
       - -c
