test: attempt to make functional test startup more reliable

JohnVillalovos · JohnVillalovos · commit 67508e8100be · 2022-07-29T08:39:42.000-07:00
The functional tests have been erratic. Current theory is that we are starting the tests before the GitLab container is fully up and running. * Add checking of the Health Check[1] endpoints. * Add a 20 second delay after we believe it is up and running. * Increase timeout from 300 to 400 seconds [1] https://docs.gitlab.com/ee/user/admin_area/monitoring/health_check.html
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -1,22 +1,48 @@
 import logging
+import pathlib
 import tempfile
 import time
 import uuid
-from pathlib import Path
 from subprocess import check_output
 
 import pytest
+import requests
 
 import gitlab
 import gitlab.base
 from tests.functional import helpers
 
+SLEEP_TIME = 10
+
 
 @pytest.fixture(scope="session")
-def fixture_dir(test_dir):
+def fixture_dir(test_dir) -> pathlib.Path:
     return test_dir / "functional" / "fixtures"
 
 
+@pytest.fixture(scope="session")
+def gitlab_service_name() -> str:
+    """The "service" name is the one defined in the `docker-compose.yml` file"""
+    return "gitlab"
+
+
+@pytest.fixture(scope="session")
+def gitlab_container_name() -> str:
+    """The "container" name is the one defined in the `docker-compose.yml` file
+    for the "gitlab" service"""
+    return "gitlab-test"
+
+
+@pytest.fixture(scope="session")
+def gitlab_docker_port(docker_services, gitlab_service_name: str) -> int:
+    return docker_services.port_for(service=gitlab_service_name, container_port=80)
+
+
+@pytest.fixture(scope="session")
+def gitlab_url(docker_ip: str, gitlab_docker_port: int) -> str:
+    return f"http://{docker_ip}:{gitlab_docker_port}"
+
+
 def reset_gitlab(gl: gitlab.Gitlab) -> None:
     """Delete resources (such as projects, groups, users) that shouldn't
     exist."""
@@ -99,8 +125,8 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope="session")
-def temp_dir():
-    return Path(tempfile.gettempdir())
+def temp_dir() -> pathlib.Path:
+    return pathlib.Path(tempfile.gettempdir())
 
 
 @pytest.fixture(scope="session")
@@ -129,15 +155,37 @@ def check_is_alive():
     Return a healthcheck function fixture for the GitLab container spinup.
     """
 
-    def _check(container: str, start_time: float) -> bool:
+    def _check(
+        *,
+        container: str,
+        start_time: float,
+        gitlab_url: str,
+    ) -> bool:
         setup_time = time.perf_counter() - start_time
         minutes, seconds = int(setup_time / 60), int(setup_time % 60)
         logging.info(
             f"Checking if GitLab container is up. "
             f"Have been checking for {minutes} minute(s), {seconds} seconds ..."
         )
         logs = ["docker", "logs", container]
-        return "gitlab Reconfigured!" in check_output(logs).decode()
+        if "gitlab Reconfigured!" not in check_output(logs).decode():
+            return False
+        logging.debug("GitLab has finished reconfiguring.")
+        for check in ("health", "readiness", "liveness"):
+            url = f"{gitlab_url}/-/{check}"
+            logging.debug(f"Checking {check!r} endpoint at: {url}")
+            try:
+                result = requests.get(url, timeout=1.0)
+            except requests.exceptions.Timeout:
+                logging.info(f"{check!r} check timed out")
+                return False
+            if result.status_code != 200:
+                logging.info(f"{check!r} check did not return 200: {result!r}")
+                return False
+            logging.debug(f"{check!r} check passed: {result!r}")
+        logging.debug(f"Sleeping for {SLEEP_TIME}")
+        time.sleep(SLEEP_TIME)
+        return True
 
     return _check
 
@@ -167,31 +215,41 @@ def _wait(timeout=30, step=0.5):
 
 
 @pytest.fixture(scope="session")
-def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_dir):
+def gitlab_config(
+    check_is_alive,
+    gitlab_container_name: str,
+    gitlab_url: str,
+    docker_services,
+    temp_dir: pathlib.Path,
+    fixture_dir: pathlib.Path,
+):
     config_file = temp_dir / "python-gitlab.cfg"
-    port = docker_services.port_for("gitlab", 80)
 
     start_time = time.perf_counter()
     logging.info("Waiting for GitLab container to become ready.")
     docker_services.wait_until_responsive(
         timeout=300,
         pause=10,
-        check=lambda: check_is_alive("gitlab-test", start_time=start_time),
+        check=lambda: check_is_alive(
+            container=gitlab_container_name,
+            start_time=start_time,
+            gitlab_url=gitlab_url,
+        ),
     )
     setup_time = time.perf_counter() - start_time
     minutes, seconds = int(setup_time / 60), int(setup_time % 60)
     logging.info(
         f"GitLab container is now ready after {minutes} minute(s), {seconds} seconds"
     )
 
-    token = set_token("gitlab-test", fixture_dir=fixture_dir)
+    token = set_token(gitlab_container_name, fixture_dir=fixture_dir)
 
     config = f"""[global]
 default = local
 timeout = 60
 
 [local]
-url = http://{docker_ip}:{port}
+url = {gitlab_url}
 private_token = {token}
 api_version = 4"""
 
@@ -208,6 +266,7 @@ def gl(gitlab_config):
     logging.info("Instantiating python-gitlab gitlab.Gitlab instance")
     instance = gitlab.Gitlab.from_config("local", [gitlab_config])
 
+    logging.info("Reset GitLab")
     reset_gitlab(instance)
 
     return instance
diff --git a/tests/functional/fixtures/docker-compose.yml b/tests/functional/fixtures/docker-compose.yml
@@ -32,6 +32,7 @@ services:
         grafana['enable'] = false
         letsencrypt['enable'] = false
         gitlab_rails['initial_license_file'] = '/python-gitlab-ci.gitlab-license'
+        gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0']
     entrypoint:
       - /bin/sh
       - -c
