chore: make reset_gitlab() better

JohnVillalovos · JohnVillalovos · commit 3c5f5109a30c · 2021-12-29T09:07:20.000-08:00
Saw issues in the CI where reset_gitlab() would fail. It would fail to
delete the group that is created when GitLab starts up. Extending the
timeout didn't fix the issue.

Changed the code so now it will check if the item is deleted and if it
isn't it will call the delete() method again to ensure that GitLab
knows it should be deleted. Since making this change I have not been
able to reproduce the failure in reset_gitlab().

Also added some logging functionality that can be seen if logging is
turned on in pytest.
diff --git a/pyproject.toml b/pyproject.toml
@@ -87,3 +87,10 @@ disable = [
     "useless-object-inheritance",
 
 ]
+
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "INFO"
+log_cli_format = "%(asctime)s.%(msecs)03d [%(levelname)8s] (%(filename)s:%(funcName)s:L%(lineno)s) %(message)s"
+log_cli_date_format = "%Y-%m-%d %H:%M:%S"
+addopts = "--color=yes"
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -1,3 +1,4 @@
+import logging
 import tempfile
 import time
 import uuid
@@ -10,54 +11,119 @@
 import gitlab.base
 
 SLEEP_INTERVAL = 0.1
-TIMEOUT = 60  # seconds before timeout will occur
+TIMEOUT = 3 * 60  # seconds before timeout will occur
 
 
 @pytest.fixture(scope="session")
 def fixture_dir(test_dir):
     return test_dir / "functional" / "fixtures"
 
 
-def reset_gitlab(gl):
-    # previously tools/reset_gitlab.py
+def reset_gitlab(gl: gitlab.Gitlab) -> None:
+    # Mark our resources for deletion. It takes time for them to actually be deleted
+    # though.
+    _reset_gitlab_delete_resources(gl=gl)
+
+    # Wait for all resources to be deleted
+    _reset_gitlab_wait_deletion_finish(gl=gl)
+
+
+def _reset_gitlab_delete_resources(gl: gitlab.Gitlab) -> None:
+    """Mark for deletion, resources (such as projects, groups, users) that shouldn't
+    exist. Once marked they will still take time to be deleted."""
     for project in gl.projects.list():
+        logging.info(f"Mark for deletion project: {project.name!r}")
         for deploy_token in project.deploytokens.list():
+            logging.info(
+                f"Mark for deletion token: {deploy_token.name!r} in "
+                f"project: {project.name!r}"
+            )
             deploy_token.delete()
         project.delete()
     for group in gl.groups.list():
+        logging.info(f"Mark for deletion group: {group.name!r}")
         for deploy_token in group.deploytokens.list():
+            logging.info(
+                f"Mark for deletion token: {deploy_token.name!r} in "
+                f"group: {group.name!r}"
+            )
             deploy_token.delete()
         group.delete()
     for variable in gl.variables.list():
+        logging.info(f"Mark for deletion variable: {variable.name!r}")
         variable.delete()
     for user in gl.users.list():
         if user.username != "root":
+            logging.info(f"Mark for deletion user: {user.username!r}")
             user.delete(hard_delete=True)
 
+
+def _reset_gitlab_wait_deletion_finish(gl: gitlab.Gitlab) -> None:
+    """Wait for all of our resources to be deleted.
+
+    If anything exists then mark it again for deletion in case initial call to delete
+    didn't work, which has been seen :("""
+
     max_iterations = int(TIMEOUT / SLEEP_INTERVAL)
 
     # Ensure everything has been reset
     start_time = time.perf_counter()
 
     def wait_for_maximum_list_length(
-        rest_manager: gitlab.base.RESTManager, description: str, max_length: int = 0
+        rest_manager: gitlab.base.RESTManager,
+        description: str,
+        max_length: int = 0,
+        should_delete_func=lambda x: True,
+        delete_kwargs={},
     ) -> None:
         """Wait for the list() length to be no greater than expected maximum or fail
         test if timeout is exceeded"""
-        for _ in range(max_iterations):
-            if len(rest_manager.list()) <= max_length:
+        for count in range(max_iterations):
+            items = rest_manager.list()
+            logging.info(
+                f"Iteration: {count}: items in {description}: {[x.name for x in items]}"
+            )
+            for item in items:
+                if should_delete_func(item):
+                    logging.info(
+                        f"Marking again for deletion {description}: {item.name!r}"
+                    )
+                    try:
+                        item.delete(**delete_kwargs)
+                    except gitlab.exceptions.GitlabDeleteError as exc:
+                        logging.info(
+                            f"Already marked for deletion: {item.name!r} {exc}"
+                        )
+            if len(items) <= max_length:
                 break
             time.sleep(SLEEP_INTERVAL)
-        assert len(rest_manager.list()) <= max_length, (
+        items = rest_manager.list()
+        elapsed_time = time.perf_counter() - start_time
+        if len(items) > max_length:
+            logging.error(
+                f"Too many items still remaining and timeout exceeded: {elapsed_time}"
+            )
+        assert len(items) <= max_length, (
             f"Did not delete required items for {description}. "
-            f"Elapsed_time: {time.perf_counter() - start_time}"
+            f"Elapsed_time: {elapsed_time}\n"
+            f"items: {[str(x) for x in items]!r}"
         )
 
     wait_for_maximum_list_length(rest_manager=gl.projects, description="projects")
     wait_for_maximum_list_length(rest_manager=gl.groups, description="groups")
     wait_for_maximum_list_length(rest_manager=gl.variables, description="variables")
+
+    def should_delete_user(user):
+        if user.username == "root":
+            return False
+        return True
+
     wait_for_maximum_list_length(
-        rest_manager=gl.users, description="users", max_length=1
+        rest_manager=gl.users,
+        description="users",
+        max_length=1,
+        should_delete_func=should_delete_user,
+        delete_kwargs={"hard_delete": True},
     )
 
 
@@ -144,7 +210,7 @@ def wait_for_sidekiq(gl):
     """
 
     def _wait(timeout=30, step=0.5):
-        for _ in range(timeout):
+        for count in range(timeout):
             time.sleep(step)
             busy = False
             processes = gl.sidekiq.process_metrics()["processes"]
@@ -153,6 +219,7 @@ def _wait(timeout=30, step=0.5):
                     busy = True
             if not busy:
                 return True
+            logging.info(f"sidekiq busy {count} of {timeout}")
         return False
 
     return _wait
@@ -163,9 +230,11 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
     config_file = temp_dir / "python-gitlab.cfg"
     port = docker_services.port_for("gitlab", 80)
 
+    logging.info("Waiting for GitLab container to become ready.")
     docker_services.wait_until_responsive(
         timeout=200, pause=5, check=lambda: check_is_alive("gitlab-test")
     )
+    logging.info("GitLab container is now ready.")
 
     token = set_token("gitlab-test", fixture_dir=fixture_dir)
 
@@ -188,8 +257,10 @@ def gitlab_config(check_is_alive, docker_ip, docker_services, temp_dir, fixture_
 def gl(gitlab_config):
     """Helper instance to make fixtures and asserts directly via the API."""
 
+    logging.info("Create python-gitlab gitlab.Gitlab object")
     instance = gitlab.Gitlab.from_config("local", [gitlab_config])
-    reset_gitlab(instance)
+
+    reset_gitlab(gl=instance)
 
     return instance
 
