chore: ensure reset_gitlab() succeeds

JohnVillalovos · JohnVillalovos · commit 112f07f2a1b3 · 2021-12-28T10:22:55.000-08:00
Ensure reset_gitlab() succeeds by waiting to make sure everything has
been deleted as expected. If the timeout is exceeded fail the test.

Not using `wait_for_sidekiq` as it didn't work. During testing I
didn't see any sidekiq processes as being busy even though not
everything was deleted.
diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py
@@ -1,8 +1,10 @@
+import dataclasses
 import tempfile
 import time
 import uuid
 from pathlib import Path
 from subprocess import check_output
+from typing import Any, Dict, List
 
 import pytest
 
@@ -30,6 +32,36 @@ def reset_gitlab(gl):
         if user.username != "root":
             user.delete(hard_delete=True)
 
+    sleep_interval = 0.1
+    timeout = 60
+    max_iterations = int(timeout / sleep_interval)
+
+    # Ensure everything has been reset
+    start_time = time.perf_counter()
+
+    def ensure_list_empty(pg_object, description: str) -> None:
+        for _ in range(max_iterations):
+            if pg_object.list() == []:
+                break
+            time.sleep(sleep_interval)
+        assert pg_object.list() == [], (
+            f"Did not delete all {description}. "
+            f"Elapsed_time: {time.perf_counter() - start_time}"
+        )
+
+    ensure_list_empty(pg_object=gl.projects, description="projects")
+    ensure_list_empty(pg_object=gl.groups, description="groups")
+    ensure_list_empty(pg_object=gl.variables, description="variables")
+
+    for _ in range(max_iterations):
+        if len(gl.users.list()) <= 1:
+            break
+        time.sleep(sleep_interval)
+    assert len(gl.users.list()) <= 1, (
+        f"Did not delete all users. "
+        f"elapsed_time: {time.perf_counter() - start_time}"
+    )
+
 
 def set_token(container, fixture_dir):
     set_token_rb = fixture_dir / "set_token.rb"
@@ -105,25 +137,59 @@ def _check(container):
     return _check
 
 
+@dataclasses.dataclass
+class WaitSidekiq:
+    iterations: int
+    step: float
+    elapsed_time: float
+    success: bool
+
+
 @pytest.fixture
-def wait_for_sidekiq(gl):
+def wait_for_sidekiq(gl: gitlab.Gitlab):
+    """
+    Return a helper function to wait until there are no busy sidekiq processes.
+
+    Use this with asserts for slow tasks (group/project/user creation/deletion).
+    """
+    return _wait_for_sidekiq(gl=gl)
+
+
+def _wait_for_sidekiq(gl: gitlab.Gitlab):
     """
     Return a helper function to wait until there are no busy sidekiq processes.
 
     Use this with asserts for slow tasks (group/project/user creation/deletion).
     """
 
-    def _wait(timeout=30, step=0.5):
-        for _ in range(timeout):
+    def _wait(
+        timeout: int = 60,
+        step: float = 0.1,
+    ) -> WaitSidekiq:
+        # timeout is approximately the timeout in seconds
+        max_iterations = int(timeout / step)
+        start_time = time.perf_counter()
+        success = False
+        for count in range(max_iterations):
+            print(f"_wait: count: {count}")
             time.sleep(step)
             busy = False
             processes = gl.sidekiq.process_metrics()["processes"]
             for process in processes:
+                print(f"_wait: process['busy']: {process['busy']}")
                 if process["busy"]:
                     busy = True
             if not busy:
-                return True
-        return False
+                success = True
+                break
+        result = WaitSidekiq(
+            iterations=count,
+            step=step,
+            elapsed_time=time.perf_counter() - start_time,
+            success=success,
+        )
+        print(f"_wait: {result}")
+        return result
 
     return _wait
 
@@ -194,6 +260,73 @@ def gitlab_runner(gl):
     check_output(docker_exec + unregister).decode()
 
 
+@pytest.fixture
+def ensure_raises_exception():
+    """
+    Return a helper function to wait until specified exception is raised
+
+    Use this with asserts for slow tasks (group/project/user creation/deletion).
+    """
+    return _ensure_raises_exception_wrapped()
+
+
+def _ensure_raises_exception(
+    *,
+    func,
+    exception: Exception,
+    args: List[Any],
+    kwargs: Dict[str, Any],
+    description: str,
+) -> None:
+    """Ensure the exception is specified when calling `func`. If no exception is raided
+    after timeout period, fail the test"""
+    sleep_interval = 0.1
+    timeout = 60
+    max_iterations = int(timeout / sleep_interval)
+
+    for _ in range(max_iterations):
+        try:
+            func(*args, **kwargs)
+        except exception:
+            return
+        time.sleep(sleep_interval)
+    pytest.fail(f"{description} did not raise exception {exception}")
+
+
+def _ensure_raises_exception_wrapped():
+    return _ensure_raises_exception
+
+
+@pytest.fixture
+def ensure_deleted():
+    """
+    Return a helper function to wait until specified object is deleted
+
+    Use this with asserts for slow tasks (group/project/user creation/deletion).
+    """
+    return _ensure_deleted_wrapped()
+
+
+def _ensure_deleted(*, pg_object, object_id: int, description: str) -> None:
+    """Ensure the object specified can not be retrieved. If object still exists after
+    timeout period, fail the test"""
+    sleep_interval = 0.1
+    timeout = 60
+    max_iterations = int(timeout / sleep_interval)
+
+    for _ in range(max_iterations):
+        try:
+            pg_object.get(object_id)
+        except gitlab.exceptions.GitlabGetError:
+            return
+        time.sleep(sleep_interval)
+    pytest.fail(f"{description} {object_id} was not deleted")
+
+
+def _ensure_deleted_wrapped():
+    return _ensure_deleted
+
+
 @pytest.fixture(scope="module")
 def group(gl):
     """Group fixture for group API resource tests."""
@@ -203,6 +336,7 @@ def group(gl):
         "path": f"group-{_id}",
     }
     group = gl.groups.create(data)
+    group_id = group.id
 
     yield group
 
@@ -211,6 +345,8 @@ def group(gl):
     except gitlab.exceptions.GitlabDeleteError as e:
         print(f"Group already deleted: {e}")
 
+    _ensure_deleted(pg_object=gl.groups, object_id=group_id, description="Group")
+
 
 @pytest.fixture(scope="module")
 def project(gl):
@@ -219,6 +355,7 @@ def project(gl):
     name = f"test-project-{_id}"
 
     project = gl.projects.create(name=name)
+    project_id = project.id
 
     yield project
 
@@ -227,6 +364,8 @@ def project(gl):
     except gitlab.exceptions.GitlabDeleteError as e:
         print(f"Project already deleted: {e}")
 
+    _ensure_deleted(pg_object=gl.projects, object_id=project_id, description="Project")
+
 
 @pytest.fixture(scope="function")
 def merge_request(project, wait_for_sidekiq):
@@ -249,8 +388,10 @@ def _merge_request(*, source_branch: str):
         # NOTE(jlvillal): Sometimes the CI would give a "500 Internal Server
         # Error". Hoping that waiting until all other processes are done will
         # help with that.
-        result = wait_for_sidekiq(timeout=60)
-        assert result is True, "sidekiq process should have terminated but did not"
+        result = wait_for_sidekiq()
+        assert (
+            result.success is True
+        ), f"sidekiq process should have terminated but did not: {result}"
 
         project.refresh()  # Gets us the current default branch
         project.branches.create(
@@ -274,8 +415,10 @@ def _merge_request(*, source_branch: str):
                 "remove_source_branch": True,
             }
         )
-        result = wait_for_sidekiq(timeout=60)
-        assert result is True, "sidekiq process should have terminated but did not"
+        result = wait_for_sidekiq()
+        assert (
+            result.success is True
+        ), f"sidekiq process should have terminated but did not: {result}"
 
         mr_iid = mr.iid
         for _ in range(60):
@@ -298,6 +441,18 @@ def _merge_request(*, source_branch: str):
             # Ignore if branch was already deleted
             pass
 
+    for mr_iid, source_branch in to_delete:
+        _ensure_deleted(
+            pg_object=project.mergerequests,
+            object_id=mr_iid,
+            description="Project mergerequest",
+        )
+        _ensure_deleted(
+            pg_object=project.branches,
+            object_id=source_branch,
+            description="Project branch",
+        )
+
 
 @pytest.fixture(scope="module")
 def project_file(project):
@@ -342,6 +497,7 @@ def user(gl):
     password = "fakepassword"
 
     user = gl.users.create(email=email, username=username, name=name, password=password)
+    user_id = user.id
 
     yield user
 
@@ -350,6 +506,8 @@ def user(gl):
     except gitlab.exceptions.GitlabDeleteError as e:
         print(f"User already deleted: {e}")
 
+    _ensure_deleted(pg_object=gl.users, object_id=user_id, description="User")
+
 
 @pytest.fixture(scope="module")
 def issue(project):
