feat: emit a warning when using a list() method returns max

JohnVillalovos · JohnVillalovos · commit ec10a5760e0d · 2022-02-05T09:26:07.000-08:00
A common cause of issues filed and questions raised is that a user
will call a `list()` method and only get 20 items. As this is the
default maximum of items that will be returned from a `list()` method.

To help with this we now emit a warning when the result from a
`list()` method is greater-than or equal to 20 (or the specified
`per_page` value) and the user is not using either `all=True`,
`all=False`, `as_list=False`, or `page=X`.
diff --git a/gitlab/__init__.py b/gitlab/__init__.py
@@ -33,6 +33,7 @@
 from gitlab.exceptions import *  # noqa: F401,F403
 
 warnings.filterwarnings("default", category=DeprecationWarning, module="^gitlab")
+warnings.filterwarnings("default", category=UserWarning, module="^gitlab")
 
 
 # NOTE(jlvillal): We are deprecating access to the gitlab.const values which
diff --git a/gitlab/client.py b/gitlab/client.py
@@ -24,6 +24,7 @@
 import requests.utils
 from requests_toolbelt.multipart.encoder import MultipartEncoder  # type: ignore
 
+import gitlab
 import gitlab.config
 import gitlab.const
 import gitlab.exceptions
@@ -35,6 +36,12 @@
     "{source!r} to {target!r}"
 )
 
+# https://docs.gitlab.com/ee/api/#offset-based-pagination
+_PAGINATION_URL = (
+    f"https://python-gitlab.readthedocs.io/en/v{gitlab.__version__}/"
+    f"api-usage.html#pagination"
+)
+
 
 class Gitlab:
     """Represents a GitLab server connection.
@@ -808,7 +815,7 @@ def http_list(
         # In case we want to change the default behavior at some point
         as_list = True if as_list is None else as_list
 
-        get_all = kwargs.pop("all", False)
+        get_all = kwargs.pop("all", None)
         url = self._build_url(path)
 
         page = kwargs.get("page")
@@ -818,7 +825,28 @@ def http_list(
 
         if page or as_list is True:
             # pagination requested, we return a list
-            return list(GitlabList(self, url, query_data, get_next=False, **kwargs))
+            gl_list = GitlabList(self, url, query_data, get_next=False, **kwargs)
+            items = list(gl_list)
+            if page is None and get_all is None and gl_list.per_page is not None:
+                if len(items) >= gl_list.per_page and (
+                    gl_list.total is None or len(items) < gl_list.total
+                ):
+                    total_items = "10,000+" if gl_list.total is None else gl_list.total
+                    # Warn the user that they are only going to retrieve `per_page`
+                    # maximum items.  This is a common cause of issues filed.
+                    utils.warn(
+                        message=(
+                            f"Calling a `list()` method without specifying `all=True` "
+                            f"or `as_list=False` will return a maximum of "
+                            f"{gl_list.per_page} items. Your query returned "
+                            f"{len(items)} of {total_items} items. See "
+                            f"{_PAGINATION_URL} for more details. If this was done "
+                            f"intentionally, then this warning can be supressed by "
+                            f"adding the argument `all=False` to the `list()` call."
+                        ),
+                        category=UserWarning,
+                    )
+            return items
 
         # No pagination, generator requested
         return GitlabList(self, url, query_data, **kwargs)
diff --git a/tests/functional/api/test_gitlab.py b/tests/functional/api/test_gitlab.py
@@ -1,3 +1,5 @@
+import warnings
+
 import pytest
 
 import gitlab
@@ -81,13 +83,13 @@ def test_template_dockerfile(gl):
 
 
 def test_template_gitignore(gl):
-    assert gl.gitignores.list()
+    assert gl.gitignores.list(all=True)
     gitignore = gl.gitignores.get("Node")
     assert gitignore.content is not None
 
 
 def test_template_gitlabciyml(gl):
-    assert gl.gitlabciymls.list()
+    assert gl.gitlabciymls.list(all=True)
     gitlabciyml = gl.gitlabciymls.get("Nodejs")
     assert gitlabciyml.content is not None
 
@@ -181,3 +183,48 @@ def test_rate_limits(gl):
     settings.throttle_authenticated_api_enabled = False
     settings.save()
     [project.delete() for project in projects]
+
+
+def test_list_default_warning(gl):
+    """When there are more than 20 items and use default `list()` then warning is
+    generated"""
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        gl.gitlabciymls.list()
+    assert len(caught_warnings) == 1
+    warning = caught_warnings[0]
+    assert isinstance(warning.message, UserWarning)
+    message = str(caught_warnings[0].message)
+    assert "Calling" in message
+    assert "return a maximum of" in message
+    assert "readthedocs" in message
+    assert __file__ == warning.filename
+
+
+def test_list_page_nowarning(gl):
+    """Using `page=X` will disable the warning"""
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        gl.gitlabciymls.list(page=1)
+    assert len(caught_warnings) == 0
+
+
+def test_list_all_false_nowarning(gl):
+    """Using `all=False` will disable the warning"""
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        gl.gitlabciymls.list(all=False)
+    assert len(caught_warnings) == 0
+
+
+def test_list_all_true_nowarning(gl):
+    """Using `all=True` will disable the warning"""
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        items = gl.gitlabciymls.list(all=True)
+    assert len(caught_warnings) == 0
+    assert len(items) > 20
+
+
+def test_list_as_list_false_nowarning(gl):
+    """Using `as_list=False` will disable the warning"""
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        items = gl.gitlabciymls.list(as_list=False)
+    assert len(caught_warnings) == 0
+    assert len(list(items)) > 20
diff --git a/tests/functional/fixtures/docker-compose.yml b/tests/functional/fixtures/docker-compose.yml
@@ -14,7 +14,7 @@ services:
       GITLAB_ROOT_PASSWORD: 5iveL!fe
       GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN: registration-token
       GITLAB_OMNIBUS_CONFIG: |
-        external_url 'http://127.0.0.1:8080'
+        external_url 'http://localhost:8080'
         registry['enable'] = false
         nginx['redirect_http_to_https'] = false
         nginx['listen_port'] = 80
diff --git a/tests/unit/test_gitlab_http_methods.py b/tests/unit/test_gitlab_http_methods.py
@@ -1,3 +1,6 @@
+import copy
+import warnings
+
 import pytest
 import requests
 import responses
@@ -329,20 +332,115 @@ def test_list_request(gl):
         match=MATCH_EMPTY_QUERY_PARAMS,
     )
 
-    result = gl.http_list("/projects", as_list=True)
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        result = gl.http_list("/projects", as_list=True)
+    assert len(caught_warnings) == 0
     assert isinstance(result, list)
     assert len(result) == 1
 
     result = gl.http_list("/projects", as_list=False)
     assert isinstance(result, GitlabList)
-    assert len(result) == 1
+    assert len(list(result)) == 1
 
     result = gl.http_list("/projects", all=True)
     assert isinstance(result, list)
     assert len(result) == 1
     assert responses.assert_call_count(url, 3) is True
 
 
+large_list_response = {
+    "method": responses.GET,
+    "url": "http://localhost/api/v4/projects",
+    "json": [
+        {"name": "project01"},
+        {"name": "project02"},
+        {"name": "project03"},
+        {"name": "project04"},
+        {"name": "project05"},
+        {"name": "project06"},
+        {"name": "project07"},
+        {"name": "project08"},
+        {"name": "project09"},
+        {"name": "project10"},
+        {"name": "project11"},
+        {"name": "project12"},
+        {"name": "project13"},
+        {"name": "project14"},
+        {"name": "project15"},
+        {"name": "project16"},
+        {"name": "project17"},
+        {"name": "project18"},
+        {"name": "project19"},
+        {"name": "project20"},
+    ],
+    "headers": {"X-Total": "30", "x-per-page": "20"},
+    "status": 200,
+    "match": MATCH_EMPTY_QUERY_PARAMS,
+}
+
+
+@responses.activate
+def test_list_request_pagination_warning(gl):
+    responses.add(**large_list_response)
+
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        result = gl.http_list("/projects", as_list=True)
+    assert len(caught_warnings) == 1
+    warning = caught_warnings[0]
+    assert isinstance(warning.message, UserWarning)
+    message = str(warning.message)
+    assert "Calling a `list()` method" in message
+    assert "python-gitlab.readthedocs.io" in message
+    assert __file__ == warning.filename
+    assert isinstance(result, list)
+    assert len(result) == 20
+    assert len(responses.calls) == 1
+
+
+@responses.activate
+def test_list_request_as_list_false_nowarning(gl):
+    responses.add(**large_list_response)
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        result = gl.http_list("/projects", as_list=False)
+    assert len(caught_warnings) == 0
+    assert isinstance(result, GitlabList)
+    assert len(list(result)) == 20
+    assert len(responses.calls) == 1
+
+
+@responses.activate
+def test_list_request_all_true_nowarning(gl):
+    responses.add(**large_list_response)
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        result = gl.http_list("/projects", all=True)
+    assert len(caught_warnings) == 0
+    assert isinstance(result, list)
+    assert len(result) == 20
+    assert len(responses.calls) == 1
+
+
+@responses.activate
+def test_list_request_all_false_nowarning(gl):
+    responses.add(**large_list_response)
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        result = gl.http_list("/projects", all=False)
+    assert len(caught_warnings) == 0
+    assert isinstance(result, list)
+    assert len(result) == 20
+    assert len(responses.calls) == 1
+
+
+@responses.activate
+def test_list_request_page_nowarning(gl):
+    response_dict = copy.deepcopy(large_list_response)
+    response_dict["match"] = [responses.matchers.query_param_matcher({"page": "1"})]
+    responses.add(**response_dict)
+    with warnings.catch_warnings(record=True) as caught_warnings:
+        gl.http_list("/projects", page=1)
+    assert len(caught_warnings) == 0
+    assert len(responses.calls) == 1
+
+
 @responses.activate
 def test_list_request_404(gl):
     url = "http://localhost/api/v4/not_there"
