fix: use keyset pagination by default for /projects > 50000

max-wittig · max-wittig · commit 44ef603a13df · 2020-06-08T14:20:08.000+02:00
Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/218504. Remove this in 13.1
diff --git a/gitlab/__init__.py b/gitlab/__init__.py
@@ -16,13 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Wrapper for the GitLab API."""
 
-from __future__ import print_function
-from __future__ import absolute_import
 import importlib
 import time
 import warnings
 
 import requests
+import requests.utils
 
 import gitlab.config
 from gitlab.const import *  # noqa
@@ -43,6 +42,8 @@
     "must update your GitLab URL to use https:// to avoid issues."
 )
 
+ALLOWED_KEYSET_ENDPOINTS = ["/projects"]
+
 
 def _sanitize(value):
     if isinstance(value, dict):
@@ -618,7 +619,7 @@ def http_list(self, path, query_data=None, as_list=None, **kwargs):
 
         Args:
             path (str): Path or full URL to query ('/projects' or
-                        'http://whatever/v4/api/projecs')
+                        'http://whatever/v4/api/projects')
             query_data (dict): Data to send as query parameters
             **kwargs: Extra options to send to the server (e.g. sudo, page,
                       per_page)
@@ -642,6 +643,12 @@ def http_list(self, path, query_data=None, as_list=None, **kwargs):
         get_all = kwargs.pop("all", False)
         url = self._build_url(path)
 
+        # use keyset pagination automatically, if all=True
+        order_by = kwargs.get("order_by")
+        if path in ALLOWED_KEYSET_ENDPOINTS and (not order_by or order_by == "id"):
+            kwargs["pagination"] = "keyset"
+            kwargs["order_by"] = "id"
+
         if get_all is True and as_list is True:
             return list(GitlabList(self, url, query_data, **kwargs))
 
@@ -781,7 +788,12 @@ def _query(self, url, query_data=None, **kwargs):
         query_data = query_data or {}
         result = self._gl.http_request("get", url, query_data=query_data, **kwargs)
         try:
-            self._next_url = result.links["next"]["url"]
+            links = result.links
+            if links:
+                next_url = links["next"]["url"]
+            else:
+                next_url = requests.utils.parse_header_links(result.headers["links"])[0]["url"]
+            self._next_url = next_url
         except KeyError:
             self._next_url = None
         self._current_page = result.headers.get("X-Page")
