-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathgithubclientbase.py
More file actions
158 lines (116 loc) · 5.11 KB
/
githubclientbase.py
File metadata and controls
158 lines (116 loc) · 5.11 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
import sys, requests, datetime
class GitHubClientBase(object):
def __init__(self, token=None, username=None, password=None, url="https://api.github.com", usesession=False):
"""Git hub access token
if token is callable it will be invoked to produce the token """
self._username = username
self._password = password
self._token = token
self._url = url
self._graphql_url = "https://api.github.com/graphql"
self._rateLimitRemaining = None
self._rateLimitReset = None
self.usesession = usesession
def _setusesession(self, usesession):
if usesession:
self._session = requests.session()
else:
self._session = requests
def _getusesession(self):
return self._session != requests
usesession = property(_getusesession, _setusesession, doc="Use requests' sessions")
def resetSession(self):
if not self.usesession:
return
self._session = requests.session()
def _requests_kwargs(self, additionHeaders=dict()):
r_kwargs = { 'headers': { 'Accept': 'application/vnd.github.v3+json' } }
r_kwargs['headers'].update(additionHeaders)
if( self._token ):
if callable(self._token):
r_kwargs['headers']["Authorization"] = f"token {self._token()}"
else:
r_kwargs['headers']["Authorization"] = f"token {self._token}"
elif self._username:
r_kwargs['auth'] = requests.auth.HTTPBasicAuth(self._username, self._password)
return r_kwargs
def _updateStats(self, headers:dict):
remaining = headers.get('X-RateLimit-Remaining')
if remaining is not None:
self._rateLimitRemaining = int(remaining)
reset = headers.get('X-RateLimit-Reset')
if reset is not None:
self._rateLimitReset = reset
##
##
##
def _getrateLimitRemaining(self):
return self._rateLimitRemaining
rateLimitRemaining = property(_getrateLimitRemaining, doc="get rateLimitRemaining")
##
##
##
def _getrateLimitReset(self):
return datetime.datetime.fromtimestamp(int(self._rateLimitReset))
rateLimitReset = property(_getrateLimitReset, doc="get local time when rate limit will reset")
@classmethod
def paginate(clazz, methodcall, *args, per_page=100, page=1, pagination_limit=sys.maxsize, extractor=lambda x: x, **kwargs):
"""Utility method that will paginate a request, gathering the results
up to the pagination_limit_parameter
arguments:
methodcall -- method that can be paginated
page -- optional starting page(defaults to 1)
pagination_limit -- maximum number of results to return
extractor -- callable to convert result into a list
"""
results = []
kwargs['per_page'] = per_page
while pagination_limit > 0:
if pagination_limit < per_page:
kwargs['per_page'] = pagination_limit
kwargs['page'] = page
data2 = methodcall(*args, **kwargs)
data = extractor(data2)
if not isinstance(data, list):
return data # an error of some description
if len(data) == 0:
return results
results.extend(data)
page += 1
pagination_limit -= len(data)
return results
@classmethod
def generate(clazz, methodcall, *args, per_page=100, page=1, pagination_limit=sys.maxsize, extractor=lambda x: x, **kwargs):
"""Utility method that will paginate a request, gathering the results
up to the pagination_limit_parameter
arguments:
methodcall -- method that can be paginated
page -- optional starting page(defaults to 1)
pagination_limit -- maximum number of results to return
extractor -- callable to convert result into a list
returns:
A generator object to iterate results
"""
kwargs['per_page'] = per_page
while pagination_limit > 0:
if pagination_limit < per_page:
kwargs['per_page'] = pagination_limit
kwargs['page'] = page
data = methodcall(*args, **kwargs)
if not isinstance(data, list) and not data.ok:
yield data # an error of some description
return
data = extractor(data)
if len(data) == 0:
return
for d in data:
yield d
page += 1
pagination_limit -= len(data)
return
def _generatorForResult(self, r, chunk_size):
isinstance(r, requests.Response)
for chunk in r.iter_content(chunk_size=chunk_size):
if not chunk:
continue
yield chunk