-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathproxy.py
More file actions
319 lines (274 loc) · 12.1 KB
/
Copy pathproxy.py
File metadata and controls
319 lines (274 loc) · 12.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
import logging
import re
from dataclasses import dataclass
from functools import cached_property
from typing import Mapping
from typing import Optional
from typing import Sequence
import requests
import werkzeug
from github_proxy.cache.backend import CacheBackend
from github_proxy.github_tokens import GitHubTokenConfig
from github_proxy.github_tokens import InstalledIntegration
from github_proxy.github_tokens import RateLimited
from github_proxy.github_tokens import construct_installed_integration
from github_proxy.github_tokens import token_generator
from github_proxy.ratelimit import get_ratelimit_reset
from github_proxy.ratelimit import is_rate_limited
from github_proxy.telemetry import TelemetryCollector
logger = logging.getLogger(__name__)
# As per RFC 2616 https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1
# hop-by-hop headers must not be forwarded by the proxy
HOP_BY_HOP_HEADERS = {
"Connection",
"Proxy-Connection",
"Keep-Alive",
"Transfer-Encoding",
"TE",
"Te",
"Trailer",
"Upgrade",
"Proxy-Authorization",
"Proxy-Authenticate",
}
# The host header should be automatically rewritten to the URL of the request target
# (ie api.github.com)
REQUEST_FILTERED_HEADERS = {"Host", *HOP_BY_HOP_HEADERS}
# Content-Length and Encoding headers are removed to prevent bad framing
RESPONSE_FILTERED_HEADERS = {"Content-Length", "Content-Encoding", *HOP_BY_HOP_HEADERS}
MATCH_ALL = re.compile(r".*")
@dataclass
class ProxyClientScope:
method: re.Pattern = MATCH_ALL # type: ignore
path: re.Pattern = MATCH_ALL # type: ignore
@dataclass
class ProxyClient:
"""
A client that is authorized to use the proxy
:param name: Human readable name of the client. Used for logging and
telemetry purposes. No two clients must share the same name.
:param token: Authorization token of the client. This is a secret shared
between the client and the proxy. No two clients must share
the same token.
:param scopes: List of scopes that determine the resources that the client
has access to. Defaults to full access.
"""
name: str
token: str
scopes: Sequence[ProxyClientScope] = (ProxyClientScope(),)
def validate_clients(clients: Sequence[ProxyClient]) -> None:
taken_tokens = set()
taken_names = set()
for client in clients:
if client.token in taken_tokens:
raise ValueError("Duplicate client token found")
if client.name in taken_names:
raise ValueError(f"Duplicate client name found: {client.name}")
taken_tokens.add(client.token)
taken_names.add(client.name)
class Proxy:
def __init__(
self,
github_api_url: str,
github_token_config: GitHubTokenConfig,
cache: CacheBackend,
rate_limited: RateLimited,
tel_collector: TelemetryCollector,
clients: Sequence[ProxyClient] = (),
) -> None:
"""
:param github_api_url: Base url of the GitHub API server
:param github_token_config: Config that collects all the available
GitHub user PATs and GitHub Apps. This config
object is used during GitHub token generation.
:param cache: The purpose of this object is to cache the responses of the
GitHub API so that future requests on the same resources can be
served by the cache.
:param rate_limited: Dictionary to store the GitHub tokens that are known
to be rate-limited so that the proxy skips them when
attempting to connect to GitHub. The `rate_limited` dict
should ideally be a TLRU cache so that tokens that
undergo a rate-limit reset, get automagically removed from
the dict.
:param tel_collector: Object collecting telemetry metrics on various points
within the control flow.
:param clients: Clients that are authorized to use the proxy. The list must not
contain duplicate client names or tokens.
"""
self.github_api_url = github_api_url
self.gh_token_config = github_token_config
validate_clients(clients)
self.client_tokens = {
client.token: (client.name, client.scopes) for client in clients
}
self.cache = cache
self.rate_limited = rate_limited
self.tel_collector = tel_collector
# Since all proxy transactions eventually hit the same GitHub host, it is
# preferred to re-use TCP connections (connection pooling). The requests.Session
# object offers this functionality, however it also persists Cookies across
# requests, which could lead to cookies being shared across completely separate
# clients.
# As of today, the GitHub REST API does not make any use of cookies,
# hence it is safe to use a single cookie persisting session across all clients.
# If this changes in the future, we could switch to having a session per client.
self.requester = requests.Session()
def auth(self, token: str, request: werkzeug.Request) -> Optional[str]:
"""
Authorize an incoming proxy request
:param token: Authorization token of the client. See ``ProxyClient.token``.
:request: The request object received by the client.
"""
if token in self.client_tokens:
name, scopes = self.client_tokens[token]
request_path = request.path[len("/api/v3") :]
for scope in scopes:
method_match = scope.method.match(
request.method.lower()
) or scope.method.match(request.method.upper())
if method_match and scope.path.match(request_path):
return name
return None
@cached_property
def integrations(self) -> Mapping[str, InstalledIntegration]:
return {
app_name: construct_installed_integration(
app_name, self.gh_token_config, self.github_api_url
)
for app_name in self.gh_token_config.github_apps
}
def request(
self, path: str, request: werkzeug.Request, client: str
) -> werkzeug.Response:
"""
Proxy a request to the GitHub origin without using the cache.
Should be used for mutative requests.
:path: Path of the requested resource, without the potential /api/v3
prefix.
:request: The request object received by the client.
:client: The name of the client (see ``ProxyClient.name``).
"""
logger.info("%s client requesting %s %s", client, request.method, path)
self.tel_collector.collect_proxy_request_metrics(client, request)
return self._send_gh_request(path, request)
def cached_request(
self, path: str, request: werkzeug.Request, client: str
) -> werkzeug.Response:
"""
Proxy a request for a cacheable resource to the GitHub origin. Return
a cached response if present and valid, else forward the request to
GitHub.
:path: Path of the requested resource, without the potential /api/v3
prefix.
:request: The request object received by the client.
:client: The name of the client (see ``ProxyClient.name``).
"""
media_type = request.accept_mimetypes.best
qs = request.query_string.decode() or None
logger.info(
"%s client requesting %s %s %s, with Etag: %s, Last-Modified: %s",
client,
path,
qs,
media_type,
request.headers.get("If-None-Match"),
request.headers.get("If-Modified-Since"),
)
# The requested media type MUST be combined with the path and the
# query string when indexing cached resources. The GitHub API may return
# a completely different response based on the requested MIME type.
# See more: https://docs.github.com/en/rest/overview/media-types
cached_response = self.cache.get(path, qs, media_type)
if cached_response is None: # cache miss
resp = self._send_gh_request(path, request)
etag_value, _ = resp.get_etag()
cache_hit = None
if etag_value or resp.last_modified:
# TODO: Writing to cache should happen asyncronously
self.cache.set(path, qs, media_type, resp)
# cache miss can only happen if resource is cacheable:
cache_hit = False
self.tel_collector.collect_proxy_request_metrics(client, request, cache_hit)
return resp
# conditional request
resp = self._send_gh_request(
path,
request,
etag=cached_response.headers.get("Etag"),
last_modified=cached_response.headers.get("Last-Modified"),
)
if resp.status_code != 304:
self.cache.set(path, qs, media_type, resp)
self.tel_collector.collect_proxy_request_metrics(
client, request, cache_hit=False
)
return resp
self.tel_collector.collect_proxy_request_metrics(
client, request, cache_hit=True
)
return cached_response # cache hit
def _send_gh_request(
self,
path: str,
request: werkzeug.Request,
etag: Optional[str] = None,
last_modified: Optional[str] = None,
) -> werkzeug.Response:
# Filter request headers
headers = {
k: v
for k, v in request.headers.items()
if k not in REQUEST_FILTERED_HEADERS
}
# Adding cache headers:
# Note that it is not necessary to send both cache headers.
# It is preferred to send only the If-Modified-Since header (if available),
# since it can be reused across different GitHub tokens.
# Etags on the other hand, are token specific.
# For example, if the token of a GitHub app is renewed,
# it cannot reuse the Etags of the previous expired token (whereas
# the Last-Modified timestamp would still work).
if last_modified is not None:
headers["If-Modified-Since"] = last_modified
elif etag is not None:
headers["If-None-Match"] = etag
for token in token_generator(
self.integrations, self.gh_token_config.github_pats, self.rate_limited
):
logger.info("Using %s %s token", token.origin.value, token.name)
# Adding auth
headers["Authorization"] = f"token {token.value}"
resp = self.requester.request(
method=request.method.lower(),
url=f'{self.github_api_url.rstrip("/")}/{path}',
data=request.data,
headers=headers,
params=request.args.to_dict(),
)
self.tel_collector.collect_gh_response_metrics(token, resp)
if is_rate_limited(resp):
reset = get_ratelimit_reset(resp)
if reset:
self.rate_limited[(token.origin, token.name)] = reset
logger.warning(
"%s %s is rate limited. Resetting at %s",
token.origin.value,
token.name,
reset,
)
else:
# Filter response headers
for h in RESPONSE_FILTERED_HEADERS:
resp.headers.pop(h, None)
return werkzeug.Response(
response=resp.content,
status=resp.status_code,
headers=resp.headers.items(),
)
raise RuntimeError("All available GitHub tokens are rate limited")
def health(self) -> bool:
"""
Check that the proxy can successfully integrate with the GitHub origin.
"""
resp = self.cached_request("zen", werkzeug.Request.from_values(), "healthcheck")
return resp.status_code == 200