-
-
Notifications
You must be signed in to change notification settings - Fork 27
Expand file tree
/
Copy pathWebInterface.py
More file actions
executable file
·176 lines (136 loc) · 5.59 KB
/
Copy pathWebInterface.py
File metadata and controls
executable file
·176 lines (136 loc) · 5.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
from pathlib import Path
from typing import Any, Union
from re import IGNORECASE, compile as re_compile
from requests import get, Session
from tenacity import retry, stop_after_attempt, wait_fixed, wait_exponential
import urllib3
from modules.Debug import log
class WebInterface:
"""
This class defines a WebInterface, which is a type of interface that
makes requests using some persistent session and returns JSON
results. This object caches requests/results for better performance.
"""
"""Maximum time allowed for a single GET request"""
REQUEST_TIMEOUT = 15
"""How many requests to cache"""
CACHE_LENGTH = 10
"""Regex to match URL's"""
_URL_REGEX = re_compile(r'^((?:https?:\/\/)?.+)(?=\/)', IGNORECASE)
"""Content to ignore if returned by any GET request"""
BAD_CONTENT = (
b'<html><head><title>',
b'<Code>AccessDenied</Code>',
)
def __init__(self,
name: str,
verify_ssl: bool = True,
*,
cache: bool = True,
) -> None:
"""
Construct a new instance of a WebInterface. This creates creates
cached request and results lists, and establishes a session for
future use.
Args:
name: Name (for logging) of this interface.
verify_ssl: Whether to verify SSL requests with this
interface.
cache: Whether to cache requests with this interface.
log: Logger for all log messages.
"""
# Store name of this interface
self.name = name
# Create session for persistent requests
self.session = Session()
# Whether to verify SSL
self.session.verify = verify_ssl
if not self.session.verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
log.debug(f'Not verifying SSL connections for {name}')
# Cache of the last requests to speed up identical sequential requests
self.__do_cache = cache
self.__cache = []
self.__cached_results = []
def __repr__(self) -> str:
"""Returns an unambiguous string representation of the object."""
return f'<WebInterface to {self.name}>'
@retry(stop=stop_after_attempt(5),
wait=wait_fixed(5)+wait_exponential(min=1, max=16),
before_sleep=lambda _:log.warning('Failed to submit GET request, retrying..'),
reraise=True)
def __retry_get(self, url: str, params: dict) -> dict:
"""
Retry the given GET request until successful (or really fails).
Args:
url: The URL of the GET request.
params: The params of the GET request.
Returns:
Dict made from the JSON return of the specified GET request.
"""
return self.session.get(
url=url,
params=params,
timeout=self.REQUEST_TIMEOUT
).json()
def get(self, url: str, params: dict, *, cache: bool = True) -> Any:
"""
Wrapper for getting the JSON return of the specified GET
request. If the provided URL and parameters are identical to the
previous request, then a cached result is returned instead (if
enabled).
Args:
url: URL to pass to GET.
Parameters to pass to GET.
Returns:
Parsed JSON return of the specified GET request.
"""
# If not caching, just query and return
if not self.__do_cache:
return self.__retry_get(url=url, params=params)
# Look through all cached results for this exact URL+params; if found,
# skip the request and return that result
for cache, result in zip(self.__cache, self.__cached_results):
if cache['url'] == url and cache['params'] == str(params):
return result
# Make new request, add to cache
self.__cached_results.append(self.__retry_get(url=url, params=params))
self.__cache.append({'url': url, 'params': str(params)})
# Delete element from cache if length has been exceeded
if len(self.__cache) > self.CACHE_LENGTH:
self.__cache.pop(0)
self.__cached_results.pop(0)
# Return latest result
return self.__cached_results[-1]
@staticmethod
def download_image(image: Union[str, bytes], destination: Path) -> bool:
"""
Download the provided image to the destination filepath.
Args:
image: URL to the image to download, or bytes of the image
to write.
destination: Destination path to download the image to.
Returns:
Whether the image was successfully downloaded.
"""
# Make parent folder structure
destination.parent.mkdir(parents=True, exist_ok=True)
# If content of image, just write directly to file
if isinstance(image, bytes):
destination.write_bytes(image)
return True
# Attempt download
url = image
try:
# Download from URL
image = get(url, timeout=30).content
if len(image) == 0:
raise ValueError(f'URL {url} returned no content')
if any(bc in image for bc in WebInterface.BAD_CONTENT):
raise ValueError(f'URL {url} returned malformed content')
# Write content to file, return success
destination.write_bytes(image)
return True
except Exception: # pylint: disable=broad-except
log.exception('Cannot download image, returned error')
return False