-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Expand file tree
/
Copy pathusage.py
More file actions
39 lines (29 loc) · 752 Bytes
/
usage.py
File metadata and controls
39 lines (29 loc) · 752 Bytes
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
import requests
proxypool_url = 'http://127.0.0.1:5555/random'
target_url = 'https://antispider5.scrape.center/'
def get_random_proxy():
"""
get random proxy from proxypool
:return: proxy
"""
return requests.get(proxypool_url).text.strip()
def crawl(url, proxy):
"""
use proxy to crawl page
:param url: page url
:param proxy: proxy, such as 8.8.8.8:8888
:return: html
"""
proxies = {'http': 'http://' + proxy}
return requests.get(url, proxies=proxies).text
def main():
"""
main method, entry point
:return: none
"""
proxy = get_random_proxy()
print('get random proxy', proxy)
html = crawl(target_url, proxy)
print(html)
if __name__ == '__main__':
main()