-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathasync_example.py
More file actions
167 lines (127 loc) · 5.24 KB
/
Copy pathasync_example.py
File metadata and controls
167 lines (127 loc) · 5.24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
"""
Scrappey - Async Python Example
This example demonstrates how to use the AsyncScrappey client
for async/await web scraping workflows.
Prerequisites:
pip install scrappey
Get your API key at: https://app.scrappey.com
"""
import asyncio
import os
from scrappey import AsyncScrappey
# Get API key from environment or replace with your key
API_KEY = os.getenv("SCRAPPEY_API_KEY", "YOUR_API_KEY")
async def basic_async_example():
"""Basic async example: Simple GET request."""
print("\n=== Basic Async Example ===\n")
async with AsyncScrappey(api_key=API_KEY) as scrappey:
result = await scrappey.get(url="https://httpbin.org/get")
if result.get("data") == "success":
print(f"Status: {result['solution']['statusCode']}")
print(f"Response: {result['solution']['response'][:200]}...")
else:
print(f"Error: {result.get('error')}")
async def parallel_requests_example():
"""Parallel requests example: Scrape multiple URLs concurrently."""
print("\n=== Parallel Requests Example ===\n")
urls = [
"https://httpbin.org/get?id=1",
"https://httpbin.org/get?id=2",
"https://httpbin.org/get?id=3",
"https://httpbin.org/get?id=4",
"https://httpbin.org/get?id=5",
]
async with AsyncScrappey(api_key=API_KEY) as scrappey:
# Create session for all requests
session_data = await scrappey.create_session()
session_id = session_data["session"]
try:
# Run requests in parallel
tasks = [
scrappey.get(url=url, session=session_id)
for url in urls
]
results = await asyncio.gather(*tasks, return_exceptions=True)
# Process results
for url, result in zip(urls, results):
if isinstance(result, Exception):
print(f" {url}: Error - {result}")
elif result.get("data") == "success":
print(f" {url}: Status {result['solution']['statusCode']}")
else:
print(f" {url}: API Error - {result.get('error')}")
finally:
# Clean up session
await scrappey.destroy_session(session_id)
print(f"\nSession destroyed: {session_id}")
async def session_management_example():
"""Async session management example."""
print("\n=== Async Session Management Example ===\n")
async with AsyncScrappey(api_key=API_KEY) as scrappey:
# Create session
session_data = await scrappey.create_session(
proxyCountry="UnitedStates",
premiumProxy=True,
)
session_id = session_data["session"]
print(f"Created session: {session_id}")
# Check if session is active
is_active = await scrappey.is_session_active(session_id)
print(f"Session active: {is_active}")
# List all sessions
sessions = await scrappey.list_sessions()
print(f"Open sessions: {sessions.get('open')}/{sessions.get('limit')}")
# Use session for login flow
result = await scrappey.browser_action(
url="https://example.com",
session=session_id,
actions=[
{"type": "wait", "wait": 1000},
{"type": "execute_js", "code": "document.title"},
],
)
if result.get("data") == "success":
print(f"Page title: {result['solution'].get('javascriptReturn')}")
# Destroy session
await scrappey.destroy_session(session_id)
print(f"Destroyed session: {session_id}")
async def rate_limited_scraping():
"""Rate-limited scraping with semaphore."""
print("\n=== Rate Limited Scraping Example ===\n")
urls = [f"https://httpbin.org/get?page={i}" for i in range(10)]
max_concurrent = 3 # Maximum concurrent requests
semaphore = asyncio.Semaphore(max_concurrent)
async def fetch_with_limit(scrappey: AsyncScrappey, url: str, session: str):
async with semaphore:
print(f"Fetching: {url}")
result = await scrappey.get(url=url, session=session)
return url, result
async with AsyncScrappey(api_key=API_KEY) as scrappey:
session_data = await scrappey.create_session()
session_id = session_data["session"]
try:
tasks = [fetch_with_limit(scrappey, url, session_id) for url in urls]
results = await asyncio.gather(*tasks)
success_count = sum(
1 for _, r in results
if not isinstance(r, Exception) and r.get("data") == "success"
)
print(f"\nCompleted: {success_count}/{len(urls)} successful")
finally:
await scrappey.destroy_session(session_id)
async def main():
"""Run all async examples."""
print("Scrappey Async Python Examples")
print("=" * 50)
try:
await basic_async_example()
await parallel_requests_example()
await session_management_example()
# Uncomment to run additional examples:
# await rate_limited_scraping()
print("\n✓ All async examples completed!\n")
except Exception as e:
print(f"\nError: {e}")
raise
if __name__ == "__main__":
asyncio.run(main())