forked from apify/apify-client-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_client_request_queue.py
More file actions
99 lines (79 loc) · 3.8 KB
/
test_client_request_queue.py
File metadata and controls
99 lines (79 loc) · 3.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
from __future__ import annotations
import re
from typing import TYPE_CHECKING
import pytest
from apify_client import ApifyClient, ApifyClientAsync
from apify_client.errors import ApifyApiError
if TYPE_CHECKING:
from pytest_httpserver import HTTPServer
_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT = """{
"data": {
"processedRequests": [
{
"requestId": "YiKoxjkaS9gjGTqhF",
"uniqueKey": "http://example.com/1",
"wasAlreadyPresent": true,
"wasAlreadyHandled": false
}
],
"unprocessedRequests": [
{
"uniqueKey": "http://example.com/2",
"url": "http://example.com/2",
"method": "GET"
}
]
}
}"""
@pytest.mark.usefixtures('patch_basic_url')
async def test_batch_not_processed_raises_exception_async(httpserver: HTTPServer) -> None:
"""Test that client exceptions are not silently ignored"""
client = ApifyClientAsync(token='placeholder_token')
httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
requests = [
{'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
{'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
]
rq_client = client.request_queue(request_queue_id='whatever')
with pytest.raises(ApifyApiError):
await rq_client.batch_add_requests(requests=requests)
@pytest.mark.usefixtures('patch_basic_url')
async def test_batch_processed_partially_async(httpserver: HTTPServer) -> None:
client = ApifyClientAsync(token='placeholder_token')
httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
)
requests = [
{'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
{'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
]
rq_client = client.request_queue(request_queue_id='whatever')
response = await rq_client.batch_add_requests(requests=requests)
assert requests[0]['uniqueKey'] in {request['uniqueKey'] for request in response['processedRequests']}
assert response['unprocessedRequests'] == [requests[1]]
@pytest.mark.usefixtures('patch_basic_url')
def test_batch_not_processed_raises_exception_sync(httpserver: HTTPServer) -> None:
"""Test that client exceptions are not silently ignored"""
client = ApifyClient(token='placeholder_token')
httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(status=401)
requests = [
{'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
{'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
]
rq_client = client.request_queue(request_queue_id='whatever')
with pytest.raises(ApifyApiError):
rq_client.batch_add_requests(requests=requests)
@pytest.mark.usefixtures('patch_basic_url')
async def test_batch_processed_partially_sync(httpserver: HTTPServer) -> None:
client = ApifyClient(token='placeholder_token')
httpserver.expect_oneshot_request(re.compile(r'.*'), method='POST').respond_with_data(
status=200, response_data=_PARTIALLY_ADDED_BATCH_RESPONSE_CONTENT
)
requests = [
{'uniqueKey': 'http://example.com/1', 'url': 'http://example.com/1', 'method': 'GET'},
{'uniqueKey': 'http://example.com/2', 'url': 'http://example.com/2', 'method': 'GET'},
]
rq_client = client.request_queue(request_queue_id='whatever')
response = rq_client.batch_add_requests(requests=requests)
assert requests[0]['uniqueKey'] in {request['uniqueKey'] for request in response['processedRequests']}
assert response['unprocessedRequests'] == [requests[1]]