Repository navigation
Expand file tree
/
Copy pathtest_workerpool.py
More file actions
151 lines (124 loc) · 4.73 KB
/
Copy pathtest_workerpool.py
File metadata and controls
151 lines (124 loc) · 4.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""Workerpool test module."""
# pylint: disable=no-self-use,too-few-public-methods,missing-docstring
import time
import threading
import pytest
from splitio.tasks.util import workerpool
from splitio.optional.loaders import asyncio
class WorkerPoolTests(object):
"""Worker pool test cases."""
def test_normal_operation(self, mocker):
"""Test normal opeation works properly."""
worker_func = mocker.Mock()
wpool = workerpool.WorkerPool(10, worker_func)
wpool.start()
for num in range(0, 100):
wpool.submit_work(str(num))
stop_event = threading.Event()
wpool.stop(stop_event)
stop_event.wait(5)
assert stop_event.is_set()
calls = worker_func.mock_calls
for num in range(0, 100):
assert mocker.call(str(num)) in calls
def test_fail_in_msg_doesnt_break(self):
"""Test that if a message cannot be parsed it is ignored and others are processed."""
class Worker(object): #pylint: disable=
def __init__(self):
self.worked = set()
def do_work(self, work):
if work == '55':
raise Exception('something')
self.worked.add(work)
worker = Worker()
wpool = workerpool.WorkerPool(50, worker.do_work)
wpool.start()
for num in range(0, 100):
wpool.submit_work(str(num))
stop_event = threading.Event()
wpool.stop(stop_event)
stop_event.wait(5)
assert stop_event.is_set()
for num in range(0, 100):
if num != 55:
assert str(num) in worker.worked
else:
assert str(num) not in worker.worked
def test_msg_acked_after_processed(self):
"""Test that events are only set after all the work in the pipeline is done."""
class Worker(object):
def __init__(self):
self.worked = set()
def do_work(self, work):
self.worked.add(work)
time.sleep(0.02) # will wait 2 seconds in total for 100 elements
worker = Worker()
wpool = workerpool.WorkerPool(50, worker.do_work)
wpool.start()
for num in range(0, 100):
wpool.submit_work(str(num))
wpool.wait_for_completion()
assert len(worker.worked) == 100
class WorkerPoolAsyncTests(object):
"""Worker pool async test cases."""
@pytest.mark.asyncio
async def test_normal_operation(self, mocker):
"""Test normal opeation works properly."""
self.calls = 0
calls = []
async def worker_func(num):
self.calls += 1
calls.append(num)
wpool = workerpool.WorkerPoolAsync(10, worker_func)
wpool.start()
jobs = []
for num in range(0, 11):
jobs.append(str(num))
task = await wpool.submit_work(jobs)
assert await task.await_completion()
await wpool.stop()
for num in range(0, 11):
assert str(num) in calls
@pytest.mark.asyncio
async def test_fail_in_msg_doesnt_break(self):
"""Test that if a message cannot be parsed it is ignored and others are processed."""
class Worker(object): #pylint: disable=
def __init__(self):
self.worked = set()
async def do_work(self, work):
if work == '55':
raise Exception('something')
self.worked.add(work)
worker = Worker()
wpool = workerpool.WorkerPoolAsync(50, worker.do_work)
wpool.start()
jobs = []
for num in range(0, 100):
jobs.append(str(num))
task = await wpool.submit_work(jobs)
assert not await task.await_completion()
await wpool.stop()
for num in range(0, 100):
if num != 55:
assert str(num) in worker.worked
else:
assert str(num) not in worker.worked
@pytest.mark.asyncio
async def test_msg_acked_after_processed(self):
"""Test that events are only set after all the work in the pipeline is done."""
class Worker(object):
def __init__(self):
self.worked = set()
async def do_work(self, work):
self.worked.add(work)
await asyncio.sleep(0.02) # will wait 2 seconds in total for 100 elements
worker = Worker()
wpool = workerpool.WorkerPoolAsync(50, worker.do_work)
wpool.start()
jobs = []
for num in range(0, 100):
jobs.append(str(num))
task = await wpool.submit_work(jobs)
assert await task.await_completion()
await wpool.stop()
assert len(worker.worked) == 100