forked from cool-RR/python_toolbox
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfuture_tools.py
More file actions
130 lines (101 loc) · 4.88 KB
/
future_tools.py
File metadata and controls
130 lines (101 loc) · 4.88 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# Copyright 2009-2017 Ram Rachum.
# This program is distributed under the MIT license.
'''
Defines tools related to the `concurrent.futures` standard library package.
'''
from __future__ import generator_stop
import time
import concurrent.futures
from python_toolbox import sequence_tools
class BaseCuteExecutor(concurrent.futures.Executor):
'''
An executor with extra functionality for `map` and `filter`.
This is a subclass of `concurrent.futures.Executor`, which is a manager for
parallelizing tasks. What this adds over `concurrent.futures.Executor`:
- A `.filter` method, which operates like the builtin `filter` except it's
parallelized with the executor.
- An `as_completed` argument for both `.map` and `.filter`, which makes
these methods return results according to the order in which they were
computed, and not the order in which they were submitted.
'''
def filter(self, filter_function, iterable, timeout=None,
as_completed=False):
'''
Get a parallelized version of `filter(filter_function, iterable)`.
Specify `as_completed=False` to get the results that were calculated
first to be returned first, instead of using the order of `iterable`.
'''
if timeout is not None:
end_time = timeout + time.time()
def make_future(item):
future = self.submit(filter_function, item)
future._item = item
return future
futures = tuple(map(make_future, iterable))
futures_iterator = concurrent.futures.as_completed(futures) if \
as_completed else futures
# Yield must be hidden in closure so that the futures are submitted
# before the first iterator value is required.
def result_iterator():
try:
for future in futures_iterator:
if timeout is None:
result = future.result()
else:
result = future.result(end_time - time.time())
if result:
yield future._item
finally:
for future in futures:
future.cancel()
return result_iterator()
def map(self, function, *iterables, timeout=None, as_completed=False):
'''
Get a parallelized version of `map(function, iterable)`.
Specify `as_completed=False` to get the results that were calculated
first to be returned first, instead of using the order of `iterable`.
'''
if timeout is not None:
end_time = timeout + time.time()
futures = [self.submit(function, *args) for args in zip(*iterables)]
futures_iterator = concurrent.futures.as_completed(futures) if \
as_completed else futures
# Yield must be hidden in closure so that the futures are submitted
# before the first iterator value is required.
def result_iterator():
try:
for future in futures_iterator:
if timeout is None:
yield future.result()
else:
yield future.result(end_time - time.time())
finally:
for future in futures:
future.cancel()
return result_iterator()
class CuteThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor,
BaseCuteExecutor):
'''
A thread-pool executor with extra functionality for `map` and `filter`.
This is a subclass of `concurrent.futures.ThreadPoolExecutor`, which is a
manager for parallelizing tasks to a thread pool. What this adds over
`concurrent.futures.ThreadPoolExecutor`:
- A `.filter` method, which operates like the builtin `filter` except it's
parallelized with the executor.
- An `as_completed` argument for both `.map` and `.filter`, which makes
these methods return results according to the order in which they were
computed, and not the order in which they were submitted.
'''
class CuteProcessPoolExecutor(concurrent.futures.ProcessPoolExecutor,
BaseCuteExecutor):
'''
A process-pool executor with extra functionality for `map` and `filter`.
This is a subclass of `concurrent.futures.ThreadPoolExecutor`, which is a
manager for parallelizing tasks to a process pool. What this adds over
`concurrent.futures.ThreadPoolExecutor`:
- A `.filter` method, which operates like the builtin `filter` except it's
parallelized with the executor.
- An `as_completed` argument for both `.map` and `.filter`, which makes
these methods return results according to the order in which they were
computed, and not the order in which they were submitted.
'''