Skip to content

Commit e0539d5

Browse files
tsnoamtsnoam
authored
Merge pull request python-telegram-bot#327 from python-telegram-bot/urllib3
Urllib3
2 parents 4f101a7 + b41f7e3 commit e0539d5

File tree

7 files changed

+164
-113
lines changed

7 files changed

+164
-113
lines changed

‎requirements.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
future
1+
future>=0.15.2
2+
urllib3>=1.8.3
3+
certifi

‎telegram/ext/dispatcher.py‎

Lines changed: 48 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,47 @@
2020

2121
import logging
2222
from functools import wraps
23-
from threading import Thread, BoundedSemaphore, Lock, Event, current_thread
23+
from threading import Thread, Lock, Event, current_thread
2424
from time import sleep
25+
from queue import Queue, Empty
2526

26-
from queue import Empty
27+
from future.builtins import range
2728

2829
from telegram import (TelegramError, NullHandler)
30+
from telegram.utils import request
2931
from telegram.ext.handler import Handler
3032
from telegram.utils.deprecate import deprecate
3133

3234
logging.getLogger(__name__).addHandler(NullHandler())
3335

34-
semaphore = None
35-
async_threads = set()
36+
ASYNC_QUEUE = Queue()
37+
ASYNC_THREADS = set()
3638
""":type: set[Thread]"""
37-
async_lock = Lock()
39+
ASYNC_LOCK = Lock() # guards ASYNC_THREADS
3840
DEFAULT_GROUP = 0
3941

4042

43+
def _pooled():
44+
"""
45+
A wrapper to run a thread in a thread pool
46+
"""
47+
while 1:
48+
try:
49+
func, args, kwargs = ASYNC_QUEUE.get()
50+
51+
# If unpacking fails, the thread pool is being closed from Updater._join_async_threads
52+
except TypeError:
53+
logging.getLogger(__name__).debug("Closing run_async thread %s/%d" %
54+
(current_thread().getName(), len(ASYNC_THREADS)))
55+
break
56+
57+
try:
58+
func(*args, **kwargs)
59+
60+
except:
61+
logging.getLogger(__name__).exception("run_async function raised exception")
62+
63+
4164
def run_async(func):
4265
"""
4366
Function decorator that will run the function in a new thread.
@@ -53,30 +76,11 @@ def run_async(func):
5376
# set a threading.Event to notify caller thread
5477

5578
@wraps(func)
56-
def pooled(*pargs, **kwargs):
57-
"""
58-
A wrapper to run a thread in a thread pool
59-
"""
60-
try:
61-
result = func(*pargs, **kwargs)
62-
finally:
63-
semaphore.release()
64-
65-
with async_lock:
66-
async_threads.remove(current_thread())
67-
return result
68-
69-
@wraps(func)
70-
def async_func(*pargs, **kwargs):
79+
def async_func(*args, **kwargs):
7180
"""
7281
A wrapper to run a function in a thread
7382
"""
74-
thread = Thread(target=pooled, args=pargs, kwargs=kwargs)
75-
semaphore.acquire()
76-
with async_lock:
77-
async_threads.add(thread)
78-
thread.start()
79-
return thread
83+
ASYNC_QUEUE.put((func, args, kwargs))
8084

8185
return async_func
8286

@@ -107,11 +111,18 @@ def __init__(self, bot, update_queue, workers=4, exception_event=None):
107111
self.__stop_event = Event()
108112
self.__exception_event = exception_event or Event()
109113

110-
global semaphore
111-
if not semaphore:
112-
semaphore = BoundedSemaphore(value=workers)
113-
else:
114-
self.logger.debug('Semaphore already initialized, skipping.')
114+
with ASYNC_LOCK:
115+
if not ASYNC_THREADS:
116+
if request.is_con_pool_initialized():
117+
raise RuntimeError('Connection Pool already initialized')
118+
119+
request.CON_POOL_SIZE = workers + 3
120+
for i in range(workers):
121+
thread = Thread(target=_pooled, name=str(i))
122+
ASYNC_THREADS.add(thread)
123+
thread.start()
124+
else:
125+
self.logger.debug('Thread pool already initialized, skipping.')
115126

116127
def start(self):
117128
"""
@@ -131,7 +142,7 @@ def start(self):
131142
self.running = True
132143
self.logger.debug('Dispatcher started')
133144

134-
while True:
145+
while 1:
135146
try:
136147
# Pop update from update queue.
137148
update = self.update_queue.get(True, 1)
@@ -145,7 +156,7 @@ def start(self):
145156
continue
146157

147158
self.logger.debug('Processing Update: %s' % update)
148-
self.processUpdate(update)
159+
self.process_update(update)
149160

150161
self.running = False
151162
self.logger.debug('Dispatcher thread stopped')
@@ -160,7 +171,7 @@ def stop(self):
160171
sleep(0.1)
161172
self.__stop_event.clear()
162173

163-
def processUpdate(self, update):
174+
def process_update(self, update):
164175
"""
165176
Processes a single update.
166177
@@ -170,7 +181,7 @@ def processUpdate(self, update):
170181

171182
# An error happened while polling
172183
if isinstance(update, TelegramError):
173-
self.dispatchError(None, update)
184+
self.dispatch_error(None, update)
174185

175186
else:
176187
for group in self.groups:
@@ -185,7 +196,7 @@ def processUpdate(self, update):
185196
'Update.')
186197

187198
try:
188-
self.dispatchError(update, te)
199+
self.dispatch_error(update, te)
189200
except Exception:
190201
self.logger.exception('An uncaught error was raised while '
191202
'handling the error')
@@ -271,7 +282,7 @@ def remove_error_handler(self, callback):
271282
if callback in self.error_handlers:
272283
self.error_handlers.remove(callback)
273284

274-
def dispatchError(self, update, error):
285+
def dispatch_error(self, update, error):
275286
"""
276287
Dispatches an error.
277288

‎telegram/ext/updater.py‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ def _gen_webhook_url(listen, port, url_path):
309309

310310
def _bootstrap(self, max_retries, clean, webhook_url, cert=None):
311311
retries = 0
312-
while True:
312+
while 1:
313313

314314
try:
315315
if clean:
@@ -346,17 +346,16 @@ def stop(self):
346346

347347
self.job_queue.stop()
348348
with self.__lock:
349-
if self.running:
349+
if self.running or dispatcher.ASYNC_THREADS:
350350
self.logger.debug('Stopping Updater and Dispatcher...')
351351

352352
self.running = False
353353

354354
self._stop_httpd()
355355
self._stop_dispatcher()
356356
self._join_threads()
357-
# async threads must be join()ed only after the dispatcher
358-
# thread was joined, otherwise we can still have new async
359-
# threads dispatched
357+
# async threads must be join()ed only after the dispatcher thread was joined,
358+
# otherwise we can still have new async threads dispatched
360359
self._join_async_threads()
361360

362361
def _stop_httpd(self):
@@ -372,13 +371,19 @@ def _stop_dispatcher(self):
372371
self.dispatcher.stop()
373372

374373
def _join_async_threads(self):
375-
with dispatcher.async_lock:
376-
threads = list(dispatcher.async_threads)
377-
total = len(threads)
378-
for i, thr in enumerate(threads):
379-
self.logger.debug('Waiting for async thread {0}/{1} to end'.format(i, total))
380-
thr.join()
381-
self.logger.debug('async thread {0}/{1} has ended'.format(i, total))
374+
with dispatcher.ASYNC_LOCK:
375+
threads = list(dispatcher.ASYNC_THREADS)
376+
total = len(threads)
377+
378+
# Stop all threads in the thread pool by put()ting one non-tuple per thread
379+
for i in range(total):
380+
dispatcher.ASYNC_QUEUE.put(None)
381+
382+
for i, thr in enumerate(threads):
383+
self.logger.debug('Waiting for async thread {0}/{1} to end'.format(i + 1, total))
384+
thr.join()
385+
dispatcher.ASYNC_THREADS.remove(thr)
386+
self.logger.debug('async thread {0}/{1} has ended'.format(i + 1, total))
382387

383388
def _join_threads(self):
384389
for thr in self.__threads:

0 commit comments

Comments
 (0)