python-telegram-bot · tsnoam · Jun 20, 2016 · May 29, 2016 · May 29, 2016 · May 29, 2016
diff --git a/requirements.txt b/requirements.txt
@@ -1 +1,3 @@
-future
+future>=0.15.2
+urllib3>=1.8.3
+certifi
diff --git a/telegram/ext/dispatcher.py b/telegram/ext/dispatcher.py
@@ -20,24 +20,47 @@
 
 import logging
 from functools import wraps
-from threading import Thread, BoundedSemaphore, Lock, Event, current_thread
+from threading import Thread, Lock, Event, current_thread
 from time import sleep
+from queue import Queue, Empty
 
-from queue import Empty
+from future.builtins import range
 
 from telegram import (TelegramError, NullHandler)
+from telegram.utils import request
 from telegram.ext.handler import Handler
 from telegram.utils.deprecate import deprecate
 
 logging.getLogger(__name__).addHandler(NullHandler())
 
-semaphore = None
-async_threads = set()
+ASYNC_QUEUE = Queue()
+ASYNC_THREADS = set()
 """:type: set[Thread]"""
-async_lock = Lock()
+ASYNC_LOCK = Lock()  # guards ASYNC_THREADS
 DEFAULT_GROUP = 0
 
 
+def _pooled():
+    """
+    A wrapper to run a thread in a thread pool
+    """
+    while 1:
+        try:
+            func, args, kwargs = ASYNC_QUEUE.get()
+
+        # If unpacking fails, the thread pool is being closed from Updater._join_async_threads
+        except TypeError:
+            logging.getLogger(__name__).debug("Closing run_async thread %s/%d" %
+                                              (current_thread().getName(), len(ASYNC_THREADS)))
+            break
+
+        try:
+            func(*args, **kwargs)
+
+        except:
+            logging.getLogger(__name__).exception("run_async function raised exception")
+
+
 def run_async(func):
     """
     Function decorator that will run the function in a new thread.
@@ -53,30 +76,11 @@ def run_async(func):
     #       set a threading.Event to notify caller thread
 
     @wraps(func)
-    def pooled(*pargs, **kwargs):
-        """
-        A wrapper to run a thread in a thread pool
-        """
-        try:
-            result = func(*pargs, **kwargs)
-        finally:
-            semaphore.release()
-
-            with async_lock:
-                async_threads.remove(current_thread())
-        return result
-
-    @wraps(func)
-    def async_func(*pargs, **kwargs):
+    def async_func(*args, **kwargs):
         """
         A wrapper to run a function in a thread
         """
-        thread = Thread(target=pooled, args=pargs, kwargs=kwargs)
-        semaphore.acquire()
-        with async_lock:
-            async_threads.add(thread)
-        thread.start()
-        return thread
+        ASYNC_QUEUE.put((func, args, kwargs))
 
     return async_func
 
@@ -107,11 +111,18 @@ def __init__(self, bot, update_queue, workers=4, exception_event=None):
         self.__stop_event = Event()
         self.__exception_event = exception_event or Event()
 
-        global semaphore
-        if not semaphore:
-            semaphore = BoundedSemaphore(value=workers)
-        else:
-            self.logger.debug('Semaphore already initialized, skipping.')
+        with ASYNC_LOCK:
+            if not ASYNC_THREADS:
+                if request.is_con_pool_initialized():
+                    raise RuntimeError('Connection Pool already initialized')
+
+                request.CON_POOL_SIZE = workers + 3
+                for i in range(workers):
+                    thread = Thread(target=_pooled, name=str(i))
+                    ASYNC_THREADS.add(thread)
+                    thread.start()
+            else:
+                self.logger.debug('Thread pool already initialized, skipping.')
 
     def start(self):
         """
@@ -131,7 +142,7 @@ def start(self):
         self.running = True
         self.logger.debug('Dispatcher started')
 
-        while True:
+        while 1:
             try:
                 # Pop update from update queue.
                 update = self.update_queue.get(True, 1)
@@ -145,7 +156,7 @@ def start(self):
                 continue
 
             self.logger.debug('Processing Update: %s' % update)
-            self.processUpdate(update)
+            self.process_update(update)
 
         self.running = False
         self.logger.debug('Dispatcher thread stopped')
@@ -160,7 +171,7 @@ def stop(self):
                 sleep(0.1)
             self.__stop_event.clear()
 
-    def processUpdate(self, update):
+    def process_update(self, update):
         """
         Processes a single update.
 
@@ -170,7 +181,7 @@ def processUpdate(self, update):
 
         # An error happened while polling
         if isinstance(update, TelegramError):
-            self.dispatchError(None, update)
+            self.dispatch_error(None, update)
 
         else:
             for group in self.groups:
@@ -185,7 +196,7 @@ def processUpdate(self, update):
                                          'Update.')
 
                         try:
-                            self.dispatchError(update, te)
+                            self.dispatch_error(update, te)
                         except Exception:
                             self.logger.exception('An uncaught error was raised while '
                                                   'handling the error')
@@ -271,7 +282,7 @@ def remove_error_handler(self, callback):
         if callback in self.error_handlers:
             self.error_handlers.remove(callback)
 
-    def dispatchError(self, update, error):
+    def dispatch_error(self, update, error):
         """
         Dispatches an error.
 

diff --git a/telegram/ext/updater.py b/telegram/ext/updater.py
@@ -309,7 +309,7 @@ def _gen_webhook_url(listen, port, url_path):
 
     def _bootstrap(self, max_retries, clean, webhook_url, cert=None):
         retries = 0
-        while True:
+        while 1:
 
             try:
                 if clean:
@@ -346,17 +346,16 @@ def stop(self):
 
         self.job_queue.stop()
         with self.__lock:
-            if self.running:
+            if self.running or dispatcher.ASYNC_THREADS:
                 self.logger.debug('Stopping Updater and Dispatcher...')
 
                 self.running = False
 
                 self._stop_httpd()
                 self._stop_dispatcher()
                 self._join_threads()
-                # async threads must be join()ed only after the dispatcher
-                # thread was joined, otherwise we can still have new async
-                # threads dispatched
+                # async threads must be join()ed only after the dispatcher thread was joined,
+                # otherwise we can still have new async threads dispatched
                 self._join_async_threads()
 
     def _stop_httpd(self):
@@ -372,13 +371,19 @@ def _stop_dispatcher(self):
         self.dispatcher.stop()
 
     def _join_async_threads(self):
-        with dispatcher.async_lock:
-            threads = list(dispatcher.async_threads)
-        total = len(threads)
-        for i, thr in enumerate(threads):
-            self.logger.debug('Waiting for async thread {0}/{1} to end'.format(i, total))
-            thr.join()
-            self.logger.debug('async thread {0}/{1} has ended'.format(i, total))
+        with dispatcher.ASYNC_LOCK:
+            threads = list(dispatcher.ASYNC_THREADS)
+            total = len(threads)
+
+            # Stop all threads in the thread pool by put()ting one non-tuple per thread
+            for i in range(total):
+                dispatcher.ASYNC_QUEUE.put(None)
+
+            for i, thr in enumerate(threads):
+                self.logger.debug('Waiting for async thread {0}/{1} to end'.format(i + 1, total))
+                thr.join()
+                dispatcher.ASYNC_THREADS.remove(thr)
+                self.logger.debug('async thread {0}/{1} has ended'.format(i + 1, total))
 
     def _join_threads(self):
         for thr in self.__threads: