Auto reconnect RTM (slackapi#297)

Roach · web-flow · commit 07cce9bf15be · 2018-03-20T16:53:13.000-07:00
* Added reconnect logic to RTM client * Added a lot more tests * updated RTM test fixture * Disable testing for 3.6 on Windows until 3.6.5 release due to a windows web socket bug: https://bugs.python.org/issue32394
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -10,8 +10,6 @@ environment:
       PYTHON_VERSION: "py34-x86"
     - PYTHON: "C:\\Python35"
       PYTHON_VERSION: "py35-x86"
-    - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "py36-x86"
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "py27-x64"
     - PYTHON: "C:\\Python33-x64"
@@ -20,8 +18,6 @@ environment:
       PYTHON_VERSION: "py34-x64"
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "py35-x64"
-    - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "py36-x64"
 
 install:
   - "%PYTHON%\\python.exe -m pip install wheel"
diff --git a/docs-src/real_time_messaging.rst b/docs-src/real_time_messaging.rst
@@ -23,7 +23,7 @@ Connecting to the Real Time Messaging API
   sc = SlackClient(slack_token)
 
   if sc.rtm_connect():
-      while True:
+    while sc.server.connected is True:
           print sc.rtm_read()
           time.sleep(1)
   else:
@@ -70,6 +70,9 @@ To do this, simply pass `with_team_state=False` into the `rtm_connect` call, lik
       print "Connection Failed"
 
 
+Passing `auto_reconnect=True` will tell the websocket client to automatically reconnect if the connection gets dropped.
+
+
 See the `rtm.start docs <https://api.slack.com/methods/rtm.start>`_ and the `rtm.connect docs <https://api.slack.com/methods/rtm.connect>`_
 for more details.
 
diff --git a/docs/real_time_messaging.html b/docs/real_time_messaging.html
@@ -155,7 +155,7 @@ <h2>Connecting to the Real Time Messaging API<a class="headerlink" href="#connec
 <span class="n">sc</span> <span class="o">=</span> <span class="n">SlackClient</span><span class="p">(</span><span class="n">slack_token</span><span class="p">)</span>
 
 <span class="k">if</span> <span class="n">sc</span><span class="o">.</span><span class="n">rtm_connect</span><span class="p">():</span>
-    <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+  <span class="k">while</span> <span class="n">sc</span><span class="o">.</span><span class="n">server</span><span class="o">.</span><span class="n">connected</span> <span class="ow">is</span> <span class="bp">True</span><span class="p">:</span>
         <span class="k">print</span> <span class="n">sc</span><span class="o">.</span><span class="n">rtm_read</span><span class="p">()</span>
         <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
 <span class="k">else</span><span class="p">:</span>
@@ -198,6 +198,7 @@ <h2>rtm.start vs rtm.connect<a class="headerlink" href="#rtm-start-vs-rtm-connec
     <span class="k">print</span> <span class="s2">&quot;Connection Failed&quot;</span>
 </pre></div>
 </div>
+<p>Passing <cite>auto_reconnect=True</cite> will tell the websocket client to automatically reconnect if the connection gets dropped.</p>
 <p>See the <a class="reference external" href="https://api.slack.com/methods/rtm.start">rtm.start docs</a> and the <a class="reference external" href="https://api.slack.com/methods/rtm.connect">rtm.connect docs</a>
 for more details.</p>
 </div>
diff --git a/docs/searchindex.js b/docs/searchindex.js
diff --git a/slackclient/client.py b/slackclient/client.py
@@ -50,7 +50,7 @@ def rtm_connect(self, with_team_state=True, **kwargs):
 
         try:
             self.server.rtm_connect(use_rtm_start=with_team_state, **kwargs)
-            return True
+            return self.server.connected
         except Exception:
             traceback.print_exc()
             return False
@@ -90,24 +90,14 @@ def api_call(self, method, timeout=None, **kwargs):
             result = json.loads(response_body)
         except ValueError as json_decode_error:
             raise ParseResponseError(response_body, json_decode_error)
-        if self.server:
+
+        if "ok" in result and result["ok"]:
             if method == 'im.open':
-                if "ok" in result and result["ok"]:
-                    self.server.attach_channel(kwargs["user"], result["channel"]["id"])
+                self.server.attach_channel(kwargs["user"], result["channel"]["id"])
             elif method in ('mpim.open', 'groups.create', 'groups.createchild'):
-                if "ok" in result and result["ok"]:
-                    self.server.attach_channel(
-                        result['group']['name'],
-                        result['group']['id'],
-                        result['group']['members']
-                    )
+                self.server.parse_channel_data([result['group']])
             elif method in ('channels.create', 'channels.join'):
-                if 'ok' in result and result['ok']:
-                    self.server.attach_channel(
-                        result['channel']['name'],
-                        result['channel']['id'],
-                        result['channel']['members']
-                    )
+                self.server.parse_channel_data([result['channel']])
         return result
 
     def rtm_read(self):
diff --git a/slackclient/server.py b/slackclient/server.py
@@ -1,13 +1,18 @@
-from .slackrequest import SlackRequest
-from requests.packages.urllib3.util.url import parse_url
 from .channel import Channel
+from .exceptions import SlackClientError
+from .slackrequest import SlackRequest
 from .user import User
 from .util import SearchList, SearchDict
-from .exceptions import SlackClientError
-from ssl import SSLError
 
-from websocket import create_connection
 import json
+import logging
+import time
+import random
+
+from requests.packages.urllib3.util.url import parse_url
+from ssl import SSLError
+from websocket import create_connection
+from websocket._exceptions import WebSocketConnectionClosedException
 
 
 class Server(object):
@@ -17,18 +22,27 @@ class Server(object):
 
     """
     def __init__(self, token, connect=True, proxies=None):
+        # Slack client configs
         self.token = token
+        self.proxies = proxies
+        self.api_requester = SlackRequest(proxies=proxies)
+
+        # Workspace metadata
         self.username = None
         self.domain = None
         self.login_data = None
-        self.websocket = None
         self.users = SearchDict()
         self.channels = SearchList()
-        self.connected = False
+
+        # RTM configs
+        self.websocket = None
         self.ws_url = None
-        self.proxies = proxies
-        self.api_requester = SlackRequest(proxies=proxies)
+        self.connected = False
+        self.last_connected_at = 0
+        self.auto_reconnect = False
+        self.reconnect_attempt = 0
 
+        # Connect to RTM on load
         if connect:
             self.rtm_connect()
 
@@ -68,8 +82,51 @@ def append_user_agent(self, name, version):
         self.api_requester.append_user_agent(name, version)
 
     def rtm_connect(self, reconnect=False, timeout=None, use_rtm_start=True, **kwargs):
+        """
+        Connects to the RTM API - https://api.slack.com/rtm
+
+        If `auto_reconnect` is set to `True` then the SlackClient is initialized, this method
+        will be used to reconnect on websocket read failures, which indicate disconnection
+
+        :Args:
+            reconnect (boolean) Whether this method is being called to reconnect to RTM
+            timeout (int): Timeout for Web API calls
+            use_rtm_start (boolean): `True` to connect using `rtm.start` or
+            `False` to connect using`rtm.connect`
+            https://api.slack.com/rtm#connecting_with_rtm.connect_vs._rtm.start
+
+        :Returns:
+            None
+
+        """
+
         # rtm.start returns user and channel info, rtm.connect does not.
         connect_method = "rtm.start" if use_rtm_start else "rtm.connect"
+
+        # If the `auto_reconnect` param was passed, set the server's `auto_reconnect` attr
+        if kwargs and kwargs["auto_reconnect"] is True:
+            self.auto_reconnect = True
+
+        # If this is an auto reconnect, rate limit reconnect attempts
+        if self.auto_reconnect and reconnect:
+            # Raise a SlackConnectionError after 5 retries within 3 minutes
+            recon_attempt = self.reconnect_attempt
+            if recon_attempt == 5:
+                logging.error("RTM connection failed, reached max reconnects.")
+                raise SlackConnectionError("RTM connection failed, reached max reconnects.")
+            # Wait to reconnect if the last reconnect was more than 3 minutes ago
+            if (time.time() - self.last_connected_at) < 180:
+                if recon_attempt > 0:
+                    # Back off after the the first attempt
+                    backoff_offset_multiplier = random.randint(1, 4)
+                    retry_timeout = (backoff_offset_multiplier * recon_attempt * recon_attempt)
+                    logging.debug("Reconnecting in %d seconds", retry_timeout)
+
+                    time.sleep(retry_timeout)
+                self.reconnect_attempt += 1
+            else:
+                self.reconnect_attempt = 0
+
         reply = self.api_requester.do(self.token, connect_method, timeout=timeout, post_data=kwargs)
 
         if reply.status_code != 200:
@@ -111,8 +168,12 @@ def connect_slack_websocket(self, ws_url):
                                                http_proxy_host=proxy_host,
                                                http_proxy_port=proxy_port,
                                                http_proxy_auth=proxy_auth)
+            self.connected = True
+            self.last_connected_at = time.time()
+            logging.debug("RTM connected")
             self.websocket.sock.setblocking(0)
         except Exception as e:
+            self.connected = False
             raise SlackConnectionError(message=str(e))
 
     def parse_channel_data(self, channel_data):
@@ -202,6 +263,13 @@ def websocket_safe_read(self):
                     # SSLWantReadError
                     return ''
                 raise
+            except WebSocketConnectionClosedException as e:
+                logging.debug("RTM disconnected")
+                self.connected = False
+                if self.auto_reconnect:
+                    self.rtm_connect(reconnect=True)
+                else:
+                    raise SlackConnectionError("Unable to send due to closed RTM websocket")
             return data.rstrip()
 
     def attach_user(self, name, user_id, real_name, tz, email):
diff --git a/tests/data/rtm.start.json b/tests/data/rtm.start.json
@@ -245,13 +245,7 @@
             "deleted": false,
             "status": null,
             "color": "9f69e7",
-            "real_name": "",
-            "tz": "America\/Los_Angeles",
-            "tz_label": "Pacific Daylight Time",
-            "tz_offset": -25200,
             "profile": {
-                "real_name": "",
-                "real_name_normalized": "",
                 "email": "fakeuser@example.com",
                 "image_24": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=24&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002-24.png",
                 "image_32": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=32&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002-32.png",
@@ -268,6 +262,28 @@
             "has_files": false,
             "presence": "away"
         },
+        {
+            "id": "U10CX1235",
+            "name": "userwithoutemail",
+            "deleted": false,
+            "status": null,
+            "color": "9f69e7",
+            "profile": {
+                "image_24": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=24&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002-24.png",
+                "image_32": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=32&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002-32.png",
+                "image_48": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=48&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F272a%2Fimg%2Favatars%2Fava_0002-48.png",
+                "image_72": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=72&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002-72.png",
+                "image_192": "https:\/\/secure.gravatar.com\/avatar\/4f1bd7fd71e645fa19620504b4c0e3ba.jpg?s=192&d=https%3A%2F%2Fslack.global.ssl.fastly.net%2F3654%2Fimg%2Favatars%2Fava_0002.png"
+            },
+            "is_admin": true,
+            "is_owner": true,
+            "is_primary_owner": true,
+            "is_restricted": false,
+            "is_ultra_restricted": false,
+            "is_bot": false,
+            "has_files": false,
+            "presence": "away"
+        },
         {
             "id": "USLACKBOT",
             "name": "slackbot",
@@ -301,5 +317,5 @@
     ],
     "bots": [],
     "cache_version": "v5-dog",
-    "url": "wss:\/\/ms9999.slack-msgs.com\/websocket\/rvyiQ_oxNhQ2C6_613rtqs1PFfT0AmivZTokv\/VOVQCmq3bk\/KarC2Z2ZMFfdMMtxn4kx9ILl6sE7JgvKv6Bct5okT0Lgru416DXsKJolJQ="
+    "url": "wss:\/\/cerberus-xxxx.lb.slack-msgs.com\/websocket\/ifkp3MKfNXd6ftbrEGllwcHn"
 }
diff --git a/tests/test_server.py b/tests/test_server.py
diff --git a/tests/test_slackclient.py b/tests/test_slackclient.py
