implement logging to SAL via logmsgbot

eevans · eevans · commit 7564435eeff9 · 2016-10-21T14:33:33.000-05:00
diff --git a/README.markdown b/README.markdown
@@ -55,6 +55,8 @@ In a WMF multi-instance environment, iteratively execute nodetool on all local C
 #### Synopsis
     usage: c-foreach-restart [-h] [-a ATTEMPTS] [-r RETRY]
                              [--execute-post-shutdown CMD] [-d DELAY]
+                             [--logmsgbot LOGMSGBOT] [--tcpircbot-host HOST]
+                             [--tcpircbot-port PORT] [--phabricator-issue ISSUE]
     
     Cassandra instance restarter
     
@@ -72,7 +74,22 @@ In a WMF multi-instance environment, iteratively execute nodetool on all local C
       -d DELAY, --delay DELAY
                             Delay between instance restarts (defaults to no
                             delay).
-    
+      --logmsgbot LOGMSGBOT
+                            Log restarts to SAL (via logmsgbot and #wikimedia-
+                            operations).
+      --tcpircbot-host HOST
+                            tcpircbot hostname. Only valid when --logmsgbot is
+                            used. Default: neon.wikimedia.org
+      --tcpircbot-port PORT
+                            tcpircbot port number. Only valid when --logmsgbot is
+                            used. Default: 9200
+      --phabricator-issue ISSUE
+                            Phabricator issue to associate these restarts with.
+                            This currently only makes sense in combination with
+                            --logmsgbot where it is included in the formatted log
+                            message.
+
+
 #### Description
 In a WMF multi-instance environment, iteratively restart instances.
 
diff --git a/c-foreach-restart b/c-foreach-restart
@@ -6,16 +6,28 @@ import logging
 import sys
 import time
 
-from cassandra.tools import get_instances
-from cassandra.tools.config import LOG_LEVEL
+from cassandra.tools           import get_instances
+from cassandra.tools.config    import LOG_LEVEL
+from cassandra.tools.tcpircbot import IrcBot
 
 logging.basicConfig(level=LOG_LEVEL, format="%(asctime)s %(levelname)-8s %(message)s")
 
 
-def main(attempts, retry, delay, post_shutdown):
+def logmsgbot_message(instance, issue=None):
+    return "Restarting {0.service_name} {1}".format(instance, "({})".format(issue) if issue else "")
+
+def main():
+    args = parse_args()
+    bot = IrcBot(args.tcpircbot_host, args.tcpircbot_port, 1.0)
     for instance in get_instances():
-        instance.restart(attempts=attempts, retry=retry, post_shutdown=post_shutdown)
-        time.sleep(delay)
+        if args.logmsgbot:
+            bot.log(logmsgbot_message(instance, args.phabricator_issue))
+        instance.restart(
+            attempts=args.attempts,
+            retry=args.retry,
+            post_shutdown=args.execute_post_shutdown
+        )
+        time.sleep(args.delay)
 
 def parse_args():
     parser = argparse.ArgumentParser(description="Cassandra instance restarter")
@@ -26,11 +38,22 @@ def parse_args():
     parser.add_argument("--execute-post-shutdown", metavar="CMD", type=str,
                         help="Command to execute after Cassandra has been shutdown, and before it "
                         "is started back up.")
-    parser.add_argument("-d", "--delay", metavar="DELAY", type=int,
+    parser.add_argument("-d", "--delay", metavar="DELAY", type=float, default=0.0,
                         help="Delay between instance restarts (defaults to no delay).")
+    parser.add_argument("--logmsgbot", action="store_true",
+                        help="Log restarts to SAL (via logmsgbot and #wikimedia-operations).")
+    parser.add_argument("--tcpircbot-host", default="neon.wikimedia.org", metavar="HOST",
+                        help="tcpircbot hostname. Only valid when --logmsgbot is used. "
+                        "Default: neon.wikimedia.org")
+    parser.add_argument("--tcpircbot-port", metavar="PORT", type=int, default=9200,
+                        help="tcpircbot port number. Only valid when --logmsgbot is used. "
+                        "Default: 9200")
+    parser.add_argument("--phabricator-issue", metavar="ISSUE",
+                        help="Phabricator issue to associate these restarts with. This currently "
+                        "only makes sense in combination with --logmsgbot where it is included in "
+                        "the formatted log message.")
     return parser.parse_args(sys.argv[1:])
 
 
 if __name__ == "__main__":
-    args = parse_args()
-    main(args.attempts, args.retry, args.delay, args.execute_post_shutdown)
+    main()
diff --git a/cassandra/tools/instances.py b/cassandra/tools/instances.py
@@ -12,8 +12,8 @@
 from   time       import sleep
 import yaml
 
-from .config   import DESCRIPTOR_DIR
-from .nodetool import Nodetool
+from .config    import DESCRIPTOR_DIR
+from .nodetool  import Nodetool
 
 
 def __get_descriptor_files():
diff --git a/cassandra/tools/tcpircbot.py b/cassandra/tools/tcpircbot.py
@@ -1,5 +1,6 @@
 
 
+import getpass
 import logging
 import socket
 
@@ -19,11 +20,17 @@ def log(self, msg, *args, **kwargs):
             sock.connect((self.host, self.port))
             sock.sendall(IrcBot.format_message(msg, *args, **kwargs))
         except (socket.timeout, socket.error, socket.gaierror), err:
-            logging.error("Unable to send to logmsgbot (SAL): %s", err.message)
+            logging.error(
+                "Unable to send to logmsgbot (SAL): socket.error (%s) %s",
+                err.errno,
+                err.strerror
+            )
         finally:
             if sock:
                 sock.close()
 
     @classmethod
     def format_message(cls, msg, *args, **kwargs):
-        return "!log " + msg.format(*args, **kwargs)
+        username = getpass.getuser()
+        hostname = socket.gethostbyaddr(socket.gethostname())[0]
+        return "!log {}@{}: ".format(username, hostname) + msg.format(*args, **kwargs)
