Kafka watcher

smalyshev · smalyshev · commit 03411ce4a16c · 2016-04-15T18:02:47.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/handlers/Dump.py b/handlers/Dump.py
@@ -0,0 +1,8 @@
+class Dump(object):
+	
+	def __init__(self, **params):
+		print(params)
+		
+	def handle(self, topic, message):
+		print(topic)
+		print(message)
diff --git a/handlers/Memcached.py b/handlers/Memcached.py
@@ -0,0 +1,19 @@
+import memcache
+
+class Memcached:
+
+    def __init__(self, hostname, **params):
+        self.mc = memcache.Client([hostname])
+
+    def handle(self, topic, message):
+        cmd = message['cmd']
+        if not hasattr(self, cmd):
+            raise Exception("Unknown command: " + cmd)
+        getattr(self, cmd)(message)
+
+    def set(self, message):
+        print("Set {0}-{1}-{2}".format(message['key'].encode('utf-8'), message['val'].encode('utf-8'), int(message['ttl'])))
+        self.mc.set(message['key'].encode('utf-8'), message['val'].encode('utf-8'), int(message['ttl']) )
+
+    def delete(self, message):
+        self.mc.delete(message['key'])
diff --git a/kafka-watcher.py b/kafka-watcher.py
@@ -0,0 +1,44 @@
+#!/usr/bin/python
+import yaml
+import argparse
+from kafka import KafkaConsumer
+import imp
+import json
+import sys
+
+parser = argparse.ArgumentParser(description='Process cache relay commands from Kafka')
+parser.add_argument('--config', required=True, help='YAML configuration file')
+script_args = parser.parse_args()
+
+config_file = open(script_args.config)
+config = yaml.safe_load(config_file)
+
+if 'listeners' not in config:
+    raise Exception('listeners config required')
+
+topics = []
+handlers = {}
+for listener in config['listeners']:
+    topics.append(listener['topic'])
+    fp, pathname, description = imp.find_module(listener['handler'], ['handlers'])
+    mod = imp.load_module(listener['handler'], fp, pathname, description)
+    klass = getattr(mod, listener['handler'])
+    handlers[listener['topic']] = klass(**listener['params'])
+
+consumer = KafkaConsumer(*topics)
+for msg in consumer:
+    if msg.topic not in handlers:
+        print("Weird, unknown topic %s" % msg.topic)
+        continue
+    try:
+        data = json.loads(msg.value)
+    except ValueError:
+        data = None
+    if not data:
+        print("Could not parse data, meh")
+        continue
+#    try:
+    handlers[msg.topic].handle(msg.topic, data)
+#    except:
+#        e = sys.exc_info()
+#        print("Oops, something happened: " + str(e))
diff --git a/watcher.yaml b/watcher.yaml
@@ -0,0 +1,6 @@
+listeners:
+    - topic: wancache-purge
+      handler: Memcached
+      params: 
+        hostname: localhost:11211
+    
