initial checkin

max-mapper · max-mapper · commit 9987c172be67 · 2011-01-09T12:37:21.000-08:00
diff --git a/README.md b/README.md
@@ -0,0 +1,5 @@
+# Python client library for Google Refine
+
+This allows you to script Refine by creating projects from data files, applying extracted JSON operation histories against the data and then exporting the transformed data back out of Refine.
+
+Work in progress! More docs soon
diff --git a/dates.txt b/dates.txt
@@ -0,0 +1,4 @@
+Date
+7 December 2001
+July 1 2002
+10/20/10
diff --git a/operations.json b/operations.json
@@ -0,0 +1,28 @@
+[
+  {
+    "op": "core/text-transform",
+    "description": "Text transform on cells in column Date using expression grel:value.toDate()",
+    "engineConfig": {
+      "facets": [],
+      "mode": "row-based"
+    },
+    "columnName": "Date",
+    "expression": "grel:value.toDate()",
+    "onError": "set-to-blank",
+    "repeat": false,
+    "repeatCount": 10
+  },
+  {
+    "op": "core/text-transform",
+    "description": "Text transform on cells in column Date using expression grel:value.datePart(\"year\")+1",
+    "engineConfig": {
+      "facets": [],
+      "mode": "row-based"
+    },
+    "columnName": "Date",
+    "expression": "grel:value.datePart(\"year\")+1",
+    "onError": "set-to-blank",
+    "repeat": false,
+    "repeatCount": 10
+  }
+]
diff --git a/refine.py b/refine.py
@@ -0,0 +1,79 @@
+# requires installation of urllib2_file from https://github.com/seisen/urllib2_file/#readme
+
+import urllib2_file
+import urllib2, urlparse, os.path, time, json
+
+class Refine:
+  def __init__(self, server='http://127.0.0.1:3333'):
+    self.server = server[0,-1] if server.endswith('/') else server
+  
+  def new_project(self, file_path, options=None):
+    file_name = os.path.split(file_path)[-1]
+    project_name = options['project_name'] if options != None and 'project_name' in options else file_name
+    data = {
+      'project-file' : {
+        'fd' : open(file_path),
+        'filename' : file_name
+      },
+      'project-name' : project_name
+    }
+
+    response = urllib2.urlopen(self.server + '/command/core/create-project-from-upload', data)
+    response.read()
+    url_params = urlparse.parse_qs(urlparse.urlparse(response.geturl()).query)
+    if 'project' in url_params:
+      id = url_params['project'][0]
+      return RefineProject(self.server, id, project_name)
+    
+    # TODO: better error reporting
+    return None
+
+class RefineProject:
+  def __init__(self, server, id, project_name):
+    self.server = server
+    self.id = id
+    self.project_name = project_name
+  
+  def wait_until_idle(self, polling_delay=0.5):
+    while True:
+      response = urllib2.urlopen(self.server + '/command/core/get-processes?project=' + self.id)
+      response_json = json.loads(response.read())
+      if 'processes' in response_json and len(response_json['processes']) > 0:
+        time.sleep(polling_delay)
+      else:
+        return
+  
+  def apply_operations(self, file_path, wait=True):
+    fd = open(file_path)
+    operations_json = fd.read()
+    
+    data = {
+      'operations' : operations_json
+    }
+    response = urllib2.urlopen(self.server + '/command/core/apply-operations?project=' + self.id, data)
+    response_json = json.loads(response.read())
+    if response_json['code'] == 'error':
+      raise Exception(response_json['message'])
+    elif response_json['code'] == 'pending':
+      if wait:
+        self.wait_until_idle()
+        return 'ok'
+    
+    return response_json['code'] # can be 'ok' or 'pending'
+  
+  def export_rows(self, format='tsv'):
+    data = {
+      'engine' : '{"facets":[],"mode":"row-based"}',
+      'project' : self.id,
+      'format' : format
+    }
+    response = urllib2.urlopen(self.server + '/command/core/export-rows/' + self.project_name + '.' + format, data)
+    return response.read()
+    
+  def delete_project(self):
+    data = {
+      'project' : self.id
+    }
+    response = urllib2.urlopen(self.server + '/command/core/delete-project', data)
+    response_json = json.loads(response.read())
+    return 'code' in response_json and response_json['code'] == 'ok'
diff --git a/refine.pyc b/refine.pyc
diff --git a/refineapi.md b/refineapi.md
@@ -0,0 +1,59 @@
+# Refine API
+
+When uploading files you will need to send the data as `multipart/form-data`, e.g.:
+
+      Content-Disposition: form-data; name="project-file"; filename="operations.json"
+
+      Content-Disposition: form-data; name="project-name"
+
+      myproject
+      
+The other operations are just normal POST parameters
+
+## Create project:
+
+POST /command/core/create-project-from-upload
+
+multipart form-data:
+
+      'project-file' : file contents...
+      'project-name' : project name...
+      
+Returns new project ID and other metadata
+
+## Apply operations
+
+POST /command/core/apply-operations
+
+multipart form-data:
+
+      'project' : project id...
+      'operations' : file contents...
+      
+Returns JSON response
+    
+## Export rows
+
+POST /command/core/export-rows
+
+      'engine' : JSON string... (e.g. '{"facets":[],"mode":"row-based"}')
+      'project' : project id...
+      'format' : format... (e.g 'tsv', 'csv')
+      
+Returns exported row data
+    
+## Delete project
+
+POST /command/core/delete-project
+
+      'project' : project id...
+      
+Returns JSON response
+    
+## Check status of async processes
+
+POST /command/core/get-processes
+
+      'project' : project id...
+      
+Returns JSON response
diff --git a/test.py b/test.py
@@ -0,0 +1,9 @@
+import sys
+sys.path.append("refine.py")
+import refine
+
+r = refine.Refine()
+p = r.new_project("dates.txt")
+p.apply_operations("operations.json")
+print p.export_rows()
+p.delete_project()
