Added email address to the MeetingRegistration model, and also to the fields imported from ietf.org/registration/attendees/NN . Changed the bin counting code for attendee-related stats to use names plus email as unique string, rather than only names.

levkowetz · levkowetz · commit b6b45166bfc3 · 2017-05-31T23:08:23.000Z
- Legacy-Id: 13481
diff --git a/ietf/bin/fetch-meeting-registration-data b/ietf/bin/fetch-meeting-registration-data
@@ -39,8 +39,8 @@ else:
     sys.exit(1)
 
 for meeting in meetings:
-    total = get_meeting_registration_data(meeting)
-    msg = "Fetched data for meeting {}: {} registrations added".format(meeting.number, total)
+    added, processed, total = get_meeting_registration_data(meeting)
+    msg = "Fetched data for meeting {}: {} processed, {} added, {} in table".format(meeting.number, processed, added, total)
     if sys.stdout.isatty():
         print(msg) # make debugging a bit easier
     else:
diff --git a/ietf/stats/backfill_data.py b/ietf/stats/backfill_data.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 import sys, os, argparse
 
 basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
diff --git a/ietf/stats/models.py b/ietf/stats/models.py
@@ -50,6 +50,7 @@ class MeetingRegistration(models.Model):
     affiliation = models.CharField(blank=True, max_length=255)
     country_code = models.CharField(max_length=2)        # ISO 3166
     person = models.ForeignKey(Person, blank=True, null=True)
+    email =  models.EmailField(blank=True, null=True)
     
     def __unicode__(self):
         return u"{} {}".format(self.first_name, self.last_name)
diff --git a/ietf/stats/resources.py b/ietf/stats/resources.py
@@ -63,6 +63,7 @@ class Meta:
             "last_name": ALL,
             "affiliation": ALL,
             "country_code": ALL,
+            "email": ALL,
             "person": ALL_WITH_RELATIONS
         }
 api.stats.register(MeetingRegistrationResource())
diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py
@@ -102,9 +102,9 @@ def test_meeting_stats(self):
         # create some data for the statistics
         make_test_data()
         meeting = MeetingFactory(type_id='ietf', date=datetime.date.today(), number="96")
-        MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', meeting=meeting)
+        MeetingRegistration.objects.create(first_name='John', last_name='Smith', country_code='US', email="john.smith@example.us", meeting=meeting)
         CountryAlias.objects.get_or_create(alias="US", country=CountryName.objects.get(slug="US"))
-        MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', meeting=meeting)
+        MeetingRegistration.objects.create(first_name='Jaume', last_name='Guillaume', country_code='FR', email="jaume.guillaume@example.fr", meeting=meeting)
         CountryAlias.objects.get_or_create(alias="FR", country=CountryName.objects.get(slug="FR"))
 
         # check redirect
@@ -193,7 +193,7 @@ def test_review_stats(self):
     def test_get_meeting_registration_data(self, mock_get):
         response = Response()
         response.status_code = 200
-        response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US"}]'
+        response._content = '[{"LastName":"Smith","FirstName":"John","Company":"ABC","Country":"US","Email":"john.doe@example.us"}]'
         mock_get.return_value = response
         meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016,7,14), number="96")
         get_meeting_registration_data(meeting)
diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py
@@ -214,24 +214,34 @@ def get_meeting_registration_data(meeting):
     """"Retrieve registration attendee data and summary statistics.  Returns number
     of Registration records created."""
     num_created = 0
+    num_processed = 0
     response = requests.get(settings.REGISTRATION_ATTENDEES_BASE_URL + meeting.number)
     if response.status_code == 200:
         decoded = []
         try:
             decoded = response.json()
         except ValueError:
-            pass
+            if response.content.strip() == 'Invalid meeting':
+                pass
+            else:
+                raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], ))
 
         for registration in decoded:
             object, created = MeetingRegistration.objects.get_or_create(
                 meeting_id=meeting.pk,
                 first_name=registration['FirstName'],
                 last_name=registration['LastName'],
                 affiliation=registration['Company'],
-                country_code=registration['Country'])
+                country_code=registration['Country'],
+                email=registration['Email'],
+            )
             if created:
                 num_created += 1
-    return num_created
+            num_processed += 1
+    else:
+        raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content))
+    num_total = MeetingRegistration.objects.filter(meeting_id=meeting.pk).count()
+    return num_created, num_processed, num_total
     
             
     
diff --git a/ietf/stats/views.py b/ietf/stats/views.py
@@ -1,8 +1,10 @@
+import os
+import calendar
 import datetime
+import email.utils
 import itertools
 import json
-import calendar
-import os
+
 from collections import defaultdict
 
 from django.shortcuts import render
@@ -780,6 +782,9 @@ def get_country_mapping(registrations):
             if alias.alias.isupper()
         }
 
+    def reg_name(r):
+        return email.utils.formataddr(((r.first_name + u" " + r.last_name).strip(), r.email))
+
     if meeting and any(stats_type == t[0] for t in possible_stats_types):
         registrations = MeetingRegistration.objects.filter(meeting=meeting)
 
@@ -794,7 +799,7 @@ def get_country_mapping(registrations):
             eu_countries = set(CountryName.objects.filter(in_eu=True))
 
             for r in registrations:
-                name = (r.first_name + " " + r.last_name).strip()
+                name = reg_name(r)
                 c = country_mapping.get(r.country_code)
                 bins[c.name if c else ""].add(name)
 
@@ -831,7 +836,7 @@ def get_country_mapping(registrations):
             country_mapping = get_country_mapping(registrations)
 
             for r in registrations:
-                name = (r.first_name + " " + r.last_name).strip()
+                name = reg_name(r)
                 c = country_mapping.get(r.country_code)
                 bins[c.continent.name if c else ""].add(name)
 
@@ -853,16 +858,16 @@ def get_country_mapping(registrations):
     elif not meeting and any(stats_type == t[0] for t in possible_stats_types):
         template_name = "overview"
 
-        registrations = MeetingRegistration.objects.filter(meeting__type="ietf")
+        registrations = MeetingRegistration.objects.filter(meeting__type="ietf").select_related('meeting')
 
         if stats_type == "overview":
             stats_title = "Number of registrations per meeting"
 
             bins = defaultdict(set)
 
-            for first_name, last_name, meeting_number in registrations.values_list("first_name", "last_name", "meeting__number"):
-                meeting_number = int(meeting_number)
-                name = (first_name + " " + last_name).strip()
+            for r in registrations:
+                meeting_number = int(r.meeting.number)
+                name = reg_name(r)
 
                 bins[meeting_number].add(name)
 
@@ -891,10 +896,10 @@ def get_country_mapping(registrations):
 
             bins = defaultdict(set)
 
-            for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"):
-                meeting_number = int(meeting_number)
-                name = (first_name + " " + last_name).strip()
-                c = country_mapping.get(country_code)
+            for r in registrations:
+                meeting_number = int(r.meeting.number)
+                name = reg_name(r)
+                c = country_mapping.get(r.country_code)
 
                 if c:
                     bins[(meeting_number, c.name)].add(name)
@@ -911,10 +916,10 @@ def get_country_mapping(registrations):
 
             bins = defaultdict(set)
 
-            for first_name, last_name, country_code, meeting_number in registrations.values_list("first_name", "last_name", "country_code", "meeting__number"):
-                meeting_number = int(meeting_number)
-                name = (first_name + " " + last_name).strip()
-                c = country_mapping.get(country_code)
+            for r in registrations:
+                meeting_number = int(r.meeting.number)
+                name = reg_name(r)
+                c = country_mapping.get(r.country_code)
 
                 if c:
                     bins[(meeting_number, c.continent.name)].add(name)

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+#!/usr/bin/env python`
	`2`	`+`
`1`	`3`	`import sys, os, argparse`
`2`	`4`
`3`	`5`	`basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))`
Original file line number	Diff line number	Diff line change
`@@ -63,6 +63,7 @@ class Meta:`
`63`	`63`	`"last_name": ALL,`
`64`	`64`	`"affiliation": ALL,`
`65`	`65`	`"country_code": ALL,`
	`66`	`+ "email": ALL,`
`66`	`67`	`"person": ALL_WITH_RELATIONS`
`67`	`68`	`}`
`68`	`69`	`api.stats.register(MeetingRegistrationResource())`