Management command to import data from Google Sheets

ballot/admin.py

@@ -14,11 +14,7 @@ class PersonAdmin(admin.ModelAdmin):
     exclude = ('filer',)
 
 
-class BallotApiAdminSite(admin.AdminSite):
-    site_header = 'Open Disclosure Ballot API Admin'
-
-
-api_admin = BallotApiAdminSite(name='ballotapi-admin')
-api_admin.register(Election)
-api_admin.register(Person, PersonAdmin)
-api_admin.register(State)
+admin.site.site_header = 'Open Disclosure Ballot API Admin'
+admin.site.register(Election)
+admin.site.register(Person, PersonAdmin)
+admin.site.register(State)

ballot_api/settings.py

@@ -1,5 +1,8 @@
+import logging
 import os
 
+logging.basicConfig(level=logging.INFO)
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 SECRET_KEY = 'y%72v=z^op&=n4$ye^u$*n0!a=v*-qs_&)#rs*wf+m3up85)(#'
@@ -18,6 +21,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'ballot',
+    'gsheets',
     'rest_framework_swagger',
 )
 

ballot_api/urls.py

@@ -1,20 +1,20 @@
 from django.conf.urls import url, include
+from django.contrib import admin
 from rest_framework import routers
 from rest_framework_swagger.views import get_swagger_view
 
-from ballot.admin import api_admin
-from ballot.views import CandidateViewSet, ElectionViewSet
+import ballot.views as ballot_views
+import gsheets.views as gsheets_views
 
-
-router = routers.DefaultRouter()
-router.register(r'candidates', CandidateViewSet)
-router.register(r'elections', ElectionViewSet)
+router = routers.SimpleRouter()
+router.register(r'elections', ballot_views.ElectionViewSet)
+router.register(r'candidates', gsheets_views.CandidateViewSet)
 
 schema_view = get_swagger_view(title='Open Disclosure Ballot API')
 
 urlpatterns = [
     url(r'^api/', include(router.urls)),
     url(r'^$', schema_view),
     url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
-    url(r'^admin/', api_admin.urls),
+    url(r'^admin/', include(admin.site.urls)),
 ]

gsheets/admin.py

@@ -0,0 +1,18 @@
+from django.contrib import admin
+
+from .models import Candidate, CandidateAlias, Committee, Referendum, ReferendumMapping
+
+
+class CandidateAliasInline(admin.TabularInline):
+    model = CandidateAlias
+
+
+class CandidateAdmin(admin.ModelAdmin):
+    inlines = (CandidateAliasInline,)
+
+
+admin.site.register(Candidate, CandidateAdmin)
+admin.site.register(CandidateAlias)
+admin.site.register(Referendum)
+admin.site.register(ReferendumMapping)
+admin.site.register(Committee)

gsheets/management/commands/gsheets_import.py

@@ -0,0 +1,93 @@
+import csv
+import io
+import logging
+import re
+import tempfile
+import urllib.request
+
+from django.core.management.base import BaseCommand
+
+from gsheets import parsers
+
+
+CANDIDATES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=0&single=true&output=csv' # noqa
+REFERENDUMS_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1693935349&single=true&output=csv' # noqa
+REFERENDUM_NAME_TO_NUMBER_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=896561174&single=true&output=csv' # noqa
+COMMITTEES_SHEET = 'https://docs.google.com/spreadsheets/d/1272oaLyQhKwQa6RicA5tBso6wFruum-mgrNm3O3VogI/pub?gid=1995437960&single=true&output=csv' # noqa
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Import data from Google Sheets'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--force',
+            action='store_true',
+            default=False,
+            help="Import the entity even if it already exists.")
+
+    def format_csv_fields(self, row):
+        """Lowercases the keys of row and removes any special characters"""
+
+        # Lowercase the keys
+        model = ((k.lower(), v) for k, v in row.items())
+
+        # Replace spaces with underscore
+        model = ((re.sub(r'\s+', '_', k), v) for k, v in model)
+
+        # Strip other characters
+        model = ((re.sub(r'[^a-z_]', '', k), v) for k, v in model)
+
+        return dict(model)
+
+    def handle(self, *args, **options):
+        self.fetch_and_parse_from_url(CANDIDATES_SHEET, parsers.CandidateParser(), **options)
+        self.fetch_and_parse_from_url(COMMITTEES_SHEET, parsers.CommitteeParser(), **options)
+        self.fetch_and_parse_from_url(REFERENDUMS_SHEET, parsers.ReferendumParser(), **options)
+        self.fetch_and_parse_from_url(REFERENDUM_NAME_TO_NUMBER_SHEET, parsers.ReferendumMappingParser(), **options)
+
+    def fetch_and_parse_from_url(self, url, parser, force=False, **options):
+        with urllib.request.urlopen(url) as request:
+            with tempfile.TemporaryFile() as csvfile:
+                csvfile.write(request.read())
+                csvfile.seek(0)
+                reader = csv.DictReader(io.TextIOWrapper(csvfile, encoding='utf-8'))
+
+                total = 0
+                imported = 0
+                import_errors = 0
+                for row in reader:
+                    total += 1
+
+                    row = self.format_csv_fields(row)
+                    logger.debug(row)
+
+                    exists = parser.exists_in_db(row)
+                    if exists and not force:
+                        # Skip this row
+                        continue
+
+                    id = row.get(parser.key)
+
+                    try:
+                        data = parser.parse(row)
+                        logger.debug(data)
+                        model, created = parser.commit(data)
+                    except Exception as err:
+                        import_errors += 1
+                        logger.error('%s "%s" could not be parsed: parse_errors=%s row=%s',
+                                     parser.name, id, err, row)
+                        logger.exception(err)
+                        continue
+
+                    imported += 1
+                    if created:
+                        logger.info('Created %s "%s"', parser.name, id)
+                    else:
+                        logger.info('Updated %s "%s"', parser.name, id)
+
+                logger.info('Import %s data complete: total=%s imported=%s errors=%s',
+                            parser.name, total, imported, import_errors)

gsheets/migrations/0001_initial.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Candidate',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('fppc', models.IntegerField(blank=True, unique=True, null=True)),
+                ('committee_name', models.CharField(blank=True, max_length=120)),
+                ('candidate', models.CharField(max_length=30)),
+                ('office', models.CharField(blank=True, help_text='Office the candidate is running for.', max_length=30)),
+                ('incumbent', models.BooleanField(default=False)),
+                ('accepted_expenditure_ceiling', models.BooleanField(default=False)),
+                ('website', models.URLField(blank=True, null=True)),
+                ('twitter', models.URLField(blank=True, null=True)),
+                ('party_affiliation', models.CharField(blank=True, max_length=15)),
+                ('occupation', models.CharField(blank=True, max_length=80)),
+                ('bio', models.TextField(blank=True)),
+                ('photo', models.URLField(blank=True, null=True)),
+                ('votersedge', models.URLField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Committee',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('filer_id', models.CharField(max_length=15)),
+                ('filer_naml', models.CharField(blank=True, max_length=200)),
+                ('committee_type', models.CharField(choices=[('RCP', 'RCP'), ('BMC', 'BMC')], blank=True, max_length=3)),
+                ('description', models.CharField(blank=True, max_length=40)),
+                ('ballot_measure', models.CharField(blank=True, max_length=3)),
+                ('support_or_oppose', models.CharField(choices=[('S', 'Support'), ('O', 'Oppose')], blank=True, max_length=1)),
+                ('website', models.URLField(blank=True, null=True)),
+                ('twitter', models.URLField(blank=True, null=True)),
+                ('facebook', models.URLField(blank=True, null=True)),
+                ('netfilelocalid', models.CharField(blank=True, max_length=15)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Referendum',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('measure_number', models.CharField(max_length=5)),
+                ('short_title', models.CharField(blank=True, max_length=30)),
+                ('full_title', models.TextField(blank=True)),
+                ('summary', models.TextField()),
+                ('votersedge', models.URLField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ReferendumMapping',
+            fields=[
+                ('id', models.AutoField(primary_key=True, verbose_name='ID', auto_created=True, serialize=False)),
+                ('measure_name', models.CharField(max_length=30)),
+                ('measure_number', models.CharField(max_length=3)),
+            ],
+        ),
+    ]

gsheets/migrations/0002_auto_20170205_0455.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gsheets', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CandidateAlias',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)),
+                ('candidate_alias', models.CharField(max_length=30)),
+                ('candidate', models.ForeignKey(to='gsheets.Candidate', related_name='aliases')),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='referendum',
+            name='short_title',
+            field=models.CharField(max_length=200, blank=True),
+        ),
+        migrations.AlterField(
+            model_name='referendummapping',
+            name='measure_name',
+            field=models.CharField(max_length=200),
+        ),
+    ]

gsheets/migrations/0003_auto_20170205_0802.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gsheets', '0002_auto_20170205_0455'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='candidatealias',
+            options={'verbose_name_plural': 'candidate aliases'},
+        ),
+        migrations.AlterField(
+            model_name='candidate',
+            name='candidate',
+            field=models.CharField(max_length=30, help_text="The candidate's full name."),
+        ),
+        migrations.AlterField(
+            model_name='candidate',
+            name='party_affiliation',
+            field=models.CharField(max_length=1, choices=[('D', 'Democrat'), ('R', 'Republican'), ('I', 'Independent'), ('O', 'Other')], blank=True),
+        ),
+    ]

gsheets/migrations/0004_auto_20170329_0408.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gsheets', '0003_auto_20170205_0802'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='referendummapping',
+            name='measure_number',
+            field=models.CharField(max_length=10),
+        ),
+    ]