Pull upstream EFF changes

Dockerfile

@@ -1,5 +1,7 @@
-FROM ubuntu:xenial
-RUN apt-get update && apt-get -y install python-pip python-dev npm git uwsgi libpq-dev curl unzip
+FROM python:2.7-buster
+RUN apt-get update && \
+  curl -sL https://deb.nodesource.com/setup_4.x | bash && \
+  apt-get -y install git uwsgi libpq-dev curl unzip nodejs npm
 
 RUN  mkdir /ngrok && \
      cd /ngrok && \
@@ -9,20 +11,13 @@ RUN  mkdir /ngrok && \
 
 WORKDIR /opt
 
-# We need to remove the os version of setuptools
-# It's incompatible with a dependency in gevent-psycopg2
-RUN easy_install -m setuptools
-RUN rm -r /usr/lib/python2.7/dist-packages/setuptools*
-RUN pip install setuptools
-
 ADD requirements.txt ./
 ADD requirements ./requirements
 RUN pip install -r requirements/production.txt -r requirements/development.txt
 
 ADD bower.json ./
 ADD .bowerrc ./
 RUN npm install -g bower
-RUN ln -s /usr/bin/nodejs /usr/bin/node
 RUN bower --allow-root --config.interactive=false install
 
 ADD alembic.ini manager.py Procfile uwsgi.ini entrypoint.sh ./

INSTALLATION.md

@@ -10,7 +10,7 @@ At a minimum, you will need to set:
 
 * SECRET_KEY, to secure login sessions cryptographically
     * This will be created for you automatically if you use the deploy to Heroku button, or you can generate one using with this Javascript one-liner: `chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-=!@#$%^&*()_+:<>{}[]".split(''); key = ''; for (i = 0; i < 50; i++) key += chars[Math.floor(Math.random() * chars.length)]; alert(key);`
-* SUNLIGHT_API_KEY, to do Congressional lookups. Sign up for one at [SunlightFoundation.com](https://sunlightfoundation.com/api/accounts/register/)
+* OPENSTATES_API_KEY, to do Congressional lookups. Sign up for one at [OpenStates.org](https://openstates.org/accounts/signup/)
 * TWILIO_ACCOUNT_SID, for an account with at least one purchased phone number
 * TWILIO_AUTH_TOKEN, for the same account
 * INSTALLED_ORG, displayed on the site homepage
@@ -66,8 +66,8 @@ To install locally and run in debug mode use:
     # create an admin user
     python manager.py createadminuser
 
-    # if testing twilio, run in another tab
-    ngrok http 5000
+    # if testing twilio, run in another tab -- you will have to sign up for an ngrok account to get your auth token
+    ngrok http --authtoken=$YOUR_AUTH_TOKEN 5000
 
     # run local server for debugging, pass external name from ngrok
     python manager.py runserver --external=SERVERID.ngrok.io

README.md

@@ -26,7 +26,7 @@ Campaign Configuration
 1) Create a campaign with one of several types, to determine how callers are matched to targets.
 
 * _Executive_ connects callers to the Whitehouse Switchboard, or to a specific office if known
-* _Congress_ connects callers to their Senators, Representative, or both. Uses data from the Sunlight Foundation.
+* _Congress_ connects callers to their Senators, Representative, or both. Uses the OpenStates API.
 * _State_ connects callers to their Governor or State Legislators. Uses the OpenStates API.
 * _Local_ connects callers to a local official. Campaigners must enter these numbers in advance.
 * _Custom_ can connect callers to corporate offices, local officals, or any other phone number entered in advance.
@@ -54,9 +54,9 @@ Read detailed instrustions at [INSTALLATION.md](INSTALLATION.md)
 Political Data
 --------------
 
-Political data is downloaded from Sunlight as CSV files stored in this repository. These are read on startup and saved in a memory cache for fast local lookup.
+Political data is downloaded as CSV files stored in this repository. These are read on startup and saved in a memory cache for fast local lookup.
 
-To update these files with new data after elections, run `cd call_server/political_data/data && make clean && make`, and `python manager.py load_political_data`
+To update these files with new data after elections, run `cd call_server/political_data/data && make clean && make`, and `python manager.py loadpoliticaldata`
 
 
 Code License

bower.json

@@ -25,7 +25,7 @@
     "bootstrap": "~3.3.5",
     "backbone": "~1.1.2",
     "html.sortable": "0.2.8",
-    "volume-meter": "https://github.com/cwilso/volume-meter.git",
+    "volume-meter": "https://github.com/cwilso/volume-meter.git#main",
     "audioRecord": "https://github.com/sb2702/audioRecord.js.git",
     "backbone-filtered-collection": "~0.4.0",
     "highcharts": "~4.1.7",

call_server/app.py

@@ -1,6 +1,22 @@
 import os
 import logging
 
+if os.environ.get('SENTRY_DSN'):
+    import sentry_sdk # must be loaded before Flask
+
+    sentry_sdk.init(
+        dsn=os.environ.get('SENTRY_DSN'),
+        environment=os.environ.get('SENTRY_ENVIRONMENT'),
+        send_default_pii=False,
+        # Set traces_sample_rate to 1.0 to capture 100%
+        # of transactions for tracing.
+        traces_sample_rate=float(os.environ.get('SENTRY_TRACES_SAMPLE_RATE')),
+        # Set profiles_sample_rate to 1.0 to profile 100%
+        # of sampled transactions.
+        # We recommend adjusting this value in production.
+        profiles_sample_rate=float(os.environ.get('SENTRY_PROFILES_SAMPLE_RATE')),
+    )
+
 from flask import Flask, g, request, session
 from flask.ext.assets import Bundle
 
@@ -190,8 +206,8 @@ def inject_sitename():
         return dict(SITENAME=app.config.get('SITENAME', 'CallPower'))
 
     @app.context_processor
-    def inject_sunlight_key():
-        return dict(SUNLIGHT_API_KEY=app.config.get('SUNLIGHT_API_KEY', ''))
+    def inject_openstates_key():
+        return dict(OPENSTATES_API_KEY=app.config.get('OPENSTATES_API_KEY', ''))
 
     # json filter
     app.jinja_env.filters['json'] = json_markup

call_server/call/views.py

@@ -5,7 +5,8 @@
 
 from flask import abort, Blueprint, request, url_for, current_app
 from flask_jsonpify import jsonify
-from twilio import TwilioRestException
+from twilio.base.exceptions import TwilioRestException
+from twilio.twiml.voice_response import VoiceResponse
 from sqlalchemy.exc import SQLAlchemyError
 
 from ..extensions import csrf, db
@@ -102,7 +103,7 @@ def intro_wait_human(params, campaign):
     Play intro message, and wait for key press to ensure we have a human on the line.
     Then, redirect to _make_calls.
     """
-    resp = twilio.twiml.Response()
+    resp = VoiceResponse()
 
     play_or_say(resp, campaign.audio('msg_intro'))
 
@@ -122,7 +123,7 @@ def intro_location_gather(params, campaign):
     If specified, play msg_intro_location audio. Otherwise, standard msg_intro.
     Then, return location_gather.
     """
-    resp = twilio.twiml.Response()
+    resp = VoiceResponse()
 
     if campaign.audio('msg_intro_location'):
         play_or_say(resp, campaign.audio('msg_intro_location'),
@@ -153,7 +154,7 @@ def make_calls(params, campaign):
 
     Required params: campaignId, targetIds
     """
-    resp = twilio.twiml.Response()
+    resp = VoiceResponse()
 
     # check if campaign target_set specified
     if not params['targetIds'] and campaign.target_set:
@@ -252,7 +253,6 @@ def create():
             to=userPhone,
             from_=from_number,
             url=url_for('call.connection', _external=True, **params),
-            timeLimit=current_app.config['TWILIO_TIME_LIMIT'],
             timeout=current_app.config['TWILIO_TIMEOUT'],
             status_callback=url_for("call.complete_status", _external=True, **params))
 
@@ -336,7 +336,7 @@ def location_parse():
         current_app.logger.debug('entered = {}'.format(location))
 
     if not target_ids:
-        resp = twilio.twiml.Response()
+        resp = VoiceResponse()
         play_or_say(resp, campaign.audio('msg_unparsed_location'))
 
         return location_gather(resp, params, campaign)
@@ -364,7 +364,7 @@ def make_single():
         db.session.add(current_target)
         db.session.commit()
 
-    resp = twilio.twiml.Response()
+    resp = VoiceResponse()
 
     if not current_target.number:
         play_or_say(resp, campaign.audio('msg_invalid_location'))
@@ -413,7 +413,7 @@ def complete():
     except SQLAlchemyError:
         current_app.logger.error('Failed to log call:', exc_info=True)
 
-    resp = twilio.twiml.Response()
+    resp = VoiceResponse()
 
     if call_data['status'] == 'busy':
         play_or_say(resp, campaign.audio('msg_target_busy'),

call_server/campaign/views.py

@@ -9,7 +9,7 @@
 import sqlalchemy
 from sqlalchemy.sql import func, desc
 
-from twilio.util import TwilioCapability
+from twilio.jwt.client import ClientCapabilityToken
 
 from ..extensions import db
 from ..utils import choice_items, choice_keys, choice_values_flat, duplicate_object
@@ -145,7 +145,7 @@ def audio(campaign_id):
     form = CampaignAudioForm()
 
     twilio_client = current_app.config.get('TWILIO_CLIENT')
-    twilio_capability = TwilioCapability(*twilio_client.auth)
+    twilio_capability = ClientCapabilityToken(*twilio_client.auth)
     twilio_capability.allow_client_outgoing(current_app.config.get('TWILIO_PLAYBACK_APP'))
 
     for field in form:

call_server/config.py

@@ -1,6 +1,5 @@
 import os
 import twilio.rest
-import sunlight
 
 
 class DefaultConfig(object):
@@ -33,7 +32,7 @@ class DefaultConfig(object):
     STORE_PROVIDER = 'flask_store.providers.local.LocalProvider'
     STORE_DOMAIN = 'http://localhost:5000' # requires url scheme for Flask-store.absolute_url to work
 
-    TWILIO_CLIENT = twilio.rest.TwilioRestClient(
+    TWILIO_CLIENT = twilio.rest.Client(
         os.environ.get('TWILIO_ACCOUNT_SID'),
         os.environ.get('TWILIO_AUTH_TOKEN'))
     TWILIO_PLAYBACK_APP = os.environ.get('TWILIO_PLAYBACK_APP')
@@ -45,10 +44,7 @@ class DefaultConfig(object):
     SECRET_KEY = os.environ.get('SECRET_KEY')
 
     GEOCODE_API_KEY = os.environ.get('GEOCODE_API_KEY')
-    SUNLIGHT_API_KEY = os.environ.get('SUNLIGHT_API_KEY')
-    if not SUNLIGHT_API_KEY:
-        SUNLIGHT_API_KEY = os.environ.get('SUNLIGHT_KEY')
-    sunlight.config.API_KEY = SUNLIGHT_API_KEY
+    OPENSTATES_API_KEY = os.environ.get('OPENSTATES_API_KEY')
 
     LOG_PHONE_NUMBERS = True
 
@@ -79,7 +75,7 @@ class ProductionConfig(DefaultConfig):
     MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', True)
 
     SQLALCHEMY_POOL_SIZE = int(os.environ.get('SQLALCHEMY_POOL_SIZE', 5))
-    SQLALCHEMY_POOL_RECYCLE = os.environ.get('SQLALCHEMY_POOL_RECYCLE', 60 * 60)  # default 1 hour
+    SQLALCHEMY_POOL_RECYCLE = os.environ.get('SQLALCHEMY_POOL_RECYCLE', 60)  # default 1 hour
     SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URI')
 
     STORE_PROVIDER = 'flask_store.providers.s3.S3Provider'

call_server/political_data/adapters.py

@@ -30,26 +30,23 @@ def adapt(self, data):
 class OpenStatesData(object):
     def adapt(self, data):
         mapped = {}
-        mapped['name'] = data['full_name']
-        if data['chamber'] == "upper":
-            mapped['title'] = "Senator"
-        if data['chamber'] == "lower":
-            mapped['title'] = "Representative"
-        if type(data['offices']) == list and 'phone' in data['offices'][0]:
-            mapped['number'] = data['offices'][0]['phone']
-        elif type(data['offices']) == dict and 'phone' in data['offices']:
-            mapped['number'] = data['offices']['phone']
+        mapped['name'] = data['name']
+        mapped['title'] = data['current_role']['title']
+        if type(data['offices']) == list and 'voice' in data['offices'][0]:
+            mapped['number'] = data['offices'][0]['voice']
+        elif type(data['offices']) == dict and 'voice' in data['offices']:
+            mapped['number'] = data['offices']['voice']
         else:
             mapped['number'] = None
-        mapped['uid'] = data['leg_id']
+        mapped['uid'] = data['id']
 
         return mapped
 
 
 class GovernorAdapter(object):
     def adapt(self, data):
         mapped = {}
-        mapped['name'] = u'{first_name} {last_name}'.format(**data)
+        mapped['name'] = data['name']
         mapped['title'] = data['title']
         mapped['number'] = data['phone']
         mapped['uid'] = data['state']

call_server/political_data/countries/us.py

@@ -66,18 +66,22 @@ def _load_legislators(self):
     def _load_districts(self):
         """
         Load US congressional district data from saved file
-        Returns a dictionary keyed by zipcode to cache for fast lookup
+        Returns a list of dictionaries keyed by zipcode to cache for fast lookup
 
         eg us:zipcode:94612 = [{'state':'CA', 'house_district': 13}]
         or us:zipcode:54409 = [{'state':'WI', 'house_district': 7}, {'state':'WI', 'house_district': 8}]
         """
         districts = collections.defaultdict(list)
 
         with open('call_server/political_data/data/us_districts.csv') as f:
-            reader = csv.DictReader(
-                f, fieldnames=['zipcode', 'state', 'house_district'])
+            reader = csv.DictReader(f)
 
-            for d in reader:
+            for row in reader:
+                d = {
+                    'state': row['state_abbr'],
+                    'zipcode': row['zcta'],
+                    'house_district': row['cd']
+                }
                 cache_key = self.KEY_ZIPCODE.format(**d)
                 districts[cache_key].append(d)
 

call_server/political_data/countries/us_state.py

@@ -1,8 +1,9 @@
 import csv
 import collections
+import os
 import random
 
-from sunlight import openstates, response_cache
+from requests import Session
 from . import DataProvider
 
 from ..constants import US_STATE_NAME_DICT
@@ -16,8 +17,6 @@ class USStateData(DataProvider):
 
     def __init__(self, cache, api_cache=None):
         self.cache = cache
-        if api_cache:
-            response_cache.enable(api_cache)
 
     def _load_governors(self):
         """
@@ -77,7 +76,7 @@ def locate_governor(self, state):
         return [self.KEY_GOVERNOR.format(state=state)]
 
     def locate_targets(self, latlon, chambers=TARGET_CHAMBER_BOTH, order=ORDER_IN_ORDER, state=None):
-        """ Find all state legistlators for a location, as comma delimited (lat,lon)
+        """ Find all state legislators for a location, as comma delimited (lat,lon)
             Returns a list of cached openstate keys in specified order.
         """
         if type(latlon) == tuple:
@@ -89,28 +88,36 @@ def locate_targets(self, latlon, chambers=TARGET_CHAMBER_BOTH, order=ORDER_IN_OR
             except ValueError:
                 raise ValueError('USStateData requires location as lat,lon')
 
-        legislators = openstates.legislator_geo_search(lat, lon)
+        params = dict(lat=float(lat), lng=float(lon), include='offices')
+        s = Session()
+        s.headers.update({'X-Api-Key': os.environ.get('OPENSTATES_API_KEY')})
+        response = s.get("https://v3.openstates.org/people.geo", params=params)
+        s.close()
+        if response.status_code != 200:
+            if response.status_code == 404:
+                raise NotFound("Not found: {0}".format(response.url))
+            else:
+                raise Exception(response.text)
+
+        legislators = response.json()['results']
         targets = []
         senators = []
         house_reps = []
 
-        # save full legislator data to cache
-        # just uids to result list
         for l in legislators:
-            if not l['active']:
-                # don't include inactive legislators
-                continue
-
-            if state and (state.upper() != l['state'].upper()):
+            parts = l['jurisdiction']['id'].partition('state:')
+            state_abbr = parts[2].partition("/")[0]
+            if state and (state.upper() != state_abbr.upper()):
                 # limit to one state
                 continue
 
             cache_key = self.KEY_OPENSTATES.format(**l)
             self.cache_set(cache_key, l)
 
-            if l['chamber'] == 'upper':
+            # limit to state legislators only
+            if l['current_role']['org_classification'] == 'upper' and l['jurisdiction']['classification'] == 'state':
                 senators.append(cache_key)
-            if l['chamber'] == 'lower':
+            if l['current_role']['org_classification'] == 'lower' and l['jurisdiction']['classification'] == 'state':
                 house_reps.append(cache_key)
 
         if chambers == TARGET_CHAMBER_UPPER:

call_server/political_data/data/Makefile

@@ -7,7 +7,7 @@ legislators-current.yaml :
 	curl -k "https://raw.githubusercontent.com/unitedstates/congress-legislators/master/legislators-current.yaml" -o "legislators-current.yaml"
 
 us_districts.csv:
-	curl -k "http://assets.sunlightfoundation.com/data/districts.csv" -o "us_districts.csv"
+	curl -k "https://raw.githubusercontent.com/OpenSourceActivismTech/us_zipcodes_congress/master/zccd.csv" -o "us_districts.csv"
 
 us_states.csv:
 	curl -k "https://raw.githubusercontent.com/spacedogXYZ/us_governors_contact/master/data.csv" -o "us_states.csv"