Prep PPCS importer for GE

README.md

@@ -60,7 +60,6 @@ Sentry is used to report errors in production. We have added a url for `sentry-d
 ```
 
 # Pre-election Tasks
- <!-- TO DO -->
 
 # Enable Candidate Leaderboard
 
@@ -70,4 +69,6 @@ We take a slice of edits in YNR and assign them to a election leaderboard.
 
 This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20
 
-We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy]
+We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy]
+
+If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL`

ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py

@@ -2,23 +2,27 @@
 
 import requests
 from candidatebot.helpers import CandidateBot
+from candidates.models import Ballot
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from elections.models import Election
 from parties.models import Party
 from people.models import Person
 from popolo.models import Membership, NotStandingValidationError
+from slugify import slugify
 
 
 class Command(BaseCommand):
     help = "Import candidates to the next parl election from a Google sheet"
 
     def add_arguments(self, parser):
         parser.add_argument("--sheet-url", action="store", required=True)
+        parser.add_argument("--election-date", action="store", required=True)
 
     def get_next_parl_election(self):
         election = (
             Election.objects.filter(slug__contains="parl.")
+            .filter(election_date=self.election_date)
             .future()
             .order_by("election_date")
             .first()
@@ -31,14 +35,15 @@ def get_next_parl_election(self):
     def handle(self, *args, **options):
         self.ballot_cache = {}
         self.party_cache = {}
+        self.election_date = options["election_date"]
         self.election = self.get_next_parl_election()
         req = requests.get(options["sheet_url"])
         req.raise_for_status()
-        self.parse_csv(req.text)
-        raise ValueError("Still testing")
+        self.parse_csv(req.content.decode("utf8").splitlines())
+        # raise ValueError("Still testing")
 
     def parse_csv(self, in_file):
-        csv_data = csv.DictReader(in_file.splitlines())
+        csv_data = csv.DictReader(in_file)
         for line in csv_data:
             self.import_line(line)
 
@@ -86,22 +91,32 @@ def get_source_from_line(self, line):
         return line.get("Source", "PPC sheet importer")
 
     def get_ballot_from_line(self, line):
-        ballot_paper_start = (
-            ".".join(line["Ballot paper ID"].split(".")[0:-1]) + "."
-        )
-        if ballot_paper_start not in self.ballot_cache:
-            self.ballot_cache[
-                ballot_paper_start
-            ] = self.election.ballot_set.get(
-                ballot_paper_id__startswith=ballot_paper_start
-            )
-        return self.ballot_cache[ballot_paper_start]
+        constituency = line["Constituency"].strip()
+
+        if constituency not in self.ballot_cache:
+            try:
+                ballot = Ballot.objects.get(
+                    election=self.election, post__label=constituency
+                )
+            except Ballot.DoesNotExist:
+                print(slugify(constituency))
+                ballot = Ballot.objects.get(
+                    election=self.election, post__slug=slugify(constituency)
+                )
+            self.ballot_cache[constituency] = ballot
+        return self.ballot_cache[constituency]
 
     def get_party_from_line(self, line):
         party_id = line["Party ID"]
         if party_id not in self.party_cache:
             try:
-                self.party_cache[party_id] = Party.objects.get(ec_id=party_id)
+                if "ynmp-party:2" in party_id:
+                    ec_id = party_id
+                elif "-" in party_id and party_id != "ynmp-party:2":
+                    ec_id = "joint-party:" + party_id
+                else:
+                    ec_id = "PP" + party_id
+                self.party_cache[party_id] = Party.objects.get(ec_id=ec_id)
             except Party.DoesNotExist:
                 print(line)
                 raise ValueError("Party not found in line")
@@ -112,16 +127,12 @@ def line_has_values(self, line):
         Some lines exist that only have a ballot ID and no actual membership
         info
         """
-        return any(
-            (line["Candidate Name"], line["Existing Candidate Profile URL"])
-        )
+
+        return any((line["Candidate Name"], line["Existing Candidate Profile"]))
 
     def add_contact_details(self, bot, person, line):
         if not person.get_email and line["Email"]:
             bot.add_email(line["Email"])
-            if line["Email Source"] and line["Source"] != line["Email Source"]:
-                # The source for the email is different, save now
-                bot.save(line["Email Source"])
 
         if line["Twitter"] and not person.tmp_person_identifiers.filter(
             value=line["Twitter"]

ynr/apps/people/helpers.py

@@ -109,9 +109,9 @@ def clean_mastodon_username(username):
 def clean_twitter_username(username):
     # Remove any URL bits around it:
     username = username.strip()
-    m = re.search(r"^.*twitter.com/(\w+)", username)
+    m = re.search(r"^.*(twitter.com|x.com)/(\@?)(\w+)", username)
     if m:
-        username = m.group(1)
+        username = m.group(3)
     # If there's a leading '@', strip that off:
     username = re.sub(r"^@", "", username)
     if not re.search(r"^\w*$", username):