diff --git a/README.md b/README.md index 1ec9c1bac6..b06f71d415 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ Sentry is used to report errors in production. We have added a url for `sentry-d ``` # Pre-election Tasks - # Enable Candidate Leaderboard @@ -70,4 +69,6 @@ We take a slice of edits in YNR and assign them to a election leaderboard. This is defined here: https://github.com/DemocracyClub/yournextrepresentative/blob/master/ynr/apps/candidates/views/mixins.py#L20 -We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] \ No newline at end of file +We can modify the old value to reflect the current election. Change, PR, merge, [currently Sym needs to deploy] + +If this is a General Election, the parliamentary candidates can be imported using a google sheet csv url with `python manage candidatebot_import_next_ppcs --sheet-url SHEET_URL` \ No newline at end of file diff --git a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py index 5ec0a071e6..75c8466e28 100644 --- a/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py +++ b/ynr/apps/candidatebot/management/commands/candidatebot_import_next_ppcs.py @@ -2,12 +2,14 @@ import requests from candidatebot.helpers import CandidateBot +from candidates.models import Ballot from django.core.management.base import BaseCommand from django.db import transaction from elections.models import Election from parties.models import Party from people.models import Person from popolo.models import Membership, NotStandingValidationError +from slugify import slugify class Command(BaseCommand): @@ -15,10 +17,12 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--sheet-url", action="store", required=True) + parser.add_argument("--election-date", action="store", required=True) def get_next_parl_election(self): election = ( Election.objects.filter(slug__contains="parl.") + .filter(election_date=self.election_date) .future() .order_by("election_date") .first() @@ -31,14 +35,15 @@ def get_next_parl_election(self): def handle(self, *args, **options): self.ballot_cache = {} self.party_cache = {} + self.election_date = options["election_date"] self.election = self.get_next_parl_election() req = requests.get(options["sheet_url"]) req.raise_for_status() - self.parse_csv(req.text) - raise ValueError("Still testing") + self.parse_csv(req.content.decode("utf8").splitlines()) + # raise ValueError("Still testing") def parse_csv(self, in_file): - csv_data = csv.DictReader(in_file.splitlines()) + csv_data = csv.DictReader(in_file) for line in csv_data: self.import_line(line) @@ -86,22 +91,32 @@ def get_source_from_line(self, line): return line.get("Source", "PPC sheet importer") def get_ballot_from_line(self, line): - ballot_paper_start = ( - ".".join(line["Ballot paper ID"].split(".")[0:-1]) + "." - ) - if ballot_paper_start not in self.ballot_cache: - self.ballot_cache[ - ballot_paper_start - ] = self.election.ballot_set.get( - ballot_paper_id__startswith=ballot_paper_start - ) - return self.ballot_cache[ballot_paper_start] + constituency = line["Constituency"].strip() + + if constituency not in self.ballot_cache: + try: + ballot = Ballot.objects.get( + election=self.election, post__label=constituency + ) + except Ballot.DoesNotExist: + print(slugify(constituency)) + ballot = Ballot.objects.get( + election=self.election, post__slug=slugify(constituency) + ) + self.ballot_cache[constituency] = ballot + return self.ballot_cache[constituency] def get_party_from_line(self, line): party_id = line["Party ID"] if party_id not in self.party_cache: try: - self.party_cache[party_id] = Party.objects.get(ec_id=party_id) + if "ynmp-party:2" in party_id: + ec_id = party_id + elif "-" in party_id and party_id != "ynmp-party:2": + ec_id = "joint-party:" + party_id + else: + ec_id = "PP" + party_id + self.party_cache[party_id] = Party.objects.get(ec_id=ec_id) except Party.DoesNotExist: print(line) raise ValueError("Party not found in line") @@ -112,16 +127,12 @@ def line_has_values(self, line): Some lines exist that only have a ballot ID and no actual membership info """ - return any( - (line["Candidate Name"], line["Existing Candidate Profile URL"]) - ) + + return any((line["Candidate Name"], line["Existing Candidate Profile"])) def add_contact_details(self, bot, person, line): if not person.get_email and line["Email"]: bot.add_email(line["Email"]) - if line["Email Source"] and line["Source"] != line["Email Source"]: - # The source for the email is different, save now - bot.save(line["Email Source"]) if line["Twitter"] and not person.tmp_person_identifiers.filter( value=line["Twitter"] diff --git a/ynr/apps/people/helpers.py b/ynr/apps/people/helpers.py index db5fbf81e5..a8df99634c 100644 --- a/ynr/apps/people/helpers.py +++ b/ynr/apps/people/helpers.py @@ -109,9 +109,9 @@ def clean_mastodon_username(username): def clean_twitter_username(username): # Remove any URL bits around it: username = username.strip() - m = re.search(r"^.*twitter.com/(\w+)", username) + m = re.search(r"^.*(twitter.com|x.com)/(\@?)(\w+)", username) if m: - username = m.group(1) + username = m.group(3) # If there's a leading '@', strip that off: username = re.sub(r"^@", "", username) if not re.search(r"^\w*$", username):