diff --git a/ynr/apps/cached_counts/management/commands/report_for_date.py b/ynr/apps/cached_counts/management/commands/report_for_date.py index 44515ce9fb..73708e9968 100644 --- a/ynr/apps/cached_counts/management/commands/report_for_date.py +++ b/ynr/apps/cached_counts/management/commands/report_for_date.py @@ -70,4 +70,6 @@ def handle(self, *args, **options): register=options["register"], nation=options["nation"], elected=options["elected"], - ) + ).print_text() + + # print(text) diff --git a/ynr/apps/cached_counts/management/commands/reports_update.py b/ynr/apps/cached_counts/management/commands/reports_update.py new file mode 100644 index 0000000000..cb81a4a4a8 --- /dev/null +++ b/ynr/apps/cached_counts/management/commands/reports_update.py @@ -0,0 +1,45 @@ +from pathlib import Path + +from django.core.management import BaseCommand + +from django.conf import settings + +from cached_counts.models import CachedReport +from cached_counts.report_helpers import ( + ALL_REPORT_CLASSES, + report_runner, + BaseReport, +) + + +class Command(BaseCommand): + help = "Create JSON versions of reports for all elections in settings.REPORT_DATES" + + def handle(self, *args, **options): + reports_dir = Path.cwd() / "ynr/apps/cached_counts/reports/" + reports_dir.mkdir(exist_ok=True) + + for group_id, report_data in settings.REPORT_DATES.items(): + election_type, report_date = group_id.split(".") + registers = report_data.get("registers", ["GB"]) + for report_class in ALL_REPORT_CLASSES: + for register in registers: + report: BaseReport = report_runner( + name=report_class, + date=report_date, + election_type=election_type, + register=register, + ) + CachedReport.objects.update_or_create( + election_date=report_date, + group_id=group_id, + report_name=report_class, + register=register, + report_json=report.as_dict(), + ) + # print(report.name) + # print(report.as_text()) + # print() + # print() + # print() + # print(report.as_dict()) diff --git a/ynr/apps/cached_counts/management/commands/stats_from_versions_for_date.py b/ynr/apps/cached_counts/management/commands/stats_from_versions_for_date.py new file mode 100644 index 0000000000..1e05913722 --- /dev/null +++ b/ynr/apps/cached_counts/management/commands/stats_from_versions_for_date.py @@ -0,0 +1,49 @@ +import json + +from django.core.management.base import BaseCommand + +from candidates.models.versions import get_version_for_date +from popolo.models import Membership + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--date", + action="store", + dest="date", + help="The election date", + required=True, + ) + + def handle(self, *args, **options): + qs = Membership.objects.filter( + ballot__election__election_date=options["date"], + ballot__ballot_paper_id__startswith="parl.", + ) + has_email = 0 + has_twitter = 0 + has_fb = 0 + has_statement = 0 + for membership in qs: + versions = json.loads(membership.person.versions) + version_for_date = get_version_for_date(options["date"], versions) + if not version_for_date: + continue + if version_for_date["data"].get("biography"): + has_statement += 1 + fb_fields = [ + version_for_date["data"]["facebook_personal_url"], + version_for_date["data"]["facebook_page_url"], + ] + if any(fb_fields): + has_fb += 1 + if version_for_date["data"]["email"]: + has_email += 1 + if version_for_date["data"]["twitter_username"]: + has_twitter += 1 + print(f"total: {qs.count()}") + print(f"has email: {has_email}") + print(f"has FB: {has_fb}") + print(f"has Twitter: {has_twitter}") + print(f"has statement: {has_statement}") diff --git a/ynr/apps/cached_counts/migrations/0006_cachedreport.py b/ynr/apps/cached_counts/migrations/0006_cachedreport.py new file mode 100644 index 0000000000..0eb7c0659f --- /dev/null +++ b/ynr/apps/cached_counts/migrations/0006_cachedreport.py @@ -0,0 +1,35 @@ +# Generated by Django 3.2 on 2022-04-04 20:28 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [("cached_counts", "0005_delete_cachedcount")] + + operations = [ + migrations.CreateModel( + name="CachedReport", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("election_date", models.DateField()), + ("register", models.CharField(max_length=2)), + ("report_name", models.CharField(max_length=255)), + ( + "report_json", + django.contrib.postgres.fields.jsonb.JSONField(), + ), + ], + ) + ] diff --git a/ynr/apps/cached_counts/migrations/0007_cachedreport_group_id.py b/ynr/apps/cached_counts/migrations/0007_cachedreport_group_id.py new file mode 100644 index 0000000000..fb0f9bbc9b --- /dev/null +++ b/ynr/apps/cached_counts/migrations/0007_cachedreport_group_id.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2 on 2022-04-04 20:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [("cached_counts", "0006_cachedreport")] + + operations = [ + migrations.AddField( + model_name="cachedreport", + name="group_id", + field=models.CharField(blank=True, max_length=300), + ) + ] diff --git a/ynr/apps/cached_counts/report_helpers.py b/ynr/apps/cached_counts/report_helpers.py index d912a5535a..04a710eced 100644 --- a/ynr/apps/cached_counts/report_helpers.py +++ b/ynr/apps/cached_counts/report_helpers.py @@ -7,6 +7,7 @@ """ import collections import sys +import abc from collections import Counter from django.db.models import ( @@ -43,7 +44,9 @@ ] -class BaseReport: +class BaseReport(abc.ABC): + name: str + def __init__( self, date, @@ -116,7 +119,9 @@ def __init__( self.f_winners / self.f_candidates, output_field=FloatField() ) - def run(self): + self.report_data = {"header": [], "lines": [], "name": self.name} + + def print_text(self): print() print() print() @@ -133,13 +138,29 @@ def run(self): print(title) print("=" * len(title)) print() - print(self.report()) + print(self.as_text()) + + def header_text(self): + text = "\t".join(self.report_data["header"]) + text = text + f"\n{'-'*len(text)}" + return text + + def as_text(self): + report_data = self.as_dict() + lines = [self.header_text()] + for report in report_data["lines"]: + lines.append("\t".join([str(field) for field in report])) + return "\n".join(lines) + + @abc.abstractmethod + def as_dict(self): + pass def report_runner(name, date, **kwargs): this_module = sys.modules[__name__] if hasattr(this_module, name): - return getattr(this_module, name)(date=date, **kwargs).run() + return getattr(this_module, name)(date=date, **kwargs) else: raise ValueError( "{} is unknown. Pick one of: {}".format( @@ -167,16 +188,16 @@ def get_qs(self): .order_by() ) - def report(self): - report = [] + def as_dict(self): + self.report_data["header"] = ["Election Type", "Candidates"] for election_type in self.get_qs(): - report.append( - "{}\t{}".format( + self.report_data["lines"].append( + [ election_type["ballot__election__for_post_role"], election_type["seats"], - ) + ] ) - return "\n".join(report) + return self.report_data class NumberOfSeats(BaseReport): @@ -187,16 +208,16 @@ def get_qs(self): seats=Sum("winner_count") ) - def report(self): - report = [] + def as_dict(self): + self.report_data["header"] = ["Election Type", "Candidates"] for election_type in self.get_qs(): - report.append( - "{}\t{}".format( + self.report_data["lines"].append( + [ election_type["election__for_post_role"], election_type["seats"], - ) + ] ) - return "\n".join(report) + return self.report_data class CandidatesPerParty(BaseReport): @@ -209,35 +230,27 @@ def get_qs(self): .order_by("-membership_count") ) - def report(self): - report = ["Party Name\tParty Register\tCandidates\tPercent of seats"] + def as_dict(self): + self.report_data["header"] = [ + "Party Name", + "Party Register", + "Candidates", + "Percent of seats", + ] total_seats = self.ballot_qs.aggregate(seats=Sum("winner_count"))[ "seats" ] for party in self.get_qs(): - report.append( - "\t".join( - [ - str(v) - for v in [ - party["party__name"], - party["party__register"], - party["membership_count"], - round( - float( - party["membership_count"] - / total_seats - * 100 - ), - 2, - ), - ] - ] - ) - ) - return "\n".join(report) + line = [ + party["party__name"], + party["party__register"], + party["membership_count"], + round(float(party["membership_count"] / total_seats * 100), 2), + ] + self.report_data["lines"].append(line) + return self.report_data class WardsContestedPerParty(BaseReport): @@ -260,37 +273,36 @@ def name(self): total_wards = self.ballot_qs.count() return f"Wards contested per party ({total_wards})" - def report(self): - report = [ - "Party Name\tParty Register\tCandidates standing\tPercent of wards" + def as_dict(self): + self.report_data["header"] = [ + "Party Name", + "Party Register", + "Candidates standing", + "Percent of wards", ] total_ballots = self.ballot_qs.count() for party in self.get_qs(): - report.append( - "\t".join( - [ - str(v) - for v in [ - party["party__name"], - party["party__register"], - party["membership_count"], - round( - float( - party["membership_count"] - / total_ballots - * 100 - ), - 2, + self.report_data["lines"].append( + [ + str(v) + for v in [ + party["party__name"], + party["party__register"], + party["membership_count"], + round( + float( + party["membership_count"] / total_ballots * 100 ), - ] + 2, + ), ] - ) + ] ) - return "\n".join(report) + return self.report_data -class UncontestedBallots(BaseReport): +class NumUncontestedBallots(BaseReport): name = "Uncontested Ballots" def get_qs(self): @@ -301,24 +313,32 @@ def get_qs(self): .order_by("ballot_paper_id") ) - def report(self): - report_list = [] + def as_dict(self): + self.report_data["header"] = ["Total"] qs = self.get_qs() - report_list.append(["{} uncontested seats".format(qs.count())]) - - report_list.append("\n") - for ballot in qs: + self.report_data["lines"].append([qs.count()]) + return self.report_data - for membership in ballot.membership_set.all(): - report_list.append( - [ - ballot.ballot_paper_id, - membership.person.name, - membership.party.name, - ] - ) - return "\n".join(["\t".join(r) for r in report_list]) +class uncontestedWinners(NumUncontestedBallots): + def as_dict(self): + self.report_data["header"] = ["Ballot", "Person", "Party"] + qs = self.get_qs() + self.report_data["lines"].append(["Total", qs.count()]) + # + # report_list.append("\n") + # for ballot in qs: + # + # for membership in ballot.membership_set.all(): + # report_list.append( + # [ + # ballot.ballot_paper_id, + # membership.person.name, + # membership.party.name, + # ] + # ) + # + # return "\n".join(["\t".join(r) for r in report_list]) class NcandidatesPerSeat(BaseReport): @@ -337,7 +357,7 @@ def get_qs(self): .filter(per_seat__gt=self.n) ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = [ @@ -396,7 +416,7 @@ def get_qs(self): .order_by("ballot_paper_id") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = [ @@ -448,7 +468,7 @@ def get_qs(self): .order_by("per_seat") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [ ["Ballot paper ID", "num candidates", "per seat", "winners"] @@ -468,7 +488,7 @@ def get_qs(self): party__date_registered__year=self.date.split("-")[0] ).order_by("party_id") - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Party Name", "Ballot Paper ID", "Candidate name"] @@ -494,7 +514,7 @@ def get_qs(self): .annotate(gender_count=Count("person__gender_guess__gender")) ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Gender", "Gender Count"] @@ -525,7 +545,7 @@ def get_qs(self): .select_related("ballot", "person", "person__gender_guess") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Name", "Guessed Gender", "Person ID", "Ballot URL", "Party"] @@ -555,7 +575,7 @@ def get_qs(self): .order_by("party__name") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Gender", "Gender Count", "Party Name"] @@ -583,7 +603,7 @@ def get_qs(self): .order_by("ballot__tags__NUTS1__value") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Gender", "Gender Count", "Region"] @@ -612,7 +632,7 @@ def get_qs(self): .order_by("ballot__election__for_post_role") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Gender", "Gender Count", "Party Name"] @@ -640,7 +660,7 @@ def get_qs(self): .order_by("ballot__winner_count") ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Seats Contested", "F", "M", "Ratio"] @@ -676,7 +696,7 @@ def get_qs(self): .values_list("ballot__ballot_paper_id", flat=True) ) - def report(self): + def as_text(self): report_list = [] headers = ["Label", "Count", "Sample"] report_list.append(headers) @@ -720,7 +740,7 @@ def get_qs(self): .filter(party_count__gt=1) ) - def report(self): + def as_text(self): qs = self.get_qs() report_list = [] headers = ["Person ID", "Person Name", "Party Count"] @@ -747,7 +767,7 @@ def report(self): class RegionalNumCandidatesPerSeat(BaseReport): name = "Number of candidates per seat per region" - def report(self): + def as_text(self): report_list = [] headers = [ "Region Name", @@ -800,7 +820,7 @@ def get_qs(self, parties=None): parties = parties or self.PARTIES return Party.objects.filter(name__in=parties, register="GB").distinct() - def report(self): + def as_text(self): report_list = [] headers = [ "Party Name", @@ -858,7 +878,7 @@ def get_qs(self): ) return people.annotate(num_candidacies=current_candidacies) - def report(self): + def as_text(self): report_list = [] headers = [ "Num Candidacies", @@ -902,7 +922,7 @@ def get_qs(self): ) ) - def report(self): + def as_text(self): report_list = [] headers = ["ID", "Name", "Num candidacies", "Ballot ID's"] report_list.append(headers) @@ -926,7 +946,7 @@ class NumCandidatesStandingInMultipleSeatsPerGender( name = "Num candidates standing in multiple per seats, per gender" - def report(self): + def as_text(self): genders = ( self.membership_qs.values_list( @@ -940,7 +960,7 @@ def report(self): headers = ["Gender", "Num Candidacies", "Num people standing"] report_list.append(headers) qs = self.get_qs() - # very arbritary safeguard but im assuming you would never have someone + # very arbitrary safeguard but im assuming you would never have someone # stand on this many ballots for the same election type and date for num in range(1, 10): for gender in genders: @@ -973,7 +993,7 @@ def collect_names(self, label, qs): for name, count in collector.most_common(30): yield [label, name, count] - def report(self): + def as_text(self): report_list = [] headers = ["Type", "Name", "Count"] report_list.append(headers) @@ -1030,7 +1050,7 @@ class CandidatesWithWithoutStatement(BaseReport): def get_qs(self): return self.membership_qs.select_related("person") - def report(self): + def as_text(self): report_list = [] headers = ["", "Number", "%"] report_list.append(headers) @@ -1059,7 +1079,7 @@ def report(self): ALL_REPORT_CLASSES = [] for x in list(locals().values()): - if type(x) == type and issubclass(x, BaseReport): + if type(x) == abc.ABCMeta and issubclass(x, BaseReport): if x.__name__ == "BaseReport": continue ALL_REPORT_CLASSES.append(x.__name__) diff --git a/ynr/apps/cached_counts/templates/report.html b/ynr/apps/cached_counts/templates/report.html new file mode 100644 index 0000000000..d91485e97d --- /dev/null +++ b/ynr/apps/cached_counts/templates/report.html @@ -0,0 +1,15 @@ +{% extends "base.html" %} +{% load humanize %} + +{% block content %} + +
+

{{ report_info.title }} {{ report_info.date }}

+ {{ reports }} +

{{ reports.NumberOfCandidates.report_json.name }}

+ {% for title, data in reports.NumberOfCandidates.report_json.report.0.items %} +

{{ data|intcomma }}

+ {% endfor %} +
+ +{% endblock %} diff --git a/ynr/apps/cached_counts/views.py b/ynr/apps/cached_counts/views.py index adccb525dd..e513e576e4 100644 --- a/ynr/apps/cached_counts/views.py +++ b/ynr/apps/cached_counts/views.py @@ -1,46 +1,40 @@ -import json - from django.db.models import Count -from django.http import HttpResponse from django.views.generic import TemplateView from candidates.models import Ballot from elections.mixins import ElectionMixin -from elections.models import Election from parties.models import Party -from popolo.models import Membership - from .models import get_attention_needed_posts -def get_counts(for_json=True): - election_id_to_candidates = {} - qs = ( - Membership.objects.all() - .values("ballot__election") - .annotate(count=Count("ballot__election")) - .order_by() - ) - - for d in qs: - election_id_to_candidates[d["ballot__election"]] = d["count"] - - grouped_elections = Election.group_and_order_elections(for_json=for_json) - for era_data in grouped_elections: - for date, elections in era_data["dates"].items(): - for role_data in elections: - for election_data in role_data["elections"]: - e = election_data["election"] - total = election_id_to_candidates.get(e.id, 0) - election_counts = { - "id": e.slug, - "html_id": e.slug.replace(".", "-"), - "name": e.name, - "total": total, - } - election_data.update(election_counts) - del election_data["election"] - return grouped_elections +# def get_counts(for_json=True): +# election_id_to_candidates = {} +# qs = ( +# Membership.objects.all() +# .values("ballot__election") +# .annotate(count=Count("ballot__election")) +# .order_by() +# ) +# +# for d in qs: +# election_id_to_candidates[d["ballot__election"]] = d["count"] +# +# grouped_elections = Election.group_and_order_elections(for_json=for_json) +# for era_data in grouped_elections: +# for date, elections in era_data["dates"].items(): +# for role_data in elections: +# for election_data in role_data["elections"]: +# e = election_data["election"] +# total = election_id_to_candidates.get(e.id, 0) +# election_counts = { +# "id": e.slug, +# "html_id": e.slug.replace(".", "-"), +# "name": e.name, +# "total": total, +# } +# election_data.update(election_counts) +# del election_data["election"] +# return grouped_elections class ReportsHomeView(TemplateView): @@ -48,16 +42,8 @@ class ReportsHomeView(TemplateView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - context["all_elections"] = get_counts() - return context - def get(self, *args, **kwargs): - if self.request.GET.get("format") == "json": - return HttpResponse( - json.dumps(get_counts(for_json=True)), - content_type="application/json", - ) - return super().get(*args, **kwargs) + return context class PartyCountsView(ElectionMixin, TemplateView): diff --git a/ynr/apps/elections/uk/urls.py b/ynr/apps/elections/uk/urls.py index 9a21b07632..e68e115b3e 100644 --- a/ynr/apps/elections/uk/urls.py +++ b/ynr/apps/elections/uk/urls.py @@ -8,6 +8,7 @@ re_path(r"^bulk_adding/", include("bulk_adding.urls")), re_path(r"^uk_results/", include("uk_results.urls")), re_path(r"^$", frontpage.HomePageView.as_view(), name="lookup-postcode"), + re_path(r"^$", frontpage.HomePageView.as_view(), name="lookup-postcode"), re_path( r"^postcode/(?P[^/]+)/$", frontpage.PostcodeView.as_view(), diff --git a/ynr/settings/base.py b/ynr/settings/base.py index a9a57558b7..7b34d05634 100644 --- a/ynr/settings/base.py +++ b/ynr/settings/base.py @@ -453,6 +453,7 @@ def root(*path): from .constants.needs_review import * # noqa from .constants.csv_fields import * # noqa from .constants.nuts import * # noqa +from .constants.reports import * # noqa from ynr_refactoring.settings import * # noqa