diff --git a/ynr/apps/candidates/urls.py b/ynr/apps/candidates/urls.py index 4e7c138bf5..4ed6ff6e88 100644 --- a/ynr/apps/candidates/urls.py +++ b/ynr/apps/candidates/urls.py @@ -1,4 +1,3 @@ -from django.conf import settings from django.urls import include, re_path from django.views.generic import TemplateView from django.views.generic.base import RedirectView @@ -12,7 +11,6 @@ r"^api-auth/", include("rest_framework.urls", namespace="rest_framework"), ), - re_path(r"^", include(settings.ELECTION_APP_FULLY_QUALIFIED + ".urls")), re_path( r"^election/(?P[^/]+)/record-winner$", views.ConstituencyRecordWinnerView.as_view(), diff --git a/ynr/apps/parties/management/commands/parties_import_pef_data.py b/ynr/apps/parties/management/commands/parties_import_pef_data.py new file mode 100644 index 0000000000..445df15885 --- /dev/null +++ b/ynr/apps/parties/management/commands/parties_import_pef_data.py @@ -0,0 +1,105 @@ +import json +import re +from datetime import datetime +from pathlib import Path + +from django.core.management.base import BaseCommand + +from parties.models import Party, PartySpending, PartyAccounts, PartyDonations + + +class Command(BaseCommand): + CACHE = {"parties": {}} + + def add_arguments(self, parser): + parser.add_argument( + "--data-path", + action="store", + help="Path to the `data` dir of the scraper", + required=True, + ) + parser.add_argument( + "--type", + action="store", + help="Currently only 'spending' supported", + default="Spending", + ) + + def populate_parties_cache(self): + for party in Party.objects.all(): + self.CACHE["parties"][party.ec_id] = party + + def add_spending(self, data: dict): + party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}" + party: Party = self.CACHE["parties"].get(party_id) + if not party: + return + print(party, data.get("RegulatedEntityName")) + spending, updated = PartySpending.objects.update_or_create( + party=party, + ec_id=data["ECRef"], + defaults={ + "raw_data": data, + "published": self.clean_date(data["PublishedDate"]), + }, + ) + return spending + + def add_accounts(self, data: dict): + party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}" + party: Party = self.CACHE["parties"].get(party_id) + if not party: + return + accounts, updated = PartyAccounts.objects.update_or_create( + party=party, + ec_id=data["ECRef"], + defaults={ + "raw_data": data, + "published": self.clean_date(data["PublishedDate"]), + }, + ) + return accounts + + def add_donations(self, data: dict): + party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}" + party: Party = self.CACHE["parties"].get(party_id) + if not party: + return + accounts, updated = PartyDonations.objects.update_or_create( + party=party, + ec_id=data["ECRef"], + defaults={ + "raw_data": data, + "published": self.clean_date(data["PublishedDate"]), + }, + ) + return accounts + + def clean_date(self, date): + try: + timestamp = re.match(r"/Date\(([\-]?\d+)\)/", date).group(1) + dt = datetime.fromtimestamp(int(timestamp) / 1000.0) + return dt.strftime("%Y-%m-%d") + except: + return None + + def handle(self, *args, **options): + self.populate_parties_cache() + + data_dir = Path(options["data_path"]) + + # spending_dir = data_dir / "Spending" + # for file in spending_dir.glob("**/*.json"): + # data = json.load(file.open()) + # self.add_spending(data=data) + + # accounts_dir = data_dir / "Accounts" + # for file in accounts_dir.glob("**/*.json"): + # data = json.load(file.open()) + # self.add_accounts(data=data) + + donations_dir = data_dir / "Donations" + for file in donations_dir.glob("**/*.json"): + data = json.load(file.open()) + donation = self.add_donations(data=data) + print(donation) diff --git a/ynr/apps/parties/migrations/0018_partyspending.py b/ynr/apps/parties/migrations/0018_partyspending.py new file mode 100644 index 0000000000..4c8faf20a1 --- /dev/null +++ b/ynr/apps/parties/migrations/0018_partyspending.py @@ -0,0 +1,50 @@ +# Generated by Django 4.1.3 on 2022-12-02 10:54 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("parties", "0017_alter_party_ec_id"), + ] + + operations = [ + migrations.CreateModel( + name="PartySpending", + fields=[ + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "ec_id", + models.CharField( + max_length=15, primary_key=True, serialize=False + ), + ), + ("raw_data", models.JSONField()), + ( + "party", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="parties.party", + ), + ), + ], + options={ + "get_latest_by": "modified", + "abstract": False, + }, + ), + ] diff --git a/ynr/apps/parties/migrations/0019_partyspending_published.py b/ynr/apps/parties/migrations/0019_partyspending_published.py new file mode 100644 index 0000000000..9f56cfe93c --- /dev/null +++ b/ynr/apps/parties/migrations/0019_partyspending_published.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.3 on 2022-12-02 11:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("parties", "0018_partyspending"), + ] + + operations = [ + migrations.AddField( + model_name="partyspending", + name="published", + field=models.DateField(null=True), + ), + ] diff --git a/ynr/apps/parties/migrations/0020_partyaccounts.py b/ynr/apps/parties/migrations/0020_partyaccounts.py new file mode 100644 index 0000000000..e5482b079c --- /dev/null +++ b/ynr/apps/parties/migrations/0020_partyaccounts.py @@ -0,0 +1,51 @@ +# Generated by Django 4.1.3 on 2022-12-02 19:36 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("parties", "0019_partyspending_published"), + ] + + operations = [ + migrations.CreateModel( + name="PartyAccounts", + fields=[ + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "ec_id", + models.CharField( + max_length=15, primary_key=True, serialize=False + ), + ), + ("published", models.DateField(null=True)), + ("raw_data", models.JSONField()), + ( + "party", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="parties.party", + ), + ), + ], + options={ + "get_latest_by": "modified", + "abstract": False, + }, + ), + ] diff --git a/ynr/apps/parties/migrations/0021_partydonations.py b/ynr/apps/parties/migrations/0021_partydonations.py new file mode 100644 index 0000000000..d3e3e04e10 --- /dev/null +++ b/ynr/apps/parties/migrations/0021_partydonations.py @@ -0,0 +1,51 @@ +# Generated by Django 4.1.3 on 2022-12-02 19:59 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("parties", "0020_partyaccounts"), + ] + + operations = [ + migrations.CreateModel( + name="PartyDonations", + fields=[ + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "ec_id", + models.CharField( + max_length=15, primary_key=True, serialize=False + ), + ), + ("published", models.DateField(null=True)), + ("raw_data", models.JSONField()), + ( + "party", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="parties.party", + ), + ), + ], + options={ + "get_latest_by": "modified", + "abstract": False, + }, + ), + ] diff --git a/ynr/apps/parties/models.py b/ynr/apps/parties/models.py index 9ae253b7a1..39b1688040 100644 --- a/ynr/apps/parties/models.py +++ b/ynr/apps/parties/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.db.models import QuerySet from django.utils import timezone from django_extensions.db.models import TimeStampedModel @@ -194,3 +195,57 @@ class Meta: def __str__(self): return '{} ("{}")'.format(self.pk, self.description) + + +class PartySpendingQueryset(QuerySet): + def recent(self): + return self.order_by("-published")[:10] + + def invoices(self): + return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False) + + +class PartySpending(TimeStampedModel): + ec_id = models.CharField(primary_key=True, max_length=15) + party = models.ForeignKey(Party, on_delete=models.CASCADE) + published = models.DateField(null=True) + raw_data = models.JSONField() + + objects = PartySpendingQueryset.as_manager() + + +class PartyAccountsQueryset(QuerySet): + def recent(self): + return self.order_by("-published")[:10] + + def invoices(self): + return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False) + + +class PartyAccounts(TimeStampedModel): + ec_id = models.CharField(primary_key=True, max_length=15) + party = models.ForeignKey(Party, on_delete=models.CASCADE) + published = models.DateField(null=True) + raw_data = models.JSONField() + + objects = PartyAccountsQueryset.as_manager() + + +class PartyDonationsQueryset(QuerySet): + def recent(self): + return self.order_by("-published")[:10] + + def invoices(self): + return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False) + + def individuals(self): + return self.filter(raw_data__DonorStatus="Individual") + + +class PartyDonations(TimeStampedModel): + ec_id = models.CharField(primary_key=True, max_length=15) + party = models.ForeignKey(Party, on_delete=models.CASCADE) + published = models.DateField(null=True) + raw_data = models.JSONField() + + objects = PartyDonationsQueryset.as_manager() diff --git a/ynr/apps/parties/templates/parties/party_detail.html b/ynr/apps/parties/templates/parties/party_detail.html new file mode 100644 index 0000000000..0743c0fed6 --- /dev/null +++ b/ynr/apps/parties/templates/parties/party_detail.html @@ -0,0 +1,117 @@ +{% extends "base.html" %} + +{% load thumbnail %} +{% load humanize %} + + +{% block hero %} +

{{ object.name }}

+{% endblock %} + +{% block content %} + + {{ object.name }} was registered on {{ object.date_registered }}. +

Descriptions

+ + {#

Emblems

#} + {# #} + + +

Latest Spending

+ + + + + + + + + + + + {% for spending in object.partyspending_set.recent %} + + + + + + + + {% endfor %} + + +
CategoryReporting Period NameSupplierAmountInvoice
{{ spending.raw_data.ExpenseCategoryName }}{{ spending.raw_data.ReportingPeriodName }}{{ spending.raw_data.SupplierName }}£{{ spending.raw_data.TotalExpenditure|intcomma }} + {% if spending.raw_data.RedactedSupportingInvoiceId %} + + Invoice (PDF) + + {% else %} + (Not online) + {% endif %} +
+ + +

Accounts

+ + + + + + + + + + + + {% for accounts in object.partyaccounts_set.recent %} + + + + + + + + {% endfor %} + +
Accounting Unit NameReporting Period TypeTotal IncomeTotal ExpenditurePDF
{{ accounts.raw_data.AccountingUnitName }}{{ accounts.raw_data.ReportingPeriodType }}£{{ accounts.raw_data.TotalIncome|intcomma }}£{{ accounts.raw_data.TotalExpenditure|intcomma }} + {% if accounts.raw_data.SOARedactedDocumentId %} + + PDF + + {% else %} + (Not online) + {% endif %} +
+ +

Donations from Individuals

+ + + + + + + + + + {% for donation in object.partydonations_set.individuals.recent %} + + + + + + {% endfor %} + +
Accounting Unit NameDonor NameAmount
{{ donation.raw_data.AccountingUnitName }}{{ donation.raw_data.DonorName }}£{{ donation.raw_data.Value|intcomma }}
+ +{% endblock %} \ No newline at end of file diff --git a/ynr/apps/parties/urls.py b/ynr/apps/parties/urls.py index 413c8f5aee..3181dc4bb9 100644 --- a/ynr/apps/parties/urls.py +++ b/ynr/apps/parties/urls.py @@ -1,11 +1,12 @@ from django.urls import re_path -from .views import CandidatesByElectionForPartyView +from .views import CandidatesByElectionForPartyView, PartyView urlpatterns = [ + re_path(r"^(?P[^/]+)", PartyView.as_view()), re_path( r"^(?P[^/]+)/elections/(?P[^/]+)/", CandidatesByElectionForPartyView.as_view(), name="candidates_by_election_for_party", - ) + ), ] diff --git a/ynr/apps/parties/views.py b/ynr/apps/parties/views.py index a3d231a1b4..2eafd78396 100644 --- a/ynr/apps/parties/views.py +++ b/ynr/apps/parties/views.py @@ -1,5 +1,5 @@ from django.shortcuts import get_object_or_404 -from django.views.generic import TemplateView +from django.views.generic import TemplateView, DetailView from candidates.models import Ballot from elections.models import Election @@ -58,3 +58,14 @@ def get_context_data(self, **kwargs): context["candidates"] = candidates_qs return context + + +class PartyView(DetailView): + model = Party + # = "ec_id" + slug_url_kwarg = "ec_id" + slug_field = "ec_id" + + def get_queryset(self): + qs = Party.objects.prefetch_related("descriptions", "emblems") + return qs diff --git a/ynr/urls.py b/ynr/urls.py index 9e337b12b1..b84062574b 100644 --- a/ynr/urls.py +++ b/ynr/urls.py @@ -27,6 +27,7 @@ def post(self, request, *args, **kwargs): re_path(r"^", include("elections.urls")), re_path(r"", include("facebook_data.urls")), re_path(r"^", include("candidates.urls")), + re_path(r"^", include("elections.uk.urls")), re_path(r"^", include("people.urls")), re_path(r"^", include("search.urls")), re_path(r"^admin/doc/", include("django.contrib.admindocs.urls")),