Party spend data and party pages

ynr/apps/candidates/urls.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.urls import include, re_path
 from django.views.generic import TemplateView
 from django.views.generic.base import RedirectView
@@ -12,7 +11,6 @@
         r"^api-auth/",
         include("rest_framework.urls", namespace="rest_framework"),
     ),
-    re_path(r"^", include(settings.ELECTION_APP_FULLY_QUALIFIED + ".urls")),
     re_path(
         r"^election/(?P<ballot_paper_id>[^/]+)/record-winner$",
         views.ConstituencyRecordWinnerView.as_view(),

ynr/apps/parties/management/commands/parties_import_pef_data.py

@@ -0,0 +1,105 @@
+import json
+import re
+from datetime import datetime
+from pathlib import Path
+
+from django.core.management.base import BaseCommand
+
+from parties.models import Party, PartySpending, PartyAccounts, PartyDonations
+
+
+class Command(BaseCommand):
+    CACHE = {"parties": {}}
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--data-path",
+            action="store",
+            help="Path to the `data` dir of the scraper",
+            required=True,
+        )
+        parser.add_argument(
+            "--type",
+            action="store",
+            help="Currently only 'spending' supported",
+            default="Spending",
+        )
+
+    def populate_parties_cache(self):
+        for party in Party.objects.all():
+            self.CACHE["parties"][party.ec_id] = party
+
+    def add_spending(self, data: dict):
+        party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}"
+        party: Party = self.CACHE["parties"].get(party_id)
+        if not party:
+            return
+        print(party, data.get("RegulatedEntityName"))
+        spending, updated = PartySpending.objects.update_or_create(
+            party=party,
+            ec_id=data["ECRef"],
+            defaults={
+                "raw_data": data,
+                "published": self.clean_date(data["PublishedDate"]),
+            },
+        )
+        return spending
+
+    def add_accounts(self, data: dict):
+        party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}"
+        party: Party = self.CACHE["parties"].get(party_id)
+        if not party:
+            return
+        accounts, updated = PartyAccounts.objects.update_or_create(
+            party=party,
+            ec_id=data["ECRef"],
+            defaults={
+                "raw_data": data,
+                "published": self.clean_date(data["PublishedDate"]),
+            },
+        )
+        return accounts
+
+    def add_donations(self, data: dict):
+        party_id = f"{data.get('RegulatedEntityTypeShortcode')}{data.get('RegulatedEntityId')}"
+        party: Party = self.CACHE["parties"].get(party_id)
+        if not party:
+            return
+        accounts, updated = PartyDonations.objects.update_or_create(
+            party=party,
+            ec_id=data["ECRef"],
+            defaults={
+                "raw_data": data,
+                "published": self.clean_date(data["PublishedDate"]),
+            },
+        )
+        return accounts
+
+    def clean_date(self, date):
+        try:
+            timestamp = re.match(r"/Date\(([\-]?\d+)\)/", date).group(1)
+            dt = datetime.fromtimestamp(int(timestamp) / 1000.0)
+            return dt.strftime("%Y-%m-%d")
+        except:
+            return None
+
+    def handle(self, *args, **options):
+        self.populate_parties_cache()
+
+        data_dir = Path(options["data_path"])
+
+        # spending_dir = data_dir / "Spending"
+        # for file in spending_dir.glob("**/*.json"):
+        #     data = json.load(file.open())
+        #     self.add_spending(data=data)
+
+        # accounts_dir = data_dir / "Accounts"
+        # for file in accounts_dir.glob("**/*.json"):
+        #     data = json.load(file.open())
+        #     self.add_accounts(data=data)
+
+        donations_dir = data_dir / "Donations"
+        for file in donations_dir.glob("**/*.json"):
+            data = json.load(file.open())
+            donation = self.add_donations(data=data)
+            print(donation)

ynr/apps/parties/migrations/0018_partyspending.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.1.3 on 2022-12-02 10:54
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("parties", "0017_alter_party_ec_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PartySpending",
+            fields=[
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "ec_id",
+                    models.CharField(
+                        max_length=15, primary_key=True, serialize=False
+                    ),
+                ),
+                ("raw_data", models.JSONField()),
+                (
+                    "party",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="parties.party",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

ynr/apps/parties/migrations/0019_partyspending_published.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2022-12-02 11:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("parties", "0018_partyspending"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="partyspending",
+            name="published",
+            field=models.DateField(null=True),
+        ),
+    ]

ynr/apps/parties/migrations/0020_partyaccounts.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.1.3 on 2022-12-02 19:36
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("parties", "0019_partyspending_published"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PartyAccounts",
+            fields=[
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "ec_id",
+                    models.CharField(
+                        max_length=15, primary_key=True, serialize=False
+                    ),
+                ),
+                ("published", models.DateField(null=True)),
+                ("raw_data", models.JSONField()),
+                (
+                    "party",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="parties.party",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

ynr/apps/parties/migrations/0021_partydonations.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.1.3 on 2022-12-02 19:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("parties", "0020_partyaccounts"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="PartyDonations",
+            fields=[
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "ec_id",
+                    models.CharField(
+                        max_length=15, primary_key=True, serialize=False
+                    ),
+                ),
+                ("published", models.DateField(null=True)),
+                ("raw_data", models.JSONField()),
+                (
+                    "party",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="parties.party",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

ynr/apps/parties/models.py

@@ -1,4 +1,5 @@
 from django.db import models
+from django.db.models import QuerySet
 from django.utils import timezone
 from django_extensions.db.models import TimeStampedModel
 
@@ -194,3 +195,57 @@ class Meta:
 
     def __str__(self):
         return '{} ("{}")'.format(self.pk, self.description)
+
+
+class PartySpendingQueryset(QuerySet):
+    def recent(self):
+        return self.order_by("-published")[:10]
+
+    def invoices(self):
+        return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False)
+
+
+class PartySpending(TimeStampedModel):
+    ec_id = models.CharField(primary_key=True, max_length=15)
+    party = models.ForeignKey(Party, on_delete=models.CASCADE)
+    published = models.DateField(null=True)
+    raw_data = models.JSONField()
+
+    objects = PartySpendingQueryset.as_manager()
+
+
+class PartyAccountsQueryset(QuerySet):
+    def recent(self):
+        return self.order_by("-published")[:10]
+
+    def invoices(self):
+        return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False)
+
+
+class PartyAccounts(TimeStampedModel):
+    ec_id = models.CharField(primary_key=True, max_length=15)
+    party = models.ForeignKey(Party, on_delete=models.CASCADE)
+    published = models.DateField(null=True)
+    raw_data = models.JSONField()
+
+    objects = PartyAccountsQueryset.as_manager()
+
+
+class PartyDonationsQueryset(QuerySet):
+    def recent(self):
+        return self.order_by("-published")[:10]
+
+    def invoices(self):
+        return self.filter(raw_data__RedactedSupportingInvoiceId__isnull=False)
+
+    def individuals(self):
+        return self.filter(raw_data__DonorStatus="Individual")
+
+
+class PartyDonations(TimeStampedModel):
+    ec_id = models.CharField(primary_key=True, max_length=15)
+    party = models.ForeignKey(Party, on_delete=models.CASCADE)
+    published = models.DateField(null=True)
+    raw_data = models.JSONField()
+
+    objects = PartyDonationsQueryset.as_manager()