SPIKE: highlight withdrawals

ynr/apps/bulk_adding/forms.py

@@ -69,9 +69,9 @@ def __init__(self, *args, **kwargs):
     def get_form_kwargs(self, index):
         kwargs = super().get_form_kwargs(index)
         kwargs["party_choices"] = self.parties
-        kwargs["previous_party_affiliations_choices"] = (
-            self.previous_party_affiliations_choices
-        )
+        kwargs[
+            "previous_party_affiliations_choices"
+        ] = self.previous_party_affiliations_choices
         return kwargs
 
     @property

ynr/apps/bulk_adding/tests/test_bulk_add_by_party.py

@@ -198,9 +198,9 @@ def test_submit_name_and_social_media_links_for_area(self):
         form[f"{ballot.pk}-0-person_identifiers_0_1"] = "homepage_url"
         form[f"{ballot.pk}-0-person_identifiers_1_0"] = "[email protected]"
         form[f"{ballot.pk}-0-person_identifiers_1_1"] = "email"
-        form[f"{ballot.pk}-0-person_identifiers_2_0"] = (
-            "https://linkedin.com/in/pamphero"
-        )
+        form[
+            f"{ballot.pk}-0-person_identifiers_2_0"
+        ] = "https://linkedin.com/in/pamphero"
         form[f"{ballot.pk}-0-person_identifiers_2_1"] = "linkedin_url"
 
         response = form.submit().follow()
@@ -304,22 +304,22 @@ def test_bulk_add_with_100_ballots(self):
         form["source"] = "https://example.com/candidates/"
         for ballot in ballots:
             form[f"{ballot.pk}-0-name"] = f"Candidate {ballot.pk}"
-            form[f"{ballot.pk}-0-biography"] = (
-                f"Biography for Candidate {ballot.pk}"
-            )
+            form[
+                f"{ballot.pk}-0-biography"
+            ] = f"Biography for Candidate {ballot.pk}"
             form[f"{ballot.pk}-0-gender"] = "female"
             form[f"{ballot.pk}-0-birth_date"] = "1990"
-            form[f"{ballot.pk}-0-person_identifiers_0_0"] = (
-                f"https://example.com/{ballot.pk}"
-            )
+            form[
+                f"{ballot.pk}-0-person_identifiers_0_0"
+            ] = f"https://example.com/{ballot.pk}"
             form[f"{ballot.pk}-0-person_identifiers_0_1"] = "homepage_url"
-            form[f"{ballot.pk}-0-person_identifiers_1_0"] = (
-                f"candidate{ballot.pk}@example.com"
-            )
+            form[
+                f"{ballot.pk}-0-person_identifiers_1_0"
+            ] = f"candidate{ballot.pk}@example.com"
             form[f"{ballot.pk}-0-person_identifiers_1_1"] = "email"
-            form[f"{ballot.pk}-0-person_identifiers_2_0"] = (
-                f"https://linkedin.com/in/candidate{ballot.pk}"
-            )
+            form[
+                f"{ballot.pk}-0-person_identifiers_2_0"
+            ] = f"https://linkedin.com/in/candidate{ballot.pk}"
             form[f"{ballot.pk}-0-person_identifiers_2_1"] = "linkedin_url"
 
         # Submit the form

ynr/apps/elections/templates/elections/includes/_sopn_debug.html

@@ -12,6 +12,15 @@ <h3>Parsing Status</h3>
             <li>AWS Textract Data: {% if textract_parsed.raw_data %}Yes{% else %}No{% endif %}</li>
             <li>AWS Textract Parsed? {% if textract_parsed.parsed_data %}Yes{% else %}
                 No{% endif %}</li>
+            <li>
+                Withdrawal detected:
+                {% if textract_parsed.withdrawal_rows %}
+                    Yes in row{{ textract_parsed.withdrawal_rows|pluralize }}
+                    {{ textract_parsed.withdrawal_rows|join:", " }}
+                {% else %}
+                    No
+                {% endif %}
+            </li>
         </ul>
 
         <h3>Camelot raw Data</h3>
@@ -28,6 +37,7 @@ <h3>Camelot table Data</h3>
             N/A
         {% endif %}
         <br/>
+        {{ textract_parsed.as_pandas.to_html|safe }}
 
 
         {% if textract_parsed and textract_parsed.as_textractor_document %}
@@ -37,6 +47,7 @@ <h5>{{ table.title.text }}</h5>
                 {{ table.to_html|safe }}
             {% endfor %}
         {% endif %}
+
 
         {% if textract_parsed.parsed_data %}
             <h3>AWS document markdown</h3>

ynr/apps/elections/templates/elections/sopn_for_ballot.html

@@ -67,6 +67,7 @@ <h2>
             {% include "elections/includes/_sopn_debug.html" %}
         {% endif %}
 
+
         <div id="sopn-{{ object.ballot_paper_id }}" class="pdf_container"></div>
 
     {% else %}
@@ -78,13 +79,14 @@ <h2>
         {% url 'admin:official_documents_ballotsopn_change' object.sopn.id as url %}
         You can <a href="{{ url }}">edit this in the admin interface</a> (e.g. to delete it)
     {% endif %}
-
     {% if object.sopn.uploaded_file.url|slice:"-3:" == "pdf" %}
         <script type="module">
-            import { SOPN_VIEWER } from '/upload_document/sopn_viewer.js';
+            import {SOPN_VIEWER} from '/upload_document/sopn_viewer.js';
+
             SOPN_VIEWER.ShowSOPNInline(
                 '{{ object.sopn.uploaded_file.url }}',
-                '{{ object.ballot_paper_id }}'
+                '{{ object.ballot_paper_id }}',
+                {{ object.sopn.awstextractparsedsopn.get_withdrawals_bboxes|safe }}
             )
         </script>
     {% else %}

ynr/apps/official_documents/migrations/0039_ballotsopn_withdrawal_detected_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.16 on 2025-04-02 08:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("official_documents", "0038_ballotsopn_replacement_reason_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ballotsopn",
+            name="withdrawal_detected",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="ballotsopnhistory",
+            name="withdrawal_detected",
+            field=models.BooleanField(default=False),
+        ),
+    ]

ynr/apps/official_documents/models.py

@@ -220,6 +220,8 @@ class BaseBallotSOPN(TimeStampedModel):
         blank=True,
     )
 
+    withdrawal_detected = models.BooleanField(default=False)
+
     class Meta:
         get_latest_by = "modified"
         abstract = True

ynr/apps/official_documents/templates/official_documents/sopn_viewer.js

@@ -9,7 +9,17 @@ var SOPN_VIEWER = (function () {
 
     var module = {};
 
-    function load_page(pdf, container, page_num) {
+    function drawRectangles(context, rectangles) {
+        rectangles.forEach(rect => {
+            context.beginPath();
+            context.rect(rect.x, rect.y, rect.width, rect.height);
+            context.lineWidth = rect.lineWidth || 1;
+            context.strokeStyle = rect.color || 'red';
+            context.stroke();
+        });
+    }
+
+    function load_page(pdf, container, page_num, rectanglesPerPage) {
         return pdf.getPage(page_num).then(function (page) {
 
             var scale = 1.2;
@@ -25,6 +35,8 @@ var SOPN_VIEWER = (function () {
                 var context = canvas.getContext("2d");
                 canvas.height = viewport.height;
                 canvas.width = viewport.width;
+                console.log(canvas.height)
+                console.log(canvas.width)
                 var renderContext = {
                     canvasContext: context,
                     viewport: viewport
@@ -33,6 +45,12 @@ var SOPN_VIEWER = (function () {
                 var renderTask = page.render(renderContext);
                 return renderTask.promise.then(function () {
                     container.append(page_container);
+
+                    if (rectanglesPerPage && rectanglesPerPage[page_num]) {
+                        drawRectangles(context, rectanglesPerPage[page_num]);
+                    }
+
+
                     return page.getTextContent({normalizeWhitespace: true});
                 }).then(function (textContent) {
                     var pdf_canvas = $(canvas),
@@ -57,14 +75,16 @@ var SOPN_VIEWER = (function () {
                         viewport: viewport,
                         textDivs: []
                     });
+
+
                 });
 
             }
 
         });
     }
 
-    function ShowSOPNInline(sopn_url, ballot_paper_id, options) {
+    function ShowSOPNInline(sopn_url, ballot_paper_id, rectanglesPerPage) {
         // The container element
         var this_pdf_container = document.getElementById("sopn-" + ballot_paper_id);
 
@@ -73,7 +93,7 @@ var SOPN_VIEWER = (function () {
         loadingTask.promise.then(function (pdf) {
             var promise = Promise.resolve();
             for (let page = 1; page <= pdf.numPages; page++) {
-                promise = promise.then(() => load_page(pdf, this_pdf_container, page));
+                promise = promise.then(() => load_page(pdf, this_pdf_container, page, rectanglesPerPage));
             }
             return promise;
         }).then(null, function (error) {

ynr/apps/parties/constants.py

@@ -60,7 +60,7 @@
     "PP504": 8345,
     # Independent Network
     "PP1951": 8015,
-    #Propel
+    # Propel
     "PP12731": 7769,
     # Chesterfield And North Derbyshire Independents (CANDI)
     "PP2883": 4670,

ynr/apps/people/forms/forms.py

@@ -107,9 +107,9 @@ def clean(self):
         if self.cleaned_data.get("value_type") in self.HTTP_IDENTIFIERS:
             # Add https schema if missing
             if not self.cleaned_data.get("value").startswith("http"):
-                self.cleaned_data["value"] = (
-                    f"https://{self.cleaned_data['value']}"
-                )
+                self.cleaned_data[
+                    "value"
+                ] = f"https://{self.cleaned_data['value']}"
             URLValidator()(value=self.cleaned_data["value"])
         if (
             "value_type" in self.cleaned_data
@@ -216,9 +216,9 @@ def __init__(self, *args, **kwargs):
         )
 
         if self.show_previous_party_affiliations:
-            self.fields["previous_party_affiliations"] = (
-                PreviousPartyAffiliationsField(membership=self.instance)
-            )
+            self.fields[
+                "previous_party_affiliations"
+            ] = PreviousPartyAffiliationsField(membership=self.instance)
 
     @property
     def show_previous_party_affiliations(self):

ynr/apps/sopn_parsing/helpers/parse_tables.py

@@ -3,6 +3,7 @@
 
 from bulk_adding.models import RawPeople
 from candidates.models import Ballot
+from django.conf import settings
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.files.base import ContentFile
 from django.core.files.storage import DefaultStorage
@@ -13,6 +14,7 @@
 from pandas import DataFrame
 from parties.models import Party, PartyDescription
 from sopn_parsing.helpers.text_helpers import clean_text
+from sopn_parsing.models import AWSTextractParsedSOPN
 from utils.db import Levenshtein
 
 FIRST_NAME_FIELDS = [
@@ -479,17 +481,20 @@ def parse_raw_data(ballot: Ballot, reparse=False):
     Given a Ballot, go and get the Camelot and the AWS Textract dataframes
     and process them
     """
-
-    camelot_model = getattr(ballot.sopn, "camelotparsedsopn", None)
     camelot_data = {}
-    textract_model = getattr(ballot.sopn, "awstextractparsedsopn", None)
-    textract_data = {}
-    if (
+    camelot_model = getattr(ballot.sopn, "camelotparsedsopn", None)
+    if getattr(settings, "CAMELOT_ENABLED", False) and (
         camelot_model
         and camelot_model.raw_data_type == "pandas"
         and (reparse or not camelot_model.parsed_data)
     ):
         camelot_data = parse_dataframe(ballot, camelot_model.as_pandas)
+
+    textract_model: AWSTextractParsedSOPN = getattr(
+        ballot.sopn, "awstextractparsedsopn", None
+    )
+    textract_data = {}
+
     if (
         textract_model
         and textract_model.raw_data
@@ -498,6 +503,9 @@ def parse_raw_data(ballot: Ballot, reparse=False):
     ):
         if not textract_model.parsed_data:
             textract_model.parse_raw_data()
+            if textract_model.withdrawal_rows():
+                ballot.sopn.withdrawal_detected = True
+                ballot.sopn.save()
         textract_data = parse_dataframe(ballot, textract_model.as_pandas)
 
     if camelot_data or textract_data:

ynr/apps/sopn_parsing/migrations/0007_awstextractparsedsopn_sopn_camelotparsedsopn_sopn.py

@@ -10,10 +10,7 @@ class Migration(migrations.Migration):
             "sopn_parsing",
             "0006_rename_sopn_awstextractparsedsopn_official_document_and_more",
         ),
-        (
-            "official_documents",
-            "0033_ballotsopnhistory_ballotsopn"
-        ),
+        ("official_documents", "0033_ballotsopnhistory_ballotsopn"),
     ]
 
     operations = [