Merge pull request #99 from GSA/add-pagination

Adds Pagination and Partial SPA-like page rerendering

app/__init__.py

@@ -3,6 +3,7 @@
 from dotenv import load_dotenv
 from flask import Flask
 from flask_bootstrap import Bootstrap
+from flask_htmx import HTMX
 from flask_migrate import Migrate
 
 from app.filters import usa_icon
@@ -19,6 +20,8 @@ def create_app():
     app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
     app.config["SECRET_KEY"] = os.getenv("FLASK_APP_SECRET_KEY")
     Bootstrap(app)
+    global htmx
+    htmx = HTMX(app)
 
     db.init_app(app)
 

app/forms.py

@@ -35,11 +35,6 @@ class HarvestSourceForm(FlaskForm):
         choices=["manual", "daily", "weekly", "biweekly", "monthly"],
         validators=[DataRequired()],
     )
-    size = SelectField(
-        "Size",
-        choices=["small", "medium", "large"],
-        validators=[DataRequired()],
-    )
     schema_type = SelectField(
         "Schema Type", choices=["strict", "other"], validators=[DataRequired()]
     )

app/paginate.py

@@ -0,0 +1,28 @@
+import math
+
+from database.interface import PAGINATE_ENTRIES_PER_PAGE
+
+
+class Pagination:
+    def __init__(self, current: int = 1, count: int = 1):
+        self.current = current
+        self.count = count
+        self.page_count = math.ceil(count / PAGINATE_ENTRIES_PER_PAGE)
+        self.per_page = PAGINATE_ENTRIES_PER_PAGE
+
+    def to_dict(self):
+        return {
+            "current": self.current,
+            "count": self.count,
+            "page_count": self.page_count,
+            "page_label": "Page",
+            "per_page": self.per_page,
+            "next": {"label": "Next"},
+            "previous": {"label": "Previous"},
+            "last_item": {
+                "label": "Last page",
+            },
+        }
+
+    def update_current(self, current: int) -> dict:
+        self.current = int(current)

app/routes.py

@@ -12,18 +12,22 @@
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from dotenv import load_dotenv
 from flask import Blueprint, flash, redirect, render_template, request, session, url_for
+from jinja2_fragments.flask import render_block
 
 from app.scripts.load_manager import schedule_first_job, trigger_manual_job
 from database.interface import HarvesterDBInterface
 
+from . import htmx
 from .forms import HarvestSourceForm, OrganizationForm
+from .paginate import Pagination
 
 logger = logging.getLogger("harvest_admin")
 
 user = Blueprint("user", __name__)
 mod = Blueprint("harvest", __name__)
 source = Blueprint("harvest_source", __name__)
 org = Blueprint("org", __name__)
+testdata = Blueprint("testdata", __name__)
 
 db = HarvesterDBInterface()
 
@@ -245,6 +249,31 @@ def cli_remove_harvest_source(id):
         print("Failed to delete harvest source")
 
 
+## Load Test Data
+# TODO move this into its own file when you break up routes
+@testdata.cli.command("load_test_data")
+def fixtures():
+    """Load database fixtures from JSON."""
+    import json
+
+    file = "./tests/fixtures.json"
+    click.echo(f"Loading fixtures at `{file}`.")
+    with open(file, "r") as file:
+        fixture = json.load(file)
+        for item in fixture["organization"]:
+            db.add_organization(item)
+        for item in fixture["source"]:
+            db.add_harvest_source(item)
+        for item in fixture["job"]:
+            db.add_harvest_job(item)
+        for item in fixture["record"]:
+            db.add_harvest_record(item)
+        for item in fixture["record_error"]:
+            db.add_harvest_record_error(item)
+
+    click.echo("Done.")
+
+
 # Helper Functions
 def make_new_source_contract(form):
     return {
@@ -434,7 +463,7 @@ def view_harvest_source_data(source_id: str):
     jobs = db.get_all_harvest_jobs_by_filter({"harvest_source_id": source.id})
     records = db.get_harvest_record_by_source(source.id)
     ckan_records = [record for record in records if record.ckan_id is not None]
-    error_records = [record for record in records if record.status == 'error']
+    error_records = [record for record in records if record.status == "error"]
     jobs = db.get_all_harvest_jobs_by_filter({"harvest_source_id": source.id})
     next_job = "N/A"
     future_jobs = db.get_new_harvest_jobs_by_source_in_future(source.id)
@@ -556,6 +585,7 @@ def clear_harvest_source(source_id):
         flash("Failed to clear harvest source")
         return {"message": "failed"}
 
+
 # Delete Source
 @mod.route("/harvest_source/config/delete/<source_id>", methods=["POST"])
 @login_required
@@ -596,18 +626,48 @@ def add_harvest_job():
 @mod.route("/harvest_job/", methods=["GET"])
 @mod.route("/harvest_job/<job_id>", methods=["GET"])
 def get_harvest_job(job_id=None):
+    record_error_count = db.get_harvest_record_errors_by_job(
+        job_id, count=True, skip_pagination=True
+    )
+    htmx_vars = {
+        "target_div": "#error_results_pagination",
+        "endpoint_url": f"/harvest_job/{job_id}",
+    }
+
+    pagination = Pagination(count=record_error_count)
+
+    if htmx:
+        page = request.args.get("page")
+        db_page = int(page) - 1
+        record_errors = db.get_harvest_record_errors_by_job(job_id, page=db_page)
+        data = {
+            "harvest_job_id": job_id,
+            "record_errors": db._to_dict(record_errors),
+            "htmx_vars": htmx_vars,
+        }
+        pagination.update_current(page)
+        return render_block(
+            "view_job_data.html",
+            "record_errors_table",
+            data=data,
+            pagination=pagination.to_dict(),
+        )
     if job_id:
         job = db.get_harvest_job(job_id)
         record_errors = db.get_harvest_record_errors_by_job(job_id)
         if request.args.get("type") and request.args.get("type") == "json":
             return db._to_dict(job) if job else ("Not Found", 404)
         else:
             data = {
+                "harvest_job_id": job_id,
                 "harvest_job": job,
                 "harvest_job_dict": db._to_dict(job),
                 "record_errors": db._to_dict(record_errors),
+                "htmx_vars": htmx_vars,
             }
-            return render_template("view_job_data.html", data=data)
+            return render_template(
+                "view_job_data.html", data=data, pagination=pagination.to_dict()
+            )
 
     source_id = request.args.get("harvest_source_id")
     if source_id:
@@ -667,7 +727,7 @@ def get_harvest_record(record_id=None):
             return "No harvest records found for this harvest source", 404
     else:
         # TODO for test, will remove later
-        record = db.get_all_harvest_records()
+        record = db.pget_harvest_records()
 
     return db._to_dict(record)
 
@@ -744,3 +804,4 @@ def register_routes(app):
     app.register_blueprint(user)
     app.register_blueprint(org)
     app.register_blueprint(source)
+    app.register_blueprint(testdata)

app/static/_scss/_uswds-theme-custom-styles.scss

@@ -56,3 +56,9 @@ ul.menu {
 .usa-card__img img {
   padding: 10px;
 }
+
+.usa-pagination {
+  &__item {
+    cursor: pointer;
+  }
+}

app/static/package.json

@@ -25,6 +25,7 @@
   "dependencies": {
     "@uswds/uswds": "3.8.0",
     "chart.js": "^4.4.2",
+    "htmx.org": "^2.0.2",
     "rollup": "^4.18.0"
   },
   "devDependencies": {

app/static/rollup.config.mjs

@@ -8,6 +8,7 @@ export default {
       targets: [
         { src: './node_modules/chart.js/dist/chart.umd.js', dest: './assets/chartjs/' },
         { src: './node_modules/chart.js/dist/chart.umd.js.map', dest: './assets/chartjs/' },
+        { src: './node_modules/htmx.org/dist/htmx.min.js', dest: './assets/htmx/' },
       ]
     })
   ]

app/templates/base.html

@@ -8,6 +8,7 @@
     <link rel="stylesheet" href="{{ url_for('static', filename='assets/uswds/css/styles.css') }}">
     <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
     <script src="{{ url_for('static', filename='assets/uswds/js/uswds-init.js') }}"></script>
+    <script src="{{ url_for('static', filename='assets/htmx/htmx.min.js') }}"></script>
     {% block script_head %}
     {% endblock %}
 </head>

app/templates/components/pagination/pagination.html

@@ -0,0 +1,23 @@
+{% from 'components/pagination/pagination_arrow.html' import pagination_arrow %}
+
+{% set overflow %}
+  <li class="usa-pagination__item usa-pagination__overflow" aria-label="ellipsis indicating non-visible pages">
+    <span>
+      …
+    </span>
+  </li>
+{% endset %}
+
+<nav aria-label="Pagination" class="usa-pagination">
+    <ul class="usa-pagination__list">
+        {% if pagination['page_count'] > 7 and pagination['current'] > 1 %}
+            {{ pagination_arrow('previous', pagination, data.htmx_vars) }}
+        {% endif %}
+
+        {% include "components/pagination/pagination_numbers.html" %}
+
+        {% if pagination['page_count'] > 7 and pagination['current'] < pagination['page_count'] %}
+            {{ pagination_arrow('next', pagination, data.htmx_vars) }}
+        {% endif %}
+    </ul>
+</nav>

app/templates/components/pagination/pagination_arrow.html

@@ -0,0 +1,37 @@
+{#
+    The full pagination data object is passed so we can access current state, aria labels, and text labels.
+  #}
+  {% macro pagination_arrow(direction, pagination, htmx_vars) %}
+    {% set page_var = ((pagination.current - 1) if direction == 'previous' else (pagination.current + 1)) | string() %}
+    {% set placeholder_link = htmx_vars.endpoint_url + "?page=" + page_var %}
+
+    {% set link_attrs = {
+      'class': 'usa-pagination__link usa-pagination__'  ~ direction ~ '-page',
+      'aria_label': pagination[direction]['label'] ~ ' ' ~ pagination.page_label | lower
+    } %}
+
+    <li class="usa-pagination__item usa-pagination__arrow">
+      <a
+        hx-get="{{ placeholder_link | default("javascript:void(0);") }}"
+        hx-target="{{htmx_vars.target_div}}"
+        class="{{ link_attrs.class }}"
+        aria-label="{{ link_attrs.aria_label }}"
+      >
+        {% if direction == 'previous' %}
+          <svg class="usa-icon" aria-hidden="true" role="img">
+            {# Global variable not applying #}
+            <use xlink:href="/assets/uswds/img/sprite.svg#navigate_before"></use>
+          </svg>
+        {% endif %}
+        <span class="usa-pagination__link-text">
+          {{ pagination[direction]['label'] }}
+        </span>
+        {% if direction == 'next' %}
+          <svg class="usa-icon" aria-hidden="true" role="img">
+            {# Global variable not applying #}
+            <use xlink:href="/assets/uswds/img/sprite.svg#navigate_next"></use>
+          </svg>
+        {% endif %}
+      </a>
+    </li>
+  {% endmacro %}

app/templates/components/pagination/pagination_button.html

@@ -0,0 +1,23 @@
+{% macro pagination_button(item, pager_opts, htmx_vars) %}
+  {% set is_current = (item == pager_opts.current) %}
+  {% set is_last = (item == pager_opts.total) %}
+  {% set labels = pager_opts.aria_labels %}
+  {# HTMX page vars#}
+  {% set item_str = item | string() %}
+  {% set placeholder_link = htmx_vars.endpoint_url + "?page=" + item_str %}
+
+  {# Display: "Last page, page X" if last item. Otherwise "Page X" #}
+  {% set aria_label = (labels.last ~ " " ~ labels.page_label | lower if is_last else labels.page_label) ~ " " ~ item %}
+
+  <li class="usa-pagination__item usa-pagination__page-no">
+    {# Global variable placeholder_link doesn't work for some reason.  #}
+    <a
+        hx-get="{{ placeholder_link | default("javascript:void(0);") }}"
+        hx-target="{{htmx_vars.target_div}}"
+        class="usa-pagination__button {{ "usa-current" if is_current}}"
+        aria-label="{{ aria_label }}"
+        {% if is_current %}aria-current="page"{% endif %}>
+      {{ item }}
+    </a>
+  </li>
+{% endmacro %}

app/templates/components/pagination/pagination_numbers.html

@@ -0,0 +1,65 @@
+{% from "components/pagination/pagination_button.html" import pagination_button %}
+
+{# Add +1 to first_five / last_five due to how ranges work in jinja #}
+{% set pager_ranges = {
+    'default': range(pagination.current - 1, pagination.current + 1),
+    'last_item': pagination.page_count,
+    'first_five': range(1, 5 + 1),
+    'last_five': range(pagination.page_count - 4, pagination.page_count + 1),
+  }
+%}
+
+{% set pager_button_opts = {
+  'current': pagination.current,
+  'total': pagination.page_count,
+  'aria_labels': {
+    'page_label': pagination.page_label,
+    'previous': pagination.previous.label,
+    'next': pagination.next.label,
+    'last': pagination.last_item.label
+    }
+} %}
+
+{# Page numbers #}
+{# List all items if less than 7 #}
+{% if pagination.page_count <= 7 %}
+  {% for item in range(1, pagination.page_count)  %}
+    {{ pagination_button(item, pager_button_opts, data.htmx_vars) }}
+  {% endfor %}
+{# User is at the start of a long dataset #}
+{# Example: 1, 2, 3, *4*, 5 … 8 #}
+{# Doesn't apply when user gets to 5 of 8 #}
+{% elif pagination.current <= 4 and pagination.page_count >= 7 %}
+  {% for item in pager_ranges.first_five  %}
+    {{ pagination_button(item, pager_button_opts, data.htmx_vars) }}
+  {% endfor %}
+
+  {{ overflow | trim | safe }}
+
+  {{ pagination_button(pager_ranges.last_item, pager_button_opts, data.htmx_vars) }}
+
+{# When user is close to the end of dataset #}
+{# Example: 1 … 4, *5*, 6, 7, 8 #}
+{% elif pagination.current >= pagination.page_count - 3 %}
+  {{ pagination_button(1, pager_button_opts, data.htmx_vars) }}
+
+  {{ overflow | trim | safe }}
+  {% for item in pager_ranges.last_five %}
+    {{ pagination_button(item, pager_button_opts, data.htmx_vars) }}
+  {% endfor %}
+{# Default case: Current - 1, Current, Current + 1 #}
+{# Example: 1 … 21, *22*, 23 … 50 #}
+{# Example: 1 … 4, *5*, 6 … 9 #}
+{% else %}
+  {{ pagination_button(1, pager_button_opts, data.htmx_vars) }}
+
+  {{ overflow | trim | safe }}
+
+  {% for item in pager_ranges.default %}
+    {{ pagination_button(item, pager_button_opts, data.htmx_vars) }}
+  {% endfor %}
+
+  {{ overflow | trim | safe }}
+
+  {{ pagination_button(pager_ranges.last_item, pager_button_opts, data.htmx_vars) }}
+{% endif %}