Download GeoJSON and GeoPackage

optimap/settings.py

@@ -210,6 +210,8 @@
     "django.contrib.sites.middleware.CurrentSiteMiddleware",
     "sesame.middleware.AuthenticationMiddleware",
     "django_currentuser.middleware.ThreadLocalUserMiddleware",
+    "django.middleware.gzip.GZipMiddleware",
+
 ]
 
 ROOT_URLCONF = 'optimap.urls'

optimap/urls.py

@@ -25,8 +25,9 @@
 }
 
 urlpatterns = [
-    path('admin/', admin.site.urls),    
-    path('', include('publications.urls')),
+    path('admin/', admin.site.urls),
+    # Updated inclusion with namespace
+    path('', include(('publications.urls', 'publications'), namespace='publications')),
     path(
         "sitemap.xml",
         sitemaps_views.index,
@@ -40,16 +41,13 @@
         name="django.contrib.sitemaps.views.sitemap",
     ),
     re_path(r'^robots.txt', RobotsView.as_view(), name="robots_file"),
-    ]
+]
 
-# https://stackoverflow.com/a/18272203/261210
+# Context processor for the site
 from django.contrib.sites.shortcuts import get_current_site
 from django.utils.functional import SimpleLazyObject
 
 def site(request):
     protocol = 'https' if request.is_secure() else 'http'
     site = SimpleLazyObject(lambda: "{0}://{1}".format(protocol, get_current_site(request)))
-
-    return {
-        'site': site,
-    }
+    return {'site': site}

publications/management/commands/schedule_geojson.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from django_q.tasks    import schedule
+from django_q.models   import Schedule
+
+class Command(BaseCommand):
+    help = "Schedule the GeoJSON regeneration task every 6 hours."
+
+    def handle(self, *args, **options):
+        func_name = 'publications.tasks.regenerate_geojson_cache'
+        if not Schedule.objects.filter(func=func_name).exists():
+            schedule(
+                func_name,
+                schedule_type='I',   # interval
+                minutes=360,         # every 6 hours
+                repeats=-1
+            )
+            self.stdout.write(self.style.SUCCESS("Scheduled GeoJSON regeneration every 6h."))
+        else:
+            self.stdout.write("GeoJSON regeneration already scheduled.")

publications/tasks.py

@@ -1,36 +1,37 @@
 import logging
 logger = logging.getLogger(__name__)
 
-from django_q.models import Schedule
-from publications.models import Publication, HarvestingEvent, Source
-from bs4 import BeautifulSoup
+import os
 import json
+import subprocess
+import gzip
+import re
+import tempfile
+import time
+import calendar
+from datetime import datetime, timedelta
 import xml.dom.minidom
-from django.contrib.gis.geos import GEOSGeometry
 import requests
-from django.core.mail import send_mail, EmailMessage
-from django.utils import timezone 
+from bs4 import BeautifulSoup
 from requests.auth import HTTPBasicAuth
-import os
+from urllib.parse import quote
 from django.conf import settings
-from django.utils.timezone import now
+from django.core.mail import send_mail, EmailMessage
+from django.core.serializers import serialize
+from django.contrib.gis.geos import GEOSGeometry
+from django.utils import timezone
+from django_q.tasks import schedule
+from django_q.models import Schedule
+from publications.models import Publication, HarvestingEvent, Source
+from .models import EmailLog, Subscription
 from django.contrib.auth import get_user_model
 User = get_user_model()
-from .models import EmailLog, Subscription
-from datetime import datetime, timedelta
 from django.urls import reverse
-from urllib.parse import quote
-from datetime import datetime
-from django_q.tasks import schedule
-from django.utils import timezone 
-from django_q.tasks import schedule
-from django_q.models import Schedule
-import time  
-import calendar
-import re
 
 BASE_URL = settings.BASE_URL
 
+DOI_REGEX = re.compile(r'10\.\d{4,9}/[-._;()/:A-Z0-9]+', re.IGNORECASE)
+
 def extract_geometry_from_html(content):
     for tag in content.find_all("meta"):
         if tag.get("name", None) == "DC.SpatialCoverage":
@@ -275,4 +276,45 @@ def schedule_subscription_email_task(sent_by=None):
             kwargs={'trigger_source': 'scheduled', 'sent_by': sent_by.id if sent_by else None} 
         )
         logger.info(f"Scheduled 'send_subscription_based_email' for {next_run_date}")
+
+def regenerate_geojson_cache():
+    cache_dir = os.path.join(tempfile.gettempdir(), "optimap_cache")
+    os.makedirs(cache_dir, exist_ok=True)
+
+    json_path = os.path.join(cache_dir, 'geojson_cache.json')
+    with open(json_path, 'w') as f:
+        serialize(
+            'geojson',
+            Publication.objects.filter(status='p'),
+            geometry_field='geometry',
+            srid=4326,
+            stream=f
+        )
 
+    gzip_path = json_path + '.gz'
+    with open(json_path, 'rb') as fin, gzip.open(gzip_path, 'wb') as fout:
+        fout.writelines(fin)
+
+    size = os.path.getsize(json_path)
+    logger.info("Cached GeoJSON at %s (%d bytes), gzipped at %s", json_path, size, gzip_path)
+    return json_path
+
+
+def convert_geojson_to_geopackage(geojson_path):
+    cache_dir = os.path.join(tempfile.gettempdir(), "optimap_cache")
+    os.makedirs(cache_dir, exist_ok=True)
+    gpkg = os.path.join(cache_dir, 'publications.gpkg')
+    cmd = ["ogr2ogr", "-f", "GPKG", gpkg, geojson_path]
+    try:
+        subprocess.check_call(cmd)
+        logger.info("Generated GeoPackage at %s", gpkg)
+    except subprocess.CalledProcessError as e:
+        logger.error("ogr2ogr failed: %s", e)
+        return None
+    return gpkg
+
+
+def regenerate_geopackage_cache():
+    json_path = regenerate_geojson_cache()
+    gpkg_path = convert_geojson_to_geopackage(json_path)
+    return json_path, gpkg_path

publications/templates/data.html

@@ -1,32 +1,63 @@
 {% extends "main.html" %}
-
 {% load optimap_extras %}
 
-{% block title %}API | {% endblock %}
+{% block title %}Data & API | {% endblock %}
 
 {% block content %}
-
 <div class="row justify-content-center">
   <div class="col-4 py-5 text-wrap">
-    <h1 class="py-2">OPTIMAP data access</h1>
-
-    <p class="lead">All publication metadata published in OPTIMAP is licensed under a Create Commons Zero (<a href='https://creativecommons.org/publicdomain/zero/1.0/'>CC-0</a>) license.</p>
-
-    <h2 class="py-2">API endpoint</h2>
-    <p>The API endpoint is <b>{{ site|addstr:"/api"|urlize }}</b>. Visit the URL in your browser to get an interactive interface for exploring the API.</p>
-
-    <p>You can query all publications with the following request (using <a href="https://stedolan.github.io/jq/" title="Link to jq project website"><code>jq</code></a> for formatting of the response):</p>
+    <h1 class="py-2">OPTIMAP Data & API Access</h1>
+    <p class="lead">
+      All publication metadata published in OPTIMAP is licensed under a Creative Commons Zero 
+      (<a href="https://creativecommons.org/publicdomain/zero/1.0/" target="_blank">CC-0</a>) license.
+    </p>
+    <h2 class="py-2">API Endpoint</h2>
+    <p>
+      The API endpoint is <b>{{ site|addstr:"/api"|urlize }}</b>. Visit the URL in your browser 
+      to get an interactive interface for exploring the API.
+    </p>
+    <p>
+      You can query all publications with the following request (using 
+      <a href="https://stedolan.github.io/jq/" title="Link to jq project website" target="_blank"><code>jq</code></a> for formatting):
+    </p>
     <pre>
-      curl -X GET {{ site|addstr:"/api" }}/api/publications/ | jq
+curl -X GET {{ site|addstr:"/api" }}/api/publications/ | jq
     </pre>
-
-    <h2 class="py-2">OpenAPI schema</h2>
-    <p>You can download an OpenAPI specification of the api at <b>{{ site|addstr:"/api/schema"|urlize }}</b>.</p>
-
-    <h2 class="py-2">OpenAPI user interface</h2>
-    <p>You can explore the API with an interactive user intreface built based on the OpenAPI schema at <b>{{ site|addstr:"/api/schema/ui"|urlize }}</b>.</p>
-
+    <h2 class="py-2">OpenAPI Schema</h2>
+    <p>
+      You can download an OpenAPI specification of the API at <b>{{ site|addstr:"/api/schema"|urlize }}</b>.
+    </p>
+    <h2 class="py-2">OpenAPI User Interface</h2>
+    <p>
+      Explore the API with an interactive user interface built based on the OpenAPI schema at <b>{{ site|addstr:"/api/schema/ui"|urlize }}</b>.
+    </p>
+    <hr>
+    <h2 class="py-2">Download Publication Data</h2>
+    <p>
+      Choose your desired file format.
+    </p>
+    <ul>
+      <li class="text-nowrap">
+        <a class="btn btn-primary" href="{% url 'publications:download_geojson' %}">
+          Download GeoJSON
+        </a>
+        <span class="text-nowrap">
+          (<a href="https://geojson.org/" target="_blank">GeoJSON spec</a>) {{ geojson_size }}
+        </span>
+      </li>
+      <li class="text-nowrap">
+        <a class="btn btn-primary" href="{% url 'publications:download_geopackage' %}">
+          Download GeoPackage
+        </a>
+        <span class="text-nowrap">
+          (<a href="https://www.geopackage.org/" target="_blank">GeoPackage spec</a>) {{ geopackage_size }}
+        </span>
+      </li>
+    </ul>
+
+    <p class="small text-muted">
+      Data dumps are recreated every 6 hours.&nbsp;Last updated: {{ last_updated }}
+    </p>
   </div>
 </div>
-
-{% endblock %}
+{% endblock %}

publications/templates/footer.html

@@ -4,7 +4,7 @@
             <span class="px-3">&copy;&nbsp;2023&nbsp;<a class="text-white" title="OPTIMETA project website" href="https://projects.tib.eu/optimeta">OPTIMETA project</a> &amp; <a class="text-white" title="KOMET project website" href="https://projects.tib.eu/komet">KOMET project</a></span>
             <a class="px-3 text-white" title="Link to source code project" href="https://github.com/GeoinformationSystems/optimap">Code</a>
             <a class="px-3 text-white" title="Privace information / Imprint" href="{% url 'optimap:privacy' %}">Privacy / Imprint / Contact</a>
-            <a class="px-3 text-white" title="API browser" href="{% url 'optimap:data' %}">API</a>
+            <a class="px-3 text-white" title="Data & API browser" href="{% url 'optimap:data_and_api' %}">API & Data</a>
             <span class="px-3">Publication data license: <a class="text-white" title="Publication metadata license" href='https://creativecommons.org/publicdomain/zero/1.0/'>CC-0</a></span>
         </p>
     </div>

publications/urls.py

@@ -19,7 +19,7 @@
     path("api/v1/", include("publications.api")),
     path('api/schema/', SpectacularAPIView.as_view(), name='schema'),
     path('api/schema/ui/sitemap', SpectacularRedocView.as_view(url_name='optimap:schema'), name='redoc'),
-    path("data/", views.data, name="data"),
+    path("data/", views.data, name="data_and_api"),
     path('feed/georss/', GeoFeed(feed_type_variant="georss"), name='georss_feed'),
     path('feed/geoatom/', GeoFeed(feed_type_variant="geoatom"), name='geoatom_feed'),
     path('feed/w3cgeo/', GeoFeed(feed_type_variant="w3cgeo"), name='w3cgeo_feed'),
@@ -37,5 +37,7 @@
     path("confirm-delete/<str:token>/", views.confirm_account_deletion, name="confirm_delete"),
     path("finalize-delete/", views.finalize_account_deletion, name="finalize_delete"),
     path("changeuser/", views.change_useremail, name="changeuser"),
+    path('download/geojson/', views.download_geojson, name='download_geojson'),
+    path('download/geopackage/', views.download_geopackage, name='download_geopackage'),
     path("confirm-email/<str:token>/<str:email_new>/", views.confirm_email_change, name="confirm-email-change"),
 ]