Download GeoJSON and GeoPackage

optimap/settings.py

@@ -59,6 +59,7 @@
     'django.contrib.staticfiles',
     'django.contrib.gis',
     'django.contrib.sitemaps',
+    'django.contrib.humanize',
     'publications',
     'rest_framework',
     'rest_framework_gis',

optimap/urls.py

@@ -26,8 +26,7 @@
 
 urlpatterns = [
     path('admin/', admin.site.urls),
-    # Updated inclusion with namespace
-    path('', include(('publications.urls', 'publications'), namespace='publications')),
+    path('', include(('publications.urls', 'optimap'), namespace='optimap')),
     path(
         "sitemap.xml",
         sitemaps_views.index,

publications/sitemaps.py

@@ -24,7 +24,7 @@ class StaticViewSitemap(Sitemap):
 
     def items(self):
         return ["main",
-                "data",
+                "data_and_api",
                 "api",
                 "privacy"]
 

publications/tasks.py

@@ -27,6 +27,16 @@
 from django.contrib.auth import get_user_model
 User = get_user_model()
 from django.urls import reverse
+from urllib.parse import quote
+from datetime import datetime
+from django_q.tasks import schedule
+from django.utils import timezone 
+from django_q.tasks import schedule
+from django_q.models import Schedule
+import time  
+import calendar
+import re
+from django.contrib.gis.geos import GeometryCollection
 
 BASE_URL = settings.BASE_URL
 

publications/templates/data.html

@@ -1,6 +1,6 @@
 {% extends "main.html" %}
 {% load optimap_extras %}
-
+{% load humanize %}
 {% block title %}Data & API | {% endblock %}
 
 {% block content %}
@@ -21,7 +21,7 @@ <h2 class="py-2">API Endpoint</h2>
       <a href="https://stedolan.github.io/jq/" title="Link to jq project website" target="_blank"><code>jq</code></a> for formatting):
     </p>
     <pre>
-curl -X GET {{ site|addstr:"/api" }}/api/publications/ | jq
+curl -X GET {{ site|addstr:"/api" }}/api/optimap/ | jq
     </pre>
     <h2 class="py-2">OpenAPI Schema</h2>
     <p>
@@ -38,15 +38,15 @@ <h2 class="py-2">Download Publication Data</h2>
     </p>
     <ul>
       <li class="text-nowrap">
-        <a class="btn btn-primary" href="{% url 'publications:download_geojson' %}">
+        <a class="btn btn-primary" href="{% url 'optimap:download_geojson' %}">
           Download GeoJSON
         </a>
         <span class="text-nowrap">
           (<a href="https://geojson.org/" target="_blank">GeoJSON spec</a>) {{ geojson_size }}
         </span>
       </li>
       <li class="text-nowrap">
-        <a class="btn btn-primary" href="{% url 'publications:download_geopackage' %}">
+        <a class="btn btn-primary" href="{% url 'optimap:download_geopackage' %}">
           Download GeoPackage
         </a>
         <span class="text-nowrap">
@@ -56,8 +56,8 @@ <h2 class="py-2">Download Publication Data</h2>
     </ul>
 
     <p class="small text-muted">
-      Data dumps are recreated every 6 hours.&nbsp;Last updated: {{ last_updated }}
-    </p>
+      Data dumps are recreated every 6 hours. Last updated: {{ last_updated|naturaltime }}
+    </p>    
   </div>
 </div>
 {% endblock %}

publications/urls.py

@@ -19,7 +19,7 @@
     path("api/v1/", include("publications.api")),
     path('api/schema/', SpectacularAPIView.as_view(), name='schema'),
     path('api/schema/ui/sitemap', SpectacularRedocView.as_view(url_name='optimap:schema'), name='redoc'),
-    path("data/", views.data, name="data_and_api"),
+    path("data_and_api/", views.data, name="data_and_api"),
     path('feed/georss/', GeoFeed(feed_type_variant="georss"), name='georss_feed'),
     path('feed/geoatom/', GeoFeed(feed_type_variant="geoatom"), name='geoatom_feed'),
     path('feed/w3cgeo/', GeoFeed(feed_type_variant="w3cgeo"), name='w3cgeo_feed'),

publications/views.py

@@ -14,7 +14,7 @@
 from django.views.decorators.cache import never_cache
 from django.urls import reverse
 import uuid
-from django.utils.timezone import now
+from django.utils.timezone import now, get_default_timezone
 from datetime import datetime
 import imaplib
 import time
@@ -26,14 +26,12 @@
 from publications.models import BlockedEmail, BlockedDomain, Subscription, UserProfile, Publication
 from django.contrib.auth import get_user_model
 User = get_user_model()
-
 import tempfile, os
-import fiona
-from fiona.crs import from_epsg
-from shapely import wkt
-from shapely.geometry import mapping
-from django.contrib.gis.geos import GEOSGeometry
 from publications.tasks import regenerate_geojson_cache
+from osgeo import ogr, osr
+
+ogr.UseExceptions()
+osr.UseExceptions()
 
 LOGIN_TOKEN_LENGTH  = 32
 LOGIN_TOKEN_TIMEOUT_SECONDS = 10 * 60
@@ -83,47 +81,50 @@ def download_geojson(request):
     return response
 
 def generate_geopackage():
-    """
-    Generates a GeoPackage file from Publication data using Fiona.
-    Geometry is converted using Shapely. The file is written to a persistent
-    cache directory and its filename is returned.
-    """
-    schema = {
-        'geometry': 'Unknown', 
-        'properties': {
-            'title': 'str',
-            'abstract': 'str',
-            'doi': 'str',
-            'source': 'str',
-        },
-    }
     cache_dir = os.path.join(tempfile.gettempdir(), "optimap_cache")
     os.makedirs(cache_dir, exist_ok=True)
-    filename = os.path.join(cache_dir, "publications.gpkg")
-    with fiona.open(filename, 'w', driver='GPKG', schema=schema, crs=from_epsg(4326)) as collection:
-        for pub in Publication.objects.all():
-            geom = None
-            if pub.geometry:
-                try:
-                    geom_wkt = pub.geometry.wkt if hasattr(pub.geometry, 'wkt') else str(pub.geometry)                    
-                    if geom_wkt.startswith("SRID="):
-                        geom_wkt = geom_wkt.split(";", 1)[1]
-                    geom_obj = wkt.loads(geom_wkt)
-                    geom = mapping(geom_obj)                                        
-                except Exception as e:
-                    logger.error("Error converting geometry for publication %s: %s", pub.id, e)
-                    geom = None
-            feature = {
-                'geometry': geom,
-                'properties': {
-                    'title': pub.title or "",
-                    'abstract': pub.abstract or "",
-                    'doi': pub.doi or "",
-                    'source': pub.source or "",
-                },
-            }
-            collection.write(feature)
-    return filename
+    gpkg_path = os.path.join(cache_dir, "publications.gpkg")
+
+    # Delete old file so driver can re-create it
+    driver = ogr.GetDriverByName("GPKG")
+    if os.path.exists(gpkg_path):
+        driver.DeleteDataSource(gpkg_path)
+
+    # Create data source & layer
+    ds = driver.CreateDataSource(gpkg_path)
+    srs = osr.SpatialReference()
+    srs.ImportFromEPSG(4326)
+    layer = ds.CreateLayer("publications", srs, ogr.wkbUnknown)
+
+    # Define fields
+    for name in ("title", "abstract", "doi", "source"):
+        field_defn = ogr.FieldDefn(name, ogr.OFTString)
+        field_defn.SetWidth(255)
+        layer.CreateField(field_defn)
+
+    # Populate features
+    layer_defn = layer.GetLayerDefn()
+    for pub in Publication.objects.all():
+        feat = ogr.Feature(layer_defn)
+        feat.SetField("title", pub.title or "")
+        feat.SetField("abstract", pub.abstract or "")
+        feat.SetField("doi", pub.doi or "")
+        feat.SetField("source", pub.source or "")
+
+        if pub.geometry:
+            # Use the GEOSGeometry WKB directly
+            wkb = pub.geometry.wkb  # bytes
+            geom = ogr.CreateGeometryFromWkb(wkb)
+            geom.AssignSpatialReference(srs)
+            feat.SetGeometry(geom)
+
+        layer.CreateFeature(feat)
+        feat = None
+
+    # Clean up
+    ds = None
+    return gpkg_path
+
 
 @require_GET
 def download_geopackage(request):
@@ -203,21 +204,27 @@ def privacy(request):
 def data(request):
     cache_dir = os.path.join(tempfile.gettempdir(), "optimap_cache")
     os.makedirs(cache_dir, exist_ok=True)
-    json_path = os.path.join(cache_dir, 'geojson_cache.json')
+    json_path = os.path.join(cache_dir, "geojson_cache.json")
     if not os.path.exists(json_path):
         json_path = regenerate_geojson_cache()
     geojson_size = format_file_size(os.path.getsize(json_path))
-    last_updated = datetime.fromtimestamp(os.path.getmtime(json_path)).strftime('%Y-%m-%d %H:%M:%S')
-    logger.info("GeoJSON cache last updated at %s", last_updated)
 
+    # Build a timezone-aware datetime from the file’s modification time
+    mtime = os.path.getmtime(json_path)
+    tz = get_default_timezone()
+
+    last_updated = datetime.fromtimestamp(mtime, tz)
     gpkg_filename = generate_geopackage()
-    geopackage_size = format_file_size(os.path.getsize(gpkg_filename)) if os.path.exists(gpkg_filename) else "0 B"
-    context = {
-        'geojson_size': geojson_size,
-        'geopackage_size': geopackage_size,
-        'last_updated': last_updated  
-    }
-    return render(request, 'data.html', context)
+    if os.path.exists(gpkg_filename):
+        geopackage_size = format_file_size(os.path.getsize(gpkg_filename))
+    else:
+        geopackage_size = "0 B"
+
+    return render(request, "data.html", {
+        "geojson_size":     geojson_size,
+        "geopackage_size":  geopackage_size,
+        "last_updated":     last_updated,  # pass the aware datetime
+    })
 
 def Confirmationlogin(request):
     return render(request, 'confirmation_login.html')
@@ -505,31 +512,6 @@ def finalize_account_deletion(request):
             del request.session[USER_DELETE_TOKEN_PREFIX]
             request.session.modified = True  
 
-@require_GET
-def download_geojson(request):
-    """
-    Returns the cached GeoJSON file as a downloadable file.
-    """
-    cache_dir = os.path.join(tempfile.gettempdir(), "optimap_cache")
-    os.makedirs(cache_dir, exist_ok=True)
-    json_path = os.path.join(cache_dir, 'geojson_cache.json')
-    if not os.path.exists(json_path):
-        json_path = regenerate_geojson_cache()
-    return FileResponse(open(json_path, 'rb'), content_type="application/json",
-                        as_attachment=True, filename="publications.geojson")
-
-# New Functionality: Download as GeoPackage using Fiona and Shapely
-@require_GET
-def download_geopackage(request):
-    """
-    Generates a GeoPackage file from Publication data and returns it as a downloadable file.
-    """
-    filename = generate_geopackage()
-    if not os.path.exists(filename):
-        return HttpResponse("Error generating GeoPackage.", status=500)
-    return FileResponse(open(filename, 'rb'), content_type="application/geopackage+sqlite3",
-                        as_attachment=True, filename="publications.gpkg")
-
 class RobotsView(View):
     http_method_names = ['get']
     def get(self, request):

requirements.txt

@@ -35,5 +35,3 @@ whitenoise==6.8.2
 psycopg2-binary==2.9.10
 packaging==21.3
 pycryptodome==3.21.0
-Fiona>=1.8.20
-Shapely>=1.8.0

tests/test_geo_data.py

@@ -94,8 +94,8 @@ def test_data_endpoint(self):
         response = self.client.get(reverse('optimap:data_and_api'))
         self.assertEqual(response.status_code, 200, "Data endpoint should return status 200")
         content = response.content.decode()
-        self.assertIn("Data dumps are recreated each night.", content,
-                      "The data page should mention nightly recreation")
+        +   self.assertIn("Data dumps are recreated every 6 hours.", content,
+                      "The data page should mention the six-hour schedule")
         match = re.search(r'Last updated:\s*(\S+)', content)
         self.assertIsNotNone(match, "Data page should display a Last updated timestamp")
         self.assertTrue(match.group(1).strip() != "", "Last updated timestamp should not be empty")