Updated scripts for issue 145. Add time stamp and logging function.

Author: BharatVe

publications/templates/data.html

@@ -39,7 +39,7 @@ <h2 class="py-2">OpenAPI UI</h2>
 
     <h2 class="py-2">Download Publication Data</h2>
     <ul class="list-unstyled mb-4">
-      {% if geojson_size %}
+      {% if last_geojson %}
       <li class="mb-3">
         <div class="d-flex align-items-center">
           <a class="btn btn-primary btn-sm" href="{% url 'optimap:download_geojson' %}">
@@ -48,28 +48,32 @@ <h2 class="py-2">Download Publication Data</h2>
           (<a href="https://geojson.org/" target="_blank" class="ms-2 small">GeoJSON spec</a>)
         </div>
         <div class="small text-muted mt-1">
-          File size: {{ geojson_size }}
+          File: {{ last_geojson }}{% if geojson_size %} &middot; Size: {{ geojson_size }}{% endif %}
         </div>
       </li>
       {% endif %}
 
-      {% if geopackage_size %}
+      {% if last_gpkg %}
       <li>
         <div class="d-flex align-items-center">
           <a class="btn btn-primary btn-sm" href="{% url 'optimap:download_geopackage' %}">
             Download GeoPackage
-         </a>
+          </a>
           (<a href="https://www.geopackage.org/" target="_blank" class="ms-2 small">GeoPackage spec</a>)
         </div>
         <div class="small text-muted mt-1">
-         File size: {{ geopackage_size }}
+          File: {{ last_gpkg }}{% if geopackage_size %} &middot; Size: {{ geopackage_size }}{% endif %}
         </div>
       </li>
       {% endif %}
     </ul>
     <p class="small text-muted text-center mb-0">
       Data dumps run every {{ interval }} hour{{ interval|pluralize }}.<br>
-      Last updated: {{ last_updated|naturaltime }}
+      {% if last_updated %}
+        Last updated: {{ last_updated|naturaltime }}
+      {% else %}
+        No dumps have been generated yet.
+      {% endif %}
     </p>
 
   </div>

publications/views.py

@@ -188,7 +188,6 @@ def loginres(request):
 def privacy(request):
     return render(request, 'privacy.html')
 
-
 @never_cache
 def data(request):
     """
@@ -197,33 +196,46 @@ def data(request):
     cache_dir = Path(tempfile.gettempdir()) / "optimap_cache"
     cache_dir.mkdir(exist_ok=True)
 
-    # locate latest dumps
+    # scan for existing dumps
     geojson_files = sorted(cache_dir.glob('optimap_data_dump_*.geojson'), reverse=True)
-    gpkg_files    = sorted(cache_dir.glob('optimap_data_dump_*.gpkg'), reverse=True)
-
-    last_geo = geojson_files[0] if geojson_files else None
-    last_gpkg = gpkg_files[0]   if gpkg_files else None
+    gpkg_files    = sorted(cache_dir.glob('optimap_data_dump_*.gpkg'),   reverse=True)
+
+    last_geo  = geojson_files[0] if geojson_files else None
+    last_gzip = Path(str(last_geo) + ".gz") if last_geo else None
+    last_gpkg = gpkg_files[0]    if gpkg_files    else None
+
+    # — Supervisor check: ensure all dump file times are within 1 hour
+    mtimes = []
+    for p in (last_geo, last_gzip, last_gpkg):
+        if p and p.exists():
+            mtimes.append(p.stat().st_mtime)
+    if mtimes and (max(mtimes) - min(mtimes) > 3600):
+        ts_map = {
+            p.name: datetime.fromtimestamp(p.stat().st_mtime, get_default_timezone())
+            for p in (last_geo, last_gzip, last_gpkg) if p and p.exists()
+        }
+        logger.warning("Data dump timestamps differ by >1h: %s", ts_map)
 
     # humanized sizes
-    geojson_size = humanize.naturalsize(last_geo.stat().st_size, binary=True) if last_geo else None
-    gpkg_size    = humanize.naturalsize(last_gpkg.stat().st_size, binary=True) if last_gpkg else None
+    geojson_size    = humanize.naturalsize(last_geo.stat().st_size, binary=True) if last_geo else None
+    geopackage_size = humanize.naturalsize(last_gpkg.stat().st_size, binary=True) if last_gpkg else None
 
-    # last updated timestamp
+    # last updated timestamp (using JSON file)
     if last_geo:
         ts = last_geo.stat().st_mtime
         last_updated = datetime.fromtimestamp(ts, get_default_timezone())
     else:
         last_updated = None
 
     return render(request, 'data.html', {
-        'geojson_size': geojson_size,
-        'geopackage_size': gpkg_size,
-        'interval': settings.DATA_DUMP_INTERVAL_HOURS,
-        'last_updated': last_updated,
-        'last_geojson': last_geo.name if last_geo else None,
-        'last_gpkg': last_gpkg.name if last_gpkg else None,
+        'geojson_size':    geojson_size,
+        'geopackage_size': geopackage_size,
+        'interval':        settings.DATA_DUMP_INTERVAL_HOURS,
+        'last_updated':    last_updated,
+        'last_geojson':    last_geo.name  if last_geo else None,
+        'last_gpkg':       last_gpkg.name if last_gpkg else None,
     })
-    
+
 def Confirmationlogin(request):
     return render(request, 'confirmation_login.html')
 

tests/test_geo_data.py

@@ -2,19 +2,21 @@
 import json
 import tempfile
 import gzip
-import json
-tempfile
 from datetime import datetime, timedelta
 from django.test import TestCase
 from django.core.serializers import serialize
 import fiona
-gzip
 from django.urls import reverse
 from django.conf import settings
 import re
 from publications.models import Publication
 from publications.views import generate_geopackage
-from publications.tasks import regenerate_geojson_cache, generate_data_dump_filename, cleanup_old_data_dumps
+from publications.tasks import (
+    regenerate_geojson_cache,
+    regenerate_geopackage_cache,
+    generate_data_dump_filename,
+    cleanup_old_data_dumps,
+)
 from pathlib import Path
 
 class GeoDataAlternativeTestCase(TestCase):
@@ -101,12 +103,9 @@ def test_data_endpoint(self):
         self.assertEqual(response.status_code, 200, "Data endpoint should return status 200")
         content = response.content.decode()
         self.assertIn(f"Data dumps run every {settings.DATA_DUMP_INTERVAL_HOURS} hour", content)
-        # find the most recent JSON dump
         cache_dir = Path(tempfile.gettempdir()) / 'optimap_cache'
         dumps = sorted(cache_dir.glob('optimap_data_dump_*.geojson'), reverse=True)
         self.assertTrue(dumps, "At least one data dump should exist for Last updated check")
-        last = dumps[0]
-        mtime = last.stat().st_mtime
         self.assertIn("Last updated:", content)
 
     def test_download_geojson_gzip(self):
@@ -122,12 +121,12 @@ def test_download_geopackage_endpoint(self):
         url = reverse('optimap:download_geopackage')
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, "GeoPackage endpoint should return 200")
-        self.assertEqual(response['Content-Type'], 'application/geopackage+sqlite3', "Content-Type should be application/geopackage+sqlite3")
+        self.assertEqual(response['Content-Type'], 'application/geopackage+sqlite3',
+                         "Content-Type should be application/geopackage+sqlite3")
         self.assertRegex(response['Content-Disposition'], r'optimap_data_dump_.*\.gpkg')
 
     def test_regenerate_geojson_cache_creates_files(self):
         cache_dir = Path(tempfile.gettempdir()) / 'optimap_cache'
-        # clean out all existing dump files
         for f in cache_dir.glob('optimap_data_dump_*'):
             f.unlink()
 
@@ -162,9 +161,8 @@ def test_download_geopackage(self):
         self.assertEqual(response['Content-Type'], 'application/geopackage+sqlite3')
         self.assertRegex(response['Content-Disposition'], r'optimap_data_dump_.*\.gpkg')
 
-    def test_data_page_hides_links_when_missing_cache(self):
+    def test_data_page_hides_and_shows_links_correctly(self):
         cache_dir = Path(tempfile.gettempdir()) / 'optimap_cache'
-        # clear only our dynamic dumps
         for f in cache_dir.glob('optimap_data_dump_*'):
             f.unlink()
 
@@ -178,3 +176,9 @@ def test_data_page_hides_links_when_missing_cache(self):
         content = response.content.decode()
         self.assertIn('Download GeoJSON', content)
         self.assertNotIn('Download GeoPackage', content)
+
+        regenerate_geopackage_cache()
+        response = self.client.get(reverse('optimap:data'))
+        content = response.content.decode()
+        self.assertIn('Download GeoJSON', content)
+        self.assertIn('Download GeoPackage', content)