Merge pull request #488 from Terralego/source-update

Improve source list and detail, adding info panel

docs/source/others/changelog.rst

@@ -3,8 +3,14 @@ Changelog
 ==========
 
 
+
 2024.02.3+dev  (XXXX-XX-XX)
 ---------------------------
+**Improvements:**
+
+- Add "In progress" status to source
+- Source list endpoint now also return ids of related layer
+- Add author to Source
 
 **Documentation:**
 

project/geosource/migrations/0012_alter_source_status.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.13 on 2023-11-20 15:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("geosource", "0011_sourcereporting_source_status_alter_source_report"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="source",
+            name="status",
+            field=models.PositiveSmallIntegerField(
+                choices=[
+                    (0, "Need sync"),
+                    (1, "Pending"),
+                    (2, "Done"),
+                    (3, "In progress"),
+                ],
+                default=0,
+            ),
+        ),
+    ]

project/geosource/migrations/0012_source_author.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.13 on 2023-12-05 08:26
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("geosource", "0011_sourcereporting_source_status_alter_source_report"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="source",
+            name="author",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="sources",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

project/geosource/migrations/0013_merge_20240205_1233.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.9 on 2024-02-05 12:33
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "geosource",
+            "0012_alter_commandsource_options_alter_csvsource_options_and_more",
+        ),
+        ("geosource", "0012_alter_source_status"),
+        ("geosource", "0012_source_author"),
+    ]
+
+    operations = []

project/geosource/models.py

@@ -10,6 +10,7 @@
 from celery.result import AsyncResult
 from celery.utils.log import LoggingProxy
 from django.conf import settings
+from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group
 from django.contrib.gis.gdal.error import GDALException
 from django.contrib.gis.geos import GEOSGeometry
@@ -33,6 +34,8 @@
 from .mixins import CeleryCallMethodsMixin
 from .signals import refresh_data_done
 
+User = get_user_model()
+
 # Decimal fields must be returned as float
 DEC2FLOAT = psycopg2.extensions.new_type(
     psycopg2.extensions.DECIMAL.values,
@@ -88,7 +91,7 @@ class Status(models.IntegerChoices):
     def reset(self):
         self.status = self.Status.PENDING.value
         self.message = ""
-        self.started = timezone.now()
+        self.started = None
         self.ended = None
         self.added_lines = 0
         self.deleted_lines = 0
@@ -102,6 +105,7 @@ class Status(models.IntegerChoices):
         NEED_SYNC = 0, _("Need sync")
         PENDING = 1, _("Pending")
         DONE = 2, _("Done")
+        IN_PROGRESS = 3, _("In progress")
 
     name = models.CharField(max_length=255, unique=True)
     credit = models.TextField(blank=True)
@@ -125,6 +129,9 @@ class Status(models.IntegerChoices):
     status = models.PositiveSmallIntegerField(
         choices=Status.choices, default=Status.NEED_SYNC
     )
+    author = models.ForeignKey(
+        User, related_name="sources", blank=True, null=True, on_delete=models.SET_NULL
+    )
 
     SOURCE_GEOM_ATTRIBUTE = "_geom_"
     MAX_SAMPLE_DATA = 5
@@ -156,9 +163,8 @@ def should_refresh(self):
         return next_run < now
 
     def refresh_data(self):
-        if self.status != self.Status.PENDING.value:
-            self.status = self.Status.PENDING.value
-            self.save()
+        self.status = self.Status.IN_PROGRESS
+        self.save()
 
         layer = self.get_layer()
         try:
@@ -184,11 +190,12 @@ def refresh_data(self):
     def _refresh_data(self, es_index=None):
         if not self.report:
             self.report = SourceReporting.objects.create(
-                started=timezone.now(), status=SourceReporting.Status.PENDING.value
+                started=timezone.now(), status=SourceReporting.Status.PENDING
             )
         else:
             self.report.reset()
-            self.report.status = SourceReporting.Status.PENDING.value
+            self.report.status = SourceReporting.Status.PENDING
+            self.report.started = timezone.now()
             self.report.save()
 
         layer = self.get_layer()

project/geosource/serializers.py

@@ -103,16 +103,13 @@ class Meta:
 
 
 class SourceReportingSerializer(serializers.ModelSerializer):
-    status = serializers.CharField(source="get_status_display")
-
     class Meta:
         fields = "__all__"
         model = SourceReporting
 
 
 class SourceSerializer(PolymorphicModelSerializer):
     fields = FieldSerializer(many=True, required=False)
-    status = serializers.SerializerMethodField()
     slug = serializers.SlugField(max_length=255, read_only=True)
     report = SourceReportingSerializer(read_only=True)
 
@@ -137,7 +134,14 @@ def create(self, validated_data):
     def update(self, instance, validated_data):
         validated_data.pop("fields")
 
-        source = super().update(instance, validated_data)
+        source = super().update(
+            instance, {**validated_data, "status": Source.Status.NEED_SYNC}
+        )
+
+        if source.report:
+            source.report.reset()
+            source.report.status = SourceReporting.Status.PENDING
+            source.report.save()
 
         self._update_fields(source)
 
@@ -154,9 +158,6 @@ def update(self, instance, validated_data):
 
         return source
 
-    def get_status(self, instance):
-        return instance.get_status_display()
-
 
 class SourceListSerializer(serializers.ModelSerializer):
     _type = serializers.SerializerMethodField()
@@ -172,6 +173,7 @@ class Meta:
             "name",
             "geom_type",
             "report",
+            "layers",
         )
 
     def get__type(self, instance):

project/geosource/tasks.py

@@ -24,9 +24,13 @@ def set_failure_state(task, method, message, instance=None):
         text = ""
         for key, value in state["meta"].items():
             text += f"{key}: {value},"
+        if not instance.report.status:
+            instance.report.status = instance.report.Status.ERROR
         instance.report.message = text
+        if not instance.report.started:
+            instance.report.started = timezone.now()
         instance.report.ended = timezone.now()
-        instance.report.save(update_fields=["ended", "message"])
+        instance.report.save(update_fields=["status", "started", "ended", "message"])
 
 
 @shared_task(bind=True)
@@ -43,14 +47,6 @@ def run_model_object_method(self, app, model, pk, method, success_state=states.S
         logger.info(f"Method {method} on {obj} ended")
 
         self.update_state(state=success_state, meta=state)
-        if obj and hasattr(obj, "report"):
-            if obj.report is not None:
-                text = ""
-                for key, value in state.items():
-                    text += f"{key}: {value},"
-                obj.report.message = text
-                obj.report.ended = timezone.now()
-                obj.report.save(update_fields=["message", "ended"])
 
     except Model.DoesNotExist:
         set_failure_state(

project/geosource/tests/test_serializers.py

@@ -4,8 +4,12 @@
 from django.test import TestCase
 from rest_framework.exceptions import ValidationError
 
-from project.geosource.models import CSVSource
-from project.geosource.serializers import CSVSourceSerializer, GeoJSONSourceSerializer
+from project.geosource.models import CSVSource, GeoJSONSource, Source, SourceReporting
+from project.geosource.serializers import (
+    CSVSourceSerializer,
+    GeoJSONSourceSerializer,
+    SourceSerializer,
+)
 from project.geosource.tests.helpers import get_file
 
 
@@ -188,3 +192,37 @@ def test_validate_fields_infos_exceptions(self, get_record_mock):
         serializer = GeoJSONSourceSerializer(data=data)
         with self.assertRaises(ValidationError):
             serializer._validate_field_infos(serializer.initial_data)
+
+
+class SourceSerializerTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls):
+        cls.report = SourceReporting.objects.create()
+        # Use GeoJSONSource for easier testing purpose
+        cls.source = GeoJSONSource.objects.create(
+            name="test-source",
+            file=get_file("test.geojson"),
+            geom_type=0,
+            id_field="id",
+            report=cls.report,
+        )
+
+    def test_update_method_reset_status(self):
+        data = {
+            "id": self.source.id,
+            "name": "new-name",
+            "_type": "GeoJSONSource",
+            "file": [get_file("test.geojson")],
+            "fields": [],
+        }
+        serializer = SourceSerializer(data=data, partial=True)
+        self.assertTrue(
+            serializer.is_valid(),
+            serializer.errors,
+        )
+        serializer.update(self.source, serializer.validated_data)
+
+        self.source.refresh_from_db()
+        self.report.refresh_from_db()
+        self.assertEqual(self.source.status, Source.Status.NEED_SYNC.value)
+        self.assertEqual(self.report.status, SourceReporting.Status.PENDING.value)

project/geosource/tests/test_tasks.py

@@ -140,3 +140,33 @@ def test_set_failure_state_task_update_report(
             source.report.refresh_from_db()
             self.assertIsInstance(source.report.message, str)
             self.assertIsInstance(source.report.ended, datetime)
+            self.assertEqual(source.report.status, SourceReporting.Status.ERROR.value)
+
+    def test_set_failure_state_task_when_report_status_none(
+        self,
+        mock_index_feature,
+        mock_es_delete,
+        mock_es_create,
+    ):
+        logging.disable(logging.ERROR)
+        source_report = SourceReporting.objects.create(status=None)
+        source = GeoJSONSource.objects.create(
+            name="exception-test",
+            geom_type=GeometryTypes.Point,
+            file=get_file("test.geojson"),
+            report=source_report,
+        )
+
+        self.assertIsNone(source.report.ended)
+        try:
+            run_model_object_method.apply(
+                (
+                    source._meta.app_label,
+                    source._meta.model_name,
+                    source.pk,
+                    "method_that_does_not_exist",
+                )
+            )
+        except AttributeError:
+            source.report.refresh_from_db()
+            self.assertEqual(source.report.status, SourceReporting.Status.ERROR.value)

project/geosource/views.py

@@ -32,6 +32,9 @@ def get_serializer_class(self):
     def get_queryset(self):
         return Source.objects.all().order_by("-id")
 
+    def perform_create(self, serializers):
+        serializers.save(author=self.request.user)
+
     @action(detail=True, methods=["get"])
     def refresh(self, request, pk):
         """Schedule a refresh now"""

project/locales/en/LC_MESSAGES/django.po

@@ -136,6 +136,9 @@ msgstr ""
 msgid "Done"
 msgstr ""
 
+msgid "In progress"
+msgstr ""
+
 msgid "Failed to refresh data"
 msgstr ""
 

project/locales/fr/LC_MESSAGES/django.po

@@ -138,6 +138,9 @@ msgstr "À synchroniser"
 msgid "Done"
 msgstr "Terminé"
 
+msgid "In progress"
+msgstr "En cours"
+
 msgid "Failed to refresh data"
 msgstr "L'actualisation des données a échoué"