Expand support for date formats in image filenames (#809)

* feat: support more date formats in filenames

* feat: update and centralize validation language

Author: mihow

ami/main/api/serializers.py

@@ -1,5 +1,6 @@
 import datetime
 
+from django.core.exceptions import ValidationError as DjangoValidationError
 from django.db.models import QuerySet
 from guardian.shortcuts import get_perms
 from rest_framework import serializers
@@ -13,7 +14,6 @@
 from ami.ml.serializers import AlgorithmSerializer
 from ami.users.models import User
 from ami.users.roles import ProjectManager
-from ami.utils.dates import get_image_timestamp_from_filename
 
 from ..models import (
     Classification,
@@ -32,6 +32,7 @@
     SourceImageUpload,
     TaxaList,
     Taxon,
+    validate_filename_timestamp,
 )
 
 
@@ -1024,13 +1025,10 @@ def create(self, validated_data):
 
     def validate_image(self, value):
         # Ensure that image filename contains a timestamp
-        timestamp = get_image_timestamp_from_filename(value.name)
-        if timestamp is None:
-            # @TODO bring back EXIF support
-            raise serializers.ValidationError(
-                "Image filename does not contain a timestamp in the format YYYYMMDDHHMMSS "
-                " (e.g. 20210101120000-snapshot.jpg). EXIF support coming soon."
-            )
+        try:
+            validate_filename_timestamp(value.name)
+        except DjangoValidationError as e:
+            raise serializers.ValidationError(str(e))
         return value
 
 
@@ -1100,8 +1098,6 @@ class Meta:
         ]
 
     def get_permissions(self, instance, instance_data):
-        request = self.context.get("request")
-
         request: Request = self.context["request"]
         user = request.user
         project = instance.get_project()

ami/main/models.py

@@ -1242,7 +1242,7 @@ def validate_filename_timestamp(filename: str) -> None:
     # Ensure filename has a timestamp
     timestamp = ami.utils.dates.get_image_timestamp_from_filename(filename)
     if not timestamp:
-        raise ValidationError("Filename must contain a timestamp in the format YYYYMMDDHHMMSS")
+        raise ValidationError("Image filename does not contain a valid timestamp (e.g. YYYYMMDDHHMMSS-snapshot.jpg).")
 
 
 def create_source_image_from_upload(image: ImageFieldFile, deployment: Deployment, request=None) -> "SourceImage":

ami/utils/dates.py

@@ -11,9 +11,13 @@
 def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.datetime | None:
     """
     Parse the date and time a photo was taken from its filename.
+    All times are assumed to be in the local timezone. Timezone information is ignored.
+    The maxium precision is seconds (milliseconds are ignored).
 
-    The timestamp must be in the format `YYYYMMDDHHMMSS` but can be
-    preceded or followed by other characters (e.g. `84-20220916202959-snapshot.jpg`).
+    Supports various formats with flexible delimiters. Supports text prefixes and suffixes.
+    - Consecutive digits: YYYYMMDDHHMMSS
+    - Date and time as separate groups: YYYYMMDD and HHMMSS with any delimiter between
+    - Delimited formats within date or time groups
 
     >>> out_fmt = "%Y-%m-%d %H:%M:%S"
     >>> # Aarhus date format
@@ -35,19 +39,34 @@ def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.d
     """
     name = pathlib.Path(img_path).stem
     date = None
+    strptime_format = "%Y%m%d%H%M%S"
+
+    # Put more specific/longer patterns first if overlap is possible.
+    # These could be combined into one pattern, but it would be less readable.
+    consecutive_pattern = r"\d{14}"  # YYYYMMDDHHMMSS
+    two_groups_pattern = r"\d{8}[^\d]+\d{6}"  # YYYYMMDD*HHMMSS
+    # Allow single non-digit delimiters within components, and one or more between DD and HH
+    delimited_pattern = r"\d{4}[^\d]\d{2}[^\d]\d{2}[^\d]+\d{2}[^\d]\d{2}[^\d]\d{2}"  # YYYY*MM*DD*+HH*MM*SS
+
+    # Combine patterns with OR '|' but keep them in their own groups
+    pattern = re.compile(f"({consecutive_pattern})|({two_groups_pattern})|({delimited_pattern})")
+
+    match = pattern.search(name)
+    if match:
+        # Get the full string matched by any of the patterns
+        matched_string = match.group(0)
+        # Remove all non-digit characters to create YYYYMMDDHHMMSS
+        consecutive_date_string = re.sub(r"[^\d]", "", matched_string)
 
-    # Extract date from a filename using regex in the format %Y%m%d%H%M%S
-    matches = re.search(r"(\d{14})", name)
-    if matches:
         try:
-            date = datetime.datetime.strptime(matches.group(), "%Y%m%d%H%M%S")
+            date = datetime.datetime.strptime(consecutive_date_string, strptime_format)
         except ValueError:
             pass
 
     if not date:
         try:
             date = dateutil.parser.parse(name, fuzzy=False)  # Fuzzy will interpret "DSC_1974" as 1974-01-01
-        except dateutil.parser.ParserError:
+        except (dateutil.parser.ParserError, ValueError, OverflowError):
             pass
 
     if not date and raise_error:

ami/utils/tests.py

@@ -0,0 +1,34 @@
+import datetime
+from unittest import TestCase
+
+
+class TestUtils(TestCase):
+    def test_extract_timestamps(self):
+        from ami.utils.dates import get_image_timestamp_from_filename
+
+        filenames_and_expected_dates = [
+            ("aarhus/20220810231507-00-07.jpg", "2022-08-10 23:15:07"),
+            ("diopsis/20230124191342.jpg", "2023-01-24 19:13:42"),
+            ("vermont_snapshots/20220622000459-108-snapshot.jpg", "2022-06-22 00:04:59"),
+            ("cyprus_snapshots/84-20220916202959-snapshot.jpg", "2022-09-16 20:29:59"),
+            ("wingscape/Project_20230801023001_4393.JPG", "2023-08-01 02:30:01"),
+            ("nikon/DSC_1974.JPG", None),  # Example with no date
+            ("not_a_date/happybirthday.jpg", None),  # Example with no date
+            ("cannon/IMG_20230801_123456.jpg", "2023-08-01 12:34:56"),
+            ("mothbox/2024_01_01 12_00_00.jpg", "2024-01-01 12:00:00"),
+            ("other_common/2024-01-01 12:00:00.jpg", "2024-01-01 12:00:00"),
+            ("other_common/2024-01-01T12:00:00.jpg", "2024-01-01 12:00:00"),
+        ]
+
+        for filename, expected_date in filenames_and_expected_dates:
+            with self.subTest(filename=filename):
+                # Convert the expected date string to a datetime object
+                if expected_date is not None:
+                    expected_date = datetime.datetime.strptime(expected_date, "%Y-%m-%d %H:%M:%S")
+                # Call the function and compare the result with the expected date
+                # Only use raise_error=True when we expect a date to be present
+                raise_error = expected_date is not None
+                result = get_image_timestamp_from_filename(filename, raise_error=raise_error)
+                self.assertEqual(
+                    result, expected_date, f"Failed for {filename}: expected {expected_date}, got {result}"
+                )

ui/src/utils/language.ts

@@ -418,7 +418,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = {
 
   /* MESSAGE */
   [STRING.MESSAGE_CAPTURE_FILENAME]:
-    'Image filename must contain a timestamp in the format YYYYMMDDHHMMSS (e.g. 20210101120000-snapshot.jpg).',
+    'Image filename must contain a timestamp with year, month, day, hours, minutes and seconds (e.g. 20210101120000-snapshot.jpg).',
   [STRING.MESSAGE_CAPTURE_LIMIT]:
     'A maximum of {{numCaptures}} images can be uploaded through the web browser. Configure a data source to upload data in bulk.',
   [STRING.MESSAGE_CAPTURE_SYNC_HIDDEN]: