Merge remote-tracking branch 'upstream/4.4.x' into 52n-4.4.x

Author: ridoo

geonode/assets/local.py

@@ -7,7 +7,7 @@
 from django.http import HttpResponse, StreamingHttpResponse
 from django.urls import reverse
 from django_downloadview import DownloadResponse
-from zipstream import ZipStream, walk
+from zipstream import ZipStream
 
 from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface
 from geonode.assets.models import LocalAsset
@@ -261,9 +261,7 @@ def create_response(
             match attachment:
                 case True:
                     logger.info(f"Zipping file '{localfile}' with name '{orig_base}'")
-                    zs = ZipStream(sized=True)
-                    for filepath in walk(LocalAssetHandler._get_managed_dir(asset)):
-                        zs.add_path(filepath, os.path.basename(filepath))
+                    zs = ZipStream(sized=True).from_path(LocalAssetHandler._get_managed_dir(asset), arcname="/")
                     # closing zip for all contents to be written
                     return StreamingHttpResponse(
                         zs,

geonode/assets/management/commands/migrate_file_to_assets.py

@@ -98,9 +98,19 @@ def handle(self, **options):
 
                 logger.info("Moving file to the asset folder")
 
-                dest = shutil.move(source, handler._create_asset_dir())
-
-                logger.info("Fixing perms")
+                if len(asset.location) == 1:
+                    # In older installations, all documents are stored in a single folder.
+                    # Instead of moving the entire folder, we can simply move the individual document.
+                    # This approach prevents the risk of breaking the other documents
+                    # that are stored in the same folder
+                    # oldpath = {MEDIA_ROOT}/documents/document/file.extension
+                    dest = shutil.move(asset.location[0], handler._create_asset_dir())
+                else:
+                    dest = shutil.move(source, handler._create_asset_dir())
+
+                logger.info(f"New destination path: {dest}")
+
+                logger.info("Fixing file/folder perms if required")
                 if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None:
                     os.chmod(os.path.dirname(dest), settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS)
 
@@ -113,7 +123,10 @@ def handle(self, **options):
 
             logger.info("Updating location field with new folder value")
 
-            asset.location = [x.replace(source, dest) for x in asset.location]
+            if len(asset.location) == 1:
+                asset.location = [dest]
+            else:
+                asset.location = [x.replace(source, dest) for x in asset.location]
             asset.save()
 
             logger.info("Checking if geoserver should be updated")

geonode/base/api/tests.py

@@ -42,6 +42,7 @@
 from django.contrib.auth import get_user_model
 
 from owslib.etree import etree
+from avatar.templatetags.avatar_tags import avatar_url
 
 from rest_framework.test import APITestCase
 from rest_framework.renderers import JSONRenderer
@@ -1045,13 +1046,13 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -1126,7 +1127,7 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
@@ -1136,13 +1137,13 @@ def test_perms_resources(self):
                         "username": norman.username,
                         "first_name": norman.first_name,
                         "last_name": norman.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "edit",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -1175,13 +1176,13 @@ def test_perms_resources(self):
                     "username": bobby.username,
                     "first_name": bobby.first_name,
                     "last_name": bobby.last_name,
-                    "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                    "avatar": build_absolute_uri(avatar_url(bobby)),
                     "permissions": "owner",
                     "is_staff": False,
                     "is_superuser": False,
                 },
                 {
-                    "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                    "avatar": build_absolute_uri(avatar_url(bobby)),
                     "first_name": "admin",
                     "id": 1,
                     "last_name": "",
@@ -1238,13 +1239,13 @@ def test_perms_resources(self):
                         "username": bobby.username,
                         "first_name": bobby.first_name,
                         "last_name": bobby.last_name,
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": False,
                         "is_superuser": False,
                     },
                     {
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "first_name": "admin",
                         "id": 1,
                         "last_name": "",
@@ -2171,7 +2172,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_superuser": False,
                         "is_staff": False,
@@ -2181,7 +2182,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_superuser": True,
                         "is_staff": True,
@@ -2212,7 +2213,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_staff": False,
                         "is_superuser": False,
@@ -2222,7 +2223,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": True,
                         "is_superuser": True,
@@ -2251,7 +2252,7 @@ def test_manager_can_edit_map(self):
                         "username": "bobby",
                         "first_name": "bobby",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/d41d8cd98f00b204e9800998ecf8427e/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "manage",
                         "is_staff": False,
                         "is_superuser": False,
@@ -2261,7 +2262,7 @@ def test_manager_can_edit_map(self):
                         "username": "admin",
                         "first_name": "admin",
                         "last_name": "",
-                        "avatar": "https://www.gravatar.com/avatar/7a68c67c8d409ff07e42aa5d5ab7b765/?s=240",
+                        "avatar": build_absolute_uri(avatar_url(bobby)),
                         "permissions": "owner",
                         "is_staff": True,
                         "is_superuser": True,

geonode/base/forms.py

@@ -343,6 +343,7 @@ def _get_thesauro_keyword_label(item, lang):
 
     @staticmethod
     def _get_thesauro_title_label(item, lang):
+        lang = remove_country_from_languagecode(lang)
         tname = ThesaurusLabel.objects.values_list("label", flat=True).filter(thesaurus=item).filter(lang=lang)
         if not tname:
             return Thesaurus.objects.get(id=item.id).title

geonode/base/tests.py

@@ -27,6 +27,7 @@
 from unittest.mock import patch, Mock
 from guardian.shortcuts import assign_perm
 
+from django.utils.module_loading import import_string
 from django.db.utils import IntegrityError, OperationalError
 from django.core.exceptions import ObjectDoesNotExist
 from django.conf import settings
@@ -57,6 +58,8 @@
     HierarchicalKeyword,
     ResourceBase,
     MenuPlaceholder,
+    License,
+    Group,
     Menu,
     MenuItem,
     Configuration,
@@ -1244,6 +1247,14 @@ def test_keyword_raise_db_error(self, add_root_mocked):
         )
 
 
+def _cache_less_call_storers(instance, custom={}):
+    storer_module_path = settings.METADATA_STORERS if hasattr(settings, "METADATA_STORERS") else []
+    storers = [import_string(storer_path) for storer_path in storer_module_path]
+    for storer in storers:
+        storer(instance, custom)
+    return instance
+
+
 class TestRegions(GeoNodeBaseTestSupport):
     def setUp(self):
         self.dataset_inside_region = GEOSGeometry(
@@ -1281,16 +1292,38 @@ def test_region_assignment_for_extent(self):
             region.is_assignable_to_geom(self.dataset_outside_region), "Extent outside a region should be assigned"
         )
 
-    @override_settings(METADATA_STORERS=["geonode.resource.regions_storer.spatial_predicate_region_assignor"])
+    @patch("geonode.resource.utils.call_storers", _cache_less_call_storers)
     def test_regions_are_assigned_if_handler_is_used(self):
-        dataset = resource_manager.create(
-            None,
-            resource_type=Dataset,
-            defaults=dict(owner=get_user_model().objects.first(), title="test_region_dataset", is_approved=True),
-        )
-        self.assertTrue(dataset.regions.exists())
-        self.assertEqual(1, dataset.regions.count())
-        self.assertEqual("Global", dataset.regions.first().name)
+        with override_settings(METADATA_STORERS=["geonode.resource.regions_storer.spatial_predicate_region_assignor"]):
+            dataset = resource_manager.create(
+                None,
+                resource_type=Dataset,
+                defaults=dict(owner=get_user_model().objects.first(), title="test_region_dataset", is_approved=True),
+            )
+            self.assertTrue(dataset.regions.exists())
+            self.assertEqual(1, dataset.regions.count())
+            self.assertEqual("Global", dataset.regions.first().name)
+
+
+class TestMetadataStorer(GeoNodeBaseTestSupport):
+
+    @patch("geonode.resource.utils.call_storers", _cache_less_call_storers)
+    def test_create_passing_custom_to_post_save(self):
+
+        with override_settings(METADATA_STORERS=["geonode.resource.metadata_storer.store_metadata"]):
+            User = get_user_model()
+            user = User.objects.create(username="test", email="test@test.com")
+            license = License.objects.all().first()
+            group = Group.objects.all().first()
+            dataset = resource_manager.create(
+                str(uuid4()),
+                resource_type=Dataset,
+                defaults=dict(owner=user, title="test"),
+                custom=dict(group=group.pk, license=license),
+            )
+            self.assertIsNotNone(dataset.license)
+            self.assertIsNotNone(dataset.group)
+            self.assertEqual(group.pk, dataset.group.pk)
 
 
 class LinkedResourcesTest(GeoNodeBaseTestSupport):

geonode/documents/admin.py

@@ -37,9 +37,19 @@ class Meta(ResourceBaseAdminForm.Meta):
 
 class DocumentAdmin(TabbedTranslationAdmin):
     exclude = ("ll_bbox_polygon", "bbox_polygon", "srid")
-    list_display = ("id", "title", "date", "category", "group", "is_approved", "is_published", "metadata_completeness")
+    list_display = (
+        "id",
+        "title",
+        "date",
+        "category",
+        "group",
+        "is_approved",
+        "is_published",
+        "advertised",
+        "metadata_completeness",
+    )
     list_display_links = ("id",)
-    list_editable = ("title", "category", "group", "is_approved", "is_published")
+    list_editable = ("title", "category", "group", "is_approved", "is_published", "advertised")
     list_filter = (
         "date",
         "date_type",

geonode/documents/forms.py

@@ -169,7 +169,7 @@ def clean(self):
             raise forms.ValidationError(_("A document cannot have both a file and a url."))
 
         if extension:
-            cleaned_data["extension"] = extension.replace(".", "")
+            cleaned_data["extension"] = extension.replace(".", "").lower()
 
         return cleaned_data
 

geonode/documents/tests.py

@@ -32,6 +32,7 @@
 
 from unittest.mock import patch
 from urllib.parse import urlparse
+from pathlib import Path
 
 from django.urls import reverse
 from django.conf import settings
@@ -42,7 +43,7 @@
 
 from guardian.shortcuts import get_anonymous_user
 
-from geonode.assets.utils import create_asset_and_link
+from geonode.assets.utils import create_asset_and_link, get_default_asset
 from geonode.base.forms import LinkedResourceForm
 from geonode.maps.models import Map
 from geonode.layers.models import Dataset
@@ -193,6 +194,25 @@ def test_create_document_url_view(self):
         d = Document.objects.get(title="GeoNode Map")
         self.assertEqual(d.doc_url, "http://www.geonode.org/map.pdf")
 
+    def test_uploaded_csv_with_uppercase_extension(self):
+        """
+        The extension of the file should always be lowercase
+        """
+
+        self.client.login(username="admin", password="admin")
+        try:
+            with open(os.path.join(os.path.dirname(__file__), "tests/data/test.CSV"), "rb") as f:
+                data = {"title": "CSV with uppercase extension", "doc_file": f, "extension": "CSV"}
+                self.client.post(reverse("document_upload"), data=data)
+            d = Document.objects.get(title="CSV with uppercase extension")
+            # verify that the extension is not lowercase
+            self.assertEqual(d.extension, "csv")
+            # be sure that also the file extension is not lowercase
+            asset = get_default_asset(d)
+            self.assertEqual(Path(asset.location[0]).suffix, ".csv")
+        finally:
+            Document.objects.filter(title="CSV with uppercase extension").delete()
+
     def test_upload_document_form(self):
         """
         Tests the Upload form.

geonode/documents/tests/data/test.CSV

@@ -0,0 +1,6 @@
+Username; Identifier;First name;Last name
+booker12;9012;Rachel;Booker
+grey07;2070;Laura;Grey
+johnson81;4081;Craig;Johnson
+jenkins46;9346;Mary;Jenkins
+smith79;5079;Jamie;Smith

geonode/documents/views.py

@@ -53,6 +53,8 @@
 from geonode.base.models import Thesaurus, TopicCategory
 from geonode.base import enumerations
 
+from pathlib import Path
+
 from .utils import get_download_response
 
 from .models import Document
@@ -162,7 +164,8 @@ def form_valid(self, form):
         if file:
             tempdir = mkdtemp()
             dirname = os.path.basename(tempdir)
-            filepath = storage_manager.save(f"{dirname}/{file.name}", file)
+            name = Path(file.name)
+            filepath = storage_manager.save(f"{dirname}/{name.stem}{name.suffix.lower()}", file)
             storage_path = storage_manager.path(filepath)
             self.object = resource_manager.create(
                 None,