feat: add robots-meta-content

Author: stefina

ckanext/switzerland/helpers.py

@@ -714,3 +714,93 @@ def uri_to_iri(uri):
         return iri
     except Exception as e:
         raise ValueError(f"Provided URI can't be converted to IRI: {e}")
+
+
+# Indexable curated list / detail pages (dataset, showcase, group).
+# Filtered URLs (any query string) and organizations / resources / harvest are noindex.
+_ROBOTS_INDEXABLE_LIST_ENDPOINTS = frozenset(
+    {
+        "dataset.search",
+        "ogdch_home.search",
+        "group.index",
+        "showcase_blueprint.index",
+    }
+)
+_ROBOTS_INDEXABLE_DETAIL_ENDPOINTS = frozenset(
+    {
+        "dataset.read",
+        "group.read",
+        "showcase_blueprint.read",
+    }
+)
+_ROBOTS_RESOURCE_ENDPOINTS = frozenset(
+    {
+        "dataset_resource.read",
+        "resource.read",
+    }
+)
+_ROBOTS_NOINDEX_ENDPOINT_PREFIXES = ("organization.", "harvest.")
+_ROBOTS_NOINDEX = "noindex, follow"
+_ROBOTS_INDEX = "index, follow"
+
+
+def get_robots_meta_content():
+    """
+    Return the robots meta content for the current request.
+
+    - Dataset / showcase / group list and detail (no query): index, follow
+    - Same pages with any query parameters (filters, search, pagination): noindex, follow
+    - Resource detail: noindex, follow
+    - Organization pages: noindex, follow
+    - Harvest pages: noindex, follow
+    - Everything else: index, follow
+    """
+    try:
+        endpoint = tk.request.endpoint
+        args = tk.request.args
+        path = tk.request.path or ""
+    except RuntimeError:
+        # Outside a request context (e.g. some CLI / test helpers)
+        return _ROBOTS_INDEX
+
+    if endpoint in _ROBOTS_RESOURCE_ENDPOINTS or _is_resource_path(path):
+        return _ROBOTS_NOINDEX
+
+    if _is_noindex_admin_path(path) or (
+        endpoint
+        and endpoint.startswith(_ROBOTS_NOINDEX_ENDPOINT_PREFIXES)
+    ):
+        return _ROBOTS_NOINDEX
+
+    has_query = bool(args)
+
+    if endpoint in _ROBOTS_INDEXABLE_LIST_ENDPOINTS:
+        return _ROBOTS_NOINDEX if has_query else _ROBOTS_INDEX
+
+    if endpoint in _ROBOTS_INDEXABLE_DETAIL_ENDPOINTS:
+        # Group detail can still have dataset filter query params
+        return _ROBOTS_NOINDEX if has_query else _ROBOTS_INDEX
+
+    return _ROBOTS_INDEX
+
+
+def _is_noindex_admin_path(path):
+    """True for organization and harvest UI paths."""
+    if not path:
+        return False
+    cleaned = path if path.startswith("/") else f"/{path}"
+    return cleaned.startswith("/organization") or cleaned.startswith("/harvest")
+
+
+def _is_resource_path(path):
+    """True for /dataset/<id>/resource/<resource_id> HTML pages."""
+    if not path:
+        return False
+    parts = [p for p in path.strip("/").split("/") if p]
+    # dataset / <id> / resource / <resource_id> [/optional subpath]
+    return (
+        len(parts) >= 4
+        and parts[0] == "dataset"
+        and parts[2] == "resource"
+        and parts[3] != "new"
+    )

ckanext/switzerland/plugin.py

@@ -98,6 +98,7 @@ def get_helpers(self):
             "ogdch_publisher_form_helper": sh.ogdch_publisher_form_helper,
             "ogdch_get_media_type_choices": sh.ogdch_get_media_type_choices,
             "ogdch_get_default_terms_of_use": sh.ogdch_get_default_terms_of_use,
+            "get_robots_meta_content": sh.get_robots_meta_content,
         }
 
     def i18n_directory(self):

ckanext/switzerland/templates/base.html

@@ -1,5 +1,10 @@
 {% ckan_extends %}
 
+{% block meta %}
+  {{ super() }}
+  <meta name="robots" content="{{ h.get_robots_meta_content() }}">
+{% endblock %}
+
 {% block links %}
   <!-- Data layer initialization -->
   <script>

ckanext/switzerland/tests/test_helpers.py

@@ -1,6 +1,6 @@
 import pytest
 
-from ckanext.switzerland.helpers import map_to_valid_format
+from ckanext.switzerland.helpers import get_robots_meta_content, map_to_valid_format
 
 CSV_URI = "http://publications.europa.eu/resource/authority/file-type/CSV"
 GEOJSON_URI = "http://publications.europa.eu/resource/authority/file-type/GEOJSON"
@@ -61,3 +61,95 @@
 )
 def test_map_to_valid_format_known_values(input_format, expected):
     assert map_to_valid_format(input_format) == expected
+
+
+INDEX = "index, follow"
+NOINDEX = "noindex, follow"
+
+
+class _FakeArgs(dict):
+    """Minimal stand-in for Flask request.args (truthy when non-empty)."""
+
+    def __bool__(self):
+        return bool(dict(self))
+
+
+class _FakeRequest:
+    def __init__(self, endpoint, path="/", args=None):
+        self.endpoint = endpoint
+        self.path = path
+        self.args = _FakeArgs(args or {})
+
+
+@pytest.mark.parametrize(
+    "endpoint,path,args,expected",
+    [
+        ("dataset.search", "/dataset/", None, INDEX),
+        ("dataset.search", "/dataset/", {"q": "fahrplan"}, NOINDEX),
+        ("dataset.search", "/dataset/", {"groups": "timetables"}, NOINDEX),
+        ("dataset.search", "/dataset/", {"page": "2"}, NOINDEX),
+        ("ogdch_home.search", "/", None, INDEX),
+        ("ogdch_home.search", "/", {"organization": "oevch"}, NOINDEX),
+        ("dataset.read", "/dataset/bike-and-car-parking", None, INDEX),
+        ("showcase_blueprint.index", "/showcase/", None, INDEX),
+        (
+            "showcase_blueprint.index",
+            "/showcase/",
+            {"q": "viz"},
+            NOINDEX,
+        ),
+        (
+            "showcase_blueprint.read",
+            "/showcase/visualisierung-der-oev-tagesentwicklungen",
+            None,
+            INDEX,
+        ),
+        ("group.index", "/group/", None, INDEX),
+        ("group.read", "/group/accessibilitydata", None, INDEX),
+        (
+            "group.read",
+            "/group/accessibilitydata",
+            {"q": "test"},
+            NOINDEX,
+        ),
+        ("organization.index", "/organization/", None, NOINDEX),
+        ("organization.read", "/organization/oevch", None, NOINDEX),
+        (
+            "organization.read",
+            "/organization/oevch",
+            {"q": "x"},
+            NOINDEX,
+        ),
+        ("harvest.search", "/harvest/", None, NOINDEX),
+        ("harvest.read", "/harvest/some-source", None, NOINDEX),
+        # Path fallback when endpoint is missing/unknown
+        (None, "/harvest/", None, NOINDEX),
+        (None, "/organization/oevch", None, NOINDEX),
+        (
+            "dataset_resource.read",
+            "/dataset/bike-and-car-parking/resource/eb892409-d24f-4484-a19b-2d6e26108d9a",
+            None,
+            NOINDEX,
+        ),
+        (
+            "resource.read",
+            "/dataset/bike-and-car-parking/resource/eb892409-d24f-4484-a19b-2d6e26108d9a",
+            None,
+            NOINDEX,
+        ),
+        # Path fallback when endpoint is missing/unknown but URL is a resource
+        (
+            None,
+            "/dataset/bike-and-car-parking/resource/eb892409-d24f-4484-a19b-2d6e26108d9a",
+            None,
+            NOINDEX,
+        ),
+        ("user.read", "/user/admin", None, INDEX),
+    ],
+)
+def test_get_robots_meta_content(monkeypatch, endpoint, path, args, expected):
+    monkeypatch.setattr(
+        "ckanext.switzerland.helpers.tk.request",
+        _FakeRequest(endpoint, path, args),
+    )
+    assert get_robots_meta_content() == expected