switch titiler.xarray to obstore+zarr and add default application (#1253)

* switch titiler.xarray to obstore+zarr and add default application

* remove python 3.10 and recreate uv.lock

* better handle aws s3 http urls

* update from comment

* lower bounds for s3fs

* assume https s3 object are public

* update dependencies and remove consolidated

Author: vincentsarago

CONTRIBUTING.md

@@ -39,6 +39,9 @@ uv run pytest src/titiler/extensions --cov=titiler.extensions --cov-report=xml -
 # titiler.mosaic
 uv run pytest src/titiler/mosaic --cov=titiler.mosaic --cov-report=xml --cov-append --cov-report=term-missing
 
+# titiler.xarray
+uv run pytest src/titiler/xarray --cov=titiler.xarray --cov-report=xml --cov-append --cov-report=term-missing
+
 # titiler.application
 uv run pytest src/titiler/application --cov=titiler.application --cov-report=xml --cov-append --cov-report=term-missing
 ```

pyproject.toml

@@ -42,10 +42,11 @@ dev = [
     "pytest-cov",
     "pytest-asyncio",
     "httpx",
-    "zarr!=3.0.9",
+    "obstore",
+    "zarr>=3,<4.0",
     "h5netcdf",
     "fsspec",
-    "s3fs",
+    "s3fs>=2025.2.0",
     "aiohttp",
     "requests",
     "cogeo-mosaic>=8.2,<9.0",

src/titiler/xarray/examples/templates/landing.html

@@ -36,38 +36,26 @@ dependencies = [
     "rio-tiler>=7.6.1,<8.0",
     "xarray",
     "rioxarray",
+    "obstore",
+    "zarr>=3.1,<4.0",
 ]
 
 [project.optional-dependencies]
-full = [
-    "zarr!=3.0.9",
+fs = [
     "h5netcdf",
     "fsspec",
-    "s3fs",
+    "s3fs>=2025.2.0",
     "aiohttp",
     "gcsfs",
+    "requests",
 ]
-minimal = [
-    "zarr!=3.0.9",
-    "h5netcdf",
-    "fsspec",
-]
-gcs = [
-    "gcsfs",
-]
-s3 = [
-    "s3fs",
-]
-http = [
-    "aiohttp",
-]
+
 [dependency-groups]
 test = [
     "pytest",
     "pytest-cov",
     "pytest-asyncio",
     "httpx",
-    "zarr!=3.0.9",
     "h5netcdf",
     "fsspec",
     "s3fs",

src/titiler/xarray/pyproject.toml

@@ -9,6 +9,7 @@
 
 from titiler.xarray.extensions import DatasetMetadataExtension, VariablesExtension
 from titiler.xarray.factory import TilerFactory
+from titiler.xarray.io import FsReader, fs_open_dataset
 
 prefix = os.path.join(os.path.dirname(__file__), "fixtures")
 
@@ -61,7 +62,30 @@ def test_tiler_factory():
 @pytest.fixture
 def app():
     """App fixture."""
-    md = TilerFactory(router_prefix="/md", extensions=[DatasetMetadataExtension()])
+    md = TilerFactory(
+        router_prefix="/md",
+        extensions=[
+            DatasetMetadataExtension(dataset_opener=fs_open_dataset),
+        ],
+        reader=FsReader,
+    )
+    assert len(md.router.routes) == 22
+
+    app = FastAPI()
+    app.include_router(md.router, prefix="/md")
+    with TestClient(app) as client:
+        yield client
+
+
+@pytest.fixture
+def app_zarr():
+    """App fixture."""
+    md = TilerFactory(
+        router_prefix="/md",
+        extensions=[
+            DatasetMetadataExtension(),
+        ],
+    )
     assert len(md.router.routes) == 22
 
     app = FastAPI()
@@ -393,7 +417,7 @@ def test_zarr_group(group, app):
 def test_preview(filename):
     """App fixture."""
     with pytest.warns(UserWarning):
-        md = TilerFactory(add_preview=True)
+        md = TilerFactory(add_preview=True, reader=FsReader)
 
     app = FastAPI()
     app.include_router(md.router)
@@ -437,3 +461,125 @@ def test_preview(filename):
             with mem.open() as dst:
                 assert dst.width == 1024
                 assert dst.height == 1024
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [dataset_3d_zarr],
+)
+def test_app_zarr(filename, app_zarr):
+    """Test endpoints with Zarr Reader."""
+    resp = app_zarr.get("/md/dataset/keys", params={"url": filename})
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+    assert resp.json() == ["dataset"]
+
+    resp = app_zarr.get("/md/dataset/dict", params={"url": filename})
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+    assert resp.json()["data_vars"]["dataset"]
+
+    resp = app_zarr.get("/md/dataset/", params={"url": filename})
+    assert resp.status_code == 200
+    assert "text/html" in resp.headers["content-type"]
+
+    resp = app_zarr.get("/md/bounds", params={"url": filename, "variable": "dataset"})
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+
+    resp = app_zarr.get("/md/info", params={"url": filename, "variable": "dataset"})
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+
+    resp = app_zarr.get(
+        "/md/tiles/WebMercatorQuad/0/0/0",
+        params={"url": filename, "variable": "dataset", "rescale": "0,500", "bidx": 1},
+    )
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "image/png"
+
+    resp = app_zarr.get(
+        "/md/WebMercatorQuad/tilejson.json",
+        params={"url": filename, "variable": "dataset", "rescale": "0,500", "bidx": 1},
+    )
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+
+    resp = app_zarr.get(
+        "/md/point/0,0", params={"url": filename, "variable": "dataset"}
+    )
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/json"
+
+    feat = {
+        "type": "Feature",
+        "properties": {},
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    (-100.0, -25.0),
+                    (40.0, -25.0),
+                    (40.0, 60.0),
+                    (-100.0, 60.0),
+                    (-100.0, -25.0),
+                ]
+            ],
+        },
+    }
+
+    resp = app_zarr.post(
+        "/md/statistics", params={"url": filename, "variable": "dataset"}, json=feat
+    )
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "application/geo+json"
+
+    feat = {
+        "type": "Feature",
+        "properties": {},
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    (-100.0, -25.0),
+                    (40.0, -25.0),
+                    (40.0, 60.0),
+                    (-100.0, 60.0),
+                    (-100.0, -25.0),
+                ]
+            ],
+        },
+    }
+
+    resp = app_zarr.post(
+        "/md/feature",
+        params={"url": filename, "variable": "dataset", "rescale": "0,500", "bidx": 1},
+        json=feat,
+    )
+    assert resp.status_code == 200
+    assert resp.headers["content-type"] == "image/jpeg"
+
+
+@pytest.mark.parametrize(
+    "group",
+    [0, 1, 2],
+)
+def test_group_open_zarr(group, app_zarr):
+    """Test /tiles endpoints."""
+    resp = app_zarr.get(
+        f"/md/tiles/WebMercatorQuad/{group}/0/0.tif",
+        params={"url": zarr_pyramid, "variable": "dataset", "group": str(group)},
+    )
+    assert resp.status_code == 200
+    # see src/titiler/xarray/tests/fixtures/generate_fixtures.ipynb
+    # for structure of zarr pyramid
+    with MemoryFile(resp.content) as mem:
+        with mem.open() as dst:
+            arr = dst.read(1)
+            assert arr.max() == group * 2 + 1
+
+    resp = app_zarr.get(
+        "/md/point/0,0",
+        params={"url": zarr_pyramid, "variable": "dataset", "group": str(group)},
+    )
+    assert resp.json()["values"] == [group * 2 + 1]

src/titiler/xarray/tests/test_factory.py

@@ -9,7 +9,7 @@
 import pytest
 import xarray
 
-from titiler.xarray.io import Reader, get_variable, xarray_open_dataset
+from titiler.xarray.io import Reader, fs_open_dataset, get_variable, open_zarr
 
 prefix = os.path.join(os.path.dirname(__file__), "fixtures")
 
@@ -220,7 +220,11 @@ def test_get_variable_datetime_tz():
 def test_reader(protocol, filename):
     """test reader."""
     src_path = protocol + os.path.join(protocol, prefix, filename)
-    with Reader(src_path, variable="dataset") as src:
+    with Reader(src_path, variable="dataset", opener=fs_open_dataset) as src:
+        assert src.info()
+        assert src.tile(0, 0, 0)
+
+    with Reader(src_path, variable="dataset", group="/", opener=fs_open_dataset) as src:
         assert src.info()
         assert src.tile(0, 0, 0)
 
@@ -267,7 +271,6 @@ def custom_netcdf_opener(  # noqa: C901
 
         fs = fsspec.filesystem(protocol)
         ds = xarray.open_dataset(fs.open(src_path), **xr_open_args)
-
         return ds
 
     with Reader(
@@ -303,14 +306,43 @@ def test_zarr_group(group):
 
 
 @pytest.mark.parametrize(
-    "src_path",
+    "src_path,options",
+    [
+        ("s3://mur-sst/zarr-v1", {"anon": True}),
+        (
+            "https://nasa-power.s3.amazonaws.com/syn1deg/temporal/power_syn1deg_monthly_temporal_lst.zarr",
+            {},
+        ),
+        (os.path.join(prefix, "dataset_3d.zarr"), {}),
+    ],
+)
+def test_io_fs_open_dataset(src_path, options):
+    """test fs_open_dataset with cloud hosted files."""
+    with fs_open_dataset(src_path, **options) as ds:
+        assert list(ds.data_vars)
+
+
+@pytest.mark.parametrize(
+    "src_path,options",
     [
-        # "s3://mur-sst/zarr-v1",
-        "https://nasa-power.s3.amazonaws.com/syn1deg/temporal/power_syn1deg_monthly_temporal_lst.zarr",
-        os.path.join(prefix, "dataset_3d.zarr"),
+        # Let's assume we don't have S3 Credentials
+        ("s3://mur-sst/zarr-v1", {"skip_signature": True}),
+        ("s3://mur-sst/zarr-v1", {"skip_signature": True, "region": "us-west-2"}),
+        # HTTS url are considered public
+        ("https://mur-sst.s3.us-west-2.amazonaws.com/zarr-v1", {}),
+        # NOTE: https://github.com/developmentseed/obstore/pull/590
+        # (
+        #     "https://nasa-power.s3.amazonaws.com/syn1deg/temporal/power_syn1deg_monthly_temporal_lst.zarr",
+        #     {},
+        # ),
+        (
+            "https://nasa-power.s3.us-west-2.amazonaws.com/syn1deg/temporal/power_syn1deg_monthly_temporal_lst.zarr",
+            {},
+        ),
+        (os.path.join(prefix, "dataset_3d.zarr"), {}),
     ],
 )
-def test_io_xarray_open_dataset(src_path):
-    """test xarray_open_dataset with cloud hosted files."""
-    with xarray_open_dataset(src_path) as ds:
+def test_io_open_zarr(src_path, options):
+    """test open_zarr with cloud hosted files."""
+    with open_zarr(src_path, **options) as ds:
         assert list(ds.data_vars)

src/titiler/xarray/tests/test_io_tools.py

@@ -13,7 +13,7 @@
 from titiler.core.resources.enums import MediaType
 from titiler.xarray.dependencies import XarrayIOParams
 from titiler.xarray.factory import TilerFactory
-from titiler.xarray.io import xarray_open_dataset
+from titiler.xarray.io import open_zarr
 
 
 @define
@@ -22,7 +22,7 @@ class VariablesExtension(FactoryExtension):
 
     # Custom dependency for /variables
     io_dependency: Type[DefaultDependency] = XarrayIOParams
-    dataset_opener: Callable[..., xarray.Dataset] = xarray_open_dataset
+    dataset_opener: Callable[..., xarray.Dataset] = open_zarr
 
     def __attrs_post_init__(self):
         """raise deprecation warning."""
@@ -54,7 +54,7 @@ class DatasetMetadataExtension(FactoryExtension):
     """Add dataset metadata endpoints to a Xarray TilerFactory."""
 
     io_dependency: Type[DefaultDependency] = XarrayIOParams
-    dataset_opener: Callable[..., xarray.Dataset] = xarray_open_dataset
+    dataset_opener: Callable[..., xarray.Dataset] = open_zarr
 
     def register(self, factory: TilerFactory):
         """Register endpoint to the tiler factory."""