Improve framework for validation

docs/specification.md

@@ -42,7 +42,7 @@ The optional `extra` object is a free-form dictionary that can hold any addition
 ## The `nodes` group
 The nodes group will contain an `ids` array and optionally a `props` group. 
 ### The `ids` array
-The `nodes\ids` array is a 1D array of node IDs of length `N` >= 0, where `N` is the number of nodes in the graph. Node ids must be unique. Node IDs can have any type supported by zarr (except floats), but we recommend integer dtypes. For large graphs, `uint64` might be necessary to provide enough range for every node to have a unique ID. In the minimal case of an empty graph, the `ids` array will be present but empty. 
+The `nodes\ids` array is a 1D array of node IDs of length `N` >= 0, where `N` is the number of nodes in the graph. Node ids must be unique. Node IDs must have an unsigned integer dtype. For large graphs, `uint64` might be necessary to provide enough range for every node to have a unique ID. In the minimal case of an empty graph, the `ids` array will be present but empty. 
 
 
 ### The `props` group and `node property` groups

justfile

@@ -8,7 +8,7 @@ test-cov:
 
 # run benchmarks
 benchmark:
-    uv run --group bench pytest tests/bench.py
+    uv run --group bench pytest tests/test_bench.py
 
 # build wheel and sdist
 build:

src/geff/_graph_libs/_api_wrapper.py

@@ -16,6 +16,7 @@
 
     from geff._typing import PropDictNpArray
     from geff.metadata._schema import GeffMetadata
+    from geff.validate.data import ValidationConfig
 
 SupportedBackend = Literal["networkx", "rustworkx", "spatial-graph"]
 
@@ -107,41 +108,45 @@ def get_construct_func(backend: SupportedBackend) -> ConstructFunc[Any]:
 @overload
 def read(
     store: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     node_props: list[str] | None = None,
     edge_props: list[str] | None = None,
     backend: Literal["networkx"] = "networkx",
+    data_validation: ValidationConfig | None = None,
 ) -> tuple[nx.Graph | nx.DiGraph, GeffMetadata]: ...
 
 
 @overload
 def read(
     store: StoreLike,
-    validate: bool,
+    structure_validation: bool,
     node_props: list[str] | None,
     edge_props: list[str] | None,
     backend: Literal["rustworkx"],
+    data_validation: ValidationConfig | None = None,
 ) -> tuple[rx.PyGraph | rx.PyDiGraph, GeffMetadata]: ...
 
 
 @overload
 def read(
     store: StoreLike,
-    validate: bool,
+    structure_validation: bool,
     node_props: list[str] | None,
     edge_props: list[str] | None,
     backend: Literal["spatial-graph"],
+    data_validation: ValidationConfig | None = None,
     *,
     position_attr: str = "position",
 ) -> tuple[sg.SpatialGraph | sg.SpatialDiGraph, GeffMetadata]: ...
 
 
 def read(
     store: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     node_props: list[str] | None = None,
     edge_props: list[str] | None = None,
     backend: SupportedBackend = "networkx",
+    data_validation: ValidationConfig | None = None,
     **backend_kwargs: Any,
 ) -> tuple[Any, GeffMetadata]:
     """
@@ -150,7 +155,7 @@ def read(
     Args:
         store (StoreLike): The path or zarr store to the root of the geff zarr, where
             the .attrs contains the geff  metadata.
-        validate (bool, optional): Flag indicating whether to perform validation on the
+        structure_validation (bool, optional): Flag indicating whether to perform validation on the
             geff file before loading into memory. If set to False and there are
             format issues, will likely fail with a cryptic error. Defaults to True.
         node_props (list of str, optional): The names of the node properties to load,
@@ -159,14 +164,18 @@ def read(
             if None all properties will be loaded, defaults to None.
         backend ({"networkx", "rustworkx", "spatial-graph"}): Flag for the chosen backend, default
             is "networkx".
+        data_validation (ValidationConfig, optional): Optional configuration for which
+            optional types of data to validate. Each option defaults to False.
         backend_kwargs (Any): Additional kwargs that may be accepted by
             the backend when reading the data.
 
     Returns:
         tuple[Any, GeffMetadata]: Graph object of the chosen backend, and the GEFF metadata.
     """
     construct_func = get_construct_func(backend)
-    in_memory_geff = read_to_memory(store, validate, node_props, edge_props)
+    in_memory_geff = read_to_memory(
+        store, structure_validation, node_props, edge_props, data_validation
+    )
     return (
         construct_func(**in_memory_geff, **backend_kwargs),
         in_memory_geff["metadata"],

src/geff/_graph_libs/_networkx.py

@@ -18,6 +18,7 @@
     from zarr.storage import StoreLike
 
     from geff._typing import PropDictNpArray
+    from geff.validate.data import ValidationConfig
 
 import logging
 
@@ -190,28 +191,33 @@ def construct_nx(
 
 def read_nx(
     store: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     node_props: list[str] | None = None,
     edge_props: list[str] | None = None,
+    data_validation: ValidationConfig | None = None,
 ) -> tuple[nx.Graph, GeffMetadata]:
     """Read a geff file into a networkx graph. Metadata properties will be stored in
     the graph properties, accessed via `G.graph[key]` where G is a networkx graph.
 
     Args:
         store (str | Path | zarr store): The path/str to the geff zarr, or the store
             itself. Opens in append mode, so will only overwrite geff-controlled groups.
-        validate (bool, optional): Flag indicating whether to perform validation on the
+        structure_validation (bool, optional): Flag indicating whether to perform validation on the
             geff file before loading into memory. If set to False and there are
             format issues, will likely fail with a cryptic error. Defaults to True.
         node_props (list of str, optional): The names of the node properties to load,
             if None all properties will be loaded, defaults to None.
         edge_props (list of str, optional): The names of the edge properties to load,
             if None all properties will be loaded, defaults to None.
+        data_validation (ValidationConfig, optional): Optional configuration for which
+            optional types of data to validate. Each option defaults to false.
 
     Returns:
         A networkx graph containing the graph that was stored in the geff file format
     """
-    in_memory_geff = read_to_memory(store, validate, node_props, edge_props)
+    in_memory_geff = read_to_memory(
+        store, structure_validation, node_props, edge_props, data_validation
+    )
     graph = construct_nx(**in_memory_geff)
 
     return graph, in_memory_geff["metadata"]

src/geff/_graph_libs/_rustworkx.py

@@ -28,6 +28,7 @@
     from zarr.storage import StoreLike
 
     from geff._typing import PropDictNpArray
+    from geff.validate.data import ValidationConfig
 
 
 def get_roi_rx(
@@ -250,9 +251,10 @@ def construct_rx(
 
 def read_rx(
     store: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     node_props: list[str] | None = None,
     edge_props: list[str] | None = None,
+    data_validation: ValidationConfig | None = None,
 ) -> tuple[rx.PyGraph | rx.PyDiGraph, GeffMetadata]:
     """Read a geff file into a rustworkx graph.
     Metadata properties will be stored in the graph.attrs dict
@@ -264,16 +266,20 @@ def read_rx(
 
     Args:
         store: The path/str to the geff zarr, or the store itself.
-        validate: Whether to validate the geff file.
+        structure_validation: Whether to validate the geff file.
         node_props: The names of the node properties to load,
             if None all properties will be loaded, defaults to None.
         edge_props: The names of the edge properties to load,
             if None all properties will be loaded, defaults to None.
+        data_validation (ValidationConfig, optional): Optional configuration for which
+            optional types of data to validate. Each option defaults to False.
 
     Returns:
         A tuple containing the rustworkx graph and the metadata.
     """
-    graph_dict = read_to_memory(store, validate, node_props, edge_props)
+    graph_dict = read_to_memory(
+        store, structure_validation, node_props, edge_props, data_validation
+    )
     graph = construct_rx(**graph_dict)
 
     return graph, graph_dict["metadata"]

src/geff/_graph_libs/_spatial_graph.py

@@ -21,6 +21,7 @@
     from zarr.storage import StoreLike
 
     from geff._typing import PropDictNpArray
+    from geff.validate.data import ValidationConfig
 
 import geff
 from geff.core_io import write_arrays
@@ -116,10 +117,11 @@ def write_sg(
 
 def read_sg(
     store: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     position_attr: str = "position",
     node_props: list[str] | None = None,
     edge_props: list[str] | None = None,
+    data_validation: ValidationConfig | None = None,
 ) -> tuple[sg.SpatialGraph | sg.SpatialDiGraph, GeffMetadata]:
     """Read a geff file into a SpatialGraph.
 
@@ -129,37 +131,31 @@ def read_sg(
     Args:
 
         store (Path | str | zarr store):
-
             The path to the root of the geff zarr, where the .attrs contains
             the geff  metadata.
-
-        validate (bool, optional):
-
+        structure_validation (bool, optional):
             Flag indicating whether to perform validation on the geff file
             before loading into memory. If set to False and there are format
             issues, will likely fail with a cryptic error. Defaults to True.
-
         position_attr (str, optional):
-
             How to call the position attribute in the returned SpatialGraph.
             Defaults to "position".
-
         node_props (list of str, optional):
-
             The names of the node properties to load, if None all properties
             will be loaded, defaults to None.
-
         edge_props (list of str, optional):
-
             The names of the edge properties to load, if None all properties
             will be loaded, defaults to None.
+        data_validation (ValidationConfig, optional): Optional configuration for which
+            optional types of data to validate. Each option defaults to False.
 
     Returns:
-
         A tuple containing the spatial_graph graph and the metadata.
     """
 
-    in_memory_geff = read_to_memory(store, validate, node_props, edge_props)
+    in_memory_geff = read_to_memory(
+        store, structure_validation, node_props, edge_props, data_validation
+    )
     graph = construct_sg(**in_memory_geff, position_attr=position_attr)
 
     return graph, in_memory_geff["metadata"]

src/geff/core_io/_base_read.py

@@ -8,6 +8,7 @@
 from geff import _path
 from geff.core_io import _utils
 from geff.metadata._schema import GeffMetadata
+from geff.validate.data import ValidationConfig, validate_data
 from geff.validate.structure import validate_structure
 
 if TYPE_CHECKING:
@@ -214,9 +215,10 @@ def build(
 #   added to this function to select between them.
 def read_to_memory(
     source: StoreLike,
-    validate: bool = True,
+    structure_validation: bool = True,
     node_props: Iterable[str] | None = None,
     edge_props: Iterable[str] | None = None,
+    data_validation: ValidationConfig | None = None,
 ) -> InMemoryGeff:
     """
     Read a GEFF zarr file to into memory as a series of numpy arrays in a dictionary.
@@ -227,9 +229,11 @@ def read_to_memory(
     Args:
         source (str | Path | zarr store): Either a path to the root of the geff zarr
             (where the .attrs contains the geff metadata), or a zarr store object
-        validate (bool, optional): Flag indicating whether to perform validation on the
-            geff file before loading into memory. If set to False and there are
-            format issues, will likely fail with a cryptic error. Defaults to True.
+        structure_validation (bool, optional): Flag indicating whether to perform metadata/structure
+            validation on the geff file before loading into memory. If set to False and
+            there are format issues, will likely fail with a cryptic error. Defaults to True.
+        data_validation (ValidationConfig, optional): Optional configuration for which
+            optional types of data to validate. Each option defaults to False.
         node_props (iterable of str, optional): The names of the node properties to load,
             if None all properties will be loaded, defaults to None.
         edge_props (iterable of str, optional): The names of the edge properties to load,
@@ -240,10 +244,14 @@ def read_to_memory(
         (metadata, node_ids, edge_ids, node_props, edge_props)
     """
 
-    file_reader = GeffReader(source, validate)
+    file_reader = GeffReader(source, structure_validation)
 
     file_reader.read_node_props(node_props)
     file_reader.read_edge_props(edge_props)
 
     in_memory_geff = file_reader.build()
+
+    if data_validation is not None:
+        validate_data(config=data_validation, memory_geff=in_memory_geff)
+
     return in_memory_geff

src/geff/testing/data.py

@@ -70,8 +70,9 @@
 if TYPE_CHECKING:
     from numpy.typing import NDArray
 
+
 DTypeStr = Literal["double", "int", "int8", "uint8", "int16", "uint16", "float32", "float64", "str"]
-NodeIdDTypeStr = Literal["int", "int8", "uint8", "int16", "uint16"]
+NodeIdDTypeStr = Literal["uint", "uint8", "uint16", "uint32", "uint64"]
 Axes = Literal["t", "z", "y", "x"]
 
 
@@ -509,7 +510,7 @@ def create_simple_2d_geff(
             >>> # graph is a networkx Graph with 2D spatial data (x, y, t)
     """
     return create_memory_mock_geff(
-        node_id_dtype="int",
+        node_id_dtype="uint",
         node_axis_dtypes={"position": "float64", "time": "float64"},
         directed=directed,
         num_nodes=num_nodes,
@@ -568,7 +569,7 @@ def create_simple_3d_geff(
             >>> x, y, z, t = node_data['x'], node_data['y'], node_data['z'], node_data['t']
     """
     return create_memory_mock_geff(
-        node_id_dtype="int",
+        node_id_dtype="uint",
         node_axis_dtypes={"position": "float64", "time": "float64"},
         directed=directed,
         num_nodes=num_nodes,
@@ -616,7 +617,7 @@ def create_simple_temporal_geff(
             >>> # Each node has only 't' coordinate, no x, y, z
     """
     return create_memory_mock_geff(
-        node_id_dtype="int",
+        node_id_dtype="uint",
         node_axis_dtypes={"position": "float64", "time": "float64"},
         directed=directed,
         num_nodes=num_nodes,