Skip wells

README.rst

@@ -95,6 +95,20 @@ To export Images or Plates via the OMERO API::
     # By default, a tile (chunk) size of 1024 is used. Specify values with
     $ omero zarr export Image:1 --tile_width 256 --tile_height 256
 
+    # To exclude Wells from a Plate export, based on Key-Value pairs
+    # (map annotations) use --skip_wells_map key:value. Supports wildcards.
+    $ omero zarr export Plate:1 --skip_wells_map my_key:my_value
+    $ omero zarr export Plate:1 --skip_wells_map my_key:my*
+    $ omero zarr export Plate:1 --skip_wells_map my_key:*value
+    $ omero zarr export Plate:1 --skip_wells_map my_key:*val*
+    $ omero zarr export Plate:1 --skip_wells_map my_key:*
+
+    # Use --metadata_only to export only metadata, no pixel data
+    $ omero zarr export Image:1 --metadata_only
+    $ omero zarr export Plate:2 --metadata_only
+
+    # To export Key-Value pairs from Wells in a Plate as a CSV file,
+    $ omero zarr export_csv Plate:1 --skip_wells_map my_key:*
 
 NB: If the connection to OMERO is lost and the Image is partially exported,
 re-running the command will attempt to complete the export.

src/omero_zarr/cli.py

@@ -29,6 +29,7 @@
 from zarr.hierarchy import open_group
 from zarr.storage import FSStore
 
+from .kvp_tables import plate_to_table
 from .masks import (
     MASK_DTYPE_SIZE,
     MaskSaver,
@@ -202,15 +203,6 @@ def _configure(self, parser: Parser) -> None:
             help=("Name of the array that will be stored. Ignored for --style=split"),
             default="0",
         )
-        polygons.add_argument(
-            "--name_by",
-            default="id",
-            choices=["id", "name"],
-            help=(
-                "How the existing Image or Plate zarr is named. Default 'id' is "
-                "[ID].ome.zarr. 'name' is [NAME].ome.zarr"
-            ),
-        )
 
         masks = parser.add(sub, self.masks, MASKS_HELP)
         masks.add_argument(
@@ -261,15 +253,6 @@ def _configure(self, parser: Parser) -> None:
                 "overlapping labels"
             ),
         )
-        masks.add_argument(
-            "--name_by",
-            default="id",
-            choices=["id", "name"],
-            help=(
-                "How the existing Image or Plate zarr is named. Default 'id' is "
-                "[ID].ome.zarr. 'name' is [NAME].ome.zarr"
-            ),
-        )
 
         export = parser.add(sub, self.export, EXPORT_HELP)
         export.add_argument(
@@ -305,24 +288,44 @@ def _configure(self, parser: Parser) -> None:
             help="Maximum number of workers (only for use with bioformats2raw)",
         )
         export.add_argument(
-            "--name_by",
-            default="id",
-            choices=["id", "name"],
-            help=(
-                "How to name the Image or Plate zarr. Default 'id' is [ID].ome.zarr. "
-                "'name' is [NAME].ome.zarr"
-            ),
+            "object",
+            type=ProxyStringType("Image"),
+            help="The Image to export.",
         )
         export.add_argument(
+            "--metadata_only",
+            action="store_true",
+            help="Only write metadata, do not export pixel data",
+        )
+
+        # CSV export
+        csv = parser.add(sub, self.export_csv, "Export Key-Value pairs as csv")
+        csv.add_argument(
             "object",
             type=ProxyStringType("Image"),
-            help="The Image to export.",
+            help="The Plate from which to export Key-Value pairs.",
         )
 
-        for subcommand in (polygons, masks, export):
+        # Need same arguments for Images and Masks
+        for subcommand in (polygons, masks, export, csv):
             subcommand.add_argument(
                 "--output", type=str, default="", help="The output directory"
             )
+            subcommand.add_argument(
+                "--skip_wells_map",
+                type=str,
+                help="For Plates, skip wells with MapAnnotation values"
+                "matching this key-value pair. e.g. 'MyKey:MyVal*'",
+            )
+            subcommand.add_argument(
+                "--name_by",
+                default="id",
+                choices=["id", "name"],
+                help=(
+                    "How to name the Image or Plate zarr. Default 'id' is "
+                    "[ID].ome.zarr. 'name' is [NAME].ome.zarr"
+                ),
+            )
         for subcommand in (polygons, masks):
             subcommand.add_argument(
                 "--overlaps",
@@ -406,6 +409,15 @@ def export(self, args: argparse.Namespace) -> None:
             plate = self._lookup(self.gateway, "Plate", args.object.id)
             plate_to_zarr(plate, args)
 
+    @gateway_required
+    def export_csv(self, args: argparse.Namespace) -> None:
+        """Export Image or Plate as a CSV file."""
+        print("export_csv...", isinstance(args.object, PlateI))
+        if isinstance(args.object, PlateI):
+            plate = self._lookup(self.gateway, "Plate", args.object.id)
+            self.ctx.out("Export Plate: %s" % plate.name)
+            plate_to_table(plate, args)
+
     @gateway_required
     def import_cmd(self, args: argparse.Namespace) -> None:
         """Import a zarr file as an Image in OMERO."""

src/omero_zarr/kvp_tables.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2023 University of Dundee & Open Microscopy Environment.
+# All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import csv
+
+import omero.clients  # noqa
+
+from .util import get_map_anns, get_zarr_name, map_anns_match
+
+
+def plate_to_table(
+    plate: omero.gateway._PlateWrapper, args: argparse.Namespace
+) -> None:
+    """
+    Exports Well KVPs to a CSV table.
+    """
+    name = get_zarr_name(plate, args.output, args.name_by)
+    skip_wells_map = args.skip_wells_map
+
+    wells = list(plate.listChildren())
+    # sort by row then column...
+    wells = sorted(wells, key=lambda x: (x.row, x.column))
+    well_count = len(wells)
+
+    well_kvps_by_id = get_map_anns(wells)
+
+    if skip_wells_map:
+        # skip_wells_map is like MyKey:MyValue.
+        # Or wild-card MyKey:* or MyKey:Val*
+        wells = [
+            well
+            for well in wells
+            if not map_anns_match(well_kvps_by_id.get(well.id, {}), skip_wells_map)
+        ]
+        print(
+            f"Skipping {well_count - len(wells)} out of {well_count} wells"
+            f" with skip_wells_map: {skip_wells_map}"
+        )
+
+    keys_set = set()
+
+    for well in wells:
+        kvps = well_kvps_by_id.get(well.id, {})
+        for key in kvps.keys():
+            keys_set.add(key)
+
+    column_names = list(keys_set)
+    column_names = sorted(column_names)
+
+    print("column_names", column_names)
+
+    plate_name = plate.getName()
+
+    # write csv file...
+    csv_name = name.replace(".ome.zarr", ".csv")
+    print(f"Writing CSV file: {csv_name}")
+    with open(csv_name, "w", newline="") as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(["Plate", "Well"] + column_names)
+
+        for well in wells:
+            kvps = well_kvps_by_id.get(well.id, {})
+            row = [plate_name, f"{well.getWellPos()}"]
+            for key in column_names:
+                row.append(";".join(kvps.get(key, [])))
+            writer.writerow(row)

src/omero_zarr/masks.py

@@ -36,10 +36,13 @@
 from omero.model import MaskI, PolygonI
 from omero.rtypes import unwrap
 from skimage.draw import polygon as sk_polygon
+from zarr.errors import GroupNotFoundError
 from zarr.hierarchy import open_group
 
 from .util import (
+    get_map_anns,
     get_zarr_name,
+    map_anns_match,
     marshal_axes,
     marshal_transformations,
     open_store,
@@ -100,7 +103,22 @@ def plate_shapes_to_zarr(
 
     count = 0
     t0 = time.time()
-    for well in plate.listChildren():
+    skip_wells_map = args.skip_wells_map
+    wells = list(plate.listChildren())
+    if skip_wells_map:
+        # skip_wells_map is like MyKey:MyValue.
+        # Or wildcard MyKey:* or MyKey:Val*
+        well_kvps_by_id = get_map_anns(wells)
+        wells = [
+            well
+            for well in wells
+            if not map_anns_match(well_kvps_by_id.get(well.id, {}), skip_wells_map)
+        ]
+
+    # sort by row then column...
+    wells = sorted(wells, key=lambda x: (x.row, x.column))
+
+    for well in wells:
         row = plate.getRowLabels()[well.row]
         col = plate.getColumnLabels()[well.column]
         for field in range(n_fields[0], n_fields[1] + 1):
@@ -293,12 +311,13 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
         # Figure out whether we can flatten some dimensions
         unique_dims: Dict[str, Set[int]] = {
             "T": {unwrap(mask.theT) for shapes in masks for mask in shapes},
+            "C": {unwrap(mask.theC) for shapes in masks for mask in shapes},
             "Z": {unwrap(mask.theZ) for shapes in masks for mask in shapes},
         }
         ignored_dimensions: Set[str] = set()
-        # We always ignore the C dimension
-        ignored_dimensions.add("C")
         print(f"Unique dimensions: {unique_dims}")
+        if unique_dims["C"] == {None} or len(unique_dims["C"]) == 1:
+            ignored_dimensions.add("C")
 
         for d in "TZ":
             if unique_dims[d] == {None}:
@@ -308,8 +327,6 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
 
         # Verify that we are linking this mask to a real ome-zarr
         source_image = self.source_image
-        print(f"source_image ??? needs to be None to use filename: {source_image}")
-        print(f"filename: {filename}", self.output, self.name_by)
         source_image_link = self.source_image
         if source_image is None:
             # Assume that we're using the output directory
@@ -320,18 +337,28 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
             assert self.plate_path, "Need image path within the plate"
             source_image = f"{source_image}/{self.plate_path}"
 
-        print(f"source_image {source_image}")
+        print(f"Exporting labels for image at {source_image}")
         image_path = source_image
         if self.output:
             image_path = os.path.join(self.output, source_image)
         src = parse_url(image_path)
         assert src, f"Source image does not exist at {image_path}"
+
+        store = open_store(image_path)
+        try:
+            # Check if labels group already exists...
+            open_group(store, path=f"labels/{name}", mode="r")
+            print(f"Labels group: {name} already exists in {image_path}")
+            # and if so, we assume that array data is already there
+            return
+        except GroupNotFoundError:
+            pass
+
         input_pyramid = Node(src, [])
         assert input_pyramid.load(Multiscales), "No multiscales metadata found"
         input_pyramid_levels = len(input_pyramid.data)
 
-        store = open_store(image_path)
-        label_group = open_group(store)
+        image_group = open_group(store)
 
         _mask_shape: List[int] = list(self.image_shape)
         mask_shape: Tuple[int, ...] = tuple(_mask_shape)
@@ -385,7 +412,7 @@ def save(self, masks: List[omero.model.Shape], name: str) -> None:
 
         write_multiscale_labels(
             label_pyramid,
-            label_group,
+            image_group,
             name,
             axes=axes,
             coordinate_transformations=transformations,
@@ -557,6 +584,7 @@ def masks_to_labels(
                 if shape.fillColor:
                     fillColors[shape_value] = unwrap(shape.fillColor)
                 binim_yx, (t, c, z, y, x, h, w) = self.shape_to_binim_yx(shape)
+                # if z, c or t are None, we apply the mask to all Z, C or T indices
                 for i_t in self._get_indices(ignored_dimensions, "T", t, size_t):
                     for i_c in self._get_indices(ignored_dimensions, "C", c, size_c):
                         for i_z in self._get_indices(