Make affine and space metadata consistent as well as updating support of latest holoscan SDK CUDA 12 version (#565)

* Make affine and space consistent as space is properly parse by MONAI transforms

Signed-off-by: M Q <[email protected]>

* Typing improvements

Signed-off-by: M Q <[email protected]>

* Make image metadata Affine and Space consistent, either LPS or RAS

Signed-off-by: M Q <[email protected]>

* Update monai/deploy/operators/monai_seg_inference_operator.py

Co-authored-by: Copilot <[email protected]>
Signed-off-by: Ming M Qin <[email protected]>

* Correct CoPilot suggested code which failed liniting

Signed-off-by: M Q <[email protected]>

* Fix complaint from new version of mypy

Signed-off-by: M Q <[email protected]>

* Support the latest holoscan SDK CUDA 12 version, holoscan-cu12

Signed-off-by: M Q <[email protected]>

* Docs gen works with Python 3.10+

Signed-off-by: M Q <[email protected]>

* Fix complaints on single quote vs dhouble quote for string

Signed-off-by: M Q <[email protected]>

* Making docs gen require python >= 3.10

Signed-off-by: M Q <[email protected]>

* Fix docs build error on readthedocs, although local builds had no issues

Signed-off-by: M Q <[email protected]>

---------

Signed-off-by: M Q <[email protected]>
Signed-off-by: Ming M Qin <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: MMelQin

.readthedocs.yml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.10"
     # You can also specify other tool versions:
     # nodejs: "20"
     # rust: "1.70"

CONTRIBUTING.md

@@ -137,10 +137,6 @@ MONAI Deploy App SDK's code coverage report is available at [CodeCov](https://co
 
 #### Building the documentation
 
-:::{note}
-Please note that the documentation builds successfully in Python 3.9 environment, but fails with Python 3.10.
-:::
-
 MONAI's documentation is located at `docs/`.
 
 ```bash

docs/requirements.txt

@@ -1,25 +1,24 @@
-Sphinx==4.1.2
-sphinx-autobuild==2021.3.14
-myst-nb==0.17.2  # this version is fine in python 3.9 and avoids pulling in multiple nbformat packages
-myst-parser==0.18.0
+Sphinx>=4.5.0
+sphinx-autobuild
+myst-nb>=0.17.2
+myst-parser>=0.18.0
 lxml_html_clean  # needed by myst-nb
-linkify-it-py==1.0.1  # https://myst-parser.readthedocs.io/en/latest/syntax/optional.html?highlight=linkify#linkify
-sphinx-togglebutton==0.2.3
-sphinx-copybutton==0.4.0
-sphinxcontrib-bibtex<2.0.0  # https://github.com/executablebooks/jupyter-book/issues/1137
-sphinxcontrib-spelling==7.2.1  # https://sphinxcontrib-spelling.readthedocs.io/en/latest/index.html
-sphinx-thebe==0.0.10
-sphinx-panels==0.6.0
-ablog==0.10.19
-docutils==0.16  # 0.17 causes error. https://github.com/executablebooks/MyST-Parser/issues/343
-pydata_sphinx_theme==0.6.3
-sphinxemoji==0.1.8
+linkify-it-py>=1.0.1  # https://myst-parser.readthedocs.io/en/latest/syntax/optional.html?highlight=linkify#linkify
+sphinx-togglebutton
+sphinx-copybutton
+sphinxcontrib-bibtex>=2.4.1  # Version 2.4.1+ supports Python 3.10
+sphinxcontrib-spelling  # https://sphinxcontrib-spelling.readthedocs.io/en/latest/index.html
+sphinx-thebe
+sphinx-design
+ablog
+pydata_sphinx_theme>=0.13.0
+sphinxemoji
 torch>=1.12.0
-sphinx-autodoc-typehints==1.12.0
-sphinxcontrib-applehelp==1.0.2
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.0
-sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-sphinxcontrib-mermaid==0.7.1
+sphinx-autodoc-typehints
+sphinxcontrib-applehelp
+sphinxcontrib-devhelp
+sphinxcontrib-htmlhelp
+sphinxcontrib-jsmath
+sphinxcontrib-qthelp
+sphinxcontrib-serializinghtml
+sphinxcontrib-mermaid

docs/source/conf.py

@@ -83,14 +83,15 @@
     "myst_nb",
     "sphinx_copybutton",
     "sphinx_togglebutton",
-    "sphinx_panels",  # https://sphinx-panels.readthedocs.io/en/latest/
+    "sphinx_design",  # https://sphinx-design.readthedocs.io/en/latest/
     "ablog",
     "sphinxemoji.sphinxemoji",
     # https://myst-parser.readthedocs.io/en/latest/sphinx/use.html#automatically-create-targets-for-section-headers
     # "sphinx.ext.autosectionlabel",  <== don't need anymore from v0.13.0
     "sphinx_autodoc_typehints",
     "sphinxcontrib.mermaid",
 ]
+bibtex_bibfiles = ["refs.bib"]
 
 autoclass_content = "both"
 add_module_names = True
@@ -180,10 +181,10 @@
 html_css_files = ["custom.css"]
 html_title = f"{project} {version} Documentation"
 
-# -- Options for sphinx-panels -------------------------------------------------
+# -- Options for sphinx-design -------------------------------------------------
 #
-# (reference: https://sphinx-panels.readthedocs.io/en/latest/)
-panels_add_bootstrap_css = False  # pydata-sphinx-theme already loads bootstrap css
+# (reference: https://sphinx-design.readthedocs.io/en/latest/)
+# No additional configuration needed - sphinx-design works with pydata-sphinx-theme
 
 # -- Options for linkcheck builder -------------------------------------------------
 #
@@ -222,16 +223,17 @@
 # -- Options for myst-nb -------------------------------------------------
 #
 # (reference: https://myst-nb.readthedocs.io/en/latest/)
-# Prevent the following error
-#     MyST NB Configuration Error:
-#    `nb_render_priority` not set for builder: doctest
-nb_render_priority = {"doctest": ()}
 # Prevent creating jupyter_execute folder in dist
 #  https://myst-nb.readthedocs.io/en/latest/use/execute.html#executing-in-temporary-folders  # noqa
 execution_in_temp = True
 jupyter_execute_notebooks = "off"
 
 
+# -- Options for sphinxcontrib.bibtex -------------------------------------------------
+#
+# (reference: https://sphinxcontrib-bibtex.readthedocs.io/)
+bibtex_bibfiles = []  # Add bibliography files here if needed
+
 # -- Options for sphinxcontrib.spelling -------------------------------------------------
 #
 # (reference: https://sphinxcontrib-spelling.readthedocs.io/en/latest/customize.html)

examples/apps/ai_remote_infer_app/spleen_seg_operator.py

@@ -127,7 +127,7 @@ def pre_process(self, img_reader, out_dir: str = "./input_images") -> Compose:
                     resample=False,
                     output_ext=".nii",
                 ),
-                Orientationd(keys=my_key, axcodes="LPS"),
+                Orientationd(keys=my_key, axcodes="RAS"),
                 Spacingd(keys=my_key, pixdim=[1.5, 1.5, 2.9], mode=["bilinear"]),
                 ScaleIntensityRanged(keys=my_key, a_min=-57, a_max=164, b_min=0.0, b_max=1.0, clip=True),
                 EnsureTyped(keys=my_key),

examples/apps/ai_unetr_seg_app/unetr_seg_operator.py

@@ -143,7 +143,7 @@ def pre_process(self, img_reader, out_dir: str = "./input_images") -> Compose:
                     output_ext=".nii",
                 ),
                 Spacingd(keys=my_key, pixdim=(1.5, 1.5, 2.0), mode=("bilinear")),
-                Orientationd(keys=my_key, axcodes="LPS"),
+                Orientationd(keys=my_key, axcodes="RAS"),
                 ScaleIntensityRanged(my_key, a_min=-175, a_max=250, b_min=0.0, b_max=1.0, clip=True),
                 CropForegroundd(my_key, source_key=my_key),
             ]

examples/apps/cchmc_ped_abd_ct_seg_app/abdomen_seg_operator.py

@@ -226,7 +226,7 @@ def pre_process(self, img_reader) -> Compose:
                 # img_reader: specialized InMemImageReader, derived from MONAI ImageReader
                 LoadImaged(keys=my_key, reader=img_reader),
                 EnsureChannelFirstd(keys=my_key),
-                Orientationd(keys=my_key, axcodes="LPS"),
+                Orientationd(keys=my_key, axcodes="RAS"),
                 Spacingd(keys=my_key, pixdim=[1.5, 1.5, 3.0], mode=["bilinear"]),
                 ScaleIntensityRanged(keys=my_key, a_min=-250, a_max=400, b_min=0.0, b_max=1.0, clip=True),
                 CropForegroundd(keys=my_key, source_key=my_key, mode="minimum"),

monai/deploy/operators/dicom_seg_writer_operator.py

@@ -27,6 +27,8 @@
 ImplicitVRLittleEndian, _ = optional_import("pydicom.uid", name="ImplicitVRLittleEndian")
 Dataset, _ = optional_import("pydicom.dataset", name="Dataset")
 FileDataset, _ = optional_import("pydicom.dataset", name="FileDataset")
+DA, _ = optional_import("pydicom.valuerep", name="DA")
+TM, _ = optional_import("pydicom.valuerep", name="TM")
 PyDicomSequence, _ = optional_import("pydicom.sequence", name="Sequence")
 sitk, _ = optional_import("SimpleITK")
 codes, _ = optional_import("pydicom.sr.codedict", name="codes")
@@ -341,8 +343,8 @@ def create_dicom_seg(self, image: np.ndarray, dicom_series: DICOMSeries, output_
         # Adding a few tags that are not in the Dataset
         # Also try to set the custom tags that are of string type
         dt_now = datetime.datetime.now()
-        seg.SeriesDate = dt_now.strftime("%Y%m%d")  # type: ignore[assignment]
-        seg.SeriesTime = dt_now.strftime("%H%M%S")  # type: ignore[assignment]
+        seg.SeriesDate = DA(dt_now.strftime("%Y%m%d"))
+        seg.SeriesTime = TM(dt_now.strftime("%H%M%S"))
         seg.TimezoneOffsetFromUTC = (
             dt_now.astimezone().isoformat()[-6:].replace(":", "")
         )  # '2022-09-27T22:36:20.143857-07:00'

monai/deploy/operators/dicom_series_to_volume_operator.py

@@ -40,19 +40,26 @@ class DICOMSeriesToVolumeOperator(Operator):
     """
 
     # Use constants instead of enums in monai to avoid dependency at this level.
+    MONAI_UTIL_ENUMS_SPACEKEYS_RAS = "RAS"
     MONAI_UTIL_ENUMS_SPACEKEYS_LPS = "LPS"
     MONAI_TRANSFORMS_SPATIAL_METADATA_NAME = "space"
+    METADATA_SPACE_RAS = {MONAI_TRANSFORMS_SPATIAL_METADATA_NAME: MONAI_UTIL_ENUMS_SPACEKEYS_RAS}
     METADATA_SPACE_LPS = {MONAI_TRANSFORMS_SPATIAL_METADATA_NAME: MONAI_UTIL_ENUMS_SPACEKEYS_LPS}
+    ATTRIBUTE_NIFTI_AFFINE = "nifti_affine_transform"
+    ATTRIBUTE_DICOM_AFFINE = "dicom_affine_transform"
 
-    def __init__(self, fragment: Fragment, *args, **kwargs):
+    def __init__(self, fragment: Fragment, *args, affine_lps_to_ras: bool = True, **kwargs):
         """Create an instance for a containing application object.
 
         Args:
             fragment (Fragment): An instance of the Application class which is derived from Fragment.
+            affine_lps_to_ras (bool): If true, the affine transform in the image metadata is RAS oriented,
+                                      otherwise it is LPS oriented. Default is True.
         """
 
         self.input_name_series = "study_selected_series_list"
         self.output_name_image = "image"
+        self.affine_lps_to_ras = affine_lps_to_ras
         # Need to call the base class constructor last
         super().__init__(fragment, *args, **kwargs)
 
@@ -89,18 +96,16 @@ def convert_to_image(self, study_selected_series_list: List[StudySelectedSeries]
             metadata.update(self._get_instance_properties(study_selected_series.study))
             selection_metadata = {"selection_name": selection_name}
             metadata.update(selection_metadata)
-            # Add the metadata to specify LPS.
-            # Previously, this was set in ImageReader class, but moving it here allows other loaders
-            # to determine this value on its own, e.g. NIfTI loader but it does not set this
-            # resulting in the MONAI Orientation transform to default the labels to RAS.
-            # It is assumed that the ImageOrientationPatient will be set accordingly if the
-            # PatientPosition is other than HFS.
-            # NOTE: This value is properly parsed by MONAI Orientation transform from v1.5.1 onwards.
-            #       Some early MONAI model inference configs incorrectly specify orientation to RAS
-            #       due part to previous MONAI versions did not correctly parse this metadata from
-            #       the input MetaTensor and defaulting to RAS. Now with LPS properly set, the inference
-            #       configs then need to be updated to specify LPS, to achieve the same result.
-            metadata.update(self.METADATA_SPACE_LPS)
+            # The affine transform and the coordinate space are set based on the flag affine_lps_to_ras.
+            # If the flag is true, the NIFTI affine (RAS) is used, otherwise the DICOM affine (LPS) is used.
+            if self.affine_lps_to_ras:
+                if hasattr(dicom_series, self.ATTRIBUTE_NIFTI_AFFINE):
+                    metadata["affine"] = getattr(dicom_series, self.ATTRIBUTE_NIFTI_AFFINE)
+                metadata.update(self.METADATA_SPACE_RAS)
+            else:
+                if hasattr(dicom_series, self.ATTRIBUTE_DICOM_AFFINE):
+                    metadata["affine"] = getattr(dicom_series, self.ATTRIBUTE_DICOM_AFFINE)
+                metadata.update(self.METADATA_SPACE_LPS)
 
             voxel_data = self.generate_voxel_data(dicom_series)
             image = self.create_volumetric_image(voxel_data, metadata)
@@ -366,7 +371,7 @@ def compute_affine_transform(self, s_1, s_n, n, series):
         zn = 0.0
 
         ip1 = None
-        ip2 = None
+        ipn = None
         try:
             ip1_de = s_1[0x0020, 0x0032]
             ipn_de = s_n[0x0020, 0x0032]
@@ -404,7 +409,7 @@ def compute_affine_transform(self, s_1, s_n, n, series):
         m1[3, 2] = 0
         m1[3, 3] = 1
 
-        series.dicom_affine_transform = m1
+        setattr(series, self.ATTRIBUTE_DICOM_AFFINE, m1)
 
         m2[0, 0] = -rx * vr
         m2[0, 1] = -cx * vc
@@ -426,7 +431,7 @@ def compute_affine_transform(self, s_1, s_n, n, series):
         m2[3, 2] = 0
         m2[3, 3] = 1
 
-        series.nifti_affine_transform = m2
+        setattr(series, self.ATTRIBUTE_NIFTI_AFFINE, m2)
 
     def create_metadata(self, series) -> Dict:
         """Collects all relevant metadata from the DICOM Series and creates a dictionary.

monai/deploy/operators/monai_seg_inference_operator.py

@@ -31,16 +31,20 @@
 ImageReader: Any = ImageReader_
 if not image_reader_ok_:
     ImageReader = object  # for 'class InMemImageReader(ImageReader):' to work
+is_no_channel, _ = optional_import("monai.data.utils", name="is_no_channel")
 decollate_batch, _ = optional_import("monai.data", name="decollate_batch")
 sliding_window_inference, _ = optional_import("monai.inferers", name="sliding_window_inference")
 simple_inference, _ = optional_import("monai.inferers", name="SimpleInferer")
 ensure_tuple, _ = optional_import(MONAI_UTILS, name="ensure_tuple")
 MetaKeys, _ = optional_import(MONAI_UTILS, name="MetaKeys")
 SpaceKeys, _ = optional_import(MONAI_UTILS, name="SpaceKeys")
+TraceKeys, _ = optional_import(MONAI_UTILS, name="TraceKeys")
 Compose_, _ = optional_import("monai.transforms", name="Compose")
 # Dynamic class is not handled so make it Any for now: https://github.com/python/mypy/issues/2477
 Compose: Any = Compose_
 
+cp, has_cp = optional_import("cupy")
+
 from monai.deploy.core import AppContext, Condition, ConditionType, Fragment, Image, OperatorSpec, Resource
 
 from .inference_operator import InferenceOperator
@@ -362,6 +366,7 @@ def compute(self, op_input, op_output, context):
                 self._executing = True
         try:
             input_image = op_input.receive(self._input_name_image)
+
             if not input_image:
                 raise ValueError("Input is None.")
             op_output.emit(self.compute_impl(input_image, context), self._output_name_seg)
@@ -592,7 +597,7 @@ def _get_meta_dict(self, img: Image) -> Dict:
         return meta_dict
 
 
-# Reuse MONAI code for the derived ImageReader
+# Reuse MONAI code for the derived ImageReader as it is not exposed
 def _copy_compatible_dict(from_dict: Dict, to_dict: Dict):
     if not isinstance(to_dict, dict):
         raise ValueError(f"to_dict must be a Dict, got {type(to_dict)}.")
@@ -601,7 +606,9 @@ def _copy_compatible_dict(from_dict: Dict, to_dict: Dict):
             datum = from_dict[key]
             if isinstance(datum, np.ndarray) and np_str_obj_array_pattern.search(datum.dtype.str) is not None:
                 continue
-            to_dict[key] = datum
+            to_dict[key] = (
+                str(TraceKeys.NONE) if datum is None else datum
+            )  # PyTorch's default_collate cannot handle None values directly
     else:
         affine_key, shape_key = MetaKeys.AFFINE, MetaKeys.SPATIAL_SHAPE
         if affine_key in from_dict and not np.allclose(from_dict[affine_key], to_dict[affine_key]):
@@ -616,12 +623,16 @@ def _copy_compatible_dict(from_dict: Dict, to_dict: Dict):
             )
 
 
-def _stack_images(image_list: List, meta_dict: Dict):
+def _stack_images(image_list: list, meta_dict: Dict, to_cupy: bool = False):
     if len(image_list) <= 1:
         return image_list[0]
-    if meta_dict.get(MetaKeys.ORIGINAL_CHANNEL_DIM, None) not in ("no_channel", None):
+    if not is_no_channel(meta_dict.get(MetaKeys.ORIGINAL_CHANNEL_DIM, None)):
         channel_dim = int(meta_dict[MetaKeys.ORIGINAL_CHANNEL_DIM])
+        if to_cupy and has_cp:
+            return cp.concatenate(image_list, axis=channel_dim)
         return np.concatenate(image_list, axis=channel_dim)
     # stack at a new first dim as the channel dim, if `'original_channel_dim'` is unspecified
     meta_dict[MetaKeys.ORIGINAL_CHANNEL_DIM] = 0
+    if to_cupy and has_cp:
+        return cp.stack(image_list, axis=0)
     return np.stack(image_list, axis=0)