Merge Nuance Updates (#377)

Signed-off-by: Iain Henderson <[email protected]>
Co-authored-by: iain_nuance <[email protected]>
Co-authored-by: Iain Henderson <[email protected]>

Author: iain-henderson

.gitignore

@@ -50,6 +50,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+cover/
 
 # Translations
 *.mo
@@ -72,6 +73,7 @@ instance/
 docs/_build/
 
 # PyBuilder
+.pybuilder/
 target/
 
 # Jupyter Notebook
@@ -82,6 +84,8 @@ profile_default/
 ipython_config.py
 
 # pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
 .python-version
 
 # pipenv
@@ -91,7 +95,22 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 
 # Celery stuff
@@ -140,3 +159,105 @@ _autosummary
 
 # model files
 *.ts
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix

integrations/nuance_pin/Dockerfile

@@ -1,55 +1,73 @@
+FROM nvcr.io/nvidia/pytorch:21.07-py3 AS foundation
+
+ARG EXTRA_PYTHON_PACKAGES
+
+RUN apt-get -y update && \
+    apt-get -y install python3-distutils python3-pip python3-venv && \
+    python3 -m pip install --no-cache-dir --upgrade pip && \
+    python3 -m pip install --no-cache-dir --ignore-installed setuptools
+
+# Create a Virtual Environment to limit the size of the application container
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:$PATH"
+RUN python3 -m pip install --upgrade pip
+
+# Copy the ai_service wheel, this is separate from requirements.txt to help with layer caching for repeated builds
+COPY lib/ai_service-*-py3-none-any.whl /tmp/
+RUN python3 -m pip install --no-cache-dir /tmp/ai_service-*-py3-none-any.whl
+
+COPY requirements.txt /tmp/
+# Add any other python packages your AI Service requires
+RUN python3 -m pip install --no-cache-dir ${EXTRA_PYTHON_PACKAGES} -r /tmp/requirements.txt
+
 FROM nvcr.io/nvidia/pytorch:21.07-py3 AS application
 
 ARG PARTNER_NAME
 ARG SERVICE_NAME
 ARG VERSION
 ARG MONAI_APP_MODULE
 ARG MODEL_PATH
-ARG EXTRA_PYTHON_PACKAGES
 
 ENV TZ=Etc/UTC
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # python3-gdcm or python-gdcm is required for decompression
 RUN apt-get -y update && \
-    apt-get -y install --no-install-recommends python3-distutils python3-gdcm && \
-    # apt-get -y install python3.7 && \
+    apt-get -y install --no-install-recommends python3-gdcm && \
     apt-get autoclean && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-ENV DEBUG=YES
-ENV KEEP_FILES=YES
-
-# make sure all messages reach the console
-ENV PYTHONUNBUFFERED=1
-
-# copy MONAI app files
-COPY . /app/.
-WORKDIR /app
-
 # copy model file to model folder
 RUN wget -q https://github.com/Project-MONAI/model-zoo/releases/download/hosting_storage_v1/lung_nodule_ct_detection_v0.2.0.zip && \
     unzip lung_nodule_ct_detection_v0.2.0.zip -d /tmp/ && \
-    cp /tmp/lung_nodule_ct_detection/models/model.ts model/. && \
+    mkdir -p /app/model && \
+    cp /tmp/lung_nodule_ct_detection/models/model.ts /app/model/ && \
     rm -rf /tmp/lung_nodule_ct_detection && \
     rm lung_nodule_ct_detection_v0.2.0.zip
 
 # non-root aiserviceuser in group aiserviceuser with UserID and GroupID as 20225
-RUN groupadd -g 20225 -r aiserviceuser && useradd -u 20225 -r -g aiserviceuser aiserviceuser && chown -R aiserviceuser:aiserviceuser /app && \
-    chown -R aiserviceuser:aiserviceuser /var
+RUN groupadd -g 20225 -r aiserviceuser && \
+    useradd -u 20225 -r -g aiserviceuser aiserviceuser && \
+    chown -R aiserviceuser:aiserviceuser /app /var
 USER aiserviceuser:aiserviceuser
 
-ENV VIRTUAL_ENV=.venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+# Enable Matplotlib cache folder
+RUN mkdir -p /app/.config/matplotlib
+ENV MPLCONFIGDIR=/app/.config/matplotlib
+
+# Copy the virtual environment from the foundation image
+ENV VIRTUAL_ENV=/app/venv
+COPY --from=foundation --chown=aiserviceuser:aiserviceuser /opt/venv "${VIRTUAL_ENV}"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+# make sure all messages reach the console
+ENV PYTHONUNBUFFERED=1
 
-RUN python -m pip install --upgrade pip && \
-    python -m pip install --upgrade --no-cache-dir ${EXTRA_PYTHON_PACKAGES} -r requirements.txt && \
-    python -m pip install --upgrade --no-cache-dir lib/ai_service-*-py3-none-any.whl && \
-    pip install --upgrade numpy && \
-    rm -rf lib && \
-    rm requirements.txt
+# copy MONAI app files
+COPY --chown=aiserviceuser:aiserviceuser app_wrapper.py /app/
+COPY --chown=aiserviceuser:aiserviceuser app/* /app/app/
+WORKDIR /app
 
 ENV AI_PARTNER_NAME ${PARTNER_NAME}
 ENV AI_SVC_NAME ${SERVICE_NAME}
@@ -58,4 +76,8 @@ ENV AI_MODEL_PATH ${MODEL_PATH}
 ENV MONAI_APP_CLASSPATH ${MONAI_APP_MODULE}
 
 ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+ENV DEBUG=NO
+ENV KEEP_FILES=NO
+
 CMD ["python", "app_wrapper.py"]

integrations/nuance_pin/app/lung_nodule.py

@@ -10,8 +10,9 @@
 # limitations under the License.
 
 import logging
-from typing import Callable
+from typing import Optional
 
+from ai_service import AiJobProcessor
 from app.inference import LungNoduleInferenceOperator
 from app.post_inference_ops import CreatePINDiagnosticsReportOp, GenerateGSPSOp
 
@@ -24,11 +25,10 @@
 @resource(cpu=1, gpu=1, memory="7Gi")
 # The monai pkg is not required by this class, instead by the included operators.
 class LungNoduleDetectionApp(Application):
-    def __init__(self, upload_document: Callable, upload_gsps: Callable, *args, **kwargs):
+    def __init__(self, pin_processor: Optional[AiJobProcessor] = None, *args, **kwargs):
         """Creates an application instance."""
         self._logger = logging.getLogger("{}.{}".format(__name__, type(self).__name__))
-        self.upload_document = upload_document
-        self.upload_gsps = upload_gsps
+        self.pin_processor = pin_processor
         super().__init__(*args, **kwargs)
 
     def run(self, *args, **kwargs):
@@ -60,8 +60,8 @@ def compose(self):
         series_selector_op = DICOMSeriesSelectorOperator(dicom_selection_rules)
         series_to_vol_op = DICOMSeriesToVolumeOperator()
         detection_op = LungNoduleInferenceOperator()
-        gsps_op = GenerateGSPSOp(upload_gsps_fn=self.upload_gsps)
-        pin_report_op = CreatePINDiagnosticsReportOp(upload_doc_fn=self.upload_document)
+        gsps_op = GenerateGSPSOp(pin_processor=self.pin_processor)
+        pin_report_op = CreatePINDiagnosticsReportOp(pin_processor=self.pin_processor)
 
         self.add_flow(study_loader_op, series_selector_op, {"dicom_study_list": "dicom_study_list"})
         self.add_flow(
@@ -87,4 +87,4 @@ def compose(self):
     #     monai-deploy exec app.py -i input -m model/model.ts
     #
     logging.basicConfig(level=logging.DEBUG)
-    app_instance = LungNoduleDetectionApp(lambda x: x, lambda x: x, do_run=True)
+    app_instance = LungNoduleDetectionApp(do_run=True)