From b71f54ab8160bd04f291c316a81751380fcbb641 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Fri, 1 Aug 2025 18:33:53 -0400 Subject: [PATCH 01/13] update the base python image to 3.12-bookworm image --- src/cirrocumulus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/Dockerfile b/src/cirrocumulus/Dockerfile index 21f127cb..201daedc 100644 --- a/src/cirrocumulus/Dockerfile +++ b/src/cirrocumulus/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM mcr.microsoft.com/vscode/devcontainers/python:3.10-buster +FROM mcr.microsoft.com/vscode/devcontainers/python:3.12-bookworm ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From b67996b2ee00af45fe08c39e16df07d7d7192a22 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 09:10:37 -0400 Subject: [PATCH 02/13] Update Dockerfile --- src/cirrocumulus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/Dockerfile b/src/cirrocumulus/Dockerfile index 201daedc..52073250 100644 --- a/src/cirrocumulus/Dockerfile +++ b/src/cirrocumulus/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM mcr.microsoft.com/vscode/devcontainers/python:3.12-bookworm +FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bookworm ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 5957c992587ee06a031fa9fe71af5c24704ef432 Mon Sep 17 00:00:00 2001 From: Yu Hu Date: Mon, 4 Aug 2025 09:17:53 -0400 Subject: [PATCH 03/13] custom --- src/cirrocumulus/.devcontainer.json | 6 +----- startupscript/post-startup.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/cirrocumulus/.devcontainer.json b/src/cirrocumulus/.devcontainer.json index 0937c0ba..770ce3d8 100644 --- a/src/cirrocumulus/.devcontainer.json +++ b/src/cirrocumulus/.devcontainer.json @@ -13,11 +13,7 @@ ], // re-mount bucket files on container start up "postStartCommand": [ - "./startupscript/remount-on-restart.sh", - "root", - "/root", - "${templateOption:cloud}", - "${templateOption:login}" + "nohup bash -c '[[ ${templateOption:login} == true ]] && ./scripts/cirro_scanner.sh &'" ], "features": { "ghcr.io/devcontainers/features/aws-cli@sha256:bbc9fd513c22e331953126c75ad7b2ed1f9044f1cd5890b7073b634810459b18": {}, diff --git a/startupscript/post-startup.sh b/startupscript/post-startup.sh index 3e849642..793c5264 100755 --- a/startupscript/post-startup.sh +++ b/startupscript/post-startup.sh @@ -34,7 +34,7 @@ source "${SCRIPT_DIR}/emit.sh" source "${CLOUD_SCRIPT_DIR}/vm-metadata.sh" -readonly RUN_AS_LOGIN_USER="sudo -u ${USER_NAME} bash -l -c" +readonly RUN_AS_LOGIN_USER="sudo -u ${USER_NAME} bash -c" export RUN_AS_LOGIN_USER # Startup script status is propagated out to VM guest attributes From 8d466960e67a60ee63584645276afe358ce37595 Mon Sep 17 00:00:00 2001 From: Yu Hu Date: Mon, 4 Aug 2025 11:24:07 -0400 Subject: [PATCH 04/13] update path --- src/cirrocumulus/.devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/.devcontainer.json b/src/cirrocumulus/.devcontainer.json index 770ce3d8..6263e543 100644 --- a/src/cirrocumulus/.devcontainer.json +++ b/src/cirrocumulus/.devcontainer.json @@ -13,7 +13,7 @@ ], // re-mount bucket files on container start up "postStartCommand": [ - "nohup bash -c '[[ ${templateOption:login} == true ]] && ./scripts/cirro_scanner.sh &'" + "nohup bash -c '[[ ${templateOption:login} == true ]] && ./workspace/scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &'" ], "features": { "ghcr.io/devcontainers/features/aws-cli@sha256:bbc9fd513c22e331953126c75ad7b2ed1f9044f1cd5890b7073b634810459b18": {}, From e47198eebda2e184f0c943b93c9e42f51e16e883 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 12:58:45 -0400 Subject: [PATCH 05/13] point to the image --- src/cirrocumulus/docker-compose.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cirrocumulus/docker-compose.yaml b/src/cirrocumulus/docker-compose.yaml index 80344336..7bf6cd09 100644 --- a/src/cirrocumulus/docker-compose.yaml +++ b/src/cirrocumulus/docker-compose.yaml @@ -2,9 +2,7 @@ version: '2.4' services: app: container_name: "application-server" - build: - context: . - dockerfile: Dockerfile + image: "cumulusprod/cumulus:1.0.0" restart: always volumes: - .:/workspace:cached From 35fb563d9fd537fe60cc4b2b6d9d16aa212ff1c0 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:46:55 -0400 Subject: [PATCH 06/13] Update docker-compose.yaml --- src/cirrocumulus/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cirrocumulus/docker-compose.yaml b/src/cirrocumulus/docker-compose.yaml index 7bf6cd09..80344336 100644 --- a/src/cirrocumulus/docker-compose.yaml +++ b/src/cirrocumulus/docker-compose.yaml @@ -2,7 +2,9 @@ version: '2.4' services: app: container_name: "application-server" - image: "cumulusprod/cumulus:1.0.0" + build: + context: . + dockerfile: Dockerfile restart: always volumes: - .:/workspace:cached From 115f989cffe36738374e91c53f388ab43742ff84 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:52:11 -0400 Subject: [PATCH 07/13] pin scipy --- src/cirrocumulus/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cirrocumulus/Dockerfile b/src/cirrocumulus/Dockerfile index 52073250..f12f3985 100644 --- a/src/cirrocumulus/Dockerfile +++ b/src/cirrocumulus/Dockerfile @@ -5,4 +5,6 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 # Install any needed packages specified in requirements.txt -RUN python -m pip install --upgrade pip && python -m pip install cirrocumulus +RUN python -m pip install --upgrade pip && \ + python -m pip install scipy==1.10.1 && \ + python -m pip install cirrocumulus From de58de4dec0b8949ae99a51b39d12386ec1bd265 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:16:37 -0400 Subject: [PATCH 08/13] Update .devcontainer.json --- src/cirrocumulus/.devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/.devcontainer.json b/src/cirrocumulus/.devcontainer.json index 6263e543..febdeaf2 100644 --- a/src/cirrocumulus/.devcontainer.json +++ b/src/cirrocumulus/.devcontainer.json @@ -13,7 +13,7 @@ ], // re-mount bucket files on container start up "postStartCommand": [ - "nohup bash -c '[[ ${templateOption:login} == true ]] && ./workspace/scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &'" + "nohup bash -c '[[ ${templateOption:login} == true ]] && ./scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &'" ], "features": { "ghcr.io/devcontainers/features/aws-cli@sha256:bbc9fd513c22e331953126c75ad7b2ed1f9044f1cd5890b7073b634810459b18": {}, From 14d0ae2eca5e4c9b1ddc93206fa0be503b299787 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:17:14 -0400 Subject: [PATCH 09/13] Update cirro_scanner.sh --- src/cirrocumulus/scripts/cirro_scanner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/scripts/cirro_scanner.sh b/src/cirrocumulus/scripts/cirro_scanner.sh index c09a4a26..3d198729 100755 --- a/src/cirrocumulus/scripts/cirro_scanner.sh +++ b/src/cirrocumulus/scripts/cirro_scanner.sh @@ -29,7 +29,7 @@ readonly -f create_dataset # Define the function to perform the scanning function scan_folders_and_create_datasets() { # Search for folders with name *.zarr and create cirro dataset if it doesn't exist - find /root/workspace -type d -name '*.zarr' | while read -r folder; do + find /root/workspace \( -type d -name '*.zarr' -o -type f -name '*.h5ad' \) | while read -r folder; do if [[ "$(matching_dataset_count "${folder}")" -eq "0" ]]; then create_dataset "${folder}" else From d43bea2b390a794209f3f9f7094aa7ec51fe80a2 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Mon, 4 Aug 2025 20:55:39 -0400 Subject: [PATCH 10/13] Update .devcontainer.json --- src/cirrocumulus/.devcontainer.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cirrocumulus/.devcontainer.json b/src/cirrocumulus/.devcontainer.json index febdeaf2..2ea1bb0f 100644 --- a/src/cirrocumulus/.devcontainer.json +++ b/src/cirrocumulus/.devcontainer.json @@ -12,9 +12,7 @@ "${templateOption:login}" ], // re-mount bucket files on container start up - "postStartCommand": [ - "nohup bash -c '[[ ${templateOption:login} == true ]] && ./scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &'" - ], + "postStartCommand": "chmod +x /workspace/scripts/cirro_scanner.sh && nohup /workspace/scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &", "features": { "ghcr.io/devcontainers/features/aws-cli@sha256:bbc9fd513c22e331953126c75ad7b2ed1f9044f1cd5890b7073b634810459b18": {}, "ghcr.io/dhoeric/features/google-cloud-cli@sha256:fa5d894718825c5ad8009ac8f2c9f0cea3d1661eb108a9d465cba9f3fc48965f": {}, From 760979277d69b0190a6d3d0f48affdef92e903af Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Thu, 21 Aug 2025 18:55:29 -0400 Subject: [PATCH 11/13] Update Dockerfile --- src/cirrocumulus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cirrocumulus/Dockerfile b/src/cirrocumulus/Dockerfile index f12f3985..587150a3 100644 --- a/src/cirrocumulus/Dockerfile +++ b/src/cirrocumulus/Dockerfile @@ -1,5 +1,5 @@ # Use an official Python runtime as a parent image -FROM mcr.microsoft.com/vscode/devcontainers/python:3.11-bookworm +FROM mcr.microsoft.com/vscode/devcontainers/python:3.12-bookworm ENV PYTHONDONTWRITEBYTECODE=1 ENV PYTHONUNBUFFERED=1 From 1669fa39267d9a44d2f5a267916ba2fa4dcd3a95 Mon Sep 17 00:00:00 2001 From: Yu Hu <93544261+yuhuyoyo@users.noreply.github.com> Date: Thu, 21 Aug 2025 19:58:30 -0400 Subject: [PATCH 12/13] Update Dockerfile --- src/cirrocumulus/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cirrocumulus/Dockerfile b/src/cirrocumulus/Dockerfile index 587150a3..84e775c5 100644 --- a/src/cirrocumulus/Dockerfile +++ b/src/cirrocumulus/Dockerfile @@ -6,5 +6,4 @@ ENV PYTHONUNBUFFERED=1 # Install any needed packages specified in requirements.txt RUN python -m pip install --upgrade pip && \ - python -m pip install scipy==1.10.1 && \ python -m pip install cirrocumulus From d6629a78d8b7fe5246974f4b8e8fc18a7cb68c70 Mon Sep 17 00:00:00 2001 From: Nicholas Bense Date: Fri, 22 Aug 2025 00:23:00 -0700 Subject: [PATCH 13/13] Update .devcontainer.json --- src/cirrocumulus/.devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cirrocumulus/.devcontainer.json b/src/cirrocumulus/.devcontainer.json index 2ea1bb0f..ee8eef1d 100644 --- a/src/cirrocumulus/.devcontainer.json +++ b/src/cirrocumulus/.devcontainer.json @@ -14,7 +14,6 @@ // re-mount bucket files on container start up "postStartCommand": "chmod +x /workspace/scripts/cirro_scanner.sh && nohup /workspace/scripts/cirro_scanner.sh > /root/.workbench/logs/cirro_scanner.log 2>&1 &", "features": { - "ghcr.io/devcontainers/features/aws-cli@sha256:bbc9fd513c22e331953126c75ad7b2ed1f9044f1cd5890b7073b634810459b18": {}, "ghcr.io/dhoeric/features/google-cloud-cli@sha256:fa5d894718825c5ad8009ac8f2c9f0cea3d1661eb108a9d465cba9f3fc48965f": {}, "ghcr.io/devcontainers/features/java@sha256:df67d6ff6e9cdd858207ae9e92a99ddb88384b789f79eecd6f873216e951d286": {} },