Feature: MPlot

.Trash-1005/info/01_dataset_artifact-output.ipynb.trashinfo

@@ -0,0 +1,3 @@
+[Trash Info]
+Path=nbs_pipeline/output/01_dataset_artifact-output.ipynb
+DeletionDate=2024-05-21T14:43:59

.Trash-1005/info/02b_encoder_MVP-output.ipynb.trashinfo

@@ -0,0 +1,3 @@
+[Trash Info]
+Path=nbs_pipeline/output/02b_encoder_MVP-output.ipynb
+DeletionDate=2024-05-21T14:44:04

.gitignore

@@ -28,7 +28,6 @@ tmp*
 tags
 
 #Macu
-nbs_pipeline/*.py
 app/shiny_logs
 ./mi-santamaria/* #No sé por qué se han copiado mis proyectos de wandb
 nbs_pipeline/test_torch.ipybn
@@ -141,6 +140,7 @@ ENV/
 .DS_Store
 .DS_Store?
 .Trashes
+.Trash-1004
 ehthumbs.db
 Thumbs.db
 .idea
@@ -165,4 +165,11 @@ checklink/cookies.txt
 /docker/.env 
 /docker_test/
 /docker/tmp_configComits.sh
-.Trash-1004
+quarto-linux-amd64.deb
+nbs_pipeline/logs_orelm.txt
+nbs_pipeline/images/
+nbs_pipeline/output/
+nbs_pipeline/calidad_aire_madrid/Anio*
+nbs_pipeline/calidad_aire_madrid_2001_2024.csv
+nbs_pipeline/horario_ensanche_vallecas.csv
+nbs_pipeline/calidad_aire_madrid

Makefile

@@ -3,14 +3,14 @@ SRC = $(wildcard ./*.ipynb)
 all: pacmel_mining_use_case docs
 
 pacmel_mining_use_case: $(SRC)
-	nbdev_build_lib
+	nbdev_export
 	touch pacmel_mining_use_case
 
 docs_serve: docs
 	cd docs && bundle exec jekyll serve
 
 docs: $(SRC)
-	nbdev_build_docs
+	nbdev_docs
 	touch docs
 
 test:

build_base_image.sh

@@ -1,20 +1,21 @@
+#!/bin/bash
 # Inicializa un array vacío
 args=()
 
 # Lee el archivo .env línea por línea
 IFS='='
 while IFS='=' read -r key value _; do
     value="${value%$'\r'}"
-    echo "key: $key"
-    echo "value: $value"
+    #echo "key: $key"
+    #echo "value: $value"
     if [[ $key != \#* && $key != '' ]]; then  # Excluye comentarios y líneas vacías
         args+=(--build-arg "$key=$value")  # Agrega --build-arg y la variable como un elemento
     else
         echo "Skipping $key = $value"
     fi
 done < docker/.env
 
-echo "Args:" "${args[@]}"
+#echo "Args:" "${args[@]}"
 
 
 # Ejecuta docker build con los argumentos
@@ -41,4 +42,4 @@ IMAGE_NAME=${PROJECT_NAME}-$DOCKER':'$VERSION
 
 
 echo "IMAGE_NAME: $IMAGE_NAME"
-docker build "${args[@]}" . -f ${DOCKERFILE} -t ${IMAGE_NAME}
+docker build "${args[@]}" . -f ${DOCKERFILE} -t ${IMAGE_NAME}

build_extra_image.sh

@@ -0,0 +1,19 @@
+DOCKER=$1
+VERSION=$2
+IMAGE_GOALS=$3 #hooks
+PROJECT_NAME=$(grep -oP '^COMPOSE_PROJECT_NAME=\K.*' ./docker/.env)
+USER_NAME=$(id -un)
+
+echo "DOCKER: $DOCKER | PROJECT_NAME: $PROJECT_NAME | USER_NAME: $USER_NAME | VERSION: $VERSION | IMAGE_GOALS: $IMAGE_GOALS"
+
+if [ "$DOCKER" = 'jupyter' ]; then
+    DOCKERFILE=./docker/Dockerfile.jupyter.${IMAGE_GOALS}
+    else 
+    DOCKERFILE=./docker/Dockerfile.rstudio.${IMAGE_GOALS}
+fi
+
+echo "Dockerfile: $DOCKERFILE"
+IMAGE_NAME=${PROJECT_NAME}-$DOCKER':'$VERSION-$IMAGE_GOALS
+
+echo "IMAGE_NAME: $IMAGE_NAME"
+docker build  . -f "${DOCKERFILE}" -t "${IMAGE_NAME}"

clean-jupyter-files.sh

@@ -14,4 +14,7 @@ for notebook in "$@"; do
     echo "Clean $notebook -->"
 done
 
-exit 0
+exit 0
+
+#Direct exec example:"
+#find nbs/ -name '*.ipynb' -not -name '.*'  -print0 | xargs -0 ./clean-jupyter-files.sh 

docker/DESCRIPTION

@@ -13,6 +13,10 @@ Depends:
     zoo,
     reticulate,
     xts,
+    plotly,
+    shinyFiles,
+    yaml,
+    xts,
     semantic.dashboard
 Remotes: 
     r-lib/later,

docker/Dockerfile.jupyter.hooks

@@ -0,0 +1,22 @@
+############################################
+# CONDA & MINICONDA WITH PYBASE DOCKERFILE #
+# For Jupyter service                      #
+############################################
+##############
+# Base image #
+##############
+##--- Setup Ubuntu
+FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04
+
+
+##--- Setup bash
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+#####################
+# Install miniconda #
+#####################
+
+RUN  source /usr/local/share/miniconda3/bin/activate env \
+     && mamba install -y -c conda-forge pre-commit \
+     && mamba clean --all --yes 
+

docker/Dockerfile.jupyter.mplots

@@ -0,0 +1,36 @@
+############################################
+# CONDA & MINICONDA WITH PYBASE DOCKERFILE #
+# For Jupyter service                      #
+############################################
+##############
+# Base image #
+##############
+##--- Setup Ubuntu
+FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks
+
+##--- Setup bash
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+##################
+# Install OCTAVE # 
+##################
+# Substitutes matlab as no inline installation found for matlab
+#RUN apt-get update \
+#    && apt-get install octave -y \
+#    && apt-get clean \
+    #&& rm -rf /var/lib/apt/lists/*
+
+#####################
+# Install miniconda #
+#####################
+
+RUN apt-get update && apt-get install -y libgl1-mesa-glx
+
+COPY docker/environment.mplots.yml /tmp/environment.mplots.yml
+
+RUN mamba env update --name env --file /tmp/environment.mplots.yml \
+    && mamba clean --all --yes
+
+# Failling the installation from environment for some reason 
+RUN mamba install aeon -c conda-forge -y \
+    && mamba clean --all --yes

docker/Dockerfile.rstudio

@@ -39,6 +39,9 @@ RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
 COPY --chown=${UID}:${GID} docker/entrypoint-rstudio.sh /usr/local/bin
 RUN chmod u+x /usr/local/bin/entrypoint-rstudio.sh
 
+RUN R -e "install.packages('semantic.dashboard', repos='http://cran.r-project.org')"
+
+
 WORKDIR $HOME
 
 # Initialize R-Stusio Server

docker/docker-compose.yml

@@ -73,6 +73,9 @@ services:
          - ../dvats:/home/${USER_NAME}/dvats
          - conda-env:/usr/local/share/miniconda3/envs/env
          - miniconda:/home/user/local/share/miniconda3 
+         # arreglar bien el error: /home/macu/work/nbs/mplot: No such file or directory
+         #- ../nbs/mplot/:/home/${USER_NAME}/work/nbs/mplot
+         ####
       deploy:
         resources:
           #limits:

docker/entrypoint-rstudio.sh

@@ -7,6 +7,11 @@ echo "conda activate ${ENV_PREFIX}" >> ~/.bashrc
 #echo "export WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.bashrc
 # echo "WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.Renviron
 
+#... added for fixing fails when rebuilding docker ...#
+### Ensuring to activate the correct conda
+source /usr/local/share/miniconda3/etc/profile.d/conda.sh
+conda activate /usr/local/share/miniconda3/envs/env
+
 # Define an array of environment variable names from the ENV_VARS Compose variable
 IFS=',' read -ra ENV_VAR_NAMES <<< "$ENV_VARS"
 

docker/entrypoint.sh

@@ -20,46 +20,13 @@ conda list | grep wandb
 # Extra pre-commit options #
 ############################
 
-########## Fix base.yaml for ensuring correct user & project directly ########## [TODO: Remove this section for directly using .env option]
-line_found=$(grep "^[[:space:]]*user: &wdb_user" /home/$USER/work/nbs_pipeline/config/base.yaml)
-
-if [ -z "$line_found" ]; then
-    echo "User line not found."
-    exit 1
-else
-    echo "User line found: '$line_found'"
-
-    modified_line="    user: &wdb_user $WANDB_ENTITY"
-    echo "--> Modified to: '    $modified_line'"
-
-    sed -i "s|^\([[:space:]]*user: &wdb_user\).*|\1 $WANDB_ENTITY|" /home/$USER/work/nbs_pipeline/config/base.yaml
-
-
-    grep "user: &wdb_user" /home/$USER/work/nbs_pipeline/config/base.yaml
-fi
-
-line_found=$(grep "^[[:space:]]*project_name: &wdb_project" /home/$USER/work/nbs_pipeline/config/base.yaml)
-
-if [ -z "$line_found" ]; then
-    echo "Project line not found."
-    exit 1
-else
-    echo "Project line found: '$line_found'"
-
-    modified_line="    project_name: &wdb_project $WANDB_PROJECT"
-    echo "--> Modify to: '    $modified_line'"
-
-    sed -i "s|^\([[:space:]]*project_name: &wdb_project\).*|\1 $WANDB_PROJECT|" /home/$USER/work/nbs_pipeline/config/base.yaml
-
-    grep "project_name: &wdb_project" /home/$USER/work/nbs_pipeline/config/base.yaml
-fi
+#Check
+conda list | grep wandb
+###
 
-[ -d "/home/$USER/data/wandb_artifacts" ] || mkdir -p "/home/$USER/data/wandb_artifacts"
 
 
-if ! grep -Fxq "./path/to/check_yml_changes.sh" $HOME/work/.git/hooks/pre-commit; then \
-        sed -i '$i./path/to/check_yml_changes.sh' $HOME/work/.git/hooks/pre-commit; \
-    fi
+echo "Aqui"
 
 exec "$@"
 

docker/environment.mplots.yml

@@ -0,0 +1,15 @@
+name: env
+channels: 
+- conda-forge
+
+dependencies:
+  #-- Mplots
+  - pyscamp-gpu=4.0.1=py310h43b6314_1
+  - stumpy=1.12.0=pyhd8ed1ab_0
+  - ipympl=0.9.3=pyhd8ed1ab_0
+  - graph
+  #-- Octave 4 Eamonn's MPlots (expecting the python code :D)
+  - octave
+  - oct2py
+  - fltk
+  - aeon