diff --git a/README.md b/README.md index e0e503f6..c36505a3 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ GH_TOKEN=your_github_pat RSTUDIO_PORT= # Password to access the Rstudio server RSTUDIO_PASSWD= +# Port in which you want shiny server to be deployed (for running the app) +SHINY_PORT= ``` Finally, in a terminal located in the folder `docker` of this repository, run: diff --git a/docker/Dockerfile.shiny b/docker/Dockerfile.shiny new file mode 100644 index 00000000..cd7ab95a --- /dev/null +++ b/docker/Dockerfile.shiny @@ -0,0 +1,93 @@ +############################# +# SHINY-SERVER # +# VOLUME DOCKERFILE # +############################# +# Set up the shiny server # +# for using deepVATS App # +############################# +FROM misantamaria/dvats-rstudio:rocker-ml4.2 +SHELL [ "/bin/bash", "--login", "-c" ] + +# Add r-studio server user +ARG USER=user \ + UID=1000 \ + GID=1000 + +ENV USER=$USER \ + UID=$UID \ + GID=$GID + +# Paths +ENV HOME=/home/$USER \ + ENV_PREFIX=$HOME/env \ + APP=$HOME/app + +# Ensure files and packages +COPY --chown=app:app docker/entrypoint-rstudio.sh /usr/local/bin +COPY --chown=app:app r_shiny_app $APP + +# Create groups and users +RUN addgroup --gid $GID $USER \ + && addgroup --system app \ + && addgroup --system shared \ + && adduser --disabled-password \ + --gecos "Non-root user" \ + --uid $UID \ + --gid $GID \ + --home $HOME \ + $USER \ + && adduser --system --ingroup app app \ + && usermod -aG shared $USER \ + && usermod -aG shared app + +# Agregar usuarios al grupo sudo después de crearlos +RUN adduser $USER sudo \ + && usermod -aG sudo app \ + && echo $USER "ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER \ + && echo "app ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/app \ + && chmod 0440 /etc/sudoers.d/$USER \ + && chmod 0440 /etc/sudoers.d/app + +ARG LOG_DIR=/var/log/shiny-server +ENV LOG_DIR=${LOG_DIR} \ + LOG_FILE=${LOG_DIR}/app.logs +COPY --chown=app:app srv/shinylog $LOG_DIR + +# Fix ownership and permissions for the home directory +RUN chown -R $USER:$USER $HOME \ + && chmod -R 775 $HOME \ + && chown -R app:app $HOME \ + && chmod -R 775 $HOME + +# Ensure the log directory and file exist with correct permissions +RUN mkdir -p $LOG_DIR \ + && touch $LOG_FILE \ + && chown -R app:app $LOG_DIR \ + && chmod -R 775 $LOG_DIR + +RUN R -e "install.packages('semantic.dashboard', repos='http://cran.r-project.org')"\ + && R -e "install.packages('randomcoloR', repos='http://cran.r-project.org')" + +### Run server +ARG SHINY_PORT +ENV SHINY_PORT=${SHINY_PORT} + +RUN chown -R app:app $APP +RUN chown -R app:app $LOG_DIR +RUN touch $LOG_FILE +RUN chown -R app:app $LOG_DIR + +RUN [ -f ${LOG_DIR} ] && echo "${LOG_DIR} is a file" || echo "${LOG_DIR} is a directory" + +#RUN ls -la $LOG_DIR && sleep 3 +ARG SHINY_PORT +ENV SHINY_PORT=$SHINY_PORT +EXPOSE ${SHINY_PORT} +EXPOSE 3838 + +COPY --chown=app:app docker/entrypoint-shiny.sh /usr/local/bin/entrypoint-shiny.sh +RUN chmod u+x /usr/local/bin/entrypoint-shiny.sh +USER app +RUN touch $LOG_FILE +RUN ls -la $LOG_DIR && sleep 3 +CMD ["/usr/local/bin/entrypoint-shiny.sh"] \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 84ccecc1..1bec56c9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -27,6 +27,7 @@ services: - ${LOCAL_DATA_PATH}:/home/${USER_NAME}/data/ - conda-env:/usr/local/share/miniconda3/envs/env - miniconda:/usr/local/share/miniconda3 + - lib:/usr/local/share/lib init: true stdin_open: true tty: true @@ -44,8 +45,10 @@ services: dockerfile: docker/Dockerfile.rstudio args: - WANDB_API_KEY=${WANDB_API_KEY} # - - RETICULATE_PYTHON_ENV=/home/env - - RETICULATE_MINICONDA_PATH=/home/miniconda3 + #- RETICULATE_PYTHON_ENV=/home/env + #- RETICULATE_MINICONDA_PATH=/home/miniconda3 + - RETICULATE_PYTHON_ENV=/usr/local/share/miniconda3/envs/env + - RETICULATE_MINICONDA_PATH=/usr/local/share/miniconda3 - USER=${USER_NAME} #* - UID=${USER_ID} #* - GID=${GROUP_ID} #* @@ -61,9 +64,7 @@ services: - GROUPID=${GROUP_ID} #* - PASSWORD=${RSTUDIO_PASSWD} #* - ROOT=FALSE - - CUDA_VISIBLE_DEVICES=0,1,2 - #- CUDA_VISIBLE_DEVICES=1 - # - CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} + - CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} - GH_TOKEN=${GH_TOKEN} #* # TODO (28/03/2023): I don't know why it is not working without this - ENV_VARS=WANDB_ENTITY,WANDB_API_KEY,WANDB_PROJECT,USER,USERID,GROUPID,PASSWORD,ROOT,CUDA_VISIBLE_DEVICES @@ -72,7 +73,8 @@ services: - ${LOCAL_DATA_PATH}:/home/${USER_NAME}/data/ - ../dvats:/home/${USER_NAME}/dvats - conda-env:/usr/local/share/miniconda3/envs/env - - miniconda:/home/user/local/share/miniconda3 + - miniconda:/usr/local/share/miniconda3 + - lib:/usr/local/share/lib deploy: resources: #limits: @@ -82,7 +84,51 @@ services: - driver: nvidia count: all capabilities: [gpu] - + shiny-server: + build: + context: ../ + dockerfile: docker/Dockerfile.shiny + args: + - WANDB_API_KEY=${WANDB_API_KEY} + - RETICULATE_PYTHON_ENV=/usr/local/share/miniconda3/envs/env + - RETICULATE_MINICONDA_PATH=/usr/local/share/miniconda3 + - USER=${USER_NAME} #* + - UID=${USER_ID} #* + - GID=${GROUP_ID} #* + - SHINY_PORT=${SHINY_PORT} + image: dvats-shiny:${USER_NAME} + ports: + - "${SHINY_PORT}:3838" #* + expose: + - "8888" + environment: + - WANDB_ENTITY=${WANDB_ENTITY} + - WANDB_API_KEY=${WANDB_API_KEY} + - WANDB_PROJECT=${WANDB_PROJECT} + - USER=${USER_NAME} #* + - USERID=${USER_ID} #* + - GROUPID=${GROUP_ID} #* + - PASSWORD=${SHINY_PASSWD} #* + - ROOT=FALSE + - CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} + - GH_TOKEN=${GH_TOKEN} #* + # TODO (28/03/2023): I don't know why it is not working without this + - ENV_VARS=WANDB_ENTITY,WANDB_API_KEY,WANDB_PROJECT,USER,USERID,GROUPID,PASSWORD,ROOT,CUDA_VISIBLE_DEVICES + volumes: + - ../r_shiny_app:/home/${USER_NAME}/app + - ${LOCAL_DATA_PATH}:/home/${USER_NAME}/data/ + - ../dvats:/home/${USER_NAME}/dvats + - conda-env:/usr/local/share/miniconda3/envs/env + - miniconda:/usr/local/share/miniconda3 + - lib:/usr/local/share/lib + - ../srv/shinylog:/var/log/shiny-server + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] volumes: conda-env: miniconda: diff --git a/docker/entrypoint-shiny.sh b/docker/entrypoint-shiny.sh new file mode 100755 index 00000000..e17c9c40 --- /dev/null +++ b/docker/entrypoint-shiny.sh @@ -0,0 +1,38 @@ +#!/bin/bash --login +set -e + +[ -f ${LOG_DIR} ] && echo "${LOG_DIR} is a file" || echo "${LOG_DIR} is a directory" +[ -f ${LOG_FILE} ] && echo "${LOG_FILE} is a file" || echo "${LOG_FILE} is a directory" + +echo ". ${HOME}/miniconda3/etc/profile.d/conda.sh" >> ${HOME}/.bashrc +# Make bash automatically activate the conda environment +echo "conda activate ${ENV_PREFIX}" >> ${HOME}/.bashrc + +#... added for fixing fails when rebuilding docker ...# +### Ensuring to activate the correct conda +source /usr/local/share/miniconda3/etc/profile.d/conda.sh +conda activate /usr/local/share/miniconda3/envs/env + +# Define an array of environment variable names from the ENV_VARS Compose variable +IFS=',' read -ra ENV_VAR_NAMES <<< "$ENV_VARS" + +echo "ENV_VAR_NAMES=${ENV_VAR_NAMES[@]}" + +# Loop through the array of environment variable names and set the variables +for ENV_VAR_NAME in "${ENV_VAR_NAMES[@]}"; do + ENV_VAR_VALUE="${!ENV_VAR_NAME:-default}" + echo "$ENV_VAR_NAME=$ENV_VAR_VALUE" >> ${HOME}/.Renviron +done + +ulimit -s 16384 +echo $LOG_FILE +#sudo chown app:app "$LOG_FILE" +sudo chown -R $UID:shared "$LOG_DIR" +sudo chmod -R 775 /var/log/shiny-server + +ls -la $LOG_DIR + +echo "DEBUG: Writing to ${LOG_FILE}" >> ${LOG_FILE} +exec >> $LOG_FILE 2>&1 + +R --quiet -e "shiny::runApp('${APP}', host='0.0.0.0', port=3838)" \ No newline at end of file diff --git a/srv/shinylog/app.logs b/srv/shinylog/app.logs new file mode 100755 index 00000000..664be5f0 --- /dev/null +++ b/srv/shinylog/app.logs @@ -0,0 +1,391 @@ +DEBUG: Writing to /var/log/shiny-server/app.logs +> shiny::runApp('/home/app', host='0.0.0.0', port=3838) +Loading required package: shiny +Warning in shiny::runApp("/home/app", host = "0.0.0.0", port = 3838) : + Shiny Server v0.3.4 or later is required; please upgrade! + +Attaching package: ‘shinyjs’ + +The following object is masked from ‘package:shiny’: + + runExample + +The following objects are masked from ‘package:methods’: + + removeClass, show + + +Attaching package: ‘jsonlite’ + +The following object is masked from ‘package:purrr’: + + flatten + +The following object is masked from ‘package:shiny’: + + validate + + +Attaching package: ‘data.table’ + +The following object is masked from ‘package:purrr’: + + transpose + + +Attaching package: ‘dplyr’ + +The following objects are masked from ‘package:data.table’: + + between, first, last + +The following objects are masked from ‘package:stats’: + + filter, lag + +The following objects are masked from ‘package:base’: + + intersect, setdiff, setequal, union + + +Attaching package: ‘shinyWidgets’ + +The following object is masked from ‘package:shinyjs’: + + alert + + +Attaching package: ‘arrow’ + +The following objects are masked from ‘package:feather’: + + read_feather, write_feather + +The following object is masked from ‘package:utils’: + + timestamp + +Loading required package: zoo + +Attaching package: ‘zoo’ + +The following objects are masked from ‘package:base’: + + as.Date, as.Date.numeric + + +######################### Warning from 'xts' package ########################## +# # +# The dplyr lag() function breaks how base R's lag() function is supposed to # +# work, which breaks lag(my_xts). Calls to lag(my_xts) that you type or # +# source() into this session won't work correctly. # +# # +# Use stats::lag() to make sure you're not using dplyr::lag(), or you can add # +# conflictRules('dplyr', exclude = 'lag') to your .Rprofile to stop # +# dplyr from breaking base R's lag() function. # +# # +# Code in packages is not affected. It's protected by R's namespace mechanism # +# Set `options(xts.warn_dplyr_breaks_lag = FALSE)` to suppress this warning. # +# # +############################################################################### + +Attaching package: ‘xts’ + +The following objects are masked from ‘package:dplyr’: + + first, last + +The following objects are masked from ‘package:data.table’: + + first, last + +[1] "CUDA AVAILABLE. Num devices: 2" +[1] "" +[1] "--> py_config " +python: /usr/local/share/miniconda3/envs/env/bin/python +libpython: /usr/local/share/miniconda3/envs/env/lib/libpython3.10.so +pythonhome: /usr/local/share/miniconda3/envs/env:/usr/local/share/miniconda3/envs/env +version: 3.10.13 | packaged by conda-forge | (main, Oct 26 2023, 18:07:37) [GCC 12.3.0] +numpy: /usr/local/share/miniconda3/envs/env/lib/python3.10/site-packages/numpy +numpy_version: 1.26.4 +torch: /usr/local/share/miniconda3/envs/env/lib/python3.10/site-packages/torch + +NOTE: Python version was forced by RETICULATE_PYTHON +[1] "py_config -->" +[1] "Querying encoders" +[1] "Done!" + +Listening on http://0.0.0.0:3838 +[1] "observeEvent encoders list enc_l | update dataset list | after freeze" +[1] "observeEvent encoders list encs_l | update dataset list -->" +[1] "--> observeEvent input_encoder | update wlen" +[1] "observeEvent input_encoder | update wlen | Before enc_ar" +[1] "--> Observe clustering options" +[1] "Observe clustering options -->" +[1] "--> observeEvent zoom_btn" +[1] "--> embedding idx" +[1] "embedding idx -->" +[1] "--> observeEvent input_dataset | update encoder list" +[1] "mi-santamaria/deepvats/gtrends_khols-normalized_yearly:v0" +[1] "observeEvent input_dataset | update encoders for dataset mi-santamaria/deepvats/gtrends_khols-normalized_yearly:v0" +[1] "observeEvent input_dataset | update encoder list -->" +[1] "--> observeEvent input_encoder | update wlen" +[1] "observeEvent input_encoder | update wlen | Before enc_ar" +[1] "eventReactive enc_ar | Enc. Artifact: mi-santamaria/deepvats/zeroshot-moment-small-embedding:v0" +[1] "envent reactive enc_ar -->" +[1] "observeEvent input_encoder | update wlen | enc_ar: " +[1] "observeEvent input_encoder | update wlen | Set wlen slider values" +[1] "observeEvent input_encoder | update wlen | enc_ar$metadata$mvp_ws 12" +[2] "observeEvent input_encoder | update wlen | enc_ar$metadata$mvp_ws 17" +[1] "observeEvent input_encoder | update wlen | Update slider input (12, 17 ) -> 17" +[1] "--> reactive enc_ar_stride" +[1] "reactive_enc_ar_stride | --> 9" +[1] "eventReactive ts_ar -->" +[1] "--> Reactive tsdf | ts artifact " +[1] "Reactive tsdf | Read feather /home/macu/data/wandb_artifacts/-942329478971956879" +[1] "Reactive tsdf | Read feather | Execution time: 0.0432560443878174 seconds" +[1] "Reactive tsdf | Column to index | Execution time: 0.0439755916595459 seconds" +[1] "--> observeEvent tsdf | update select variables" +[1] "observeEvent tsdf | select variables volume" +[2] "observeEvent tsdf | select variables timeindex" +[1] "--> observeEvent tsdf | update select variables -->" +[1] "--> observeEvent input_wlen | update slide stride value | wlen 17" +[1] "enc_ar_stride: 9" +[1] "oserveEvent input_wlen | update slide stride value | Update stride to 9" +[1] "Check reactiveness | X | wlen, stride |" +[1] "--> ReactiveVal X | Update Sliding Window" +[1] "reactive X | wlen 17 | stride 9 | Let's prepare data" +[1] "reactive X | SWV" +[1] "reactive X | SWV: 0.0104906558990479 secs " +[1] "reactive X | Update sliding window | Apply stride 9 | enc_input ~ 423-->" +[2] "reactive X | Update sliding window | Apply stride 9 | enc_input ~ 1-->" +[3] "reactive X | Update sliding window | Apply stride 9 | enc_input ~ 17-->" +[1] "| Update | X" +[1] "| Outside| X" +[1] "--> ts_plot | Before req 1" +[1] "--> ts_plot_base" +[1] "ts_plot_base | start_date: 2004-01-01 end_date: 2012-05-31" +[1] "ts_plot_base | tsdf_xts time0.0116572380065918" + volume +2004-01-01 0.09091226 +2004-01-08 0.09091226 +2004-01-15 0.09091226 +2004-01-22 0.00000000 +2004-01-29 0.00000000 +2004-02-05 0.09091226 + volume +2012-04-26 0.06896603 +2012-05-03 0.12069014 +2012-05-10 0.10344822 +2012-05-17 0.05172411 +2012-05-24 0.06896603 +2012-05-31 0.12069014 +[1] "ts_plot_base -->" +[1] "ts_plot | bp" +[1] "ts_plot | embedings idxs " +[1] "--> embedding idx" +[1] "--> eventReactive enc | load encoder " +py function: prepare_forecasting_data : +--> Exec with feather | reading input from /home/macu/data/wandb_artifacts/-942329478971956879 +--> Exec with feather | Apply function /home/macu/data/wandb_artifacts/-942329478971956879 +Exec with feather | time: 0.008246898651123047 +Exec with feather --> /home/macu/data/wandb_artifacts/-942329478971956879 +[1] "embedding idx -->" +[1] "ts_plot -->" +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +[1] "--> Projections_plot" +[1] "--> Projections" +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +[1] "Check reactiveness | X | wlen, stride |" +[1] "--> Projections_plot" +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +[1] "--> Projections_plot" +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp +Warning: Error in : ModuleNotFoundError: No module named 'momentfm' + + 234: + 233: stop + 232: py_call_impl + 231: pickle$load + 230: py_load_object + 229: eventReactiveValueFunc [/home/app/server.R#401] + 185: enc + 178: [/home/app/server.R#414] + 162: embs + 155: [/home/app/server.R#513] + 139: prj_object + 137: [/home/app/server.R#711] + 121: embedding_ids + 120: [/home/app/server.R#766] + 104: ts_plot + 97: :: +htmlwidgets +shinyRenderWidget [/home/app/server.R#1072] + 96: func + 83: renderFunc + 82: output$ts_plot_dygraph + 1: shiny::runApp