aws-samples · KeitaW · Aug 5, 2024 · Aug 6, 2024 · Aug 23, 2024 · Aug 23, 2024
diff --git a/3.test_cases/14.bionemo/0.Dockerfile b/3.test_cases/14.bionemo/0.Dockerfile
@@ -1,9 +1,11 @@
-FROM nvcr.io/nvidia/clara/bionemo-framework:latest
+FROM nvcr.io/nvidia/clara/bionemo-framework:1.7
 
-ARG EFA_INSTALLER_VERSION=1.30.0
-ARG AWS_OFI_NCCL_VERSION=v1.7.4-aws
+ARG EFA_INSTALLER_VERSION=1.33.0
+ARG AWS_OFI_NCCL_VERSION=v1.9.2-aws
+ARG GDRCOPY_VERSION=v2.4.1
 ARG NCCL_TESTS_VERSION=master
-ARG NCCL_VERSION=v2.18.6-1
+ARG NCCL_VERSION=v2.21.5-1
+
 RUN apt-get update -y
 RUN apt-get remove -y --allow-change-held-packages \
     libmlx5-1 ibverbs-utils libibverbs-dev libibverbs1 libnccl2 libnccl-dev
@@ -34,21 +36,25 @@ RUN mkdir -p /var/run/sshd
 RUN sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config && \
     echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
     sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/lib:$LD_LIBRARY_PATH
-ENV PATH /opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:$PATH
+ENV LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/lib:$LD_LIBRARY_PATH
+ENV PATH=/opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:$PATH
 RUN curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
- && python3 /tmp/get-pip.py \
+    && python3 /tmp/get-pip.py \
     && pip3 install awscli pynvml
 
 #################################################
-# Install NVIDIA GDRCopy
-RUN git clone https://github.com/NVIDIA/gdrcopy.git /opt/gdrcopy \
-    && cd /opt/gdrcopy \
-    && make lib_install install \
-    && cd /opt/gdrcopy/tests \
-    && make \
-    && make install \
-    && mv gdrcopy_copylat gdrcopy_copybw gdrcopy_sanity gdrcopy_apiperf /usr/bin/
+## Install NVIDIA GDRCopy
+##
+## NOTE: if `nccl-tests` or `/opt/gdrcopy/bin/sanity -v` crashes with incompatible version, ensure
+## that the cuda-compat-xx-x package is the latest.
+RUN git clone -b ${GDRCOPY_VERSION} https://github.com/NVIDIA/gdrcopy.git /tmp/gdrcopy \
+    && cd /tmp/gdrcopy \
+    && make prefix=/opt/gdrcopy install
+
+ENV LD_LIBRARY_PATH=/opt/gdrcopy/lib:/usr/local/cuda/compat:$LD_LIBRARY_PATH
+ENV LIBRARY_PATH=/opt/gdrcopy/lib:/usr/local/cuda/compat/:$LIBRARY_PATH
+ENV CPATH=/opt/gdrcopy/include:$CPATH
+ENV PATH=/opt/gdrcopy/bin:$PATH
 
 #################################################
 ## Install EFA installer
@@ -59,21 +65,32 @@ RUN cd $HOME \
     && ./efa_installer.sh -y -g -d --skip-kmod --skip-limit-conf --no-verify \
     && rm -rf $HOME/aws-efa-installer
 
+###################################################
+## Install NCCL
+RUN git clone -b ${NCCL_VERSION} https://github.com/NVIDIA/nccl.git  /opt/nccl \
+    && cd /opt/nccl \
+    && make -j $(nproc) src.build CUDA_HOME=/usr/local/cuda \
+    NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_86,code=sm_86 -gencode=arch=compute_89,code=sm_89 -gencode=arch=compute_90,code=sm_90"
+
+
 ###################################################
 ## Install AWS-OFI-NCCL plugin
-RUN apt-get install libtool autoconf cmake nasm unzip pigz parallel nfs-common build-essential hwloc libhwloc-dev libjemalloc2 libnuma-dev numactl libjemalloc-dev preload htop iftop liblapack-dev libgfortran5 ipcalc wget curl devscripts debhelper check libsubunit-dev fakeroot pkg-config dkms -y
-RUN export OPAL_PREFIX="" \
-    && git clone https://github.com/aws/aws-ofi-nccl.git /opt/aws-ofi-nccl \
-    && cd /opt/aws-ofi-nccl \
-    && git checkout ${AWS_OFI_NCCL_VERSION} \
-    && ./autogen.sh \
-    && ./configure --prefix=/opt/aws-ofi-nccl \
-    --with-libfabric=/opt/amazon/efa \
-    --with-cuda=/usr/local/cuda \
-    --with-nccl=/opt/nccl/build \
-    --with-mpi=/opt/amazon/openmpi \
-    --enable-platform-aws
-    && make && make install
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libhwloc-dev
+#Switch from sh to bash to allow parameter expansion
+SHELL ["/bin/bash", "-c"]
+RUN curl -OL https://github.com/aws/aws-ofi-nccl/releases/download/${AWS_OFI_NCCL_VERSION}/aws-ofi-nccl-${AWS_OFI_NCCL_VERSION//v}.tar.gz \
+    && tar -xf aws-ofi-nccl-${AWS_OFI_NCCL_VERSION//v}.tar.gz \
+    && cd aws-ofi-nccl-${AWS_OFI_NCCL_VERSION//v} \
+    && ./configure --prefix=/opt/aws-ofi-nccl/install \
+        --with-mpi=/opt/amazon/openmpi \
+        --with-libfabric=/opt/amazon/efa \
+        --with-cuda=/usr/local/cuda \
+        --enable-platform-aws \
+    && make -j $(nproc) \
+    && make install \
+    && cd .. \
+    && rm -rf aws-ofi-nccl-${AWS_OFI_NCCL_VERSION//v} \
+    && rm aws-ofi-nccl-${AWS_OFI_NCCL_VERSION//v}.tar.gz
 
 ###################################################
 ## Install NCCL-tests
@@ -87,16 +104,10 @@ RUN git clone https://github.com/NVIDIA/nccl-tests.git /opt/nccl-tests \
     NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_86,code=sm_86 -gencode=arch=compute_90,code=sm_90"
 
 
-
 RUN rm -rf /var/lib/apt/lists/*
-ENV LD_PRELOAD /opt/nccl/build/lib/libnccl.so
 
 
 ##############################################
-## BioNemo dependencies
-COPY requirements.txt /workspace/
-RUN pip3 install -r /workspace/requirements.txt
-
 COPY prepare_uniref50.py /workspace/bionemo
 
 WORKDIR /workspace/bionemo/
diff --git a/3.test_cases/14.bionemo/README.md b/3.test_cases/14.bionemo/README.md
@@ -14,134 +14,9 @@ NVIDIA BioNeMo is a domain-specific machine learning framework for training and
 | 6 |      [ProtT5nv](https://docs.nvidia.com/bionemo-framework/latest/models/prott5nv.html)       |
 
 
-This project provides a guide to run [Nvidia's BioNemo](https://docs.nvidia.com/bionemo-framework/latest/index.html) on AWS ParallelCluster and pretrain the popular [ESM models](https://github.com/facebookresearch/esm) specifically the [ESM1nv](https://docs.nvidia.com/bionemo-framework/latest/notebooks/model_training_esm1nv.html) model.
+This project provides a guide to run [Nvidia's BioNemo](https://docs.nvidia.com/bionemo-framework/latest/index.html) and pretrain the popular [ESM models](https://github.com/facebookresearch/esm) specifically the [ESM1nv](https://docs.nvidia.com/bionemo-framework/latest/notebooks/model_training_esm1nv.html) model. We provide guides for Slurm (Kubernetes guide is coming soon!). For detailed instructions, proceed to the [slurm](slurm) or [kubernetes](kubernetes) subdirectory.
 
 
-## 0. Prerequisites
-
-0. You have access to the bionemo container. To get the access to BioNeMo, visit the [information website](https://www.nvidia.com/en-us/clara/bionemo/).
-
-1. Have a slurm based AWS ParallelCluster created with a FSx for Lustre filesystem mounted. Below we are presenting instructions for a cluster with compute nodes instantiated with an Ubuntu based AMI.
-
-## 1. Install Nvidia Container CLI
-
-### 1.1 If you have created your cluster with the AWS ParallelCluster Base AMI or [DLAMI](https://aws.amazon.com/machine-learning/amis/) or your custom AMI, please make sure `libnvidia-container cli` is installed. You can follow the instructions below to install it.   
-
-### 1.2 To install libnvidia-container cli:
-We need [libnvidia-container cli](https://github.com/NVIDIA/libnvidia-container) to train models in an Nvidia container. We follow the instructions [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). This installation needs to be done in each compute node.
-
-```
-curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
-  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
-    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
-    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list \
-  && \
-    sudo apt-get update \
-  && sudo apt-get install libnvidia-container1 \
-  && sudo apt-get install libnvidia-container-tools
-```
-### 1.3 You can set the Nemo Multimodal version and others as environment variables:
-
-SSH into the head node of your cluster and run:
-
-```
-export PYTHON_VERSION=3.10
-# We are using Python version 3.10 in this work. For a different Python version select the right Miniconda file from https://repo.anaconda.com/miniconda/
-export MINICONDA_INSTALLER=Miniconda3-py310_23.5.2-0-Linux-x86_64
-export TARGET_PATH=/apps/bionemo-src   # Must be a shared filesystem. This is where Nemo launcher scripts will reside.
-export DOCKER_IMAGE_NAME=bionemo
-export TAG=latest
-export ENROOT_IMAGE=/apps/${DOCKER_IMAGE_NAME}
-export DATASET_PATH=/fsx/
-```
-
-## 1.4. Pull this github repo
-
-```bash
-cd /apps/
-git clone https://github.com/aws-samples/awsome-distributed-training.git
-cp -r /apps/awsome-distributed-training/3.test_cases/14.bionemo/* ./apps/
-```
-
-## 2. Pull Image
-
-```bash
-cd /apps/
-docker pull nvcr.io/nvidia/clara/bionemo-framework:1.2
-```
-
-## 3. Create Conda env
-We need a conda environment that has the necessary dependencies for submitting multiple arrays of slurm jobs via [HYDRA](https://github.com/facebookresearch/hydra) which NeMo uses to configuring both NeMo models and the PyTorch Lightning Trainer. 
-```
-# Miniconda is already installed if you are using the DLAMI but needs installation with Base AMI
-
-wget -O miniconda.sh "https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}.sh" \
-    && bash miniconda.sh -b -p /apps/.conda \
-          &&  /apps/.conda/bin/conda init bash  
-
-source ~/.bashrc    
-conda create --name bionemo python=${PYTHON_VERSION}
-
-source activate bionemo
-
-pip3 install -r requirements.txt
-
-```
-All package versions in the above `requirements.txt` file is recommended from Nvidia. An older version of the package `opencv-python-headless==4.8.0.74` has to be installed to avoid this [error](https://github.com/rom1504/img2dataset/issues/355) with [img2dataset](https://github.com/rom1504/img2dataset) package.
-
-
-
-## 4. Build customized docker image
-To achieve target performance of Nemo-Multimodal with EFA on P5 and P4de instances, we provide a customized 
-`3.test_cases/14.nemo-multimodal/0.Dockerfile` and we can build a image like below:
-
-```
-docker build -t ${DOCKER_IMAGE_NAME}:${TAG} -f 0.Dockerfile .
-```
-
-## 5. Convert image
-Convert the Docker container image to an [Enroot](https://github.com/NVIDIA/enroot) squash file that will be stored in `/apps`. This step takes a few minutes.
-```
-enroot import -o ${ENROOT_IMAGE}.sqsh dockerd://${DOCKER_IMAGE_NAME}
-
-```
-
-## 6. Download and preprocess data
-We will use the popular [UniRef50](https://www.uniprot.org/help/uniref) dataset for pretraining. We will use BioNemo's in-built functionality to download and pre-process data. To this end, we provide `prepare_uniref50.py` file to do so. You can edit the above to download and process [UniRef90]((https://www.uniprot.org/help/uniref)). To run the above python code on your slurm cluster in the BioNemo cluster execute the following:
-
-```bash
-sbatch 1.uniref50.slurm
-```
-
-This will download raw data in `/fsx/raw/` and save pre-processed `train, validation and test` csv files in `/fsx/processed/`. The log files for submitted jobs are written to the local directory. To check the status of the datasets download job, you can tail the log file:
-
-```bash
-tail -f slurm-uniref-<slurm_job_id>.out
-```
-
-
-
-## 7. Pretrain ESM models
-Now we are ready to submit distributed training jobs to pretrain `ESM1nv` models. We provide the `2.esm1nv_pretrain.slurm` script to run training 4 `p4de.24xlarge` nodes with `8xA100 80 GB` GPUs. Make sure data paths and model configuration is correct if you are running on custom data. To kick off distributed training execute:
-
-```bash
-sbatch 2.esm1nv_pretrain.slurm
-
-```
-
-Before kicking off training, first train, validation and test datasets are indexed and dataloaders are created and then you should see an example output like below:
-
-```bash
-Epoch 0:   3%|▎         | 34103/1100000 [5:28:58<171:22:21,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.510, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
-Epoch 0:   3%|▎         | 34106/1100000 [5:29:00<171:22:19,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
-Epoch 0:   3%|▎         | 34109/1100000 [5:29:02<171:22:09,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
-Epoch 0:   3%|▎         | 34112/1100000 [5:29:03<171:22:00,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
-```
-
-## 8. Run container on Head Node [Troubleshooting]
-Once the above image is pulled, you can run the container on the head node like below. This step could be used for troubleshooting purposes. Here we are running the container just to be able to copy launcher scripts on the host machine. If you need to run the container on the compute nodes, you would need to add `--gpus all` flag to the run command. It is recommended to have the docker run flags like below, as recommended by Nvidia PyTorch containers, otherwise you may potentially run into an error like [this](https://github.com/NVIDIA/Megatron-LM/issues/516)
-
-```
- docker run -it nvcr.io/nvidia/clara/bionemo-framework:latest bash
-```
+## Prerequisites
 
+You must have access to the bionemo container. To get the access to BioNeMo, visit the [information website](https://www.nvidia.com/en-us/clara/bionemo/).
diff --git a/3.test_cases/14.bionemo/1.uniref50.slurm → ...t_cases/14.bionemo/slurm/1.uniref50.slurm b/3.test_cases/14.bionemo/1.uniref50.slurm → ...t_cases/14.bionemo/slurm/1.uniref50.slurm
diff --git a/..._cases/14.bionemo/2.esm1nv_pretrain.slurm → .../14.bionemo/slurm/2.esm1nv_pretrain.slurm b/..._cases/14.bionemo/2.esm1nv_pretrain.slurm → .../14.bionemo/slurm/2.esm1nv_pretrain.slurm
diff --git a/3.test_cases/14.bionemo/slurm/README.md b/3.test_cases/14.bionemo/slurm/README.md
@@ -0,0 +1,111 @@
+## 0. Prerequisites
+
+The guide assumes that you have the following:
+
+* A functional Slurm cluster on AWS, whose compute instances are based on DeepLearning AMI.
+* An FSx for Lustre filesystem mounted on `/fsx`.
+* `enroot` if you want to run the container example.
+
+We recommend that you setup a Slurm cluster using the templates in the architectures [directory](../../1.architectures). 
+
+
+```
+export PYTHON_VERSION=3.10
+# We are using Python version 3.10 in this work. For a different Python version select the right Miniconda file from https://repo.anaconda.com/miniconda/
+export MINICONDA_INSTALLER=Miniconda3-py310_23.5.2-0-Linux-x86_64
+export TARGET_PATH=/apps/bionemo-src   # Must be a shared filesystem. This is where Nemo launcher scripts will reside.
+export DOCKER_IMAGE_NAME=bionemo
+export TAG=latest
+export ENROOT_IMAGE=/apps/${DOCKER_IMAGE_NAME}
+export DATASET_PATH=/fsx/
+```
+
+## 1.4. Pull this github repo
-## 1.4. Pull this github repo
+## 1.4. Clone this github repo
-## 1.4. Pull this github repo
+## 1.4. Clone this github repo
+
+```bash
+cd /apps/
+git clone https://github.com/aws-samples/awsome-distributed-training.git
+cp -r /apps/awsome-distributed-training/3.test_cases/14.bionemo/* ./apps/
+```
+
+## 2. Pull Image
+
+```bash
+cd /apps/
+docker pull nvcr.io/nvidia/clara/bionemo-framework:1.2
+```
+
+## 3. Create Conda env
+We need a conda environment that has the necessary dependencies for submitting multiple arrays of slurm jobs via [HYDRA](https://github.com/facebookresearch/hydra) which NeMo uses to configuring both NeMo models and the PyTorch Lightning Trainer. 
+```
+# Miniconda is already installed if you are using the DLAMI but needs installation with Base AMI
+
+wget -O miniconda.sh "https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}.sh" \
+    && bash miniconda.sh -b -p /apps/.conda \
+          &&  /apps/.conda/bin/conda init bash  
+
+source ~/.bashrc    
+conda create --name bionemo python=${PYTHON_VERSION}
+
+source activate bionemo
+
+pip3 install -r requirements.txt
+
+```
+All package versions in the above `requirements.txt` file is recommended from Nvidia. An older version of the package `opencv-python-headless==4.8.0.74` has to be installed to avoid this [error](https://github.com/rom1504/img2dataset/issues/355) with [img2dataset](https://github.com/rom1504/img2dataset) package.
+
+
+
+## 4. Build customized docker image
+To achieve target performance of Nemo-Multimodal with EFA on P5 and P4de instances, we provide a customized 
+`3.test_cases/14.nemo-multimodal/0.Dockerfile` and we can build a image like below:
+
+```
+docker build -t ${DOCKER_IMAGE_NAME}:${TAG} -f 0.Dockerfile .
+```
+
+## 5. Convert image
+Convert the Docker container image to an [Enroot](https://github.com/NVIDIA/enroot) squash file that will be stored in `/apps`. This step takes a few minutes.
+```
+enroot import -o ${ENROOT_IMAGE}.sqsh dockerd://${DOCKER_IMAGE_NAME}
+
+```
+
+## 6. Download and preprocess data
+We will use the popular [UniRef50](https://www.uniprot.org/help/uniref) dataset for pretraining. We will use BioNemo's in-built functionality to download and pre-process data. To this end, we provide `prepare_uniref50.py` file to do so. You can edit the above to download and process [UniRef90]((https://www.uniprot.org/help/uniref)). To run the above python code on your slurm cluster in the BioNemo cluster execute the following:
+
+```bash
+sbatch 1.uniref50.slurm
+```
+
+This will download raw data in `/fsx/raw/` and save pre-processed `train, validation and test` csv files in `/fsx/processed/`. The log files for submitted jobs are written to the local directory. To check the status of the datasets download job, you can tail the log file:
+
+```bash
+tail -f slurm-uniref-<slurm_job_id>.out
+```
+
+
+
+## 7. Pretrain ESM models
+Now we are ready to submit distributed training jobs to pretrain `ESM1nv` models. We provide the `2.esm1nv_pretrain.slurm` script to run training 4 `p4de.24xlarge` nodes with `8xA100 80 GB` GPUs. Make sure data paths and model configuration is correct if you are running on custom data. To kick off distributed training execute:
+
+```bash
+sbatch 2.esm1nv_pretrain.slurm
+
+```
+
+Before kicking off training, first train, validation and test datasets are indexed and dataloaders are created and then you should see an example output like below:
+
+```bash
+Epoch 0:   3%|▎         | 34103/1100000 [5:28:58<171:22:21,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.510, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34106/1100000 [5:29:00<171:22:19,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34109/1100000 [5:29:02<171:22:09,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34112/1100000 [5:29:03<171:22:00,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+```
+
+## 8. Run container on Head Node [Troubleshooting]
+Once the above image is pulled, you can run the container on the head node like below. This step could be used for troubleshooting purposes. Here we are running the container just to be able to copy launcher scripts on the host machine. If you need to run the container on the compute nodes, you would need to add `--gpus all` flag to the run command. It is recommended to have the docker run flags like below, as recommended by Nvidia PyTorch containers, otherwise you may potentially run into an error like [this](https://github.com/NVIDIA/Megatron-LM/issues/516)
+
+```
+ docker run -it nvcr.io/nvidia/clara/bionemo-framework:latest bash
+```