From 01bddcf7bbfe2dc3b3f3adea82ae191c039cd884 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 23 Apr 2025 16:49:47 -0700 Subject: [PATCH 01/15] initial draft of slinky-slurm --- .../slinky-slurm/Docker-Build-README.md | 2 + .../slinky-slurm/README.md | 803 +++++++++++++ .../slinky-slurm/dlc-slurmd.Dockerfile | 91 ++ .../slinky-slurm/download_c4.py | 6 + .../slinky-slurm/lustre-pvc-slurm.yaml | 12 + .../slinky-slurm/openzfs-pvc-slurm.yaml | 12 + .../slinky-slurm/openzfs-storageclass.yaml | 19 + .../slinky-slurm/values.yaml | 1013 +++++++++++++++++ 8 files changed, 1958 insertions(+) create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-pvc-slurm.yaml create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-pvc-slurm.yaml create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-storageclass.yaml create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md new file mode 100644 index 000000000..20daebd31 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md @@ -0,0 +1,2 @@ +# Docker Build for the Slurmd Deep Learning Container + diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md new file mode 100644 index 000000000..84f7b9c24 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -0,0 +1,803 @@ +# Running Slurm on HyperPod EKS with Slinky + +### What is the Slinky Project? +--- + +### Architecture + +--- + +### Release Notes + +The following was tested on 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) for hosting the Worker Pod NodeSet. 2 `m5.2xlarge` instances were also allocated for separately hosting the Controller Pod and Login Pod. Other components (Accounting, MariaDB, RestAPI, Token Creation, Exporter) were also colocated across the 2 `m5.2xlarge` instances for simplicity, but you may wish to deploy them on separate instance depending on your specific needs. This can be accomplished by modifying the associated node affinity configurations, which is discussed in more detail below. + +Testing used [Slurm Operator v0.2.1](https://github.com/slinkyproject/slurm-operator/pkgs/container/slurm-operator) (pulled from the Slinky GHCR) and [Slurm Cluster v0.3.0](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) (packaged and deployed locally using the main Slinky repo branch) in order to include the NoteSet volume mount and Login Pod features. These features are expected to be included in the official Slurm Cluster v0.3.0 release when it becomes available on the Slinky GHCR repo, along with a new version of the Slurm Operator with corresponding validating webhooks. + +Note that the [Slinky Project](https://github.com/SlinkyProject) is under active development and could introduce breaking changes that may require modified deployment steps and configuration changes. + +Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image. See [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. + +* * * + + +### Set Up the HyperPod Cluster: + +Follow the [Prerequisites](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup)and [Cluster Configuration](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster) steps of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). + +Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. + +Add an access entry (if needed): + +``` +export AWS_ACCOUNT_ID= + +export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster + +export ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_ID:role/Administrator + +export PLCY_ARN=arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy + +export AWS_REGION=us-west-2 + +aws eks create-access-entry \ + --cluster-name $EKS_CLUSTER_NAME \ + --principal-arn $ROLE_ARN \ + --type STANDARD \ + --region $AWS_REGION + +aws eks associate-access-policy \ + --cluster-name $EKS_CLUSTER_NAME \ + --principal-arn $ROLE_ARN \ + --policy-arn $PLCY_ARN \ + --access-scope type=cluster \ + --region $AWS_REGION + +``` + +Update your kubectl context: + +``` + +aws eks update-kubeconfig --name $EKS_CLUSTER_NAME + +kubectl get nodes + +``` + +* * * + +### Create an FSx for Lustre Storage Class: + +Follow the [Setup FSx for Lustre File System](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster/06-fsx-for-lustre) of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). + +Verify` fsx-sc` Storage Class: + +``` +kubectl get storageclass fsx-sc -oyaml +``` + +* * * + +### Create an FSx for OpenZFS Storage Class: + +Install the[OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/blob/main/docs/install.md). Set up permissions using IAM roles for service accounts, and taint the nodes as recommended: + +``` + +eksctl create iamserviceaccount \ + --name fsx-openzfs-csi-controller-sa \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonFSxFullAccess \ + --approve \ + --role-name AmazonEKSFSxOpenZFSCSIDriverFullAccess \ + --region $AWS_REGION + +kubectl taint nodes --all fsx.openzfs.csi.aws.com/agent-not-ready:NoExecute + +helm repo add aws-fsx-openzfs-csi-driver \ + https://kubernetes-sigs.github.io/aws-fsx-openzfs-csi-driver + +helm repo update + +helm upgrade --install aws-fsx-openzfs-csi-driver \ + --namespace kube-system \ + --set controller.serviceAccount.create=false \ + aws-fsx-openzfs-csi-driver/aws-fsx-openzfs-csi-driver + +kubectl get pods -n kube-system \ + -l app.kubernetes.io/part-of=aws-fsx-openzfs-csi-driver + +``` + +Follow the [Dynamic Provisioning](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/tree/main/examples/kubernetes/dynamic-provisioning) guide to create an FSx for OpenZFS Storage Class: + +``` + +export PRIVATE_SUBNET_ID= +export SECURITY_GROUP_ID= + +kubectl apply -f openzfs-storageclass.yaml + +kubectl get sc openzfs-sc -oyaml + +``` + +* * * + +### Install the AWS Load Balancer Controller: + +Following [these instructions](https://docs.aws.amazon.com/eks/latest/userguide/lbc-helm.html): + +``` +export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster +export VPC_ID= +export AWS_REGION=us-west-2 +export AWS_ACCOUNT_ID= + +# manually update crds +kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master" + +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.12.0/docs/install/iam_policy.json + +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy-v2.12.0 \ + --policy-document file://iam_policy.json + +eksctl create iamserviceaccount \ + --cluster=$EKS_CLUSTER_NAME \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn=arn:aws:iam::$AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy-v2.12.0 \ + --override-existing-serviceaccounts \ + --region $AWS_REGION \ + --approve + +helm repo add eks https://aws.github.io/eks-charts +helm repo update + +helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=$EKS_CLUSTER_NAME \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller \ + --set region=$AWS_REGION \ + --set vpcId=$VPC_ID + +kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller + +kubectl get sa aws-load-balancer-controller -n kube-system -oyaml + +``` + +* * * + +### Instill Prerequisites (Cert Manager and Prometheus): + +Follow the [QuickStart Guide](http://curl%20-l%20https//raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.1.0/helm/slurm-operator/values.yaml%20/%20%20%20-o%20values-operator.yaml%20helm%20install%20slurm-operator%20oci://ghcr.io/slinkyproject/charts/slurm-operator%20/%20%20%20--values=values-operator.yaml%20--version=0.1.0%20--namespace=slinky%20--create-namespace) to install Cert Manager and Prometheus as [Pre-Requisites](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites). + +Verify Pre-Requisites Instillation: + +``` + kubectl get all -n cert-manager + kubectl get all -n prometheus +``` + +* * * + +### Install the Slurm Operator: + +For [Slurm Operator](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites) Installation, we'll install release v0.2.1, which is the latest release available at the time of testing. + + Note: We will locally build and deploy a pre-release v0.3.0 of the [Slurm Cluster](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) from the main branch of the Slinky Project repository. The project is being actively developed, so there is a risk of pulling down breaking changes, but it includes the features to [add additional volume mounts to compute NodeSets](https://github.com/SlinkyProject/slurm-operator/commit/b0e111b0a8434e38b5fb37a2051e7525d5679319) and [deploy Login Pods](https://github.com/SlinkyProject/slurm-operator/commit/37f020f041556164b9c935f799b51df65d22aefe). + +``` + +curl -L https://raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.2.1/helm/slurm-operator/values.yaml \ + -o values-operator-0.2.1.yaml + +# Delete any stale crds (if you deployed an older version) +kubectl delete crd clusters.slinky.slurm.net +kubectl delete crd nodesets.slinky.slurm.net + +helm install slurm-operator oci://ghcr.io/slinkyproject/charts/slurm-operator \ + --values=values-operator-0.2.1.yaml --version=0.2.1 --namespace=slinky --create-namespace + +``` + +Verify Slurm Operator Instillation: + +``` +kubectl get all -n slinky +``` + +* * * + +### Install the Slurm Cluster: + +To deploy the **slurm cluster**, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, again, in order to test the latest changes in release **v0.3.0**, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. For your convenience, we've provided a copy of the [values.yaml](./values.yaml) file with most of the configuration changes mentioned below already implemented. + +Clone the Slurm Operator repository, which also contains the Helm chart artifacts for the Slurm Cluster: +``` +git clone https://github.com/SlinkyProject/slurm-operator.git + +``` +(Optional) If you wish to start from scratch, open the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file associated with the Slurm Cluster Helm Chart: +``` +code slurm-operator/helm/slurm/values.yaml + +``` +Otherwise, you can use the [values.yaml](./values.yaml) file we've provided. + +Verify the existence of the instance type label for controller affinity: + +``` +export GEN_INSTANCE_TYPE=ml.m5.2xlarge + +kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE + +``` + +#### Controller Modifications: + +Configure controller affinity: + +``` +controller: +... + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: "In" + values: + - "ml.m5.2xlarge" +... +``` + +#### Compute NodeSet Modifications: + +Verify the existence of the instance type label for compute node selector: + +``` +ACCEL_INSTANCE_TYPE=ml.g5.8xlarge + + kubectl get nodes -l node.kubernetes.io/instance-type=$ACCEL_INSTANCE_TYPE + +``` + +Change the compute node name, replica count, and node selector:: + +``` +compute: +... + nodeSets: + - name: hp-node + ... + replicas: 4 + ... + nodeSelector: + kubernetes.io/os: linux + node.kubernetes.io/instance-type: ml.g5.8xlarge +... +``` + +Create the slurm namespace: + +``` +kubectl create ns slurm +``` + +Create a FSx for Lustre Persistent Volume Claim (PVC) in the slurm namespace: + +This is needed to reference for node volume mounts later. + +``` +kubectl apply -f lustre-pvc-slurm.yaml +``` + +Verify FSx for Lustre PVC creation: + +``` +kubectl get pvc -n slurm + +# check for a bound state +kubectl get pvc fsx-claim -n slurm -ojson \ + | jq -r .status.phase + +# get the the volume ID +kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ + | jq -r .spec.volumeName) -ojson \ + | jq -r .spec.csi.volumeHandle + +``` + +Create an FSx for OpenZFS PVC in the slurm namespace: + +``` +kubectl apply -f openzfs-pvc-slurm.yaml + +``` + +Verify FSx for OpenZFS PVC creation: + +``` +kubectl get pvc -n slurm + +# check for a bound state +kubectl get pvc openzfs-claim -n slurm -ojson \ + | jq -r .status.phase + +# get the volume ID +kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ + | jq -r .spec.volumeName) -ojson \ + | jq -r .spec.csi.volumeHandle + +``` + +Add the FSx for Lustre and OpenZFS PVCs to the list of compute node volumes: + +``` +compute: + nodesets: + - name: hp-node + ... + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + ... +``` + +Configure resources for compute nodes: +Note: limits are required, otherwise the compute nodes will not deploy. + +``` +compute: + nodesets: + - name: hp-node + ... + resources: + limit: + cpu: "32" + memory: "128Gi" + nvidia.com/gpu: "1" + requests: + cpu: "1" + memory: "1Gi" + nvidia.com/gpu: "1" + ... +``` + +Modify the compute node container image to use the [Slurmd Deep Learning Container](./Docker-Build-README.md) (Slurmd DLC) build: + +``` +compute: + nodesets: + - name: compute-node + ... + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ".dkr.ecr.us-west-2.amazonaws.com/dlc-slurmd" + # + # -- (string) + # Set the image tag to use. + tag: "24.11.4-ubuntu24.04" + ... +``` +For your convenience, we've pre-build a Slurmd DLC for and made it available in an ECR public repository, but you can use the provided [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) to modify and build your own. + +#### Login Node Modifications: + +Add the FSx for Lustre and OpenZFS PVCs to the list of login node volumes: + +``` +login: + ... + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + ... +``` + +Generate an SSH key for root authorization: + +``` + +export EMAIL_ADDR= + +ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_slurm -C "${EMAIL_ADDR}" + +cat ~/.ssh/id_ed25519_slurm.pub + +# ssh-ed25519 janedoe@example.com + +``` + +Specify the root SSH authorized key in `values.yaml`: + +``` +login: + ... + rootSshAuthorizedKeys: + - "ssh-ed25519 janedoe@example.com" + ... +``` + +Disable SSSD: the `nsswitchConf` (Name Service Switch configuration) file tells Linux how to resolve different types of system information like users, groups, passwords, etc. By setting everything to just `files` we're telling the system to only use local files for authentication and not to try SSSD or other sources. This is simpler and more reliable when you just want to use SSH key authentication for root access, as the SSH keys are stored in local files anyway (/root/.ssh/authorized_keys). + +``` +... +login: + ... + nsswitchConf: + passwd: files + group: files + shadow: files + gshadow: files + sudoers: files + hosts: files + networks: files + protocols: files + services: files + ethers: files + rpc: files + netgroup: files + automount: files + ... +... +``` + +Define the content of the sshd_config file: + +``` +... +login: + sshdConfig: + # This is the actual content of the sshd_config file + AcceptEnv: "LANG LC_*" + AuthorizedKeysFile: "/root/.ssh/authorized_keys" + ChallengeResponseAuthentication: "no" + ClientAliveCountMax: "3" + ClientAliveInterval: "60" + LogLevel: "INFO" + PasswordAuthentication: "no" + PermitRootLogin: "yes" + Port: "22" + PrintMotd: "no" + Protocol: "2" + PubkeyAuthentication: "yes" + Subsystem: "sftp internal-sftp" + TCPKeepAlive: "yes" + UseDNS: "no" + UsePAM: "no" + X11Forwarding: "no" +... +``` + +Update the **slurm-login** service port: + +``` +login: + ... + servicePort: 22 + ... +``` + +#### Deploy the Slurm Cluster + +Locally package and deploy the **slurm cluster** using the modified `values.yaml` file: + +``` +helm dependency update slurm-operator/helm/slurm + +helm package slurm-operator/helm/slurm + +slurm-0.3.0.tgz + +# Dry run +helm install --dry-run slurm slurm-0.3.0.tgz \ +--values=values.yaml \ +--namespace=slurm + +helm install slurm slurm-0.3.0.tgz \ +--values=values.yaml \ +--namespace=slurm + +``` + +Note: Release v0.2.1 of the slurm-operator validating webhook may throw a few warning about not recognizing `spec.template.spec.volumes[].mountPath` fields. This is not surprising given we are using the newer pre-release v0.3.0 of the slurm cluster, but it doesn’t appear to cause any functional errors. + + +Watch the deployment status of the Slurm cluster: + +``` +kubectl --namespace=slurm get pods -l app.kubernetes.io/instance=slurm --watch +``` + +Verify deployment status of all components: + +``` +kubectl get all -n slurm +``` + +#### Configure Network Load Balancer provisioning using the AWS Load Balancer Controller + +Manually add annotation to the slurm-login service: + +``` +export PUBLIC_SUBNET_ID_1= +export PUBLIC_SUBNET_ID_2= + +kubectl annotate service slurm-login -n slurm \ + service.beta.kubernetes.io/aws-load-balancer-type="nlb" \ + service.beta.kubernetes.io/aws-load-balancer-scheme="internet-facing" \ + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type="ip" \ + service.beta.kubernetes.io/aws-load-balancer-subnets="$PUBLIC_SUBNET_ID_1,$PUBLIC_SUBNET_ID_2" \ + service.beta.kubernetes.io/aws-load-balancer-healthcheck-port="22" \ + --overwrite + +kubectl describe service slurm-login -n slurm + +``` + +Any annotations added to the slurm cluster `values.yaml` file for the slurm-login service are currently ignored, but AWS Load Balancer Controller actively watches for and implements annotation changes. It Automatically adds inbound rules to the node security group to allow traffic from the NLB security group on the target port (22 in this case). +* * * + +### Basic Tests: + +SSH into the login node as root from the NLB endpoint: + +``` + +SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" +ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME + +``` + +Check the available nodes: + +``` + +sinfo + +PARTITION AVAIL TIMELIMIT NODES STATE NODELIST +hp-node up infinite 4 idle hp-node-[0-3] +all* up infinite 4 idle hp-node-[0-3] + +``` + +Verify FSx for Lustre and OpenZFS filesystem mounts on the login pod: + +``` + +df -h + +Filesystem Size Used Avail Use% Mounted on +overlay 500G 30G 471G 6% / +tmpfs 64M 0 64M 0% /dev +tmpfs 63G 0 63G 0% /sys/fs/cgroup +10.1.12.93@tcp:/7c5dpb4v 1.2T 7.8M 1.2T 1% /fsx +fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home +tmpfs 115G 4.0K 115G 1% /etc/slurm +/dev/nvme0n1p1 100G 23G 78G 23% /run +/dev/nvme1n1 500G 30G 471G 6% /etc/hostname +shm 64M 0 64M 0% /dev/shm +tmpfs 115G 4.0K 115G 1% /etc/sssd/sssd.conf +tmpfs 115G 12K 115G 1% /etc/ssh/ssh_host_rsa_key +tmpfs 63G 0 63G 0% /proc/acpi +tmpfs 63G 0 63G 0% /sys/firmware + +exit + +``` + +Verify FSx for Lustre and OpenZFS filesystem mounts on the worker node pods: + +``` + +kubectl -n slurm exec -it pod/slurm-compute-hp-node-0 -- bash --login + +df -h + +Filesystem Size Used Avail Use% Mounted on +overlay 500G 31G 470G 7% / +tmpfs 64M 0 64M 0% /dev +tmpfs 63G 0 63G 0% /sys/fs/cgroup +10.1.12.93@tcp:/7c5dpb4v 1.2T 7.5M 1.2T 1% /fsx +fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home +tmpfs 115G 4.0K 115G 1% /etc/slurm +/dev/nvme0n1p1 100G 23G 78G 23% /run +/dev/nvme1n1 500G 31G 470G 7% /etc/hostname +shm 64M 0 64M 0% /dev/shm +tmpfs 115G 0 115G 0% /var/log/slurm + +``` + +Check the installed CUDA compiler version on worker node pods: + +``` + +nvcc --version + +# nccl-slurmd +nvcc: NVIDIA (R) Cuda compiler driver +Copyright (c) 2005-2023 NVIDIA Corporation +Built on Tue_Aug_15_22:02:13_PDT_2023 +Cuda compilation tools, release 12.2, V12.2.140 +Build cuda_12.2.r12.2/compiler.33191640_0 + +# dlc-slurmd +nvcc: NVIDIA (R) Cuda compiler driver +Copyright (c) 2005-2024 NVIDIA Corporation +Built on Tue_Oct_29_23:50:19_PDT_2024 +Cuda compilation tools, release 12.6, V12.6.85 +Build cuda_12.6.r12.6/compiler.35059454_0 + +``` + +Check the NCCL version on worker node pods: + +``` +ldconfig -v | grep "libnccl.so" | tail -n1 | sed -r 's/^.*\.so\.//' + +2.23.4 +``` + +Confirm NCCL headers are installed worker node pods: + +``` +find /usr/local/lib/ -name "nccl.h" 2>/dev/null + +/usr/local/lib/python3.12/site-packages/torch/include/torch/csrc/cuda/nccl.h + +exit +``` + +* * * + +### FSDP Test + +SSH into the login pod as root, clone the repo, and create a checkpoints directory: + +``` + +SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" +ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME + +# install git +apt update +apt install -y git +git --version + +# install vim +`apt install ``-``y vim ` +vim --version + +cd /fsx +git clone https://github.com/aws-samples/awsome-distributed-training/ +cd awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm + +mkdir checkpoints +``` + +Download the c4 dataset to avoid throttling errors from HuggingFace: + +``` +mkdir -p /fsx/datasets/c4 + +apt install -y python3.12-venv +python3 -m venv env +source env/bin/activate +pip install --upgrade pip +pip install datasets + +python3 download_c4.py + +deactivate +``` + +Kick-off training: + +``` +sbatch llama2_7b-training.sbatch +``` + +Watch the output logs from login pod: + +``` + +tail -f logs/llama2_7b-FSDP_$(squeue -h -u $USER -o "%i" | head -1).out + +``` + +Watch the error logs from `slurm-compute-hp-node-0`: + +``` +# from a new terminal window +kubectl -n slurm exec -it pod/slurm-compute-hp-node-0 -- bash --login + +cd /fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm + +watch "grep 'Batch.*Loss' logs/llama2_7b-FSDP_65.err" + +tail -f logs/llama2_7b-FSDP_$(squeue -h -u $USER -o "%i" | head -1).err | grep --line-buffered 'Batch.*Loss' + +``` + +Watch squeue from `slurm-compute-hp-node-1`: + +``` +# from a new terminal window +kubectl -n slurm exec -it pod/slurm-compute-hp-node-1 -- bash --login + +# 1 second updates +watch -n 1 squeue + +``` + +Watch checkpoints from `slurm-compute-hp-node-2`: + +``` +# from a new terminal window +kubectl -n slurm exec -it pod/slurm-compute-hp-node-2 -- bash --login + +cd /fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm + +# highlight changes, show timestamps, 5 second updates +watch -n 5 -d "ls -lh checkpoints" + +``` + +* * * + +### Clean Up: + +``` + +rm -rf checkpoints/* + +rm -rf logs/* + +kubectl delete pvc fsx-lustre-pvc -n slurm + +helm uninstall slurm -n slurm +helm uninstall slurm-operator -n slinky + +helm uninstall prometheus -n prometheus +helm uninstall cert-manager -n cert-manager + +kubectl delete pvc fsx-claim -n slurm +kubectl delete pvc openzfs-claim + +helm uninstall aws-fsx-csi-driver -n kube-system +helm uninstall aws-fsx-openzfs-csi-driver -n kube-system + +eksctl delete iamserviceaccount \ + --name fsx-csi-controller-sa \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME + +eksctl delete iamserviceaccount \ + --name fsx-openzfs-csi-controller-sa \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME + +``` diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile new file mode 100644 index 000000000..8a3b97fb7 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile @@ -0,0 +1,91 @@ +# First stage - DLC PyTorch 2.6, Python 3.12, CUDA 12.6, Ubuntu 22.04 +FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.6.0-gpu-py312-cu126-ubuntu22.04-ec2 AS dlc + +# Second stage - Slurm compute node +FROM ghcr.io/slinkyproject/slurmd:24.11.4-ubuntu24.04 + +ARG PYTHON_SHORT_VERSION=3.12 + +# Create required directory +RUN mkdir -p /var/spool/slurmd + +# Environment variables from DLC +ENV CUDA_HOME="/usr/local/cuda" +ENV EFA_PATH="/opt/amazon/efa" +ENV LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu" +ENV PATH="${EFA_PATH}/bin:${CUDA_HOME}/bin:${PATH}" +ENV NCCL_DEBUG=INFO +ENV NCCL_SOCKET_IFNAME=^docker0 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 + +# Install critical system dependencies missing in base Slurm image +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + git \ + libcurl4-openssl-dev \ + libssl-dev \ + libnuma1 \ + libnuma-dev \ + libibverbs-dev \ + libtool \ + autoconf \ + pkg-config \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy CUDA/NCCL/EFA stack from DLC +COPY --from=dlc /usr/local/cuda /usr/local/cuda +COPY --from=dlc /opt/amazon/efa /opt/amazon/efa +COPY --from=dlc /usr/local/lib/libnccl* /usr/local/lib/ +COPY --from=dlc /etc/nccl.conf /etc/nccl.conf + +# Copy Python installation +COPY --from=dlc /usr/local/bin/python${PYTHON_SHORT_VERSION}* /usr/local/bin/ +COPY --from=dlc /usr/local/lib/python${PYTHON_SHORT_VERSION} /usr/local/lib/python${PYTHON_SHORT_VERSION} +COPY --from=dlc /usr/local/lib/libpython${PYTHON_SHORT_VERSION}* /usr/local/lib/ +COPY --from=dlc /usr/local/include/python${PYTHON_SHORT_VERSION}* /usr/local/include/ + +# Fix Python symlinks +RUN rm -f /usr/local/bin/python3 && \ + rm -f /usr/local/bin/python && \ + ln -s /usr/local/bin/python${PYTHON_SHORT_VERSION} /usr/local/bin/python3 && \ + ln -s /usr/local/bin/python${PYTHON_SHORT_VERSION} /usr/local/bin/python + +# Additional requirements +RUN /usr/local/bin/python3 -m pip install --no-cache-dir --no-deps \ + transformers==4.37.2 \ + datasets==2.17.1 + +# Install OpenSSH, allow OpenSSH to talk to containers without asking for confirmation +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssh-client openssh-server \ + && mkdir -p /var/run/sshd \ + && cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ + && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ + && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Configure OpenSSH so that nodes can communicate with each other +RUN mkdir -p /var/run/sshd \ + && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +RUN rm -rf /root/.ssh/ \ + && mkdir -p /root/.ssh/ \ + && ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config + +WORKDIR /home diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py new file mode 100644 index 000000000..9990e0f17 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py @@ -0,0 +1,6 @@ +from datasets import load_dataset + +# Download and cache the English C4 dataset +dataset = load_dataset("allenai/c4", + name="en", + cache_dir="/fsx/datasets/c4") \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-pvc-slurm.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-pvc-slurm.yaml new file mode 100644 index 000000000..c3e6fc1e0 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-pvc-slurm.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: fsx-claim + namespace: slurm +spec: + accessModes: + - ReadWriteMany + storageClassName: fsx-sc + resources: + requests: + storage: 1200Gi \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-pvc-slurm.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-pvc-slurm.yaml new file mode 100644 index 000000000..51218f34d --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-pvc-slurm.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: openzfs-claim + namespace: slurm +spec: + accessModes: + - ReadWriteMany + storageClassName: openzfs-sc + resources: + requests: + storage: 64Gi \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-storageclass.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-storageclass.yaml new file mode 100644 index 000000000..5b84aae1d --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/openzfs-storageclass.yaml @@ -0,0 +1,19 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: openzfs-sc +provisioner: fsx.openzfs.csi.aws.com +parameters: + ResourceType: "filesystem" #REQUIRED + DeploymentType: '"SINGLE_AZ_HA_2"' #REQUIRED + ThroughputCapacity: '160' #REQUIRED + SubnetIds: '["${PRIVATE_SUBNET_ID}"]' #REQUIRED + SkipFinalBackupOnDeletion: 'true' #REQUIRED + AutomaticBackupRetentionDays: '30' + SecurityGroupIds: '["${SECURITY_GROUP_ID}"]' + CopyTagsToBackups: 'true' + CopyTagsToVolumes: 'true' + DailyAutomaticBackupStartTime: '"19:00"' + OptionsOnDeletion: '["DELETE_CHILD_VOLUMES_AND_SNAPSHOTS"]' + RootVolumeConfiguration: '{"DataCompressionType": "NONE", "NfsExports": [{"ClientConfigurations": [{"Clients": "*", "Options": ["rw", "no_root_squash", "crossmnt"]}]}]}' + WeeklyMaintenanceStartTime: '"1:04:00"' \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml new file mode 100644 index 000000000..19ba9cd72 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml @@ -0,0 +1,1013 @@ +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-License-Identifier: Apache-2.0 + +# +# Debug configuration. +# @ignored +debug: + # + # -- (bool) + # Enables debug configuration. + enabled: false + # + # -- (bool) + # Allow a locally running operator to communicate with slurm cluster via port-forward. + # NOTE: use when running the operator in a local debugger. + localOperator: true + +# +# -- (string) +# Overrides the name of the release. +nameOverride: "" + +# +# -- (string) +# Overrides the full name of the release. +fullnameOverride: "" + +# +# -- (string) +# Overrides the namespace of the release. +namespaceOverride: "" + +# +# -- (list) +# Set the secrets for image pull. +# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] + # - name: regcred + +# +# -- (string) +# Set the image pull policy. +imagePullPolicy: IfNotPresent + +# +# -- (string) +# Set the priority class to use. +# Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass +priorityClassName: "" + +# +# Slurm JWT authentication. +jwt: + # + # JWT hs256 configurations. + hs256: + # + # -- (string) + # The existing secret to use otherwise one will be generated. + existingSecret: "" + +# +# Slurm configurations. +slurm: + # + # Slurm authentication configurations. + auth: + # + # -- (string) + # The existing secret to use otherwise one will be generated. + existingSecret: "" + # + # -- (map[string]string | map[string][]string) + # Extra slurmdbd configuration lines to append to `slurmdbd.conf`. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurmdbd.conf.html + extraSlurmdbdConf: + CommitDelay: 1 + ### LOGGING ### + DebugLevel: info + DebugFlags: [] + LogTimeFormat: + - iso8601_ms + - format_stderr + # PLUGINS & PARAMETERS + CommunicationParameters: [] + HashPlugin: hash/k12 + ### ARCHIVE ### + ArchiveDir: /tmp + #ArchiveEvents: YES + #ArchiveJobs: YES + #ArchiveResvs: YES + #ArchiveSteps: NO + #ArchiveSuspend: NO + #ArchiveTXN: NO + #ArchiveUsage: NO + ### PURGE ### + #PurgeEventAfter: 12month + #PurgeJobAfter: 12month + #PurgeResvAfter: 2month + #PurgeStepAfter: 2month + #PurgeSuspendAfter: 1month + #PurgeTXNAfter: 12month + #PurgeUsageAfter: 12month + # + # -- (map[string]string | map[string][]string) + # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurm.conf.html + extraSlurmConf: + MaxNodeCount: 1024 + ReturnToService: 2 + EnforcePartLimits: "NO" + ### PLUGINS & PARAMETERS ### + AuthInfo: + - use_client_ids + SchedulerType: sched/backfill + SchedulerParameters: + - defer_batch + SelectType: select/cons_tres + SelectTypeParameters: + - CR_Core_Memory + SlurmctldParameters: + - enable_configless + - enable_stepmgr + SlurmdParameters: + - contain_spank + CommunicationParameters: + - block_null_hash + LaunchParameters: + - enable_nss_slurm + - use_interactive_step + - ulimit_pam_adopt + ReconfigFlags: + - KeepPartInfo + - KeepPartState + PrologFlags: Contain + HashPlugin: hash/k12 + ### LOGGING ### + SlurmctldDebug: info + SlurmSchedLogLevel: 1 + SlurmdDebug: info + DebugFlags: [] + LogTimeFormat: + - iso8601_ms + - format_stderr + # + # -- (map[string]string) + # Optional raw Slurm configuration files, as a map. + # The map key represents the config file by name; the map value represents config file contents as a string. + # Ref: https://slurm.schedmd.com/man_index.html#configuration_files + configFiles: {} + # acct_gather.conf: | + # # Ref: https://slurm.schedmd.com/acct_gather.conf.html + # burst_buffer.conf: | + # # Ref: https://slurm.schedmd.com/burst_buffer.conf.html + # gres.conf: | + # # Ref: https://slurm.schedmd.com/gres.conf.html + # helpers.conf: | + # # Ref: https://slurm.schedmd.com/helpers.conf.html + # job_container.conf: | + # # Ref: https://slurm.schedmd.com/job_container.conf.html + # mpi.conf: | + # # Ref: https://slurm.schedmd.com/mpi.conf.html + # oci.conf: | + # # Ref: https://slurm.schedmd.com/oci.conf.html + # plugstack.conf: | + # # Ref: https://slurm.schedmd.com/plugstack.conf.html + # topology.conf: | + # # Ref: https://slurm.schedmd.com/topology.conf.html + # + # -- (map[string]string) + # The Prolog scripts for compute nodesets, as a map. + # The map key represents the filename; the map value represents the script contents. + # WARNING: The script must include a shebang (!) so it can be executed correctly by Slurm. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog + # Ref: https://slurm.schedmd.com/prolog_epilog.html + # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) + prologScripts: {} + # empty: | + # #!/usr/bin/env bash + # exit 0 + # + # -- (map[string]string) + # The Epilog scripts for compute nodesets, as a map. + # The map key represents the filename; the map value represents the script contents. + # WARNING: The script must include a shebang (!) so it can be executed correctly by Slurm. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog + # Ref: https://slurm.schedmd.com/prolog_epilog.html + # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) + epilogScripts: {} + # empty: | + # #!/usr/bin/env bash + # exit 0 + +# +# Shared configurations. +sharedConfig: + # + # -- (list) + # List of volumes to be mounted on each Login and NodeSet pods. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: [] + # - name: nfs-home + # mountPath: /home + # persistentVolumeClaim: + # claimName: nfs-home + # - name: nfs-data + # mountPath: /mnt/data + # persistentVolumeClaim: + # claimName: nfs-data + +# +# Slurm authcred (sackd) configurations. +authcred: + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/sackd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# Slurm controller (slurmctld) configurations. +controller: + # + # -- (bool) + # Enables the controller node. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmctld + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: "In" + values: + - "ml.m5.2xlarge" + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + # + # Define a persistent volume for the slurm controller to store its save-state. + # Used to recover from system failures or from pod upgrades. + persistence: + # + # -- (bool) + # Enables save-state persistence. + enabled: false + # + # -- (string) + # Name of an existing `PersistentVolumeClaim` to use instead of creating one from definition. + # NOTE: When not empty, the other persistence fields will be ignored. + existingClaim: "" + # + # -- (object) + # Create a `PersistentVolumeClaim` with these annotations. + annotations: {} + # + # -- (object) + # Create a `PersistentVolumeClaim` with these labels. + labels: {} + # + # -- (string) + # Create a `PersistentVolumeClaim` with this storage class. + storageClass: standard + # + # -- (list) + # Create a `PersistentVolumeClaim` with these access modes. + accessModes: + - ReadWriteOnce + # + # -- (string) + # Create a `PersistentVolumeClaim` with this storage size. + size: 4Gi + # + # -- (object) + # Selector to match an existing `PersistentVolume`. + selector: {} + # matchLabels: + # app: foo + +# +# Login node configurations. +login: + # + # -- (bool) + # Enables login nodes. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/login + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # The restapi service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: + type: LoadBalancer + # annotations: + # service.beta.kubernetes.io/aws-load-balancer-type: "external" + # service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + # service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" + # externalIPs: [] + # externalName: my.login.example.com + # + # -- (integer) + # The external service port number. + servicePort: 22 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 32222 + # + # -- (list) + # The `/root/.ssh/authorized_keys` file to write, represented as a list. + rootSshAuthorizedKeys: + - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICe9Hm9zk+q0I9rTQWtAdTS1uIuIRtN+6drJYt0k6JWN natharno@amazon.com" + # + # -- (map) + # The `/etc/ssh/sshd_config` file to use, represented as a map. + # Ref: https://man.openbsd.org/sshd_config + sshdConfig: + # This is the actual content of the sshd_config file + AcceptEnv: "LANG LC_*" + AuthorizedKeysFile: "/root/.ssh/authorized_keys" + ChallengeResponseAuthentication: "no" + ClientAliveCountMax: "3" + ClientAliveInterval: "60" + LogLevel: "INFO" + PasswordAuthentication: "no" + PermitRootLogin: "yes" + Port: "22" + PrintMotd: "no" + Protocol: "2" + PubkeyAuthentication: "yes" + Subsystem: "sftp internal-sftp" + TCPKeepAlive: "yes" + UseDNS: "no" + UsePAM: "no" + X11Forwarding: "no" + + + # Include: "/etc/ssh/sshd_config.d/*.conf" + # Subsystem: sftp internal-sftp + # + # The `/etc/sssd/sssd.conf` represented by as a map. + sssdConf: + # + # -- (map) + # The `/etc/sssd/sssd.conf` [sssd] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#The_%5Bsssd%5D_section + sssd: + # debug_level: 9 + config_file_version: 2 + services: nss, pam + domains: DEFAULT + # + # -- (map[map]) + # The `/etc/sssd/sssd.conf` [domain/$DOMAIN] sections, represented as a map of map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#DOMAIN_SECTIONS + domains: + DEFAULT: + # debug_level: 9 + auth_provider: ldap + id_provider: ldap + ldap_uri: ldap://ldap.example.com + ldap_search_base: dc=example,dc=com + ldap_user_search_base: ou=Users,dc=example,dc=com + ldap_group_search_base: ou=Groups,dc=example,dc=com + # + # -- (map) + # The `/etc/sssd/sssd.conf` [nss] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#NSS_configuration_options + nss: + # debug_level: 9 + filter_groups: root,slurm + filter_users: root,slurm + # + # -- (map) + # The `/etc/sssd/sssd.conf` [pam] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#PAM_configuration_options + pam: {} + # debug_level: 9 + # + # --(map) + # The `/etc/nsswitch.conf` file to use, represented as a map. + # Ref: https://man7.org/linux/man-pages/man5/nsswitch.conf.5.html + nsswitchConf: + passwd: files + group: files + shadow: files + gshadow: files + sudoers: files + hosts: files + networks: files + protocols: files + services: files + ethers: files + rpc: files + netgroup: files + automount: files + # + # -- (list) + # List of volumes to be mounted on each login pod. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + # - name: nfs-home + # mountPath: /home + # persistentVolumeClaim: + # claimName: nfs-home + # - name: nfs-data + # mountPath: /mnt/data + # persistentVolumeClaim: + # claimName: nfs-data + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: "In" + values: + - "ml.m5.2xlarge" + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# Slurm compute (slurmd) configurations. +compute: + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Default image for the nodeset pod (slurmd) + # Each nodeset may override this setting. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmd + # + # -- (string) + # Set the image tag to use. + # @default -- The Release appVersion. + tag: 24.11-ubuntu24.04 + # + # -- (list) + # Slurm NodeSets by object list. + nodesets: + # + # -- (string) + # Name of NodeSet. Must be unique. + - name: hp-node + # + # -- (bool) + # Enables the NodeSet in Slurm. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 4 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: "159553542841.dkr.ecr.us-west-2.amazonaws.com/dlc-slurmd" + # + # -- (string) + # Set the image tag to use. + tag: "24.11.4-ubuntu24.04" + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: + limits: + cpu: "32" + memory: "128Gi" + nvidia.com/gpu: "1" + requests: + cpu: "1" + memory: "1Gi" + nvidia.com/gpu: "1" + # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + nodeSelector: + kubernetes.io/os: linux + node.kubernetes.io/instance-type: ml.g5.8xlarge + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "kubernetes.io/os" + # operator: In + # values: + # - linux + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - topologyKey: "kubernetes.io/hostname" + # labelSelector: + # matchExpressions: + # - key: "app.kubernetes.io/name" + # operator: In + # values: + # - slurmctld + # - slurmdbd + # - slurmrestd + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set the update strategy configuration. + updateStrategy: + # + # -- (string) + # Set the update strategy type. + # Can be either: "RollingUpdate"; "OnDelete". + type: RollingUpdate + # + # -- (object) + # Define the rolling update policy. + # Only used when "updateStrategy.type=RollingUpdate". + rollingUpdate: + # + # -- (string) + # The maximum number of pods that can be unavailable during the update. + # Value can be an absolute number (ex: 5) or a percentage of desired + # pods (ex: 10%). Absolute number is calculated from percentage by + # rounding up. This can not be 0. Defaults to 1. + maxUnavailable: 20% + # + # -- (object) + # The policy used for PVCs created from the NodeSet VolumeClaimTemplates. + persistentVolumeClaimRetentionPolicy: + # + # -- (string) + # WhenDeleted specifies what happens to PVCs created from NodeSet + # VolumeClaimTemplates when the NodeSet is deleted. The default policy + # of `Retain` causes PVCs to not be affected by NodeSet deletion. The + # `Delete` policy causes those PVCs to be deleted. + whenDeleted: Retain + # + # -- (string) + # WhenScaled specifies what happens to PVCs created from NodeSet + # VolumeClaimTemplates when the NodeSet is scaled down. The default + # policy of `Retain` causes PVCs to not be affected by a scale-in. The + # `Delete` policy causes the associated PVCs for any excess pods to be + # deleted. + whenScaled: Retain + # + # -- (list) + # List of PVCs to be created from template and mounted on each NodeSet pod. + # PVCs are given a unique identity relative to each NodeSet pod. + # Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#volume-claim-templates + volumeClaimTemplates: [] + # - metadata: + # name: scratch + # spec: + # mountPath: /mnt/scratch + # storageClassName: standard + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 1Gi + # + # -- (list) + # List of volumes to be mounted on each NodeSet pod. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + # - name: nfs-bin + # mountPath: /usr/local/bin + # nfs: + # server: nfs-server.example.com + # path: /opt/bin + # readOnly: true + # - name: nfs-data + # mountPath: /mnt/data + # persistentVolumeClaim: + # claimName: nfs-data + # + # -- (object) + # Partition describes the partition created specifically for this NodeSet to be added. + partition: + # + # -- (bool) + # Enables this NodeSet's partition line to be added in Slurm. + enabled: true + # + # -- (map[string]string | map[string][]string) + # Extra Slurm partition configuration appended onto the partition line. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAI + config: + State: UP + MaxTime: UNLIMITED + # + # -- (string) + # Set Slurm node GRES. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1 + nodeGres: "" + # + # -- (list) + # Set Slurm node Features as a list(string). + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Features + nodeFeatures: [] + # + # -- (string) + # Set Slurm node weight for Slurm scheduling. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Weight + nodeWeight: 1 + # + # -- (list) + # Slurm Partitions by object list. + partitions: + # + # -- (string) + # Name of Partition. Must be unique. + - name: all + # + # -- (bool) + # Enables the partition in Slurm. + enabled: true + # + # -- (list) + # NodeSets to put into this Partition by name/key. + # NOTE: 'ALL' is a Slurm meta value to mean all nodes in the system. + nodesets: + - ALL + # + # -- (map[string]string | map[string][]string) + # Extra Slurm partition configuration appended onto the partition line. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAI + config: + State: UP + Default: "YES" + MaxTime: UNLIMITED + +# +# Slurm accounting (slurmdbd) configurations. +accounting: + # + # -- (bool) + # Enables accounting services. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmdbd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: "In" + values: + - "ml.m5.2xlarge" + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + # + # Configuration for an external accounting instance (slurmdbd). + external: + # + # -- (bool) + # Use an external acounting instance (slurmdbd) instead of deploying one. + enabled: false + # + # -- (string) + # The external acounting instance (slurmdbd) host. + host: "" + # + # -- (integer) + # The external acounting instance (slurmdbd) port. + port: 6819 + +# +# `bitnami/mariadb` subchart configurations. +# Ref: https://github.com/bitnami/charts/blob/main/bitnami/mariadb/values.yaml +mariadb: + enabled: true + auth: + username: slurm + database: slurm_acct_db + tls: + enabled: false + tde: + enabled: false + primary: + # NOTE: https://slurm.schedmd.com/accounting.html#slurm-accounting-configuration-before-build + configuration: |- + [mysqld] + skip-name-resolve + explicit_defaults_for_timestamp + basedir=/opt/bitnami/mariadb + datadir=/bitnami/mariadb/data + plugin_dir=/opt/bitnami/mariadb/plugin + port={{ .Values.primary.containerPorts.mysql }} + socket=/opt/bitnami/mariadb/tmp/mysql.sock + tmpdir=/opt/bitnami/mariadb/tmp + innodb_buffer_pool_size=4096M + innodb_lock_wait_timeout=900 + innodb_log_file_size=1024M + max_allowed_packet=16M + bind-address=* + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + log-error=/opt/bitnami/mariadb/logs/mysqld.log + character-set-server=UTF8 + collation-server=utf8_general_ci + slow_query_log=0 + long_query_time=10.0 + binlog_expire_logs_seconds=2592000 + {{- if .Values.tls.enabled }} + ssl_cert=/opt/bitnami/mariadb/certs/{{ .Values.tls.certFilename }} + ssl_key=/opt/bitnami/mariadb/certs/{{ .Values.tls.certKeyFilename }} + {{- if (include "mariadb.tlsCACert" .) }} + ssl_ca={{ include "mariadb.tlsCACert" . }} + {{- end }} + {{- end }} + {{- if .Values.tde.enabled }} + plugin_load_add=file_key_management + file_key_management_filename=/opt/bitnami/mariadb/tde/{{ .Values.tde.encryptedKeyFilename }} + file_key_management_filekey=FILE:/opt/bitnami/mariadb/tde/{{ .Values.tde.randomKeyFilename }} + file_key_management_encryption_algorithm={{ .Values.tde.fileKeyManagementEncryptionAlgorithm }} + innodb_encrypt_tables={{ .Values.tde.innodbEncryptTables }} + innodb_encrypt_log={{ .Values.tde.innodbEncryptLog }} + innodb_encrypt_temporary_tables={{ .Values.tde.innodbEncryptTemporaryTables }} + innodb_encryption_threads={{ .Values.tde.innodbEncryptionThreads }} + encrypt_tmp_disk_tables={{ .Values.tde.encryptTmpDiskTables }} + encrypt_tmp_files={{ .Values.tde.encryptTmpTiles }} + encrypt_binlog={{ .Values.tde.encryptBINLOG }} + aria_encrypt_tables={{ .Values.tde.ariaEncryptTables }} + {{- end }} + + [client] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + default-character-set=UTF8 + plugin_dir=/opt/bitnami/mariadb/plugin + + [manager] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + persistence: + enabled: false + existingClaim: "" + storageClass: standard + labels: {} + annotations: {} + accessModes: + - ReadWriteOnce + size: 8Gi + selector: {} + priorityClassName: "" + tolerations: [] + metrics: + enabled: false + serviceMonitor: + enabled: false + affinity: {} + resources: {} + +# +# Slurm REST API (slurmrestd) configurations. +restapi: + # + # -- (bool) + # Enables restapi services. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmrestd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # The restapi service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: {} + # type: LoadBalancer + # externalIPs: [] + # externalName: my.slurmrestd.example.com + # + # -- (integer) + # The external service port number. + servicePort: 6820 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 36820 + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# `slurm-exporter` subchart configurations. +# Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml +slurm-exporter: + enabled: true + exporter: + enabled: true + secretName: "slurm-token-exporter" + # image: + # repository: ghcr.io/slinkyproject/slurm-exporter + # tag: 0.2.0 From a3ebffa60fbf3b1df051392e079ea3ed607013a9 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Fri, 25 Apr 2025 14:28:36 -0400 Subject: [PATCH 02/15] updated dockerfile and readme --- .../slinky-slurm/.gitignore | 3 + .../slinky-slurm/Docker-Build-README.md | 152 +++++++ .../slinky-slurm/README.md | 428 +++++++++--------- .../slinky-slurm/dlc-slurmd.Dockerfile | 72 +-- .../slinky-slurm/llama2_7b-training.sbatch | 122 +++++ .../slinky-slurm/slinky-slurm-hp-eks.png | Bin 0 -> 172916 bytes .../slinky-slurm/values.yaml | 61 ++- 7 files changed, 569 insertions(+), 269 deletions(-) create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky-slurm-hp-eks.png diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore new file mode 100644 index 000000000..b6d262826 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore @@ -0,0 +1,3 @@ +slurm*/ + +*.tgz \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md index 20daebd31..b16d7a918 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md @@ -1,2 +1,154 @@ # Docker Build for the Slurmd Deep Learning Container +This build includes Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + +Clone the AWSome Distributed Training repo: +``` +https://github.com/aws-samples/awsome-distributed-training.git +cd awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/ + +``` + +Build the container image: + +``` + +# Authenticate to DLC repo (Account 763104351884 is publicly known) +aws ecr get-login-password --region us-east-1 \ +| docker login --username AWS \ +--password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com + +# on a Mac +docker buildx build --platform linux/amd64 -t dlc-slurmd:24.11.4-ubuntu24.04 -f dlc-slurmd.Dockerfile . + +# on Linux +# docker build -t dlc-slurmd:24.11.4-ubuntu24.04 -f dlc-slurmd.Dockerfile . + +``` + +Test locally: + +Verify Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + +``` + +docker run --platform linux/amd64 -it --entrypoint=/bin/bash dlc-slurmd:24.11.4-ubuntu24.04 + +python3 --version +# Python 3.12.8 + +which python3 +# /usr/local/bin/python3 + +nvcc --version +# nvcc: NVIDIA (R) Cuda compiler driver +# Copyright (c) 2005-2024 NVIDIA Corporation +# Built on Tue_Oct_29_23:50:19_PDT_2024 +# Cuda compilation tools, release 12.6, V12.6.85 +# Build cuda_12.6.r12.6/compiler.35059454_0 + +python3 -c "import torch; print(torch.__version__)" +# 2.6.0+cu126 + +python3 -c "import torch; print(torch.cuda.nccl.version())" +# (2, 23, 4) + +ls -l /usr/local/lib/libnccl* +# -rwxr-xr-x 1 root root 263726576 Mar 6 23:36 /usr/local/lib/libnccl.so +# -rwxr-xr-x 1 root root 263726576 Mar 6 23:36 /usr/local/lib/libnccl.so.2 +# -rwxr-xr-x 1 root root 263726576 Mar 6 23:36 /usr/local/lib/libnccl.so.2.23.4 +# -rw-r--r-- 1 root root 277972056 Mar 6 23:36 /usr/local/lib/libnccl_static.a + +cat /etc/nccl.conf +# NCCL_DEBUG=INFO +# NCCL_SOCKET_IFNAME=^docker0 + +``` + +Create a private ECR repo: + +``` + +aws ecr create-repository --repository-name dlc-slurmd + +``` + +Authenticate to the repo: + +``` +export AWS_ACCOUNT_ID= +export AWS_REGION= + +aws ecr get-login-password --region $AWS_REGION \ + | docker login --username AWS \ + --password-stdin ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com + +``` + +Tag the image: + +``` + +docker tag dlc-slurmd:24.11.4-ubuntu24.04 \ + ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 + +``` + +Push the image to an ECR repo: + +``` + +docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 + +``` + +Test ECR access: + +``` + +kubectl run test-pod \ + --image=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 \ + --restart=Never \ + --image-pull-policy=Always + +# verify slurm version +kubectl exec -it test-pod -- slurmd -V + +kubectl describe pod test-pod + +# verify additional requirements +kubectl exec -it test-pod -- ls /usr/local/lib/python3.12/site-packages/ \ + | egrep "datasets|fsspec|numpy|torch|torchaudio|torchvision|transformers" + +kubectl delete pod test-pod + +``` + +(Optional) Update the container image used by the Slinky NodeSet: + +Note: this step is not required if you specify the image repository and tag in the [values.yaml](./values.yaml) file, but is useful if you want to test a new image build without redeploying the entire Slurm cluster. + +``` +export NODESET_NAME=$(kubectl get nodeset -n slurm -o custom-columns=NAME:metadata.name --no-headers) + +kubectl -n slurm patch nodeset.slinky.slurm.net \ + $NODESET_NAME \ + --type='json' \ + -p="[ + {\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/image\", \"value\":\"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04\"}, + {\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/imagePullPolicy\", \"value\":\"Always\"} + ]" + +``` + +Scale the Slinky NodeSet down and back up to trigger replacement: + +``` + +kubectl -n slurm scale nodeset/$NODESET_NAME --replicas=0 + + +kubectl -n slurm scale nodeset/$NODESET_NAME --replicas=4 + +``` + diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 84f7b9c24..d409dd6cd 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -1,21 +1,41 @@ # Running Slurm on HyperPod EKS with Slinky ### What is the Slinky Project? + +The [Slinky Project](https://github.com/SlinkyProject/slurm-operator/tree/main) is an open-source solution maintained by SchedMD (the main developer of Slurm) that deploys Slurm on Kubernetes. When paired with HyperPod EKS, the Slinky Project unlocks the ability for enterprises who have standardized infrastructure management on Kubernetes to deliver a Slurm-based experience to their ML scientists. It also enables training, experimentation, and inference to happen on the same cluster of accelerated nodes with the build-in resiliency provided by HyperPod. + --- -### Architecture +### Slinky on HypePod EKS Architecture +![Image Description](./slinky-slurm-hp-eks.png) + +The diagram above depicts the resulting proof-of-concept deployment outlined in this guide. An Amazon EKS cluster acts as an orchestration layer, while a HyperPod cluster deliver a resilient instance group of GPU accelerated compute nodes. The Slinky Slurm operator is installed to extend Kubernetes with custom resources and actions, and a containerized Slurm cluster is deployed using Kubernetes pods via Helm chart. This Slurm cluster includes the following components: +| Component | Description | +|-----------|-------------| +| Controller (slurmctld) | The central management daemon that monitors resources, accepts jobs, and assigns work to compute nodes. | +| Accounting (slurmdbd) | Handles job accounting and user/project management through a MariaDB database backend. | +| Compute (slurmd) | The worker nodes that execute jobs, organized into NodeSets which can be grouped into different partitions. | +| Login | Provides SSH access points for users to interact with the Slurm cluster and submit jobs. | +| REST API (slurmrestd) | Offers HTTP-based API access to Slurm functionality for programmatic interaction with the cluster. | +| Authentication (sackd) | Manages credential authentication for secure access to Slurm services. | +| MariaDB | The database backend used by the accounting service to store job, user, and project information. | +| Slurm Exporter | Collects and exports Slurm metrics for monitoring purposes. | + +The login LoadBalancer type service is annotated to dynamically create an AWS Network Load Balancer using the [AWS Load Balancer Controller](https://github.com/kubernetes-sigs/aws-load-balancer-controller), allowing ML scientists to SSH into their login pods without interfacing with the Kubernetes API server via kubectl. + +The login and compute node pods also have FSx for Lustre and FSx for OpenZFS shared filesystems mounted. Having containerized compute node pods allows many dependencies that would traditionally be installed manually using Conda or a Python virtual environment to be baked into the container image, but shared filesystems are still beneficial for storing training artifacts, data, logs, and checkpoints. If Conda environments are still required, FSx for OpenZFS has proven optimal to avoid IOPS saturation with many small files. --- ### Release Notes -The following was tested on 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) for hosting the Worker Pod NodeSet. 2 `m5.2xlarge` instances were also allocated for separately hosting the Controller Pod and Login Pod. Other components (Accounting, MariaDB, RestAPI, Token Creation, Exporter) were also colocated across the 2 `m5.2xlarge` instances for simplicity, but you may wish to deploy them on separate instance depending on your specific needs. This can be accomplished by modifying the associated node affinity configurations, which is discussed in more detail below. +The following was tested on 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) for hosting the compute NodeSet pods. For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the [values.yaml](./values.yaml) file to modify how they are spread across your nodes. -Testing used [Slurm Operator v0.2.1](https://github.com/slinkyproject/slurm-operator/pkgs/container/slurm-operator) (pulled from the Slinky GHCR) and [Slurm Cluster v0.3.0](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) (packaged and deployed locally using the main Slinky repo branch) in order to include the NoteSet volume mount and Login Pod features. These features are expected to be included in the official Slurm Cluster v0.3.0 release when it becomes available on the Slinky GHCR repo, along with a new version of the Slurm Operator with corresponding validating webhooks. +Testing used [Slurm Operator v0.2.1](https://github.com/slinkyproject/slurm-operator/pkgs/container/slurm-operator) (pulled as OCI artifacts from the Slinky container registry) and [Slurm Cluster v0.3.0](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) (packaged and deployed locally using the main branch of the Slinky git repository) in order to include the NoteSet volume mount and Login Pod features. These features are expected to be included in the official Slurm Cluster v0.3.0 release when it becomes available, along with a new version of the Slurm Operator with corresponding validating webhooks. -Note that the [Slinky Project](https://github.com/SlinkyProject) is under active development and could introduce breaking changes that may require modified deployment steps and configuration changes. +Note that the [Slinky Project](https://github.com/SlinkyProject) is under active development and could introduce breaking changes that may require modified deployment and configuration steps. -Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image. See [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. +Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image. See the [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. * * * @@ -26,7 +46,7 @@ Follow the [Prerequisites](https://catalog.workshops.aws/sagemaker-hyperpod-eks/ Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. -Add an access entry (if needed): +(Optional) Add an access entry (if needed): ``` export AWS_ACCOUNT_ID= @@ -172,7 +192,7 @@ kubectl get sa aws-load-balancer-controller -n kube-system -oyaml * * * -### Instill Prerequisites (Cert Manager and Prometheus): +### Instill Slinky Prerequisites (Cert Manager and Prometheus): Follow the [QuickStart Guide](http://curl%20-l%20https//raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.1.0/helm/slurm-operator/values.yaml%20/%20%20%20-o%20values-operator.yaml%20helm%20install%20slurm-operator%20oci://ghcr.io/slinkyproject/charts/slurm-operator%20/%20%20%20--values=values-operator.yaml%20--version=0.1.0%20--namespace=slinky%20--create-namespace) to install Cert Manager and Prometheus as [Pre-Requisites](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites). @@ -215,49 +235,87 @@ kubectl get all -n slinky ### Install the Slurm Cluster: -To deploy the **slurm cluster**, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, again, in order to test the latest changes in release **v0.3.0**, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. For your convenience, we've provided a copy of the [values.yaml](./values.yaml) file with most of the configuration changes mentioned below already implemented. +To deploy the slurm cluster, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, in order to test the latest changes in release v0.3.0, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. For your convenience, we've provided a copy of the [values.yaml](./values.yaml) file with most of the configuration changes mentioned below already implemented, so you'll only need to make additional changes as needed to further customize your deployment. + +--- +#### Clone the Repos Clone the Slurm Operator repository, which also contains the Helm chart artifacts for the Slurm Cluster: ``` git clone https://github.com/SlinkyProject/slurm-operator.git ``` -(Optional) If you wish to start from scratch, open the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file associated with the Slurm Cluster Helm Chart: + +Clone the AWSome Distributed Training repo to use the [values.yaml](./values.yaml) file we've provided: ``` -code slurm-operator/helm/slurm/values.yaml +git clone https://github.com/aws-samples/awsome-distributed-training.git +cd awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm +``` + +(Optional) If you wish to start from scratch, open the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml) file associated with the Slurm Cluster Helm Chart: +``` +code slurm-operator/helm/slurm/values.yaml ``` -Otherwise, you can use the [values.yaml](./values.yaml) file we've provided. +--- + +#### Component Affinity: -Verify the existence of the instance type label for controller affinity: +Verify the existence of the instance type label for non-compute component affinity: ``` export GEN_INSTANCE_TYPE=ml.m5.2xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE - ``` -#### Controller Modifications: - -Configure controller affinity: +Verify the name pod labels applied to each component: +``` +kubectl get pods -n slurm -L app.kubernetes.io/name + +# NAME READY STATUS RESTARTS AGE NAME +# slurm-accounting-0 1/1 Running 0 12m slurmdbd +# slurm-compute-hp-node-0 2/2 Running 0 12m slurmd +# slurm-compute-hp-node-1 2/2 Running 0 12m slurmd +# slurm-compute-hp-node-2 2/2 Running 0 12m slurmd +# slurm-compute-hp-node-3 2/2 Running 0 12m slurmd +# slurm-controller-0 2/2 Running 0 12m slurmctld +# slurm-exporter-86448948f4-rqtg8 1/1 Running 0 12m slurm-exporter +# slurm-login-78b8fc9cd-rz8qj 1/1 Running 0 12m login +# slurm-mariadb-0 1/1 Running 0 12m mariadb +# slurm-restapi-55d998b698-gdzc6 1/1 Running 0 12m slurmrestd +# slurm-token-create-b297g 0/3 Completed 0 12m token +``` + +For each non-compute component, we apply both a Node Affinity and a Pod Anti-affinity in [values.yaml](./values.yaml) to ensure they are hosted only on the 2 `m5.2xlarge` instances while also being evenly spread between the hosts. + +``` +# Inter-pod anti-affinity and node affinity for non-compute components +commonAffinity: &commonAffinity + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - "ml.m5.2xlarge" + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: ["slurmdbd", "slurmctld", "slurm-exporter", "login", "mariadb", "slurmrestd"] + topologyKey: "kubernetes.io/hostname" +``` +You can modify this common affinity setting, or apply unique affinity settings for individual components for further customization. -``` -controller: -... - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: "node.kubernetes.io/instance-type" - operator: "In" - values: - - "ml.m5.2xlarge" -... -``` +--- -#### Compute NodeSet Modifications: +#### Compute Node Selector: Verify the existence of the instance type label for compute node selector: @@ -268,7 +326,7 @@ ACCEL_INSTANCE_TYPE=ml.g5.8xlarge ``` -Change the compute node name, replica count, and node selector:: +The instance type label is used as a node selector to ensure the compute pods only run on the `ml.g5.8xlarge` GPU accelerated instances: ``` compute: @@ -283,6 +341,9 @@ compute: node.kubernetes.io/instance-type: ml.g5.8xlarge ... ``` +--- + +#### Create an FSx for Lustre Persistent Volume Claim (PVC) in the slurm namespace: Create the slurm namespace: @@ -290,8 +351,6 @@ Create the slurm namespace: kubectl create ns slurm ``` -Create a FSx for Lustre Persistent Volume Claim (PVC) in the slurm namespace: - This is needed to reference for node volume mounts later. ``` @@ -313,8 +372,9 @@ kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ | jq -r .spec.csi.volumeHandle ``` +--- -Create an FSx for OpenZFS PVC in the slurm namespace: +#### Create an FSx for OpenZFS PVC in the slurm namespace: ``` kubectl apply -f openzfs-pvc-slurm.yaml @@ -337,26 +397,41 @@ kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ ``` -Add the FSx for Lustre and OpenZFS PVCs to the list of compute node volumes: +Add the FSx for Lustre and OpenZFS PVCs to the list of `volumes` for both the login service and and compute nodes in [values.yaml](./values.yaml): ``` +login: + ... + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + ... + compute: - nodesets: - - name: hp-node - ... - volumes: - - name: fsx-lustre - mountPath: /fsx - persistentVolumeClaim: - claimName: fsx-claim - - name: fsx-openzfs - mountPath: /home - persistentVolumeClaim: - claimName: openzfs-claim - ... + nodesets: + - name: hp-node + ... + volumes: + - name: fsx-lustre + mountPath: /fsx + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + mountPath: /home + persistentVolumeClaim: + claimName: openzfs-claim + ... ``` +--- + +#### Configure Compute Node Resources: -Configure resources for compute nodes: Note: limits are required, otherwise the compute nodes will not deploy. ``` @@ -375,8 +450,13 @@ compute: nvidia.com/gpu: "1" ... ``` +--- -Modify the compute node container image to use the [Slurmd Deep Learning Container](./Docker-Build-README.md) (Slurmd DLC) build: +#### Build and Set the Compute Node Container Image: + +Use the provided [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) to build a [Slurmd Deep Learning Container](./Docker-Build-README.md) (Slurmd DLC), following [the instructions here](./Docker-Build-README.md). + +then modify the compute node container image to use your Slurmd DLC build: ``` compute: @@ -395,27 +475,15 @@ compute: tag: "24.11.4-ubuntu24.04" ... ``` -For your convenience, we've pre-build a Slurmd DLC for and made it available in an ECR public repository, but you can use the provided [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) to modify and build your own. +The Slurm DLC has Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image, but you can modify the [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) for further customization. -#### Login Node Modifications: +--- -Add the FSx for Lustre and OpenZFS PVCs to the list of login node volumes: +#### Login Access: -``` -login: - ... - volumes: - - name: fsx-lustre - mountPath: /fsx - persistentVolumeClaim: - claimName: fsx-claim +Access to the login service can be configured through several authentication and networking mechanisms. The login service can be exposed either as a `LoadBalancer` (default) or `NodePort` type service, with the external port configurable via `servicePort` (default 22) or `serviceNodePort` (default 32222) respectively. Authentication can be integrated with LDAP through SSSD configuration, where users and groups can be managed via the `sssdConf` settings that define LDAP URIs, search bases, and domain configurations. SSH access can be customized through both `sshdConfig` and `rootSshAuthorizedKeys` parameters, allowing for specific SSH daemon configurations and authorized key management. Additionally, the name service switch configuration (`nsswitchConf`) can be customized to control how various databases like passwd, group, and hosts are resolved, with support for multiple sources including files, SSS, and database lookups. - - name: fsx-openzfs - mountPath: /home - persistentVolumeClaim: - claimName: openzfs-claim - ... -``` +For simplicity of demonstration, we've disabled SSSD by setting everything in `nsswitchConf` to `files` so the system to only uses local files for authentication and does not to try SSSD or other sources. This is simpler and more reliable when you just want to use SSH key authentication for root access, as the SSH keys are stored in local files (/root/.ssh/authorized_keys). Generate an SSH key for root authorization: @@ -440,107 +508,51 @@ login: - "ssh-ed25519 janedoe@example.com" ... ``` +--- -Disable SSSD: the `nsswitchConf` (Name Service Switch configuration) file tells Linux how to resolve different types of system information like users, groups, passwords, etc. By setting everything to just `files` we're telling the system to only use local files for authentication and not to try SSSD or other sources. This is simpler and more reliable when you just want to use SSH key authentication for root access, as the SSH keys are stored in local files anyway (/root/.ssh/authorized_keys). - -``` -... -login: - ... - nsswitchConf: - passwd: files - group: files - shadow: files - gshadow: files - sudoers: files - hosts: files - networks: files - protocols: files - services: files - ethers: files - rpc: files - netgroup: files - automount: files - ... -... -``` - -Define the content of the sshd_config file: +#### Deploy the Slurm Cluster: -``` -... -login: - sshdConfig: - # This is the actual content of the sshd_config file - AcceptEnv: "LANG LC_*" - AuthorizedKeysFile: "/root/.ssh/authorized_keys" - ChallengeResponseAuthentication: "no" - ClientAliveCountMax: "3" - ClientAliveInterval: "60" - LogLevel: "INFO" - PasswordAuthentication: "no" - PermitRootLogin: "yes" - Port: "22" - PrintMotd: "no" - Protocol: "2" - PubkeyAuthentication: "yes" - Subsystem: "sftp internal-sftp" - TCPKeepAlive: "yes" - UseDNS: "no" - UsePAM: "no" - X11Forwarding: "no" -... -``` - -Update the **slurm-login** service port: +Locally package and deploy the slurm cluster using the modified `values.yaml` file: +Assuming you are still sitting in the `slinky-slurm` directory of the AWSome Distributed Training repo that we cloned and navigated into earlier, and assuming you cloned the Slinky repo into your home directory (adjust the path as needed), copy the Helm chart artifacts in for packaging: ``` -login: - ... - servicePort: 22 - ... +cp -r ~/slurm-operator/helm/slurm . ``` -#### Deploy the Slurm Cluster - -Locally package and deploy the **slurm cluster** using the modified `values.yaml` file: +Locally package and deploy the Slurm cluster Helm chart v0.3.0: ``` -helm dependency update slurm-operator/helm/slurm +helm dependency update slurm -helm package slurm-operator/helm/slurm - -slurm-0.3.0.tgz +helm package slurm # Dry run helm install --dry-run slurm slurm-0.3.0.tgz \ ---values=values.yaml \ ---namespace=slurm +-f values.yaml \ +-n slurm helm install slurm slurm-0.3.0.tgz \ ---values=values.yaml \ ---namespace=slurm - +-f values.yaml \ +-n slurm ``` -Note: Release v0.2.1 of the slurm-operator validating webhook may throw a few warning about not recognizing `spec.template.spec.volumes[].mountPath` fields. This is not surprising given we are using the newer pre-release v0.3.0 of the slurm cluster, but it doesn’t appear to cause any functional errors. - - Watch the deployment status of the Slurm cluster: ``` -kubectl --namespace=slurm get pods -l app.kubernetes.io/instance=slurm --watch +kubectl -n slurm get pods -l app.kubernetes.io/instance=slurm --watch ``` -Verify deployment status of all components: +Verify the deployment status of all components: ``` kubectl get all -n slurm ``` -#### Configure Network Load Balancer provisioning using the AWS Load Balancer Controller +--- + +#### Configure Login Network Load Balancer provisioning using the AWS Load Balancer Controller: -Manually add annotation to the slurm-login service: +Manually add annotation to the `slurm-login` service: ``` export PUBLIC_SUBNET_ID_1= @@ -555,130 +567,117 @@ kubectl annotate service slurm-login -n slurm \ --overwrite kubectl describe service slurm-login -n slurm - ``` Any annotations added to the slurm cluster `values.yaml` file for the slurm-login service are currently ignored, but AWS Load Balancer Controller actively watches for and implements annotation changes. It Automatically adds inbound rules to the node security group to allow traffic from the NLB security group on the target port (22 in this case). -* * * + +--- ### Basic Tests: SSH into the login node as root from the NLB endpoint: ``` - SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME - ``` +--- Check the available nodes: ``` - sinfo PARTITION AVAIL TIMELIMIT NODES STATE NODELIST hp-node up infinite 4 idle hp-node-[0-3] all* up infinite 4 idle hp-node-[0-3] - ``` +--- Verify FSx for Lustre and OpenZFS filesystem mounts on the login pod: ``` - df -h -Filesystem Size Used Avail Use% Mounted on -overlay 500G 30G 471G 6% / -tmpfs 64M 0 64M 0% /dev -tmpfs 63G 0 63G 0% /sys/fs/cgroup -10.1.12.93@tcp:/7c5dpb4v 1.2T 7.8M 1.2T 1% /fsx -fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home -tmpfs 115G 4.0K 115G 1% /etc/slurm -/dev/nvme0n1p1 100G 23G 78G 23% /run -/dev/nvme1n1 500G 30G 471G 6% /etc/hostname -shm 64M 0 64M 0% /dev/shm -tmpfs 115G 4.0K 115G 1% /etc/sssd/sssd.conf -tmpfs 115G 12K 115G 1% /etc/ssh/ssh_host_rsa_key -tmpfs 63G 0 63G 0% /proc/acpi -tmpfs 63G 0 63G 0% /sys/firmware +# Filesystem Size Used Avail Use% Mounted on +# overlay 500G 30G 471G 6% / +# tmpfs 64M 0 64M 0% /dev +# tmpfs 63G 0 63G 0% /sys/fs/cgroup +# 10.1.12.93@tcp:/7c5dpb4v 1.2T 7.8M 1.2T 1% /fsx +# fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home +# tmpfs 115G 4.0K 115G 1% /etc/slurm +# /dev/nvme0n1p1 100G 23G 78G 23% /run +# /dev/nvme1n1 500G 30G 471G 6% /etc/hostname +# shm 64M 0 64M 0% /dev/shm +# tmpfs 115G 4.0K 115G 1% /etc/sssd/sssd.conf +# tmpfs 115G 12K 115G 1% /etc/ssh/ssh_host_rsa_key +# tmpfs 63G 0 63G 0% /proc/acpi +# tmpfs 63G 0 63G 0% /sys/firmware exit - ``` +--- -Verify FSx for Lustre and OpenZFS filesystem mounts on the worker node pods: +Verify FSx for Lustre and OpenZFS filesystem mounts on the compute node pods: ``` - kubectl -n slurm exec -it pod/slurm-compute-hp-node-0 -- bash --login df -h -Filesystem Size Used Avail Use% Mounted on -overlay 500G 31G 470G 7% / -tmpfs 64M 0 64M 0% /dev -tmpfs 63G 0 63G 0% /sys/fs/cgroup -10.1.12.93@tcp:/7c5dpb4v 1.2T 7.5M 1.2T 1% /fsx -fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home -tmpfs 115G 4.0K 115G 1% /etc/slurm -/dev/nvme0n1p1 100G 23G 78G 23% /run -/dev/nvme1n1 500G 31G 470G 7% /etc/hostname -shm 64M 0 64M 0% /dev/shm -tmpfs 115G 0 115G 0% /var/log/slurm - +# Filesystem Size Used Avail Use% Mounted on +# overlay 500G 31G 470G 7% / +# tmpfs 64M 0 64M 0% /dev +# tmpfs 63G 0 63G 0% /sys/fs/cgroup +# 10.1.12.93@tcp:/7c5dpb4v 1.2T 7.5M 1.2T 1% /fsx +# fs-03221b7c7d3767607.fsx.us-west-2.amazonaws.com:/fsx 64G 0 64G 0% /home +# tmpfs 115G 4.0K 115G 1% /etc/slurm +# /dev/nvme0n1p1 100G 23G 78G 23% /run +# /dev/nvme1n1 500G 31G 470G 7% /etc/hostname +# shm 64M 0 64M 0% /dev/shm +# tmpfs 115G 0 115G 0% /var/log/slurm ``` +--- -Check the installed CUDA compiler version on worker node pods: +Check the installed CUDA compiler version on compute node pods: ``` - nvcc --version -# nccl-slurmd -nvcc: NVIDIA (R) Cuda compiler driver -Copyright (c) 2005-2023 NVIDIA Corporation -Built on Tue_Aug_15_22:02:13_PDT_2023 -Cuda compilation tools, release 12.2, V12.2.140 -Build cuda_12.2.r12.2/compiler.33191640_0 - -# dlc-slurmd -nvcc: NVIDIA (R) Cuda compiler driver -Copyright (c) 2005-2024 NVIDIA Corporation -Built on Tue_Oct_29_23:50:19_PDT_2024 -Cuda compilation tools, release 12.6, V12.6.85 -Build cuda_12.6.r12.6/compiler.35059454_0 +# nvcc: NVIDIA (R) Cuda compiler driver +# Copyright (c) 2005-2024 NVIDIA Corporation +# Built on Tue_Oct_29_23:50:19_PDT_2024 +# Cuda compilation tools, release 12.6, V12.6.85 +# Build cuda_12.6.r12.6/compiler.35059454_0 ``` +--- -Check the NCCL version on worker node pods: +Check the NCCL version on compute node pods: ``` ldconfig -v | grep "libnccl.so" | tail -n1 | sed -r 's/^.*\.so\.//' -2.23.4 +# 2.23.4 ``` +--- Confirm NCCL headers are installed worker node pods: ``` find /usr/local/lib/ -name "nccl.h" 2>/dev/null -/usr/local/lib/python3.12/site-packages/torch/include/torch/csrc/cuda/nccl.h +# /usr/local/lib/python3.12/site-packages/torch/include/torch/csrc/cuda/nccl.h exit ``` - -* * * +--- ### FSDP Test SSH into the login pod as root, clone the repo, and create a checkpoints directory: ``` - SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME @@ -687,46 +686,53 @@ apt update apt install -y git git --version -# install vim -`apt install ``-``y vim ` +# install vim (optional) +apt install -y vim vim --version cd /fsx git clone https://github.com/aws-samples/awsome-distributed-training/ cd awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm -mkdir checkpoints +mkdir -p checkpoints ``` +--- Download the c4 dataset to avoid throttling errors from HuggingFace: ``` mkdir -p /fsx/datasets/c4 +export SLINKY_PATH=/fsx/awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm + apt install -y python3.12-venv python3 -m venv env source env/bin/activate pip install --upgrade pip pip install datasets -python3 download_c4.py +python3 ${SLINKY_PATH}/download_c4.py deactivate ``` -Kick-off training: - +--- +Copy the modified sbatch file and kick-off training: ``` +cp ${SLINKY_PATH}/llama2_7b-training.sbatch . + sbatch llama2_7b-training.sbatch ``` +--- -Watch the output logs from login pod: +Watch the output logs from the login pod: ``` +export JOB_ID=$(squeue -h -u root -o "%i" | head -1) -tail -f logs/llama2_7b-FSDP_$(squeue -h -u $USER -o "%i" | head -1).out - +tail -f logs/llama2_7b-FSDP_${JOB_ID}.out ``` +--- Watch the error logs from `slurm-compute-hp-node-0`: @@ -735,11 +741,13 @@ Watch the error logs from `slurm-compute-hp-node-0`: kubectl -n slurm exec -it pod/slurm-compute-hp-node-0 -- bash --login cd /fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm +export JOB_ID=$(squeue -h -u root -o "%i" | head -1) -watch "grep 'Batch.*Loss' logs/llama2_7b-FSDP_65.err" +watch "grep 'Batch.*Loss' logs/llama2_7b-FSDP_${JOB_ID}.err" -tail -f logs/llama2_7b-FSDP_$(squeue -h -u $USER -o "%i" | head -1).err | grep --line-buffered 'Batch.*Loss' +# or +tail -f logs/llama2_7b-FSDP_${JOB_ID}.err | grep --line-buffered 'Batch.*Loss' ``` Watch squeue from `slurm-compute-hp-node-1`: diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile index 8a3b97fb7..5d2015356 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile @@ -10,17 +10,20 @@ ARG PYTHON_SHORT_VERSION=3.12 RUN mkdir -p /var/spool/slurmd # Environment variables from DLC -ENV CUDA_HOME="/usr/local/cuda" -ENV EFA_PATH="/opt/amazon/efa" -ENV LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu" -ENV PATH="${EFA_PATH}/bin:${CUDA_HOME}/bin:${PATH}" -ENV NCCL_DEBUG=INFO -ENV NCCL_SOCKET_IFNAME=^docker0 -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PYTHONIOENCODING=UTF-8 -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 +ENV CUDA_HOME="/usr/local/cuda" \ + EFA_PATH="/opt/amazon/efa" \ + OPEN_MPI_PATH="/opt/amazon/openmpi" + +ENV LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${OPEN_MPI_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu" \ + PATH="${EFA_PATH}/bin:${OPEN_MPI_PATH}/bin:${CUDA_HOME}/bin:${PATH}" \ + NCCL_DEBUG=INFO \ + NCCL_SOCKET_IFNAME=^docker0 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + NVTE_FRAMEWORK=pytorch # Install critical system dependencies missing in base Slurm image ENV DEBIAN_FRONTEND=noninteractive @@ -43,14 +46,28 @@ RUN apt-get update && \ libsm6 \ libxext6 \ libxrender-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -# Copy CUDA/NCCL/EFA stack from DLC +# Copy CUDA stack from DLC COPY --from=dlc /usr/local/cuda /usr/local/cuda + +# Copy EFA stack from DLC COPY --from=dlc /opt/amazon/efa /opt/amazon/efa +COPY --from=dlc /opt/amazon/openmpi /opt/amazon/openmpi + +# Copy NCCL configuration COPY --from=dlc /usr/local/lib/libnccl* /usr/local/lib/ COPY --from=dlc /etc/nccl.conf /etc/nccl.conf +# Configure OpenMPI +RUN mv ${OPEN_MPI_PATH}/bin/mpirun ${OPEN_MPI_PATH}/bin/mpirun.real \ + && echo '#!/bin/bash' > ${OPEN_MPI_PATH}/bin/mpirun \ + && echo "${OPEN_MPI_PATH}/bin/mpirun.real --allow-run-as-root \"\$@\"" >> ${OPEN_MPI_PATH}/bin/mpirun \ + && chmod a+x ${OPEN_MPI_PATH}/bin/mpirun \ + && echo "hwloc_base_binding_policy = none" >> ${OPEN_MPI_PATH}/etc/openmpi-mca-params.conf \ + && echo "rmaps_base_mapping_policy = slot" >> ${OPEN_MPI_PATH}/etc/openmpi-mca-params.conf + # Copy Python installation COPY --from=dlc /usr/local/bin/python${PYTHON_SHORT_VERSION}* /usr/local/bin/ COPY --from=dlc /usr/local/lib/python${PYTHON_SHORT_VERSION} /usr/local/lib/python${PYTHON_SHORT_VERSION} @@ -64,28 +81,31 @@ RUN rm -f /usr/local/bin/python3 && \ ln -s /usr/local/bin/python${PYTHON_SHORT_VERSION} /usr/local/bin/python # Additional requirements -RUN /usr/local/bin/python3 -m pip install --no-cache-dir --no-deps \ +RUN /usr/local/bin/python3 -m pip install --no-cache-dir \ transformers==4.37.2 \ datasets==2.17.1 +# Remove problematic typing.py to avoid conflicts +RUN rm -f /usr/local/lib/python${PYTHON_SHORT_VERSION}/site-packages/typing.py + # Install OpenSSH, allow OpenSSH to talk to containers without asking for confirmation RUN apt-get update \ - && apt-get install -y --no-install-recommends openssh-client openssh-server \ - && mkdir -p /var/run/sshd \ - && cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ - && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ - && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config \ - && rm -rf /var/lib/apt/lists/* \ - && apt-get clean + && apt-get install -y --no-install-recommends openssh-client openssh-server \ + && mkdir -p /var/run/sshd \ + && cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new \ + && echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new \ + && mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean # Configure OpenSSH so that nodes can communicate with each other RUN mkdir -p /var/run/sshd \ - && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + && sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd RUN rm -rf /root/.ssh/ \ - && mkdir -p /root/.ssh/ \ - && ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \ - && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ - && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config + && mkdir -p /root/.ssh/ \ + && ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa \ + && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys \ + && printf "Host *\n StrictHostKeyChecking no\n" >> /root/.ssh/config WORKDIR /home diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch new file mode 100644 index 000000000..88616d62a --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch @@ -0,0 +1,122 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +#SBATCH --nodes=4 # number of nodes to use +#SBATCH --job-name=llama2_7b-FSDP # name of your job +#SBATCH --output=logs/%x_%j.out # logfile for stdout +#SBATCH --error=logs/%x_%j.err # logfile for stderr, remove it to merge both outputs +#SBATCH --exclusive # job has exclusive use of the resource, no sharing +#SBATCH --ntasks-per-node=1 # one task per node +#SBATCH --cpus-per-task=32 # match the number of CPUs per node +set -ex; + +GPUS_PER_NODE=1 # 4 for G5.12x, 8 for P4/P5 + +# Set environment variables +export CUDA_VISIBLE_DEVICES=0 +export NVIDIA_VISIBLE_DEVICES=all +# Set LD_LIBRARY_PATH to prioritize pip-installed CUDA 12.4 libraries + +# Check if NCCL exists before setting LD_PRELOAD +if [ -f "/usr/local/lib/libnccl.so.2" ]; then + export LD_PRELOAD="/usr/local/lib/libnccl.so.2" +fi + +# Basic NCCL settings +export NCCL_DEBUG=INFO +export NCCL_SOCKET_IFNAME=^docker,lo,veth,eth + +# Performance settings +export NCCL_IB_DISABLE=0 # Enable InfiniBand +export NCCL_IB_GID_INDEX=3 # Specify GID index for IB +export NCCL_IB_TC=106 # Traffic class for IB +export NCCL_IB_SL=3 # Service level for IB +export NCCL_NET_GDR_LEVEL=2 # GPU Direct RDMA level + +# Timeout settings +export NCCL_TIMEOUT=1800 # 30 minutes timeout (in seconds) +export NCCL_SOCKET_TIMEOUT=300 # wait 5 minutes for TCP connections between nodes +export NCCL_ASYNC_ERROR_HANDLING=1 # Enable async error handling + +# Buffer settings +export NCCL_BUFFSIZE=2097152 # 2MB buffer size +export NCCL_IB_CUDA_SUPPORT=1 # Enable CUDA support for IB + +# Remove this if not debugging specific issues +# export NCCL_DEBUG_SUBSYS=ALL # Very verbose debugging + +## Set HuggingFace metadata timeout (in seconds) for large clusters +export HF_HUB_ETAG_TIMEOUT=60 + +# TCP connection settings +export TORCH_DISTRIBUTED_DETAILED_LOGGING=1 +export GLOO_SOCKET_IFNAME=eth0 +export TP_SOCKET_IFNAME=eth0 +export NCCL_SOCKET_IFNAME=eth0 + +# TCP Store timeout settings +export TORCHELASTIC_MAX_CALLTIME=3600 +export PYTORCH_TIMEOUT=3600 +export TORCH_DISTRIBUTED_TIMEOUT=3600 + +# PyTorch specific settings +export TORCH_DISTRIBUTED_DEBUG=DETAIL # Enable distributed debug info +export TORCH_CPP_LOG_LEVEL=INFO # C++ front-end logging +export CUDA_LAUNCH_BLOCKING=0 # Async CUDA operation + +########################### +####### Torch Dist ####### +########################### + +# Debug Slurm environment +echo "=== Slurm Environment ===" +echo "SLURM_JOB_ID: $SLURM_JOB_ID" +echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" +echo "SLURM_NODELIST: $SLURM_NODELIST" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" + +declare -a TORCHRUN_ARGS=( + --nproc_per_node=$GPUS_PER_NODE + --nnodes=$SLURM_JOB_NUM_NODES + --rdzv_id=$SLURM_JOB_ID + --rdzv_backend=c10d + --rdzv_endpoint=$(hostname) +) + +export PATH="/usr/local/bin:$PATH" +export PYTHONPATH="/usr/local/lib/python3.12/site-packages:$PYTHONPATH" +export TORCHRUN="/usr/local/bin/python3 -m torch.distributed.run" +export TRAIN_SCRIPT="/fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/src/train.py" + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +############################ +# llama2_7b Training Params ## +############################ +declare -a TRAINING_ARGS=( + --max_context_width=512 + --num_key_value_heads=8 + --intermediate_size=2048 + --hidden_width=1024 + --num_layers=8 + --num_heads=16 + --model_type=llama_v2 + --tokenizer="hf-internal-testing/llama-tokenizer" + --checkpoint_freq=100 + --validation_freq=100 + --max_steps=1000 + --checkpoint_dir=./checkpoints + --dataset='c4' \ + --dataset_path='/fsx/datasets/c4/allenai___c4' # Point to downloaded dataset + --dataset_config_name='en' + --resume_from_checkpoint=./checkpoints + --train_batch_size=1 + --val_batch_size=1 + --gradient_checkpointing=True + --mixed_precision=bf16 + --sharding_strategy="full" # https://pytorch.org/docs/stable/fsdp.html + --offload_activations=1 +) + +srun --export=ALL -l ${TORCHRUN} "${TORCHRUN_ARGS[@]}" $TRAIN_SCRIPT "${TRAINING_ARGS[@]}" \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky-slurm-hp-eks.png b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky-slurm-hp-eks.png new file mode 100644 index 0000000000000000000000000000000000000000..d63ff8f46e3fbd6562c009df586de8082ef0552c GIT binary patch literal 172916 zcmeEP2V4_L7e_?Jj=d|2V4;{wvC^fZAOfO5AORvJ7(x?8WABOzDyZ17A|mzzVnsmo ztcd+ouz-5@uHT#NPS^w#xWj@U_wzKF%+Actyz+l#W@Znw8``u{$3_YY3Qa97OdS*y zlu{KG6mym9L&>AoQE%a&Iw1~2jTKVPc7CK#XIPrZ%taI!;lmH$DX23|h)?Panx7y< zq|P)|XE4kHc-|tRAP7D}Imee5ME+na7|#y~;HWdq**Y}%skbf1n;#?+g!rj5jp4gx zkcbxm|3WeRZSM$wxxoK)ng^Zjq1OjK8U+Uj@SJ&G)_f6ag{h~*)P>^S))pgdZPXbi z@Hv3*&x3!4^1S^8=n<}v6RyJ-Ky`+$4nqeG$;3}62!u-JbS6uOrlV_Mpsz#M(}!P5 zhi=Yf>(C{&9XUQ6A)nG6Y9fp$4B^8oN?sT@jD84)_YEP7i3tet<^=GF5;{@hB@l9X zLZXO;ikN2VG$VKz{b8DL`5a#%ClI~_abTE|E`~v~+T`PWE>Y&8>m)D|nwYYDhTG_l zHyUa0s248Z8g4T%VyKW4>}M+!3WF0h}G7q5w z$1*dDN}c&!5fBohKz%kjJXCJM#G7cmar-@jR0K03lWVS?O#DrOfnSniGWDYL7FwFn7@PAItLLa?DL4 zxY~3kv7}*~fKYPIiGmPOBqm^?AT)@}LrVmNY8>vz7x5f}Io_yxIM5fA`H2Dp;2Rx2 zf`|w(5d;Vjr3VRupptP2$OV5ME>veQX>5+ZKC}nykH{&QQ~HABI|HYQcqF+>ioQSb zrPDu@2TU#!MnbdX<1Afr(nw-wU}6X_B^_NRS;vn;HZ0UC8Xt!wS6^w{sI)Rm3Ux?| z`2P5jM3p0)@>h#a&JSgEcwFEOaED=jsaUBA*gHUa@Go@CX=@P#B7y9h@*?;m7pRGrAANR3pLN*eH&Zmx_$N~|%lOMMo{$f#A(48LvLIL))DR*R zplM+wr6u&Wv?<9q+}1bABt#G@l+Yp6D0jkOq0m~1I6_|@DGkc4N2il(jb;-D2#@_+ zvJ>(GI3j)+wUytAgvc48@VmKV1Wb_+a4dsn6AH`Y~-7z}F^LaB&47u{H;I;xI zh9O-5t&zA*k8VJ`JjoJl15!m-47{kL{AT-L6u{J?+5K-FFv0#bCuX8DZ!M_~h8f0^ z-+zXzEy0Er)vO$ZEQ|LY!CbQC_=7NJ&>TgfTx2#^sQ*pCriA)tT)s#G90-G;76C{w zi$vLzxCJYA0v5>XaH6bKhf5I8uR{Pdx{OBAfz{?Xd;m4|bT{)XnuN<2ctM=bp-2+aO$WV$sL6IjFiV=&*V4iT802vY{ z2#FyfRBJ_GOGw@h_&<^%fqX6(`Jc)d64D>A7zyC{AcK_Zg8GSOgj9&Y5acZgA>o4j zDH5eJUalGYbzxAjvUT^KA^Xj0H=q#5}WZ3=|J zEB1dwo$7U#l5|8-O(6lH!az_q2=!CQtE^vv>Z77a78wVK#1S4M>j=k)r;1Jys?!`? zF${e@wpqEhrxb|EJI-_v)gm7gP^*lSNr%Lfa-}dgP#NZd>X0HGRI+6T2H$8{Px7>m zfuyNYua=|{NvD)D1`r+eS6Wx%u9A1(_=kG&ghAji4$9Bff<*C&;u5oI4>+q$ssb zWilruhNyfRMkQigdK?a&u1DIv$dHx9;r~lu?VEVK+Ja@UDrjMTqFq}VG*pU-wh0XzW0m+&73@I&K+ zx_YuNQ{C}FCbDnQwEtg)2XVw%Ub}mXAV*U&9kqUwTEEGEv;_@tb_@2KAQuy2wHgP8 z*=50S87`4`)yetXe)voPohB^{!iGH(HFP6!U+dr7jLQHhY}9SV8=VLS(32$A}ZO67f( zl2kKaNUaTqAAPZxQYH~i@~a2Wo8iF^3K4OFym=mIL`3IhgY2gIy1I~#Et{A}P6YNV z8Qp zm;7D>H9Zbh)G~c7)7LUR_;bLK0Q-|GX8M{8@3FK0drVJwdnAs^$~a3HH1PJI^A9+L zZwY(^FqxIx8}*F|$JzGSiGtma6e+9ABn=g+6NSklV;R4X6NO2l!tYHuFNg~vB+2;< zWgUXDyV|Q1(+uLmu4Qy!0dkR*^?sGRLmU&xe-AHG77K>}scVJHD!q^VKSd!Kx+J-f zB*g|djy~N$SC2u53@sM&JtN0FrkiSvNmj1Ie;tGT<@@Z)W>gYWNr|8RAo~7)r)XJE zGaUEx!)TZ^c^SI85JzV~QVWB|q|q?bkQA;*(_zu|X-tNmzOKGL8--J-ELC$UoP3dA zA;Iy1BsxrR##7Q#bZC%bUe(Sj}%Q%Wi3z@3!;h|;}25=2=3@H zXe>6J4w!~+fUQtcF8@3ef&#zF8xLm98p|oz6s*+*V1oWS8~OV%gbC?SEJ!}3>oN_P zEKm`$@V@%>LIoYxTxzfN2r(F#<$qUduK;vcKeboTVa=s>(j!|mlLX{th@s_At8Kk zFu^Ga{M8Y!lj;1YLQwfleHP^`UOw zVq(=vHDlDTY%(EI5@T84ohn}-fI0uN`^lMpb0{9*D%kF@+TVN}0B0hc8~ zd!z)B@3F>eW*lq&p2(!-478C(I#zu}DTK@5%h%MJ+&86Tdk34x;=UPyuIB zVyIC41TAmx%J*1fMZ3Oa3ysV{#RRYZu(bM8d$-nO^G%AD2fsg_qUC$6xfD$*pK9{p z)Pj}LB$}VIg-U8`l5j;{)>c0+$oKf?02Y1H1gT{P>QSC@pXD}({$~3LWgYGEV!Ha7 z;m?ISWL&M1yih$s_j1sVG|ukwbA#^K)&Gatw(|4}4rCb@d-ZdOG#X1PDES_1EOas` z&(A|#9|vb427v)qY8B{?3l*Sy!r;6|bT_>b{3!JG>Wz-8gpc^2K2QU`qJt&1eK>*q z0GKjV)q)og#zWT~Q+_87heW^98Np!bNgpITMZW|Ign=B`X)gUOj3eZ8pm8|Iks}I) zyNQYRzgf>4&ZhiEm2mQ|dejdK-ayhx09@qG6Ka!3L6&_|APn~71d&am&;>j!;zfwG z$qRH)W8`s?_!oW<7cxDGrfB#@CC`Y2&=g#v4EH({FB^nzY=h@OB?-c1I|&yExw6BO zwxu1;^YZ75v?Xm4J_c=4si9`2^(04tO4@*Khz`Tyeg~pUFOIiA%8dx(YRfG#0|l|r zB2zz*3oYrA%kvfp;m&qJkhaK=@9iJN3n9nO4}x2*@%qWvr7ZJz>rsXyQ;*9J2@Zh! zb0x3B4?-7r!yvo!1A_%ZI5-dyD}46}5OB!JC(l5D8xl!chEI~q*ey-XDAy(uq)+@( z#x_PT0S%XTnkpq%i+HRuhg?+V@BulzLOMlsajH9(O89#Pc!ouK0I>7q*JqKSTdhEd zQ!h`#%faP`;YGql-;_)e`XUHgj=1I5+Yw!31|8Z6eECSqSqMTzxOq4vUX}m|dFUmS z9|u&F86A3qQgO-neFz0M{}S{29YVoaftqjh`w1q{Lt-_MQBvd^m)m(p+ z!ZP9pN(9#ko)A8Of-SxDS5|oazCsB*qDi5ohoucgdzE|2C9d}ysjsrU zF|&4rwc!LJkQL+=LSQXIU{REZ3BYBc09KJUEP#Vh3{j&34HH7WWV75u@~Ow(OD>>6 z1+s-O_A+HCCr+*qAwaoDp@b-vo^nTAw=7?X8j~-R^dMV^zR8wI`m2FgIGdU3p+ndI z^EL3&o0{bhub2LSFYUw^B~uV+MMwLWy|UGh6AWg7>=B({UkO5m-aO(CQ8?K@guG2u z<{(d=KfwJmw>L|zAS8o+EEJ}L1t|k@BnBz!mm*DS%I(c$?3@B6kpst36e+o20Zbb| zfv*5$zFBFhvFzTE(mFN*0f-#QPLHu**hZ343vr8`34 zCsQP`5qYC$fyAC9$))7-tIm;TxPdS?N`_GU^;+gQ!*@tH8G-PF#{|CLHq#A?9 zhijL=Nqf@E#3=@iiIXWbTm~c9nV8t%;tGJlZ{VDe?!zbPg`o#CB`1*Kus5QNYII^c z`+eY@6opdO$QB$?a5yQKgI9zC=7vR%5D2~f@GrmrTG272CfACoPlli<%JY}4m4xSn z0_1uw!*TeCN_0o9t$@puavmyXWBCwX(p={3Nq)xJV38vyqr z-arMUhK2$<$sN9-d@gcxqalzLMFK&9h>slGltS{bT$wTsI{j6)nvVc3qN6?+46jwD zz|or{fg7cLqH0*-La9FXHllJ1(vvP3jr78yK#oK^LuK+!;0h>kYZHA*iptI-p3QHp zIPo*CR7S><{Zy=z%8}3H)KY=!e`$^rIbyi2K|~1Mnkx#lFbWQHv~$%a`=AICm>gJm z3f7ZFaurB&>h;v5bDMDgQ!o>a&C%C~6$f89>Wcb)csM*EI4db6@U3N$pZ&5YTC=eD z#$q1@hETT@QKE}4wV{fAHg*U661iXQV!^QG*5)VTaEz~AVOdD@jG(s^JpU*h{!JpG z!sv3Rs~i|zZasXD*-wPgh;GegQ~;Hdu{41?q(J#T{g)#<%6-wY*$3v15x@!%?}NW> zLV;6}r>7#{kfR?*$m7Bju-lav5*Z@mLFA%*|4^idHc<};;2|^$o3w}wash2<`5&$eR#uduH7||Q- zd35_D)m88_Jxf5a9>6ai0R*>{buvi@_)WouUa*?98xun^tljiswGrb`wA*jzWhz%T zk=ynuxZ`K)Hgw%7WxF%^x$?@5pnVKRk@u$zH&l#zsQ#XE=LoJ6l`QU0R8-$rJXEE{ zH2I#GMD^ltxv1%>j?vZc5y6+wb8wB_gu_pNf|FQ##4Ow}2(s$lh zA6-jK8IW{FG2s2zX=~UD<_Q9MA|bpl#ndftF{&Hqkj?^%QR4_Ep)JQB6i3DI#v(G9 zCmbd~0LKJLdC5ZoMDEUa^DAl*%I}F#XmlfeeFSsL$mFUa6lsu49-&Boz|k)<&_s!5 z{7i@v0>mCDA@;K91jpqF&j0nMcg+ANzcc_w)1lF1(1{Kl&qRQg$^#T>6pRx1as|{a z4T_aEB@+evCY@AheUpj26@nTTDXsq>L5(g>aw9PTlSMkfstMG{w-2YQ5er(i_N$0F zezp29n?fFfvMWKMBeX-YFv?~eQ1e$P(p$!Wm3TlQr^y@nsL5<)tP$jmS`i9%m$7K2 zrY_Z63Z^teUyp58uJIsMRPsqR64`)Ee{^IU#UDV^Aw2?erI2@t`~-5Pgo{A-c>@$( zF0Wy9HXoFu6j_-LQFTeDQl|uT1ZyEvSK@8}ue(fQhkvLSPZ-1#@j`S^3=7Rbu#ima z2nHZlR*gxGY^-sm1>Y1{)o69Iv6YS4mQJ!Q#p=ev2&zjNUju^y_D{`oF4v+go90~0 zDnPNw%cfJyDgft+_}=#oxKbOlZ1_b_F8l&F5E2|(<5&gzA1GGkFXI)d0>OGm)>m4q zR#4nqry{i?fdfC*1d`VjaGzjJC4X|@wd7w8eyWkG;wO;5y!5ZZ@rg== z74$e9Ivrn_=^}~ELlp48#4^erqZR^2yaslm%yyhSfEG`Q`n0?bk483yw+3;{tCHB=yOQ%vy&FXFPu|JN2@hfE5LbOLjeW6uQ)bti zM08#@dTFY!s|(^+HY1Cii1Hl+*jq}*Cy^7IALMJp^MUBLDX1@KN;c_B+J%eFOHL!KcJzF!4<#blGg|TgJnw zxqMajTn~amDO+7?krBjaf#G3d$XqqN?9}jJW%As=nZG2DjC2ijSPWeT!+=eb03>pi zaKF_^{Um)?&|%G`?@HuMRDr(n!J;VX;J@TaR>gv-qQ>~c6aj)edT^r&n-15S0WV-H zl%!ksPctDX@TVAia;oKlHIgDP-JRVW;q{ceWv|C2@WeHg-o^d}aa{z2De8ZcR) zA_(DtiLd(gLIoYxTxzdGguRy9e>m>>KDAc>x~resE9kK1QhOyr%T=KEUx9)shfpw? z67Yr2nsAla;RbM}7of4K9>`urU`4DxVF#3}kx1v}9|pjHCs=fr9#dDJ1&8>u;kZ7D zBZkCZ73ha*WKpS^F4Mo9yRGJOXeHvnzd46ik)8e&HX#|I0@YECj8HkwtFfGblRGG} z3_Bb0xMzw}O6I^cL8u7bmuMmg3gUUAeDreC(WPgQek1)~ZsWVY3Cr9Zj@gwu@l-Ud=zuN#Q#Bz5Lz)>l1_olxG+#|{L@Z!J) zP2Q3tQLD(k)p;Tci?lp3VdHIKl-Om>IO+?+13xoJ0hh-5iL|*Kp?_~U07&1P1tFBy^#6i@wA39Ex$6|^a$U;G? zUvR%bB~rA0c}|?Jy!DGt&O?(+nQ$haTbT(*wNS<1Xu?Ulpx|s`h!jy<1&gy1{(gpm z9-D`Fw-k{eoKEywVFnT2-u0&-2$E;@Knj`0h&hA|WA*_+EM{ZbjGIRFMM+68XP#ME-C_Av+)$6YEVZv1=}yRBwt;sqTUbNkxP-`9_a5mx8M|iKkT7W6|zPI@i%Jl?OOg;e;xR`9H-RAi=sU29byA z)z1mC{XvQyG5s}Pl>hYno?7b`oxe6rzz-6ED97ZC(FiGMaaup7uj(ggdC@N4W6c%q zKR399u9T?t*Hlc=^5FN!Q?z`KHJ75x#(N0u^Y1*6wP1z8___OeSqS|9_87u#wrGU_;n>6`=wbdlnkmWpV&|i+L2}z zyMWaz{UR}eiTz(cDC77~3w3=X5C#5YnV4wxbBHt+OPU|@J=Rk|LoX0O}8e96%mCQF~1-+}tHOP+D>bHBIhRY4EI3j>0A?N{+KGtq9H( zbis>kO`@>8veKu&eQd4Vnaa{6R%((9m}+LGa0fLCrW2kve6ltCBE9`jCTHbaDXY@k zXmJI6Ih6DVoVAKCgQbYvDkrU$o1iM>5psaI!w(L~4k2&zlR561=MQIU8wYT_cmcx% zA$$=ZNS|p+o=siWg-E)&sxkM5PLn$yiGeQ_r{sdvjI>ppe@ivHxV}6`QKaOUa)D6f zC-4;naRSUrOO0jqYiS)D0lKzQqAkbrM50JX{&=)^o)an(z=BbzMY^G@3-xAj^VSmP>aad{gR!6y}fTlwMmPn}dsPS|Iu?BVZXc zIFKA|${-(xY9w`3nD9l2rOF(uQU1jr+B6-ezP?<3HZjZ}!d0ctZ%kPNAD<8&c*9ZR zUuB8^ClXb;G7ev!^d8tW=&osy8pzL=r1*rv;E01cg zOsc&yNdh2#uztZlSCZ#-F>0prJVwq`1@jwTnZzm^R|Kiem%NPq;oISZM@;c(G3Kq-v5tGIhu&H=>G*JGQNyDv)W0Fsw; zIy9O@lq(x5<4enFMAE5L3PMM) ziqJ-BU5S%S-epwfd%DX9y;x1~gd!eJT|tWC-&-GYcnzP_L}4e)VN`Cyr2G45Q0()d zz_l_OREG8?WJbC74!L(=)wq%k$H+w~!x0otk@ew_JNM+soCSwoNz%;A@SnUZu#8Px zMgA&d8<%^=fIK;dg>iM@hCdPVfqmQ2RK`mF_hd$A$PpS#A9xQ5D+;CIi{q)3#@5%t z`)7V5Vk5^+L1)N?=2&Ark>A~-&(+uE(!9|Y{j&M7#2S{P^DFJ7{6FK*fBp=e>{{EJ{8akn)g zC|a&Rv@GnCxOmWn%L$Yk=L=;^mhIoG3Gjuo4CHcg0rjD&`N6(Wk`cA#X!|#Jjh0Ja zI+p)4kDrpq!IFzAD;LT435oCk4)PEH z|BUxRQGG%#l>x~NS{XR>?*$?g`!Y!R1m{Z=-BMulZx_8US1OTWTu#INV8~n+u*zC* zka(iU;n3;mz6@es<=|-bEanf^U8E;~El(>r(1HJkEIeGyU3kW-2E*}KOCzh~ipaMwot|oH|C`fawe}^k+{&jg{KU|2xsrs8-qd_a z!k|$zbjp2KiM0P0>f_IqI0kYO2L}XdB@W65lb1Lw99aLOB?9RzpjbTi!6?;|gYt$6 zvj_5sxg7Y$3ku^41wnz3))zvkE{YtfR07411$^>BZ5E3JP87f=twVK`8vM$jcDb^P z+z3jU-=E_n6KH6O5iary&AgXi}z`srVtnnzajK!gE0C#y1uZ3l7^#e%; z7ULL+mDB6L#*)%1GgGHQS{NAd_z&(4!lr(<0xmB=rjF!SE}sLZI|q_K zkPkoz3Bs+Yxsx9o%~eoPRj@QQ8WG`MvZ~=#o5GLpp6xh&%S$^ZEva=ZSJTaMuW{U> zRorEZwEf>Kns}RMJ=`L0k%juGK0M2~H15K*FFQUy3cvOBuy@f!-=bB8D_5L5pwsS_ z!2{R9Z<)`|Oel(e`tnF|{tl0J&6Eul6jdfGD5)ycp?@!}Vjzet|I zr<>1tVYp5YsFUQ!bSCR`61%kQ?7YLBz3a%MjR{^pT~0Vm@YdLD)0*sgJ@icP7_qS_ z(ep4xyB3}82R@$`P>`ms<5#fDciE}=ovil--Rv27ZMXZ>1ENO`eOyx4+&{a@?Zc}_ zYBQU6i{oG1c+%#dRV{BXZ>ZdAIyrx;o{GxJlh2RqPE2*?d!>~<^hbEc_I0xAA6DWw zHtfl*bT_MII)U5~yL|Ev*Izemt?`9z&i%GHZELHT`#v|e1D;`(<7kMTCv0p+3^90d z=GM^%(nQa04bt&r`(I&w)p0+(zvTKp`%??M3BG*&{Pwo>%c8s_n-vVkgm=N`1GcYV z`sVm1PSm)*PoUSqt&?8R(qq#b=iWFFqV9ca-iwR&LoAE0Y|p#*{^f&Z!`GWzW-nGR z+_ocQaSJtb3$bE7?&C{a7OPvgy=cGF>tyGh5i47I49(;&!c=6sR6$X5-MdGf$q98u zGZ-&+%9s%O=t9z5KkaQ@283B$-s&`ebj}LqrV*Qm8J%4{Nb}LfP0PM~{+Juj_UFPx zI&5)J=U>_!6XWQewSc|w^~3WfT_f+ehL`tD(jak*2|@j^ z6gT`m`S(Zd8LM8mL9?ZzM;BITZH2||bbdpD11vQCSoXxmfA>n9&?7-lpexS*;M$;h z*QNEFcC-&~s0ZNuT#*-|EQpnOAb-05fIK6Z?I=2fw>B)!|6=M1%3KHb<7c zJU2wot?`~_%UU(tbRxf@RtGiA=P~NQ6T>{CXWq(#udle-?!7#{ak~wcOEfeb z8;jeVPqKP|U-E=LO!B;b3LBeC=ya#8>L{fvv6{|3O=dLl*>-zrAIA-5bK7d9j@taW zaG8hho|F+me4n^m2ZlAb_S7D=`*eWOr76$G4?VueKi_Iq|Iqs|yBn_#P}JO;sCK3+ zF>wlYqEtZGs5P6XNX}#n#q&KXF8sePEa8^}T_4drf0KM=4|grKAPj?5{M^bfm^AV=NTK?||2ml?mBl*rZkP zoXi*MrlXdAJ{_aR@IAXSVw1l{Y*52XPs9zoY_d;`Gnv`UGG+9Ep0@F<3=#ih$htlm zduQe;bSI^&avzuyi{}^gNg;4cX`NB<=7^5E0n4*@UzoRbXxGRSxBfb=f6#B1p#8Y> zZSZpGYD~^d8yDQZBT=Aw2|c%RP}E5C`5o1rwvCJ(+HE51(&^5Zqgf~Nhi7)s!Vh)_ z3E8N_sE#v;b-tRYFy;6cx>b(yY=OhWaoV_A7zoKUM^^jBWcz27R629&cP)B4OR!vg zu?wzNUTbgLk2w-OCdECyZ}yU2Z$Gb7PPCk$)>+{oNm+)felUYZCYB~7Sv+Wwy6aBF z-PygkxpCU#hVAk_r!&8kUfcBRp4WEs2OSH#d2o--O8xWqyhlAuw_c`Grvd-w!O#mE zEL#~w{NtG!IPIhHfGoWmS`V9J3h)JWV=y3k9!V05dMbvs1zrM5RR=-SIxqTKdepw9 zI%7s{v`x^+i)X*o7o@BG4Mcm!yYHc=86Sfl-9B>U{a5XMH+q14YP`IiqNb;4v@#)E zK~pWZQ4q&+r?JQ-4A!Z0sBv;mtfuwbr#J5@O|V!Tkaxso_^f z!LmH;%&@MihwuQGrSSt4Z>U!rX+wNu#31W-aLA zuxG4O2fEj^`)5`-_8qZV)%V=$*jJA(CTMK~9nwCV+a*P@`1Yi)Y)0|gnOzrm9W?&* z;%0XqKAJO-HN!^^uWCZ1I|Qj zb>5+4o1n{9EV;JFKaF#I#y?7$lif=?-&WXoTpbTpUKE#TNks8(5XHQ-B$1})sd*iE zAluD*x9L615NO1kkvylsT>6eoZr|(htIjtaHg^~?%FVjol^sXEjP1G1X9EAp zcIbG;Mp5ydXGgvU?N!^F*?-cP!&{E-8GE_!l2u79^HOH6y65lgGp{iwqo|f3NSHeF z8j*seEkQ;iw?y?MQSNFVH`ke8KHWVz7wKfR)y=vM)+$Kjz3iPjVg%p(^4V1b?sd(7 z6Kd`C_p#|_`}&URVU@J>4LhvlY6N{)Op=@O!mfkHMyC|G+Yf%fVaqvQ{_(TV_h>Rt z)g!5F@}P#QsU;oSnUe*|jq40(+^mhwQ+Dd;waFew+_ySy)0p_-RSeAbjnvq4p^ex2 zI67@{Ftc0LUtRB}Xi1FSC{1hTE>YCFZJ8II%sMz`9E@(`p}^~Vv)7u;oHsBey>WL% zO)cxCc2b}>x%=cyukIwM@LE23Nj)r_g->r7tYGlw?Djj~!@cL2dRGc_m*vGB|NHpN zgdQcMO2#m^+pQkh$8BHG{9dUUUq1NvaZXuT&|~w-TZxfH9r*bvs|v=>%AE6euf~p2 zA+QdFK#P<1q;3$OZW)}G=^%V^eV?A&XSX2=C82vx1qm>|!C1x6c4iQa(3yf0^$wX~mNn`)-CM*au#J-g8q{{sNm0t+2F< zfeD(^Bz!!H;I2-Y>{xH+oXtST*>{dkwM~~u>$U@*=qhhIl^@i(Lv!haz2LzKP1@HX zrKy+F%D(!q7KqoI&u3i!*mdHo^Ru4sipsK^zvJcs#$XJhkJ&4T+e8<=A|+31(>h8s z1C3iO7OUJ|o({K;A3$gM~y>wcHh)r z+0o;=m}`jn%ycC{j=SAG6-beAxm z_IQ<2__JHuAe@(MbKa42WxE@%Fb67F#VXaiGV$%rUh_M$7Z>Fv%x$z21Z4KR=l7Dl zPR?QYUrE*<0U|1WbLIta7}>@#o4VM|*>UkI!yV7oF>9DD_gk;Xv81+vt{2!3`8*IL0+xFmDz?Tqx@{EHeX^Fb2M@+6nfVSpJUcDRp$x$MOr zdx^GZhbY%8p2g?hIvj}r;zy-(Gh?9P!8S7;u}GN30TZQtv%#2c^(}+7M(_WtwcFR? z9Y+vq8+UX-)SG0mBh2D;1N$EN^kQ!7e(s}R=}eCw&g)`7==IWviJ&KzKbiTogG(>F zm9%iyrDCu=G@EwPU3wtoL41F=@ei*U=bgFGM7tMSsU`avBc&@l*#; z@BLbR8n0_BdX~Q<@zaOm9X<((Iotak`Q*@b(x>P8XAiy>fP&CCl(Qr=p2nH7Wp#(Jx6J6v~d%uYKfdbHSYTtuuR0MVt+?vW=0+5`Z;$$n)O(_=c{ z@m%iGK1-T(=|2K=UEy-4d%NHDudh{CYA}|7MB@y#n@Mu;M-^b$zZ9PWG+qP(Im_tM zlgY3qv#&GugF;r@BO0)#D+o*xZ?G0G0<(k&%?xequ)iVgbGnspKr7MG%(>kac9HZ@!TByDwkMpItVnH3lIkGr@3^yM8M9>StCeWuoL+Om!F)CSGt zZlyxVW6gu7=V@hEC&K~&uL@VK|hr>JQ2d_D^dXO73k<+$=lR-N_WlXC;`@$e(Qrxb4n5Q_1BrRK21MvQzu+nRW#E+%e9at?2*oueE8OM-}^ks>zN6(C0F` zew>fjma89L=XbS}7MRu>m(}2SB_8Q+bc1f~ovn%i)kjYB{WQ^x-=`a$!FpTT<)4M1qr|OvW$0Jlw zoDJ)tK4(@dQfi*J&dgRU24TIppT}UwoWL?_gnB;eU zU7CNs&K9?QDW72Ai@Tf5*zbruAC4PsVnFQY%xR-})wq3n?&=jrr1^QrP*GW{LH+sN z$+e$i27Os~csTM{oVJ_Q(JA!`;T>47cLp7LqU*S5;-$^Q$1Q*J^rrvb)?eOS7YIMR z;zz#wduAaxfR+Jv&sw$K)p?YH`@U{x6ZD#k+a#Nr55Nm-sH%+2w8<8&$dXMhgAZmN zjQ)J^!1_DLRRl&Dv)WyKYr* ze7`5~y6zV6Ni}ItGwd&V==s$6>fS)Ng2Ry^hyN-9PuYcyw#CbX2WV$3sFQ1mMX+*N zy{4f*LW|!be7l&}WyqIz>-*dYYM89)oMHj+gLgHRGw#9Cejc}Ote{1`DJk+Dy0B|( zTkTQHx6in8K4W532L^Z62D??)l(n{RD~tyBotYJIV4{KsqvfWP`E<_?y)j@Z-HQu%K0X^ zon`Khwf9vs-xsmkkuvJoDex*X@*oow=%U2%{K$I72YB*XqePm9oIr+~A%*J`Ie9 z{a2T2I@^JgPzUs^JpOR>hey34?}!_0;ASsQ1N$`M$&imFyVAzL9{V1IPEWzD!+NbF z`#EfwbzsTbFXv6Nz_mPt54PH0SsnbQhRqeMd3mQlR`jZ1b~G(DJ60)Wb({35``er0 zi37iWddUzy92WA{FnzA)wvpTKCkQfP(xcv5fg+vKar>AOF=rFCqjQ(_%WcqYf+~3LTEu#SFB8~Oa697y@b{Vp z^gi3(BfN5Jr z%^8UnHl*xT5HCX7xih^BIfo_InRbfXV1OHoF2i&P6y98Nw6_G|VEe`qxHfNZ`i!QX za#A_Rdm@KuKmo&b&)8VMPT=W{`S|RJ2Vg?4v&A}rAR`6WBOvvR=7<-yAf!TngU zPmq|MIlg%NJ{$APAf#+=J8c~i1Kb|(KZcolC1s4Jy4BJ}$o#mFWL=2dcr9%{ns^q3 zuVOOrg0BQ*SNN$tqb^CvnJ~GjjoRsGog4UaUbdfb$nLIApqUCEZA#QtwBEJZVWUkS zG$?Bk&q42QCUM>PFHu9N411=b(OBiAt)twmYjhhNscopj;Froap>%}z8V zM4$Mfx@Rx3)AmtpoLu29044Kq;%XWbPgxHHhTpfRDUwrmLiwtt=a%J<&##}^XU%H5_bGi1 zkV`SWR!0~Q{`|@jnYiGxKFM_lGITsR(Z2Cv;6M+g=z)857lQXR=d|X;(e>hj+8us- zD=NK_R;=UPw%RRY-N!#L*}kGsCv~*H=Ya-I#cg)-Y_hOYH~~B{vWe1)KIBv#Zjrhq zvw!4^4dBljwDHs>z&tjCl7V}NkG+!2kNyJ6X!)@x2dDpoOqw*Seb2UtE^MC!%fx)W z{S`fS*r-ISeRapCYkDZf1)1!AYP{LF9)l`3pqkfCXos_pBOGPETk7SWdh z;ZK)>6Di@?!z;sgxDNvFgw>5z4YcW@C0FT=?fen^_qN(sC*z-EjeUv}yB@qO03!;S z!V49x_jP~V)-Zj^-w^e%8Kaf=>cZWpDDW}Kt%D1QGWJ)Ex|%JwEgg*2(B#R=igTSv zZy%UMVXa(7_TJq#F6is76EUwt@-jOb2sdnrd<`qSLR&n(opJGgP?MeYgAKoYe(T{l zdCY@$xiiPy>y%69JjFAqWDe`uUJuJ((|Q0On^}8K1d<$nbjYP8!J{Uaz$sQ}5`wM;Fu!j!ZfNqTMX$$mchyC#>i0 zDej`OeSpfr@c66uI32i?S6$W4sAD%Z7pt(z3T8K4>yrH9qT$)@=(sPRBKZYrNoNv= z4AJR)Kv>8KZ+~e(-5GcF0o00$uV|;(Yk#=4o9`I;`E|Z+`0PQe!FCEsxz*-8M8B?f z2X}1Ty{v@^hq{@6NVT4zmM8uqos7YC7^y3}nfPEI`tr$LlxE&C_y@?Xb)G05To$dh zT!sN6Rs(Y=2Haio@y_6zXc)2Jmd=Rh8$QqLz%&Q(s16EY8N^LagD>rVfo~$)tMA$@ zC(qC)*DlY}3l}3w%3f&@@>f=gU~1T`9Y?&2CSZ0{y)#=uac*cGV%88%sB{J#v;^F# zx#0RM6gTKPYwis7y~6vayRRR*Fb-l^R^YNgu()U?ikd-`@5~V=Vd&Evu3_NlXw!GZ zB4jWk*XE1+*}`}KaCT{r1NVo<0IOXGN9geK;+;TyssZyZC=NL`t>KG*jyJcS-BP{f z#LnV2$L%}g(BEXkt023MEm+x%oUJ$AJsXJG5anCJ;x8__dFX9@8W-0$U>C&9GyqF4eD&XzaYb{JkO)OiN{kW9wFA-a<~HS>baRY z#yjr2A+$d1=cm1O($~+yfGry8;S*-hJT^a85Lc1n@Hds>AuD znT>XWSK`8_(Ytmh59s2)KiCN3adjQfZ3VB@IM>m)x;k@CwLR88`YtULlh!din4hrw zBJA~RUT1)EG606fqFTlq=Tc5jYCKAg{CR!iL-cX`m;8{Ul6 zp!C-x)#7)biVdRQ*gtN4ov8OOc<&+pm=|^&{DSdWn`{%>+Iu$bI`Ar&|7fpSe&AGy zQ0Nsn9<-&e%^LFQ9}}gxiE(r_EQeL=BRfp>v`q`r4x0}=7q?*hh%FB0b|^O2dem1i zbP6v!d$iI1CuNB9iX_EtQE#7a+;|xx$$4GsI*r(}=0eAjiP0bUqvt0-zn7Jexb4%6 zo~gSNfx>bj!W_7_!`LE#Ifpiky}b28D#v8kW!l=TJ$u`Ba2dEmDrOAd!8nd(#jhqy z_NsZd6DRFiFDeGxF}+>Z!}CMCvE4qiOswXv2gi3JKW`h*f;AYT8^E(z^eNvx%GP{A zXa1K}5U+O3f-t#oUUb4;z&%9G%VE5)7r4kK&5Cgry3W)&;Vg<er{J4^f?)Ab?| zG6|R)>JcNJ?A=h`EZEQI%Cv^9Y{n!5rY_id`1qjUO(*njj4q9%J!k=Ow0=JG+LN9e zuqzg&XD7VWzzz z0E#6U6FdHnhI0OLALWN%c-=FytScm=l9Rr;0)y&5v7wywb+qI~Ae5XnGrkVR%Xb z*S7#8X^w*Epj5pNRAPO!N{YP|I;>t?j#li*8!@zjpB~jCmI@T3(p(sZOHY0STynj| zu*jslIsLW7yw4tHaXH%5OrI!}p)+9cJe;8;R&5#F&c!Goej|Phhf()ewLI9 z(}5RD+B7$>OA@|<(gesNcoEzB0iLlt5%8K>GZ!_)#;J;2Hq9(EpiGx}{X_6_s&<4% z*L9zBrVbu>A{0zGcES?Rx?yG*tk&Mf4eLsc*AY;#wukj0X%*GosHD)k7?)gyRmn9! z>e(9?7-lwyp$+jI*B4X9SqP~M!#=eigbQ|}P{E;{zD2|{;^ip0=)fW-2e0C~X)woD zirFkYNfs!iGR}M@i3?!PN?^`-S(|AZ%7y^JdU~)JeT!Pmw(L3Abki#cEa@D17r((p zzlTOqPEW)b5k3BT|5wLMB@55L{JS-XP+tZUc@&nuQGK&(C`am!cygm;|3%Le3=NZ=C51 zFPu1hoFwS-6(;r5J>IJMnT_Uka()kJw6IgJvoN9Smcc8eUhR>pc<+>xin0c<4~>7J zEt->d2vNq>Rn(*fjV|m2fK;$jAEKqR+)=sLTuDnts>@;Kk8ZGZLzAZBV(DI2Ik~6W z2C))!=+YGim<*&=Z?B??i+J3!A*JMy%Vl5npZ7V>wfcfV_LfK((W(1;F^{Y^XlWcZMW$=Q@*~s+Pxs6 zWXY+8dz(qNE^Mw9a zeH@gvu1}jHh8|kr#fi7{JJAHsa<5t5vx<0hU!B8TTJdv`c!jI1C0?VN}=3BY+<3A|71#k?KY16 zn+>|K@7A5r#(y7Z$g>U9dRweUZ=f`CR@1%3AZqR>`d+t%ho; zL6%eCIi;d|rszG)=hMMAyFgVXKwP*wv~bjez6I-I2GfVPD2$IRock`VV>4$o{7u7z z4wDgyPPqqe5!JD1P9v}BJlx_jNhz+4f_;I@({Oqdvs92`YpfDf{#gj(Yl!s<*Av)xHt%29eOC(p1SWJskHVqjMacE9w1t-tTnU#0uIcZ zk(x@t(R=(uYn&zw+G*6w$&>meDx*mX-}UVwAPyRcx+@xHAPMHL>ebvS5ebu&7=B?DyiCH&qm2^*8(gY zhQIb~s3qv3xje+ovmnfpnc5zhY|dIwrW35iVWoA$J-4tfdf(eQRWEaQ+DR3czsLB# zD(VK*FzK*Arbd^Xn<* zrf4o62U%}zixE5vRs5pd-TBdyOHZbx+-RlwC5Y$an*VV5Ym3KY>fKb2+YxSkDWiqU z&M#*;U&25SrkS*EUsp+0B^Hr%>-bIU9{4jo8l3Ocl-|Qf-jU8M+Od51$*>>I+ zb5FBE;evR}?y2iEmfAq{QF)P}s$%XSlfIcLC&$>RUkP&9wej%$i!J@Wq=J6w7@9KG z@E=&ZDOro-kUl@5rqWDY*f6rUQrsMd^)T0o4kuRCdq&~$qdo5|+`UuD!E0TgozK;i z-W_t^cD+gHp|zp!m<{Z`K}5Etcb!Si<1wux_G{@fQh&q(*AC8|7IvA}E@@cXaYH5+ zB^P`ME!KXt-jhe0ZUMn0Ge`{pW>~B7PgY`9PwyGm3yTsQBc9}Rd*7-FL}|3PdfswU z5x039Yw=M<8L~n~`t83UfiaeIGB3>AG`QZJtm|9zpJi>I?GT!`RCsPtnohI(I@4x{ zY4wQ9bU3=_`8%WMyu+)nU5?z=`=WR5vg^FZXWpm1pSvUPYvUoG7VQogWuFt<_wU?A zpZfbg`%>3o?Rb-mS?gTy-5qm%LukAAB`PjwcyxpE;w`$J@G3dfn#8 zhobxcfRvcNkUhvqwWU)1g)3DrKG0g6^l}qq=H6&GyZgJF4<2{~Et{b|AoB3jmwTRN zKk#_E=(*#A)7P38om|D7@QS-)U`WQ9_;+ywhaaAG#%=eABJEKcQExu`oQ*tqrqzWc z6YK1W&u;4N?xZ7*U7EaYnreNMSs<<5H;z`QpcgA zN26AL2gQa;%E-gtXj#hG@hQ$PBeN!~SmSYpHMYP~H)V80-8}=Io$1>xWwfTZ#<|5! zHQO%F(vD<@=k)Fafvgv=^@g0DYkO_Zm#@!{P3OMd@sWG!-J=)z*^V#13|OMpBXvsm z6)oM2Eg#ht$F?6MPgiv&)r&i%aKT0+)^HSUOwxkT{d2p&{V-zu7*L|=Eng+Ac3Sgp z&yy3U5?rpwW%dvLCv~9Hp4Dd(KJ0C+`C`_AlEg`0W3!pJVh*wsmuRhEW*?lNkf-`8e7rvXlgyDI${reZi zg2FEo4kY~Ju8oS)+JVZyWHPHCRmGhaM;d+4rG+`ec? zBs+JcB-UM!q29%f&Gx<+=5sd6`AbId@U2cd{Q^CLHtajGWK`2pmg(0nbFY<5XFPf$ z7M{ELwaw`VrDR`Xc;{lChgJTqsfBy?=J8JK=dUZUMj3FQK0Z_5a((xzfzDeFYrVgt zW|^Qdf46^uMcjl`tG&^Cj$5qh{@H!mh(3-vXA`4iW?y`ey|>G4>!XVA3kx+~Z#cca zUg6YOf9{)4Z|~afie5GL#j%h!g-hnUao;WP^CIiKx5MGn>o4q!8hSv@-{CZDyx?$X!oYWigF6gfGn2dj^WKOjE{gu=_9c!#v0wE5vA&osy3}*Uk^6O57;yFv z9QE+%{`@WZQO#$+Ej)Yc^O4-4{<_N}4|ZF7HdB~>+>`cf^uvY=Bq$@HqwAG%X&1k? z9J4?<*|fztMWp~sFf4LYX}v*XbyR-WH}&vr&9@yLS2I@TaKG5Mcbz#TT%pOxQ%g6w z9N4%0!MpevUc+RRN_Fwvk)FNmSH;Djb7pm;%^5hVx6aGQ;I@2WI(<}8UL<68E^6Vu z(c;#Pw^rwL0(qY|CFX7$RR8p%j2&}>nofv1)+93J?nQ>zuzrz$Cpq7bd-Z68*^=>T zuXCR;_@@S>-P;u_EP2;ZEtM5zRqyhY4&Hf}M(r;4QHl?;KHo&^=7PVsdsus3-ZI>5 zYrde5cE@x-)7a?EW3TMp_9{Ak>ix5^j_%I}xjeqKx$pLbBR$`?(dh+A*fB*-J(iw& z?Qiu##9z;Oe?6=j0M;2Z#73&wO8rB-se(IY>Bv0y#aUL)><*S6u5oy;in2o&iu^90 z;J$pkto?^W2Lra;ePNZgnlaw5+m_L5S1z6OMZM35tH&K4Oy8Pz_n!TkXV09T<+c3m zP?(?oF|YlQj}sEkOnO^)*F<}>*o-jN77y#K1KsRS*{)FYJ-2GvbY@o0a!V$!Jhl<*EV$=$36kGb`73)pY@-k-KBKK-yJ zCl#!S@dMAXJ8ABW-h(Wj#2}V}coqmdGo=A2F@2|DW;=1=4$~JsXDsD;xjNs9h{|1l z;Z1VK(Ux2Ig_dX4mv-5BPqbYedUuCon&@Pf#u6)L{42i~4uOwuZHaw9c}+LNz5|_d zEOb6EN-ciiVf%UOt-3?wZ8mOfIyTF)cN>!rU&p?7kDK33dvQs@(?#I0vEI39tM!qt zJFHfZHprW~h@D%M!@S&ax!@@zF7%mh?GzK*EyH_oXGp?pu6gT&llXOU#_juux*k31w7Yt?aiYB~&_qo8_v+0xDc@yTIYcU?IyT6zl8z`Gj znVg|9@B_DS|25|iNk_Z5GIBF-jk&9IWo5QnlIi(lX#poYKDyz*X2L#R?0ZF{NkiL> z|0kM%aoyUOk-XJf8ef-uUD1!}UTiQdEc(DmhmD5|Y>LmV&q^|$Kf~N5G~nilTO9{D zWnA5B5_e{WU)TIIn;Kp2bb8~$cDaYFopah<&zSwu*!0{W;eGoK2Jhxa%+eA&-T5-% zdGwd7YR$7djk(@Qe7SJGkxw!t2(dgwhX?ph$=2D`EY|^=#z_vz;YTF+Ri($`k8 ztM7VMa^uwQB)6NdUwajN>hmIdliU5%-LlRsYdzktY1XsotLm-HGdeVn>F1{GYR9GO&Ly>S)9s^fRMEVkvvMQ>Tpuzs+^W6l1qb}BAkax_;Z4spmy z4ABk~+;lRkpZ;cC&p5q_z;R!PLh6?Ta>%Ik0065sNqwn*mu)AeEgEt$q@8Pjo%}=H z>t9K7ZKc}z&Nxv=rG{k$gy;f7BSGPh;lO;ft^!l{^B z`1p0v@r6M{&d0mOWq$blSBo8=w`N`t-1Z2(_&?6xJ1VN}+4pT)Km`<#EGkJrpaB69 zB`ZluA~}O3$p}i$Dk@nJP(Y9w%~tPcGtQ+dGEZCT8vy(NuWhB-Q?A4RP!3V^H-?mFg*+bWGELEcgbWa*m~* zO4^qmKGO~=V)j6cZ|XLghDVjYs5UR3BEcdRo7l<()6B$@-F`oMY5%}`Py=f+y;Z(3 zpHF5#Tna%>j?`F_LbOduqmBS&Hzq$;14GX$D)n_{;i24WLc(!9VrW!GQ&4>%aQ z2J{HBZTyd$S?yN#dtWiA7sIYbF+jJ_2>1-um`(=XI zl0|kl<*Ndtw<`WzwiJ~Nq8S(zRZ0+tyUs=yS z3;RusuOrk%usuA+MShmbu1qCsprmKe@6oA>gR6P6Vh{WJ z?)P#2Djj#r+@jqm!MJ(*9j-N4s-#yNmmc%lx`?@M{X7XsOOuUkH8JnEn)rI?HOT30 z7aYx4yF6}JQ%5ag|H=R@=NEBc|DbcZz_O#!#hIZ`{(5#{dQ*=xODt#D_`x~8++b}! zY-?mD%c$HzY03SH6066)v)U4EbgrqJo5=F#)S}y_v0m@LUl*oWK%*ZbQTR54*>hO@ zlry;Od0CkBmrL4vOfR3sCfT0w6=#JsOg7H>2V0H0#aSipJ~1;~9J)a$HyNSx+Lp{C z!->Dp=5AolphZuoiQ-~+f}*3!+oKKv!Pu-c`*)9H87nIs?_wV<(v3GJMyp-c|4lXc zx%iw!V6{g2gi_7;Mp%kmtoqUoTdR`i9(jhAV^JhWPr`mev)@i8oe5--Y;DKA=9X`> zspXg4uu{^wIGD?9qwoH0Y(Kj5-BaFB2<_jDd!8&>ZP8Ytg?(l5xSRg`z>1Zy((`$;D8 zkwdE8AC1}m)l>2-n2wQtAsR&*zM;N8$a8rO?LE~kiRur5`jb3U*3h9XN156@`;NLE zvTa-cAA>fJc)hmy^H1J(n;j%Onu~C9Y8vFI`qPdSXCkLSaN#Lgs~hLY>e;BjCMmn- z!sy+FPX2Gr&?dp&MTeUN0XN5zL_01-^7lyBGz=HVarqCLx7SLFyEJ@f-Qr8IR+l+A z2-_SnHTqcp(%hMTE4{!n?8)}(!_&k^k3^{#)WrrRurUi!qMXe+G}`l>CpI-Ug91Bl zXb9`G;q+I|sODv^8pu<0T6 zz=oHrL5T7SX^FaElh~}7y!D8aPd)vf@X<(AU45!lAR$PQc z?O8fr%cQ*>-QV)5eiQH8qYVSL9KPQK_sY8I(=Y_zj85d57vKLXZ~L&e%O^VMvd@%y zczZCmPS*URiNU)33vqKq=J^(>j>z`+Kv0CQ%StTf~ zt8iM0u^v10^Vr)iozq!JlYJj?urX((cDK%?#^UCs)1a;UYBEoS(iiahdt}R)+ML>& z^?0BqdmPju;&ptF&AZc)O4C31A&n+4m--;}rqg+xv~hD#>^~Ma+=H)wO}ZvyxW}HO zid#NG(RFtkyX{uv9=0Q%Z)wB%=iBJ5dd@s_9ND0h82q}erVli5ft@A0Ctp{lf7#ZT zqIuwsRn^~bDIPJ1$avFtJ3n|9&o`ZMs>u2duDU9x7z^ZghWvfR&~5J_K~Kca*_z{6 zco?bql|(bgxI4dc;P-~=!ew96v`Cs!<2!~8PKVKuLz$YStbTF^(=pKVtyP;q)oAbU z0&4TAp98&)9+J9kAH}Dpmuk#F{bQ93MAToBGpBq(nER8(L7|bupP_3S&K-?>%c@OT z2#&Ag%ybWDaD@$LVtq~8kB^bHit*t>g=S$lERemyP+^D&8foShz;?x2 znY4U-h%_dC$w?9e;-QEQ0wGRWA~A507m%CP?*p5 zB`8?q5ezzu$lbVza#{XZ&y`H|xUZDpRh>gey2B4#mA=&(qN}H{3;ATq8_`rI{qBX+ z6b7FPr#z!iR!?uN9~4Sl?DsxC(g4>`{~YW&yi0 z+aKT?7ssFKh$G8ReppSF;+{6}S-;oQudOA<^eOY8i1R^zT6~tX$l-C*rxc^T6M`9s zGyHj)xFl~Ja-XIATMOWTX3t?!{KRVa-~*{Z@1^0wmN&j4v)y-z@r4igv10EIU7xm% z-!9l2dlk|!`g=_+EfGst8ODg;-5rm&2zt z=2VaJo7CvGrAe{}nMdD^zRr-mA%~5W(%j39B!yxxfL`2_VUf)SukC@#4l`NX_9dIb z|M`Y;>YB)?{E0s1_@)YAlnM~Phul2KwF~yD6Vl0%?!hd@m-j69-HCjy= z$}wfyWN4f`&eF^}*?LNI-6P9dGoOCymdvQ1iJ_E^WHd4EG#NFYMO0OxOLUNXX07l} zvxEc0LcEVgbX~ponhL!3JX${d-0^vHW}nOI@YSQ$2iDJ>C^Q9EQ|@6jEFy+3yQgE_Q#K~06QxrFMivCK^+Bcllc6IK=g$j{R^~7Q(BZJvK zp~6~P6-C}!`d?J^M2E|yUWG<8dS4P@X$TZ%_pEd#Kcdw8K`USyf-xvoL_ANEj&QBc zI5BSt{DCsI!r-9MGKa*gP;GzyML>awyZ1ZFU~Hr3o7+#JAt+`&gw)b)LEsPHe6^Km zO5AOC#YXuUHV#pJ?$Sx8%x5cI${Vxg8W)R}ubw8p9QODVmrbY2?VBdQaEJel-f=CE zBwg%JjYBSrf?qGx92VB-9JYq+3p*Z0OJjZO0tT9SxzD`Ot=a|suIf3_;-G@1r~%J; ze=i0ehp}g_Kg%C8!wTqoJ3IdtkSF-{gCldmHbUg!{Jqr2t+Kf}D<mT+c05fU{UC-$qe7X3PGJ10S0XeskSJUH;@RqH#;noT@RbTGcwKTZB`tmqPE!VlWFtHRX!H}1=uHzZ**I>sigk<7x3kIG0$My_K z8M;fe-l#G&x3-Y-x& z?|UIH=!%;^B?{wJuZ_F^81J#F)9FcHB}~%U)OT5=C};K*wCGNWw?1;H47)2Ns>4?4 zJD*h&VEu?Mk?}XtRy)7KHp~;z8&FaeV9T&L>|F-uvZ=4gtwfsX9h=(h zu!FQ_oQShNW#=Uh5Jca{pPC2{$&P7GZPPH>U1-^mFK2}lzf!pq*_WZ*(sev8JpJrw zM;}k+2hKOs%_Zi!v2&knB9A4SJ}iH+D5PqQ;x28ovWac`{G&EJZ02)IhysV{ab=sy zoRFMJrl2+TYZny-!y?0Fn#ByoYlCqq z%9MABRuKO@6!RVyJg!y!GOQ`8B|7O5{d2L)W^}^KxppWeZqGrylr~=26I!k5SbxjY zaz^u+zBRKkcz;ptQ{?+4Rle!wZRa`P7q3mb-mK5H1s`nu3fFKax;~0CTHAB$j;~t5 zka17adZ5F~694etCU;Mgr}N^vHZ9X+*79g&;UwDw)m?6;(k=<&<~KYl?Irv1dLyk1 z{(L6?Lp<7ZNL^_=S6wr(h=q{z+68o*p5z{%{(}p0rR4gc2Wd)WZhurFz*vo2)T%$) zm!#rm6{m-ue9cGriq1$nXj!q2&UaRB-BC`mobrkD zsqO#7uMR@7nHcn0veqg~3ly3Ir|GN+6FH1U(puz-<%7dZ8xv-YXs z3fqjTO6%12_>*{Ry+2-kRD#lm!$$(FGcUxaa8*Su%zCLlVcyBph}cocFF60n9LN7a z#2P8()|Wo1tnZQdr7!29lrB^qZaE*tWy~*9p>j(s%yM)Nj|ph_T%$bpyOVJ7qO8tg zs&l+n=2KNhDZ{yJV#=_*W@Q~FL>ew%B0dT`j-27>pdMH@q|h|5K4|nHO?=(C(i|q1 zYVl%3-vE1qX^@VAoXt0N)4s!y;8*9%v!=#=5{7s+;pG?5iAb-Bh_M@@CQMyJ?B1kTZJUYJVoA{X+&`6%c8}bhcqh#T?EX|Ba+&Mc1x-8R zMNT}zF*~oQl}?O3l+C%z7RICZO(fSGZ%`6*>S3X{b-NcFmrEAdQS&?cor8A>|ALY>Hy7-!bQ+;eJRl8@Z;o`LR zDY?{m0GWl?-$U)-~rxo1@xiJ}}*WxoqKcOP)1 zbiMc!8>C`aTX~v@mZR88_2#gOo4H6mIMCO%6jsq~+sr31HGp4*9Se1XPqpD z3f?VG)QH~kP1PAN_@Xzio1DaMGjdXsWq-86*rewivl3W1#~lW}u!bc(6CjP*qp_eC zO&7SZ{`kl}d6XyjMcqSt#Bp(Ey*Nidd;T&M@S~6KVX}oBU4Jyv&aK=xfdY>a6PZ1x z&54T4=R|HR#_@h6ZwTPBPd~%-?yn{!jJ(B_j$U;kxh4QCQ3vUpA74_vbJO z=+ZIXkuhB4-?wq!D|pa6#8zlJ@`2&=wVZ&&HjSJQMrCdbr9YV(75M0`g9Kz<6ZatY zX0fY0_M>?mdq)BUlSpm8Lq9P&2rb5za*K&NBpid5jFmOotmpK-`pNfgbZbAm%9mcY z{Oa_FD9bf4N_O(A-Q-)&HMidk;fzua?`Y=O^D1_(U<`5$%trbuc`Vf<2{7X~lj$53 z<8CuyZ(4^POUT69gq-#gyY$|d-}3MolJ)x)x0u|qtsQavUgSXXch)V(ox8Z|Yw`O< zxKQNr#z!qJ&#Ac8EHKW|e9pG~i{ossTA=B0K>yj})f^4%ZP%I&Z|8(8kSIDe zpgPtJA4_$rxAFXXCVmvZzJ_nH^9jzohD?Kyu}@;T7~PFTkKOKyTfWr zE5(Y1FOI@)`huLGdjTusr?N!gX*)DAU*WNJY0$5%~)4Eb4BWVFdxP2dc; z8TH!SL*ljPk!iyb7IST ze=hRPA70QD^4^zP;{Q$G+jNf4xx&q=oqW&qx0i~E=Ui1of%U954PRESZpE|v`j+co z$h~&IA`qHOTm~!55X4CGD|mY zxRLM~cYx!vd(6|~)t0?l?q=iE%+}Y0X)WC|tSLI55;yin{+M7jgxXJEtUgXWz1h~M zMJR`!wSvX2;JLcsn-(!EvR|~DarpxBns)=Dp9oG0EtR{Q0Mw1D;9w8Ig{9mHl zudV39o+1T}O}y8(27{b0<8cQ-y05AkuqeqF>uCf>ChQ5f{^|qo<1AAhR@xk9VX<|;uIoTFg0#qK-%*(;QC zhKq5LvZ9;M!{qi~jS6nPcs^17pp<0E9#T<=HmCCb%hQAR9Y0hJGxCPLrxNzOeGfKU z`gmS2%gxDuuB*V_tnXaW{!l6IdDKDNlVvjqc!_~1aXT7KZ80o}IYCc)Z0ItSKa-D_ zB zY$`4;em+{%BX<5h{1`WCK-@J$s;O*+)omWrA*ymy4SPdC3Ond^{RcTsJdbn!qP}Ug zZ72rP!o2n)!o!6pJ}pXtJ1;n2A^)8Cdmb96Z#8(%r&*VwCfCYwyx z`;V1KmY~8&^uC7MjdYdqF%QY#l$p*UAUcRG9}ZUyxVUHc-aI6gg^c>jw|BZ5X_GGp z^36Qv0?DgZQdaw0W zuJj`o<~DYQcqnGfHXq6V?}9zhS>A}>!i{pTW-}-#J|xu~shP;ngAmb^(aO}&PtGv= z*yJ5Xzv8a))d!DCUTMAtX*&q>3IE6a|0tUlN8{GBLA^kM2ZhF}%J~u~Oj<7rV1WzD zJb>jXQl(9Ti%z7}T-Cw%4AT(gG{99)D9!?rsOa8mhRS2f(snHxX~Cuk0^Sw{fdLe* zSI>l{_?sgIQlHIvsUY16`cVyQ<4SDK#IRVbYVN z0c{}yFJcjPjThOQl)3_ZESr!lok|!~1?mk~cK5ATR6S0f-0UZO9v19ZM{q`jY8d!( zZJ&J~X(Em=qZEA~T5diyTDivdcj2wF$&4;qI*#PFPj0#pj%VN8-;RX8t)cu8K!TDS zI8bfp{`Hx6&bHsDN=@a5v)VK48PlzfM5|A43jk;5E)h7@lDC`Kz|~gtC!ayeuSrWk?OVNIJdBVlzg4X?( zG6yP+42~^-*2Y8AgcXLo<+2zWl?3=;-|(5gy|52u>%ghGpt`!N$Csl^x0%`q6>qXs zPstUas7mUq2|K7!;<)HAfsu!IM~~4L-;7F))Le;iq*ze@s`Q2pVoc@U0lA|AvZH5biOyU(;DBmZ-;i%p|%> z85#5LUgjSmW`!4V%c7SG`IOTnNF$8(m*A%JjmfiZU^WQ1+(gDk*Iq%;;p0a$BOz(n zu!}a_hYTj$tJTFE@8hKW~`W`GxB@5oL8<*(M08abgcXVq~_{i zd0UDhA?8ogT1`l=&NT?0bW?bSD9{}EFFGSYHh00j%kZK8;lj8IwVEzovUnx89Io#y zA#vhY=X*s8?|aWFkd%MQ9M*8NRW2c6dD*HGhk08DUbdqJ>|mI!4?0n;!pz)h3aciP2PZ%s~g3MLkX!sSgLW?xA&C{ z(%QOOPXYY1&&OIZ7g`sP&i3umEF zvPCWw{{qq_?h1c9Uy=R-tVr0bMdcLK_s(v_2a}!!lojL9;tfgI*9_1xj2MECvS0`} zc*14_JvJgbO#QQzC@daez~+8Gv+kk{yCnQ>s1>|*b9$2dNJ(r#K|bTdgz@)fjDJh% z6_M6`*%WG4M4i7*dggsy&!Wqyy14>>zRmM^6SYOm@XWo)(Y4ic&U{1^(zxpB%{@6# zyO)0+QS(`$=`NM0D;sd87a?;3p(azFCiGQ8**xSo3#O+OOopX&wPxVF{ck?utZ(Xzf$SGCBB@g20#}3< z{s^lYEs`#5dpR#S;h;G(aLYv*;B^jwX&}KulTzN8RFv7HVTPGj&p2rBI*KOK+mtbs z(h!bD@EE>V$=*cN$!^p!jX`dy9_P)*DyinfD~6xJbY7LM%RzH=`^xO%;;O$qCI06Q zY$F18#fu0QW6A?gY@N}Y|Hs~f9k7%Ik9Db$236lTP&J76N;Cs)<=iIk>)5VF%KBvT z7a0THi*Y%NwA3sT(dJ zpZIiN?dJv584RxawRn!7+O>()LN<+8kU!>;`P<2`e4pMDH>xIo=8Y{H`m#e;&+-bL z8A7^A^2A?i8!ql-khQ(OALGG1ThFc2KIx`;GEkfSw5r30Q`5KMW@2VNGwLH(o@VV= z1NX1yJi1r`YU9*%$b;!Xw*@r)Mv9!3jI6u+Waw!ap0TQv_so@`+dcTW%Xh``259jD zMon*pT$K5A`qX@sCh78}Tu=JbGQ!*glkB)$%M(VsDdo8-VLAiGXR_8fW1pNwRo0#_ z@cLFd_k36dR_}U~yiUHHI3DHe6I}emGBqfR$UmWSz{9mhT3x;+Tu-4rJ9HE*#PdS@ z25FgV_q}q}G?g0)Eu>G^Sx`mdWo8*)wOH1TEDH{_Bo4;bzzbvDXAyfwk5=AjTJ#LKTS8*2>twi{Pol?IgsZiygGx zG!rIy!;G%V4NNtcIngE6ycw_#3?>ADc84x(0Kd|0`wOCtFW9`!yi-|gPfT@s-&g$B znKfzFOA&UB;Gec9xQ#gSQRZvRR4WOI9nKHM_Qbq9h2cW%I`7`_^V zz%4G_c4%%0q+$lxf@tVco$slQehq{B+1kZb`h@@~Mfd@igk8GkomQNZ#wW{0gQ;K* zm65Mw+d|hyfqOkHwbAim=p!H)=1)(93Huq}r~eK1AWI%5r|*f}5&_}LHnPv-@Ur0= zISa4uGT@S6Lo$+dfY=B)B4ZN(uqp#WfkwdgBXCLwOboYI$8}f~Vy+^zJ)$eZnwEX( zkED0Mo~CPlL&{_VEX|E}z8}qR860EEOW16nt4SPjTSb1nys zsU~vq2yT8)>!6MMPUAOC^neC4@}TjH-l;FX+k+dA1jef0UW9$6`~PBJwRa2KPrpQ1 zQIy>J7--_GtlAl|LV!#edX*)lny@W+(3U`81VkO?Z*M)YP8Dq_IJH&XwiX!h2*}?; zS8nS0}fDTw6K>CNab!TW`aRfk_Yv&D71bc2x|{WA?8BaijlVB+W=@8gBut zs0mP~9>sf3jSn9MyHD}SjYP8N4S4@i@44vzHM5Ok33CSX+jl-#(IlQco;R=Pt-CUA zROd^^)x_Y(|Ngo}y~l}`BCsb^XVUb*cd|$J#7EM{x4@0zXwTmVW)L&*ZF!+l++wUO z|D&*LPhYic?9J>jRp_kc*HUBu0M#9)?*h_*-Wp?_17aj9Acino{Hlq+8XD;g$a$lM+*An2Q&nWEo7 zS)XztsD1n2^CE3_#9TZG=OrcN9TqO}y0dFFTu+>IEyUpFIrCkz6z#M+HoOJZ*rrSw z5bD|BMlnZk@r<*l^UCpMOn7W6yt9Gc``az?B9&o=t7y5@2wP&++jDMki*vpNZVG~nc!?57>s-N(2ga2#TP;j>)Dq(>;ZRq^7pLR(MU50z8 z47q3wIarG{!oFV6u0dWRy3PV>breYdC}meo718Ew(|+vkA%IK&3|vC=ePxX~aECq$ z2_pCmXtQS6t{e!T0WB9;ZuJuqnt;{e*TGd&vZ!Vs*S`hmuxL@w3J>UMH*0$Y{5XZ` z%xs;~R_Isma0R;gO`9kVjZSM)06o;UOabyKj?@Tf$2<=5#8BJx;!6E0+}B^F;2K5i z+)k_xAd2eRIH!h%SE)#*sa?z2;mOPlrWa@Y<JCb|-@f-Z(F)%lt{aMQ*l zv~w+i|L5>SakwALxYLsqWT??XR*@?0^=qr>P453Ynlye0qdHX zpzX^y`(u~>tb*aVz@B6khTK3pahN-Pq)oLr(|2@mCCK>du9|1W#l4qdm@-)fS7tWJ}vleN{A zt<4%m{O)^!9&)Wd`qx1G->+4m`u(-?No+9>g{k0{3nhngHp5?}&{|lhu4tYsD~~c) z@Aoq(c_Vu*Qf=U5+fHPQ2m5$JI-rigiwJIz5k>UHY(vC8}(niy44>&m5~-7?zX)YEj0CNA6i<&fh#3F0k8-q z=UI(|f8N!zk$uyZT z>YSh4`-?UK2Z3CkEZ|zK?^T!!k^0WwPD0+RAR6R|q1Ll2YEGRmcfp36bS2y=0E{x5 zZdqQsd<7snzlWpP^IM}hIpCIi2cWeUfNiCMF)nbJmDS2O@E5OwOz~-j4q;44Hpj#hrg_N__5J z_>br1@z49C;>7NKF=W@PC_;Fw6J7^4x1Co%lBz`2^>CXZ|Qat?zy0I1D%s1jXX^XVpfZ%)o2UbaVfP+m^XjrBk-62bi0=Z8x4w)SAbyL8j=&5kD zVhYop?0Vc9aX%i29Z+x(;M$u@hp#HQNb5!MaNlMd+?AmA9)Y!y|D3GgQrIPMv~S}A zB&igjBJ)}gU4I(y#g}9Et4N_*y_Vqt&?(c`dhA+PJm|s3ebxPh++EiPRq{p(GErT7 zivxdvr4o(7bl?vR76IL5?6EIDki-$LV7t{rzcA|Ag*1Uv%^*YKmT5kH7Kp>pg9(OPQs?j&yc4DzZLrW?BDwl2((?u(FKm&ES+Es5y zc^vP}!xwOt(U%$z{ypA!t{J_Ta4$iL`fS--o+;cjZ+$5mXMY0GYqBfTq-US*>3k=@ z)`T`G@kxN(u^;KwNq3~%>Jnr`8mWMI)^9!795!>3?|el` zom`HZ>(0h0N!|-4VL=J>*Y4bB49C3_j`KFiq~*4p^!*E0Wgg;^;+m;$=E|;`omC8yV|3Y=3TWA5_axO z!5j96hU-5bw|^id%mdQPeE_@%oXWe!5xv4LGq!h&;RTP0Aq|Cn<472f7ip!h zCnk9koqJIv0Y%=GghEA8Lb1F#`5 zykF2bm@8>pLO?^D{qf15kNhklVKR7AZ%*Uo+^Of6-_?Axd=5p3YHOy(t3%iE^2P@h zFk@d_l|XfVo*oJK=!o0^_8)(Pg;!0MK8KUHTV??8=M2DJCzUWtq(N5`K3?7}oe$W% zTv`P$UCw#*#hrBHquSEn@C}3ntp^UBTQ;USqIBtBz`;xKp@c^+O82WN;6VA9CcfV7 zP&njnhVbyh@Pl|7&HhXcDnGm1w3dL$1cQ1TzUbXGZLHR*HmbVm8EUY4RF0CnbMbW7 zfb-29uv&1=3#BxHY9PiN3k`kzBe^tOH91p{;FR&B<4JH ze-Zk}-lhS#g@H^jJ&NAqP@W{8p(Fma127){Afw~zZ!&m$AB0cD1Xjt!P6e*?XKC@> zJQWyPx4G`rSwB4)IYu4NtPm3s#ieH%@$1!4eyd%<$(176)7gp=%ljJg>a`ZzeESkO z*QRuEP#DSAvE-<_g09#@?p}|g-wh)&n2Z^8%gEUiv@Tp~qn$j7ggxlLykc4(a|Z;a)6 z->?}IYs^`^EX0kXL`4OCJ}a0k!;Y%0!5P`IIOBuolgjPTe%bHz{ioafZ53)>?~eW?s0n_UZBi;Gto8dc zpZ{A4zL_JUYUQ{ZZCKvbL{LXSOzdss>{0cWaGpi+qbB)JN7;8rjpJOGaq_mm{#uD69IT2grts{JsYOoYN#^=q!9NA7ra4z;0r`4Yo+V=EVEP;#F(kPW!c z*fjRgmU|xpzSSK-(7!c*z5kvO*6lbBPk@)C&Qjm?_H4VR?Dkc@ff`!EJZfE@ zg@-ZdoaxcgGVh=naZ2PUE^+($T^`?|zwJ7HE~bd9HSPHdd#V#&bHTqhgQF%q@j@j^ zQ(cm5&Ii4N*uz?4pxJ=U;8ko(;FOY*>gM|)ECD$B1aKpJ01?Lq;VL6`31&E`f?%Ls zVjbIQf-`w+ip05kU4pH)5C@T>Q`tO`ecmIYRw7LzFr zc|IujMl5sS>GKp#-xe+`AQy@s)>CZzj)o1Vyy=-&5vqd{Pk4Q@TOoMaMC;#O8$H>k z6sU*7@FkxuN%RpWKpD2o5N2It!O?3Z}-p zdml$yP<5N>k@_}q1O-_evCFlmvwQB?dF)-JYRxC~F@1L9>lO01Dqw4GA3axFR6)U> zw*4FwK%B?O#KZ;y1mh{Wb9I7G=Yp3wL7s9CcNaV{7~4)v@Akjc8h ziMs}HzzZwNlgcKw_JgIFQ!*`Q@**m9JPw~UUYwMn60x$@id)2k)u1>m{oz8W==S!; z%+JXT`nG>f!Skaz2B#VJ1@c-vtaH$7bYd1_kK89c-A~trH>ZWtiG2own+iDF8J-6_ zefh>RF}I(7MrnEwy~XOYy=@&Ot%pP4&-femO)^0B1DIYz{E-OsS*;|!C7l3xNBd3` zhu8Q}l!GOp13C1>`lbfWkM9-y9BwLn;;|NxUWh{MFPC!G%IwS2_c^W{SNmd)qqFR| zivHc{)swQ{bDU>Q{>18g8($*#L7!-#P=wu6EUuhp@cC^{)~?`#ggY-@ zK9UmhEWBkG5sl(d!jBXy7B0-J!eLKsjMBeHnU#ikY0rZh_!`>|=`EVR)vAa`sd*7@ zxpxJ9=rihNzg(2%f57N;#KDp|S+q}cl!zi<%SNFnlfEgycE7)%AS&A&T$n#y3Ldb( zJjNjS&H&unFhoW-sjE^aCYRP3Ujx{VtIeMkH7m` z@)C`W#K7tM*>+}uxpMxxJa+XqI&Azaf>#uNvaTxcuA7fcOhxi%_uN}FQF{Y0O&%@d+D_wqSsZM_!T*+EyWK5KQm-8Utj}2v=(P+Gu>PNQJ z_Ou39+0T*Tex{GdIes{v0>8+@-#Bx>b(cPa3U$56$gm+>Qj@4IU~=eY>3yuxaksQw z?5N<22bJ2()ZK%BA7GQxNFz>Py&<=vKY$#b zYXpQTwbx8nXX0skT z716@Bv>WkFyN8QSO3id6MhWCm`8O7Ew1lHpGDQRzqElz>I^ylPRK;Vp(^Dx7d5Sj2 zDo2e;1q63oG#gL19F9LYAeT^xiqvQBM`eFYE?DHZTey{Zi0_lyVe>}M(M^k@c1it( zz7mSyLn3Mw+pa`$a|L-%T(dm6EO>{eh5$@dB+nym_h3Ja55NB5l{ z0%Yt~s=Q4upNP83*u>?1)hxe$n6K$aXMLy+P7IK$KpW!G-lC>c!Wq7DwS3ocCgXj) zW+^&n%S0QGoI(GzH(r_PZE5q!NmO2+46%_$wK}}2eTqa@cvaHKl~R)*6ZEm<_-8q6kuz3G%lhrnq#Zowv#}Zh4e%6^AaS^ z*q?eqyZ{hyFB?vSnvV!+d&16*kS&j$C#V8w(5zWJxHV4inSq80NiZ$9aNXs8Pq{pI-3-Y0UNRLUUQ zkhkwE+{Ddw$|uzl%U=dtzwD~jGH$&@E7h6fJu!Iv@3P@%+TEyI8ZKlNaar?e4Dxp+ z)V&V(ht(DgVCuHYt)3f{4`Y-{hnrRv25TCA=6hzHKFI>wP-X( zprgsJrJ}fhU@)_fFJ0IENfG)mL?+<*?`YZXApYO@LDjo7niF-AMpyd^a#4BcSD_Y{ zP2AsZsYzV(T46KT{&I-JX`rQE6TNd)165k?#4D9q zsq$VbGdf$~!;4qKOCRC?cE%N!o}gX;bl-%VHX~iYyuO3@X)wb<3FIP4`~i>`79loC zgtP~Y*Evdu2^dD7*VdjaK}RR*K5X63;Piilxd$Bd{cvYl!2BVYNhkQI6x8=$XY*X* zArvG1b}{|{;P6E-N^T_I_%w{Z@?(+zAP(i-N#9-*_rmzeyYTs+J@z*IZv)(00R#yl z&z<4NML>0ji98n(cao8j(Hz!RHA|BxM#woQ?<=-BhM%JYrhgoM{IdbdVxVI%HP3TM zzmdMQ++McIdxT2k^Z_Y`&}U#KsVEL6T4I+vm}_QP~NwnQyi z0M4B@d7Wq6zv1~aw>Mu^LoVG;2f-Ds7<@$}hS#K-9z}903&wzUftDv_zYZ&Y_O0X5 zz9TFxS+U(@JThLy9heCl(JG8W!Z|vs5AH~$ zrFgw=fe58BzJI8Q*0ABUa=evUUIVA-2jzgg&BvA@^ll?tr>lyF*kdw9S8g8%U-Y;6 zZQMjEJ+?43;}m`6>QO&Q&&H0J9SxV;g#H;HsjCoDX-!k*XRhos2Tm4cU#x~p7X*$~ z?7u2OYOe(FU?>yjA#9Hjc6A7Ax#7A04Cg(yAtZM~kQ7-!KtR&tcvoo?DIU~CAtwwn z$z)nOqA>54#1bU!wm_;nLp@OI?VZo~N`)v2_Nx?RwUR&xBPjds6GVFo6aUV}VTOq9 zl^)QUNrp9v8WEDe=u8&o7z19UJ&N)h|EtGi)J~AY4WQ)Gy%#TYhQ<;EPdi4TOvA6w zi(qI7yLM6fP+=qzm~#}beQdM!nO(U@pZmbZlilDgM@>9Y^>@;*C#DjvZ38{wezPx$ zvLG5#foQ0tZ>l~xL}bz&Rwwv5Dj+;Sr1N`^%Gx+poe{N}^JUDxap#$>70G3MSrY#l&kM7wG8`QByMfG-AO)^Jr)6U9 zGPq}pTO^PF#(1lfwcIFB#O4=Z4ns1G0%>q1Ok}Oou6UMltEUvw?>ro#V9pb6U2GoK zMnYpa6#+k}ubTddk8^2<<&gB`^x~sGg~8V%%5xEx;ITb?42tdoVDl-Wbv=l99@W3x z+Fdu6jT0N%8+fw69=PDSF5lNGJTOz1A)DK7+?Ly{Eu9f+n(S4_3xu` z@`Qa!2TP_Kqs7Ol7v6}Sygy4(yAv6OxmKJY=c^o`eLNMB8L z@$co^vzQ2wYyYG_Q0QQ;jR6r<+!x1~a7`#Ndb(5YPz#C^wpcw3k5q2%O z_F2?^T2edD(AUS%0~1!58gfO1y`UhXlmw#aG_;ca0M(DCrlVW#91oKuAwwV%#?Ww7 zeoc7P>_ejr&ECRKQ1kF^shr9yd*TlH6)8QC`7F;qX_R3qHy0kAD11y@eYnvk7}+Zz z51YlalE6!q-Y0do!y`WPVL=}CmWvfpj*BgxHJc|jz5UFmgSM#)B+K4S=u_L<9&!w! zDieYWHhQgIr-rxi@~V6>l?8}%f8uqk<2i;n{5ZsL5l`C8)NtKxnd_JBzf#w6&EOlo z6%7%Z!_S`m)c;eEq z(7pW2uM~-jkpXl8`3I*oE1f_nGSDRsB<$MMA*bnC-;Z$9CTfky%BD~06;_2c>7eB` z7pM0z?YdDP^myHoX$8L^weH4oFYOJ&8xy1RzNjc(Y2SNJCXYabUe79QYTlq zG$-m|x6!{p*%qzo)Wl`Q_F^(9>65KT@yN`p)qOWdg+9*?Uhe2R!Iu#y! z1t=w5CS**IeWQH`8@>*J<9Dn=VVBsuaP4>Q?3N`yo)NfGz}eZ_qw*w|mpbyn1UagM9wSZVn%p5FJQEVLBrS z{8$jdP83zwlUAkflWG?=C5>Zs;oiCYOp!09 zYq2|O#<+hkZ20F%l>4nY5Irnk*~qVfog28E4YV<#XQxM^IN>Ww#;FHsL0*VLX&yU% z3l6C%BXEi!A`bt_nv}8LC+Ahl$W&KQcpk#Zf8@+v0>$q+H5_lpy?RS6gP#BW0cE+p z|NSlX`!CTTjE>H-pfH}TNNfgqgr_nsYxFef#}weYQwW^@_jG?9EG5a${Jn2dI=Pj7 zG5LR{08IXn44n3vWyJ@961{dosL^9SG5 zs*a8j<877AO9=xq1~&w5y)g$-4Ka(o^W8$eUF56^gFb2gGimF#J3ypH%5o)&taktV+h`khkc- zqIA-BoiqI=1J|sKnvEUXs&+*LC!b_6_adbo;jUZqM$U(VB2L@pw%@5MkIcSbF+2sg zMIc2x?xPk!V6;21jqqC1We!pocI((nt`Dn3*U&v zQsB>fhVqTspd9Fi!>@ZV9fNYb#xoY|m=yR0q}$0KC*Jzbj=w;JURo5PfZ$)fw?O#b zmll&p%tbyJ5mcb>cZfU4y@W?{*R%&7y*dq!B8_{mkqZ80l1go1%3clpS_=byY4N|; znTesrLqbne&`0DLiCxiIxi$L%FhYuvejt#$<{C;JBx`dhHB(KwM+y$dwd&1Ip&OQc z*O1)L0yvZ8h}d5Omk-Rkxms*J91auFA0Ts%%-`=vp%I;i+o38H(fR(#^T1s~Nm6%U zr^6O$KvRUm6i)@P78UM97CeeRG*iVa-rKYwm5W#$t2I8N2!KsneIo1r{k zWYjW@S0hP%@F^ZLqL^;3ZBD#@1b4W}yUz&-x~UR+1<4bKTZS0BuHI6jjJW8Nil@u= z6r0gRQI$5W{kFwU`nN)$h}JXlWy|%8P%M_=H=$G)4GvLfwAZ&U)v()`ADXzJv%uTw z1~)JRJOLv=qEy6Vj1jQAf{d*ya{ptyz6m3nl#(RvGfrJkGOg{it#rymYAuvc0a-S^PtzV+3#@UBpvF4IdEAcYf8GcZ z7VIgEe+mB0{0>%d`}qG56o)rU#G?WyQ?|h6e+U9;0|88NU<|?Mjj-%a!h;Aa_yUXk zz4;%X(qQPr8J~vjd=PUPk7tr!K%|rm3Sz*qqTN54uInKr`x;~dg3{8lZaTc8_m*(v zCd8w-eM}=fx7leYRitZaYTN24O?$tl4>>$tF3>RD)S0>`fv8GF>^=vA*#U{Q?TF0v z>+%C1ey>9p?Xo8fNSRjzB(cLh)u7+|vibSqR&QlKOpCrbF%s*MV2hzjg+jo??&#d8Utuy$<&pZ$4EHKe$ z!-n6>$}cUAVTo1yP6>U@Be&Fw*U#O$p1JOU^N$B(Owq$ zf+r?Dclv&@Uw(B)T^-xucfIwScepA#IlE#3dCVr!{eRK+mSI)>UDxQ|lnN-IGy;ks zjf5!DA*di-n{E(M>8`CPNQ0zENOyxEA)u6?(jhHKmxR)A=JJ2vPrdJRo$Gu!AN(Nb z+H3!E&N0Ur^UtnL)OXvaJDDCtW$?DY*JX5$vcxyLFAz#Tm4xNT6Cu5OkubivQ61n{ zpOssobm%rng4vmvq{(f%V!7!y81@#XAVA24Or$x*sLm}C8j~yStWRcVr}!j5^>`7w zp+#R+q=kGiYA?Cu0m04vq0@^uRu~oOcEb5SYTW%UNf7JnIwRMXT#tI)@mP6fEHS)j545_ThmfyxP!*dj@QGf_VGWeVDeh1w<{x0NO`NA2{_0oqncRM zqsDEutXaxoeNU=hDxWL}GWddvKZabQqheeq#{e%dDf!z6G7GN48B5CAx7%er0@#VH z8cE810Kuv4Un8_WJTtYO;Rt+8!7yN{*5Iu&Jv;?OoBNDDyQ%FT92{}~o>m{w|1PlK z`XR+1pCH||z^n(TX1!38ZPSBdmx2}HOD^@EYAy}Ap%UAEfa6rbE%WkREydLNyI-yq z5MZEok_uf_LLoQiMh#LKc}W4=v4-2>`E^J+?BB5dQ~k#K&(E2a>U~IhNubVjQlteI zXYMr-rEYZTLB>wv)Xj1;sz0gNQ3#|QY6E5`Q&=0PYbJoxasFDeO<&eo=n-^3kqxQ2 zLz5=W{-e1soIIfwuIq-tL*;f}?3Jx+4D*GvS%NRt#&2u#eZ_h0I9gFR-cWiGt!P72 z5q-uw=xrL!rNfW@2E1D|*1{#{m%UV#l~(QmDRN9cF34Ftg`DSCl?v8oGERLqCN#6@ z(QHjy-56SDZm{k#v@yzFzz&Bl>tydL+K8q}K%Dar>~36zL)hrCUcYnuMnBwlTuQHC z%z+4_dmApBJgny79#EX{!KiO%vZZrLDMefnN?>o>+wa1_B?%D-dH#6>lSlQ0R_MY| z?{r9>!)!-j3JJY%v_A^sPU$`M}A{Zm`PMNPQ}Puo4Hs)kA4~kN$J%;G@_M1nKbFb&>b|jtbUZ z=XUAK9#&jbTS;x#lU>{_G?nr$OgdjXeYHVTxI=`SKj+Al7pBMoxwWc%rA?d(8-e#p z0UYE|NW0{L-4%vgg52hZ?_WJ_(b0fkP;4T!to|cQj3$)TIDm3f@JPQeDE!=9XI!|) zpA~i=A7Bp4M_mm?S>!yOW&L>WqR^$=`*2L=Kmc>ox`{g-nX?@vS51)yTxV%e4sQ8N8k^34-}|duH5M9{qD`F(Q|0 z>zv{?qYwJV8$;5^ocu&KW%<+x{fMNcTvOy)xSN;`W>k>;BiA_}>*X*Xg5& z&q^?GzuS=eHWku-V_igK^ISX19I&7S@N zY{lk#*Zma%${=eys8*uz!2SWlFO8`e7I8HRfof+l-ijjQf0@KvN{GWoKP5G8d9~i@9(T%`xfB9K`-(`{bx0! zl{4QZq%ZxD4{+Q(v$H-3?qm;$T zx;wui<#P-3!~eKu9cWR%#&$v~E7_+%xs|V!BELcExo5wKZE4;K3%a07LJkQUDaj+WVQ0B}U*O=JN%tjAuZ|aH{mI+754|$+ z3Z}U@=dKdR?+UQs1_#$Tnv#@9BLRW`$Jd%)t|EQ$e7>>v#5<3D@YZ7{U7e`cyU1t$ zq!(K0?n5k?W6>Fl6T)|5>;gC&jh=bj8uxSJ2r+fHK9=lG68s^Rm{4uslpPENSmQ zW~Y!|!q;8b^wAB}VWr;Rw$&u*JRM*XkyxHgR7m=P!Xujmr%%T{74|38ie}xfPa|h* z>-y6uG zK-G)p67RePoe|MKqIBwV;#%!(2j^MjnK@=1{VB8u=O1xq+6!M~5-GvH1 z!@@XkX6m*zr)XagUS2og5ECO<>MVNjRZ~ z0L7b^XtqnWIKAisIBx<4;vq(%nhMOFa-Hcu1{%U}%9g4) zk49F=ykjB)oEvmeDSM!yZf0PML!xMh-#_1}8{iU|wl}NYoZ$S=8Ts|?(#V^$V zAS~S8m1@4(I=dgw?*5P=h+Q$lWrEyT9H$ZR(GTr>uF?NY3yRCp?DM&0r7$BgROi`W zF*~bH4d2O@<1Y{jDWVTQ0e0g5yG9kgh@zy@m{0fhIt*KugFE*H+(TCgGdyqjOA`lA zQ`u{`Zbi!Y%X{6Ac+A&0QexuAhIt22&$QVy&@QDy1QuMLg0S!xeE+QIsJ3-zH^5n; z3RGIzr!L>7k&ArwdwBq~vzD9m+6on->Ugc~B^TWl%41lEk>on|<;k;jl{w3`&H&&7 z)$9LY+9>%%;5B~K8%w;{0P%^DWAR$aIs6rbyJC36S6fP`e{_9LkfSS1-7wX(z`MNQ z-l-C|boof$W08iD$RC=JSZScS)~*9G7fI7n2)xhf$Nq^HEyf{6B%gm5SKEtm`UGFH zGPUp7g(CjzWcN9mX)a!)qupvItCxHk)6h#%Ow~AfGN_$5RG=~1dCR^H`q~AXZXD*x z6X?S=E!Y{C2jSx$0VWUcb~Y<{4kqJGf1cgr^0NnRDlJR~vR`>PA^j=PI5=^iazUZ zBvaU9@v?Toit32E`MSi77-8p!2f@i=zoUogXvt4g;BVuguWUDkhLLeRtorfu)V=zR zC4$pa<_*^2YLizIi*Vth=|Sm165E~u^#$`yg2D3z5F|EnyxfRo!xD3WcNg<{_P120 zsY~(47N}FxTA zYVy!GI}2-g$@sZabq}Ak?COm~V$^mkkh}f}MLIu^nxR4w36_{28JZZVG@eE(jn9Kh zE+a^7nH$dXuY4dI;4t@-cN5`-bf1A-E+=Q)A6ET@8-8Oi18Z5Box>p#ZhYRa|5)=D zONJ!7rzuQi5LTu62HqeV`+^#a!$7!zk0bZ;B^~`0js7Q}KBFXU>u`*%!!PLE5xV7r zL@EE<3ji)iED5k+L>QVC7lq+gdW_>J<~=^~kh69(@J zMLfj#_37-5<`+mmPS3DUabWS+YxJH%6T@0hzpHC&!1;ArJCyBsb?XYt-3ZB{`>Hpe zs4Z!*JBPMhT1CF;h?R!}-V>(~71i3aG*28Vi-j$3^KR(AJ+L;J>CSoNkzM7S7(p0ocR%dA$@-TnR_B4?K7rOm*<<$%>jYQzb&8|7JM{(DiT9JxC8qrDiDQG5 z@^`7!BxGYq#4|!vm-s&*2ZKO!u>jM}Zr70T6WOARvY)r2vqtY!1xmbU_nnuB5&z&4 za^-nz8xAUVBjKaz0|YM|Ic{4sc#Fa<*eq_4D}3X<+x(>F551gKm%-bX@~f>re>w+~ zoWwYS47uXNYyQy48HiM5s1;MD7mB#_G+R4bPsd9|UpCO~Iq99f`>;Qch+f40RY$CQ zj8OivTQmET#G{M!qF&JsVkt@cPByIksj)>EMXPVg*}-u<>d?N{`awgH{op4k&Z7pO zv2+HO*Nwb;z`Wme;z2Y1!7)zP`%s)-+#@9G3r4cx(dfMwoAarcz1_L9bMzXzZci#F z_%fGuOm`iOljK|fd1kvfxGJKNugmgqsH_wd6-A4RjjDT5DW^ZCBrw>YmH1j-b*n1Y z`h4@*wvy|Qsgp7_E@=pqWH=E}&T2y2GfurY3MY?8ifYH^)XoRRk3VW{)@9<4sfpt@ zmhrTsN(%aD1zz~WQIh(@YNfY@khifTLY%u^^GE2g!RATBeuqMZ>45N3uP0j!mufUe zjBk>m`<`sa8&)coLnHbELqEA5+uA?9n=kruJnFMF=94)3-#;$Li1hUAJlN=ET-ds1 z)jP&+mMi&TOe0lJrsIuB$jJ4Bm+u~tWZw#o?QM+-e(<@B(RwU{X4@n8JdSw1mgtik zFRoz)Jr1wec!&l0le}c7?KnAEFC1m6U8SL zu5n!#|Gl(%=BUEbEYaQl>(>$I>|-%uwW5-X{plxmc}BGam9_)Js;VgLESu|#LsN8# zv^d_yYXCgV7%c3c-NnHqO0|90)TOnuaxV`c=HlkaTPDJ0DzR=q!+8ifyIMZ^z7+Z% zg)snXxzPpehF5X(o4my-I1Q_;iH=GhiXGL6>7#&kS=KY$wXgQ$+1`Rj|5*;bO7YA? zWkUJb>+FSxmU9={@M!FuSNLAKJULAE8K+cI9Cf)+S)!J4muH2F-Qb1)pH=znuH^Lh z!5`A;m*d>I$%{`J#}rRQ$dhwR8wgjBE{oW2GIFS~Nw_myK6Ge36xw>XGOS>3!7jNn zx~|=`{XO`>co6<8kM4sqRLRBJq6i}{lvaBHb_O3{hdLsI2DA~d+_YkDns|kx8~MdU zK!PwtPALQA^MdX_5-#gF_=ohD>+H9TbXLA;>~!LPT=0kew(djphWgJ zeMz&cKC)&3zc+M^$AMvUgHk3hCGKmeX4t%Mt(s1a<2(<&h|jCmv+MJknw2P<;rP!; z0vP$U(tG~N(wNOlHp#BWLht3P8ppq1DWkZ$+&KphBbbz0>IUvx6OTS)m$bbTk23zA zDm>Q@*ITUNEU(hPud5#lPj^wl7&hLGF>a&P}==p=Iz(1 zk~*n$3W>r@9&i7iQtyWDufDcDZ`QpeUDMOqM;%`;9?14BE@r2Mm8&abTkyxO>V%rjawzu+VU#O!Gqy&WbZf|^I_k71S z-EO>d{E3}o=_;6e1`>GP2z$3aCbN|lnqIYhJ?q9-@r*%5iC<8@5OtGm=@@*QoHVLZ z@0#B;NZ+6yz<_JrRV>k4@jVy^ry!iRPQf;o^MZ|pdzpEK#T}uq@m0^~l<8Q5-VZ$* zekPuyt$-?+o}Em+Cbvzcqdn@fmZdDds?hKlqZF0(HM7sEtM9ogF1vW+zSrh`-Ab1z zHlY+IS^gyfH)2pf+#a@9!)bQRB&FH)+5dE%5hr_4!wF+*4FPV+)ZX<(6m6qnlyGApz>}J*5 zwxQ)syKHZ*W-7gIl!y%2jK%Wk*Fl5YEUMSW?uMr?+&OJ^?J(9qG@1FKu6-;fT&ZyJ zs7BDbFP6SP&0BdtN9wWa0Zgm-jqmRiBucHGcx~JD4mU-!sLN@3bo8Dx+3Xe_m^b+9^=~<D_&pI6=Nm{s(}T zI@w?4nf`{&1x@NU2>5ytxca8;1i7(18sn9?{@N;;%$OZeIwxR_AqXEOM_fXmrRK%ncr+{t zq7I5OpnOM)@BO^6u?14lKF;DgCX~ivZ)PsSuyeAXDJaIlCwy7JVSDGa`(r>{vP;d{ zf{qc8$oFsRbbQph-w&Z>rXeTrJ!3f=Nh2PR$p+jX4Y9ws-8T-H-{fm zyB8D-#;ad!&VG~qUT(oRQep82F6O9&N>O|JT6+SM&-plHRHoM%1{a=PeL?6X!M)E& z8>84Kd!hct1z<7| zJcEbLyvO5o12`&NQO{rKd?GIqz~vMcPN0^gU{`y7jzRP;D5EJWi{P8fAks><>7t95 zp_5+2A{ei{g}Qx^uJ#8M<%t)-sHG^IpY4$v`C5fSsO7?{iSy1y!Y(Up`}S8g|EzqM zy&6|V%*DgpwXCwgWiL%&-QFlo5Nl#|o4U*ZK7G^xi!b?6n zkxlo2Pth^3y!ieL4ZE5Dw$m1z+dx}J4fcq=xh-F91+-dm!6!%T-r{u<+tsW5&VSA^kSiG`#>j(ADxUKDdF%Wc4rBgfU-Ag7@*BiHich!9A334L?_FXPAh z$uhBq=$GVmI(fBjOUFH$gV4 z_JaeU0B+f+gEm+yp|pZ{jSmM4Xpo9C=*tyXN2{nv1Ix3Yp)vY(-k>(x9(SGWft&&e zi_&SkA}2!?*6Jp2-av0u8&mreAUBd0zYcES;=2tX2bbU^(Y8I3Ieq(eL5s$eF!6An zxPI^3sc3;aH_#X%qUG6-_-4z-e2oc0k9jJsk1tbvdGp3U?Dg%llb(0X>CZCUN6V~vKbE$^C;<9EoX3q6_epVC8BEE?9q2GM{VqCOk0w5r2EWQLS){ zZ#r}3T7N)m_(jP94($?_eK^M=$Cca%IY*QM(jEcQbM?PW$Bs_A>GG`sB z?6yJXS!I9Ks*{nc;!4R?*CErVICy=H1^AI`U9x>5u4`Ojzc)5MfPi0hd9bjBOj=yG z9TiMezO^CwMBmQ)_;3%0e?wKuN8!@{P~tAHjn}3PYQ}JAJ^+DmG{1#Bx@DK!57e>g zh+>I6bUtm3xCGeN(m*^vZ_D=c5#JNy(N|X$<)MgI$?Z05OH8x+DA4|Sx6&S8RTD&I zG+b<7q|oW)y9aJ~-Umk6WSPL&9Q=zL*#8KQ3$&&T+93IqcBqU@QlURSNF)ReG8{+nfk1c z<_fnTC0OOS#=ZM^%$o9t!nd{fX{Na`--E43#HkMEzH4W&f|eM@4)qFAzF&DdtK57U zHj^eeX@a(6fd17ooL%^?TClz!O6Rom@M(}^gI(sshZV8JP4MMGD=tHr@-P$E(vmCqexC3Inq!Fh_Jw9*_GuL-4=<||M; zPC5D6n!+xzRmlf=Oq+hahvr4PpVCVZS5;^W5GPgJ--uNWoj13O6|$ekM3FFvl3ogJ zinW?a*`gx`LFQ(&m0aPSVEVuG-D~8h5!sq?Xg4k)rl@gs)U}!JL?}3r?i1HHQPcQJ z$C(`_y@i$c!Pg(&y|%FzO+&QMn0f_=*PfD$k?3c-i&se4k2p1jVDw=v8$w~G}X znCG7vG%l_hpS@=#dXZGLT>6{XdsJgSl_@0=W8 zzN9y6{W0h1b;pVGwBl=>WNzvtJo{1ZUzGEo$HwoWtK8Hk!Vz-1qCLnCYFA zTdq$#Gz*m{uO%qa`Fy9G=YCEICz2&^Orcsh0_O;O_iC8msMpBiP(1RA7xJgZEtw0x zNwdi(DxR=2>c7q|8)R;pY{se-n4tYwc!vVN`;8ar7G<2c-<`h7WmPHM0+XZKo?sTv zK1fM(f9A`+c>L|68K$lBL)w!u^4Dg>okE=3qj|!Y(=#UNb@l zqv(r`_gIyu zY^d19^JdRTLA}Y@d@*cQSmvp1+{x{Fg3$A?)r&uDo(HaWMAlbB=C|hk#5>w*tuboebYQjcnSGdm-3(kgAf#~X4* z#7)>vMEdCSN0nK8J+GH@oF8~);m4-{jqT7Sr{%%Nrl(t-^NvNm`mIi&gQ1F_u;6V2 zUZbsn0ms*`5uOjW^oR3~?`(;soTomd=LQXyqb;>P~W5APAGFxmu2z&&Cg z!+?qvIP%x*L@<)N4^17?7_^Vs)D+o_^^cpAZFwrZz6`35>ORtC;*GtxV}Au;n!7r& zmGP1LPc}tbf%vR{fE<>GmK5A-FV8c(1^G;RV*ecOgAamDLyEFKYtX2&%?-6S20v4% z)_=gSeDdeAK~SA^sICvk!w2>=GhSYak*l8XRz+OZ)3s7wNnk5YW|?#=Eicdq3$Z*j z+URMWTlIXI9y(TCMB&F@l-bz!XP1qUDsg^=PkgS;bE>svah}^yuHmFit?qH;o8sd- z-@&Ig_V(U9wp-)Azb;*Bgc~Z}O(2#2?48r0s{i8}NoBxRF5j-mdD>#zb6Y?gO@Pt2 z;flUY=|?I)ZDpwtn_ZC;^};d{7E!D983h$AtRm@)lITE+sVfc$7(8EQe*Vr%{|#}& zHXXgi-Yl|~@5TD{JjZK=jDwx^;1TWQP@(~&x{LSB{QsyzGm^WnHzlHZrEnsfW^Kf;>{w)Qi!>4O zmxOFHKf*_x3V`o5Mu@+S;xeR?foQ@}tYW#@Ak{vHsxuD#Jbu~QRX485_DM~#qgAxA zwk{;;2-#Q~&1U=_=jM}R1?W6ZNuw#@CFV!?v-9yD?Mgf;s*`$PwZA8xzIu}IuP?xJ z`Xzjf((s$Up_f%+t!^p=q|@7&GP5K8-Lt69b6sXiQW6~eEPkTlhzcvIMAiJGI7c10Xr@=TL-CR7D-&C?WLOM&cVh^1JwCyombj6JO& z|CqBSI?zr5K%QXMQW70_@~f5$m{Ulm-zbBI?~bCB?vaO6sz8IzaSuj7VPY6SAjU@1 z6eV4%S%HNSP5D*Y2q>+rSK@OzW+8zUfI?^S-(owF-QU{evh?7~xiMROROgYn%g1x* zlApiNqhdW#T9@o7BnZb*fOr0^e`jp*O#@yLVRM2g=w&vFD!*ZGoqOYt^i8|6{~=9WR8ZiUO$W%kS?{mbV|P|GhK;yXv^hRBU0(fAkhBqMt@7X}MQ zD{MzO_LYOkOKanAc#>_<9T5CpyU*C4Z?U^J_+(6a&tX1o<$GE8^W1XlkI-k%@z*F} zzIKr$w}elMgSqFo`7zhEiA*Y^{QGoTJe#AolZ*iiKhgsdHRd2P+X`9AldFnj~S`a@-v~G-4N#R3Zb#_If-(2eB4-)j^CgN_aW| zoEhqs@LM5R2Lc+ZV9Bw9_S={8)N=I;m<>M#|R(r+I(TGD`K4_ezPl_`J{d0@><>Czg=ni^AtuSAT!E-L1a1xHkR( zJyTcN#oCnt)$YLXKPwVzZ1A_+&OzBvhNc-dq2!`RcUN%$~Qe z<-RmpFQypJ|K`7DC6hZV+4Tn}8MoBiT0Nexh9&bR{dsJ_61I)R<$n{eEO&q1n@3n^ z?ydxOo!PC05}92AJK2hQF5N8(kxu(7?jtp3R1_Q$0dgdZiVf>xlH>c{S66&`53iIK z?G~ziCCOF&G;-Z~U$lmfUQtn^KtWMFhm-%4&!PL3O!upT&g;WL9noA$g~GqHp!k$Y z8A^xTBK;pmC6~JA%s7r7^0BAGjX+&BoSV)kV%I={;@UYFJ|lwH_-aT2b_cjXU^#6s z1^ZqFNjvO@|GxIIC=9)>;N@>>5XB2_Y}CKritRr33&1DMQ_W-=@T?Nn%;|~PoI95e zAaNS^=qKxMX_*!FvrGwsCv^W5UKV)zf0_lHOw@3fB`w{wzEitHPpYFaW;MKU4@n*5 zQ`U4I4i=pnDB8H;jcK6#s&`2LVtqnc;A&_H`oT$JCOXAismtQq-5LL#bY;)8ski~h zX@EbU78-m%WSk&iK}_MfC;Lx&V%?%_V?AB*zQ0say@Nj1kAV0@8e6sRoY&W_dI5iB zX<#;rHHG_)ET0I5@_N)VEYn1;8YrytDt%w>wDpy7gU0N*)Rnr{~~&j`ir1 zQf#orjJ?c(3$oAl@G*&%{U0>VQm>EBh7@^J=BpoGc}|=ns`sKSGsos2Hk!qez$4d5 z6J0P&d6PgFD6AKBj6qISW#DPOQ4pBc6(4BS^WKpP54Gj`-0sgu+r_ax1*kH>0q`xU zLGsh7KcuL~lTT%sf`bM6q1JpCe``J~oa6dKc+2o501PDJ;iG)BJr>m zAvO@P|JREFIrmuLi=$%)s|Z`e%g(=l^^y_f*$FQ6;aR^fxUwI*Rje6Dm&yUr?uo1E zCGuZ+2+IJMP)GxfkHil-IqofYW;--(Bt5H&(&kn@PGe(uO{5cQ7ew~mzn@&bV8vZ; zkjo`2XBdiG;vf3@KJ203uZ?-ywX_57*+DAvGs-opEoGUvEj7Hbl@@J0v^NV<7UG~{ zAb;?PcEgoGR&(ik>ZGsEy{-%;7Qpe9eHr%NKvGG1%9!sZ?XBwUg-@~zJB&iNjl$@b zsEiSCLAc^LUOtwM7XC%z(y&=X`=!oU!C)b+T~)u4k2g+2c8qq$AC|thDyx}P`Ze83 z%%yV_;!H;WzW){Q2Br1{ykz(I?*}leG-}~L21q1r^#9JRBR@z z!wm@QhFBRt!vq_%^Md!`L@Mj*_su9Fjve5%c;w#KL@MwjCGK>xhVLnf!gO7iWRNff zAG1E>9q>@gQ{nUqiE#^Lf^N6n`z|qV%p~ZKjkwZDz99Tb;hZljP{N}h6oN1R5H%Q+ zKa?XKy0f_RRRYl}Orr|Y+nCspQkSj;*Fv@&?bp z8RhVEN9XpG5&Pga_AXp^iqnOylsoR%B88qLWJBdYdaWH8{Q6F!up-1tsk#G`LE!(e zQjGadn?|7dKdVzilZ6>HaCRM9&)nwQfq=RD)bEG*c$l&9)f-;x^pUEFBU%Ryb>_XO z1P@nh&bO+wBcTs|Xd9_OeU%64ax~_k?aZTt3Sa&vPA3 z!pN+)#Z6>-;0FeR|qMm+{>K^B-va$JgkYE-!|l zevrgq>Ry*_)`sGt)XTn@2jd|*|8eFwC3=fscAX)=0CgQPKW8cQE%HV|X>n}Ji%~k| z`t93Twkdh#teC+ETiT+AS!T~cM=BP4tkT#3+vg=I8z~OqSoK~pN)rlKS?2emTGw_F zlFqG74vV`fTIyGOf9KFe^UwNf)Hfku;gRDUSEh2@jdzr>l%G!^GCj{Ke;fZC}l%1?=SEA1dkDM`tO44a8sV)wKl1%&eVGi={R8$e!CYnnS*lllxtKP+! zF1@8ioG0ELBF+;*`S0GWEr-0kGHmw_2{ivOYLP&osqi<@Ed9UE^*oB}f3t-s>u-`K z(Bhr+33-8|3=!Ci90+<*gm-hWNCbVNW|bXw-ajT<;p3;==qF8SZ?V@2r)V!THg0O3 zaQvT64F>45Sc|yL_@P38n%)nT4Fs(+VUpUy=g>L(?=?%Ljp*WK!l}0suyU++C zqkSUZIeJoOVYTS)c^RcO<>t^Qai3s<+oKwRz^go|4-iE~B;oKG!!f6Hv+S@i zVo^HELKGKWR%tdIW)7}`gQ9yj@`Ops{ z>J2Mheju+E9JJOfe)uLlz#CNnDofL@A9oN>`_SCYI*AVw7+(9?K7DXdoPv z%R8f|f(1IVDheM4B0>eO>U-7K+qeQ#DQ*I@pqiKnJ{UzPifA}DQ28K1&B$i+nX!&L z9OR1T##Qi_UPEE3bEME_$fsy2V_*V7@|lL>FL@S6MuTaxIZXsVkVM!1^A@F*R~atJ z6SZ$@+Xk(65R_~eeQu8jUl%pS;b$-j2rUqXDMO?2+lXS$Qx=Q6#)yOzMjgq}+^{We zr*9E;mgD2-{4i{7Cb>(4jvaS~xsu*2d!hIy2R`*aIaSu*De-_0?9JC7v;Vs{jq%|f z*hohZES98XIL(SOmNtR{6tJVHMvu)}LSTRmU4za z@Mzt^tF2IiM=V#g-v2v{1MmfMV>@i`pt#JOo>Q=qav)1{-r5}c4K=S3a|sHTY&SBo zlykii0xA=ud;wQ@-hqG zC0eT$(B{VPJ5z?w!gu9SZTuF%esg>Q(hvn{vQaF~WjI_ZHfsQ=a|6E$jOE)ZQVy~- zeAA71I3xDX@^GURxT~kntQL7^KB9#@--ZaHADD~8#hV4RFmrL7#`B-ZQL&wVM5D06 zrq1B6)XjaQKSmZ6gR#4%Q7*oAP+b2-mhJ5aZy#|9q7;a#)#e7(!#hLPnNYDbc^@s= z5Sm}dMB4<+ZG;wDgfR<14X=e2B%-@l*I}e{MR=2f8K27BR&zhAzZqPycxlJCor1Asjik{ zPM18rOtXMVp{?J>ve;e%gHisD4+-`0-`m(4@XO_Hjt}9d5uBnJh;{Z(1-t#gtv8Ih@Nn z5Oy!tz`WLk3D(KHK8JAY2_LBZ=VTDL1~?5Fu%_XDxW|!!|8b66_0sgTOHg%5l7l6O z>@9)VGIOQ-X-ZEGSl5DVo)79OE)|Gm$3fKgM)zTGjVE0v=oEdAXVaF)hKD>7xMW9X ziVX-F-Y(VniLGR$a`(Xe_)<5F0zq<`iBLoDBeUAPy&BBffKKWhehEbzhC)bP8 zfz|eTnI2j7a{l)jj8si;HYbb<TC05uf87RR6CZ2~Cvg^o7H`c$cO&y83aCjo1 zE8^JdfWP7ukJISjZg=mUm}sK>Tm-2>SkllKX^j5wclM!n=NhMD2-N!xoK;983t)iP z(V&CY?x7;Y(*g*vL=JJcMUj{2qi0QmCSOumZqSwbAHMh7r{J<)NM7|Qr(%#l>zvNs z`pfQNPyuhADq5fOI-#YBtNxWl{E~va02Z4b7ZexKwR7b1?^T)g!76Dl!fx`i z90(5-6`MvDX{6rOt{gwR91x)PDdYNAQU)c(C`x}pHmuDHf>JJIZIq)Ziw7UQWN}2E z7JQ8~p64gbKH;r6vrq6eAt`i~xp7i;brKWLBUTg_>+K{lo90isVl`%i)yRQGM^-H- zL}JWNYO`6x1{9~q7`u(qCO8We4qByb>7Z8kX^cchMH1m}Nd8A?M4~P?*jVPsxS~}% zAIDpfp@$5;WMYEuX`TE&_!FDmUmjv7nQ zHk8xlFa2f5kt-wfNXNy{QRe=`ucIx3AL~&HpFgNQ`xvSN(NRLfCN&EG5D9lt_RrxA z4YRvIY5f+*Mg0Ezv&J+Fl<}MCji4k8ybR8!ADaUShuyzArx|J$!L?D3>J&JT4`i>h z022}&_sT~7)VB~BvAKQ6_rz;-czU4}UlAEckg9&xTsV|~T~A7=qy=BGD9X9qbQOhJ z!^3Qp7e`|p^e6$LpwK?^g-jj>?8U1p`Z{eC*C`Pag`rGr5(#j=YV;{L{UE3SvM)C_ zX-NMLkyX<)x@7E3Nu7le`;VVlTmZil@tlum%0L!9yQ_$6ISMo3U1f6eU|x7&E=HTH z_v;+RZps8)gOJC{$c(grS?n?`*ZmhbFgluC*D`8hu1JjcIzFi3j4L9ebJ=_Ez z9ERq<9ypW^`2>xK1;YZmj8Vqf0E`n;io(we1%>*nzC0o739XVXR5~{Uav?{ox3tfTtt+)Oi=gOTcVv zFFuX>tL4PggDep}&-wP0G5eiS>LPLyg(WXC<_W2g)>NY6VL^Uugt5%eAqA`FX9%?A zgGow6eZToybpmi_C8t+s5SWNt^z+}Vja@IA=x&#A=(@4nVl}t_eS6V6B1gqb)aSzt!t;v8VlW4d* z#JK*x@;|Z>cx9UZ?Umtst;} z6hKY3Q@D^%ofN(_*y@mpD--|cT)^iv5@KVDT9jl0!#^F^`5@8mNG3jsoIB$ zJ46<_k=YmHB$$Ok>(JNF{!S?8%U|H3!Im5eCLauhf;?2?EULz>BO~OHDS_Knh=V0p zriW(K(D>WW%V|0PTKWSs-Kq?*DiUdGpWriXdI^LW1nM1uXaTe;;=@1(u_7UF*+ z6nEfyLe#KtnQ%KB667WpAT|%z=e&g7A9%3-{+)_+O~gKXp4YF-79k!5Bd9Qw+*tXM zFs)`DDC)U3z<1b?|7_Azcnq&E@gbzHSirY39t}fqnLr;7H)N5U5dX6#Xe>GxNfDLC z*uF|GOA&4h&!_D6m~=Mx|Dgl< zDO7hfbffh9Pj?8bGrb6~76!!C6fZH^Zz~1f4;aBDwvnM?_h`)smSHuQDM52@Ux^Uq<>GJdWKr!-UAAX#q%Kh$JM`6)q>SNn8r=o{fCIc%5r6Me_KC4 z(Xau`lUVpy*O#??(Z;VC_2edoUN`7|f%^mE_!p=D-XF-0Ru)N@CBZlJ=#(L&6o1T({KP)2+W}&Wg{3U0wr07X7NMcIaoLDeGHv`S6){yu~C|MHCQBqfkEuuziOn|P8YN1L%BZ*(z| z+Lb~n0ENQv%f$%VY!Vo`&s7N4EayFSwwTRfoiKb!i)lSL4~2P1vMUjGtbst`6=>DK z;eX5ze`P6mEqRB50oq0v0ADi_C%ON5pZ`(%42pxHtE5eD#!dD&)!pWnRl~_D22oHUSv+(6+Y2)WjAfH`PqTM^IdBoTl3X14jH1j#;<2I3K8WTe>>b zLhM{(d%=EHKL5R6asd0+s9kDfSiwmLyz(UZD?+f~!bo)~*f+nO`!o@pROer9VJ|P4{ks9}{tB9p3!v2Pa3yvBE`e5Y)_UIS36TmB05&F2Kfy7-M zXynh{QB5{NqmTuVtDGC*L23Phl-qn$^v1B2Ve4CO)^y0I_I85cumk~j1N!fh%$yi{ zvK&avWlmDVomI++Y;Jxdn=9+fUyQgdv9QCH+3Z$^`8-aJcC3NfU7CVe-y?nu(AaM) zpZZfW*t_QGRq+%WH6)AgPT|(LZKVCM2Rqi39w4_^0|C$ekwA5xYC6FYSY{v^^6{;* zB=FztyE*A~!6qFj+t9v{tK$_*@RQnPTwr(Iz4l?3Nelw{2cB@$<8l?=!4(UqcbFRl zcia8nweuTt`BF5n5Vg9gSS}C?(c_?uUz3ub$?kR|m^6SrhEf`%%-e?J z{a{1SRl!eUz!v9445qmCt9ip#fteoz0&|7LTbVbZ#l`tiH>ik1J2s?9t>LVU8tr7;`WUyxo z02=T^+(-2rC;Gne}HsGCj z4h@N77`6M{7DcUlwrs0=)Y}A#tx=--?+X8t4TqjFfRn`Ov%i$er3bJBpT+XN1qd2@ zKGn~U{PZoXCfItYZaoA8CFSZ|@TEbzJ$U>c(9Q*?xBfI)BE&=(NW?+sneR%1nMN8| z-t-|F3vimvfE^)c_6cCtFM^5Y$E%Dw z&?c-Qz8pH@Xp|UB?H_J+)#Gsa8xh9?VfhYN#b<&8S>SOZ*&vvtBDxTR2A?GGhh}Qj zg}zXM;rM?Rdi*-#t})gc`P&n39@2aFmovWGfyW?DIys(Ok2LscfB$G!zOjyEJ{1?sW)zaBo_{~Nvq zk%@@kA)qY7th^MtGKUa$N59t;yzMiyd9}&cb%1%+Ir@ zM~oZz%y0YQxIlhr@nQepH_oM>C`vzyZ%(G&6#l4x3Jg{^NPFv9zL3AR`cdpVzyWf}) z89JD*+IbTnoDMa=vWF6*#cmtO9cJF<{0<)pmC80n^QIOMfLa#|P`%V1evenA&o* zi($sC;nXhbF7Fbjqf^%>PZr2P97y?dE7m|r!?z8+YhT_Mcmk|!^50F=#(a1yU{X>7 z-52CHZdnyT(7zL-MN}d*%HFMa`U=mZ@N#P9;bum*%6B;5RYRJ6-=q@XITZsS%wD+2 z6%YIg~@T=hB%Uigy=cyKj@RUrxxmo+|Yzbm_!*u+nxu+0(zi zOUGut^3U;V*Zh&2^N``P&gB2Y-dl!6+4b$CIFh2IfOJVnih^{Dpa_W6&?AV10wUdv z5)u+pN_WRl(l9Cm2+|1B0@5Yju-Ap&&wJnR^Tz(~V}IBm_Hpn*FNQViTI<(!&Q;eh zc}!Fcq^@qi$bR)vQf}wz?ZKFJ?3j?MdJ=e!A@zHCMkbPYEbN$r3lY^t8jQRE`29OJ zi~|L~zuw8#u*n5rI2IhV3*fQuPMTDPgX zI9^}t5)j6@ajfkjO-abpAe;}ld&)cpJtaLoro)p^)No*BClMZYZI}Yi(+Kxnuh%&G zB_JsU`y$=~XIH4fP8unBH@zp ztzByl7JZ*AP64O07kTx#xpK28c1m$puBF}e`(%MeybGDkt|@#H7L4i7hHK>oXPwoB9P!=NmmoA37N-gFp-i0x#}<0x#c zW{TdhE;z{VQsYy4`NyK)z#QEiSa8eKAqviEXW$_qZhW~*ygtp7@XLu`Uz*3`bD!D#1H=-Rob zBOiD(Gcxdz->U3PnSZ%B$503d%Shd_ocl=nsWoc(0uQBT34`(tg{PktVf@z&ZeEMf z%6opP$8#?c?zF*Q`^FGClj*Vk;@Nvbdj2`)S{uz>DmBOjgC>3WrgADwI8mhlHxaA#X@E_$UH7u>Q@XVbzn!? zj5(LD)|M~5taA(XJj%Et=6>E?hcq{+%)~E9v%~~vWXxRD+OuNJyI6#jj>i@L;};Rz z(xahIEpr9743F2NB;B3t2t)~UK`$49=;zw9qUu>1J53va+PVOH3MzA^hzbC(rK*E2 zfRY8Xo6muMK<+fOiKP~>CTdO&mM|r^yC_Oqy_g_7m&385fThg6i0bb37Ys!lykE|##9(>!=XEU;;~+g1xx`r_ zW*P^y?jSN!|^}LWm&qJsF21D@?`KxTG0(JNlDyMq6w7qFe zec-d)>$T{q#0Ow7C%Jfl9KYNt=$gfNY77{v4!A=jkAZRuGSoa;dC6lqB?R)v7)Xy0MzXnX0J^|KvIt!@D$7d4~ z(!!TX;x2n+*W&zq{n%tZNeR|YrgPKdMhjRbD;W5+~ui&K@B)lH2VrpA79cXh3$ za2G^U@D){gjsy$>6zIg~ox{#~2t z(pWaJD+?;E<0}~)m`B=Dt``wuGCAlAU`W*n2|PknZ%S#FsGvJyYV0XSy0ZDh@W!DL zBkM&fctlgQkQ%>}u`FWf(EX!r;e9grji%`6#^*@0t|?cF%&K06rDunl*MD?c^Lg z(iJ1gjae1qck=mSFpt2CIc^gGF6(_e3N?gZ8`jBM>V$zsa^67<`xja*!TSi3Vkff; zJpDy^_v9PBAeNTOzcJ{zn2y00BlkhyYY%mTDbJ-5SUuxGxx`YOa`b*C<9^+x5Nn<} z^)1~vV=e}LVPcYkt60vPa;0mm(=fBQ&|+m)c#Ppz>N*pO?9-E<%3{hsE{rFUIGweq zJ$&tg0O!XGoTUz(`lYTVhYg(WuWbjoij^XNPNZOg>l}MFUE1Aicq?jG3)s9h^y6IGSR1UL zZ^ulRyV+ZPI$m%pe=u0bc2NXL*OZQ6xLrf+x@aZ1~`hp%gFzXim9@_gdxDoe`~NAAlN#)nNa<^2M- zFP=PUq@t%zahje}YpKJ8(eNJKf!P1fzt|rON$~*bxHYfd_WmZvF)*_oz$m{(gLIY; zpo=hA{G(pE#iVXQM1x=WD*_5#*-^ht6FhBjXalP`yzbhsM~xp=Wqp^VsA1)F*p22> z#GaqTRLFY1-{y#`TflZb{SILxsSo;LvikcavF#oc>_l*kIGP9ub|Qw~ zSQU&f<(+Abke{QWD5(eXd8=EVxCg+lNCpeVp-W*V5kdgdqibr@3PAzm)|A7qtwY{L z?B(UsQt87G-WB{p#yinUgc2-!(i9*RJ7}hM{jX1i4q!Sojj&C^ zyLljE)pkei!${=n;uk6gjnY92dJh%eI$ROX)5spT!11y z6@SE~U7*CO7@G04sQS(_3c?Qom^YzDZVPPCuu_#IiT419=spO1+_WAl^;@l)CUQGm zslxkK=rvH@FZUz^mM@|Y679|JR2 zQf*xA+u(QH$F8I}t(a+5)(r*I6#hF&N5{~!Vho)yH7#i~^7d6un9}79c0L(NMptSe{8-)LC2qduD`T@YI+Ycn(ZE5pF$C%fS zz?BmO10fE6eI{N2_FU?SP3@a;&t1de57xBM2tR040SHlO9n|6?DX48FkePzlE{GA= zV@Vy^@V@_y89`D=QMOF&C@kKXTPo*2BkaR1$HVKEb41J(6BjM+ax- zD{rqnxm=WE_>Cgq&uoDX^Wa2n5Er1h z5u6MFb`3^pG2-vxJRr^#Zd1Rl&78&H`jgZOc&qGy4gxT1FN{IoIhp4%FoPX16gsa@ zvec3p?CB`)g3$|^z)RKu6IT5Y(_?U+1o(BkiyqG+oP!KK)-g@acS@p6C!tvq%qW0C zejfx}$fkn)sCG%+e1Ab*?B1Ol!I%+fjx>OvaR8%SxsDBAp*I@QFO-c0?qw<2pv7d<}-oJ?>>Z>T1ISR%ZLTG-m8jmCV*?NSe*WT+f(1CjwKmd>{ z;+C1Uvo}$!l|rLtn;HXF(Nl&Gd4l#V!8}_KhT=K?J_$xLG_CTEGh-A90Sze<%k%rq zxwzm_FzWsc@FQ=(Ii89+l{|WMt|itfHX{P-&s=~no20kLEt|WbfFBsTV>uwDDrk%IC-B;=Z5O zl7gI!wRln~tYfvxrvsf#OVsp@l*cyMZJ+G8_r5`nxyi0jrj%;zj;IOhz4UVqj{OH%8rKO`8Xa5&sCFjU?6?$Jf85D-ut=+pkJ% z!-KU?z}gRH<&(_H35Vp@ewJ!HEm@s>iGGPL)sVNT?70iOT`T|keN&A@;)H&gXtcJi zXwoC$XtuIXrwM*q-;_($Db~9-(^$gTuR4g_$!^Ghv^v0D!L<(ux0s@5{N3r)F&+(Hq6+Qr2RZ9j8Y6h>_-6X+mHIDbGyk z1nv~}1$=?;8jJ4lG%AT3knQit;TPP_Z%f>LP}?J#?lhJw8)Cu-*}c`T&=0#rI9X*7 zfWk#I#}&*x)0w=L8I|cy{2P@_EWw!jh1#5g>3{*VRp9Kq7rO{+f@wB=VI&(7^&hN9 zf|LhcV=sH#Gcit1ix;H5*=*x3e$DDz;)I9>u9C;i$oDhl5`)?wFA1Yg#^n(E8VJ98x#j6kdK&A)oe_RseUA!W zPTMLq5fwCfvzhTZJooMPVmRVFS1_b67vgNZ;tNqN3 z>|N~0RX;ztK*U4TyA(X?_$B`MUgX3>D%>1H^XhMGww-Brp8mV_D2rGyNY=xEBV|Nl zCfF@@CvC|udZyLWj`JCven(QoheHZNyUt2t)80_(J}7v64Zyn)Y&4O_Uz==lmB$}< za71`HJn|WKjFB|lnB}o5-=T#LoI3h1KM@PsZs&Hc)-xb?N#SCb&-PE`c!l8=5hJ^f z?cGcwjh*JE^mO#b1_KZM#;l{{5*n*wsqTV9Yy4);y8i?|a2C2a|FZ|f419*Sd}-lC zkn`|(AhMvrVedNi8WD36H@0?j$WpkluFyLRX|d}vD`H(qM6UJ>LgT-F@9p1-ff8H0 zR{N-Y^Hvj5aUgk78>>1ozu|U%HCv<5w7z$()`yk z;vm_|m`cub-%|N)Ttw%;!Vh#NxYc#1g&uC?MV?CJT4ro!rMC#Zp+u$F&komqm(RA4 zk7lkpTche(enr+Cs1rcfgW1|8vItqk7ak>i9(80)uS1WRo>ONjszBfAt zPat0yRCA3KioJa(yK;x!++2~>O1LNQv%42FWOzpcN0}PW3>*A?K}x?{@1`#h^Y?0A z%Zd4bf4tRID?R9yh5a&EUVc33q$=gY!GeNksi=x7tVS72sIBjToZPkw}T%#nUDC$L?14O>yct5^wS9w^;bSB-h_$4+uRjY2<%D zyJE_#fJd{goms_=FC593oj|iiC(k$^r#P%Vu&T6K^`{iyA$7^%!)#y$!@aoAl+v6I z_#bfd{s)!?^!fEAbTq$DWaV>)1}T~@EwI1^*4}9-JDD(j_ol9E;9!`IYcN%DQ1VhU zR>f2P_L`XvqkfB-|M46+GtU3VX$`KR(@m$Y=zGgpJ|d$4RKVVoLzN9D47rG zRW6vEf_4?Q($A`tXd*)=Bi;4+IAP&-9Ua`9vF}o;#C3}W9B8fnzZu*AlCEc01@zl@L7?84P|G?+on$`dlNI>yz7>*a)nB#7 zi&YIE1%sVVzp=bZz=0vZ%@F;9$BryNv&El=1w6-MZOVp z(OrBk4zO#yXe9g34%Oq-H$xD+&JhFkG@TNHg~UNpMCqxyB|%(!mr7j;5BkTArMue3 zSsw2k^s(o8ibZy_eyV6E-dSw93=S=1=q8&j*HbD_cQweX00ON4CQ%zc^k=fU+Zl5N zV996)QVBPyC%r6&T|j0<4k4OvQ?9K7 zb|&6XD-)ZXWVgQ@*PCqaFAwYn$Lt@0gf)!;^Y&)p$@AGQEHzI7cN^x_m;kv zxNxCjiPYs{JQA`~U;aB!XgeUI1xJ0H9#oo4_;gB~ltZ=@p0>xhHlfG6@~(T?F|9z+ zt8UL4BI{KvfnpWmLA$;W`ZhXoE+fptJF(#xwT#aQAZa>k*?&___1VllYpo-^;~w@^^l`A{@U zz*8RK8p-AHE~F#TTW4PyWD7!t34(5Lfh(sdW>T+R$Df>&Wrwu;;h(o*`k@AUzlz?N z6BOVkVTuzmV^>Moc)@;oY_&H2U~Bn)ai?d1QKERV*HEcSdG&slUVo*I4wvB~%gEli z+uV6KtFKqxmLu?Tw&_)#HBycLDw{EW*eJ_7A0~Zz@3hB|H@?@jo-yUwTw?6=#vY?( z>3Qmd33tXP2X=u|3YH5m2XnPr7`C=6mu?<=9Av0#<{wmSeJ#_NulKvgVie=OZ8ezI z;_jSHCt9i;_AP@5uoCKgyVtDBWo9VE0bC=iFC0BKsSB!*O>JV>N3z&dE|waEtHuM}$(zAuQk0 z`KBhJeW{*wmufw=IC?EhjVpe7WU#e4Yu$g4#2|QxG0(M{3EJxG(w`st(&rccUf4Wu z<;Ut`f3QuC?wgwNzJVR_$xG74C2^}O<2p*H54<+n_Fdw!1BDYx^>`Pv+wU3Q=U2TF z@qD7kNWx|QdS+Hu)~QP$zkcrYvFH4WDwuZ8km!f5sreM$U!+xb(eP(iau}s&X-KoQ z^s6UQ(s4nC@ORq9$Dv7AZu!#)<*+s(!WsWqi;wfLEdV|gK%M<^a+~}a0Wz9Ngvv;M zrP)7Z*fAAfz_$caJ=vdLfzPYPbK+>WG@;AyF#cS|jK? z&rcX1j9Mt}eK%*sGG4JY|9MwqJ1^Q|zLg-V)IroRoS)yZc9+lUTb)3ngkz9id2RCW zhiCWtp47`~G!thB^K&JQ6SuQa%1+<8_?2WXh`%w5$dEBLY_@}CJk{~%y!!CoH2+qI z^F-epq5Pf{MaB#;@3t2JNwfO)eu`4Nw!2@QV;?ZkzZX*z73cPHcyuIkc+{0vCHa^= zGf6~tcfy^j;@P;!XA#9vy38!=+6NJza295$nqQ{}(`l?M6|;^PQl4zNnux57R>T`6 zuNxeMi8P$YBko;!vq<}p-s;oua8EL8A{&;s;V%GxtOCHqn6-)w-tV1v1zER1=Mofp z>a@SeFn+cxSxq6BGPCjYBp#aK%INXy94{Q@1L4d;H4!bRAQW>u{+4wb6om#ZaV`jI zYSz0s(p*RQ_y~c)u%MJELWEBQ)SJ1jR1D^T2n~c_4sev1oDf6&*9P^{oYZTJC}?pv z=pn1;+FvRJDYGW!893`F;mZ?Vn&gf23B6YiiSP?sF>XSYHamxn3OUl*$eZL zV)6Oek=3+^Y|7o%mzDwz;e(o-v0lUjwTwJu;we&I=C|ym-FH^A2cO)}D3f1xo06)L z6!cmAmdobEC3$ldN3U_6gSjk}L0pws`j#d>Go`*6_h4!Qm_nSrQ7FN^l!|?pO3H;9{_Omsr%baVdp$;5AJa0M6d4*%Z}@>=m4U$JiLinE z)5xtEorzvN+l~y_{$`-GIZ`VJ;gx4R=f4%cH<54XE>vH8-Oo+>BT3M11ocXfm$U@snU5jI>xQrIzuLA}doq-^U$*cz_~yhLjCg{ckgc5XV9{Nj!DpPlT7`>Rw@t0=gId-8F4I+#PcQFgMwA3 zIfj*1m!|i^AgeE-&0sJ$jFNR;4r@YWo`)QodA+kTY$hgVu?<7cgmCE5-e(#h{gkd4 z7MT8GK=mr67EdkwiRbJKRleCUZoe8D2M$yC~ zPJ2r?#=88aRa9+}2|crf!=7#1d)C??E#j@(-rKyF7;oO)YW&ebjjMBiBenf^nlu17 zOn&6X0}TLfFz;bzP`HZ6SN>t5g5$$o!u`i9=;10m){gf#712MOD{UuK`5O~s1Z&?EWX7t*Zvv5GiC?1ybKS3* zkNZ9kl{^(5>qk*^c&{jMd7J6lNj(1+S78#+lTA4Ha6L`ohMx^V0j}@_(_>P?g!c9* z$`!XAdY*%Y{)TVfu%)k^+1;UHPD?6>r$^k8N0Z_ojNSd9uTlhx6(W#V%yHY$8Ozq32$?q@K?>P<&pdCh29ZvCb*f`Sh&gD>3B$9=TQ zYeS`s(8}-AG!U66Uw?mCRKObB=yCX7f8pB;y5E}o+JF}97Zn@IXX@1462yT80RoKO z;5PszKOduSL%M6MtB4N@lFVzidX*RAZ5v3P4Hd}YM{^)CME)F(cta=p7L?(|$s)!) zHapTLKrSrM$s-{ga|I}k;%k*>O2g(GP4hYb4g;y^=ZySWF3kETknxzA*p+@p*1HTw3@@NR z%JYs{S-ClS(+v3t`RyQ-ed^?z)Qi^LCna@L0bvZKnrVI6e&`^lFv?29 ztuHt1_G`BJPN^pzeakeZW$u6@O6G@aj)i-KTMl}gNWmEGa|d>PPZs;j8msbylMft= zvIEooU(@SHI3YUR9WMwq*6KHnZU}Sn8UAtu_hoyUywy7W`L=|I?UfdTrJAQ0gTh3k zJ5;J3%$@JrgG>hE_EIOK$sM)GdeuoQyyJ18)J*n43KisJ&VeF$V4pQXYPi{9DWMjPKhK2kIh=r|%*!C3`z6cX=i=~L2?0!+k)u{YV26*iNw^xcg zQIEyeZ4J27+77`u75VyiUQ!bSB~=HOfs~Yw%rju==a7(Kog2jW_;93L(!Nl`HCT0v z9ruOm?}eni|8GSmuz_N! z7R?9{Dp#Xq1$D^csJfI;0a-LIaymD6igx`16<2tw&;$Yb$_X{?n3&C{-G_eTrZZLN z>kxF{OKczd6Gx6eS98+*c?rfnM6{y@83-Dzm`h+)fi5g8h^cUW|@9Lp275|gJ zIbrAhL$f}6Gp5QOy9XYoN$o$w*=D$&@Z>&&iWZkn_dSH6;p9E5!8~2u2GEO=UO3{F z4#VL~IojX2fZSdeFUxhv4UU?7)KoRMu(z)1wy_=%9X2y}4iw=*k;@ApmT#V^p$yJP zIor*FjL>S3fVW&r0R-MCbio-t7I;a84P0fy#Vh34CL}r9+1JL#@~F3$hnefYB?!;; zQJC>loK`Q^;BFx*-7SDm7v(mEy*=NXv|6>I-Q(1@rDJ)5cde=A#HOY}O!A=h*kMQ4 zyyr)Da%OMly@zeRX8mU4DH=qs`NcZ*eqLNT?yRXxw=J)4RJYGP^IIUs4&Asj^9lx$0yGyl(t> zU8jid{qSk2nfNySUis@8r0PmjXobbJDwq zVN$tqWQFP6nuBqemx;OVg7UqGr?RX$TAS>0{bNHXUU}O4R-{W`MQx#49n0lDwFj9? zv_0G!E$VNqqA><*t4Ab-`s>V&RgKPBJD>4(GqdkVDsreE$%k!w`FyaRAa#^j+M9m; z@z`Qotr6c}i%Tm-bw$7LXRJ}`hE7ao>UoBqf*YfRpiUf59JM@m31kS$dUM%8NGel9 zQY;XsV#$|sIFAJcrGl2g|J|ENR?D1Wo4-}`fFD8Xry1D zL>{(^qn@bHD+=XnQ7T?4&i(cPpCzr&;aB&gJeP0LCOYa}Qm=p~fLH5y8J2Gm@ktPJ zWET6cacShd zgP&Y*`w_v>6q<9+)4ltbK&0hvQvKHGJV*KmVoAC0nS-Y4Ez^5eMmoC7j}NU|_Q`D6i)gOt$HzGk3@$l!E-Y0o2_v2@-2EfWz zQ;5=DTm;7Jr&~8O}k=uSQ>3b_%TY@*@i}Zfi12P zK{^9E`RcrlYqb|ZF)aQIM)%#7Y?|%W*2}WMlUO)d)SAb{8dE(COzu8mHJoBt!GClW z`E4b&1;ba;9WecqUW>c&D2~DF*_Gqo3z+%=7egJoquK2BJWL>$!m~)Af$l7Xh?q~DsIPrNY5lR zQj@;bI=P;x!sE;yLYFW}w)^0t;M!Q&eEjxApKpYG&UZh*Tre}wL3iQd0xbUvuVb#+ zX0yZcce|QAxASgxpYZd5Y5z{rdy{!Wp<-!Q(xh@g&Dfje#B*-V-B;v=8=f{e2Jc_^ zcFV;r-p=myVq3y1Bd2)L)mvl-X6?6X>h4?Goet#NPozd=wp(J-Di>U=xC{-;m@hH* z$L?9o9B45HSEW@jO0HwJlt&9q11G1~*JVPBt%FCJhy;}pg+>Q_Q0xqwlJrjnZ6o4e z${*x=zokdQAJnQ^q!w}kWuMoYw4n1l%y$kb;s@uM5Esui!qEHGXT#y_Dv1gj<_+u$i1i}&1{W_7(v7+a z;BKxvTr_KUVt7Mya78QEllG1GlW<^VPqi0gdOq^U)Uk$=m%cs(O2m%idll(<482az zuzg+Q%26VFC=CmrPnaM>*m)$yp^n^{xpiVJ{eQ#}vOh-FZuNc+BPMsgsjGO-HrGA9 zD*X6oZX_wrz)NBC?i1;gj8#wEz$90W9+iqB^4Q4D)!gis`;@TP{MvJKEh_Q38eJz- zMLb@68xAp2AgWGOzi}WzBX(ouBpb5RJ-WV)zK?7o%Iywddr-jY>v6l-GQC`Rl&T|H z1!u!pX3U-Ve&b+p*acM@an09)4bE13^LMB48P{qSPagwmHVyTj37jI3o#(HInP;xf#;n&>#7`yuV;*xkDH zJa`XF6q_>T!DjLKC=N9nG4W4_63risoga464dp6LzN9`lT|&zOKrQ@ePOsE?#R2<+ zElWF|h&jwe#+Bh+8DRdNULrPbh|VSL=3V+IMx_ zD`}1t;*j291?fq8_zpMrUm78X9rf$o-4u|edx9I9<0%il0oOkj0uzE0NnJLPF*xC+ zwwz-%kyV{qcAXz;r2&4|egJ;SS$-{Ebx4MX2@95WR$(acmtgQg%UuG->y>*xING_v z%H5Fh@!ivG-XVlQ|F<1K%q+Kjrkt`gZjlfIGuILdzM0?u2QOj4z|utczbnV_!c5^H zp*29(0_q&`r_P-tx$Xyri0*&+bMnAGu)@NY0YB(-zmr6GKYA{oJ;1_-os|N8`p@rp z{anMdB!2fw_lWwB8sFF*&R);DH%JCr2K>l~114Y8_)}+v1)2BYdqZVFRIL{%@r4lz z!fQI62@C6LiTgBflCDQ5c^o~e$v4wj*wh2wVa-u#uY-DEnxI|V{Wy`CnJobTm@WH; zwm+3`w>nQWs<&R?j`j?9wUlN54&4_)dRTGgh4- zaS;!?JMe>hMj1T+pbQ^7Y&gPD`@p0I^_m0=| z(MW~fIzk2BGxi|8G(RH0Y~kj(Dh?SQ%;|gYkDDbxo=ybvJRrl9%X!~y>rq?wR(s4! zKg!G6EzMXhZtv6mxT9Wf&5Y-=G@@Gq2UaU$2T#U!w>F|m7LKYWddQ74Y-9N|Am1UC zmE%EM?~H?NLtX#Mcr7s6ngQQR_ot4qZY{r-Eq55=8ZL8|KK`)jtk+<5DY~)r;v0q? z5zp-ow?*+^mOX3d6NdwqFS10ANzxkFsg1($pei|_R8+D!nj`U;?po~@A}Xz6`J?# z-CkpNUnP6RKuNUbH&>KEq&sB5_9&3SbI*&p1z0Vd^+%ts-PvxC14*09 z>P$1*oluT7eE1eO>;|b5O%V?6JI7l}6S)jXnqE26W2%?PD^wu#tTNse6=*21SGjDIq zICR@e2mYe^-;DIBdM4>JS6jB|>R0+da>E2P20b)p{>ux&Li*8Qotwr-0mzmYuSwwb ztBs0u3s1dtbQVWTF4arSSu>;fxQs$TPEy$L`TEuU`s}26d1;#iwk&j>R{g!2j7cw7 zMVX>NUzb8zhNmxJ3GAECP2B3Ns`<4|&;T=4prZYGgi zH%si2tqfP_%yVFfXgHSf`@t96Tm`P^guIYORg9?Xu5 zJ;wM2phr3WN&SxWfyFrWm!{IW7&qbGlVfk-eSU7apGD50nz4qq8mc25vl}UV*)B=E zm02S^Kq8+031)roJ2JpiM}joJrax zW4T6~<2!C%qZ$G@+VSr0y&&h7x~RoXfTWq)ucvU0ZlyD0ACM@TQBn7R$o0=F% zg?f5hCJi3_*S99eZ0eSHTf@#uD#N)6?D5Wlhz1a-mr#ym(0FT5C>-h-@h~OJp5S}h+ zeGw;JIDPJ1G8xb&Zz}Qv$K>mjExu};&#|07fIOFDt%j1os#ketWp~OwzUgcsPMOPh zHgUF^Psg(@i@#KE8z# zlO{qX;YK98s;lU66$4@72+cT;?`30csmDz&c3;GmGf3qRG=j4(Sq`pB_Ls^#57(O~ z?`eM|!|-L?h&3Y9L9EkjX=F8WHSC9n5fmLe$@_!SIRy+={=`Uq@|Dq;DP-d z7QzCywZ=P}R2~VkSJ#OOwkJ-NBAYK#Og#8rJ(u2J8*l)>MiJZKYV^yV4eSEap8{zaqcWV zHTh2w3~}UhncqRxb1s57e^}J#%-VSV4{PJhN)lmWf+|vHuq5JM{x+Lr&}Z3#s(-Na znbk)}a%Nz?z+zC5{cQ#^At2W0F!*n+Q&Ft2tUs?tavd@m$w849)MfI=WQ63$IXvvM zm?E8zqQ;p(IY)wj1v&bVvH^9a^l}3Zgt5Y2ZaQNwPEy(cqpZIY}Cb@k6ZRZUv5ScEsI3 zD|5TW{rEN^L~2iP4 zxdeRowl2bU02vKU`)WhiPjUqYpT}nZ_AVy?JB!zf?`KggY^hm}ZQfrFBbnio3Xv1j z8}u8T_1%a(J#>vKh3?i8R7QIiPy$5K{SJ##jD`a*b>H8I2u3K*;Gv=V4j-a`+iX6L z^g*B103jYlseHwO?31e`XgNjLZHIF(JFtn=+rK50PG3tw@O9~Z0Y9L`bl&1j=t(fStXJvHN*0P zrLR5ZF|+2`Q)pRS4&qdE5XJOMJL4*nRvEB|%h>oi9KZ`o61emGDbuGKBCQ`BU`ekw z_Sv&tk>Mj!SqfK`C5mv-E%g1`yILNbw}&^HiL}`suIba zhaU5T{Z8XJhENNFoyoR~%$_Yg4(*qA^@`RdaFId=J>1a3SCqjSm}&%|FM&OnJt!sN zI(rH&`zkV^`z8>Ja_W#^!NPycf3_PEu&2Sm%WC5r%WPW+9W5;IoxVrgYHb>%QQ&{I6bkL<1s%?ekmA zE@w~u!EH5XTtE&3+Aup$76*NmOgMNe@{DJuMMhv#8=!U+lHEuIDY}LazQ1$$j2U&5 z^yn)wL~?XwfL@rd3*sTM0e!P!T`K;)SrK+XF{Y1QWKF@wAZ~#?VIPo&>0W;+)_f7@ zx`Grly3I4CU<>W{l5AvLC<5&%cMb zi4$TQEbJSAX7O;&!4?4_=bDF*{(kxzRU}X0Qlq<2Sb>;L7`I)gfR{} z>eMT3#^yg0OJ{rzAl3i?vKFo3%us6G4Rn^yCpwR(LS=H(`VAhULeIX36VZ$Pq<)Cc z)75Q?0Q+ww2j{&;-F)N#m8i>8lAMPKW-l3B<4IQuqj~G61%Fx_5;Cagt-sRNSj=sU z6DqU26ULyztzD1;(nNu|YM_T3s?Iu?OZJ}6j5TVzii+XYzXOKMyhLx9rXo}!P68wd z+=zdY;1436ejiuty$?BHoCXAOo(r(qO<-XDR3>Oi3;|%zN}#nr8|wQ8nDdw})dUgd zlS9efQF(xx?vK8~KlAzubYo;d4n&KH*Ac*+z?}`T`8h$~hz0us7>AGA&1H}q2I@D` z!PGAe&<>dEv@)!jtx8|BkQL7fx|_?rj@|M=D`)gI4gbZ#VtywpP#y?Z;+2y7-stV! zGY;mq4aoxldPygQYxMo!yv{{{NKJpscYuq8I5h_wB8C6pl+>4K{>CGgRkU8|V}kni zW?H%XlAB*|n&g05o<2~ExB%L&sT2wLYnN|89n}_K+}V`UBme>Ll^;eudgVgwjL2;z zJHUq43~uxQRt-(?56NLC9={eg*?o5orawWe)+g^i36CzsqX_P>b$P zP~mJ52>i=$0H5UqiZYZ1HBX5x z2T2UN-?HH^{fi4d!-bxq$-$euyrh4H3v4)TG+F4v8AmIEH{ZmU@Yj+Bu=Wr36mjiTO^|0&S!eFIpS!z>gdssl81_1Ohu zgL75#dCm8`i#F=aymC=QACN2?VqY2R!glN0-CV9q1id4bM*?Edky|K&2zhN36B8py zN^d-RD8L4I(Gn^oQc=8 z5+(&7zJ~mN^}{4;h-l8_^^-h$+TN^&jZ264&+%eBwY{tekeRpQfho41<_t-)>!Dyj zcGu+5xZve)T(`uh1{5ufZ9>2kqjQw3R2r1oZ%f;ILcWFsMK!I-(gEm)@j76;9zEjF z*vMJMc!ei3q!|Lp1-!5aJGjqef}z+r8*o`+08e0C(LJ;P`eq`#~a4`v+qqfmCL zW<#Cq?;4A_fKtl{N!pp#7+wvzk|O)cW^g+j{(nD5plkWe@W0CduE%A+#0;)t&tFCm zIrH;1)92UDqbRqf)>f(gDP=LBs{6NQYp7{F_V#axIWArn%F&7(*R8wyxvR4yE}D-| zJA})~Nc(7iyiopKJ>sxE?y?*&7PAB?}i@w0B(^)Xu7_ zw(n!aBkB&SnCM>U%U;;2ab!YC`;T3?;QH50{e6*KLDCElZ@egn1#@8kM&Rm_t?$u#_1agh=x_%f-q zxxjX#!Zpo8$2WFaVU+2GQAQPMg!jRbw}!Q3aizsB^}f!5vG9^9HkzFMfw zl+yc~?Zo zzMNm2>(xOM#j5WV*Z3GrK*h{B=ze*-0OP(a(q0(MR~MeAU3HI6dbGrxSg6?9el!7L zNJSOVk4pl7Xa|NhU;@X_TsIFO#oi^$f0gkDlzT9TPYQI(UaU+N`oi&(j}~P_l6)3{ z+$Y(N1>K<8oWb0v&4f$0oO7o0edNILR%BKGaY9*_DiwUD-QbSGh07$&Go8`Obi6<3 z0(kW;K4&H8uB!+?+zAJ7<6b^sJA8YFuU_!D3u2*Ah)SA?{?6dPiP)56FZb#2C&n;W z>^9XgTcd8s9z)~9dO$zv%QDKqyK8X$lfT6A0BpQw+VMNDI|SQ9uEaCOsfklqvxPqzNh@y%#}m>8Nz1 zBTA9p6hz9|3w_RW-|utI@BR3GdHv*1bM4N~&dkot&i=>tg7^OReYtBJOa^tkcLiOX z+cr6U4>m+Mdk0?i<&VFglEPd9R@M~23D|oo5?)S6{N{rc-~(%d1cLoO@gzJC7w7l5 z!TFM^XCruhS|tU_7a}#N9|4B3qr!MVtD0A1ct>P@!?9DDO?igl8My{8a4byqT>uM0 z#pmWgw^hqrO5YbqfwWGukq^uLHKlTJqR9t5|ioKRfLCRa6Gp{_MNT^mm@r8&JqXL!F-C;^Q z6mP8o;skz0*nxrjFlP&UH@xcFp-^~T z=BwIjZj3Eg;H(7r1a+3?EK$*;X^P;vBilGQVE;p_#gXqA@w{FQnhlTYQ}g{x9?ITLmYfS7E%5{IDJ{TT$?BF9+yOWg6^~$1qQRWaGPCUKuNQZxJdpWlLuPseX}Inm z@`1uPXVb#?(M8*VcgD@9zkVJwjtK%K+R_=eg~ABZMIBpmka1!Sutt#1Wx5S{S5=kx z9SexZJ8Jm@01^j^|G_FUqVQ|(1^}RFSQdp}*F?cJ<@7d@aP*xJB5eRu6hFf7g2Mi; zG(-we*?)`bZ&JWF06#N|;)bZ69y^EYdQqcHg#8OPZUV(?i4s^5ea1e$VgYV zS}n}3aPYuIV~JmG!z=@CI^n#7r{F@;erw|);IKztg5*Dd2qi!WR9in?0i=RUAXkx( z#b)9u5bB%3;s^>ds=-*MpAZ&9ftr_U%y0a?6>M5s$Zl0QxPvSO4s`TPk4iKS1R@6b8Z z0&&Zuxcgn~jA3O*BBJQ|z*8)w+0?QWQ>4XzD+4b+Z?X(qJn1JU3>2@(aV3HPw_aSaL5F*r1?-1;3Fyd@iy8ySS<2`JQq3 z!M>fBV}0Z$weo55TVP8-HVb5T$>1kI@~KBk8u?y4z!u&y|3YcS<6C8MW!(46Bwba1 z6E44$tlQed2m3EJ)09uL1IC(6y>9fhY73);xTud>t+giTmd`&rdpDz}T||>)OzHw| z__C>O3u4#8)Z!)^ykmhD++zXvunY3NZ&1lgsndt?7u_@J@75Ly<2~LXxG?cEZFlu! z{v}Y(BM3wVxS7B0COgAwvUL#;lV+Y$kii9?bNg}lXyY|Cc|sHaV>23L{@3l#4$#P} z&{M#P?6X~r5BQ6$8^OzFw&Q1hzyf&F1iXNFNGUdo!aIiTvK35}^H^?qFk2g1EBL(> zHa~a`7Ju$c7!9q6pi)=l$&mk;vw^A4KYwQA{J~0NN6;JE#ZdROV`OxF zmEtb(C^+T^Ks?RwOad0e@1c%rdYPm+3Y~Q6VZ`>}$js<&VUTB^R1GNi(iHpyuyVD(XI2-&)7tW? zb})h=NDlh8z|c?X!D}=rU^u1pb;4pV5L~ z)OI=>0}w4v>(+tVokFBtzdAJ(Sl68OHZ|*Tb^*LF)zbOgdwJH^%x`z z+lfRY8V*kGroIb2>7|X90Px@=qpN;c9v&0H%X|r(Xj@{k-Jt{LoJ`&-TmdYIq?+Sl z!bDBlr0!BT5e53!X zL8ygWt}w^F)<_s!^StinVQEtc9+|P!-a;Y7rRS1UUvxkJoORxE%EUFW@1;r2NNXOXqn?Ppw0e0sw zH?Bz43Vdf7x1c3Ocujb;(lju6%Lhw|xnAl|8maxOoc#E**T%ltSz_&$>Z zbiL~ub8sNe+;9s`3dxm_*x~}3gv;(%+5EJ`dd#o+)zNh<441^6j2pobl2R_dJ5*L` zB$(+mlEyp`r+{!}Ps2R4n!GSePpO?g1`2LUSwHhGc*{%5HNQD-z#KDJK0Q4;#(EOo z0q%1SrJVN9b%?zgv_PSqJ?3*aWNY)Mbfh$n$SN#^>hdxyS?qY%Ol$8Ty1BrXJxA0% zrgdx=3SoCJXZf~%&+%Hl8{jo{zHN_yk43kLJzc_ezmec+8wbxF&FP*=6ka5tSqxH3 z0yQ`d^dy3%mQY>lL90zm5EAI|3f=;SAv>E;dD~=sLd$~d!z%#I&E_HGU+*xkUX=)n z;O?JjWwA47S0?XsQ;j9aNfRE|J&jFw4rX{c!RzC7+V0<(cnH9xu!n8CZH`A_ zanwJ1Dj6nh*VKesC;`52<3SA>aNR}#*9+B)PQPi=?3#(a(y1J70~o$`=^3O zp}o-Q25_M%-*$H9@FYGD<;W4iH&jq!@`SbSdNtKuhInIa{&rHzU9tN`)E6^Z9rzPP zwg}%IOL9|l$b%?T3NsmReaCyt;B0`+n+N5LLYo#MLjVAND=;_rblN`7@F3(Qf-N-zVu z>xQU{rnOjn{eTaYKry#4dVTYfa_EN(rG#Eve!HGo(YKc~H+Tj?@{^W7OfF9f0PbI2 zv%1rUaHZ$!UT&EB*8G3LM(laWSL7L7=W%IB1X&xYyp;wNgEeT zgOwE@p?CG$RZ$SN@6eIk?V*5oh$g+e089c7#{kx+j_!WMjM(KX!oDD(zNF&E7N*@h zWk4kjTh3n>@c{5k9A5`XA_7*)d*7&qQkre7b}?P^2Xj|w7QXY^UN!18;O_|VuzUhX76S(UjP4sDZ8 zCY&gIjt%YFVP@VE1ilYoS>8reb7%!EMm)d(tiAUo4sO@d3N!6kzWee3?F1Qgc#Tj4!m%D+#xO++ z7>ANt=u2ozHslS)-V>hCWZ3m;plQ0!drzF)hcnOCWI@Bt9Uti>e~We5yK<(8-i}F z$(=vbMS*73BdY}1QhaJTHTi4lIP1S0;4hast(KM+ltxP2<%~JSuga0iZdBH+F>?wV z5s=(+%4)jx{cwq?ke@we!1VLmmLiZWpaY1h!I44NyU3AM3XQkuOMu7ZmRKS=W9D`o zZ!vLy+HKrSK!+CGyHB#gBzf-vti{VzG@?&e#|ScQLXrLNhkKS;I)aJlA~WJc zu0v`6*ZpcU_hdOz+<+imPYH&Izvmggx6*c~-w?-QkTT8`?qLk>U!3q`lg?vtCH-^LJgR4$pBNaE{MX9s9Id$IP`R za%a|cuIjM&c!(b{_9xfdg*x)ZK}0WkKqLLC1|gVpLyxyeVWl~@a#L92CC4vQq{nTv z$xoGL31mFpeaf{YLa=Vq64bwKIkoLha2Pir9Q#T6_p}!PT4mWoE(^%xxuOLwD#Qt{ zD}t~><*-!n46Hr#%AJ7^1=iz_;=}IksnpOZ&rnhoZp_%nSc-u!rV$Fk$LN|B_=bb0 zCGJzPD1xW@N|C$DPbztf!7;&3ci^ObZ~5()zUg&$A(2R<%0%mdvO!D9+;I+bL{QoJt!tmSh*FcMQV56pz3dvKNc2$5$uz zaXI-Nybs^XxH9{*^tjlXc#`zS`HOxW_I$f5>av!nO-8|szIF!uh4U@9`9D~@6V{|Y zdjzkg1+P%|a)pM;)4?SM*&RCphT-qi9vuxN1)k(Sp<}{AKc!^pEKxJq!EuIP2Rz0H zP=meTb1eZZv8{9AFwP^fjM4QV#{?Xf!hDVDIaUNK#h=zt%{A z%~HOBaFOL06Z3+4L#m$ED@oTLa6gadS`p+EeRQ3HnzJ!zu*bH}PER3S*x# zVMtno^-1Er+TukdmVc`KgzT`3VR>M=59tXTEW)UsK z#4Uff6Wme@8m)PLdVKn;`@1V;v@OFx<7A8Sf2uMX*DSqr&Z`*x^}a<-0+_zG^f1U$ z;Ls4H+T-9fU4?rqf4eLWpx^0o$}5V|IKRnJ1Daki(rd;j8J-Q`%{#EQpz;AOiWHls z8*h!VY3^*`{xqo2h;3oG2o+M`D1co)jl4wPr5nT}m;~V|Hflh4inI4xn7I2Dd19FS zDm5kCo>`5@Cee+9Bd05Zm>etb5x`V)L}>VcEhDS3MDW4ERv4%G^4q(m6`MP+6wVd{ zZe|?v5=Rdfr%dR6#gA}h;&K;ti$XjK?gjpKQzg}U_5!%~0GajGEc0PUN#hKR=8Oo| zeB(v}%`l*CPP=z3l;iV5%F(R+E2P=t^Bl^Gw=CrT_3xL)?O; z8^3Bk<4{2Mt>9{7(;5k!SrYKUG6bRB1Qf{9a40KgnWCa~K*$o?>WlCBjMClCUJ8%A z?z0E?}d?}Tx^dsdWL{#K#n5B~Q%HKL1sim^xCaSz#%f{O9ZDr?po!UC-SAatRE{CB(Su+Hw*W9hvss<#tnud9s zm#}Ya_zhpz)!*~P)FxE@`nHV*dPhxl<0LYlFvC$w0V2=2!Z!f6pOCnel6gm?y?L--qu^3g*_zt#jag^1B#}i`~PUM*5jfHHAzT0DEBb5;LW_YbmzVTFbA*Hc-3$%i^!u5|U9`Vr zG#YGepelvLEprwpRc6ie3&`vDe8SDnJI%6YPDIiv5ZYeg{Xx!Z`eU7?AvoAiKpqO3 z3MqdA)~kfn%x@E*1U`zs9qslf!j_lkGygB?Wn|p0)+XcnCAbOj!8%oyaM3*q3wG|W z&VLQMxqIl9`^qgE zDv@E{b>7A}-IzJ5d!Hx(`N^MKkV}3-gc2?8R||N{wbJzlf0Yy5PZ}Rocz1qt6ufkb z(>l_k&#>fN;8y>?i`fv5K_U&uxYBE<7?hZV|N6eOj>{gr6$4=rmum}sX-TQ3fN@Uo zfLmSZgS6E#z1^D`nS!pABy7Mmd4#p`u;)bHa0%nd;d{`%wd@a8=(c+noGd<1CyCm0 zfXE>3e-jx?`?)r9w2DRuK=X9T7GkD_T|*OZ2TTu+8FhB&4NFXWh`s=qcI4+a5pv6hsS(+gvf8e~4&+uw z`cRS{g51`aFG%2J2JUqOOzcbRVa7n4ZK0M(e?3D#3j*WuJp9nh5+e!4Kqzn)iIN*u zUAS5gD60`M!x;BGh~xBAuk8c_je~)ve>KQ3JVhf6-_EUKAj9wuU=;>auQ9^HP-jZR z>ynZrm5#!5P^(u#w;4%~>&m8s#w^t=qG`tX;8}4qg7Posw_UYmn6Z^<4^IH(**-zG z+(H#FE$8wOB*#!Fuuhn|D=vqM-ttCfjO3Y(kQefe$Ke(<`D&`$`W(EELDGQ5(S4m= zqzLnV=#`UY86%8VHlB=(ep_8hUcWH^yu~0X-!q-F8gTyWx{?#*a{`OltUxIxk7I$O zxN5B58nd@1pRuq#3Vh6m#AC5Kc$i2~Rr$|4qqDhY>Q>(a`fd%nKxW>}v zu7s@LHD^(`cv-X1VKLKR+4HiZ?;|t!C#aMw;W%_c5s8X$3lUfLkOwnpR02^QW4*ol zxy<>C%9ArUbw)jv?Tg$N453mk&=>SV#+I8`+q1fF?~WFO6@KDQPqq%@@v>few{CC# z{I1TAl0|KyG5d~_)U4-#IcBy8u`LqO)kEL)I_{0suqq()>YanwO^kY@x{xqoT- z8f6zs+l)UR@sO1R8ufu12zS}9VxptB|)DAi05(93C^@6Vg=%1~TWrK@xp;p?s~0`P@`pa*Om z0Jv%9RILO3eX(X$SB|kup}(|58FhPT`|80oVvy)bA0iV|qUgK!tt*>6OJ$w~x0bH2 zWCR)MUPBheYTc9?WFQ3h-g19H1}Bljkrf6=Ms#`2m?yk&h%pJ8sVfq|e`{@=FdqV-&>SnFnWK{>$bntXk!=;B0S{6cBdCGsI8 zieU5QgE>784eqKaQ^}A+nTpWnm=Vi?3J8dZkN`_N0Qarl&#@VDt2~T`9ie#l@hxcB zdUf|gp3Ce9^#;GabqYKUGd3(OMwvL>g}X`(M@n&}^IErVVP7J}+G>lno-UZ=iSB%| zI&KlX-6LVj^@FeDA|t`(<;Qw@T;nZR93@BL=T9kd@DoS!=lr=+WZZH$=lg~Hq<&3z zBz%cFs`9buft-odok`pfsQd(IZHl>hdAvKV*I(Zx&=aJkV~+&-ZsnkN-pjnaIPf&@-AX6a!2%E_=y&fI;5NX?6kbytr79Zc zpcBCX14>>NOsh*cUcc|N@R8&YF&@KI-#ofg`g2Qm-fVC{l|1WLw(qAp(X1B^6+X@$ zzMKerfk)U53Ws}zQ{`~&gI}pKfQhV6V6oO;>!{%kyj!<7dnUEo_zVxNV7s z4-Bnu4~~Vu0$l2*cz`2=agz72yZUSrF4vIz&?sbvRwR%}%?Ejtpyiwmz#EjXZbc21 z*~KlZl)o6SeC^rKe*Qj@5w8p=%BeHVajn% zG~WeO^hT(IwIFDzPz1I>n;|U(QbFSSWnaA=(vxU%6fXTJj8ie>$J4m@8|b?k24yOn z9u}`?wxvurW$VdchF*$>(AVqKD1tbv92F{PZjsNocStf<1s6eF;ZhXo+$mc5&>pPc zt@xQLKUTYZr8Hjaex3^)f$xOFk!XS%k*ouS?YM!#p<=1L=aYLX`+~4wi4Mr1;}T~t z4NlA1Cwh+?EPLYVndA*xcH4u_nsRV6Hwz;0*Ef^QF;K*Yz_X>hXj;J$T+OzRM6@Oz zu9rx`-_U>o?i7^DE_1ynnH!IJa5wghUCLD@gN2ey%}I^2YiMG@oQa*`oas(FWdRt} zsv-JWbbw3hnD{ZfipBQ%*C%15<$c|YKn1al@-3fdu9#Q`oJM}m)|-5Dl*Nr%rytnG zI-DBUky)AAFRVSchvNHvCE?hyy(}*T>JFZGKk~2l8D|kFh(jnfVbs;u#%hu8RXir> zXG0nY#_c8qK7fvF1d!wcXLs!!DvsM>Ud@$+h>DHe+zn+rymh_siF=i@~mxX$yqottRLqyMX! zq^bB<{Kz%YZ@s2Bd>l&j4gB62He8O+;Vz**)cXXL7H^ekk)DY+#sEB%Uftj-m2kFXA9ZJnABs z>HJ`Ol{mNWsqkYX$%Uo;*Kq+NN<=2xFGJ_y$V3)&ql8l@$8UMSp=Dv$6hlro4zCV> zwArH(`YO~r+xp?vcZvD2N+-XdqDBvbwy#eht$?rEiIXS4co0$>R$uQfvlOaVIw}vH z)9Oz3O%+xazEh$P$MTOT=mKH9PX;5=nOoESmLhc56JjAy!A7rNMui<4xBRgy>~ezM zX{Y0#Up5nvYOGm_^UeNPU;pfzS6tn%aXSIQdBMVl_1FumN&B?=F!<{ahA`9F*z*Wv z5i1>#i^ga$JQtp%{98(3{0N_;Y)>fh2?rzSDD9WgdUO-~6Z3bS-oOkuqEP40oom~@ znK!);OC#AtK)zbgf0j z(`yW}YK+PsKIK)vMeWRlwf4P>P@NJp1 zpPNYh3s??z$79)Cf7H=FAX z5u{CM@87?5pA~*KR3eo3%MpRE`K9_D2vQNS9HGgIc7GTd4J;P0jOO#s>SZ{!(s(4_ zY6MjgoRh2~3RFoTq~;P!ZEPUR0fue0_+v#qY5o^0dg70jd&Z0uu%0wa@G@f%M)ln0 z4W!z&gy%O>5|_Zbq+!QoW^Nm^{Zac8mc!rP^cYuoB3xTiv!X)GuJzXk(;uU`)8$+x z{W#>0480^jWL5U51SJ}9R)iDa4T)qzt1!YZuw^>HCQF+df(LvMwE>I2e>;>~`TyDa zA8b{kKt z>}0<*xuL07x!A8kwE6f%_K%4E*{5+NO5$Fx#_ajGx1|Re_^C0CcQcP3hy2PvmKXeO zjsb1m)i0{uFJbzm#-`8^j`bcGrTQZhzr_tS$%uZF49KCs(NRs3(nY~Yj1K9@Kf?H% zC|m*=*Zl%NyIj&xPoJ=NIRjy+bVlcI^8hn12`C;f>#2SuX8NK0U3eg}D8=@O^1n%e z{#JuPvblcO)GO~KXy_ysM!69hqZUL{(aV6rk(#g~C ze~{e8i)&5KyY?j4rsAwyhs7%d{zrZ_bp%;DZKj!WI*hC%{QMQfKS)m3Bu2{{ zHJml*xNSF2syCr3S~Pay|Jyv^067_O+IljWC=jr+EKx48V9$hZ>=~Ax|D%q7XA+zq z-AMhc`!?g>|MB Date: Fri, 25 Apr 2025 15:38:48 -0400 Subject: [PATCH 03/15] updated values.yaml --- .../7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml index b7f6a976b..0cc60662b 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml @@ -404,7 +404,7 @@ login: # -- (list) # The `/root/.ssh/authorized_keys` file to write, represented as a list. rootSshAuthorizedKeys: - - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICe9Hm9zk+q0I9rTQWtAdTS1uIuIRtN+6drJYt0k6JWN natharno@amazon.com" + - ".dkr.ecr..amazonaws.com/dlc-slurmd" # # -- (string) # Set the image tag to use. From c863a8b9be5c7c1723b730461bef4f1764292bbb Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 30 Apr 2025 15:38:01 -0400 Subject: [PATCH 04/15] README.md improvements --- .../7.sagemaker-hyperpod-eks/slinky-slurm/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index d409dd6cd..742212901 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -53,7 +53,7 @@ export AWS_ACCOUNT_ID= export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster -export ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_ID:role/Administrator +export ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_ID:role/ export PLCY_ARN=arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy @@ -100,7 +100,9 @@ kubectl get storageclass fsx-sc -oyaml ### Create an FSx for OpenZFS Storage Class: -Install the[OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/blob/main/docs/install.md). Set up permissions using IAM roles for service accounts, and taint the nodes as recommended: +Install the [OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/blob/main/docs/install.md). + +Set up permissions using IAM roles for service accounts, and taint the nodes as recommended: ``` @@ -147,7 +149,7 @@ kubectl get sc openzfs-sc -oyaml ### Install the AWS Load Balancer Controller: -Following [these instructions](https://docs.aws.amazon.com/eks/latest/userguide/lbc-helm.html): +Following the instructions below, which are a consolidation of the full [Install with Helm](https://docs.aws.amazon.com/eks/latest/userguide/lbc-helm.html) instructions found in the Amazon EKS documentation: ``` export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster @@ -194,7 +196,7 @@ kubectl get sa aws-load-balancer-controller -n kube-system -oyaml ### Instill Slinky Prerequisites (Cert Manager and Prometheus): -Follow the [QuickStart Guide](http://curl%20-l%20https//raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.1.0/helm/slurm-operator/values.yaml%20/%20%20%20-o%20values-operator.yaml%20helm%20install%20slurm-operator%20oci://ghcr.io/slinkyproject/charts/slurm-operator%20/%20%20%20--values=values-operator.yaml%20--version=0.1.0%20--namespace=slinky%20--create-namespace) to install Cert Manager and Prometheus as [Pre-Requisites](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites). +Follow the steps included in the [Slinky QuickStart Guide | Pre-Requisites](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites) section to install Cert Manager and Prometheus. Verify Pre-Requisites Instillation: From a3b95fc980fabb2a7af08e5702018ecb0f75cccf Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 7 May 2025 08:35:16 -0400 Subject: [PATCH 05/15] including p5 example artifacts --- .../slinky-slurm/Docker-Build-README.md | 2 +- .../slinky-slurm/README.md | 196 ++-- .../slinky-slurm/dlc-slurmd.Dockerfile | 3 +- .../slinky-slurm/download_c4.py | 6 - .../g5/g5-llama2_7b-training.sbatch | 123 +++ .../{values.yaml => g5/g5-values.yaml} | 250 ++--- .../slinky-slurm/llama2_7b-training.sbatch | 122 --- .../p5/p5-llama2_7b-training.sbatch | 100 ++ .../slinky-slurm/p5/p5-values.yaml | 954 ++++++++++++++++++ 9 files changed, 1410 insertions(+), 346 deletions(-) delete mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-llama2_7b-training.sbatch rename 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/{values.yaml => g5/g5-values.yaml} (88%) delete mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-llama2_7b-training.sbatch create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md index b16d7a918..a0b54dec3 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md @@ -1,6 +1,6 @@ # Docker Build for the Slurmd Deep Learning Container -This build includes Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 +This build includes Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EFA Installer 1.38.0 (bundled with OFI NCCL plugin) Clone the AWSome Distributed Training repo: ``` diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 742212901..591fdde5e 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -29,20 +29,23 @@ The login and compute node pods also have FSx for Lustre and FSx for OpenZFS sha ### Release Notes -The following was tested on 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) for hosting the compute NodeSet pods. For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the [values.yaml](./values.yaml) file to modify how they are spread across your nodes. +The following was tested in two infrastructure scenarios for hosting the compute NodeSet pods: +1. On 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) +2. On 2 `p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 + +For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. Testing used [Slurm Operator v0.2.1](https://github.com/slinkyproject/slurm-operator/pkgs/container/slurm-operator) (pulled as OCI artifacts from the Slinky container registry) and [Slurm Cluster v0.3.0](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) (packaged and deployed locally using the main branch of the Slinky git repository) in order to include the NoteSet volume mount and Login Pod features. These features are expected to be included in the official Slurm Cluster v0.3.0 release when it becomes available, along with a new version of the Slurm Operator with corresponding validating webhooks. Note that the [Slinky Project](https://github.com/SlinkyProject) is under active development and could introduce breaking changes that may require modified deployment and configuration steps. -Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image. See the [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. +Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EFA Installer 1.38.0 (bundled with OFI NCCL plugin) pre-installed in the container image. See the [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. * * * - ### Set Up the HyperPod Cluster: -Follow the [Prerequisites](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup)and [Cluster Configuration](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster) steps of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). +Follow the [Prerequisites](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup) and [Cluster Configuration](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster) steps of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. @@ -270,25 +273,6 @@ export GEN_INSTANCE_TYPE=ml.m5.2xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE ``` - -Verify the name pod labels applied to each component: -``` -kubectl get pods -n slurm -L app.kubernetes.io/name - -# NAME READY STATUS RESTARTS AGE NAME -# slurm-accounting-0 1/1 Running 0 12m slurmdbd -# slurm-compute-hp-node-0 2/2 Running 0 12m slurmd -# slurm-compute-hp-node-1 2/2 Running 0 12m slurmd -# slurm-compute-hp-node-2 2/2 Running 0 12m slurmd -# slurm-compute-hp-node-3 2/2 Running 0 12m slurmd -# slurm-controller-0 2/2 Running 0 12m slurmctld -# slurm-exporter-86448948f4-rqtg8 1/1 Running 0 12m slurm-exporter -# slurm-login-78b8fc9cd-rz8qj 1/1 Running 0 12m login -# slurm-mariadb-0 1/1 Running 0 12m mariadb -# slurm-restapi-55d998b698-gdzc6 1/1 Running 0 12m slurmrestd -# slurm-token-create-b297g 0/3 Completed 0 12m token -``` - For each non-compute component, we apply both a Node Affinity and a Pod Anti-affinity in [values.yaml](./values.yaml) to ensure they are hosted only on the 2 `m5.2xlarge` instances while also being evenly spread between the hosts. ``` @@ -322,15 +306,20 @@ You can modify this common affinity setting, or apply unique affinity settings f Verify the existence of the instance type label for compute node selector: ``` +# for g5 instances ACCEL_INSTANCE_TYPE=ml.g5.8xlarge + +# for p5 instances +ACCEL_INSTANCE_TYPE=ml.p5.48xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$ACCEL_INSTANCE_TYPE ``` -The instance type label is used as a node selector to ensure the compute pods only run on the `ml.g5.8xlarge` GPU accelerated instances: +The instance type label is used as a node selector to ensure the compute pods only run on either the `ml.g5.8xlarge` or `ml.p5.48xlarge` GPU accelerated instances: ``` +# for g5 instances compute: ... nodeSets: @@ -342,6 +331,19 @@ compute: kubernetes.io/os: linux node.kubernetes.io/instance-type: ml.g5.8xlarge ... + +# for p5 instances +compute: +... + nodeSets: + - name: hp-node + ... + replicas: 4 + ... + nodeSelector: + kubernetes.io/os: linux + node.kubernetes.io/instance-type: ml.p5.48xlarge +... ``` --- @@ -399,37 +401,51 @@ kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ ``` -Add the FSx for Lustre and OpenZFS PVCs to the list of `volumes` for both the login service and and compute nodes in [values.yaml](./values.yaml): +FSx for Lustre and OpenZFS PVCs are added to the list of `extraVolumeMounts` and `extraVolumes` for both the login service and compute nodes: ``` login: ... - volumes: + extraVolumeMounts: - name: fsx-lustre - mountPath: /fsx - persistentVolumeClaim: - claimName: fsx-claim + mountPath: /fsx - name: fsx-openzfs - mountPath: /home - persistentVolumeClaim: - claimName: openzfs-claim + mountPath: /home ... + extraVolumes: + - name: fsx-lustre + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + persistentVolumeClaim: + claimName: openzfs-claim compute: nodesets: - name: hp-node ... - volumes: + extraVolumeMounts: - name: fsx-lustre mountPath: /fsx + - name: fsx-openzfs + mountPath: /home + - name: shmem + mountPath: /dev/shm + ... + extraVolumes: + - name: fsx-lustre persistentVolumeClaim: claimName: fsx-claim - name: fsx-openzfs - mountPath: /home - persistentVolumeClaim: + persistentVolumeClaim: claimName: openzfs-claim - ... + - name: shmem + hostPath: + path: /dev/shm ``` + +Note that for the compute nodes we've also added `/dev/shm` to provide access to the EC2 host's shared memory segment. This shared memory is used to for inter-process communication. + --- #### Configure Compute Node Resources: @@ -437,21 +453,34 @@ compute: Note: limits are required, otherwise the compute nodes will not deploy. ``` +# for g5 instances compute: nodesets: - name: hp-node ... resources: limit: - cpu: "32" - memory: "128Gi" nvidia.com/gpu: "1" requests: - cpu: "1" - memory: "1Gi" nvidia.com/gpu: "1" ... + +# for p5 instances +compute: + nodesets: + - name: hp-node + ... + resources: + limits: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + requests: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + ... ``` +Note that for p5 capacity, we are allocating half the available GPU and EFA network interfaces to each pod so that two pods can run on one instances. This can be adjusted to accomodate other pod topologies. + --- #### Build and Set the Compute Node Container Image: @@ -470,14 +499,14 @@ compute: # # -- (string) # Set the image repository to use. - repository: ".dkr.ecr.us-west-2.amazonaws.com/dlc-slurmd" + repository: ".dkr.ecr..amazonaws.com/dlc-slurmd" # # -- (string) # Set the image tag to use. tag: "24.11.4-ubuntu24.04" ... ``` -The Slurm DLC has Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-installed in the container image, but you can modify the [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) for further customization. +The Slurm DLC has Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EFA Installer 1.38.0 (bundled with OFI NCCL plugin) pre-installed in the container image, but you can modify the [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) for further customization. --- @@ -485,7 +514,7 @@ The Slurm DLC has Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 pre-in Access to the login service can be configured through several authentication and networking mechanisms. The login service can be exposed either as a `LoadBalancer` (default) or `NodePort` type service, with the external port configurable via `servicePort` (default 22) or `serviceNodePort` (default 32222) respectively. Authentication can be integrated with LDAP through SSSD configuration, where users and groups can be managed via the `sssdConf` settings that define LDAP URIs, search bases, and domain configurations. SSH access can be customized through both `sshdConfig` and `rootSshAuthorizedKeys` parameters, allowing for specific SSH daemon configurations and authorized key management. Additionally, the name service switch configuration (`nsswitchConf`) can be customized to control how various databases like passwd, group, and hosts are resolved, with support for multiple sources including files, SSS, and database lookups. -For simplicity of demonstration, we've disabled SSSD by setting everything in `nsswitchConf` to `files` so the system to only uses local files for authentication and does not to try SSSD or other sources. This is simpler and more reliable when you just want to use SSH key authentication for root access, as the SSH keys are stored in local files (/root/.ssh/authorized_keys). +For simplicity of demonstration, we'll use SSH key authentication for root access. Generate an SSH key for root authorization: @@ -521,20 +550,33 @@ Assuming you are still sitting in the `slinky-slurm` directory of the AWSome Dis cp -r ~/slurm-operator/helm/slurm . ``` -Locally package and deploy the Slurm cluster Helm chart v0.3.0: +Locally package the Slurm cluster Helm chart v0.3.0: ``` helm dependency update slurm helm package slurm +``` +Option 1: Deploy the Slurm cluster on g5 instances: +``` +# Dry run +helm install --dry-run slurm slurm-0.3.0.tgz \ +-f g5/g5-values.yaml \ +-n slurm +helm install slurm slurm-0.3.0.tgz \ +-f g5/g5-values.yaml \ +-n slurm +``` +Option 2: Deploy the Slurm cluster on p5 instances: +``` # Dry run helm install --dry-run slurm slurm-0.3.0.tgz \ --f values.yaml \ +-f p5/p5-values.yaml \ -n slurm helm install slurm slurm-0.3.0.tgz \ --f values.yaml \ +-f p5/p5-values.yaml \ -n slurm ``` @@ -593,7 +635,10 @@ sinfo PARTITION AVAIL TIMELIMIT NODES STATE NODELIST hp-node up infinite 4 idle hp-node-[0-3] all* up infinite 4 idle hp-node-[0-3] + ``` +Note that in both scenarios (using 4 `ml.g5.8xlarge` instances or 2 `ml.p5.48xlarge` instances) we should see the same number of slurm compute nodes. When running on 4 `ml.g5.8xlarge` instances, each slurm compute node is mapped to 1 available A10G GPU, whereas when running on 2 `ml.p5.48xlarge` instances, each slurm compute node is mapped to 4 available H100 GPUs and 16 EFA network interfaces. + --- Verify FSx for Lustre and OpenZFS filesystem mounts on the login pod: @@ -670,9 +715,28 @@ Confirm NCCL headers are installed worker node pods: find /usr/local/lib/ -name "nccl.h" 2>/dev/null # /usr/local/lib/python3.12/site-packages/torch/include/torch/csrc/cuda/nccl.h - -exit ``` +For p5 capacity, check EFA availability: +``` +ls /sys/class/infiniband/ +fi_info -p efa +``` +Check that the EFA libraries are properly mounted +``` +ls /opt/amazon/efa/lib +ls /opt/amazon/ofi-nccl/lib/x86_64-linux-gnu +``` +Verify EFA device allocation: +``` +ls -l /dev/infiniband/ +``` +Verify intra-node GPU topology: +``` +nvidia-smi topo -m +``` +The GPU topology should show all GPUs are connected via NVLink (NV18 indicates 18 NVLink connections). +The GPUs are split across two NUMA nodes (0-3 on NUMA 0, 4-7 on NUMA 1). + --- ### FSDP Test @@ -699,30 +763,27 @@ cd awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm mkdir -p checkpoints ``` --- - -Download the c4 dataset to avoid throttling errors from HuggingFace: - +Copy the modified sbatch file: ``` -mkdir -p /fsx/datasets/c4 - export SLINKY_PATH=/fsx/awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm -apt install -y python3.12-venv -python3 -m venv env -source env/bin/activate -pip install --upgrade pip -pip install datasets +# for g5 instances +cp ${SLINKY_PATH}/g5/g5-llama2_7b-training.sbatch ./llama2_7b-training.sbatch -python3 ${SLINKY_PATH}/download_c4.py +# for p5 instances +cp ${SLINKY_PATH}/p5/p5-llama2_7b-training.sbatch ./llama2_7b-training.sbatch +``` +--- +Add your Hugging Face token to stream the [allenai/c4](https://huggingface.co/datasets/allenai/c4) dataset without throttling: +``` +NEW_TOKEN="your_new_token_here" +sed -i "s/export HF_TOKEN=.*$/export HF_TOKEN=$NEW_TOKEN/" llama2_7b-training.sbatch -deactivate ``` --- -Copy the modified sbatch file and kick-off training: +Kick-off the training job: ``` -cp ${SLINKY_PATH}/llama2_7b-training.sbatch . - sbatch llama2_7b-training.sbatch ``` --- @@ -786,8 +847,6 @@ rm -rf checkpoints/* rm -rf logs/* -kubectl delete pvc fsx-lustre-pvc -n slurm - helm uninstall slurm -n slurm helm uninstall slurm-operator -n slinky @@ -800,6 +859,8 @@ kubectl delete pvc openzfs-claim helm uninstall aws-fsx-csi-driver -n kube-system helm uninstall aws-fsx-openzfs-csi-driver -n kube-system +helm uninstall aws-load-balancer-controller -n kube-system + eksctl delete iamserviceaccount \ --name fsx-csi-controller-sa \ --namespace kube-system \ @@ -810,4 +871,9 @@ eksctl delete iamserviceaccount \ --namespace kube-system \ --cluster $EKS_CLUSTER_NAME +eksctl delete iamserviceaccount \ + --name aws-load-balancer-controller \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME + ``` diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile index 5d2015356..d3181bf62 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile @@ -14,7 +14,7 @@ ENV CUDA_HOME="/usr/local/cuda" \ EFA_PATH="/opt/amazon/efa" \ OPEN_MPI_PATH="/opt/amazon/openmpi" -ENV LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${OPEN_MPI_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu" \ +ENV LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${OPEN_MPI_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu:/opt/nccl/build/lib:/opt/amazon/ofi-nccl/lib/x86_64-linux-gnu:/usr/local/nvidia/lib" \ PATH="${EFA_PATH}/bin:${OPEN_MPI_PATH}/bin:${CUDA_HOME}/bin:${PATH}" \ NCCL_DEBUG=INFO \ NCCL_SOCKET_IFNAME=^docker0 \ @@ -55,6 +55,7 @@ COPY --from=dlc /usr/local/cuda /usr/local/cuda # Copy EFA stack from DLC COPY --from=dlc /opt/amazon/efa /opt/amazon/efa COPY --from=dlc /opt/amazon/openmpi /opt/amazon/openmpi +COPY --from=dlc /opt/amazon/ofi-nccl /opt/amazon/ofi-nccl # Copy NCCL configuration COPY --from=dlc /usr/local/lib/libnccl* /usr/local/lib/ diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py deleted file mode 100644 index 9990e0f17..000000000 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/download_c4.py +++ /dev/null @@ -1,6 +0,0 @@ -from datasets import load_dataset - -# Download and cache the English C4 dataset -dataset = load_dataset("allenai/c4", - name="en", - cache_dir="/fsx/datasets/c4") \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-llama2_7b-training.sbatch b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-llama2_7b-training.sbatch new file mode 100644 index 000000000..9635ff16d --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-llama2_7b-training.sbatch @@ -0,0 +1,123 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +#SBATCH --nodes=4 # number of nodes to use +#SBATCH --job-name=llama2_7b-FSDP # name of your job +#SBATCH --output=logs/%x_%j.out # logfile for stdout +#SBATCH --error=logs/%x_%j.err # logfile for stderr, remove it to merge both outputs +#SBATCH --exclusive # job has exclusive use of the resource, no sharing +#SBATCH --ntasks-per-node=1 # one task per node +#SBATCH --cpus-per-task=32 # match the number of CPUs per node +set -ex; + +########################### +###### User Variables ##### +########################### + +GPUS_PER_NODE=1 + +########################### +## Environment Variables ## +########################### + +export CUDA_HOME="/usr/local/cuda" +export EFA_PATH="/opt/amazon/efa" +export OPEN_MPI_PATH="/opt/amazon/openmpi" +export OFI_NCCL_PATH="/opt/amazon/ofi-nccl" +export LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${OPEN_MPI_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu:/opt/nccl/build/lib:${OFI_NCCL_PATH}/lib/x86_64-linux-gnu:/usr/local/nvidia/lib" + +# LD_PRELOAD is required for PyTorch to find the NCCL library +export LD_PRELOAD="/usr/local/lib/libnccl.so.2" + +export CUDA_VISIBLE_DEVICES=0 # Restrict PyTorch to only use the first GPU (GPU 0) +export NVIDIA_VISIBLE_DEVICES=all # Make all GPUs visible to NVIDIA container runtime + +# Debug settings +export NCCL_DEBUG=INFO # Set NCCL debug level for troubleshooting +export NCCL_DEBUG_SUBSYS=ALL # Enable detailed debugging output for all NCCL subsystems + +# Timeout settings +export NCCL_TIMEOUT=1800 # Set overall NCCL operation timeout to 30 minutes (in seconds) +export NCCL_SOCKET_TIMEOUT=300 # Allow 5 minutes for TCP socket connections between nodes +export NCCL_ASYNC_ERROR_HANDLING=1 # Enable asynchronous error handling for better fault tolerance + +# Buffer settings +export NCCL_BUFFSIZE=2097152 # Set NCCL communication buffer size to 2MB for larger transfers + +# TCP connection settings +export TORCH_DISTRIBUTED_DETAILED_LOGGING=1 # Enable verbose logging for PyTorch distributed operations +export GLOO_SOCKET_IFNAME=eth0 # Use eth0 network interface for Gloo collective operations +export TP_SOCKET_IFNAME=eth0 # Use eth0 for tensor parallelism communication +export NCCL_SOCKET_IFNAME=eth0 # Use eth0 (primary EC2 network interface) for NCCL communication + +# TCP Store timeout settings +export TORCHELASTIC_MAX_CALLTIME=3600 # Set maximum call time for TorchElastic operations to 1 hour +export PYTORCH_TIMEOUT=3600 # Set PyTorch RPC timeout to 1 hour +export TORCH_DISTRIBUTED_TIMEOUT=3600 # Set PyTorch distributed timeout to 1 hour + +# PyTorch specific settings +export TORCH_DISTRIBUTED_DEBUG=DETAIL # Enable detailed debugging for distributed operations +export TORCH_CPP_LOG_LEVEL=INFO # Set C++ frontend logging level to INFO +export CUDA_LAUNCH_BLOCKING=0 # Allow asynchronous CUDA kernel launches (0=async, 1=sync) + +# HuggingFace settings +export HF_HUB_ETAG_TIMEOUT=60 # Metadata timeout (in seconds) for large clusters +export HF_TOKEN= # Token used to avoid throttling for data streaming + +########################### +####### Torch Dist ####### +########################### + +# Debug Slurm environment +echo "=== Slurm Environment ===" +echo "SLURM_JOB_ID: $SLURM_JOB_ID" +echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" +echo "SLURM_NODELIST: $SLURM_NODELIST" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" +echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" +echo "=======================" + +declare -a TORCHRUN_ARGS=( + --nproc_per_node=$GPUS_PER_NODE + --nnodes=$SLURM_JOB_NUM_NODES + --rdzv_id=$SLURM_JOB_ID + --rdzv_backend=c10d + --rdzv_endpoint=$(hostname) +) + +export PATH="/usr/local/bin:$PATH" +export TRAIN_SCRIPT="/fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/src/train.py" +export PYTHONPATH="/usr/local/lib/python3.12/site-packages:$PYTHONPATH" +export TORCHRUN="/usr/local/bin/python3 -m torch.distributed.run" + +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +############################ +# llama2_7b Training Params ## +############################ +declare -a TRAINING_ARGS=( + --max_context_width=512 + --num_key_value_heads=8 + --intermediate_size=2048 + --hidden_width=1024 + --num_layers=8 + --num_heads=16 + --model_type=llama_v2 + --tokenizer="hf-internal-testing/llama-tokenizer" + --checkpoint_freq=100 + --validation_freq=100 + --max_steps=1000 + --checkpoint_dir=./checkpoints + --dataset='allenai/c4' + --dataset_config_name='en' + --resume_from_checkpoint=./checkpoints + --train_batch_size=1 + --val_batch_size=1 + --gradient_checkpointing=True + --mixed_precision=bf16 + --sharding_strategy="full" # https://pytorch.org/docs/stable/fsdp.html + --offload_activations=1 +) + +srun --export=ALL -l ${TORCHRUN} "${TORCHRUN_ARGS[@]}" $TRAIN_SCRIPT "${TRAINING_ARGS[@]}" \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml similarity index 88% rename from 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml rename to 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml index 0cc60662b..32654cd01 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml @@ -1,6 +1,5 @@ # SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. # SPDX-License-Identifier: Apache-2.0 - # Inter-pod anti-affinity and node affinity for non-compute components commonAffinity: &commonAffinity nodeAffinity: @@ -21,6 +20,7 @@ commonAffinity: &commonAffinity operator: In values: ["slurmdbd", "slurmctld", "slurm-exporter", "login", "mariadb", "slurmrestd"] topologyKey: "kubernetes.io/hostname" + # # Debug configuration. # @ignored @@ -94,82 +94,41 @@ slurm: # Extra slurmdbd configuration lines to append to `slurmdbd.conf`. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurmdbd.conf.html - extraSlurmdbdConf: - CommitDelay: 1 + extraSlurmdbdConf: {} + # CommitDelay: 1 ### LOGGING ### - DebugLevel: info - DebugFlags: [] - LogTimeFormat: - - iso8601_ms - - format_stderr - # PLUGINS & PARAMETERS - CommunicationParameters: [] - HashPlugin: hash/k12 - ### ARCHIVE ### - ArchiveDir: /tmp - #ArchiveEvents: YES - #ArchiveJobs: YES - #ArchiveResvs: YES - #ArchiveSteps: NO - #ArchiveSuspend: NO - #ArchiveTXN: NO - #ArchiveUsage: NO + # DebugLevel: debug2 + # DebugFlags: [] ### PURGE ### - #PurgeEventAfter: 12month - #PurgeJobAfter: 12month - #PurgeResvAfter: 2month - #PurgeStepAfter: 2month - #PurgeSuspendAfter: 1month - #PurgeTXNAfter: 12month - #PurgeUsageAfter: 12month + # PurgeEventAfter: 12month + # PurgeJobAfter: 12month + # PurgeResvAfter: 2month + # PurgeStepAfter: 2month + # PurgeSuspendAfter: 1month + # PurgeTXNAfter: 12month + # PurgeUsageAfter: 12month # # -- (map[string]string | map[string][]string) # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurm.conf.html - extraSlurmConf: - MaxNodeCount: 1024 - ReturnToService: 2 - EnforcePartLimits: "NO" - ### PLUGINS & PARAMETERS ### - AuthInfo: - - use_client_ids - SchedulerType: sched/backfill - SchedulerParameters: - - defer_batch - SelectType: select/cons_tres - SelectTypeParameters: - - CR_Core_Memory - SlurmctldParameters: - - enable_configless - - enable_stepmgr - SlurmdParameters: - - contain_spank - CommunicationParameters: - - block_null_hash - LaunchParameters: - - enable_nss_slurm - - use_interactive_step - - ulimit_pam_adopt - ReconfigFlags: - - KeepPartInfo - - KeepPartState - PrologFlags: Contain - HashPlugin: hash/k12 + extraSlurmConf: {} + # MinJobAge: 2 + # MaxNodeCount: 1024 ### LOGGING ### - SlurmctldDebug: info - SlurmSchedLogLevel: 1 - SlurmdDebug: info - DebugFlags: [] - LogTimeFormat: - - iso8601_ms - - format_stderr + # SlurmctldDebug: debug2 + # SlurmSchedLogLevel: 1 + # SlurmdDebug: debug2 + # DebugFlags: [] + ### PLUGINS & PARAMETERS ### + # SchedulerParameters: + # - defer_batch # # -- (map[string]string) # Optional raw Slurm configuration files, as a map. # The map key represents the config file by name; the map value represents config file contents as a string. # Ref: https://slurm.schedmd.com/man_index.html#configuration_files - configFiles: {} + configFiles: # acct_gather.conf: | # # Ref: https://slurm.schedmd.com/acct_gather.conf.html # burst_buffer.conf: | @@ -197,8 +156,9 @@ slurm: # Ref: https://slurm.schedmd.com/prolog_epilog.html # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) prologScripts: {} - # empty: | + # 00-empty.sh: | # #!/usr/bin/env bash + # set -euo pipefail # exit 0 # # -- (map[string]string) @@ -209,28 +169,11 @@ slurm: # Ref: https://slurm.schedmd.com/prolog_epilog.html # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) epilogScripts: {} - # empty: | + # 00-empty.sh: | # #!/usr/bin/env bash + # set -euo pipefail # exit 0 -# -# Shared configurations. -sharedConfig: - # - # -- (list) - # List of volumes to be mounted on each Login and NodeSet pods. - # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ - volumes: [] - # - name: nfs-home - # mountPath: /home - # persistentVolumeClaim: - # claimName: nfs-home - # - name: nfs-data - # mountPath: /mnt/data - # persistentVolumeClaim: - # claimName: nfs-data - -# # Slurm authcred (sackd) configurations. authcred: # @@ -263,14 +206,6 @@ authcred: # # Slurm controller (slurmctld) configurations. controller: - # - # -- (bool) - # Enables the controller node. - enabled: true - # - # -- (integer) - # Set the number of replicas to deploy. - replicas: 1 # # -- (string) # Set the image pull policy. @@ -287,10 +222,27 @@ controller: # Set the image tag to use. tag: 24.11-ubuntu24.04 # + # -- (object) + # The controller service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: {} + # type: LoadBalancer + # externalIPs: [] + # externalName: my.slurmctld.example.com + # + # -- (integer) + # The external service port number. + servicePort: 6817 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 36817 + # # -- (string) # Set the priority class to use. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass - priorityClassName: + priorityClassName: "" # # -- (object) # Set affinity for Kubernetes Pod scheduling. @@ -381,14 +333,10 @@ login: tag: 24.11-ubuntu24.04 # # -- (object) - # The restapi service configuration. + # The login service configuration. # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ service: type: LoadBalancer - # annotations: - # service.beta.kubernetes.io/aws-load-balancer-type: "external" - # service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - # service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip" # externalIPs: [] # externalName: my.login.example.com # @@ -404,13 +352,17 @@ login: # -- (list) # The `/root/.ssh/authorized_keys` file to write, represented as a list. rootSshAuthorizedKeys: - - "" # # -- (map) # The `/etc/ssh/sshd_config` file to use, represented as a map. # Ref: https://man.openbsd.org/sshd_config sshdConfig: - # This is the actual content of the sshd_config file + # LogLevel: DEBUG3 + # Include: "/etc/ssh/sshd_config.d/*.conf" + # X11Forwarding: "yes" + # UsePAM: "yes" + # Subsystem: sftp /usr/libexec/openssh/sftp-server AcceptEnv: "LANG LC_*" AuthorizedKeysFile: "/root/.ssh/authorized_keys" ChallengeResponseAuthentication: "no" @@ -428,10 +380,6 @@ login: UseDNS: "no" UsePAM: "no" X11Forwarding: "no" - - - # Include: "/etc/ssh/sshd_config.d/*.conf" - # Subsystem: sftp internal-sftp # # The `/etc/sssd/sssd.conf` represented by as a map. sssdConf: @@ -472,42 +420,39 @@ login: pam: {} # debug_level: 9 # - # --(map) - # The `/etc/nsswitch.conf` file to use, represented as a map. - # Ref: https://man7.org/linux/man-pages/man5/nsswitch.conf.5.html - nsswitchConf: - passwd: files - group: files - shadow: files - gshadow: files - sudoers: files - hosts: files - networks: files - protocols: files - services: files - ethers: files - rpc: files - netgroup: files - automount: files - # - # -- (list) - # List of volumes to be mounted on each login pod. + # --(list) + # List of volume mounts. # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ - volumes: + extraVolumeMounts: - name: fsx-lustre mountPath: /fsx - persistentVolumeClaim: - claimName: fsx-claim - name: fsx-openzfs mountPath: /home - persistentVolumeClaim: - claimName: openzfs-claim # - name: nfs-home # mountPath: /home + # - name: nfs-data + # mountPath: /mnt/data + # + # --(list) + # Define list of pod volumes. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumes: + - name: fsx-lustre + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + persistentVolumeClaim: + claimName: openzfs-claim + # - name: nfs-home # persistentVolumeClaim: # claimName: nfs-home # - name: nfs-data - # mountPath: /mnt/data + # persistentVolumeClaim: + # - name: nfs-home + # nfs: + # server: nfs-server.example.com + # path: /exports/home/ + # - name: nfs-data # persistentVolumeClaim: # claimName: nfs-data # @@ -555,7 +500,6 @@ compute: # # -- (string) # Set the image tag to use. - # @default -- The Release appVersion. tag: 24.11-ubuntu24.04 # # -- (list) @@ -599,12 +543,8 @@ compute: # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container resources: limits: - cpu: "32" - memory: "128Gi" nvidia.com/gpu: "1" requests: - cpu: "1" - memory: "1Gi" nvidia.com/gpu: "1" # # -- (map) @@ -697,27 +637,39 @@ compute: # requests: # storage: 1Gi # - # -- (list) - # List of volumes to be mounted on each NodeSet pod. + # --(list) + # List of volume mounts. # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ - volumes: + extraVolumeMounts: - name: fsx-lustre mountPath: /fsx + - name: fsx-openzfs + mountPath: /home + - name: shmem + mountPath: /dev/shm + # - name: nfs-home + # mountPath: /home + # - name: nfs-data + # mountPath: /mnt/data + # + # --(list) + # Define list of pod volumes. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumes: + - name: fsx-lustre persistentVolumeClaim: claimName: fsx-claim - name: fsx-openzfs - mountPath: /home - persistentVolumeClaim: + persistentVolumeClaim: claimName: openzfs-claim - - # - name: nfs-bin - # mountPath: /usr/local/bin + - name: shmem + hostPath: + path: /dev/shm + # - name: nfs-home # nfs: # server: nfs-server.example.com - # path: /opt/bin - # readOnly: true + # path: /exports/home/ # - name: nfs-data - # mountPath: /mnt/data # persistentVolumeClaim: # claimName: nfs-data # @@ -785,10 +737,6 @@ accounting: # Enables accounting services. enabled: true # - # -- (integer) - # Set the number of replicas to deploy. - replicas: 1 - # # -- (string) # Set the image pull policy. imagePullPolicy: IfNotPresent @@ -1005,4 +953,4 @@ slurm-exporter: exporter: enabled: true secretName: "slurm-token-exporter" - affinity: *commonAffinity + affinity: *commonAffinity \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch deleted file mode 100644 index 88616d62a..000000000 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/llama2_7b-training.sbatch +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: MIT-0 - -#SBATCH --nodes=4 # number of nodes to use -#SBATCH --job-name=llama2_7b-FSDP # name of your job -#SBATCH --output=logs/%x_%j.out # logfile for stdout -#SBATCH --error=logs/%x_%j.err # logfile for stderr, remove it to merge both outputs -#SBATCH --exclusive # job has exclusive use of the resource, no sharing -#SBATCH --ntasks-per-node=1 # one task per node -#SBATCH --cpus-per-task=32 # match the number of CPUs per node -set -ex; - -GPUS_PER_NODE=1 # 4 for G5.12x, 8 for P4/P5 - -# Set environment variables -export CUDA_VISIBLE_DEVICES=0 -export NVIDIA_VISIBLE_DEVICES=all -# Set LD_LIBRARY_PATH to prioritize pip-installed CUDA 12.4 libraries - -# Check if NCCL exists before setting LD_PRELOAD -if [ -f "/usr/local/lib/libnccl.so.2" ]; then - export LD_PRELOAD="/usr/local/lib/libnccl.so.2" -fi - -# Basic NCCL settings -export NCCL_DEBUG=INFO -export NCCL_SOCKET_IFNAME=^docker,lo,veth,eth - -# Performance settings -export NCCL_IB_DISABLE=0 # Enable InfiniBand -export NCCL_IB_GID_INDEX=3 # Specify GID index for IB -export NCCL_IB_TC=106 # Traffic class for IB -export NCCL_IB_SL=3 # Service level for IB -export NCCL_NET_GDR_LEVEL=2 # GPU Direct RDMA level - -# Timeout settings -export NCCL_TIMEOUT=1800 # 30 minutes timeout (in seconds) -export NCCL_SOCKET_TIMEOUT=300 # wait 5 minutes for TCP connections between nodes -export NCCL_ASYNC_ERROR_HANDLING=1 # Enable async error handling - -# Buffer settings -export NCCL_BUFFSIZE=2097152 # 2MB buffer size -export NCCL_IB_CUDA_SUPPORT=1 # Enable CUDA support for IB - -# Remove this if not debugging specific issues -# export NCCL_DEBUG_SUBSYS=ALL # Very verbose debugging - -## Set HuggingFace metadata timeout (in seconds) for large clusters -export HF_HUB_ETAG_TIMEOUT=60 - -# TCP connection settings -export TORCH_DISTRIBUTED_DETAILED_LOGGING=1 -export GLOO_SOCKET_IFNAME=eth0 -export TP_SOCKET_IFNAME=eth0 -export NCCL_SOCKET_IFNAME=eth0 - -# TCP Store timeout settings -export TORCHELASTIC_MAX_CALLTIME=3600 -export PYTORCH_TIMEOUT=3600 -export TORCH_DISTRIBUTED_TIMEOUT=3600 - -# PyTorch specific settings -export TORCH_DISTRIBUTED_DEBUG=DETAIL # Enable distributed debug info -export TORCH_CPP_LOG_LEVEL=INFO # C++ front-end logging -export CUDA_LAUNCH_BLOCKING=0 # Async CUDA operation - -########################### -####### Torch Dist ####### -########################### - -# Debug Slurm environment -echo "=== Slurm Environment ===" -echo "SLURM_JOB_ID: $SLURM_JOB_ID" -echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" -echo "SLURM_NODELIST: $SLURM_NODELIST" -echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" - -declare -a TORCHRUN_ARGS=( - --nproc_per_node=$GPUS_PER_NODE - --nnodes=$SLURM_JOB_NUM_NODES - --rdzv_id=$SLURM_JOB_ID - --rdzv_backend=c10d - --rdzv_endpoint=$(hostname) -) - -export PATH="/usr/local/bin:$PATH" -export PYTHONPATH="/usr/local/lib/python3.12/site-packages:$PYTHONPATH" -export TORCHRUN="/usr/local/bin/python3 -m torch.distributed.run" -export TRAIN_SCRIPT="/fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/src/train.py" - -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - -############################ -# llama2_7b Training Params ## -############################ -declare -a TRAINING_ARGS=( - --max_context_width=512 - --num_key_value_heads=8 - --intermediate_size=2048 - --hidden_width=1024 - --num_layers=8 - --num_heads=16 - --model_type=llama_v2 - --tokenizer="hf-internal-testing/llama-tokenizer" - --checkpoint_freq=100 - --validation_freq=100 - --max_steps=1000 - --checkpoint_dir=./checkpoints - --dataset='c4' \ - --dataset_path='/fsx/datasets/c4/allenai___c4' # Point to downloaded dataset - --dataset_config_name='en' - --resume_from_checkpoint=./checkpoints - --train_batch_size=1 - --val_batch_size=1 - --gradient_checkpointing=True - --mixed_precision=bf16 - --sharding_strategy="full" # https://pytorch.org/docs/stable/fsdp.html - --offload_activations=1 -) - -srun --export=ALL -l ${TORCHRUN} "${TORCHRUN_ARGS[@]}" $TRAIN_SCRIPT "${TRAINING_ARGS[@]}" \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-llama2_7b-training.sbatch b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-llama2_7b-training.sbatch new file mode 100644 index 000000000..528973704 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-llama2_7b-training.sbatch @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +#SBATCH --nodes=4 # number of nodes to use +#SBATCH --job-name=llama2_7b-FSDP # name of your job +#SBATCH --output=logs/%x_%j.out # logfile for stdout +#SBATCH --error=logs/%x_%j.err # logfile for stderr, remove it to merge both outputs +#SBATCH --exclusive # job has exclusive use of the resource, no sharing + +set -ex; + +########################### +###### User Variables ##### +########################### + +GPUS_PER_NODE=4 + +########################### +## Environment Variables ## +########################### + +export CUDA_HOME="/usr/local/cuda" +export EFA_PATH="/opt/amazon/efa" +export OPEN_MPI_PATH="/opt/amazon/openmpi" +export OFI_NCCL_PATH="/opt/amazon/ofi-nccl" +export LD_LIBRARY_PATH="lib:${EFA_PATH}/lib:${OPEN_MPI_PATH}/lib:${CUDA_HOME}/lib64:/usr/local/lib:/lib/x86_64-linux-gnu:/opt/nccl/build/lib:${OFI_NCCL_PATH}/lib/x86_64-linux-gnu:/usr/local/nvidia/lib" + +# LD_PRELOAD is required for PyTorch to find the NCCL library +export LD_PRELOAD="/usr/local/lib/libnccl.so.2" + +# NCCL settings for EFA +export NCCL_PROTO=simple # Use a simpler communication protocol, often more reliable for EFA +export NCCL_ALGO=ring # Use ring algorithm for collective operations, typically best for EFA +export NCCL_NET_GDR_LEVEL=5 # Enable GPUDirect RDMA for direct GPU-to-network transfers +export NCCL_DEBUG=INFO # Set NCCL debug level for troubleshooting +export NCCL_DEBUG_SUBSYS=ALL # Enable detailed debugging output for all NCCL subsystems +export NCCL_SOCKET_IFNAME=^lo # Exclude loopback interface +export NCCL_NET_MAX_REQUESTS=8 # Maximum number of concurrent network requests, optimized for EFA +export NCCL_MIN_NCHANNELS=8 # Minimum number of channels for NCCL communications, increases parallelism +export NCCL_NSOCKS_PERTHREAD=8 # Number of sockets per thread for network operations + +# HuggingFace settings +export HF_HUB_ETAG_TIMEOUT=60 # Metadata timeout (in seconds) for large clusters +export HF_TOKEN= # Token used to avoid throttling for data streaming + +########################### +####### Torch Dist ####### +########################### + +# Debug Slurm environment +echo "=== Slurm Environment ===" +echo "SLURM_JOB_ID: $SLURM_JOB_ID" +echo "SLURM_JOB_NUM_NODES: $SLURM_JOB_NUM_NODES" +echo "SLURM_NODELIST: $SLURM_NODELIST" +echo "SLURM_JOB_NODELIST: $SLURM_JOB_NODELIST" +echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" +echo "=======================" + + +declare -a TORCHRUN_ARGS=( + --nproc_per_node=$GPUS_PER_NODE + --nnodes=$SLURM_JOB_NUM_NODES + --rdzv_id=$SLURM_JOB_ID + --rdzv_backend=c10d + --rdzv_endpoint=$(hostname) +) + +export PATH="/usr/local/bin:$PATH" +export TRAIN_SCRIPT="/fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/src/train.py" +export PYTHONPATH="/usr/local/lib/python3.12/site-packages:$PYTHONPATH" +export TORCHRUN="/usr/local/bin/python3 -m torch.distributed.run" + +############################ +# llama2_7b Training Params ## +############################ +declare -a TRAINING_ARGS=( + --max_context_width=4096 + --num_key_value_heads=32 + --intermediate_size=11008 + --hidden_width=4096 + --num_layers=32 + --num_heads=32 + --model_type=llama_v2 + --tokenizer="hf-internal-testing/llama-tokenizer" + --checkpoint_freq=5000 + --validation_freq=500 + --max_steps=5000 + --checkpoint_dir=./checkpoints + --dataset='allenai/c4' + --dataset_config_name='en' + --resume_from_checkpoint=./checkpoints + --train_batch_size=1 + --val_batch_size=1 + --sharding_strategy="full" # https://pytorch.org/docs/stable/fsdp.html + --offload_activations=1 +) + +srun -l ${TORCHRUN} "${TORCHRUN_ARGS[@]}" $TRAIN_SCRIPT "${TRAINING_ARGS[@]}" \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml new file mode 100644 index 000000000..54715c271 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml @@ -0,0 +1,954 @@ +# SPDX-FileCopyrightText: Copyright (C) SchedMD LLC. +# SPDX-License-Identifier: Apache-2.0 +# Inter-pod anti-affinity and node affinity for non-compute components +commonAffinity: &commonAffinity + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node.kubernetes.io/instance-type" + operator: In + values: + - "ml.m5.2xlarge" + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: ["slurmdbd", "slurmctld", "slurm-exporter", "login", "mariadb", "slurmrestd"] + topologyKey: "kubernetes.io/hostname" + +# +# Debug configuration. +# @ignored +debug: + # + # -- (bool) + # Enables debug configuration. + enabled: false + # + # -- (bool) + # Allow a locally running operator to communicate with slurm cluster via port-forward. + # NOTE: use when running the operator in a local debugger. + localOperator: true + +# +# -- (string) +# Overrides the name of the release. +nameOverride: "" + +# +# -- (string) +# Overrides the full name of the release. +fullnameOverride: "" + +# +# -- (string) +# Overrides the namespace of the release. +namespaceOverride: "" + +# +# -- (list) +# Set the secrets for image pull. +# Ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +imagePullSecrets: [] + # - name: regcred + +# +# -- (string) +# Set the image pull policy. +imagePullPolicy: IfNotPresent + +# +# -- (string) +# Set the priority class to use. +# Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass +priorityClassName: "" + +# +# Slurm JWT authentication. +jwt: + # + # JWT hs256 configurations. + hs256: + # + # -- (string) + # The existing secret to use otherwise one will be generated. + existingSecret: "" + +# +# Slurm configurations. +slurm: + # + # Slurm authentication configurations. + auth: + # + # -- (string) + # The existing secret to use otherwise one will be generated. + existingSecret: "" + # + # -- (map[string]string | map[string][]string) + # Extra slurmdbd configuration lines to append to `slurmdbd.conf`. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurmdbd.conf.html + extraSlurmdbdConf: {} + # CommitDelay: 1 + ### LOGGING ### + # DebugLevel: debug2 + # DebugFlags: [] + ### PURGE ### + # PurgeEventAfter: 12month + # PurgeJobAfter: 12month + # PurgeResvAfter: 2month + # PurgeStepAfter: 2month + # PurgeSuspendAfter: 1month + # PurgeTXNAfter: 12month + # PurgeUsageAfter: 12month + # + # -- (map[string]string | map[string][]string) + # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurm.conf.html + extraSlurmConf: {} + # MinJobAge: 2 + # MaxNodeCount: 1024 + ### LOGGING ### + # SlurmctldDebug: debug2 + # SlurmSchedLogLevel: 1 + # SlurmdDebug: debug2 + # DebugFlags: [] + ### PLUGINS & PARAMETERS ### + # SchedulerParameters: + # - defer_batch + # + # -- (map[string]string) + # Optional raw Slurm configuration files, as a map. + # The map key represents the config file by name; the map value represents config file contents as a string. + # Ref: https://slurm.schedmd.com/man_index.html#configuration_files + configFiles: + # acct_gather.conf: | + # # Ref: https://slurm.schedmd.com/acct_gather.conf.html + # burst_buffer.conf: | + # # Ref: https://slurm.schedmd.com/burst_buffer.conf.html + # gres.conf: | + # # Ref: https://slurm.schedmd.com/gres.conf.html + # helpers.conf: | + # # Ref: https://slurm.schedmd.com/helpers.conf.html + # job_container.conf: | + # # Ref: https://slurm.schedmd.com/job_container.conf.html + # mpi.conf: | + # # Ref: https://slurm.schedmd.com/mpi.conf.html + # oci.conf: | + # # Ref: https://slurm.schedmd.com/oci.conf.html + # plugstack.conf: | + # # Ref: https://slurm.schedmd.com/plugstack.conf.html + # topology.conf: | + # # Ref: https://slurm.schedmd.com/topology.conf.html + # + # -- (map[string]string) + # The Prolog scripts for compute nodesets, as a map. + # The map key represents the filename; the map value represents the script contents. + # WARNING: The script must include a shebang (!) so it can be executed correctly by Slurm. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Prolog + # Ref: https://slurm.schedmd.com/prolog_epilog.html + # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) + prologScripts: {} + # 00-empty.sh: | + # #!/usr/bin/env bash + # set -euo pipefail + # exit 0 + # + # -- (map[string]string) + # The Epilog scripts for compute nodesets, as a map. + # The map key represents the filename; the map value represents the script contents. + # WARNING: The script must include a shebang (!) so it can be executed correctly by Slurm. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Epilog + # Ref: https://slurm.schedmd.com/prolog_epilog.html + # Ref: https://en.wikipedia.org/wiki/Shebang_(Unix) + epilogScripts: {} + # 00-empty.sh: | + # #!/usr/bin/env bash + # set -euo pipefail + # exit 0 + +# Slurm authcred (sackd) configurations. +authcred: + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/sackd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# Slurm controller (slurmctld) configurations. +controller: + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmctld + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # The controller service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: {} + # type: LoadBalancer + # externalIPs: [] + # externalName: my.slurmctld.example.com + # + # -- (integer) + # The external service port number. + servicePort: 6817 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 36817 + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: *commonAffinity + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + # + # Define a persistent volume for the slurm controller to store its save-state. + # Used to recover from system failures or from pod upgrades. + persistence: + # + # -- (bool) + # Enables save-state persistence. + enabled: false + # + # -- (string) + # Name of an existing `PersistentVolumeClaim` to use instead of creating one from definition. + # NOTE: When not empty, the other persistence fields will be ignored. + existingClaim: "" + # + # -- (object) + # Create a `PersistentVolumeClaim` with these annotations. + annotations: {} + # + # -- (object) + # Create a `PersistentVolumeClaim` with these labels. + labels: {} + # + # -- (string) + # Create a `PersistentVolumeClaim` with this storage class. + storageClass: standard + # + # -- (list) + # Create a `PersistentVolumeClaim` with these access modes. + accessModes: + - ReadWriteOnce + # + # -- (string) + # Create a `PersistentVolumeClaim` with this storage size. + size: 4Gi + # + # -- (object) + # Selector to match an existing `PersistentVolume`. + selector: {} + # matchLabels: + # app: foo + +# +# Login node configurations. +login: + # + # -- (bool) + # Enables login nodes. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/login + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # The login service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: + type: LoadBalancer + # externalIPs: [] + # externalName: my.login.example.com + # + # -- (integer) + # The external service port number. + servicePort: 22 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 32222 + # + # -- (list) + # The `/root/.ssh/authorized_keys` file to write, represented as a list. + rootSshAuthorizedKeys: + - "" + # + # -- (map) + # The `/etc/ssh/sshd_config` file to use, represented as a map. + # Ref: https://man.openbsd.org/sshd_config + sshdConfig: + # LogLevel: DEBUG3 + # Include: "/etc/ssh/sshd_config.d/*.conf" + # X11Forwarding: "yes" + # UsePAM: "yes" + # Subsystem: sftp /usr/libexec/openssh/sftp-server + AcceptEnv: "LANG LC_*" + AuthorizedKeysFile: "/root/.ssh/authorized_keys" + ChallengeResponseAuthentication: "no" + ClientAliveCountMax: "3" + ClientAliveInterval: "60" + LogLevel: "INFO" + PasswordAuthentication: "no" + PermitRootLogin: "yes" + Port: "22" + PrintMotd: "no" + Protocol: "2" + PubkeyAuthentication: "yes" + Subsystem: "sftp internal-sftp" + TCPKeepAlive: "yes" + UseDNS: "no" + UsePAM: "no" + X11Forwarding: "no" + # + # The `/etc/sssd/sssd.conf` represented by as a map. + sssdConf: + # + # -- (map) + # The `/etc/sssd/sssd.conf` [sssd] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#The_%5Bsssd%5D_section + sssd: + # debug_level: 9 + config_file_version: 2 + services: nss, pam + domains: DEFAULT + # + # -- (map[map]) + # The `/etc/sssd/sssd.conf` [domain/$DOMAIN] sections, represented as a map of map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#DOMAIN_SECTIONS + domains: + DEFAULT: + # debug_level: 9 + auth_provider: ldap + id_provider: ldap + ldap_uri: ldap://ldap.example.com + ldap_search_base: dc=example,dc=com + ldap_user_search_base: ou=Users,dc=example,dc=com + ldap_group_search_base: ou=Groups,dc=example,dc=com + # + # -- (map) + # The `/etc/sssd/sssd.conf` [nss] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#NSS_configuration_options + nss: + # debug_level: 9 + filter_groups: root,slurm + filter_users: root,slurm + # + # -- (map) + # The `/etc/sssd/sssd.conf` [pam] section, represented as a map. + # Ref: https://man.archlinux.org/man/sssd.conf.5#PAM_configuration_options + pam: {} + # debug_level: 9 + # + # --(list) + # List of volume mounts. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumeMounts: + - name: fsx-lustre + mountPath: /fsx + - name: fsx-openzfs + mountPath: /home + # - name: nfs-home + # mountPath: /home + # - name: nfs-data + # mountPath: /mnt/data + # + # --(list) + # Define list of pod volumes. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumes: + - name: fsx-lustre + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + persistentVolumeClaim: + claimName: openzfs-claim + # - name: nfs-home + # nfs: + # server: nfs-server.example.com + # path: /exports/home/ + # - name: nfs-data + # persistentVolumeClaim: + # claimName: nfs-data + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: *commonAffinity + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# Slurm compute (slurmd) configurations. +compute: + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Default image for the nodeset pod (slurmd) + # Each nodeset may override this setting. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (list) + # Slurm NodeSets by object list. + nodesets: + # + # -- (string) + # Name of NodeSet. Must be unique. + - name: hp-node + # + # -- (bool) + # Enables the NodeSet in Slurm. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 4 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: Always + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ".dkr.ecr..amazonaws.com/dlc-slurmd" + # + # -- (string) + # Set the image tag to use. + tag: "24.11.4-ubuntu24.04" + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: + limits: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + requests: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + nodeSelector: + kubernetes.io/os: linux + node.kubernetes.io/instance-type: ml.p5.48xlarge + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "kubernetes.io/os" + # operator: In + # values: + # - linux + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - topologyKey: "kubernetes.io/hostname" + # labelSelector: + # matchExpressions: + # - key: "app.kubernetes.io/name" + # operator: In + # values: + # - slurmctld + # - slurmdbd + # - slurmrestd + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set the update strategy configuration. + updateStrategy: + # + # -- (string) + # Set the update strategy type. + # Can be either: "RollingUpdate"; "OnDelete". + type: RollingUpdate + # + # -- (object) + # Define the rolling update policy. + # Only used when "updateStrategy.type=RollingUpdate". + rollingUpdate: + # + # -- (string) + # The maximum number of pods that can be unavailable during the update. + # Value can be an absolute number (ex: 5) or a percentage of desired + # pods (ex: 10%). Absolute number is calculated from percentage by + # rounding up. This can not be 0. Defaults to 1. + maxUnavailable: 20% + # + # -- (object) + # The policy used for PVCs created from the NodeSet VolumeClaimTemplates. + persistentVolumeClaimRetentionPolicy: + # + # -- (string) + # WhenDeleted specifies what happens to PVCs created from NodeSet + # VolumeClaimTemplates when the NodeSet is deleted. The default policy + # of `Retain` causes PVCs to not be affected by NodeSet deletion. The + # `Delete` policy causes those PVCs to be deleted. + whenDeleted: Retain + # + # -- (string) + # WhenScaled specifies what happens to PVCs created from NodeSet + # VolumeClaimTemplates when the NodeSet is scaled down. The default + # policy of `Retain` causes PVCs to not be affected by a scale-in. The + # `Delete` policy causes the associated PVCs for any excess pods to be + # deleted. + whenScaled: Retain + # + # -- (list) + # List of PVCs to be created from template and mounted on each NodeSet pod. + # PVCs are given a unique identity relative to each NodeSet pod. + # Ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#volume-claim-templates + volumeClaimTemplates: [] + # - metadata: + # name: scratch + # spec: + # mountPath: /mnt/scratch + # storageClassName: standard + # accessModes: + # - ReadWriteOnce + # resources: + # requests: + # storage: 1Gi + # + # --(list) + # List of volume mounts. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumeMounts: + - name: fsx-lustre + mountPath: /fsx + - name: fsx-openzfs + mountPath: /home + - name: shmem + mountPath: /dev/shm + # - name: nfs-home + # mountPath: /home + # - name: nfs-data + # mountPath: /mnt/data + # + # --(list) + # Define list of pod volumes. + # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ + extraVolumes: + - name: fsx-lustre + persistentVolumeClaim: + claimName: fsx-claim + - name: fsx-openzfs + persistentVolumeClaim: + claimName: openzfs-claim + - name: shmem + hostPath: + path: /dev/shm + # - name: nfs-home + # nfs: + # server: nfs-server.example.com + # path: /exports/home/ + # - name: nfs-data + # persistentVolumeClaim: + # claimName: nfs-data + # + # -- (object) + # Partition describes the partition created specifically for this NodeSet to be added. + partition: + # + # -- (bool) + # Enables this NodeSet's partition line to be added in Slurm. + enabled: true + # + # -- (map[string]string | map[string][]string) + # Extra Slurm partition configuration appended onto the partition line. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAI + config: + State: UP + MaxTime: UNLIMITED + # + # -- (string) + # Set Slurm node GRES. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1 + nodeGres: "" + # + # -- (list) + # Set Slurm node Features as a list(string). + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Features + nodeFeatures: [] + # + # -- (string) + # Set Slurm node weight for Slurm scheduling. + # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Weight + nodeWeight: 1 + # + # -- (list) + # Slurm Partitions by object list. + partitions: + # + # -- (string) + # Name of Partition. Must be unique. + - name: all + # + # -- (bool) + # Enables the partition in Slurm. + enabled: true + # + # -- (list) + # NodeSets to put into this Partition by name/key. + # NOTE: 'ALL' is a Slurm meta value to mean all nodes in the system. + nodesets: + - ALL + # + # -- (map[string]string | map[string][]string) + # Extra Slurm partition configuration appended onto the partition line. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAI + config: + State: UP + Default: "YES" + MaxTime: UNLIMITED + +# +# Slurm accounting (slurmdbd) configurations. +accounting: + # + # -- (bool) + # Enables accounting services. + enabled: true + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmdbd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: *commonAffinity + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + # + # Configuration for an external accounting instance (slurmdbd). + external: + # + # -- (bool) + # Use an external acounting instance (slurmdbd) instead of deploying one. + enabled: false + # + # -- (string) + # The external acounting instance (slurmdbd) host. + host: "" + # + # -- (integer) + # The external acounting instance (slurmdbd) port. + port: 6819 + +# +# `bitnami/mariadb` subchart configurations. +# Ref: https://github.com/bitnami/charts/blob/main/bitnami/mariadb/values.yaml +mariadb: + enabled: true + auth: + username: slurm + database: slurm_acct_db + tls: + enabled: false + tde: + enabled: false + primary: + # NOTE: https://slurm.schedmd.com/accounting.html#slurm-accounting-configuration-before-build + configuration: |- + [mysqld] + skip-name-resolve + explicit_defaults_for_timestamp + basedir=/opt/bitnami/mariadb + datadir=/bitnami/mariadb/data + plugin_dir=/opt/bitnami/mariadb/plugin + port={{ .Values.primary.containerPorts.mysql }} + socket=/opt/bitnami/mariadb/tmp/mysql.sock + tmpdir=/opt/bitnami/mariadb/tmp + innodb_buffer_pool_size=4096M + innodb_lock_wait_timeout=900 + innodb_log_file_size=1024M + max_allowed_packet=16M + bind-address=* + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + log-error=/opt/bitnami/mariadb/logs/mysqld.log + character-set-server=UTF8 + collation-server=utf8_general_ci + slow_query_log=0 + long_query_time=10.0 + binlog_expire_logs_seconds=2592000 + {{- if .Values.tls.enabled }} + ssl_cert=/opt/bitnami/mariadb/certs/{{ .Values.tls.certFilename }} + ssl_key=/opt/bitnami/mariadb/certs/{{ .Values.tls.certKeyFilename }} + {{- if (include "mariadb.tlsCACert" .) }} + ssl_ca={{ include "mariadb.tlsCACert" . }} + {{- end }} + {{- end }} + {{- if .Values.tde.enabled }} + plugin_load_add=file_key_management + file_key_management_filename=/opt/bitnami/mariadb/tde/{{ .Values.tde.encryptedKeyFilename }} + file_key_management_filekey=FILE:/opt/bitnami/mariadb/tde/{{ .Values.tde.randomKeyFilename }} + file_key_management_encryption_algorithm={{ .Values.tde.fileKeyManagementEncryptionAlgorithm }} + innodb_encrypt_tables={{ .Values.tde.innodbEncryptTables }} + innodb_encrypt_log={{ .Values.tde.innodbEncryptLog }} + innodb_encrypt_temporary_tables={{ .Values.tde.innodbEncryptTemporaryTables }} + innodb_encryption_threads={{ .Values.tde.innodbEncryptionThreads }} + encrypt_tmp_disk_tables={{ .Values.tde.encryptTmpDiskTables }} + encrypt_tmp_files={{ .Values.tde.encryptTmpTiles }} + encrypt_binlog={{ .Values.tde.encryptBINLOG }} + aria_encrypt_tables={{ .Values.tde.ariaEncryptTables }} + {{- end }} + + [client] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + default-character-set=UTF8 + plugin_dir=/opt/bitnami/mariadb/plugin + + [manager] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + persistence: + enabled: false + existingClaim: "" + storageClass: standard + labels: {} + annotations: {} + accessModes: + - ReadWriteOnce + size: 8Gi + selector: {} + priorityClassName: "" + tolerations: [] + affinity: *commonAffinity + metrics: + enabled: false + serviceMonitor: + enabled: false + affinity: {} + resources: {} + +# +# Slurm REST API (slurmrestd) configurations. +restapi: + # + # -- (bool) + # Enables restapi services. + enabled: true + # + # -- (integer) + # Set the number of replicas to deploy. + replicas: 1 + # + # -- (string) + # Set the image pull policy. + imagePullPolicy: IfNotPresent + # + # Set the image to use. + image: + # + # -- (string) + # Set the image repository to use. + repository: ghcr.io/slinkyproject/slurmrestd + # + # -- (string) + # Set the image tag to use. + tag: 24.11-ubuntu24.04 + # + # -- (object) + # The restapi service configuration. + # Ref: https://kubernetes.io/docs/concepts/services-networking/service/ + service: {} + # type: LoadBalancer + # externalIPs: [] + # externalName: my.slurmrestd.example.com + # + # -- (integer) + # The external service port number. + servicePort: 6820 + # + # -- (integer) + # The external service node port number. + # Ignored unless `service.type == NodePort`. + serviceNodePort: 36820 + # + # -- (string) + # Set the priority class to use. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass + priorityClassName: "" + # + # -- (object) + # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: *commonAffinity + # + # -- (list) + # Configure pod tolerations. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + tolerations: [] + # + # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: {} + # requests: + # cpu: 1 + # memory: 1Gi + # limits: + # cpu: 2 + # memory: 4Gi + +# +# `slurm-exporter` subchart configurations. +# Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml +slurm-exporter: + enabled: true + exporter: + enabled: true + secretName: "slurm-token-exporter" + affinity: *commonAffinity \ No newline at end of file From 67a7bc99cb02eee25ddd8b7fdbf0efeb662628a6 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 7 May 2025 08:53:15 -0400 Subject: [PATCH 06/15] added reference to distinct values.yaml files --- .../slinky-slurm/README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 591fdde5e..b9a521b1a 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -240,7 +240,11 @@ kubectl get all -n slinky ### Install the Slurm Cluster: -To deploy the slurm cluster, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, in order to test the latest changes in release v0.3.0, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. For your convenience, we've provided a copy of the [values.yaml](./values.yaml) file with most of the configuration changes mentioned below already implemented, so you'll only need to make additional changes as needed to further customize your deployment. +To deploy the slurm cluster, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, in order to test the latest changes in release v0.3.0, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. + +For your convenience, we've provided [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml) files with most of the configuration changes mentioned below already implemented, so you'll only need to make additional changes as needed to further customize your deployment. + +The two things you must minimally modify are the container image that the slurm compute nodes use ([instructions here](#build-and-set-the-compute-node-container-image)) and the root ssh key used for accessing the login node ([instructions here](#login-access)). --- @@ -543,7 +547,7 @@ login: #### Deploy the Slurm Cluster: -Locally package and deploy the slurm cluster using the modified `values.yaml` file: +Locally package and deploy the slurm cluster using the modified `values.yaml` file (either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml)): Assuming you are still sitting in the `slinky-slurm` directory of the AWSome Distributed Training repo that we cloned and navigated into earlier, and assuming you cloned the Slinky repo into your home directory (adjust the path as needed), copy the Helm chart artifacts in for packaging: ``` @@ -557,7 +561,7 @@ helm dependency update slurm helm package slurm ``` -Option 1: Deploy the Slurm cluster on g5 instances: +**Option 1**: Deploy the Slurm cluster on `ml.g5.8xlarge` instances: ``` # Dry run helm install --dry-run slurm slurm-0.3.0.tgz \ @@ -568,7 +572,7 @@ helm install slurm slurm-0.3.0.tgz \ -f g5/g5-values.yaml \ -n slurm ``` -Option 2: Deploy the Slurm cluster on p5 instances: +**Option 2**: Deploy the Slurm cluster on `ml.p5.48xlarge` instances: ``` # Dry run helm install --dry-run slurm slurm-0.3.0.tgz \ @@ -716,6 +720,8 @@ find /usr/local/lib/ -name "nccl.h" 2>/dev/null # /usr/local/lib/python3.12/site-packages/torch/include/torch/csrc/cuda/nccl.h ``` +--- + For p5 capacity, check EFA availability: ``` ls /sys/class/infiniband/ @@ -778,7 +784,6 @@ Add your Hugging Face token to stream the [allenai/c4](https://huggingface.co/da ``` NEW_TOKEN="your_new_token_here" sed -i "s/export HF_TOKEN=.*$/export HF_TOKEN=$NEW_TOKEN/" llama2_7b-training.sbatch - ``` --- From 95966c517461e38a82b3e6093492377496cbf6e5 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 7 May 2025 09:05:48 -0400 Subject: [PATCH 07/15] made IAM role for OpenZFS CSI driver unique --- 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index b9a521b1a..7697cabf3 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -115,7 +115,7 @@ eksctl create iamserviceaccount \ --cluster $EKS_CLUSTER_NAME \ --attach-policy-arn arn:aws:iam::aws:policy/AmazonFSxFullAccess \ --approve \ - --role-name AmazonEKSFSxOpenZFSCSIDriverFullAccess \ + --role-name FSXOCSI-${EKS_CLUSTER_NAME}-${AWS_REGION} \ --region $AWS_REGION kubectl taint nodes --all fsx.openzfs.csi.aws.com/agent-not-ready:NoExecute From 460d0b8879c7447b713ee49f734d74e7865edeee Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Wed, 7 May 2025 11:40:28 -0400 Subject: [PATCH 08/15] updated openzfs instructions for clarity --- .../slinky-slurm/README.md | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 7697cabf3..4a6e8f5c0 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -74,17 +74,14 @@ aws eks associate-access-policy \ --policy-arn $PLCY_ARN \ --access-scope type=cluster \ --region $AWS_REGION - ``` Update your kubectl context: ``` - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME kubectl get nodes - ``` * * * @@ -103,12 +100,9 @@ kubectl get storageclass fsx-sc -oyaml ### Create an FSx for OpenZFS Storage Class: -Install the [OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/blob/main/docs/install.md). - -Set up permissions using IAM roles for service accounts, and taint the nodes as recommended: +Install the [OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver) following the steps provided below: ``` - eksctl create iamserviceaccount \ --name fsx-openzfs-csi-controller-sa \ --namespace kube-system \ @@ -117,8 +111,6 @@ eksctl create iamserviceaccount \ --approve \ --role-name FSXOCSI-${EKS_CLUSTER_NAME}-${AWS_REGION} \ --region $AWS_REGION - -kubectl taint nodes --all fsx.openzfs.csi.aws.com/agent-not-ready:NoExecute helm repo add aws-fsx-openzfs-csi-driver \ https://kubernetes-sigs.github.io/aws-fsx-openzfs-csi-driver @@ -132,20 +124,17 @@ helm upgrade --install aws-fsx-openzfs-csi-driver \ kubectl get pods -n kube-system \ -l app.kubernetes.io/part-of=aws-fsx-openzfs-csi-driver - ``` Follow the [Dynamic Provisioning](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/tree/main/examples/kubernetes/dynamic-provisioning) guide to create an FSx for OpenZFS Storage Class: ``` - export PRIVATE_SUBNET_ID= export SECURITY_GROUP_ID= kubectl apply -f openzfs-storageclass.yaml kubectl get sc openzfs-sc -oyaml - ``` * * * @@ -192,7 +181,6 @@ helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller kubectl get sa aws-load-balancer-controller -n kube-system -oyaml - ``` * * * @@ -217,7 +205,6 @@ For [Slurm Operator](https://github.com/SlinkyProject/slurm-operator/blob/main/d Note: We will locally build and deploy a pre-release v0.3.0 of the [Slurm Cluster](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) from the main branch of the Slinky Project repository. The project is being actively developed, so there is a risk of pulling down breaking changes, but it includes the features to [add additional volume mounts to compute NodeSets](https://github.com/SlinkyProject/slurm-operator/commit/b0e111b0a8434e38b5fb37a2051e7525d5679319) and [deploy Login Pods](https://github.com/SlinkyProject/slurm-operator/commit/37f020f041556164b9c935f799b51df65d22aefe). ``` - curl -L https://raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.2.1/helm/slurm-operator/values.yaml \ -o values-operator-0.2.1.yaml @@ -227,7 +214,6 @@ kubectl delete crd nodesets.slinky.slurm.net helm install slurm-operator oci://ghcr.io/slinkyproject/charts/slurm-operator \ --values=values-operator-0.2.1.yaml --version=0.2.1 --namespace=slinky --create-namespace - ``` Verify Slurm Operator Instillation: @@ -252,7 +238,6 @@ The two things you must minimally modify are the container image that the slurm Clone the Slurm Operator repository, which also contains the Helm chart artifacts for the Slurm Cluster: ``` git clone https://github.com/SlinkyProject/slurm-operator.git - ``` Clone the AWSome Distributed Training repo to use the [values.yaml](./values.yaml) file we've provided: @@ -317,7 +302,6 @@ ACCEL_INSTANCE_TYPE=ml.g5.8xlarge ACCEL_INSTANCE_TYPE=ml.p5.48xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$ACCEL_INSTANCE_TYPE - ``` The instance type label is used as a node selector to ensure the compute pods only run on either the `ml.g5.8xlarge` or `ml.p5.48xlarge` GPU accelerated instances: @@ -378,7 +362,6 @@ kubectl get pvc fsx-claim -n slurm -ojson \ kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ | jq -r .spec.volumeName) -ojson \ | jq -r .spec.csi.volumeHandle - ``` --- @@ -386,7 +369,6 @@ kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ ``` kubectl apply -f openzfs-pvc-slurm.yaml - ``` Verify FSx for OpenZFS PVC creation: @@ -402,7 +384,6 @@ kubectl get pvc openzfs-claim -n slurm -ojson \ kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ | jq -r .spec.volumeName) -ojson \ | jq -r .spec.csi.volumeHandle - ``` FSx for Lustre and OpenZFS PVCs are added to the list of `extraVolumeMounts` and `extraVolumes` for both the login service and compute nodes: @@ -523,7 +504,6 @@ For simplicity of demonstration, we'll use SSH key authentication for root acces Generate an SSH key for root authorization: ``` - export EMAIL_ADDR= ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_slurm -C "${EMAIL_ADDR}" @@ -531,7 +511,6 @@ ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519_slurm -C "${EMAIL_ADDR}" cat ~/.ssh/id_ed25519_slurm.pub # ssh-ed25519 janedoe@example.com - ``` Specify the root SSH authorized key in `values.yaml`: @@ -639,7 +618,6 @@ sinfo PARTITION AVAIL TIMELIMIT NODES STATE NODELIST hp-node up infinite 4 idle hp-node-[0-3] all* up infinite 4 idle hp-node-[0-3] - ``` Note that in both scenarios (using 4 `ml.g5.8xlarge` instances or 2 `ml.p5.48xlarge` instances) we should see the same number of slurm compute nodes. When running on 4 `ml.g5.8xlarge` instances, each slurm compute node is mapped to 1 available A10G GPU, whereas when running on 2 `ml.p5.48xlarge` instances, each slurm compute node is mapped to 4 available H100 GPUs and 16 EFA network interfaces. @@ -700,7 +678,6 @@ nvcc --version # Built on Tue_Oct_29_23:50:19_PDT_2024 # Cuda compilation tools, release 12.6, V12.6.85 # Build cuda_12.6.r12.6/compiler.35059454_0 - ``` --- @@ -826,7 +803,6 @@ kubectl -n slurm exec -it pod/slurm-compute-hp-node-1 -- bash --login # 1 second updates watch -n 1 squeue - ``` Watch checkpoints from `slurm-compute-hp-node-2`: @@ -839,7 +815,6 @@ cd /fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm # highlight changes, show timestamps, 5 second updates watch -n 5 -d "ls -lh checkpoints" - ``` * * * @@ -847,7 +822,6 @@ watch -n 5 -d "ls -lh checkpoints" ### Clean Up: ``` - rm -rf checkpoints/* rm -rf logs/* @@ -879,6 +853,5 @@ eksctl delete iamserviceaccount \ eksctl delete iamserviceaccount \ --name aws-load-balancer-controller \ --namespace kube-system \ - --cluster $EKS_CLUSTER_NAME - + --cluster $EKS_CLUSTER_NAME ``` From 85334a12ae71f77fa7b5de5013cd0186f17ce3df Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Sun, 18 May 2025 22:53:58 -0400 Subject: [PATCH 09/15] updated readme instructions and added lustre storage class file --- .../slinky-slurm/README.md | 226 +++++++++++++----- .../slinky-slurm/lustre-storageclass.yaml | 16 ++ 2 files changed, 187 insertions(+), 55 deletions(-) create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-storageclass.yaml diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 4a6e8f5c0..1c544594f 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -45,23 +45,42 @@ Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.2 ### Set Up the HyperPod Cluster: -Follow the [Prerequisites](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup) and [Cluster Configuration](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster) steps of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). +Deploy the [HyperPod EKS CloudFormation Stack](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/own-account/workshop-infra/02-workshop-infra-cfn). Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. -Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. +To test on g5 capacity using the [g5-values.yaml](./g5/g5-values.yaml) file: +- Set `AcceleratedInstanceType` to `ml.g5.8xlarge` (the default) +- Set`AcceleratedInstanceCount` to `4` -(Optional) Add an access entry (if needed): +To test on p5 capacity using the [p5-values.yaml](./p5/p5-values.yaml) file: +- Set `AcceleratedInstanceType` to `ml.p5.48xlarge` +- Set`AcceleratedInstanceCount` to `2` +In both cases, set the `GeneralPurposeInstanceCount` to 2 + +Run the `create_config.sh` script to set your environment variables using the output of the deployed CloudFormation stack: ``` -export AWS_ACCOUNT_ID= +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) + +export STACK_ID=hyperpod-eks-full-stack + +curl -O https://raw.githubusercontent.com/aws-samples/awsome-distributed-training/refs/heads/main/1.architectures/7.sagemaker-hyperpod-eks/create_config.sh -export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster +chmod +x create_config.sh +./create_config.sh + +source env_vars +``` +Verify that the required environment variables are set: +``` +echo $AWS_ACCOUNT_ID $AWS_REGION $EKS_CLUSTER_NAME $VPC_ID $PRIVATE_SUBNET_ID $SECURITY_GROUP_ID +``` +(Optional) Add an EKS access entry (if needed): +``` export ROLE_ARN=arn:aws:iam::$AWS_ACCOUNT_ID:role/ export PLCY_ARN=arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy -export AWS_REGION=us-west-2 - aws eks create-access-entry \ --cluster-name $EKS_CLUSTER_NAME \ --principal-arn $ROLE_ARN \ @@ -88,20 +107,58 @@ kubectl get nodes ### Create an FSx for Lustre Storage Class: -Follow the [Setup FSx for Lustre File System](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/01-cluster/06-fsx-for-lustre) of the [HyperPod EKS Workshop](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US). +Create an [IAM OpenID Connect (OIDC)](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) identity provider for your cluster: +``` +eksctl utils associate-iam-oidc-provider --cluster $EKS_CLUSTER_NAME --approve +``` +Create a service account with an IAM role mapped to it for use with the FSx for Lustre CSI driver: +``` +eksctl create iamserviceaccount \ + --name fsx-csi-controller-sa \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonFSxFullAccess \ + --approve \ + --role-name FSXLCSI-${EKS_CLUSTER_NAME}-${AWS_REGION} \ + --region $AWS_REGION +``` +Verify proper annotation of the service account with the IAM role ARN: +``` +kubectl get sa fsx-csi-controller-sa -n kube-system -oyaml +``` +Install the [FSx for Lustre CSI Driver](https://github.com/kubernetes-sigs/aws-fsx-csi-driver) using Helm: +``` +helm repo add aws-fsx-csi-driver \ + https://kubernetes-sigs.github.io/aws-fsx-csi-driver + +helm repo update -Verify` fsx-sc` Storage Class: +helm upgrade --install aws-fsx-csi-driver \ + --namespace kube-system \ + --set controller.serviceAccount.create=false \ + aws-fsx-csi-driver/aws-fsx-csi-driver +``` +Verify instillation of the FSx for Lustre CSI driver: +``` +kubectl get pods -n kube-system \ + -l app.kubernetes.io/name=aws-fsx-csi-driver +``` +Create an FSx for Lustre storage class: +``` +envsubst < lustre-storageclass.yaml | kubectl apply -f - +``` +Note: This example uses [envsubst](https://github.com/a8m/envsubst) to inject the `PRIVATE_SUBNET_ID` and `SECURITY_GROUP_ID` environment variables into the storage class Kubernetes manifest. If you don't have envsubst in your development environment, install it by following the [instructions here.](https://github.com/a8m/envsubst?tab=readme-ov-file#installation) +Verify the `fsx-sc` storage class was created: ``` -kubectl get storageclass fsx-sc -oyaml +kubectl get sc fsx-sc -oyaml ``` * * * -### Create an FSx for OpenZFS Storage Class: - -Install the [OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver) following the steps provided below: +### (Optional) Create an FSx for OpenZFS Storage Class: +Create a service account with an IAM role mapped to it for use with the FSx for OpenZFS CSI driver: ``` eksctl create iamserviceaccount \ --name fsx-openzfs-csi-controller-sa \ @@ -111,7 +168,13 @@ eksctl create iamserviceaccount \ --approve \ --role-name FSXOCSI-${EKS_CLUSTER_NAME}-${AWS_REGION} \ --region $AWS_REGION - +``` +Verify proper annotation of the service account with the IAM role ARN: +``` +kubectl get sa fsx-openzfs-csi-controller-sa -n kube-system -oyaml +``` +Install the [FSx for OpenZFS CSI driver](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver) using Helm: +``` helm repo add aws-fsx-openzfs-csi-driver \ https://kubernetes-sigs.github.io/aws-fsx-openzfs-csi-driver @@ -121,19 +184,20 @@ helm upgrade --install aws-fsx-openzfs-csi-driver \ --namespace kube-system \ --set controller.serviceAccount.create=false \ aws-fsx-openzfs-csi-driver/aws-fsx-openzfs-csi-driver - +``` +Verify instillation of the FSx for OpenZFS CSI driver: +``` kubectl get pods -n kube-system \ -l app.kubernetes.io/part-of=aws-fsx-openzfs-csi-driver ``` - -Follow the [Dynamic Provisioning](https://github.com/kubernetes-sigs/aws-fsx-openzfs-csi-driver/tree/main/examples/kubernetes/dynamic-provisioning) guide to create an FSx for OpenZFS Storage Class: - +Create an FSx for OpenZFS Storage Class: ``` -export PRIVATE_SUBNET_ID= -export SECURITY_GROUP_ID= - -kubectl apply -f openzfs-storageclass.yaml +envsubst < openzfs-storageclass.yaml | kubectl apply -f - +``` +Note: This example uses [envsubst](https://github.com/a8m/envsubst) to inject the `PRIVATE_SUBNET_ID` and `SECURITY_GROUP_ID` environment variables into the storage class Kubernetes manifest. If you don't have envsubst in your development environment, install it by following the [instructions here.](https://github.com/a8m/envsubst?tab=readme-ov-file#installation) +Verify the `openzfs-sc` storage class was created: +``` kubectl get sc openzfs-sc -oyaml ``` @@ -143,21 +207,17 @@ kubectl get sc openzfs-sc -oyaml Following the instructions below, which are a consolidation of the full [Install with Helm](https://docs.aws.amazon.com/eks/latest/userguide/lbc-helm.html) instructions found in the Amazon EKS documentation: +Create the IAM policy to give the AWS Load Balancer Controller permission to make calls to AWS APIs on your behalf: ``` -export EKS_CLUSTER_NAME=sagemaker-hyperpod-eks-cluster -export VPC_ID= -export AWS_REGION=us-west-2 -export AWS_ACCOUNT_ID= - -# manually update crds -kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master" - curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.12.0/docs/install/iam_policy.json aws iam create-policy \ --policy-name AWSLoadBalancerControllerIAMPolicy-v2.12.0 \ --policy-document file://iam_policy.json - +``` + +Create a service account with an IAM role mapped to it for use with the AWS Load Balancer Controller: +``` eksctl create iamserviceaccount \ --cluster=$EKS_CLUSTER_NAME \ --namespace=kube-system \ @@ -166,7 +226,13 @@ eksctl create iamserviceaccount \ --override-existing-serviceaccounts \ --region $AWS_REGION \ --approve - +``` +Verify proper annotation of the service account with the IAM role ARN: +``` +kubectl get sa aws-load-balancer-controller -n kube-system -oyaml +``` +Install the AWS Load Balancer Controller using Helm: +``` helm repo add eks https://aws.github.io/eks-charts helm repo update @@ -177,19 +243,34 @@ helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ --set serviceAccount.name=aws-load-balancer-controller \ --set region=$AWS_REGION \ --set vpcId=$VPC_ID - +``` +Verify instillation of the AWS Load Balancer Controller: +``` kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller - -kubectl get sa aws-load-balancer-controller -n kube-system -oyaml ``` * * * -### Instill Slinky Prerequisites (Cert Manager and Prometheus): +### Instill Slinky Prerequisites: + +Follow the steps below to install [cert-manager](https://github.com/cert-manager/cert-manager) and the [Prometheus operator](https://github.com/prometheus-operator/kube-prometheus?tab=readme-ov-file#kube-prometheus): + +``` +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add jetstack https://charts.jetstack.io + +helm repo update + +helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager --create-namespace --set crds.enabled=true -Follow the steps included in the [Slinky QuickStart Guide | Pre-Requisites](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites) section to install Cert Manager and Prometheus. +helm install prometheus prometheus-community/kube-prometheus-stack \ + --namespace prometheus --create-namespace --set installCRDs=true +``` -Verify Pre-Requisites Instillation: +Verify pre-requisite instillation: ``` kubectl get all -n cert-manager @@ -230,7 +311,15 @@ To deploy the slurm cluster, we first need to make some modifications to the [va For your convenience, we've provided [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml) files with most of the configuration changes mentioned below already implemented, so you'll only need to make additional changes as needed to further customize your deployment. -The two things you must minimally modify are the container image that the slurm compute nodes use ([instructions here](#build-and-set-the-compute-node-container-image)) and the root ssh key used for accessing the login node ([instructions here](#login-access)). +The following was tested in two infrastructure scenarios for hosting the compute NodeSet pods: +1. On 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) using the [g5-values.yaml](./g5/g5-values.yaml) file +2. On 2 `p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 using the [p5-values.yaml](./p5/p5-values.yaml) file + +For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. + +The two things you must minimally modify are: +- The container image that the slurm compute nodes use ([instructions here](#build-and-set-the-compute-node-container-image)) +- The root ssh key used for accessing the login node ([instructions here](#login-access)) --- @@ -240,7 +329,7 @@ Clone the Slurm Operator repository, which also contains the Helm chart artifact git clone https://github.com/SlinkyProject/slurm-operator.git ``` -Clone the AWSome Distributed Training repo to use the [values.yaml](./values.yaml) file we've provided: +Clone the AWSome Distributed Training repo to use the [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) file we've provided: ``` git clone https://github.com/aws-samples/awsome-distributed-training.git @@ -262,7 +351,7 @@ export GEN_INSTANCE_TYPE=ml.m5.2xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE ``` -For each non-compute component, we apply both a Node Affinity and a Pod Anti-affinity in [values.yaml](./values.yaml) to ensure they are hosted only on the 2 `m5.2xlarge` instances while also being evenly spread between the hosts. +For each non-compute component, we apply both a Node Affinity and a Pod Anti-affinity in [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml) to ensure they are hosted only on the 2 `m5.2xlarge` instances while also being evenly spread between the hosts. ``` # Inter-pod anti-affinity and node affinity for non-compute components @@ -304,7 +393,7 @@ ACCEL_INSTANCE_TYPE=ml.p5.48xlarge kubectl get nodes -l node.kubernetes.io/instance-type=$ACCEL_INSTANCE_TYPE ``` -The instance type label is used as a node selector to ensure the compute pods only run on either the `ml.g5.8xlarge` or `ml.p5.48xlarge` GPU accelerated instances: +The instance type label is used as a node selector to ensure the compute nodes only run on either the `ml.g5.8xlarge` or `ml.p5.48xlarge` GPU accelerated instances: ``` # for g5 instances @@ -370,9 +459,7 @@ kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ ``` kubectl apply -f openzfs-pvc-slurm.yaml ``` - Verify FSx for OpenZFS PVC creation: - ``` kubectl get pvc -n slurm @@ -386,7 +473,7 @@ kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ | jq -r .spec.csi.volumeHandle ``` -FSx for Lustre and OpenZFS PVCs are added to the list of `extraVolumeMounts` and `extraVolumes` for both the login service and compute nodes: +FSx for Lustre and OpenZFS PVCs are added to the list of `extraVolumeMounts` and `extraVolumes` for both the login service and compute nodes in [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml): ``` login: @@ -435,10 +522,10 @@ Note that for the compute nodes we've also added `/dev/shm` to provide access to #### Configure Compute Node Resources: -Note: limits are required, otherwise the compute nodes will not deploy. + You'll find the compute nodes pre-configured with the following resources: +In [g5-values.yaml](./g5/g5-values.yaml#L544): ``` -# for g5 instances compute: nodesets: - name: hp-node @@ -450,7 +537,9 @@ compute: nvidia.com/gpu: "1" ... -# for p5 instances +``` +In [p5-values.yaml](./p5/p5-values.yaml#L539): +``` compute: nodesets: - name: hp-node @@ -464,7 +553,7 @@ compute: vpc.amazonaws.com/efa: 16 ... ``` -Note that for p5 capacity, we are allocating half the available GPU and EFA network interfaces to each pod so that two pods can run on one instances. This can be adjusted to accomodate other pod topologies. +Note that for p5 capacity, we are allocating half the available GPUs (4 of 8) and EFA network interfaces (16 of 32) to each pod so that two pods can run on one `ml.p5.48xlarge` instances. This can be adjusted to accomodate other pod topologies. --- @@ -821,25 +910,41 @@ watch -n 5 -d "ls -lh checkpoints" ### Clean Up: +(Optional) From the login pod, clear out the checkpoints and logs directories as needed to make room for additional training runs: ``` +cd /fsx/awsome-distributed-training/3.test_cases/pytorch/FSDP/slurm + rm -rf checkpoints/* rm -rf logs/* +exit +``` +Uninstall the Slurm cluster and the Slurm operator: +``` helm uninstall slurm -n slurm helm uninstall slurm-operator -n slinky - +``` +Uninstall the Prometheus operator and cert-manager: +``` helm uninstall prometheus -n prometheus helm uninstall cert-manager -n cert-manager - +``` +Delete the FSx persistent volume claims: +``` kubectl delete pvc fsx-claim -n slurm -kubectl delete pvc openzfs-claim - +kubectl delete pvc openzfs-claim -n slurm +``` +Delete the FSx storage classes: +``` +kubectl delete sc fsx-sc +kubectl delete sc openzfs-sc +``` +Uninstall the FSx CSI drivers and delete the IAM roles mapped to their service accounts: +``` helm uninstall aws-fsx-csi-driver -n kube-system helm uninstall aws-fsx-openzfs-csi-driver -n kube-system -helm uninstall aws-load-balancer-controller -n kube-system - eksctl delete iamserviceaccount \ --name fsx-csi-controller-sa \ --namespace kube-system \ @@ -849,9 +954,20 @@ eksctl delete iamserviceaccount \ --name fsx-openzfs-csi-controller-sa \ --namespace kube-system \ --cluster $EKS_CLUSTER_NAME +``` +Uninstall the AWS Load Balancer Controller and delete the IAM role mapped to its service account: +``` +helm uninstall aws-load-balancer-controller -n kube-system eksctl delete iamserviceaccount \ --name aws-load-balancer-controller \ --namespace kube-system \ --cluster $EKS_CLUSTER_NAME + +aws iam delete-policy --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy-v2.12.0 +``` + +Delete the HyperPod EKS CloudFormation stack: ``` +aws cloudformation delete-stack --stack-name $STACK_ID --region $AWS_REGION +``` \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-storageclass.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-storageclass.yaml new file mode 100644 index 000000000..c73cf1de1 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-storageclass.yaml @@ -0,0 +1,16 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: fsx-sc +provisioner: fsx.csi.aws.com +parameters: + subnetId: ${PRIVATE_SUBNET_ID} + securityGroupIds: ${SECURITY_GROUP_ID} + deploymentType: PERSISTENT_2 + automaticBackupRetentionDays: "0" + copyTagsToBackups: "true" + perUnitStorageThroughput: "250" + dataCompressionType: "LZ4" + fileSystemTypeVersion: "2.15" +mountOptions: + - flock From 752fbedd0c3dd794ef08428d151e69f2ee812d00 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Mon, 19 May 2025 09:13:47 -0400 Subject: [PATCH 10/15] updated README and values.yaml files with latest from SchedMD --- .../slinky-slurm/README.md | 6 +- .../slinky-slurm/g5/g5-values.yaml | 113 +++++++++++------ .../slinky-slurm/p5/p5-values.yaml | 115 ++++++++++++------ 3 files changed, 160 insertions(+), 74 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 1c544594f..8657879bb 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -209,7 +209,7 @@ Following the instructions below, which are a consolidation of the full [Install Create the IAM policy to give the AWS Load Balancer Controller permission to make calls to AWS APIs on your behalf: ``` -curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.12.0/docs/install/iam_policy.json +curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/refs/heads/release-2.13/docs/install/iam_policy.json aws iam create-policy \ --policy-name AWSLoadBalancerControllerIAMPolicy-v2.12.0 \ @@ -602,7 +602,7 @@ cat ~/.ssh/id_ed25519_slurm.pub # ssh-ed25519 janedoe@example.com ``` -Specify the root SSH authorized key in `values.yaml`: +Specify the root SSH authorized key in either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml): ``` login: @@ -615,7 +615,7 @@ login: #### Deploy the Slurm Cluster: -Locally package and deploy the slurm cluster using the modified `values.yaml` file (either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml)): +Locally package and deploy the slurm cluster using either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml): Assuming you are still sitting in the `slinky-slurm` directory of the AWSome Distributed Training repo that we cloned and navigated into earlier, and assuming you cloned the Slinky repo into your home directory (adjust the path as needed), copy the Helm chart artifacts in for packaging: ``` diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml index 32654cd01..bffcd25bb 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml @@ -76,8 +76,8 @@ jwt: hs256: # # -- (string) - # The existing secret to use otherwise one will be generated. - existingSecret: "" + # The existing secret containing the jwt_hs256.key, otherwise one will be generated. + secretName: "" # # Slurm configurations. @@ -87,8 +87,8 @@ slurm: auth: # # -- (string) - # The existing secret to use otherwise one will be generated. - existingSecret: "" + # The existing secret containing the slurm.key, otherwise one will be generated. + secretName: "" # # -- (map[string]string | map[string][]string) # Extra slurmdbd configuration lines to append to `slurmdbd.conf`. @@ -112,7 +112,8 @@ slurm: # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurm.conf.html - extraSlurmConf: {} + extraSlurmConf: + SlurmctldHost: slurm-controller # MinJobAge: 2 # MaxNodeCount: 1024 ### LOGGING ### @@ -244,6 +245,12 @@ controller: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -352,7 +359,7 @@ login: # -- (list) # The `/root/.ssh/authorized_keys` file to write, represented as a list. rootSshAuthorizedKeys: - - "" + - "" # # -- (map) # The `/etc/ssh/sshd_config` file to use, represented as a map. @@ -461,6 +468,12 @@ login: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -555,6 +568,7 @@ compute: # # -- (object) # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -688,20 +702,14 @@ compute: State: UP MaxTime: UNLIMITED # - # -- (string) - # Set Slurm node GRES. - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1 - nodeGres: "" - # - # -- (list) - # Set Slurm node Features as a list(string). - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Features - nodeFeatures: [] - # - # -- (string) - # Set Slurm node weight for Slurm scheduling. - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Weight - nodeWeight: 1 + # --(map[string]string || map[string][]string) + # Extra Slurm node configuration appended into the --conf arguments. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAE + nodeConfig: {} + # Features: [] + # Gres: [] + # Weight: 1 # # -- (list) # Slurm Partitions by object list. @@ -752,6 +760,45 @@ accounting: # Set the image tag to use. tag: 24.11-ubuntu24.04 # + # Configuration for an external database (e.g. mariadb, mysql). + external: + # + # -- (bool) + # Use an external database instead of creating one. + # WARNING: `accounting.external.enabled=true` is mutually exclusive with `mariadb.enabled=true`. + enabled: false + # + # -- (string) + # The database user to use. + user: slurm + # + # -- (string) + # The database context to use. + database: slurm_acct_db + # + # -- (string) + # The plaintext user password to the database. + # NOTE: ignored when secretName is not empty. + password: "" + # + # -- (string) + # The existing secret containing the user password to the database. + secretName: "" + # + # -- (string) + # The host or service to communicate with. + host: "" + # + # -- (integer) + # The database port to use. + port: 3306 + # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -772,21 +819,6 @@ accounting: # limits: # cpu: 2 # memory: 4Gi - # - # Configuration for an external accounting instance (slurmdbd). - external: - # - # -- (bool) - # Use an external acounting instance (slurmdbd) instead of deploying one. - enabled: false - # - # -- (string) - # The external acounting instance (slurmdbd) host. - host: "" - # - # -- (integer) - # The external acounting instance (slurmdbd) port. - port: 6819 # # `bitnami/mariadb` subchart configurations. @@ -873,6 +905,9 @@ mariadb: enabled: false serviceMonitor: enabled: false + nodeSelector: + kubernetes.io/os: linux + affinity: {} resources: {} # @@ -924,6 +959,12 @@ restapi: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -949,7 +990,7 @@ restapi: # `slurm-exporter` subchart configurations. # Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml slurm-exporter: - enabled: true + enabled: false exporter: enabled: true secretName: "slurm-token-exporter" diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml index 54715c271..bcdd6571d 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml @@ -76,8 +76,8 @@ jwt: hs256: # # -- (string) - # The existing secret to use otherwise one will be generated. - existingSecret: "" + # The existing secret containing the jwt_hs256.key, otherwise one will be generated. + secretName: "" # # Slurm configurations. @@ -87,8 +87,8 @@ slurm: auth: # # -- (string) - # The existing secret to use otherwise one will be generated. - existingSecret: "" + # The existing secret containing the slurm.key, otherwise one will be generated. + secretName: "" # # -- (map[string]string | map[string][]string) # Extra slurmdbd configuration lines to append to `slurmdbd.conf`. @@ -112,7 +112,8 @@ slurm: # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurm.conf.html - extraSlurmConf: {} + extraSlurmConf: + SlurmctldHost: slurm-controller # MinJobAge: 2 # MaxNodeCount: 1024 ### LOGGING ### @@ -244,6 +245,12 @@ controller: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -444,6 +451,11 @@ login: persistentVolumeClaim: claimName: openzfs-claim # - name: nfs-home + # persistentVolumeClaim: + # claimName: nfs-home + # - name: nfs-data + # persistentVolumeClaim: + # - name: nfs-home # nfs: # server: nfs-server.example.com # path: /exports/home/ @@ -456,6 +468,12 @@ login: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -552,6 +570,7 @@ compute: # # -- (object) # Set affinity for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} # nodeAffinity: # requiredDuringSchedulingIgnoredDuringExecution: @@ -685,20 +704,14 @@ compute: State: UP MaxTime: UNLIMITED # - # -- (string) - # Set Slurm node GRES. - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Gres_1 - nodeGres: "" - # - # -- (list) - # Set Slurm node Features as a list(string). - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Features - nodeFeatures: [] - # - # -- (string) - # Set Slurm node weight for Slurm scheduling. - # Ref: https://slurm.schedmd.com/slurm.conf.html#OPT_Weight - nodeWeight: 1 + # --(map[string]string || map[string][]string) + # Extra Slurm node configuration appended into the --conf arguments. + # WARNING: Values can override existing ones. + # Ref: https://slurm.schedmd.com/slurm.conf.html#lbAE + nodeConfig: {} + # Features: [] + # Gres: [] + # Weight: 1 # # -- (list) # Slurm Partitions by object list. @@ -749,6 +762,45 @@ accounting: # Set the image tag to use. tag: 24.11-ubuntu24.04 # + # Configuration for an external database (e.g. mariadb, mysql). + external: + # + # -- (bool) + # Use an external database instead of creating one. + # WARNING: `accounting.external.enabled=true` is mutually exclusive with `mariadb.enabled=true`. + enabled: false + # + # -- (string) + # The database user to use. + user: slurm + # + # -- (string) + # The database context to use. + database: slurm_acct_db + # + # -- (string) + # The plaintext user password to the database. + # NOTE: ignored when secretName is not empty. + password: "" + # + # -- (string) + # The existing secret containing the user password to the database. + secretName: "" + # + # -- (string) + # The host or service to communicate with. + host: "" + # + # -- (integer) + # The database port to use. + port: 3306 + # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -769,21 +821,6 @@ accounting: # limits: # cpu: 2 # memory: 4Gi - # - # Configuration for an external accounting instance (slurmdbd). - external: - # - # -- (bool) - # Use an external acounting instance (slurmdbd) instead of deploying one. - enabled: false - # - # -- (string) - # The external acounting instance (slurmdbd) host. - host: "" - # - # -- (integer) - # The external acounting instance (slurmdbd) port. - port: 6819 # # `bitnami/mariadb` subchart configurations. @@ -870,6 +907,8 @@ mariadb: enabled: false serviceMonitor: enabled: false + nodeSelector: + kubernetes.io/os: linux affinity: {} resources: {} @@ -922,6 +961,12 @@ restapi: # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # + # -- (map) + # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector + nodeSelector: + kubernetes.io/os: linux + # # -- (object) # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity @@ -947,7 +992,7 @@ restapi: # `slurm-exporter` subchart configurations. # Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml slurm-exporter: - enabled: true + enabled: false exporter: enabled: true secretName: "slurm-token-exporter" From b03b0f858b0aa06601fc724cdeb7caeaf2f51b6e Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Mon, 23 Jun 2025 07:29:14 -0500 Subject: [PATCH 11/15] updates for Slinky v0.3.0 --- .../slinky-slurm/.gitignore | 4 +- .../slinky-slurm/Docker-Build-README.md | 21 +- .../slinky-slurm/README.md | 210 ++++++++++-------- .../slinky-slurm/dlc-slurmd.Dockerfile | 2 +- .../slinky-slurm/g5/g5-custom.tfvars | 23 ++ .../slinky-slurm/g5/g5-params.json | 74 ++++++ .../slinky-slurm/g5/g5-values.yaml | 97 ++++---- .../slinky-slurm/p5/p5-custom.tfvars | 23 ++ .../slinky-slurm/p5/p5-params.json | 74 ++++++ .../slinky-slurm/p5/p5-values.yaml | 97 ++++---- 10 files changed, 448 insertions(+), 177 deletions(-) create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-custom.tfvars create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-params.json create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-custom.tfvars create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-params.json diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore index b6d262826..3e04c7d79 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/.gitignore @@ -1,3 +1,5 @@ slurm*/ -*.tgz \ No newline at end of file +*.tgz + +new-values.yaml \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md index a0b54dec3..1687056af 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md @@ -4,7 +4,7 @@ This build includes Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EF Clone the AWSome Distributed Training repo: ``` -https://github.com/aws-samples/awsome-distributed-training.git +git clone https://github.com/aws-samples/awsome-distributed-training.git cd awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/ ``` @@ -19,10 +19,10 @@ aws ecr get-login-password --region us-east-1 \ --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com # on a Mac -docker buildx build --platform linux/amd64 -t dlc-slurmd:24.11.4-ubuntu24.04 -f dlc-slurmd.Dockerfile . +docker buildx build --platform linux/amd64 -t dlc-slurmd:25.05.0-ubuntu24.04 -f dlc-slurmd.Dockerfile . # on Linux -# docker build -t dlc-slurmd:24.11.4-ubuntu24.04 -f dlc-slurmd.Dockerfile . +# docker build -t dlc-slurmd:25.05.0-ubuntu24.04 -f dlc-slurmd.Dockerfile . ``` @@ -32,7 +32,7 @@ Verify Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 ``` -docker run --platform linux/amd64 -it --entrypoint=/bin/bash dlc-slurmd:24.11.4-ubuntu24.04 +docker run --platform linux/amd64 -it --entrypoint=/bin/bash dlc-slurmd:25.05.0-ubuntu24.04 python3 --version # Python 3.12.8 @@ -63,6 +63,7 @@ cat /etc/nccl.conf # NCCL_DEBUG=INFO # NCCL_SOCKET_IFNAME=^docker0 +exit ``` Create a private ECR repo: @@ -76,7 +77,7 @@ aws ecr create-repository --repository-name dlc-slurmd Authenticate to the repo: ``` -export AWS_ACCOUNT_ID= +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) export AWS_REGION= aws ecr get-login-password --region $AWS_REGION \ @@ -89,8 +90,8 @@ Tag the image: ``` -docker tag dlc-slurmd:24.11.4-ubuntu24.04 \ - ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 +docker tag dlc-slurmd:25.05.0-ubuntu24.04 \ + ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:25.05.0-ubuntu24.04 ``` @@ -98,7 +99,7 @@ Push the image to an ECR repo: ``` -docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 +docker push ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:25.05.0-ubuntu24.04 ``` @@ -107,7 +108,7 @@ Test ECR access: ``` kubectl run test-pod \ - --image=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04 \ + --image=${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:25.05.0-ubuntu24.04 \ --restart=Never \ --image-pull-policy=Always @@ -135,7 +136,7 @@ kubectl -n slurm patch nodeset.slinky.slurm.net \ $NODESET_NAME \ --type='json' \ -p="[ - {\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/image\", \"value\":\"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:24.11.4-ubuntu24.04\"}, + {\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/image\", \"value\":\"${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:25.05.0-ubuntu24.04\"}, {\"op\": \"replace\", \"path\": \"/spec/template/spec/containers/0/imagePullPolicy\", \"value\":\"Always\"} ]" diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index 8657879bb..eaea30710 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -2,14 +2,14 @@ ### What is the Slinky Project? -The [Slinky Project](https://github.com/SlinkyProject/slurm-operator/tree/main) is an open-source solution maintained by SchedMD (the main developer of Slurm) that deploys Slurm on Kubernetes. When paired with HyperPod EKS, the Slinky Project unlocks the ability for enterprises who have standardized infrastructure management on Kubernetes to deliver a Slurm-based experience to their ML scientists. It also enables training, experimentation, and inference to happen on the same cluster of accelerated nodes with the build-in resiliency provided by HyperPod. +The [Slinky Project](https://github.com/SlinkyProject/slurm-operator/tree/main) is an open-source solution maintained by SchedMD (the main developers of Slurm) that deploys Slurm on Kubernetes. When paired with HyperPod EKS, the Slinky Project unlocks the ability for enterprises who have standardized infrastructure management on Kubernetes to deliver a Slurm-based experience to their ML scientists. It also enables training, experimentation, and inference to happen on the same cluster of accelerated nodes with the build-in resiliency provided by HyperPod. --- ### Slinky on HypePod EKS Architecture ![Image Description](./slinky-slurm-hp-eks.png) -The diagram above depicts the resulting proof-of-concept deployment outlined in this guide. An Amazon EKS cluster acts as an orchestration layer, while a HyperPod cluster deliver a resilient instance group of GPU accelerated compute nodes. The Slinky Slurm operator is installed to extend Kubernetes with custom resources and actions, and a containerized Slurm cluster is deployed using Kubernetes pods via Helm chart. This Slurm cluster includes the following components: +The diagram above depicts the resulting proof-of-concept deployment outlined in this guide. An Amazon EKS cluster acts as an orchestration layer, while a HyperPod cluster delivers a resilient instance group of GPU accelerated compute nodes. The Slinky Slurm operator is installed to extend Kubernetes with custom resources and actions, and a containerized Slurm cluster is deployed using Kubernetes pods via Helm chart. This Slurm cluster includes the following components: | Component | Description | |-----------|-------------| | Controller (slurmctld) | The central management daemon that monitors resources, accepts jobs, and assigns work to compute nodes. | @@ -30,14 +30,12 @@ The login and compute node pods also have FSx for Lustre and FSx for OpenZFS sha ### Release Notes The following was tested in two infrastructure scenarios for hosting the compute NodeSet pods: -1. On 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) -2. On 2 `p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 +1. On 4 `ml.g5.8xlarge` instances (1 A10G Tensor Core GPU each) +2. On 2 `ml.p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 -For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. +For simplicity, 2 `ml.m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. -Testing used [Slurm Operator v0.2.1](https://github.com/slinkyproject/slurm-operator/pkgs/container/slurm-operator) (pulled as OCI artifacts from the Slinky container registry) and [Slurm Cluster v0.3.0](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) (packaged and deployed locally using the main branch of the Slinky git repository) in order to include the NoteSet volume mount and Login Pod features. These features are expected to be included in the official Slurm Cluster v0.3.0 release when it becomes available, along with a new version of the Slurm Operator with corresponding validating webhooks. - -Note that the [Slinky Project](https://github.com/SlinkyProject) is under active development and could introduce breaking changes that may require modified deployment and configuration steps. +Testing used [Slurm Operator v0.3.0](https://github.com/orgs/slinkyproject/packages/container/package/charts/slurm-operator) and [Slurm Cluster v0.3.0](https://github.com/orgs/slinkyproject/packages/container/package/charts/slurm) Helm charts pulled as OCI artifacts from the Slinky container registry. Slinky v0.3.0 includes the NoteSet volume mount and Login Pod features. Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EFA Installer 1.38.0 (bundled with OFI NCCL plugin) pre-installed in the container image. See the [Docker Build for the Slurmd Deep Learning Container](./Docker-Build-README.md) for details. @@ -45,23 +43,45 @@ Worker pods were built with Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.2 ### Set Up the HyperPod Cluster: -Deploy the [HyperPod EKS CloudFormation Stack](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/own-account/workshop-infra/02-workshop-infra-cfn). Be sure to modify the Accelerated and General Purpose instance groups as needed to deploy the desired instance type and number of nodes. +Deploy the [HyperPod EKS CloudFormation Stack](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/00-workshop-infra-cfn) or the [HyperPod EKS Terraform Modules](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/01-workshop-infra-tf) using the provided configurations below. -To test on g5 capacity using the [g5-values.yaml](./g5/g5-values.yaml) file: -- Set `AcceleratedInstanceType` to `ml.g5.8xlarge` (the default) -- Set`AcceleratedInstanceCount` to `4` +#### Clone the AWSome Distributed Training Repo +``` +git clone https://github.com/aws-samples/awsome-distributed-training.git +cp -r awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm . +cd slinky-slurm +``` -To test on p5 capacity using the [p5-values.yaml](./p5/p5-values.yaml) file: -- Set `AcceleratedInstanceType` to `ml.p5.48xlarge` -- Set`AcceleratedInstanceCount` to `2` +#### Deploy Using CloudFormation -In both cases, set the `GeneralPurposeInstanceCount` to 2 +Use one of the provided `*-params.json` files to set the CloudFormation stack parameters. +For using 4 `ml.g5.8xlarge` instances: +``` +export PARAMS="g5/g5-params.json" +``` +For using 2 `ml.p5.48xlarge` instances: +``` +export PARAMS="p5/p5-params.json" +``` +Curl the `main-stack.yaml` template and issue an `aws cloudformation create-stack` command to deploy the specified HyperPod cluster infrastructure: +``` +export AWS_REGION= # e.g. us-west-2 + +curl -O https://raw.githubusercontent.com/aws-samples/awsome-distributed-training/refs/heads/main/1.architectures/7.sagemaker-hyperpod-eks/cfn-templates/nested-stacks/main-stack.yaml + +aws cloudformation create-stack \ +--stack-name hp-eks-slinky-stack \ +--template-body file://main-stack.yaml \ +--region $AWS_REGION \ + --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM \ +--parameters file://$PARAMS +``` Run the `create_config.sh` script to set your environment variables using the output of the deployed CloudFormation stack: ``` export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -export STACK_ID=hyperpod-eks-full-stack +export STACK_ID=hp-eks-slinky-stack curl -O https://raw.githubusercontent.com/aws-samples/awsome-distributed-training/refs/heads/main/1.architectures/7.sagemaker-hyperpod-eks/create_config.sh @@ -71,6 +91,49 @@ chmod +x create_config.sh source env_vars ``` +--- + +#### Deploy Using Terraform + +Copy the Terraform modules: +``` +cd .. +cp -r awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/terraform-modules . +cd terraform-modules/hyperpod-eks-tf +``` +Use one of the provided `*-custom.tfvars` files to set the Terraform Module parameters. + +For using 4 `ml.g5.8xlarge` instances: +``` +cp ../../slinky-slurm/g5/g5-custom.tfvars . +export PARAMS="g5-custom.tfvars" +``` +For using 2 `ml.p5.48xlarge` instances: +``` +cp ../../slinky-slurm/p5/p5-custom.tfvars . +export PARAMS="p5-custom.tfvars" +``` +Initialize the Terraform modules: +``` +terraform init +``` +Generate an execution plan to validate the configuration of the Terraform modules: +``` +terraform plan -var-file=$PARAMS +``` +Apply the Terraform modules to deploy the specified HyperPod cluster infrastructure: +``` +terraform apply -var-file=$PARAMS +``` +Run the `terraform_outputs.sh` script, which populates the `env_vars.sh` script with your environment variables: +``` +cd .. +chmod +x terraform_outputs.sh +./terraform_outputs.sh +cat env_vars.sh +source env_vars.sh +``` +--- Verify that the required environment variables are set: ``` echo $AWS_ACCOUNT_ID $AWS_REGION $EKS_CLUSTER_NAME $VPC_ID $PRIVATE_SUBNET_ID $SECURITY_GROUP_ID @@ -281,20 +344,18 @@ Verify pre-requisite instillation: ### Install the Slurm Operator: -For [Slurm Operator](https://github.com/SlinkyProject/slurm-operator/blob/main/docs/quickstart.md#pre-requisites) Installation, we'll install release v0.2.1, which is the latest release available at the time of testing. - - Note: We will locally build and deploy a pre-release v0.3.0 of the [Slurm Cluster](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm) from the main branch of the Slinky Project repository. The project is being actively developed, so there is a risk of pulling down breaking changes, but it includes the features to [add additional volume mounts to compute NodeSets](https://github.com/SlinkyProject/slurm-operator/commit/b0e111b0a8434e38b5fb37a2051e7525d5679319) and [deploy Login Pods](https://github.com/SlinkyProject/slurm-operator/commit/37f020f041556164b9c935f799b51df65d22aefe). +Install the [Slurm Operator](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm-operator#slurm-operator) release v0.3.0, the latest release available at the time of testing, along with the default `values-operator.yaml` file provided by SchedMD: ``` -curl -L https://raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.2.1/helm/slurm-operator/values.yaml \ - -o values-operator-0.2.1.yaml +curl -L https://raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.3.0/helm/slurm-operator/values.yaml \ + -o values-operator.yaml # Delete any stale crds (if you deployed an older version) kubectl delete crd clusters.slinky.slurm.net kubectl delete crd nodesets.slinky.slurm.net helm install slurm-operator oci://ghcr.io/slinkyproject/charts/slurm-operator \ - --values=values-operator-0.2.1.yaml --version=0.2.1 --namespace=slinky --create-namespace + --values=values-operator.yaml --version=0.3.0 --namespace=slinky --create-namespace ``` Verify Slurm Operator Instillation: @@ -307,42 +368,22 @@ kubectl get all -n slinky ### Install the Slurm Cluster: -To deploy the slurm cluster, we first need to make some modifications to the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml)` file. After that, in order to test the latest changes in release v0.3.0, we’ll locally package and deploy the helm chart from the main branch of the cloned repo. +To deploy the slurm cluster, we first need to make some modifications to the default [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/release-0.3/helm/slurm/values.yaml)` file. For your convenience, we've provided [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml) files with most of the configuration changes mentioned below already implemented, so you'll only need to make additional changes as needed to further customize your deployment. The following was tested in two infrastructure scenarios for hosting the compute NodeSet pods: -1. On 4 `g5.8xlarge` instances (1 A10G Tensor Core GPU each) using the [g5-values.yaml](./g5/g5-values.yaml) file -2. On 2 `p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 using the [p5-values.yaml](./p5/p5-values.yaml) file +1. On 4 `ml.g5.8xlarge` instances (1 A10G Tensor Core GPU each) using the [g5-values.yaml](./g5/g5-values.yaml) file +2. On 2 `ml.p5.48xlarge` instances (8 H100 Tensor Core GPUs each) with EFAv2 using the [p5-values.yaml](./p5/p5-values.yaml) file -For simplicity, 2 `m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. +For simplicity, 2 `ml.m5.2xlarge` instances were also allocated for separately hosting other components like the Controller and Login pods. You can adjust the number and type of instances associated with your HyperPod cluster, as well as the component affinity rules in the respective [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files to modify how they are spread across your nodes. The two things you must minimally modify are: - The container image that the slurm compute nodes use ([instructions here](#build-and-set-the-compute-node-container-image)) - The root ssh key used for accessing the login node ([instructions here](#login-access)) - ---- - -#### Clone the Repos -Clone the Slurm Operator repository, which also contains the Helm chart artifacts for the Slurm Cluster: -``` -git clone https://github.com/SlinkyProject/slurm-operator.git -``` - -Clone the AWSome Distributed Training repo to use the [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) file we've provided: -``` -git clone https://github.com/aws-samples/awsome-distributed-training.git - -cd awsome-distributed-training/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm -``` - -(Optional) If you wish to start from scratch, open the [values.yaml](https://github.com/SlinkyProject/slurm-operator/blob/dd65faba359702a8eda6cce9484b702f2fd2ae2e/helm/slurm/values.yaml) file associated with the Slurm Cluster Helm Chart: -``` -code slurm-operator/helm/slurm/values.yaml -``` --- -#### Component Affinity: +#### Review Component Affinity: Verify the existence of the instance type label for non-compute component affinity: @@ -379,7 +420,7 @@ You can modify this common affinity setting, or apply unique affinity settings f --- -#### Compute Node Selector: +#### Review Compute Node Selector: Verify the existence of the instance type label for compute node selector: @@ -432,7 +473,7 @@ Create the slurm namespace: kubectl create ns slurm ``` -This is needed to reference for node volume mounts later. +Create a PVC named `fsx-claim` in the slurm namespace: ``` kubectl apply -f lustre-pvc-slurm.yaml @@ -454,8 +495,9 @@ kubectl get pv $(kubectl get pvc fsx-claim -n slurm -ojson \ ``` --- -#### Create an FSx for OpenZFS PVC in the slurm namespace: +#### (Optional) Create an FSx for OpenZFS PVC in the slurm namespace: +Create a PVC named `openzfs-claim` in the slurm namespace: ``` kubectl apply -f openzfs-pvc-slurm.yaml ``` @@ -472,7 +514,8 @@ kubectl get pv $(kubectl get pvc openzfs-claim -n slurm -ojson \ | jq -r .spec.volumeName) -ojson \ | jq -r .spec.csi.volumeHandle ``` - +--- +#### Review Volume Mounts: FSx for Lustre and OpenZFS PVCs are added to the list of `extraVolumeMounts` and `extraVolumes` for both the login service and compute nodes in [g5-values.yaml](./g5/g5-values.yaml) and [p5-values.yaml](./p5/p5-values.yaml): ``` @@ -520,7 +563,7 @@ Note that for the compute nodes we've also added `/dev/shm` to provide access to --- -#### Configure Compute Node Resources: +#### Review Compute Node Configuration: You'll find the compute nodes pre-configured with the following resources: @@ -561,7 +604,7 @@ Note that for p5 capacity, we are allocating half the available GPUs (4 of 8) an Use the provided [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) to build a [Slurmd Deep Learning Container](./Docker-Build-README.md) (Slurmd DLC), following [the instructions here](./Docker-Build-README.md). -then modify the compute node container image to use your Slurmd DLC build: +then modify the compute node container image to use your Slurmd DLC build in either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml): ``` compute: @@ -577,7 +620,7 @@ compute: # # -- (string) # Set the image tag to use. - tag: "24.11.4-ubuntu24.04" + tag: "25.05.0-ubuntu24.04" ... ``` The Slurm DLC has Python 3.12.8 + PyTorch 2.6.0 + CUDA 12.6 + NCCL 2.23.4 + EFA Installer 1.38.0 (bundled with OFI NCCL plugin) pre-installed in the container image, but you can modify the [dlc-slurmd.Dockerfile](./dlc-slurmd.Dockerfile) for further customization. @@ -615,41 +658,25 @@ login: #### Deploy the Slurm Cluster: -Locally package and deploy the slurm cluster using either [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml): - -Assuming you are still sitting in the `slinky-slurm` directory of the AWSome Distributed Training repo that we cloned and navigated into earlier, and assuming you cloned the Slinky repo into your home directory (adjust the path as needed), copy the Helm chart artifacts in for packaging: -``` -cp -r ~/slurm-operator/helm/slurm . -``` - -Locally package the Slurm cluster Helm chart v0.3.0: - -``` -helm dependency update slurm +Install the [Slurm Cluster](https://github.com/SlinkyProject/slurm-operator/tree/main/helm/slurm#slurm) release v0.3.0, the latest release available at the time of testing, along with one of the custom [g5-values.yaml](./g5/g5-values.yaml) or [p5-values.yaml](./p5/p5-values.yaml) files privided: -helm package slurm -``` **Option 1**: Deploy the Slurm cluster on `ml.g5.8xlarge` instances: ``` # Dry run -helm install --dry-run slurm slurm-0.3.0.tgz \ --f g5/g5-values.yaml \ --n slurm +helm install --dry-run slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values=g5/g5-values.yaml --version=0.3.0 --namespace=slurm -helm install slurm slurm-0.3.0.tgz \ --f g5/g5-values.yaml \ --n slurm +helm install slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values=g5/g5-values.yaml --version=0.3.0 --namespace=slurm ``` **Option 2**: Deploy the Slurm cluster on `ml.p5.48xlarge` instances: ``` # Dry run -helm install --dry-run slurm slurm-0.3.0.tgz \ --f p5/p5-values.yaml \ --n slurm +helm install --dry-run slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values=p5/p5-values.yaml --version=0.3.0 --namespace=slurm -helm install slurm slurm-0.3.0.tgz \ --f p5/p5-values.yaml \ --n slurm +helm install slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values=p5/p5-values.yaml --version=0.3.0 --namespace=slurm ``` Watch the deployment status of the Slurm cluster: @@ -666,14 +693,21 @@ kubectl get all -n slurm --- -#### Configure Login Network Load Balancer provisioning using the AWS Load Balancer Controller: +#### Configure a Login Network Load Balancer using the AWS Load Balancer Controller: + +Identify two public subnets in your VPC to reference. An Elastic Network Interface (ENI) will be provisioned in each of these subnets to act as entry points for traffic into your `slurm-login` service. + +If you used the [default VPC configuration](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/02-additional-info#default-vpc-networking-architecture) provided in the [HyperPod EKS CloudFormation Stack](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/00-workshop-infra-cfn) or the [HyperPod EKS Terraform Modules](https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/01-workshop-infra-tf), two public subnets were provisioned for you, and you can use the following commands to set environment variables to reference them: +``` +export PUBLIC_SUBNET_ID_1=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=map-public-ip-on-launch,Values=true" --query "Subnets[0].SubnetId" --output text) -Manually add annotation to the `slurm-login` service: +export PUBLIC_SUBNET_ID_2=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=map-public-ip-on-launch,Values=true" --query "Subnets[1].SubnetId" --output text) +echo $PUBLIC_SUBNET_ID_1 $PUBLIC_SUBNET_ID_2 ``` -export PUBLIC_SUBNET_ID_1= -export PUBLIC_SUBNET_ID_2= +Add annotations to the `slurm-login` service to make it internet facing using the public subnets: +``` kubectl annotate service slurm-login -n slurm \ service.beta.kubernetes.io/aws-load-balancer-type="nlb" \ service.beta.kubernetes.io/aws-load-balancer-scheme="internet-facing" \ @@ -685,7 +719,7 @@ kubectl annotate service slurm-login -n slurm \ kubectl describe service slurm-login -n slurm ``` -Any annotations added to the slurm cluster `values.yaml` file for the slurm-login service are currently ignored, but AWS Load Balancer Controller actively watches for and implements annotation changes. It Automatically adds inbound rules to the node security group to allow traffic from the NLB security group on the target port (22 in this case). +The AWS Load Balancer Controller actively watches for and implements annotation changes. It Automatically adds inbound rules to the node security group to allow traffic from the NLB security group on the target port (22 in this case). --- @@ -695,6 +729,7 @@ SSH into the login node as root from the NLB endpoint: ``` SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" + ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME ``` --- @@ -788,7 +823,7 @@ find /usr/local/lib/ -name "nccl.h" 2>/dev/null ``` --- -For p5 capacity, check EFA availability: +Check EFA availability: ``` ls /sys/class/infiniband/ fi_info -p efa @@ -806,7 +841,7 @@ Verify intra-node GPU topology: ``` nvidia-smi topo -m ``` -The GPU topology should show all GPUs are connected via NVLink (NV18 indicates 18 NVLink connections). +For `ml.p5.48xlarge` instances, the GPU topology should show all GPUs are connected via NVLink (NV18 indicates 18 NVLink connections). The GPUs are split across two NUMA nodes (0-3 on NUMA 0, 4-7 on NUMA 1). --- @@ -817,6 +852,7 @@ SSH into the login pod as root, clone the repo, and create a checkpoints directo ``` SLURM_LOGIN_HOSTNAME="$(kubectl get services -n slurm -l app.kubernetes.io/instance=slurm,app.kubernetes.io/name=login -o jsonpath="{.items[0].status.loadBalancer.ingress[0].hostname}")" + ssh -i ~/.ssh/id_ed25519_slurm -p 22 root@$SLURM_LOGIN_HOSTNAME # install git diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile index d3181bf62..7c2b3ab34 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile @@ -2,7 +2,7 @@ FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:2.6.0-gpu-py312-cu126-ubuntu22.04-ec2 AS dlc # Second stage - Slurm compute node -FROM ghcr.io/slinkyproject/slurmd:24.11.4-ubuntu24.04 +FROM ghcr.io/slinkyproject/slurmd:25.05.0-ubuntu24.04 ARG PYTHON_SHORT_VERSION=3.12 diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-custom.tfvars b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-custom.tfvars new file mode 100644 index 000000000..6df39213c --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-custom.tfvars @@ -0,0 +1,23 @@ +kubernetes_version = "1.32" +eks_cluster_name = "slinky-eks-cluster" +hyperpod_cluster_name = "slinky-hp-cluster" +resource_name_prefix = "slinky-hp-eks" +availability_zone_id = "usw2-az2" +instance_groups = { + accelerated-instance-group-1 = { + instance_type = "ml.g5.8xlarge", + instance_count = 4, + ebs_volume_size = 500, + threads_per_core = 2, + enable_stress_check = true, + enable_connectivity_check = true, + lifecycle_script = "on_create.sh" + }, + general-instance-group-2 = { + instance_type = "ml.m5.2xlarge", + instance_count = 2, + ebs_volume_size = 500, + threads_per_core = 1, + lifecycle_script = "on_create.sh" + } +} \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-params.json b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-params.json new file mode 100644 index 000000000..c9ef08a19 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-params.json @@ -0,0 +1,74 @@ +[ + { + "ParameterKey": "KubernetesVersion", + "ParameterValue": "1.32" + }, + { + "ParameterKey": "EKSClusterName", + "ParameterValue": "slinky-eks-cluster" + }, + { + "ParameterKey": "HyperPodClusterName", + "ParameterValue": "slinky-hp-cluster" + }, + { + "ParameterKey": "ResourceNamePrefix", + "ParameterValue": "slinky-hp-eks" + }, + { + "ParameterKey": "AvailabilityZoneId", + "ParameterValue": "usw2-az2" + }, + { + "ParameterKey": "AcceleratedInstanceGroupName", + "ParameterValue": "accelerated-instance-group-1" + }, + { + "ParameterKey": "AcceleratedInstanceType", + "ParameterValue": "ml.g5.8xlarge" + }, + { + "ParameterKey": "AcceleratedInstanceCount", + "ParameterValue": "4" + }, + { + "ParameterKey": "AcceleratedEBSVolumeSize", + "ParameterValue": "500" + }, + { + "ParameterKey": "AcceleratedThreadsPerCore", + "ParameterValue": "2" + }, + { + "ParameterKey": "EnableInstanceStressCheck", + "ParameterValue": "true" + }, + { + "ParameterKey": "EnableInstanceConnectivityCheck", + "ParameterValue": "true" + }, + { + "ParameterKey": "CreateGeneralPurposeInstanceGroup", + "ParameterValue": "true" + }, + { + "ParameterKey": "GeneralPurposeInstanceGroupName", + "ParameterValue": "general-instance-group-2" + }, + { + "ParameterKey": "GeneralPurposeInstanceType", + "ParameterValue": "ml.m5.2xlarge" + }, + { + "ParameterKey": "GeneralPurposeInstanceCount", + "ParameterValue": "2" + }, + { + "ParameterKey": "GeneralPurposeEBSVolumeSize", + "ParameterValue": "500" + }, + { + "ParameterKey": "GeneralPurposeThreadsPerCore", + "ParameterValue": "1" + } +] \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml index bffcd25bb..20434bd5f 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml @@ -112,8 +112,7 @@ slurm: # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurm.conf.html - extraSlurmConf: - SlurmctldHost: slurm-controller + extraSlurmConf: {} # MinJobAge: 2 # MaxNodeCount: 1024 ### LOGGING ### @@ -134,6 +133,16 @@ slurm: # # Ref: https://slurm.schedmd.com/acct_gather.conf.html # burst_buffer.conf: | # # Ref: https://slurm.schedmd.com/burst_buffer.conf.html + # + # Ref: https://slurm.schedmd.com/cgroup.conf.html + cgroup.conf: | + CgroupPlugin=autodetect + IgnoreSystemd=yes + EnableControllers=yes + ConstrainCores=yes + ConstrainRAMSpace=yes + ConstrainDevices=yes + ConstrainSwapSpace=yes # gres.conf: | # # Ref: https://slurm.schedmd.com/gres.conf.html # helpers.conf: | @@ -191,7 +200,7 @@ authcred: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # Set container resource requests and limits for Kubernetes Pod scheduling. @@ -221,7 +230,7 @@ controller: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The controller service configuration. @@ -337,7 +346,7 @@ login: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The login service configuration. @@ -359,7 +368,7 @@ login: # -- (list) # The `/root/.ssh/authorized_keys` file to write, represented as a list. rootSshAuthorizedKeys: - - "" + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgiXVQ3l7+huQ2vC6gvpTqd94YljwjneCdH/irPNc1d natharno@amazon.com # # -- (map) # The `/etc/ssh/sshd_config` file to use, represented as a map. @@ -427,6 +436,15 @@ login: pam: {} # debug_level: 9 # + # -- (object) + # The security context given to the container. + # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: + privileged: false + # capabilities: + # add: + # - SYS_CHROOT + # # --(list) # List of volume mounts. # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ @@ -513,7 +531,7 @@ compute: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (list) # Slurm NodeSets by object list. @@ -540,28 +558,20 @@ compute: # # -- (string) # Set the image repository to use. - repository: ".dkr.ecr..amazonaws.com/dlc-slurmd" + repository: "659924747436.dkr.ecr.us-west-2.amazonaws.com/dlc-slurmd" # # -- (string) # Set the image tag to use. - tag: "24.11.4-ubuntu24.04" + tag: "25.05.0-ubuntu24.04" # # -- (string) # Set the priority class to use. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # - # -- (object) - # Set container resource requests and limits for Kubernetes Pod scheduling. - # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container - resources: - limits: - nvidia.com/gpu: "1" - requests: - nvidia.com/gpu: "1" - # # -- (map) # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector nodeSelector: kubernetes.io/os: linux node.kubernetes.io/instance-type: ml.g5.8xlarge @@ -570,25 +580,21 @@ compute: # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: "kubernetes.io/os" - # operator: In - # values: - # - linux + # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - topologyKey: "kubernetes.io/hostname" - # labelSelector: - # matchExpressions: - # - key: "app.kubernetes.io/name" - # operator: In - # values: - # - slurmctld - # - slurmdbd - # - slurmrestd + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # topologyKey: kubernetes.io/hostname + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - slurmctld + # - slurmdbd + # - slurmrestd + # - mariadb # # -- (list) # Configure pod tolerations. @@ -596,6 +602,19 @@ compute: tolerations: [] # # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: + limits: + nvidia.com/gpu: "1" + requests: + nvidia.com/gpu: "1" + # + # -- (bool) + # Enable to propagate the pod `resources.limits` into slurmd. + useResourceLimits: true + # + # -- (object) # Set the update strategy configuration. updateStrategy: # @@ -758,7 +777,7 @@ accounting: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # Configuration for an external database (e.g. mariadb, mysql). external: @@ -935,7 +954,7 @@ restapi: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The restapi service configuration. @@ -990,7 +1009,7 @@ restapi: # `slurm-exporter` subchart configurations. # Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml slurm-exporter: - enabled: false + enabled: true exporter: enabled: true secretName: "slurm-token-exporter" diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-custom.tfvars b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-custom.tfvars new file mode 100644 index 000000000..26b02ed4e --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-custom.tfvars @@ -0,0 +1,23 @@ +kubernetes_version = "1.32" +eks_cluster_name = "slinky-eks-cluster" +hyperpod_cluster_name = "slinky-hp-cluster" +resource_name_prefix = "slinky-hp-eks" +availability_zone_id = "usw2-az2" +instance_groups = { + accelerated-instance-group-1 = { + instance_type = "ml.p5.48xlarge", + instance_count = 2, + ebs_volume_size = 500, + threads_per_core = 2, + enable_stress_check = true, + enable_connectivity_check = true, + lifecycle_script = "on_create.sh" + }, + general-instance-group-2 = { + instance_type = "ml.m5.2xlarge", + instance_count = 2, + ebs_volume_size = 500, + threads_per_core = 1, + lifecycle_script = "on_create.sh" + } +} \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-params.json b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-params.json new file mode 100644 index 000000000..f9ebb6cb2 --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-params.json @@ -0,0 +1,74 @@ +[ + { + "ParameterKey": "KubernetesVersion", + "ParameterValue": "1.32" + }, + { + "ParameterKey": "EKSClusterName", + "ParameterValue": "slinky-eks-cluster" + }, + { + "ParameterKey": "HyperPodClusterName", + "ParameterValue": "slinky-hp-cluster" + }, + { + "ParameterKey": "ResourceNamePrefix", + "ParameterValue": "slinky-hp-eks" + }, + { + "ParameterKey": "AvailabilityZoneId", + "ParameterValue": "usw2-az2" + }, + { + "ParameterKey": "AcceleratedInstanceGroupName", + "ParameterValue": "accelerated-instance-group-1" + }, + { + "ParameterKey": "AcceleratedInstanceType", + "ParameterValue": "ml.p5.48xlarge" + }, + { + "ParameterKey": "AcceleratedInstanceCount", + "ParameterValue": "2" + }, + { + "ParameterKey": "AcceleratedEBSVolumeSize", + "ParameterValue": "500" + }, + { + "ParameterKey": "AcceleratedThreadsPerCore", + "ParameterValue": "2" + }, + { + "ParameterKey": "EnableInstanceStressCheck", + "ParameterValue": "true" + }, + { + "ParameterKey": "EnableInstanceConnectivityCheck", + "ParameterValue": "true" + }, + { + "ParameterKey": "CreateGeneralPurposeInstanceGroup", + "ParameterValue": "true" + }, + { + "ParameterKey": "GeneralPurposeInstanceGroupName", + "ParameterValue": "general-instance-group-2" + }, + { + "ParameterKey": "GeneralPurposeInstanceType", + "ParameterValue": "ml.m5.2xlarge" + }, + { + "ParameterKey": "GeneralPurposeInstanceCount", + "ParameterValue": "2" + }, + { + "ParameterKey": "GeneralPurposeEBSVolumeSize", + "ParameterValue": "500" + }, + { + "ParameterKey": "GeneralPurposeThreadsPerCore", + "ParameterValue": "1" + } +] \ No newline at end of file diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml index bcdd6571d..7d1c93c3f 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/p5/p5-values.yaml @@ -112,8 +112,7 @@ slurm: # Extra slurm configuration lines to append to `slurm.conf`, represetned as a string or a map. # WARNING: Values can override existing ones. # Ref: https://slurm.schedmd.com/slurm.conf.html - extraSlurmConf: - SlurmctldHost: slurm-controller + extraSlurmConf: {} # MinJobAge: 2 # MaxNodeCount: 1024 ### LOGGING ### @@ -134,6 +133,16 @@ slurm: # # Ref: https://slurm.schedmd.com/acct_gather.conf.html # burst_buffer.conf: | # # Ref: https://slurm.schedmd.com/burst_buffer.conf.html + # + # Ref: https://slurm.schedmd.com/cgroup.conf.html + cgroup.conf: | + CgroupPlugin=autodetect + IgnoreSystemd=yes + EnableControllers=yes + ConstrainCores=yes + ConstrainRAMSpace=yes + ConstrainDevices=yes + ConstrainSwapSpace=yes # gres.conf: | # # Ref: https://slurm.schedmd.com/gres.conf.html # helpers.conf: | @@ -191,7 +200,7 @@ authcred: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # Set container resource requests and limits for Kubernetes Pod scheduling. @@ -221,7 +230,7 @@ controller: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The controller service configuration. @@ -337,7 +346,7 @@ login: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The login service configuration. @@ -427,6 +436,15 @@ login: pam: {} # debug_level: 9 # + # -- (object) + # The security context given to the container. + # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: + privileged: false + # capabilities: + # add: + # - SYS_CHROOT + # # --(list) # List of volume mounts. # Ref: https://kubernetes.io/docs/concepts/storage/volumes/ @@ -513,7 +531,7 @@ compute: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (list) # Slurm NodeSets by object list. @@ -544,26 +562,16 @@ compute: # # -- (string) # Set the image tag to use. - tag: "24.11.4-ubuntu24.04" + tag: "25.05.0-ubuntu24.04" # # -- (string) # Set the priority class to use. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/#priorityclass priorityClassName: "" # - # -- (object) - # Set container resource requests and limits for Kubernetes Pod scheduling. - # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container - resources: - limits: - nvidia.com/gpu: 4 - vpc.amazonaws.com/efa: 16 - requests: - nvidia.com/gpu: 4 - vpc.amazonaws.com/efa: 16 - # # -- (map) # Selector which must match a node's labels for the pod to be scheduled on that node. + # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector nodeSelector: kubernetes.io/os: linux node.kubernetes.io/instance-type: ml.p5.48xlarge @@ -572,25 +580,21 @@ compute: # Set affinity for Kubernetes Pod scheduling. # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity affinity: {} - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: "kubernetes.io/os" - # operator: In - # values: - # - linux + # podAntiAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # - topologyKey: "kubernetes.io/hostname" - # labelSelector: - # matchExpressions: - # - key: "app.kubernetes.io/name" - # operator: In - # values: - # - slurmctld - # - slurmdbd - # - slurmrestd + # preferredDuringSchedulingIgnoredDuringExecution: + # - weight: 100 + # podAffinityTerm: + # topologyKey: kubernetes.io/hostname + # labelSelector: + # matchExpressions: + # - key: app.kubernetes.io/name + # operator: In + # values: + # - slurmctld + # - slurmdbd + # - slurmrestd + # - mariadb # # -- (list) # Configure pod tolerations. @@ -598,6 +602,21 @@ compute: tolerations: [] # # -- (object) + # Set container resource requests and limits for Kubernetes Pod scheduling. + # Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-requests-and-limits-of-pod-and-container + resources: + limits: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + requests: + nvidia.com/gpu: 4 + vpc.amazonaws.com/efa: 16 + # + # -- (bool) + # Enable to propagate the pod `resources.limits` into slurmd. + useResourceLimits: true + # + # -- (object) # Set the update strategy configuration. updateStrategy: # @@ -760,7 +779,7 @@ accounting: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # Configuration for an external database (e.g. mariadb, mysql). external: @@ -937,7 +956,7 @@ restapi: # # -- (string) # Set the image tag to use. - tag: 24.11-ubuntu24.04 + tag: 25.05-ubuntu24.04 # # -- (object) # The restapi service configuration. @@ -992,7 +1011,7 @@ restapi: # `slurm-exporter` subchart configurations. # Ref: https://github.com/SlinkyProject/slurm-exporter/-/blob/main/helm/slurm-exporter/values.yaml slurm-exporter: - enabled: false + enabled: true exporter: enabled: true secretName: "slurm-token-exporter" From 85d8f27f26fd2468eab7de5acf69fc32409c235a Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Mon, 23 Jun 2025 16:05:42 -0500 Subject: [PATCH 12/15] Slinky v0.3.0 updates --- .../7.sagemaker-hyperpod-eks/slinky-slurm/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md index eaea30710..0d0432658 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/README.md @@ -132,6 +132,7 @@ chmod +x terraform_outputs.sh ./terraform_outputs.sh cat env_vars.sh source env_vars.sh +cd .. ``` --- Verify that the required environment variables are set: @@ -1003,7 +1004,13 @@ eksctl delete iamserviceaccount \ aws iam delete-policy --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy-v2.12.0 ``` -Delete the HyperPod EKS CloudFormation stack: +Delete the HyperPod EKS CloudFormation stacks: ``` aws cloudformation delete-stack --stack-name $STACK_ID --region $AWS_REGION +``` +Delete the HyperPod EKS Terraform modules: +``` +cd terraform-modules/hyperpod-eks-tf +terraform plan -destroy -var-file=custom.tfvars +terraform destroy -var-file=$PARAMS ``` \ No newline at end of file From 7b83877374f1f3efaf6de4fa6627d396b44f8d50 Mon Sep 17 00:00:00 2001 From: bluecrayon52 <16687465+bluecrayon52@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:33:08 -0500 Subject: [PATCH 13/15] code tidy --- .../7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml index 20434bd5f..1b36f3d29 100644 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml @@ -368,7 +368,7 @@ login: # -- (list) # The `/root/.ssh/authorized_keys` file to write, represented as a list. rootSshAuthorizedKeys: - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOgiXVQ3l7+huQ2vC6gvpTqd94YljwjneCdH/irPNc1d natharno@amazon.com + - "" # # -- (map) # The `/etc/ssh/sshd_config` file to use, represented as a map. @@ -558,7 +558,7 @@ compute: # # -- (string) # Set the image repository to use. - repository: "659924747436.dkr.ecr.us-west-2.amazonaws.com/dlc-slurmd" + repository: ".dkr.ecr..amazonaws.com/dlc-slurmd" # # -- (string) # Set the image tag to use. From 4ef521068527f0279730d6778af67787c9e5dbe3 Mon Sep 17 00:00:00 2001 From: Basira Daqiq Date: Mon, 14 Jul 2025 16:58:51 +0000 Subject: [PATCH 14/15] modifying this script for slinky cluster creation --- .../automate-SlinkyCluster-creation.sh | 1251 +++++++++++++++++ 1 file changed, 1251 insertions(+) create mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh new file mode 100644 index 000000000..4864044bb --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh @@ -0,0 +1,1251 @@ +#!/bin/bash + +# Workshop Automation Script +# This script automates the steps of the workshop by executing CLI commands + +# Exit immediately if a command exits with a non-zero status. Print commands and their arguments as executed +set -e + +#===Global=== +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) +export AWS_REGION=${AWS_DEFAULT_REGION:-$(aws configure get region)} +TOTAL_STEPS=5 +CURRENT_STEP=0 + +#===Style Definitions=== +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Function to print a yellow header +print_header() { + echo -e "\n${BLUE}=================================================${NC}" + echo -e "\n${YELLOW}==== $1 ====${NC}\n" + echo -e "\n${BLUE}=================================================${NC}" + +} + +# UX Function for a Progress Bar :) +progress_bar() { + local duration=$1 + local steps=$2 + local width=50 + local progress=0 + + for ((i=0; i /dev/null; then + echo -e "${YELLOW}⚠️ AWS CLI is not installed. Installing...${NC}" + install_aws_cli + else + echo -e "${GREEN}✅ AWS CLI found. Checking version...${NC}" + CLI_VERSION=$(aws --version | awk '{print $1}' | cut -d/ -f2) + + echo -e "${BLUE}Current version: ${YELLOW}$CLI_VERSION${NC}" + echo -e "${BLUE}Min. required version: ${YELLOW}2.17.1${NC}" + + if [[ "$(printf '%s\n' "2.17.1" "$CLI_VERSION" | sort -V | head -n1)" != "2.17.1" ]]; then + echo -e "${YELLOW}⚠️ AWS CLI version $CLI_VERSION is lower than required.${NC}" + echo -e "${YELLOW} Updating AWS CLI...${NC}" + install_aws_cli + else + echo -e "${GREEN}✅ AWS CLI version $CLI_VERSION is up to date.${NC}" + fi + fi + + echo -e "${BLUE}=== AWS CLI Check Complete ===${NC}\n" + +} + +# Function to check if Git is installed and configured +check_git() { + if ! command -v git &> /dev/null; then + echo "Git is not installed. Please install Git and try again." + exit 1 + fi +} + +# clone_adt() { +# REPO_NAME="awsome-distributed-training" +# if [ -d "$REPO_NAME" ]; then +# echo -e "${YELLOW}⚠️ The directory '$REPO_NAME' already exists.${NC}" +# echo -e "${GREEN}Do you want to remove it and clone again? (yes/no): ${NC}" +# read -e REMOVE_AND_CLONE +# if [[ $REMOVE_AND_CLONE == "yes" ]]; then +# echo -e "${YELLOW}Removing existing directory...${NC}" +# rm -rf "$REPO_NAME" +# echo -e "${BLUE}Cloning repository...${NC}" +# git clone --depth=1 https://github.com/aws-samples/awsome-distributed-training/ +# echo -e "${GREEN}✅ Repository cloned successfully${NC}" +# else +# echo -e "${BLUE}Using existing directory...${NC}" +# fi +# else +# echo -e "${BLUE}Cloning repository $REPO_NAME...${NC}" +# git clone --depth=1 https://github.com/aws-samples/awsome-distributed-training/ +# echo -e "${GREEN}✅ Repository cloned successfully${NC}" +# fi +# } + +clone_adt() { + REPO_NAME="awsome-distributed-training" + BRANCH_NAME="feature/slinkly-slurm-hyperpod-eks" + if [ -d "$REPO_NAME" ]; then + echo -e "${YELLOW}⚠️ The directory '$REPO_NAME' already exists.${NC}" + echo -e "${GREEN}Do you want to remove it and clone again? (yes/no): ${NC}" + read -e REMOVE_AND_CLONE + if [[ $REMOVE_AND_CLONE == "yes" ]]; then + echo -e "${YELLOW}Removing existing directory...${NC}" + rm -rf "$REPO_NAME" + echo -e "${BLUE}Cloning repository...${NC}" + git clone -b $BRANCH_NAME --depth=1 https://github.com/aws-samples/awsome-distributed-training/ + echo -e "${GREEN}✅ Repository cloned successfully${NC}" + else + echo -e "${BLUE}Using existing directory...${NC}" + fi + else + echo -e "${BLUE}Cloning repository $REPO_NAME...${NC}" + git clone -b $BRANCH_NAME --depth=1 https://github.com/aws-samples/awsome-distributed-training/ + echo -e "${GREEN}✅ Repository cloned successfully${NC}" + fi +} +# Function for multi-headnode feature for SMHP SLURM cluster +#MH +multi_headnode() { + source env_vars + echo -e "${BLUE}=== Multi-Headnode Feature ===${NC}" + MULTI_HEADNODE=$(get_input "Do you want to enable multi-headnode feature?" "no") + if [[ $MULTI_HEADNODE == "yes" ]]; then + export MH=true + local SHOULD_DEPLOY=true + # Query for BackupPrivateSubnet and FSxLustreFilesystemDNSname in create_config.sh + # DONE + + export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the name for the SageMaker HyperPod Multiheadnode stack to be deployed" "sagemaker-hyperpod-mh") + + # Check if stack already exists and has required outputs + if aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} >/dev/null 2>&1; then + echo -e "${YELLOW}⚠️ A stack with name '${MULTI_HEAD_SLURM_STACK}' already exists${NC}" + echo -e "${YELLOW}Note: The new stack's AZs must match the existing stack's AZs for the multi-headnode feature to work properly (${SUBNET_ID}, ${BACKUP_SUBNET})${NC}" + echo -e "${BLUE}Would you like to deploy a stack with a different name? (yes/no)${NC}" + read -e DEPLOY_NEW_STACK + + if [[ $DEPLOY_NEW_STACK != "yes" ]]; then + echo -e "${YELLOW}Using existing stack '${MULTI_HEAD_SLURM_STACK}'${NC}" + SHOULD_DEPLOY=false + else + export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the NEW name for the SageMaker HyperPod Multiheadnode stack to be deployed)" "sagemaker-hyperpod-mh") + fi + fi + + # Source env_vars + source env_vars + + if [[ $SHOULD_DEPLOY == true ]]; then + # Ask user to input EMAIL and DB_USER_NAME + export EMAIL=$(get_input "Input your SNSSubEmailAddress here (this is the email address that will be used to send notifications about your head node status)" "johndoe@example.com") + export DB_USER_NAME=$(get_input "Input your DB_USER_NAME here (this is the username that will be used to access the SlurmDB)" "johndoe") + # export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the name for the SageMaker HyperPod Multiheadnode stack to be deployed)" "sagemaker-hyperpod-mh") + + echo -e "${YELLOW}The following CloudFormation command will be executed:${NC}" + echo -e "${GREEN}aws cloudformation deploy \\ + --template-file awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/sagemaker-hyperpod-slurm-multi-headnode.yaml \\ + --stack-name ${MULTI_HEAD_SLURM_STACK} \\ + --parameter-overrides \\ + SlurmDBSecurityGroupId=${SECURITY_GROUP} \\ + SlurmDBSubnetGroupId1=${SUBNET_ID} \\ + SlurmDBSubnetGroupId2=${BACKUP_SUBNET} \\ + SNSSubEmailAddress=${EMAIL} \\ + SlurmDBUsername=${DB_USER_NAME} \\ + --capabilities CAPABILITY_NAMED_IAM${NC}" + echo -e "\n${YELLOW}This will create the following resources in your account:${NC}" + echo -e "- Amazon RDS instance for SLURM database" + echo -e "- Amazon SNS topic for head node failover notifications" + echo -e "- IAM roles and policies for multi-head node functionality" + + echo -e "\n${BLUE}Would you like to proceed with the deployment? Please acnowledge that you allow CloudFormation to create resources in your account by hitting ENTER${NC}" + read + + # Deploy the multi-head CF stack + aws cloudformation deploy \ + --template-file awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/sagemaker-hyperpod-slurm-multi-headnode.yaml \ + --stack-name ${MULTI_HEAD_SLURM_STACK} \ + --parameter-overrides \ + SlurmDBSecurityGroupId=${SECURITY_GROUP} \ + SlurmDBSubnetGroupId1=${SUBNET_ID} \ + SlurmDBSubnetGroupId2=${BACKUP_SUBNET} \ + SNSSubEmailAddress=${EMAIL} \ + SlurmDBUsername=${DB_USER_NAME} \ + --capabilities CAPABILITY_NAMED_IAM + + # Wait for stack to be created + echo -e "${BLUE}Waiting for multi-headnode stack creation to complete...${NC}" + aws cloudformation wait stack-create-complete \ + --stack-name ${MULTI_HEAD_SLURM_STACK} + else + # Get the outputs for EMAIL and DB_USER_NAME (used in provisioning_parameters.json!!!) + echo "From Stack: ${MULTI_HEAD_SLURM_STACK}" + export EMAIL=$(aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} --query 'Stacks[0].Outputs[?OutputKey==`SNSSubEmailAddress`].OutputValue' --output text) + export DB_USER_NAME=$(aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBUsername`].OutputValue' --output text) + + echo -e "Set Email: ${EMAIL}, DB Username: ${DB_USER_NAME}" + fi + + # Query new stack for SlurmDBEndpointAddress SlurmDBSecretArn SlurmExecutionRoleArn SlurmFailOverSNSTopicArn and write these to env_vars + SLURM_DB_ENDPOINT_ADDRESS=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBEndpointAddress`].OutputValue' --output text) + SLURM_DB_SECRET_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBSecretArn`].OutputValue' --output text) + SLURM_EXECUTION_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmExecutionRoleArn`].OutputValue' --output text) + SLURM_SNS_FAILOVER_TOPIC_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmFailOverSNSTopicArn`].OutputValue' --output text) + + echo "export SLURM_DB_ENDPOINT_ADDRESS=${SLURM_DB_ENDPOINT_ADDRESS}" >> env_vars + echo "export SLURM_DB_SECRET_ARN=${SLURM_DB_SECRET_ARN}" >> env_vars + echo "export SLURM_EXECUTION_ROLE_ARN=${SLURM_EXECUTION_ROLE_ARN}" >> env_vars + echo "export SLURM_SNS_FAILOVER_TOPIC_ARN=${SLURM_SNS_FAILOVER_TOPIC_ARN}" >> env_vars + echo "export EMAIL=${EMAIL}" >> env_vars + echo "export DB_USER_NAME=${DB_USER_NAME}" >> env_vars + + if [[ -z "$SLURM_DB_ENDPOINT_ADDRESS" ]] || [[ -z "$SLURM_DB_SECRET_ARN" ]] || [[ -z "$SLURM_EXECUTION_ROLE_ARN" ]] || [[ -z "$SLURM_SNS_FAILOVER_TOPIC_ARN" ]]; then + echo -e "${YELLOW}⚠️ Failed to retrieve all required values from the CloudFormation stack${NC}" + echo -e "${YELLOW}Please ensure the stack deployed correctly and all outputs are available${NC}" + return 1 + fi + + SLURM_EXECUTION_ROLE=$(echo $SLURM_EXECUTION_ROLE_ARN | awk -F'/' '{print $NF}') + + echo -e "${GREEN}✅ Multi-headnode feature enabled${NC}" + + # Create IAM policy for multi-headnode feature + echo -e "\n${BLUE}Creating IAM policy for SLURM execution role...${NC}" + + create_and_attach_policy() { + aws iam create-policy \ + --policy-name AmazonSageMakerExecutionPolicy \ + --policy-document file://awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json --output json && \ + aws iam attach-role-policy \ + --role-name $SLURM_EXECUTION_ROLE \ + --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/AmazonSageMakerExecutionPolicy + } + + if error_output=$(create_and_attach_policy 2>&1); then + echo -e "${GREEN}✅ IAM policy created and attached successfully${NC}" + else + echo -e "${YELLOW}⚠️ Error occurred while creating/attaching IAM policy:${NC}" + echo -e "${YELLOW}$error_output${NC}" + + if [[ $error_output == *"EntityAlreadyExists"* ]]; then + echo -e "\n${YELLOW}If the error you received is that the policy already exists, you can either:${NC}" + echo -e "\n${GREEN} 1. Continue the script with the existing policy (make sure the permissions match the ones in https://github.com/aws-samples/awsome-distributed-training/blob/main/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json) and manually attach it to your role ${SLURM_EXECUTION_ROLE}, or${NC}" + echo -e "\n${GREEN} 2. You can create a new policy with a name different than 'AmazonSageMakerExecutionPolicy' manually and attach it to your 'AmazonSageMakerExecutionRole' with the following command. Once you do that, you can continue with the rest of the script:${NC}" + + echo -e "\n${YELLOW} Creating an IAM policy (required for option 2 above)${NC}" + echo -e "\n${BLUE} aws iam create-policy \\ + --policy-name \\ + --policy-document file://awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json${NC}" + + echo -e "\n${YELLOW} Attach an IAM policy to an IAM role (required for options 1 & 2 above)${NC}" + echo -e "\n${BLUE} aws iam attach-role-policy \\ + --role-name ${SLURM_EXECUTION_ROLE} \\ + --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/${NC}" + fi + + echo -e "Options:" + echo -e "1. [RECOMMENDED, PLEASE READ ABOVE] Press Enter to continue with the rest of the script" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + fi + else + echo -e "${YELLOW}Skipping multi-headnode configuration...${NC}" + export MH=false + fi + echo -e "\n${BLUE}=== Multi-Headnode Configuration Complete ===${NC}" +} + +# Function to setup environment variables +setup_env_vars() { + echo -e "${BLUE}=== Setting Up Environment Variables ===${NC}" + echo -e "${GREEN}Cloning awsome-distributed-training${NC}" + clone_adt + + echo -e "${BLUE}Enter the name of the SageMaker VPC CloudFormation stack that was deployed as a prerequisite (default: sagemaker-hyperpod):${NC}" + read -e STACK_ID_VPC + export STACK_ID_VPC=${STACK_ID_VPC:-sagemaker-hyperpod} + + if [ "$CF_STACK_NAME" != "sagemaker-hyperpod" ]; then + echo -e "${GREEN}✅ Configuration script updated with stack name: $STACK_ID_VPC${NC}" + else + echo -e "${GREEN}Using default stack name: sagemaker-hyperpod${NC}" + fi + + # Clear env_vars from previous runs + > env_vars + + echo -e "${YELLOW}Generating new environment variables...${NC}" + + generate_env_vars() { + bash awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/create_config.sh + # bash create_config.sh + } + + # Capture stdout + stderr + if error_output=$(generate_env_vars 2>&1); then + echo -e "${GREEN}✅ New environment variables generated and sourced${NC}" + else + echo -e "${YELLOW}⚠️ Error occurred while generating environment variables:${NC}" + echo -e "${YELLOW}$error_output${NC}" + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + fi + fi + + # FEAT: Add support for multiple headnodes + #MH + multi_headnode + + source env_vars + + echo -e "\n${BLUE}=== Environment Variables Summary ===${NC}" + echo -e "${YELLOW}Note: You may ignore the INSTANCES parameter for now${NC}" + echo -e "${GREEN}Current environment variables:${NC}" + cat env_vars + + echo -e "\n${BLUE}=== Environment Setup Complete ===${NC}" +} + +# Function to setup lifecycle scripts +setup_lifecycle_scripts() { + echo -e "${BLUE}=== Setting Up Lifecycle Scripts ===${NC}" + + cd awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/ + + echo -e "${YELLOW}Are you using Neuron-based instances (Trainium/Inferentia)? (yes/no)${NC}" + read -e USING_NEURON + + if [ "$USING_NEURON" == "yes" ]; then + echo -e "${BLUE}Enabling Neuron in LCS...${NC}" + sed -i.bak 's/enable_update_neuron_sdk = False/enable_update_neuron_sdk = True/' base-config/config.py + rm base-config/config.py.bak + echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! Neuron enabled in config.py${NC}" + else + echo -e "${BLUE}Continuing with Neuron disabled in LCS...${NC}" + fi + + # Check if FSx OpenZFS was deployed in the stack + echo -e "${BLUE}Checking if FSx OpenZFS was deployed in the stack...${NC}" + + export ENABLE_FSX_OPENZFS="false" + + FSX_OPENZFS_DNS=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_ID_VPC}" \ + --query 'Stacks[0].Outputs[?OutputKey==`FSxOpenZFSFileSystemDNSname`].OutputValue' \ + --output text) + + if [ -n "$FSX_OPENZFS_DNS" ]; then + echo -e "${BLUE}FSx OpenZFS detected in stack. DNS: ${FSX_OPENZFS_DNS}${NC}" + echo -e "${BLUE}Enabling FSx OpenZFS in LCS...${NC}" + + # Get the FSx OpenZFS File System ID as well + FSX_OPENZFS_ID=$(aws cloudformation describe-stacks \ + --stack-name "${STACK_ID_VPC}" \ + --query 'Stacks[0].Outputs[?OutputKey==`FSxOpenZFSFileSystemId`].OutputValue' \ + --output text) + + ENABLE_FSX_OPENZFS="true" + echo "export FSX_OPENZFS_DNS=${FSX_OPENZFS_DNS}" >> env_vars + echo "export FSX_OPENZFS_ID=${FSX_OPENZFS_ID}" >> env_vars + + # Update config.py + sed -i.bak 's/enable_fsx_openzfs = False/enable_fsx_openzfs = True/' base-config/config.py + rm base-config/config.py.bak + + echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! FSx OpenZFS enabled in config.py${NC}" + else + echo -e "${BLUE}No FSx OpenZFS detected in stack. Continuing with FSx OpenZFS disabled in LCS...${NC}" + fi + + echo -e "${YELLOW}Did you deploy the optional hyperpod-observability CloudFormation stack? (yes/no)${NC}" + read -e DEPLOYED_OBSERVABILITY + + if [ "$DEPLOYED_OBSERVABILITY" == "yes" ]; then + echo -e "${BLUE}Enabling observability in LCS...${NC}" + sed -i.bak 's/enable_observability = False/enable_observability = True/' base-config/config.py + rm base-config/config.py.bak + echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! Observability enabled in config.py${NC}" + + echo -e "${BLUE}Attaching IAM policies for observability to $ROLENAME${NC}" + + # Helper function for attaching IAM policies (specific to observability stack only!) + attach_policies() { + aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess --output json + aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess --output json + } + + # Capture stdout + stderr + + if ! error_output=$(attach_policies 2>&1); then + echo -e "${YELLOW}⚠️ Failed to attach IAM policies. This operation requires admin permissions${NC}" + echo -e "${YELLOW} This was the error received${NC}" + echo -e "${YELLOW}$error_output${NC}" + echo -e "Options:" + echo -e "1. Run 'aws configure' as an admin user as part of this script." + echo -e "2. Press Ctrl+C to exit and run 'aws configure' as an admin user outside this script." + echo -e "3. Press Enter to continue with the rest of the script without configuring this step." + + read -e -p "Choose an option (1, 2, or 3): " choice + + case $choice in + 1) + echo -e "${BLUE}Running 'aws configure'. Please enter your **admin** credentials..${NC}" + aws configure + echo -e "${GREEN}✅ AWS CLI configured successfully${NC}" + echo -e "${BLUE}Retrying to attach IAM policies!${NC}" + if ! attach_policies; then + echo -e "${YELLOW}⚠️ Failed to attach IAM policies. Please attach the following policies manually:${NC}" + echo -e "1. AmazonPrometheusRemoteWriteAccess" + echo -e "2. AWSCloudFormationReadOnlyAccess" + echo -e "Press Enter to continue with the rest of the script without configuring this step." + read -e -p "Press Enter to continue: " + echo -e "${BLUE}Continuing with the rest of the script without configuring this step.${NC}" + else + echo -e "${GREEN}✅ IAM policies attached successfully${NC}" + fi + ;; + 2) + echo -e "${BLUE}Please run 'aws configure' as an admin user outside this script.${NC}" + exit 1 + ;; + 3) + echo -e "${BLUE}Continuing with the rest of the script without configuring this step.${NC}" + ;; + *) + echo -e "${BLUE}Invalid choice. Continuing with the rest of the script without configuring this step.${NC}" + ;; + esac + else + echo -e "${GREEN}✅ IAM policies attached successfully${NC}" + fi + echo -e "${GREEN}✅ Observability setup complete!${NC}" + else + echo -e "${YELLOW}Observability not enabled. Continuing with default configuration${NC}" + fi + + echo -e "${BLUE}Uploading your lifecycle scripts to S3 bucket ${YELLOW}${BUCKET}${NC}" + # upload data + upload_to_s3() { + aws s3 cp --recursive base-config/ s3://${BUCKET}/src --output json + } + + if error_output=$(upload_to_s3 2>&1); then + echo -e "${GREEN}✅ Lifecycle scripts uploaded successfully${NC}" + else + echo -e "${YELLOW}⚠️ Error occurred while uploading lifecycle scripts to S3 bucket:${NC}" + echo -e "${YELLOW}$error_output${NC}" + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + fi + + # move back to env_var directory + cd ../../../.. + + echo -e "\n${BLUE}=== Lifecycle Scripts Setup Complete ===${NC}" +} + +# Helper function to get user inputs with default values specified +get_input() { + local prompt="$1" + local default="$2" + local input + read -e -p "$prompt [$default]: " input + echo "${input:-$default}" +} + +# Function to write the cluster-config.json file +create_config() { + echo -e "\n${BLUE}=== Lifecycle Scripts Setup Complete ===${NC}" + + # Get controller machine details + CONTROLLER_NAME=$(get_input "Enter the name for the controller instance group" "controller-machine") + CONTROLLER_TYPE=$(get_input "Enter the instance type for the controller" "ml.m5.12xlarge") + + # Initialize instance groups array + INSTANCE_GROUPS="[" + + # Add login group + echo -e "${GREEN}Do you want to add a login group? (yes/no): ${NC}" + read -e ADD_LOGIN_GROUP + + if [[ $ADD_LOGIN_GROUP == "yes" ]]; then + LOGIN_TYPE=$(get_input "Enter the instance type for the login group" "ml.m5.4xlarge") + + INSTANCE_GROUPS+="{ + \"InstanceGroupName\": \"login-group\", + \"InstanceType\": \"$LOGIN_TYPE\", + \"InstanceStorageConfigs\": [ + { + \"EbsVolumeConfig\": { + \"VolumeSizeInGB\": 500 + } + } + ], + \"InstanceCount\": 1, + \"LifeCycleConfig\": { + \"SourceS3Uri\": \"s3://${BUCKET}/src\", + \"OnCreate\": \"on_create.sh\" + }, + \"ExecutionRole\": \"${ROLE}\", + \"ThreadsPerCore\": 2 + }," + + echo -e "${GREEN}✅ Login Group added${NC}" + fi + + CONTROLLER_COUNT=$([ "${MH:-false}" = true ] && echo "2" || echo "1") + EXECUTION_ROLE=$([ "${MH:-false}" = true ] && echo "${SLURM_EXECUTION_ROLE_ARN}" || echo "${ROLE}") + + # Add controller group + INSTANCE_GROUPS+="{ + \"InstanceGroupName\": \"$CONTROLLER_NAME\", + \"InstanceType\": \"$CONTROLLER_TYPE\", + \"InstanceStorageConfigs\": [ + { + \"EbsVolumeConfig\": { + \"VolumeSizeInGB\": 500 + } + } + ], + \"InstanceCount\": ${CONTROLLER_COUNT}, + \"LifeCycleConfig\": { + \"SourceS3Uri\": \"s3://${BUCKET}/src\", + \"OnCreate\": \"on_create.sh\" + }, + \"ExecutionRole\": \"${EXECUTION_ROLE}\", + \"ThreadsPerCore\": 2 + }" + + # Loop to add worker instance groups + WORKER_GROUP_COUNT=1 + echo -e "\n${BLUE}=== Worker Group Configuration ===${NC}" + while true; do + if [[ $WORKER_GROUP_COUNT -eq 1 ]]; then + ADD_WORKER=$(get_input "Do you want to add a worker instance group? (yes/no):" "yes") + else + ADD_WORKER=$(get_input "Do you want to add another worker instance group? (yes/no):" "no") + fi + + if [[ $ADD_WORKER != "yes" ]]; then + break + fi + + echo -e "${YELLOW}Configuring Worker Group $WORKER_GROUP_COUNT${NC}" + INSTANCE_TYPE=$(get_input "Enter the instance type for worker group $WORKER_GROUP_COUNT" "ml.c5.4xlarge") + INSTANCE_COUNT=$(get_input "Enter the instance count for worker group $WORKER_GROUP_COUNT" "4") + + echo -e "${GREEN}Are you using training plans? (yes/no): ${NC}" + read -e USE_TRAINING_PLAN + + INSTANCE_GROUPS+=", + { + \"InstanceGroupName\": \"worker-group-$WORKER_GROUP_COUNT\", + \"InstanceType\": \"$INSTANCE_TYPE\", + \"InstanceCount\": $INSTANCE_COUNT, + \"InstanceStorageConfigs\": [ + { + \"EbsVolumeConfig\": { + \"VolumeSizeInGB\": 500 + } + } + ], + \"LifeCycleConfig\": { + \"SourceS3Uri\": \"s3://${BUCKET}/src\", + \"OnCreate\": \"on_create.sh\" + }, + \"ExecutionRole\": \"${ROLE}\", + \"ThreadsPerCore\": 1" + + if [[ $USE_TRAINING_PLAN == "yes" ]]; then + echo -e "\n${BLUE}=== Training Plan Configuration ===${NC}" + # aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AmazonEC2FullAccess + + TRAINING_PLAN=$(get_input "Enter the training plan name" "") + + count=0 + while true; do + # Attempt to describe the training plan + echo -e "${YELLOW}Attempting to retrieve training plan details...${NC}" + + if ! TRAINING_PLAN_DESCRIPTION=$(aws sagemaker describe-training-plan --training-plan-name "$TRAINING_PLAN" --output json 2>&1); then + echo -e "${BLUE}❌Error: Training plan '$TRAINING_PLAN' not found. Please try again.${NC}" + echo -e "${GREEN}Are you using training plans (Beta feature)? (yes/no)${NC}" + read -e USE_TRAINING_PLAN + if [[ $USE_TRAINING_PLAN != "yes" ]]; then + echo -e "${YELLOW}Exiting training plan configuration.${NC}" + break + else + TRAINING_PLAN=$(get_input "Enter the training plan name" "") + fi + else + # Extract relevant information from the description + TRAINING_PLAN_ARN=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.TrainingPlanArn') + AVAILABLE_INSTANCE_COUNT=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.AvailableInstanceCount') + TOTAL_INSTANCE_COUNT=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.TotalInstanceCount') + TRAINING_PLAN_AZ=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.ReservedCapacitySummaries[0].AvailabilityZone') + TP_INSTANCE_TYPE=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.ReservedCapacitySummaries[0].InstanceType') + + CF_AZ=$(aws ec2 describe-subnets --subnet-ids $SUBNET_ID --output json | jq -r '.Subnets[0].AvailabilityZone') + + # Only print if count=0 + if [[ $count -eq 0 ]]; then + echo -e "${GREEN}Training Plan Details:${NC}" + echo -e " ${YELLOW}Name:${NC} $TRAINING_PLAN" + echo -e " ${YELLOW}Available Instance Count:${NC} $AVAILABLE_INSTANCE_COUNT" + echo -e " ${YELLOW}Total Instance Count:${NC} $TOTAL_INSTANCE_COUNT" + echo -e " ${YELLOW}Training Plan Availability Zone:${NC} $TRAINING_PLAN_AZ" + echo -e " ${YELLOW}Training Plan Instance Type:${NC} $TP_INSTANCE_TYPE" + fi + + # Compare INSTANCE_COUNT with AVAILABLE_INSTANCE_COUNT + INSTANCE_COUNT_OK="n" + if [[ $INSTANCE_COUNT -gt $AVAILABLE_INSTANCE_COUNT ]]; then + echo -e "${YELLOW}Warning: The requested instance count ($INSTANCE_COUNT) is greater than the available instances in the training plan ($AVAILABLE_INSTANCE_COUNT).${NC}" + echo -e "${BLUE}Do you want to continue anyway?(yes/no)${NC}" + read -e CONTINUE + if [[ $CONTINUE != "yes" ]]; then + NEW_INSTANCE_COUNT=$(get_input "Enter the new number of instances" "1") + # Update INSTANCE_GROUPS with new INSTANCE_COUNT for the current worker group + INSTANCE_GROUPS=$(echo "$INSTANCE_GROUPS" | perl -pe ' + BEGIN { + $group = "worker-group-'"$WORKER_GROUP_COUNT"'"; + $count = '"$NEW_INSTANCE_COUNT"'; + $in_group = 0; + } + if (/"InstanceGroupName":\s*"$group"/) { + $in_group = 1; + } + if ($in_group && /"InstanceCount":\s*\d+/) { + s/("InstanceCount":\s*)\d+/$1$count/; + $in_group = 0; + } + ') + INSTANCE_COUNT=$NEW_INSTANCE_COUNT + echo -e "${GREEN}Updated instance count for worker-group-$WORKER_GROUP_COUNT to $INSTANCE_COUNT${NC}" + fi + INSTANCE_COUNT_OK="y" + else + INSTANCE_COUNT_OK="y" + fi + + if [[ $INSTANCE_COUNT_OK == "y" ]]; then + INSTANCE_TYPE_OK="n" + # Compare INSTANCE_TYPE with TP_INSTANCE_TYPE + if [[ $INSTANCE_TYPE != $TP_INSTANCE_TYPE ]]; then + echo -e "${YELLOW}Warning: The requested instance type ($INSTANCE_TYPE) does not match the instance type in the training plan ($TP_INSTANCE_TYPE).${NC}" + echo -e "${BLUE}Do you want to continue anyway? If you choose "no", then the script will update instance type for you and proceed. (yes/no)${NC}" + read -e CONTINUE + if [[ $CONTINUE != "yes" ]]; then + NEW_INSTANCE_TYPE=$TP_INSTANCE_TYPE + # Update INSTANCE_GROUPS with new INSTANCE_TYPE for the current worker group + INSTANCE_GROUPS=$(echo "$INSTANCE_GROUPS" | perl -pe ' + BEGIN { + $group = "worker-group-'$WORKER_GROUP_COUNT'"; + $type = "'$NEW_INSTANCE_TYPE'"; + $in_group = 0; + } + if (/"InstanceGroupName":\s*"$group"/) { + $in_group = 1; + } + if ($in_group && /"InstanceType":\s*"[^"]*"/) { + s/("InstanceType":\s*")[^"]*"/$1$type"/; + $in_group = 0; + } + ') + INSTANCE_TYPE=$NEW_INSTANCE_TYPE + echo -e "${GREEN}Updated instance type for worker-group-$WORKER_GROUP_COUNT to $INSTANCE_TYPE${NC}" + fi + INSTANCE_TYPE_OK="y" + else + INSTANCE_TYPE_OK="y" + fi + + if [[ $INSTANCE_TYPE_OK == "y" ]]; then + # Compare TRAINING_PLAN_AZ with CF_AZ + if [[ $TRAINING_PLAN_AZ != $CF_AZ ]]; then + echo -e "${YELLOW}Warning: The training plan availability zone ($TRAINING_PLAN_AZ) does not match the cluster availability zone ($CF_AZ).${NC}" + echo -e "${BLUE}Do you want to continue anyway? (yes/no)${NC}" + read -e CONTINUE + if [[ $CONTINUE != "yes" ]]; then + echo -e "${YELLOW}Please ensure that your VPC is in the same Availability Zone as your training plan (or vice versa). If you used the workshop, this should be the CF stack \"sagemaker-hyperpod\". Exiting training plan configuration.${NC}" + continue + fi + fi + fi + fi + + echo -e "${GREEN}Adding Training Plan ARN to instance group configuration.${NC}" + INSTANCE_GROUPS+=", + \"TrainingPlanArn\": \"$TRAINING_PLAN_ARN\"" + break + fi + count+=1 + done + fi + + INSTANCE_GROUPS+=" + }" + + echo -e "${GREEN}✅ Worker Group $WORKER_GROUP_COUNT added${NC}" + ((WORKER_GROUP_COUNT++)) + done + + INSTANCE_GROUPS+="]" + + read -e -p "What would you like to name your cluster? (default: ml-cluster): " CLUSTER_NAME + CLUSTER_NAME=${CLUSTER_NAME:-ml-cluster} + + # Create the cluster-config.json file + cat > cluster-config.json << EOL + { + "ClusterName": "$CLUSTER_NAME", + "InstanceGroups": $INSTANCE_GROUPS, + "VpcConfig": { + "SecurityGroupIds": ["$SECURITY_GROUP"], + "Subnets":["$SUBNET_ID"] + } + } +EOL + + echo -e "${GREEN}✅ cluster-config.json created successfully${NC}" + + source env_vars + + echo -e "\n${YELLOW}Creating provisioning_parameters.json...${NC}" + WORKER_GROUPS="[" + + # Loop through worker groups + for ((i=1; i<=WORKER_GROUP_COUNT-1; i++)); do + if [ $i -gt 1 ]; then + WORKER_GROUPS+="," + fi + + instance_type=$(jq -r ".InstanceGroups[] | select(.InstanceGroupName == \"worker-group-$i\").InstanceType" cluster-config.json) + + WORKER_GROUPS+=" + { + \"instance_group_name\": \"worker-group-$i\", + \"partition_name\": \"$instance_type\" + }" + done + + WORKER_GROUPS+=" + ]" + + # OpenZFS + if [[ $ENABLE_FSX_OPENZFS == "true" ]]; then + FSX_OPENZFS_CONFIG=", + \"fsx_openzfs_dns_name\": \"${FSX_OPENZFS_ID}.fsx.${AWS_REGION}.amazonaws.com\"" + else + FSX_OPENZFS_CONFIG="" + fi + + #MH + if [[ $MH == "true" ]]; then + SLURM_CONFIGURATIONS=" + { + \"slurm_database_secret_arn\": \"$SLURM_DB_SECRET_ARN\", + \"slurm_database_endpoint\": \"$SLURM_DB_ENDPOINT_ADDRESS\", + \"slurm_shared_directory\": \"/fsx\", + \"slurm_database_user\": \"$DB_USER_NAME\", + \"slurm_sns_arn\": \"$SLURM_SNS_FAILOVER_TOPIC_ARN\" + }" + fi + + if [[ $ADD_LOGIN_GROUP == "yes" ]]; then + if [[ $MH == "true" ]]; then + cat > provisioning_parameters.json << EOL + { + "version": "1.0.0", + "workload_manager": "slurm", + "controller_group": "$CONTROLLER_NAME", + "login_group": "login-group", + "worker_groups": $WORKER_GROUPS, + "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", + "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG}, + "slurm_configurations": $SLURM_CONFIGURATIONS + } +EOL + else + cat > provisioning_parameters.json << EOL + { + "version": "1.0.0", + "workload_manager": "slurm", + "controller_group": "$CONTROLLER_NAME", + "login_group": "login-group", + "worker_groups": $WORKER_GROUPS, + "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", + "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG} + } +EOL + fi + else + if [[ $MH == "true" ]]; then + cat > provisioning_parameters.json << EOL + { + "version": "1.0.0", + "workload_manager": "slurm", + "controller_group": "$CONTROLLER_NAME", + "worker_groups": $WORKER_GROUPS, + "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", + "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG}, + "slurm_configurations": $SLURM_CONFIGURATIONS + } +EOL + else + cat > provisioning_parameters.json << EOL + { + "version": "1.0.0", + "workload_manager": "slurm", + "controller_group": "$CONTROLLER_NAME", + "worker_groups": $WORKER_GROUPS, + "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", + "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG} + } +EOL + fi + fi + + echo -e "${GREEN}✅ provisioning_parameters.json created successfully${NC}" + + # copy to the S3 Bucket + echo -e "\n${BLUE}Copying configuration to S3 bucket...${NC}" + + # upload data + upload_to_s3() { + aws s3 cp provisioning_parameters.json s3://${BUCKET}/src/ --output json + } + + if error_output=$(upload_to_s3 2>&1); then + echo -e "${GREEN}✅ Provisioning Parameters uploaded successfully${NC}" + else + echo -e "${YELLOW}⚠️ Error occurred while uploading lifecycle scripts to S3 bucket:${NC}" + echo -e "${YELLOW}$error_output${NC}" + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (Not Recommended)" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + fi + + echo -e "\n${BLUE}=== Cluster Configuration Complete ===${NC}" +} + +validate_cluster_config() { + echo "Validating your cluster configuration..." + # TODO: MAKE SURE PACKAGES ARE INSTALLED HERE!! + + curl -O https://raw.githubusercontent.com/aws-samples/awsome-distributed-training/main/1.architectures/5.sagemaker-hyperpod/validate-config.py + + # check config for known issues + python3 validate-config.py --cluster-config cluster-config.json --provisioning-parameters provisioning_parameters.json --region $AWS_REGION +} + +# Function to display the prerequisites before starting this workshop +display_important_prereqs() { + echo -e "${BLUE}Before running this script, please ensure the following:${NC}\n" + + echo -e "${GREEN}1. 🔑 IAM Credentials:${NC}" + echo " You have Administrator Access Credentials in IAM." + echo " This is crucial as we'll be using CloudFormation to create IAM roles and policies." + echo " Run 'aws configure' to set up your credentials." + + echo -e "\n${GREEN}2. 🌐 VPC Stack:${NC}" + echo " Deploy the sagemaker-hyperpod VPC stack using:" + echo " https://catalog.workshops.aws/sagemaker-hyperpod/en-US/00-setup/02-own-account" + echo " This creates essential resources: VPC, subnets, FSx Lustre volumes," + echo " S3 bucket, and IAM role for your SageMaker HyperPod cluster." + echo " ⚠️⚠️ IMPORTANT: If you choose a multi-head node configuration (i.e., multiple head nodes), then make sure that" + echo " the VPC stack has the \"(Optional) Availability zone id to deploy the backup private subnet\"". + + echo -e "\n${GREEN}3. 📊 Observability Stack:${NC}" + echo " It's highly recommended to deploy the observability stack as well." + echo " Navigate to https://catalog.workshops.aws/sagemaker-hyperpod/en-US/00-setup/02-own-account#2.-deploy-cluster-observability-stack-(recommended) to deploy the stack" + + echo -e "\n${GREEN}4. 💻 Development Environment:${NC}" + echo " Ensure you have a Linux-based development environment (macOS works great too)." + + echo -e "\n${GREEN}5. 🔧 Packages required for this script to run:${NC}" + echo " Ensure you install the following: pip, jq, boto3, and jsonschema" + + echo -e "\n${YELLOW}Ready to proceed? Press Enter to continue or Ctrl+C to exit...${NC}" + read +} + +region_check() { + echo -e "${BLUE}Please confirm that your AWS region is ${GREEN}$AWS_REGION${BLUE} (default).${NC}" + echo -e "${BLUE}If not, enter the AWS region where you want to set up your cluster (e.g., us-west-2):${NC}" + + read -p "> " NEW_REGION + + if [[ -z "$NEW_REGION" ]]; then + echo -e "${GREEN}✅ Using default region: ${YELLOW}$AWS_REGION${NC}" + else + export AWS_REGION="$NEW_REGION" + echo -e "${GREEN}✅ Region updated to: ${YELLOW}$AWS_REGION${NC}" + fi + + echo -e "\n${BLUE}Your region is set to: ${YELLOW}$AWS_REGION${NC}" + echo -e "${BLUE}Ensure your chosen region supports SageMaker HyperPod.${NC}" + echo -e "${GREEN}You can check out https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-hyperpod.html#sagemaker-hyperpod-available-regions to learn about supported regions.${NC}" + echo -e "${BLUE}Press Enter to continue...${NC}" + read +} + +# Function to create users in cluster +configure_cluster_users() { + echo -e "\n${BLUE}=== User Configuration ===${NC}" + + CONFIGURE_USERS=$(get_input "Would you like to configure users? If not, you can still use the ubuntu user (yes/no)" "no") + + FIRST_SSM_INTEGRATION=true + + if [[ "${CONFIGURE_USERS}" == "yes" ]]; then + echo -e "${BLUE}Creating shared_users.txt file...${NC}" + + # Initialize or clear the shared_users.txt file + > shared_users.txt + + # Initialize the user ID counter + next_user_id=2001 + + echo -e "${YELLOW}Enter user details (Press Ctrl+D when finished)${NC}" + echo -e "${BLUE}========================================${NC}" + + while IFS= read -p "Enter username: " username; do + # If username is empty, skip this iteration + if [[ -z "$username" ]]; then + continue + fi + + # Get user ID with default value + user_id=$(get_input "Enter user ID" "$next_user_id") + + # Write to shared_users.txt + echo "${username},${user_id},/fsx/${username}" >> shared_users.txt + + # SSM Integration + ASSOCIATE_IAM=$(get_input "[REQUIRES ADMIN] Would you like to associate this POSIX user with an IAM user? (yes/no)" "no") + + while [[ "${ASSOCIATE_IAM}" == "yes" ]]; do + if [[ "$FIRST_SSM_INTEGRATION" == true ]]; then + echo -e "\n${BLUE}=== SSM Run As Configuration ===${NC}" + echo -e "Now that we've created a new POSIX user, how do we ensure that users only connect as their user and not ssm-user when connecting via SSM? To do this, we use SSM run as tags, which allows us to tag an IAM user with the POSIX user (aka cluster user) they should connect to via SSM." + read -p "Hit ENTER if you understand, or type "no" to skip this: " CONTINUE + + if [[ -z "$CONTINUE" ]]; then + echo -e "\n${YELLOW}Please complete the following steps:${NC}" + + echo -e "1. Navigate to the Session Manager Preferences Console" + echo -e " (https://console.aws.amazon.com/systems-manager/session-manager/preferences)" + read -p "Hit ENTER once you are there: " + + echo -e "\n2. Under 'Specify Operating System user for sessions'," + echo -e " ✅ check the 'Enable Run As Support for Linux Instances'" + read -p "Hit ENTER once step is complete: " + + echo -e "\n3. Change the Linux shell profile." + echo -e " It should have '/bin/bash -c 'export HOME=/fsx/\$(whoami) && cd \${HOME} && exec /bin/bash' in its first and only line" + read -p "Hit ENTER once you've added this line in: " + + echo -e "\n${GREEN}✅ SSM Run As support configured successfully${NC}" + else + echo -e "${YELLOW}Skipping SSM Run As configuration instructions...${NC}" + break + fi + FIRST_SSM_INTEGRATION=false + fi + + IAM_USERNAME=$(get_input "Enter the IAM username to associate with POSIX user ${username}" "$username") + + if ! aws iam get-user --user-name "${IAM_USERNAME}" --output json >/dev/null 2>&1; then + echo -e "${YELLOW}⚠️ IAM user ${IAM_USERNAME} does not exist${NC}" + CREATE_IAM=$(get_input "Would you like to create this IAM user? (Note: You'll need to add permissions later) (yes/no)" "no") + + if [[ "${CREATE_IAM}" == "yes" ]]; then + if ! output=$(aws iam create-user --user-name "$IAM_USERNAME" --output json 2>&1); then + echo -e "${YELLOW}⚠️ Error creating IAM user ${IAM_USERNAME}:${NC}" + echo -e "${YELLOW}$output${NC}" + ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") + continue + else + echo -e "${GREEN}✅ IAM user ${IAM_USERNAME} created successfully. Reminder to add permissions to this user as required!${NC}" + fi + else + ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") + continue + fi + fi + + if ! output=$(aws iam tag-user \ + --user-name "$IAM_USERNAME" \ + --tags "[{\"Key\": \"SSMSessionRunAs\",\"Value\": \"$username\"}]" --output json 2>&1); then + echo -e "${YELLOW}⚠️ Error adding SSM Run As tag for ${IAM_USERNAME}:${NC}" + echo -e "${YELLOW}$output${NC}" + ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") + continue + else + echo -e "${GREEN}✅ SSM Run As tag added for ${IAM_USERNAME} (will run as ${username})${NC}" + break + fi + done + + # Increment the next_user_id + if [[ "$user_id" == "$next_user_id" ]]; then + ((next_user_id++)) + fi + + echo -e "${BLUE}========================================${NC}" + done + + echo -e "${GREEN}✅ User configuration completed. Users have been written to shared_users.txt${NC}" + echo -e "\n${BLUE}Please review the user configuration below. Press Enter to confirm and upload to S3, or Ctrl+C to exit${NC}" + echo -e "${YELLOW}Contents of shared_users.txt:${NC}" + cat shared_users.txt + + read + + echo -e "${BLUE}Uploading shared_users.txt to S3 bucket: $BUCKET...${NC}" + + if ! output=$(aws s3 cp shared_users.txt s3://${BUCKET}/src/ --output json 2>&1); then + echo -e "${YELLOW}⚠️ Error occurred while uploading shared_users.txt to S3 bucket:${NC}" + echo -e "${YELLOW}$output${NC}" + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (If you do this, please make sure you upload the file manually before creating the cluster)" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + else + echo -e "${GREEN}✅ User configuration file uploaded successfully to s3://${BUCKET}/src/shared_users.txt${NC}" + fi + else + echo -e "${YELLOW}Skipping user configuration...${NC}" + fi + echo -e "\n${BLUE}=== User Configuration Complete ===${NC}" +} + +# Function to create the cluster +create_cluster() { + echo -e "${GREEN}✅ Creating cluster for you!${NC}" + + if ! output=$(aws sagemaker create-cluster \ + --cli-input-json file://cluster-config.json \ + --region $AWS_REGION \ + --output json 2>&1); then + + echo -e "${YELLOW}⚠️ Error occurred while creating the cluster:${NC}" + echo -e "${YELLOW}$output${NC}" + + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (you will run the command below yourself!)" + echo -e "2. Press Ctrl+C to exit the script." + + # Command to create the cluster + echo -e "${GREEN} aws sagemaker create-cluster \\" + echo -e "${GREEN} --cli-input-json file://cluster-config.json \\" + echo -e "${GREEN} --region $AWS_REGION --output json${NC}\n" + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + else + echo -e "${GREEN}✅ Cluster creation request submitted successfully. To monitor the progress of cluster creation, you can either check the SageMaker console, or you can run:.${NC}" + echo -e "${YELLOW}watch -n 1 aws sagemaker list-clusters --output table${NC}" + fi +} + +# Warning message function +warning() { + echo -e "${BLUE}⚠️ Please note:${NC}" + echo -e " - Cluster creation may take some time (~15-20 min)" + echo -e " - This operation may incur costs on your AWS account" + echo -e " - Ensure you understand the implications before proceeding\n" +} + +# Function to display goodbye message +goodbye() { + # Final goodbye message + echo -e "${GREEN}Thank you for using the SageMaker HyperPod Cluster Creation Script!${NC}" + echo -e "${GREEN}For any issues or questions, please refer to the AWS documentation.${NC}" + echo "https://docs.aws.amazon.com/sagemaker/latest/dg/smcluster-getting-started.html" + + # Exit message + echo -e "\n${BLUE}Exiting script. Good luck with your SageMaker HyperPod journey! 👋${NC}\n" +} + +#===Main Script=== +main() { + print_header "🚀 Welcome to the SageMaker HyperPod Slurm Cluster Creation Script! 🚀" + + # Prerequisites + display_important_prereqs + + # Checking AWS Account ID + echo -e "\n${BLUE}🔍 AWS Account Verification${NC}" + echo -e "Your AWS Account ID is: ${GREEN}$AWS_ACCOUNT_ID${NC}" + echo "Press Enter to confirm ✅ or Ctrl+C to exit❌..." + read + + # Checking Git installation + check_git + + # Checking AWS CLI version and installation + echo -e "\n${BLUE}📦 1a: AWS CLI Installation and Verification${NC}" + check_and_install_aws_cli + + # Checking Region + echo -e "\n${BLUE}🌎 AWS Region Configuration${NC}" + region_check + + # Lifecycle Scripts Setup + echo -e "\n${BLUE}🔧 Setting Up Lifecycle Scripts${NC}" + echo -e "${BLUE}1b. Configuring environment variables and lifecycle scripts...${NC}" + setup_env_vars + setup_lifecycle_scripts + echo -e "${GREEN}✅ Lifecycle scripts setup completed${NC}" + + + # Cluster Configuration + echo -e "\n${BLUE}🚀 Creating the Cluster${NC}" + echo -e "${BLUE}1c. Generating cluster configuration...${NC}" + create_config + echo -e "${GREEN}✅ Cluster configuration created successfully${NC}" + echo -e "${BLUE}ℹ️ Validating the generated configuration before proceeding${NC}" + + if error_output=$(validate_cluster_config 2>&1); then + echo -e "${GREEN}✅ Cluster configuration validated!${NC}" + else + echo -e "${YELLOW}⚠️ Error occurred while validating cluster config script:${NC}" + echo -e "${YELLOW}$error_output${NC}" + echo -e "Options:" + echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" + echo -e "2. Press Ctrl+C to exit the script." + + read -e -p "Select an option (Enter/Ctrl+C): " choice + + if [[ -z "$choice" ]]; then + echo -e "${BLUE}Continuing with the rest of the script...${NC}" + else + exit 1 + fi + fi + + + echo -e "${BLUE}ℹ️ For your viewing, here's the cluster configuration generated. Please make sure it looks right before proceeding. Press enter to continue, or Ctrl+C to exit and make changes${NC}" + echo -e "${YELLOW}$(cat cluster-config.json | jq . --color-output)${NC}" + read + + configure_cluster_users + + print_header "🎉 Cluster Creation Script Completed! 🎉" + + # Instructions for next steps + echo -e "${GREEN}Congratulations! You've completed all the preparatory steps.${NC}" + echo -e "${YELLOW}Next Steps:${NC}" + + CREATE_CLUSTER=$(get_input "Do you want the script to create the cluster for you now? (yes/no):" "yes") + # read -e -p "Do you want the script to create the cluster for you now? (yes/no): " CREATE_CLUSTER + if [[ "$CREATE_CLUSTER" == "yes" ]]; then + warning + create_cluster + goodbye + else + echo -e "${YELLOW}Run the following command to create the cluster. Exiting this script!${NC}" + + # Command to create the cluster + echo -e "${GREEN} aws sagemaker create-cluster \\" + echo -e "${GREEN} --cli-input-json file://cluster-config.json \\" + echo -e "${GREEN} --region $AWS_REGION --output json${NC}\n" + + echo -e "${YELLOW}To monitor the progress of cluster creation, you can either check the SageMaker console, or you can run:.${NC}" + echo -e "${GREEN}watch -n 1 aws sagemaker list-clusters --output table${NC}" + + \ + warning + goodbye + fi +} + +main From 79e2c3a7da26d5b1130a6f267bb4bc7adc467320 Mon Sep 17 00:00:00 2001 From: Basira Daqiq Date: Thu, 31 Jul 2025 21:16:40 +0000 Subject: [PATCH 15/15] Automate slurm deployment on EKS HP cluster(Slinky) This commit introduces a Bash script that automates the deployment of Slinky on an Amazon SageMaker HyperPod EKS cluster. The script streamlines the setup process with the following key features: - Cluster configuration generation based on a CloudFormation stack - Creation of an FSx for Lustre storage class - Installation of the AWS Load Balancer Controller - Setup of Slinky prerequisites (Prometheus, cert-manager, Slurm Operator) - Dynamic Slurm cluster configuration based on instance types - Creation and verification of FSx Persistent Volume Claims (PVCs) - Slurm cluster deployment using Helm - Configuration of a Network Load Balancer (NLB) for login node access The script includes error handling, progress tracking, and detailed logging to ensure a smooth deployment experience. It is designed to support both G5 and P5 instance types by dynamically allocating resources based on GPU count and EFA support. Note: This script has only been tested on G5 instance types. Dynamic resource allocation is handled by the dynamic_pods_allocation function, which is currently invoked within the set_slurm_values function. This dynamic behavior has not been fully tested. If you encounter issues with G5 instance deployments, consider commenting out the call to dynamic_pods_allocation in set_slurm_values. The script should still work correctly in this mode to function for P5 instance types, where a single pod is deployed per node. slinky automation tidy up added back a deleted file --- .../7.sagemaker-hyperpod-eks/create_config.sh | 0 .../automate-SlinkyCluster-creation.sh | 1251 ----------------- .../slinky-slurm/slinky_installation.sh | 1126 +++++++++++++++ 3 files changed, 1126 insertions(+), 1251 deletions(-) mode change 100755 => 100644 1.architectures/7.sagemaker-hyperpod-eks/create_config.sh delete mode 100644 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh create mode 100755 1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky_installation.sh diff --git a/1.architectures/7.sagemaker-hyperpod-eks/create_config.sh b/1.architectures/7.sagemaker-hyperpod-eks/create_config.sh old mode 100755 new mode 100644 diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh deleted file mode 100644 index 4864044bb..000000000 --- a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/automate-SlinkyCluster-creation.sh +++ /dev/null @@ -1,1251 +0,0 @@ -#!/bin/bash - -# Workshop Automation Script -# This script automates the steps of the workshop by executing CLI commands - -# Exit immediately if a command exits with a non-zero status. Print commands and their arguments as executed -set -e - -#===Global=== -export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) -export AWS_REGION=${AWS_DEFAULT_REGION:-$(aws configure get region)} -TOTAL_STEPS=5 -CURRENT_STEP=0 - -#===Style Definitions=== -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Function to print a yellow header -print_header() { - echo -e "\n${BLUE}=================================================${NC}" - echo -e "\n${YELLOW}==== $1 ====${NC}\n" - echo -e "\n${BLUE}=================================================${NC}" - -} - -# UX Function for a Progress Bar :) -progress_bar() { - local duration=$1 - local steps=$2 - local width=50 - local progress=0 - - for ((i=0; i /dev/null; then - echo -e "${YELLOW}⚠️ AWS CLI is not installed. Installing...${NC}" - install_aws_cli - else - echo -e "${GREEN}✅ AWS CLI found. Checking version...${NC}" - CLI_VERSION=$(aws --version | awk '{print $1}' | cut -d/ -f2) - - echo -e "${BLUE}Current version: ${YELLOW}$CLI_VERSION${NC}" - echo -e "${BLUE}Min. required version: ${YELLOW}2.17.1${NC}" - - if [[ "$(printf '%s\n' "2.17.1" "$CLI_VERSION" | sort -V | head -n1)" != "2.17.1" ]]; then - echo -e "${YELLOW}⚠️ AWS CLI version $CLI_VERSION is lower than required.${NC}" - echo -e "${YELLOW} Updating AWS CLI...${NC}" - install_aws_cli - else - echo -e "${GREEN}✅ AWS CLI version $CLI_VERSION is up to date.${NC}" - fi - fi - - echo -e "${BLUE}=== AWS CLI Check Complete ===${NC}\n" - -} - -# Function to check if Git is installed and configured -check_git() { - if ! command -v git &> /dev/null; then - echo "Git is not installed. Please install Git and try again." - exit 1 - fi -} - -# clone_adt() { -# REPO_NAME="awsome-distributed-training" -# if [ -d "$REPO_NAME" ]; then -# echo -e "${YELLOW}⚠️ The directory '$REPO_NAME' already exists.${NC}" -# echo -e "${GREEN}Do you want to remove it and clone again? (yes/no): ${NC}" -# read -e REMOVE_AND_CLONE -# if [[ $REMOVE_AND_CLONE == "yes" ]]; then -# echo -e "${YELLOW}Removing existing directory...${NC}" -# rm -rf "$REPO_NAME" -# echo -e "${BLUE}Cloning repository...${NC}" -# git clone --depth=1 https://github.com/aws-samples/awsome-distributed-training/ -# echo -e "${GREEN}✅ Repository cloned successfully${NC}" -# else -# echo -e "${BLUE}Using existing directory...${NC}" -# fi -# else -# echo -e "${BLUE}Cloning repository $REPO_NAME...${NC}" -# git clone --depth=1 https://github.com/aws-samples/awsome-distributed-training/ -# echo -e "${GREEN}✅ Repository cloned successfully${NC}" -# fi -# } - -clone_adt() { - REPO_NAME="awsome-distributed-training" - BRANCH_NAME="feature/slinkly-slurm-hyperpod-eks" - if [ -d "$REPO_NAME" ]; then - echo -e "${YELLOW}⚠️ The directory '$REPO_NAME' already exists.${NC}" - echo -e "${GREEN}Do you want to remove it and clone again? (yes/no): ${NC}" - read -e REMOVE_AND_CLONE - if [[ $REMOVE_AND_CLONE == "yes" ]]; then - echo -e "${YELLOW}Removing existing directory...${NC}" - rm -rf "$REPO_NAME" - echo -e "${BLUE}Cloning repository...${NC}" - git clone -b $BRANCH_NAME --depth=1 https://github.com/aws-samples/awsome-distributed-training/ - echo -e "${GREEN}✅ Repository cloned successfully${NC}" - else - echo -e "${BLUE}Using existing directory...${NC}" - fi - else - echo -e "${BLUE}Cloning repository $REPO_NAME...${NC}" - git clone -b $BRANCH_NAME --depth=1 https://github.com/aws-samples/awsome-distributed-training/ - echo -e "${GREEN}✅ Repository cloned successfully${NC}" - fi -} -# Function for multi-headnode feature for SMHP SLURM cluster -#MH -multi_headnode() { - source env_vars - echo -e "${BLUE}=== Multi-Headnode Feature ===${NC}" - MULTI_HEADNODE=$(get_input "Do you want to enable multi-headnode feature?" "no") - if [[ $MULTI_HEADNODE == "yes" ]]; then - export MH=true - local SHOULD_DEPLOY=true - # Query for BackupPrivateSubnet and FSxLustreFilesystemDNSname in create_config.sh - # DONE - - export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the name for the SageMaker HyperPod Multiheadnode stack to be deployed" "sagemaker-hyperpod-mh") - - # Check if stack already exists and has required outputs - if aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} >/dev/null 2>&1; then - echo -e "${YELLOW}⚠️ A stack with name '${MULTI_HEAD_SLURM_STACK}' already exists${NC}" - echo -e "${YELLOW}Note: The new stack's AZs must match the existing stack's AZs for the multi-headnode feature to work properly (${SUBNET_ID}, ${BACKUP_SUBNET})${NC}" - echo -e "${BLUE}Would you like to deploy a stack with a different name? (yes/no)${NC}" - read -e DEPLOY_NEW_STACK - - if [[ $DEPLOY_NEW_STACK != "yes" ]]; then - echo -e "${YELLOW}Using existing stack '${MULTI_HEAD_SLURM_STACK}'${NC}" - SHOULD_DEPLOY=false - else - export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the NEW name for the SageMaker HyperPod Multiheadnode stack to be deployed)" "sagemaker-hyperpod-mh") - fi - fi - - # Source env_vars - source env_vars - - if [[ $SHOULD_DEPLOY == true ]]; then - # Ask user to input EMAIL and DB_USER_NAME - export EMAIL=$(get_input "Input your SNSSubEmailAddress here (this is the email address that will be used to send notifications about your head node status)" "johndoe@example.com") - export DB_USER_NAME=$(get_input "Input your DB_USER_NAME here (this is the username that will be used to access the SlurmDB)" "johndoe") - # export MULTI_HEAD_SLURM_STACK=$(get_input "Enter the name for the SageMaker HyperPod Multiheadnode stack to be deployed)" "sagemaker-hyperpod-mh") - - echo -e "${YELLOW}The following CloudFormation command will be executed:${NC}" - echo -e "${GREEN}aws cloudformation deploy \\ - --template-file awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/sagemaker-hyperpod-slurm-multi-headnode.yaml \\ - --stack-name ${MULTI_HEAD_SLURM_STACK} \\ - --parameter-overrides \\ - SlurmDBSecurityGroupId=${SECURITY_GROUP} \\ - SlurmDBSubnetGroupId1=${SUBNET_ID} \\ - SlurmDBSubnetGroupId2=${BACKUP_SUBNET} \\ - SNSSubEmailAddress=${EMAIL} \\ - SlurmDBUsername=${DB_USER_NAME} \\ - --capabilities CAPABILITY_NAMED_IAM${NC}" - echo -e "\n${YELLOW}This will create the following resources in your account:${NC}" - echo -e "- Amazon RDS instance for SLURM database" - echo -e "- Amazon SNS topic for head node failover notifications" - echo -e "- IAM roles and policies for multi-head node functionality" - - echo -e "\n${BLUE}Would you like to proceed with the deployment? Please acnowledge that you allow CloudFormation to create resources in your account by hitting ENTER${NC}" - read - - # Deploy the multi-head CF stack - aws cloudformation deploy \ - --template-file awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/sagemaker-hyperpod-slurm-multi-headnode.yaml \ - --stack-name ${MULTI_HEAD_SLURM_STACK} \ - --parameter-overrides \ - SlurmDBSecurityGroupId=${SECURITY_GROUP} \ - SlurmDBSubnetGroupId1=${SUBNET_ID} \ - SlurmDBSubnetGroupId2=${BACKUP_SUBNET} \ - SNSSubEmailAddress=${EMAIL} \ - SlurmDBUsername=${DB_USER_NAME} \ - --capabilities CAPABILITY_NAMED_IAM - - # Wait for stack to be created - echo -e "${BLUE}Waiting for multi-headnode stack creation to complete...${NC}" - aws cloudformation wait stack-create-complete \ - --stack-name ${MULTI_HEAD_SLURM_STACK} - else - # Get the outputs for EMAIL and DB_USER_NAME (used in provisioning_parameters.json!!!) - echo "From Stack: ${MULTI_HEAD_SLURM_STACK}" - export EMAIL=$(aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} --query 'Stacks[0].Outputs[?OutputKey==`SNSSubEmailAddress`].OutputValue' --output text) - export DB_USER_NAME=$(aws cloudformation describe-stacks --stack-name ${MULTI_HEAD_SLURM_STACK} --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBUsername`].OutputValue' --output text) - - echo -e "Set Email: ${EMAIL}, DB Username: ${DB_USER_NAME}" - fi - - # Query new stack for SlurmDBEndpointAddress SlurmDBSecretArn SlurmExecutionRoleArn SlurmFailOverSNSTopicArn and write these to env_vars - SLURM_DB_ENDPOINT_ADDRESS=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBEndpointAddress`].OutputValue' --output text) - SLURM_DB_SECRET_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmDBSecretArn`].OutputValue' --output text) - SLURM_EXECUTION_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmExecutionRoleArn`].OutputValue' --output text) - SLURM_SNS_FAILOVER_TOPIC_ARN=$(aws cloudformation describe-stacks --stack-name $MULTI_HEAD_SLURM_STACK --query 'Stacks[0].Outputs[?OutputKey==`SlurmFailOverSNSTopicArn`].OutputValue' --output text) - - echo "export SLURM_DB_ENDPOINT_ADDRESS=${SLURM_DB_ENDPOINT_ADDRESS}" >> env_vars - echo "export SLURM_DB_SECRET_ARN=${SLURM_DB_SECRET_ARN}" >> env_vars - echo "export SLURM_EXECUTION_ROLE_ARN=${SLURM_EXECUTION_ROLE_ARN}" >> env_vars - echo "export SLURM_SNS_FAILOVER_TOPIC_ARN=${SLURM_SNS_FAILOVER_TOPIC_ARN}" >> env_vars - echo "export EMAIL=${EMAIL}" >> env_vars - echo "export DB_USER_NAME=${DB_USER_NAME}" >> env_vars - - if [[ -z "$SLURM_DB_ENDPOINT_ADDRESS" ]] || [[ -z "$SLURM_DB_SECRET_ARN" ]] || [[ -z "$SLURM_EXECUTION_ROLE_ARN" ]] || [[ -z "$SLURM_SNS_FAILOVER_TOPIC_ARN" ]]; then - echo -e "${YELLOW}⚠️ Failed to retrieve all required values from the CloudFormation stack${NC}" - echo -e "${YELLOW}Please ensure the stack deployed correctly and all outputs are available${NC}" - return 1 - fi - - SLURM_EXECUTION_ROLE=$(echo $SLURM_EXECUTION_ROLE_ARN | awk -F'/' '{print $NF}') - - echo -e "${GREEN}✅ Multi-headnode feature enabled${NC}" - - # Create IAM policy for multi-headnode feature - echo -e "\n${BLUE}Creating IAM policy for SLURM execution role...${NC}" - - create_and_attach_policy() { - aws iam create-policy \ - --policy-name AmazonSageMakerExecutionPolicy \ - --policy-document file://awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json --output json && \ - aws iam attach-role-policy \ - --role-name $SLURM_EXECUTION_ROLE \ - --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/AmazonSageMakerExecutionPolicy - } - - if error_output=$(create_and_attach_policy 2>&1); then - echo -e "${GREEN}✅ IAM policy created and attached successfully${NC}" - else - echo -e "${YELLOW}⚠️ Error occurred while creating/attaching IAM policy:${NC}" - echo -e "${YELLOW}$error_output${NC}" - - if [[ $error_output == *"EntityAlreadyExists"* ]]; then - echo -e "\n${YELLOW}If the error you received is that the policy already exists, you can either:${NC}" - echo -e "\n${GREEN} 1. Continue the script with the existing policy (make sure the permissions match the ones in https://github.com/aws-samples/awsome-distributed-training/blob/main/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json) and manually attach it to your role ${SLURM_EXECUTION_ROLE}, or${NC}" - echo -e "\n${GREEN} 2. You can create a new policy with a name different than 'AmazonSageMakerExecutionPolicy' manually and attach it to your 'AmazonSageMakerExecutionRole' with the following command. Once you do that, you can continue with the rest of the script:${NC}" - - echo -e "\n${YELLOW} Creating an IAM policy (required for option 2 above)${NC}" - echo -e "\n${BLUE} aws iam create-policy \\ - --policy-name \\ - --policy-document file://awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/1.AmazonSageMakerClustersExecutionRolePolicy.json${NC}" - - echo -e "\n${YELLOW} Attach an IAM policy to an IAM role (required for options 1 & 2 above)${NC}" - echo -e "\n${BLUE} aws iam attach-role-policy \\ - --role-name ${SLURM_EXECUTION_ROLE} \\ - --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/${NC}" - fi - - echo -e "Options:" - echo -e "1. [RECOMMENDED, PLEASE READ ABOVE] Press Enter to continue with the rest of the script" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - fi - else - echo -e "${YELLOW}Skipping multi-headnode configuration...${NC}" - export MH=false - fi - echo -e "\n${BLUE}=== Multi-Headnode Configuration Complete ===${NC}" -} - -# Function to setup environment variables -setup_env_vars() { - echo -e "${BLUE}=== Setting Up Environment Variables ===${NC}" - echo -e "${GREEN}Cloning awsome-distributed-training${NC}" - clone_adt - - echo -e "${BLUE}Enter the name of the SageMaker VPC CloudFormation stack that was deployed as a prerequisite (default: sagemaker-hyperpod):${NC}" - read -e STACK_ID_VPC - export STACK_ID_VPC=${STACK_ID_VPC:-sagemaker-hyperpod} - - if [ "$CF_STACK_NAME" != "sagemaker-hyperpod" ]; then - echo -e "${GREEN}✅ Configuration script updated with stack name: $STACK_ID_VPC${NC}" - else - echo -e "${GREEN}Using default stack name: sagemaker-hyperpod${NC}" - fi - - # Clear env_vars from previous runs - > env_vars - - echo -e "${YELLOW}Generating new environment variables...${NC}" - - generate_env_vars() { - bash awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/create_config.sh - # bash create_config.sh - } - - # Capture stdout + stderr - if error_output=$(generate_env_vars 2>&1); then - echo -e "${GREEN}✅ New environment variables generated and sourced${NC}" - else - echo -e "${YELLOW}⚠️ Error occurred while generating environment variables:${NC}" - echo -e "${YELLOW}$error_output${NC}" - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - fi - fi - - # FEAT: Add support for multiple headnodes - #MH - multi_headnode - - source env_vars - - echo -e "\n${BLUE}=== Environment Variables Summary ===${NC}" - echo -e "${YELLOW}Note: You may ignore the INSTANCES parameter for now${NC}" - echo -e "${GREEN}Current environment variables:${NC}" - cat env_vars - - echo -e "\n${BLUE}=== Environment Setup Complete ===${NC}" -} - -# Function to setup lifecycle scripts -setup_lifecycle_scripts() { - echo -e "${BLUE}=== Setting Up Lifecycle Scripts ===${NC}" - - cd awsome-distributed-training/1.architectures/5.sagemaker-hyperpod/LifecycleScripts/ - - echo -e "${YELLOW}Are you using Neuron-based instances (Trainium/Inferentia)? (yes/no)${NC}" - read -e USING_NEURON - - if [ "$USING_NEURON" == "yes" ]; then - echo -e "${BLUE}Enabling Neuron in LCS...${NC}" - sed -i.bak 's/enable_update_neuron_sdk = False/enable_update_neuron_sdk = True/' base-config/config.py - rm base-config/config.py.bak - echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! Neuron enabled in config.py${NC}" - else - echo -e "${BLUE}Continuing with Neuron disabled in LCS...${NC}" - fi - - # Check if FSx OpenZFS was deployed in the stack - echo -e "${BLUE}Checking if FSx OpenZFS was deployed in the stack...${NC}" - - export ENABLE_FSX_OPENZFS="false" - - FSX_OPENZFS_DNS=$(aws cloudformation describe-stacks \ - --stack-name "${STACK_ID_VPC}" \ - --query 'Stacks[0].Outputs[?OutputKey==`FSxOpenZFSFileSystemDNSname`].OutputValue' \ - --output text) - - if [ -n "$FSX_OPENZFS_DNS" ]; then - echo -e "${BLUE}FSx OpenZFS detected in stack. DNS: ${FSX_OPENZFS_DNS}${NC}" - echo -e "${BLUE}Enabling FSx OpenZFS in LCS...${NC}" - - # Get the FSx OpenZFS File System ID as well - FSX_OPENZFS_ID=$(aws cloudformation describe-stacks \ - --stack-name "${STACK_ID_VPC}" \ - --query 'Stacks[0].Outputs[?OutputKey==`FSxOpenZFSFileSystemId`].OutputValue' \ - --output text) - - ENABLE_FSX_OPENZFS="true" - echo "export FSX_OPENZFS_DNS=${FSX_OPENZFS_DNS}" >> env_vars - echo "export FSX_OPENZFS_ID=${FSX_OPENZFS_ID}" >> env_vars - - # Update config.py - sed -i.bak 's/enable_fsx_openzfs = False/enable_fsx_openzfs = True/' base-config/config.py - rm base-config/config.py.bak - - echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! FSx OpenZFS enabled in config.py${NC}" - else - echo -e "${BLUE}No FSx OpenZFS detected in stack. Continuing with FSx OpenZFS disabled in LCS...${NC}" - fi - - echo -e "${YELLOW}Did you deploy the optional hyperpod-observability CloudFormation stack? (yes/no)${NC}" - read -e DEPLOYED_OBSERVABILITY - - if [ "$DEPLOYED_OBSERVABILITY" == "yes" ]; then - echo -e "${BLUE}Enabling observability in LCS...${NC}" - sed -i.bak 's/enable_observability = False/enable_observability = True/' base-config/config.py - rm base-config/config.py.bak - echo -e "${GREEN}✅ Lifecycle Scripts modified successfully! Observability enabled in config.py${NC}" - - echo -e "${BLUE}Attaching IAM policies for observability to $ROLENAME${NC}" - - # Helper function for attaching IAM policies (specific to observability stack only!) - attach_policies() { - aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess --output json - aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AWSCloudFormationReadOnlyAccess --output json - } - - # Capture stdout + stderr - - if ! error_output=$(attach_policies 2>&1); then - echo -e "${YELLOW}⚠️ Failed to attach IAM policies. This operation requires admin permissions${NC}" - echo -e "${YELLOW} This was the error received${NC}" - echo -e "${YELLOW}$error_output${NC}" - echo -e "Options:" - echo -e "1. Run 'aws configure' as an admin user as part of this script." - echo -e "2. Press Ctrl+C to exit and run 'aws configure' as an admin user outside this script." - echo -e "3. Press Enter to continue with the rest of the script without configuring this step." - - read -e -p "Choose an option (1, 2, or 3): " choice - - case $choice in - 1) - echo -e "${BLUE}Running 'aws configure'. Please enter your **admin** credentials..${NC}" - aws configure - echo -e "${GREEN}✅ AWS CLI configured successfully${NC}" - echo -e "${BLUE}Retrying to attach IAM policies!${NC}" - if ! attach_policies; then - echo -e "${YELLOW}⚠️ Failed to attach IAM policies. Please attach the following policies manually:${NC}" - echo -e "1. AmazonPrometheusRemoteWriteAccess" - echo -e "2. AWSCloudFormationReadOnlyAccess" - echo -e "Press Enter to continue with the rest of the script without configuring this step." - read -e -p "Press Enter to continue: " - echo -e "${BLUE}Continuing with the rest of the script without configuring this step.${NC}" - else - echo -e "${GREEN}✅ IAM policies attached successfully${NC}" - fi - ;; - 2) - echo -e "${BLUE}Please run 'aws configure' as an admin user outside this script.${NC}" - exit 1 - ;; - 3) - echo -e "${BLUE}Continuing with the rest of the script without configuring this step.${NC}" - ;; - *) - echo -e "${BLUE}Invalid choice. Continuing with the rest of the script without configuring this step.${NC}" - ;; - esac - else - echo -e "${GREEN}✅ IAM policies attached successfully${NC}" - fi - echo -e "${GREEN}✅ Observability setup complete!${NC}" - else - echo -e "${YELLOW}Observability not enabled. Continuing with default configuration${NC}" - fi - - echo -e "${BLUE}Uploading your lifecycle scripts to S3 bucket ${YELLOW}${BUCKET}${NC}" - # upload data - upload_to_s3() { - aws s3 cp --recursive base-config/ s3://${BUCKET}/src --output json - } - - if error_output=$(upload_to_s3 2>&1); then - echo -e "${GREEN}✅ Lifecycle scripts uploaded successfully${NC}" - else - echo -e "${YELLOW}⚠️ Error occurred while uploading lifecycle scripts to S3 bucket:${NC}" - echo -e "${YELLOW}$error_output${NC}" - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - fi - - # move back to env_var directory - cd ../../../.. - - echo -e "\n${BLUE}=== Lifecycle Scripts Setup Complete ===${NC}" -} - -# Helper function to get user inputs with default values specified -get_input() { - local prompt="$1" - local default="$2" - local input - read -e -p "$prompt [$default]: " input - echo "${input:-$default}" -} - -# Function to write the cluster-config.json file -create_config() { - echo -e "\n${BLUE}=== Lifecycle Scripts Setup Complete ===${NC}" - - # Get controller machine details - CONTROLLER_NAME=$(get_input "Enter the name for the controller instance group" "controller-machine") - CONTROLLER_TYPE=$(get_input "Enter the instance type for the controller" "ml.m5.12xlarge") - - # Initialize instance groups array - INSTANCE_GROUPS="[" - - # Add login group - echo -e "${GREEN}Do you want to add a login group? (yes/no): ${NC}" - read -e ADD_LOGIN_GROUP - - if [[ $ADD_LOGIN_GROUP == "yes" ]]; then - LOGIN_TYPE=$(get_input "Enter the instance type for the login group" "ml.m5.4xlarge") - - INSTANCE_GROUPS+="{ - \"InstanceGroupName\": \"login-group\", - \"InstanceType\": \"$LOGIN_TYPE\", - \"InstanceStorageConfigs\": [ - { - \"EbsVolumeConfig\": { - \"VolumeSizeInGB\": 500 - } - } - ], - \"InstanceCount\": 1, - \"LifeCycleConfig\": { - \"SourceS3Uri\": \"s3://${BUCKET}/src\", - \"OnCreate\": \"on_create.sh\" - }, - \"ExecutionRole\": \"${ROLE}\", - \"ThreadsPerCore\": 2 - }," - - echo -e "${GREEN}✅ Login Group added${NC}" - fi - - CONTROLLER_COUNT=$([ "${MH:-false}" = true ] && echo "2" || echo "1") - EXECUTION_ROLE=$([ "${MH:-false}" = true ] && echo "${SLURM_EXECUTION_ROLE_ARN}" || echo "${ROLE}") - - # Add controller group - INSTANCE_GROUPS+="{ - \"InstanceGroupName\": \"$CONTROLLER_NAME\", - \"InstanceType\": \"$CONTROLLER_TYPE\", - \"InstanceStorageConfigs\": [ - { - \"EbsVolumeConfig\": { - \"VolumeSizeInGB\": 500 - } - } - ], - \"InstanceCount\": ${CONTROLLER_COUNT}, - \"LifeCycleConfig\": { - \"SourceS3Uri\": \"s3://${BUCKET}/src\", - \"OnCreate\": \"on_create.sh\" - }, - \"ExecutionRole\": \"${EXECUTION_ROLE}\", - \"ThreadsPerCore\": 2 - }" - - # Loop to add worker instance groups - WORKER_GROUP_COUNT=1 - echo -e "\n${BLUE}=== Worker Group Configuration ===${NC}" - while true; do - if [[ $WORKER_GROUP_COUNT -eq 1 ]]; then - ADD_WORKER=$(get_input "Do you want to add a worker instance group? (yes/no):" "yes") - else - ADD_WORKER=$(get_input "Do you want to add another worker instance group? (yes/no):" "no") - fi - - if [[ $ADD_WORKER != "yes" ]]; then - break - fi - - echo -e "${YELLOW}Configuring Worker Group $WORKER_GROUP_COUNT${NC}" - INSTANCE_TYPE=$(get_input "Enter the instance type for worker group $WORKER_GROUP_COUNT" "ml.c5.4xlarge") - INSTANCE_COUNT=$(get_input "Enter the instance count for worker group $WORKER_GROUP_COUNT" "4") - - echo -e "${GREEN}Are you using training plans? (yes/no): ${NC}" - read -e USE_TRAINING_PLAN - - INSTANCE_GROUPS+=", - { - \"InstanceGroupName\": \"worker-group-$WORKER_GROUP_COUNT\", - \"InstanceType\": \"$INSTANCE_TYPE\", - \"InstanceCount\": $INSTANCE_COUNT, - \"InstanceStorageConfigs\": [ - { - \"EbsVolumeConfig\": { - \"VolumeSizeInGB\": 500 - } - } - ], - \"LifeCycleConfig\": { - \"SourceS3Uri\": \"s3://${BUCKET}/src\", - \"OnCreate\": \"on_create.sh\" - }, - \"ExecutionRole\": \"${ROLE}\", - \"ThreadsPerCore\": 1" - - if [[ $USE_TRAINING_PLAN == "yes" ]]; then - echo -e "\n${BLUE}=== Training Plan Configuration ===${NC}" - # aws iam attach-role-policy --role-name $ROLENAME --policy-arn arn:aws:iam::aws:policy/AmazonEC2FullAccess - - TRAINING_PLAN=$(get_input "Enter the training plan name" "") - - count=0 - while true; do - # Attempt to describe the training plan - echo -e "${YELLOW}Attempting to retrieve training plan details...${NC}" - - if ! TRAINING_PLAN_DESCRIPTION=$(aws sagemaker describe-training-plan --training-plan-name "$TRAINING_PLAN" --output json 2>&1); then - echo -e "${BLUE}❌Error: Training plan '$TRAINING_PLAN' not found. Please try again.${NC}" - echo -e "${GREEN}Are you using training plans (Beta feature)? (yes/no)${NC}" - read -e USE_TRAINING_PLAN - if [[ $USE_TRAINING_PLAN != "yes" ]]; then - echo -e "${YELLOW}Exiting training plan configuration.${NC}" - break - else - TRAINING_PLAN=$(get_input "Enter the training plan name" "") - fi - else - # Extract relevant information from the description - TRAINING_PLAN_ARN=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.TrainingPlanArn') - AVAILABLE_INSTANCE_COUNT=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.AvailableInstanceCount') - TOTAL_INSTANCE_COUNT=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.TotalInstanceCount') - TRAINING_PLAN_AZ=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.ReservedCapacitySummaries[0].AvailabilityZone') - TP_INSTANCE_TYPE=$(echo "$TRAINING_PLAN_DESCRIPTION" | jq -r '.ReservedCapacitySummaries[0].InstanceType') - - CF_AZ=$(aws ec2 describe-subnets --subnet-ids $SUBNET_ID --output json | jq -r '.Subnets[0].AvailabilityZone') - - # Only print if count=0 - if [[ $count -eq 0 ]]; then - echo -e "${GREEN}Training Plan Details:${NC}" - echo -e " ${YELLOW}Name:${NC} $TRAINING_PLAN" - echo -e " ${YELLOW}Available Instance Count:${NC} $AVAILABLE_INSTANCE_COUNT" - echo -e " ${YELLOW}Total Instance Count:${NC} $TOTAL_INSTANCE_COUNT" - echo -e " ${YELLOW}Training Plan Availability Zone:${NC} $TRAINING_PLAN_AZ" - echo -e " ${YELLOW}Training Plan Instance Type:${NC} $TP_INSTANCE_TYPE" - fi - - # Compare INSTANCE_COUNT with AVAILABLE_INSTANCE_COUNT - INSTANCE_COUNT_OK="n" - if [[ $INSTANCE_COUNT -gt $AVAILABLE_INSTANCE_COUNT ]]; then - echo -e "${YELLOW}Warning: The requested instance count ($INSTANCE_COUNT) is greater than the available instances in the training plan ($AVAILABLE_INSTANCE_COUNT).${NC}" - echo -e "${BLUE}Do you want to continue anyway?(yes/no)${NC}" - read -e CONTINUE - if [[ $CONTINUE != "yes" ]]; then - NEW_INSTANCE_COUNT=$(get_input "Enter the new number of instances" "1") - # Update INSTANCE_GROUPS with new INSTANCE_COUNT for the current worker group - INSTANCE_GROUPS=$(echo "$INSTANCE_GROUPS" | perl -pe ' - BEGIN { - $group = "worker-group-'"$WORKER_GROUP_COUNT"'"; - $count = '"$NEW_INSTANCE_COUNT"'; - $in_group = 0; - } - if (/"InstanceGroupName":\s*"$group"/) { - $in_group = 1; - } - if ($in_group && /"InstanceCount":\s*\d+/) { - s/("InstanceCount":\s*)\d+/$1$count/; - $in_group = 0; - } - ') - INSTANCE_COUNT=$NEW_INSTANCE_COUNT - echo -e "${GREEN}Updated instance count for worker-group-$WORKER_GROUP_COUNT to $INSTANCE_COUNT${NC}" - fi - INSTANCE_COUNT_OK="y" - else - INSTANCE_COUNT_OK="y" - fi - - if [[ $INSTANCE_COUNT_OK == "y" ]]; then - INSTANCE_TYPE_OK="n" - # Compare INSTANCE_TYPE with TP_INSTANCE_TYPE - if [[ $INSTANCE_TYPE != $TP_INSTANCE_TYPE ]]; then - echo -e "${YELLOW}Warning: The requested instance type ($INSTANCE_TYPE) does not match the instance type in the training plan ($TP_INSTANCE_TYPE).${NC}" - echo -e "${BLUE}Do you want to continue anyway? If you choose "no", then the script will update instance type for you and proceed. (yes/no)${NC}" - read -e CONTINUE - if [[ $CONTINUE != "yes" ]]; then - NEW_INSTANCE_TYPE=$TP_INSTANCE_TYPE - # Update INSTANCE_GROUPS with new INSTANCE_TYPE for the current worker group - INSTANCE_GROUPS=$(echo "$INSTANCE_GROUPS" | perl -pe ' - BEGIN { - $group = "worker-group-'$WORKER_GROUP_COUNT'"; - $type = "'$NEW_INSTANCE_TYPE'"; - $in_group = 0; - } - if (/"InstanceGroupName":\s*"$group"/) { - $in_group = 1; - } - if ($in_group && /"InstanceType":\s*"[^"]*"/) { - s/("InstanceType":\s*")[^"]*"/$1$type"/; - $in_group = 0; - } - ') - INSTANCE_TYPE=$NEW_INSTANCE_TYPE - echo -e "${GREEN}Updated instance type for worker-group-$WORKER_GROUP_COUNT to $INSTANCE_TYPE${NC}" - fi - INSTANCE_TYPE_OK="y" - else - INSTANCE_TYPE_OK="y" - fi - - if [[ $INSTANCE_TYPE_OK == "y" ]]; then - # Compare TRAINING_PLAN_AZ with CF_AZ - if [[ $TRAINING_PLAN_AZ != $CF_AZ ]]; then - echo -e "${YELLOW}Warning: The training plan availability zone ($TRAINING_PLAN_AZ) does not match the cluster availability zone ($CF_AZ).${NC}" - echo -e "${BLUE}Do you want to continue anyway? (yes/no)${NC}" - read -e CONTINUE - if [[ $CONTINUE != "yes" ]]; then - echo -e "${YELLOW}Please ensure that your VPC is in the same Availability Zone as your training plan (or vice versa). If you used the workshop, this should be the CF stack \"sagemaker-hyperpod\". Exiting training plan configuration.${NC}" - continue - fi - fi - fi - fi - - echo -e "${GREEN}Adding Training Plan ARN to instance group configuration.${NC}" - INSTANCE_GROUPS+=", - \"TrainingPlanArn\": \"$TRAINING_PLAN_ARN\"" - break - fi - count+=1 - done - fi - - INSTANCE_GROUPS+=" - }" - - echo -e "${GREEN}✅ Worker Group $WORKER_GROUP_COUNT added${NC}" - ((WORKER_GROUP_COUNT++)) - done - - INSTANCE_GROUPS+="]" - - read -e -p "What would you like to name your cluster? (default: ml-cluster): " CLUSTER_NAME - CLUSTER_NAME=${CLUSTER_NAME:-ml-cluster} - - # Create the cluster-config.json file - cat > cluster-config.json << EOL - { - "ClusterName": "$CLUSTER_NAME", - "InstanceGroups": $INSTANCE_GROUPS, - "VpcConfig": { - "SecurityGroupIds": ["$SECURITY_GROUP"], - "Subnets":["$SUBNET_ID"] - } - } -EOL - - echo -e "${GREEN}✅ cluster-config.json created successfully${NC}" - - source env_vars - - echo -e "\n${YELLOW}Creating provisioning_parameters.json...${NC}" - WORKER_GROUPS="[" - - # Loop through worker groups - for ((i=1; i<=WORKER_GROUP_COUNT-1; i++)); do - if [ $i -gt 1 ]; then - WORKER_GROUPS+="," - fi - - instance_type=$(jq -r ".InstanceGroups[] | select(.InstanceGroupName == \"worker-group-$i\").InstanceType" cluster-config.json) - - WORKER_GROUPS+=" - { - \"instance_group_name\": \"worker-group-$i\", - \"partition_name\": \"$instance_type\" - }" - done - - WORKER_GROUPS+=" - ]" - - # OpenZFS - if [[ $ENABLE_FSX_OPENZFS == "true" ]]; then - FSX_OPENZFS_CONFIG=", - \"fsx_openzfs_dns_name\": \"${FSX_OPENZFS_ID}.fsx.${AWS_REGION}.amazonaws.com\"" - else - FSX_OPENZFS_CONFIG="" - fi - - #MH - if [[ $MH == "true" ]]; then - SLURM_CONFIGURATIONS=" - { - \"slurm_database_secret_arn\": \"$SLURM_DB_SECRET_ARN\", - \"slurm_database_endpoint\": \"$SLURM_DB_ENDPOINT_ADDRESS\", - \"slurm_shared_directory\": \"/fsx\", - \"slurm_database_user\": \"$DB_USER_NAME\", - \"slurm_sns_arn\": \"$SLURM_SNS_FAILOVER_TOPIC_ARN\" - }" - fi - - if [[ $ADD_LOGIN_GROUP == "yes" ]]; then - if [[ $MH == "true" ]]; then - cat > provisioning_parameters.json << EOL - { - "version": "1.0.0", - "workload_manager": "slurm", - "controller_group": "$CONTROLLER_NAME", - "login_group": "login-group", - "worker_groups": $WORKER_GROUPS, - "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", - "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG}, - "slurm_configurations": $SLURM_CONFIGURATIONS - } -EOL - else - cat > provisioning_parameters.json << EOL - { - "version": "1.0.0", - "workload_manager": "slurm", - "controller_group": "$CONTROLLER_NAME", - "login_group": "login-group", - "worker_groups": $WORKER_GROUPS, - "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", - "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG} - } -EOL - fi - else - if [[ $MH == "true" ]]; then - cat > provisioning_parameters.json << EOL - { - "version": "1.0.0", - "workload_manager": "slurm", - "controller_group": "$CONTROLLER_NAME", - "worker_groups": $WORKER_GROUPS, - "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", - "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG}, - "slurm_configurations": $SLURM_CONFIGURATIONS - } -EOL - else - cat > provisioning_parameters.json << EOL - { - "version": "1.0.0", - "workload_manager": "slurm", - "controller_group": "$CONTROLLER_NAME", - "worker_groups": $WORKER_GROUPS, - "fsx_dns_name": "${FSX_ID}.fsx.${AWS_REGION}.amazonaws.com", - "fsx_mountname": "${FSX_MOUNTNAME}"${FSX_OPENZFS_CONFIG} - } -EOL - fi - fi - - echo -e "${GREEN}✅ provisioning_parameters.json created successfully${NC}" - - # copy to the S3 Bucket - echo -e "\n${BLUE}Copying configuration to S3 bucket...${NC}" - - # upload data - upload_to_s3() { - aws s3 cp provisioning_parameters.json s3://${BUCKET}/src/ --output json - } - - if error_output=$(upload_to_s3 2>&1); then - echo -e "${GREEN}✅ Provisioning Parameters uploaded successfully${NC}" - else - echo -e "${YELLOW}⚠️ Error occurred while uploading lifecycle scripts to S3 bucket:${NC}" - echo -e "${YELLOW}$error_output${NC}" - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (Not Recommended)" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - fi - - echo -e "\n${BLUE}=== Cluster Configuration Complete ===${NC}" -} - -validate_cluster_config() { - echo "Validating your cluster configuration..." - # TODO: MAKE SURE PACKAGES ARE INSTALLED HERE!! - - curl -O https://raw.githubusercontent.com/aws-samples/awsome-distributed-training/main/1.architectures/5.sagemaker-hyperpod/validate-config.py - - # check config for known issues - python3 validate-config.py --cluster-config cluster-config.json --provisioning-parameters provisioning_parameters.json --region $AWS_REGION -} - -# Function to display the prerequisites before starting this workshop -display_important_prereqs() { - echo -e "${BLUE}Before running this script, please ensure the following:${NC}\n" - - echo -e "${GREEN}1. 🔑 IAM Credentials:${NC}" - echo " You have Administrator Access Credentials in IAM." - echo " This is crucial as we'll be using CloudFormation to create IAM roles and policies." - echo " Run 'aws configure' to set up your credentials." - - echo -e "\n${GREEN}2. 🌐 VPC Stack:${NC}" - echo " Deploy the sagemaker-hyperpod VPC stack using:" - echo " https://catalog.workshops.aws/sagemaker-hyperpod/en-US/00-setup/02-own-account" - echo " This creates essential resources: VPC, subnets, FSx Lustre volumes," - echo " S3 bucket, and IAM role for your SageMaker HyperPod cluster." - echo " ⚠️⚠️ IMPORTANT: If you choose a multi-head node configuration (i.e., multiple head nodes), then make sure that" - echo " the VPC stack has the \"(Optional) Availability zone id to deploy the backup private subnet\"". - - echo -e "\n${GREEN}3. 📊 Observability Stack:${NC}" - echo " It's highly recommended to deploy the observability stack as well." - echo " Navigate to https://catalog.workshops.aws/sagemaker-hyperpod/en-US/00-setup/02-own-account#2.-deploy-cluster-observability-stack-(recommended) to deploy the stack" - - echo -e "\n${GREEN}4. 💻 Development Environment:${NC}" - echo " Ensure you have a Linux-based development environment (macOS works great too)." - - echo -e "\n${GREEN}5. 🔧 Packages required for this script to run:${NC}" - echo " Ensure you install the following: pip, jq, boto3, and jsonschema" - - echo -e "\n${YELLOW}Ready to proceed? Press Enter to continue or Ctrl+C to exit...${NC}" - read -} - -region_check() { - echo -e "${BLUE}Please confirm that your AWS region is ${GREEN}$AWS_REGION${BLUE} (default).${NC}" - echo -e "${BLUE}If not, enter the AWS region where you want to set up your cluster (e.g., us-west-2):${NC}" - - read -p "> " NEW_REGION - - if [[ -z "$NEW_REGION" ]]; then - echo -e "${GREEN}✅ Using default region: ${YELLOW}$AWS_REGION${NC}" - else - export AWS_REGION="$NEW_REGION" - echo -e "${GREEN}✅ Region updated to: ${YELLOW}$AWS_REGION${NC}" - fi - - echo -e "\n${BLUE}Your region is set to: ${YELLOW}$AWS_REGION${NC}" - echo -e "${BLUE}Ensure your chosen region supports SageMaker HyperPod.${NC}" - echo -e "${GREEN}You can check out https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-hyperpod.html#sagemaker-hyperpod-available-regions to learn about supported regions.${NC}" - echo -e "${BLUE}Press Enter to continue...${NC}" - read -} - -# Function to create users in cluster -configure_cluster_users() { - echo -e "\n${BLUE}=== User Configuration ===${NC}" - - CONFIGURE_USERS=$(get_input "Would you like to configure users? If not, you can still use the ubuntu user (yes/no)" "no") - - FIRST_SSM_INTEGRATION=true - - if [[ "${CONFIGURE_USERS}" == "yes" ]]; then - echo -e "${BLUE}Creating shared_users.txt file...${NC}" - - # Initialize or clear the shared_users.txt file - > shared_users.txt - - # Initialize the user ID counter - next_user_id=2001 - - echo -e "${YELLOW}Enter user details (Press Ctrl+D when finished)${NC}" - echo -e "${BLUE}========================================${NC}" - - while IFS= read -p "Enter username: " username; do - # If username is empty, skip this iteration - if [[ -z "$username" ]]; then - continue - fi - - # Get user ID with default value - user_id=$(get_input "Enter user ID" "$next_user_id") - - # Write to shared_users.txt - echo "${username},${user_id},/fsx/${username}" >> shared_users.txt - - # SSM Integration - ASSOCIATE_IAM=$(get_input "[REQUIRES ADMIN] Would you like to associate this POSIX user with an IAM user? (yes/no)" "no") - - while [[ "${ASSOCIATE_IAM}" == "yes" ]]; do - if [[ "$FIRST_SSM_INTEGRATION" == true ]]; then - echo -e "\n${BLUE}=== SSM Run As Configuration ===${NC}" - echo -e "Now that we've created a new POSIX user, how do we ensure that users only connect as their user and not ssm-user when connecting via SSM? To do this, we use SSM run as tags, which allows us to tag an IAM user with the POSIX user (aka cluster user) they should connect to via SSM." - read -p "Hit ENTER if you understand, or type "no" to skip this: " CONTINUE - - if [[ -z "$CONTINUE" ]]; then - echo -e "\n${YELLOW}Please complete the following steps:${NC}" - - echo -e "1. Navigate to the Session Manager Preferences Console" - echo -e " (https://console.aws.amazon.com/systems-manager/session-manager/preferences)" - read -p "Hit ENTER once you are there: " - - echo -e "\n2. Under 'Specify Operating System user for sessions'," - echo -e " ✅ check the 'Enable Run As Support for Linux Instances'" - read -p "Hit ENTER once step is complete: " - - echo -e "\n3. Change the Linux shell profile." - echo -e " It should have '/bin/bash -c 'export HOME=/fsx/\$(whoami) && cd \${HOME} && exec /bin/bash' in its first and only line" - read -p "Hit ENTER once you've added this line in: " - - echo -e "\n${GREEN}✅ SSM Run As support configured successfully${NC}" - else - echo -e "${YELLOW}Skipping SSM Run As configuration instructions...${NC}" - break - fi - FIRST_SSM_INTEGRATION=false - fi - - IAM_USERNAME=$(get_input "Enter the IAM username to associate with POSIX user ${username}" "$username") - - if ! aws iam get-user --user-name "${IAM_USERNAME}" --output json >/dev/null 2>&1; then - echo -e "${YELLOW}⚠️ IAM user ${IAM_USERNAME} does not exist${NC}" - CREATE_IAM=$(get_input "Would you like to create this IAM user? (Note: You'll need to add permissions later) (yes/no)" "no") - - if [[ "${CREATE_IAM}" == "yes" ]]; then - if ! output=$(aws iam create-user --user-name "$IAM_USERNAME" --output json 2>&1); then - echo -e "${YELLOW}⚠️ Error creating IAM user ${IAM_USERNAME}:${NC}" - echo -e "${YELLOW}$output${NC}" - ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") - continue - else - echo -e "${GREEN}✅ IAM user ${IAM_USERNAME} created successfully. Reminder to add permissions to this user as required!${NC}" - fi - else - ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") - continue - fi - fi - - if ! output=$(aws iam tag-user \ - --user-name "$IAM_USERNAME" \ - --tags "[{\"Key\": \"SSMSessionRunAs\",\"Value\": \"$username\"}]" --output json 2>&1); then - echo -e "${YELLOW}⚠️ Error adding SSM Run As tag for ${IAM_USERNAME}:${NC}" - echo -e "${YELLOW}$output${NC}" - ASSOCIATE_IAM=$(get_input "Would you like to try associating with a different IAM user? (yes/no)" "yes") - continue - else - echo -e "${GREEN}✅ SSM Run As tag added for ${IAM_USERNAME} (will run as ${username})${NC}" - break - fi - done - - # Increment the next_user_id - if [[ "$user_id" == "$next_user_id" ]]; then - ((next_user_id++)) - fi - - echo -e "${BLUE}========================================${NC}" - done - - echo -e "${GREEN}✅ User configuration completed. Users have been written to shared_users.txt${NC}" - echo -e "\n${BLUE}Please review the user configuration below. Press Enter to confirm and upload to S3, or Ctrl+C to exit${NC}" - echo -e "${YELLOW}Contents of shared_users.txt:${NC}" - cat shared_users.txt - - read - - echo -e "${BLUE}Uploading shared_users.txt to S3 bucket: $BUCKET...${NC}" - - if ! output=$(aws s3 cp shared_users.txt s3://${BUCKET}/src/ --output json 2>&1); then - echo -e "${YELLOW}⚠️ Error occurred while uploading shared_users.txt to S3 bucket:${NC}" - echo -e "${YELLOW}$output${NC}" - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (If you do this, please make sure you upload the file manually before creating the cluster)" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - else - echo -e "${GREEN}✅ User configuration file uploaded successfully to s3://${BUCKET}/src/shared_users.txt${NC}" - fi - else - echo -e "${YELLOW}Skipping user configuration...${NC}" - fi - echo -e "\n${BLUE}=== User Configuration Complete ===${NC}" -} - -# Function to create the cluster -create_cluster() { - echo -e "${GREEN}✅ Creating cluster for you!${NC}" - - if ! output=$(aws sagemaker create-cluster \ - --cli-input-json file://cluster-config.json \ - --region $AWS_REGION \ - --output json 2>&1); then - - echo -e "${YELLOW}⚠️ Error occurred while creating the cluster:${NC}" - echo -e "${YELLOW}$output${NC}" - - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (you will run the command below yourself!)" - echo -e "2. Press Ctrl+C to exit the script." - - # Command to create the cluster - echo -e "${GREEN} aws sagemaker create-cluster \\" - echo -e "${GREEN} --cli-input-json file://cluster-config.json \\" - echo -e "${GREEN} --region $AWS_REGION --output json${NC}\n" - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - else - echo -e "${GREEN}✅ Cluster creation request submitted successfully. To monitor the progress of cluster creation, you can either check the SageMaker console, or you can run:.${NC}" - echo -e "${YELLOW}watch -n 1 aws sagemaker list-clusters --output table${NC}" - fi -} - -# Warning message function -warning() { - echo -e "${BLUE}⚠️ Please note:${NC}" - echo -e " - Cluster creation may take some time (~15-20 min)" - echo -e " - This operation may incur costs on your AWS account" - echo -e " - Ensure you understand the implications before proceeding\n" -} - -# Function to display goodbye message -goodbye() { - # Final goodbye message - echo -e "${GREEN}Thank you for using the SageMaker HyperPod Cluster Creation Script!${NC}" - echo -e "${GREEN}For any issues or questions, please refer to the AWS documentation.${NC}" - echo "https://docs.aws.amazon.com/sagemaker/latest/dg/smcluster-getting-started.html" - - # Exit message - echo -e "\n${BLUE}Exiting script. Good luck with your SageMaker HyperPod journey! 👋${NC}\n" -} - -#===Main Script=== -main() { - print_header "🚀 Welcome to the SageMaker HyperPod Slurm Cluster Creation Script! 🚀" - - # Prerequisites - display_important_prereqs - - # Checking AWS Account ID - echo -e "\n${BLUE}🔍 AWS Account Verification${NC}" - echo -e "Your AWS Account ID is: ${GREEN}$AWS_ACCOUNT_ID${NC}" - echo "Press Enter to confirm ✅ or Ctrl+C to exit❌..." - read - - # Checking Git installation - check_git - - # Checking AWS CLI version and installation - echo -e "\n${BLUE}📦 1a: AWS CLI Installation and Verification${NC}" - check_and_install_aws_cli - - # Checking Region - echo -e "\n${BLUE}🌎 AWS Region Configuration${NC}" - region_check - - # Lifecycle Scripts Setup - echo -e "\n${BLUE}🔧 Setting Up Lifecycle Scripts${NC}" - echo -e "${BLUE}1b. Configuring environment variables and lifecycle scripts...${NC}" - setup_env_vars - setup_lifecycle_scripts - echo -e "${GREEN}✅ Lifecycle scripts setup completed${NC}" - - - # Cluster Configuration - echo -e "\n${BLUE}🚀 Creating the Cluster${NC}" - echo -e "${BLUE}1c. Generating cluster configuration...${NC}" - create_config - echo -e "${GREEN}✅ Cluster configuration created successfully${NC}" - echo -e "${BLUE}ℹ️ Validating the generated configuration before proceeding${NC}" - - if error_output=$(validate_cluster_config 2>&1); then - echo -e "${GREEN}✅ Cluster configuration validated!${NC}" - else - echo -e "${YELLOW}⚠️ Error occurred while validating cluster config script:${NC}" - echo -e "${YELLOW}$error_output${NC}" - echo -e "Options:" - echo -e "1. Press Enter to continue with the rest of the script (Not Recommended, unless you know how to set the environment variables manually!)" - echo -e "2. Press Ctrl+C to exit the script." - - read -e -p "Select an option (Enter/Ctrl+C): " choice - - if [[ -z "$choice" ]]; then - echo -e "${BLUE}Continuing with the rest of the script...${NC}" - else - exit 1 - fi - fi - - - echo -e "${BLUE}ℹ️ For your viewing, here's the cluster configuration generated. Please make sure it looks right before proceeding. Press enter to continue, or Ctrl+C to exit and make changes${NC}" - echo -e "${YELLOW}$(cat cluster-config.json | jq . --color-output)${NC}" - read - - configure_cluster_users - - print_header "🎉 Cluster Creation Script Completed! 🎉" - - # Instructions for next steps - echo -e "${GREEN}Congratulations! You've completed all the preparatory steps.${NC}" - echo -e "${YELLOW}Next Steps:${NC}" - - CREATE_CLUSTER=$(get_input "Do you want the script to create the cluster for you now? (yes/no):" "yes") - # read -e -p "Do you want the script to create the cluster for you now? (yes/no): " CREATE_CLUSTER - if [[ "$CREATE_CLUSTER" == "yes" ]]; then - warning - create_cluster - goodbye - else - echo -e "${YELLOW}Run the following command to create the cluster. Exiting this script!${NC}" - - # Command to create the cluster - echo -e "${GREEN} aws sagemaker create-cluster \\" - echo -e "${GREEN} --cli-input-json file://cluster-config.json \\" - echo -e "${GREEN} --region $AWS_REGION --output json${NC}\n" - - echo -e "${YELLOW}To monitor the progress of cluster creation, you can either check the SageMaker console, or you can run:.${NC}" - echo -e "${GREEN}watch -n 1 aws sagemaker list-clusters --output table${NC}" - - \ - warning - goodbye - fi -} - -main diff --git a/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky_installation.sh b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky_installation.sh new file mode 100755 index 000000000..652754aec --- /dev/null +++ b/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/slinky_installation.sh @@ -0,0 +1,1126 @@ +#!/bin/bash + +# Workshop Automation Script +# This script automates the steps of the workshop by executing CLI commands + +# Exit immediately if a command exits with a non-zero status. Print commands and their arguments as executed +set -e + +#===Global=== +export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text) +export AWS_REGION=${AWS_DEFAULT_REGION:-$(aws configure get region)} +TOTAL_STEPS=5 +CURRENT_STEP=0 + +#===Style Definitions=== +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Function to print a yellow header +print_header() { + echo -e "\n${BLUE}=================================================${NC}" + echo -e "\n${YELLOW}==== $1 ====${NC}\n" + echo -e "\n${BLUE}=================================================${NC}" + +} + +# UX Function for a Progress Bar :) +progress_bar() { + local duration=$1 + local steps=$2 + local width=50 + local progress=0 + + for ((i=0; i /dev/null; then + echo -e "${YELLOW}⚠️ AWS CLI is not installed. Installing...${NC}" + install_aws_cli + else + echo -e "${GREEN}✅ AWS CLI found. Checking version...${NC}" + CLI_VERSION=$(aws --version | awk '{print $1}' | cut -d/ -f2) + + echo -e "${BLUE}Current version: ${YELLOW}$CLI_VERSION${NC}" + echo -e "${BLUE}Min. required version: ${YELLOW}2.17.1${NC}" + + if [[ "$(printf '%s\n' "2.17.1" "$CLI_VERSION" | sort -V | head -n1)" != "2.17.1" ]]; then + echo -e "${YELLOW}⚠️ AWS CLI version $CLI_VERSION is lower than required.${NC}" + echo -e "${YELLOW} Updating AWS CLI...${NC}" + install_aws_cli + else + echo -e "${GREEN}✅ AWS CLI version $CLI_VERSION is up to date.${NC}" + fi + fi + + echo -e "${BLUE}=== AWS CLI Check Complete ===${NC}\n" + +} + +# Function to check if Git is installed and configured +check_git() { + if ! command -v git &> /dev/null; then + echo "Git is not installed. Please install Git and try again." + exit 1 + fi +} + +# Function to display the prerequisites before starting this workshop +display_important_prereqs() { + echo -e "${BLUE}Before running this script, please ensure the following prerequisites:${NC}\n" + + echo -e "${GREEN}1. 🔑 IAM Credentials:${NC}" + echo " You have Administrator Access Credentials in IAM." + echo " This is crucial as we'll be using CloudFormation to create IAM roles and policies." + echo " Run 'aws configure' to set up your credentials." + + echo -e "\n${GREEN}2. Deploy Sagemaker Hyperpod on EKS stack using this link https://catalog.workshops.aws/sagemaker-hyperpod-eks/en-US/00-setup/00-workshop-infra-cfn ${NC}" + echo " Set the number of the GeneralPurposeInstanceCount at least to 2. " + echo " Make sure the cloufromation stack creation is successful, the eks and hyperpod clusters are \"Inservice\" status and the nodes are \"Running\" " + echo " (It may take up to an hour for DeepHealthChecks on the node to be finished and for the node to be in the \"running\" state)." + + echo -e "\n${GREEN}3. Build a Slurmd Deep Learning Container:${NC}" + echo " Build a Slurm DLC using this dockerfile: https://github.com/aws-samples/awsome-distributed-training/blob/feature/slinkly-slurm-hyperpod-eks/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/dlc-slurmd.Dockerfile " + echo " following this direction: https://github.com/aws-samples/awsome-distributed-training/blob/feature/slinkly-slurm-hyperpod-eks/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/Docker-Build-README.md" + + echo -e "\n${GREEN}4. 🔧 Packages required for this script to run:${NC}" + echo " Ensure you install the following: eksctl, kubectl, helm, jq, and yq (install:sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq ) " + echo -e "\n${YELLOW}Ready to proceed? Press Enter to continue or Ctrl+C to exit...${NC}" + read +} + + +# Helper function to get user inputs with default values specified +get_input() { + local prompt="$1" + local default="$2" + local input + read -e -p "$prompt [$default]: " input + echo "${input:-$default}" +} + +get_prompt() { + local prompt="$1" + local input + read -e -p "$prompt: " input + echo "$input" +} +region_check() { + + NEW_REGION=$(get_input "Please, enter the AWS region where you want to set up your cluster" "$AWS_REGION") #eks cluster name + + if [[ -z "$NEW_REGION" ]]; then + echo -e "${GREEN}✅ Using default region: ${YELLOW}$AWS_REGION${NC}" + else + export AWS_REGION="$NEW_REGION" + echo -e "${GREEN}✅ Region updated to: ${YELLOW}$AWS_REGION${NC}" + fi + + echo -e "\n${BLUE}Your region is set to: ${YELLOW}$AWS_REGION${NC}" + echo -e "${BLUE}Ensure your chosen region supports SageMaker HyperPod.${NC}" + echo -e "${GREEN}You can check out https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-hyperpod.html#sagemaker-hyperpod-available-regions to learn about supported regions.${NC}" + echo -e "${BLUE}Press Enter to continue...${NC}" + read +} + + + +# Warning message function +warning() { + echo -e "${BLUE}⚠️ Please note:${NC}" + echo -e " - Cluster creation may take some time (~15-20 min)" + echo -e " - This operation may incur costs on your AWS account" + echo -e " - Ensure you understand the implications before proceeding\n" +} + +# Function to display goodbye message +goodbye() { + # Final goodbye message + echo -e "${GREEN}Thank you for using the SageMaker HyperPod Cluster Creation Script!${NC}" + echo -e "${GREEN}For any issues or questions, please refer to the AWS documentation.${NC}" + echo "https://docs.aws.amazon.com/sagemaker/latest/dg/smcluster-getting-started.html" + + # Exit message + echo -e "\n${BLUE}Exiting script. Good luck with your SageMaker HyperPod journey! 👋${NC}\n" +} + + +# Function to setup environment variables +setup_env_vars() { + + # Clear env_vars from previous runs + > env_vars + + echo -e "${YELLOW}Generating new environment variables...${NC}" + + # -------------------------- + # Write instance mappings + # -------------------------- + echo "export EKS_CLUSTER_NAME=${EKS_CLUSTER_NAME}" >> env_vars + echo "[INFO] EKS_CLUSTER_NAME = ${EKS_CLUSTER_NAME}" + echo "export ACCEL_INSTANCE_TYPE=${ACCEL_INSTANCE_TYPE}" >> env_vars + echo "export ACCEL_INSTANCE_COUNT=${ACCEL_INSTANCE_COUNT}" >> env_vars + # Export General Purpose Instance details without INFO messages + echo "export GEN_INSTANCE_TYPE=${GEN_INSTANCE_TYPE}" >> env_vars + echo "export GEN_INSTANCE_COUNT=${GEN_INSTANCE_COUNT}" >> env_vars + + EKS_CLUSTER_INFO=$(aws eks describe-cluster --name "$EKS_CLUSTER_NAME" --region "$AWS_REGION") #Eks cluster information + + # -------------------------- + # Get EKS_CLUSTER_ARN from CloudFormation + # -------------------------- + EKS_CLUSTER_ARN=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[?OutputKey==`EKSClusterArn`].OutputValue' \ + --output text) + + if [[ -n "$EKS_CLUSTER_ARN" && "$EKS_CLUSTER_ARN" != "None" ]]; then + echo "export EKS_CLUSTER_ARN=${EKS_CLUSTER_ARN}" >> env_vars + echo "[INFO] EKS_CLUSTER_ARN = ${EKS_CLUSTER_ARN}" + else + echo "[ERROR] Failed to retrieve EKS_CLUSTER_ARN from CloudFormation." + return 1 + fi + + # -------------------------- + # Get S3_BUCKET_NAME + # -------------------------- + S3_BUCKET_NAME=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[?OutputKey==`S3BucketName`].OutputValue' \ + --output text) + + if [[ -n "$S3_BUCKET_NAME" && "$S3_BUCKET_NAME" != "None" ]]; then + echo "export S3_BUCKET_NAME=${S3_BUCKET_NAME}" >> env_vars + echo "[INFO] S3_BUCKET_NAME = ${S3_BUCKET_NAME}" + else + echo "[ERROR] Failed to retrieve S3_BUCKET_NAME from CloudFormation." + return 1 + fi + + #SageMakerIAMRoleArn + EXECUTION_ROLE=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[?OutputKey==`SageMakerIAMRoleArn`].OutputValue' \ + --output text) + + + if [[ -n "$EXECUTION_ROLE" && "$EXECUTION_ROLE" != "None" ]]; then + echo "export EXECUTION_ROLE=${EXECUTION_ROLE}" >> env_vars + echo "[INFO] EXECUTION_ROLE = ${EXECUTION_ROLE}" + else + echo "[ERROR] Failed to retrieve EXECUTION_ROLE from CloudFormation." + return 1 + fi + + # -------------------------- + # Get VPC_ID + # -------------------------- + VPC_ID=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[?OutputKey==`VpcId`].OutputValue' \ + --output text) + + if [[ -n "$VPC_ID" && "$VPC_ID" != "None" ]]; then + echo "export VPC_ID=${VPC_ID}" >> env_vars + echo "[INFO] VPC_ID = ${VPC_ID}" + else + echo "[ERROR] Failed to retrieve VPC_ID from CloudFormation." + return 1 + fi + + #EKS Cluster subnet + + # -------------------------- + # Get PRIVATE_SUBNET_ID directly from EKS cluster + # -------------------------- + echo "[INFO] Retrieving subnet information from EKS cluster ${EKS_CLUSTER_NAME}..." + + # Get cluster VPC configuration + # Extract the first private subnet ID + export PRIVATE_SUBNET_ID=$(echo "$EKS_CLUSTER_INFO" | jq -r '.cluster.resourcesVpcConfig.subnetIds[0]') + + if [[ -n "$PRIVATE_SUBNET_ID" && "$PRIVATE_SUBNET_ID" != "null" ]]; then + echo "export PRIVATE_SUBNET_ID=${PRIVATE_SUBNET_ID}" >> env_vars + echo "[INFO] PRIVATE_SUBNET_ID = ${PRIVATE_SUBNET_ID}" + else + echo "[ERROR] Failed to retrieve PRIVATE_SUBNET_ID from EKS cluster." + return 1 + fi + + # -------------------------- + # Get SECURITY_GROUP_ID + # -------------------------- + SECURITY_GROUP_ID=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Outputs[?OutputKey==`SecurityGroupId`].OutputValue' \ + --output text) + + if [[ -n "$SECURITY_GROUP_ID" && "$SECURITY_GROUP_ID" != "None" ]]; then + echo "export SECURITY_GROUP_ID=${SECURITY_GROUP_ID}" >> env_vars + echo "[INFO] SECURITY_GROUP_ID = ${SECURITY_GROUP_ID}" + else + echo "[ERROR] Failed to retrieve SECURITY_GROUP_ID from CloudFormation." + return 1 + fi + + # -------------------------- + # Source the generated variables + # -------------------------- + source env_vars + + # Update kubectl config to point to the correct cluster + echo -e "${YELLOW}Updating kubectl configuration to use cluster: ${EKS_CLUSTER_NAME}${NC}" + aws eks update-kubeconfig --name $EKS_CLUSTER_NAME --region $AWS_REGION + + # -------------------------- + # Summary + # -------------------------- + echo -e "\n${BLUE}=== Environment Variables Summary ===${NC}" + echo -e "${GREEN}Current environment variables:${NC}" + cat env_vars + echo -e "\n${BLUE}=== Environment Setup Complete ===${NC}" +} + + +# Function to write the cluster-config.json file +create_config() { + + STACK_ID=$(get_input "Enter the name of the cloud formaiton stack created in step 2 of the prerequisite" "hyperpod-eks-full-stack") + #get the eks cluster name + EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`EKSClusterName`].ParameterValue' \ + --output text) + + + ACCEL_INSTANCE_TYPE=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`AcceleratedInstanceType`].ParameterValue' \ + --output text) + + ACCEL_INSTANCE_GROUP=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`AcceleratedInstanceGroupName`].ParameterValue' \ + --output text) + + ACCEL_INSTANCE_COUNT=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`AcceleratedInstanceCount`].ParameterValue' \ + --output text) + + GEN_INSTANCE_TYPE=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`GeneralPurposeInstanceType`].ParameterValue' \ + --output text) + + GEN_INSTANCE_COUNT=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`GeneralPurposeInstanceCount`].ParameterValue' \ + --output text) + + HP_CLUSTER_NAME=$(aws cloudformation describe-stacks \ + --stack-name "$STACK_ID" \ + --region "$AWS_REGION" \ + --query 'Stacks[0].Parameters[?ParameterKey==`HyperPodClusterName`].ParameterValue' \ + --output text) + + #sourcing the env var + setup_env_vars #sets and sources the env variables +} + +# Function to create FSx for Lustre Storage Class +create_fsx_lustre_storage_class() +{ + echo + echo -e "${BLUE}=== Creating FSx for Lustre Storage Class ===${NC}" + + FSX_SERVICE_ACCOUNT_NAME="fsx-csi-controller-sa-${EKS_CLUSTER_NAME}" + FSX_ROLE_NAME="FSXLCSI-${EKS_CLUSTER_NAME}-${AWS_REGION}" + + # Create an IAM OpenID Connect (OIDC) identity provider for the cluster + echo -e "${YELLOW}Creating IAM OIDC identity provider...${NC}" + eksctl utils associate-iam-oidc-provider --cluster $EKS_CLUSTER_NAME --approve + # Create a service account with an IAM role for the FSx for Lustre CSI driver + + # Wait a moment for cleanup + + echo -e "${YELLOW}Creating service account with IAM role for use with FSx for Lustre CSI driver...(${FSX_SERVICE_ACCOUNT_NAME})${NC}" + + eksctl create iamserviceaccount \ + --name ${FSX_SERVICE_ACCOUNT_NAME} \ + --namespace kube-system \ + --cluster $EKS_CLUSTER_NAME \ + --attach-policy-arn arn:aws:iam::aws:policy/AmazonFSxFullAccess \ + --approve \ + --role-name ${FSX_ROLE_NAME} \ + --region $AWS_REGION + + # Verify service account annotation + echo -e "${YELLOW}Verifying service account annotation...${NC}" + kubectl get sa ${FSX_SERVICE_ACCOUNT_NAME} -n kube-system -oyaml #retrives information about theFSXLCSI-${EKS_CLUSTER_NAME}-${AWS_REGION}" service account + + echo -e "${YELLOW} Adding the FSx for Lustre CSI Driver to helm repos...${NC}" + # Check if repo already exists before adding it + if ! helm repo list | grep -q "aws-fsx-csi-driver"; then + helm repo add aws-fsx-csi-driver https://kubernetes-sigs.github.io/aws-fsx-csi-driver + else + echo -e "${YELLOW}Helm repository aws-fsx-csi-driver already exists, skipping add...${NC}" + fi + + # Check if FSx CSI driver pods exist + if kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-fsx-csi-driver 2>/dev/null | grep -q .; then + echo -e "${YELLOW}Existing FSx CSI driver pods found. Cleaning up...${NC}" + + # Show existing pods before cleanup + echo -e "${YELLOW}Current FSx CSI driver pods:${NC}" + kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-fsx-csi-driver + + # Delete the helm release + echo -e "${YELLOW}Uninstalling existing FSx CSI driver...${NC}" + helm uninstall aws-fsx-csi-driver -n kube-system + + # Delete any remaining pods (backup cleanup) + kubectl delete pods -n kube-system -l app.kubernetes.io/name=aws-fsx-csi-driver 2>/dev/null || true + + # Wait a moment for cleanup + echo -e "${YELLOW}Waiting for pods to be cleaned up...${NC}" + sleep 10 + else + echo -e "${YELLOW}No existing FSx CSI driver pods found. Proceeding with installation...${NC}" + fi + + echo -e "${YELLOW}Installing the FSx for Lustre CSI driver...${NC}" + helm repo update + helm upgrade --install aws-fsx-csi-driver \ + --namespace kube-system \ + --set controller.serviceAccount.create=false \ + --set controller.serviceAccount.name=${FSX_SERVICE_ACCOUNT_NAME} \ + aws-fsx-csi-driver/aws-fsx-csi-driver + + # Verify installation of the FSx for Lustre CSI driver + echo -e "${YELLOW}Verifying FSx for Lustre CSI driver installation...${NC}" + kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-fsx-csi-driver + + # Install the FSx for Lustre Storage Class using Helm + echo -e "${YELLOW}Installing FSx for Lustre Storage Class...${NC}" + cat > /tmp/lustre-storageclass.yaml << EOL +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: fsx-sc +provisioner: fsx.csi.aws.com +parameters: + subnetId: \${PRIVATE_SUBNET_ID} + securityGroupIds: \${SECURITY_GROUP_ID} + deploymentType: PERSISTENT_2 + automaticBackupRetentionDays: "0" + copyTagsToBackups: "true" + perUnitStorageThroughput: "250" + dataCompressionType: "LZ4" + fileSystemTypeVersion: "2.15" +mountOptions: + - flock +EOL + + # Create an FSx for Lustre storage class + echo -e "Creating FSx for Lustre storage class..." + envsubst < /tmp/lustre-storageclass.yaml | kubectl apply -f - + + # Verify the storage class was created + echo -e "${YELLOW}Verifying storage class creation...${NC}" + kubectl get sc fsx-sc -oyaml + + # Clean up the temporary YAML file + rm -f /tmp/lustre-storageclass.yaml + + echo -e "${GREEN}✅ FSx for Lustre Storage Class setup completed${NC}" + echo +} + + +install_aws_load_balancer_controller() +{ + echo -e "${BLUE}=== Installing AWS Load Balancer Controller ===${NC}" + + # Create the IAM policy + echo -e "${YELLOW}Creating IAM policy for AWS Load Balancer Controller...${NC}" + curl -O https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.0/docs/install/iam_policy.json + + # Check if policy already exists + if ! aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy-v2.13.0 \ + --policy-document file://iam_policy.json 2>/dev/null; then + echo -e "${YELLOW}Policy AWSLoadBalancerControllerIAMPolicy-v2.13.0 already exists, continuing...${NC}" + fi + + # Create a service account with IAM role + echo -e "${YELLOW}Creating service account with IAM role...${NC}" + eksctl create iamserviceaccount \ + --cluster=$EKS_CLUSTER_NAME \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn=arn:aws:iam::$AWS_ACCOUNT_ID:policy/AWSLoadBalancerControllerIAMPolicy-v2.13.0 \ + --override-existing-serviceaccounts \ + --region $AWS_REGION \ + --approve + + # Verify service account annotation + echo -e "${YELLOW}Verifying Load balance contoller service account annotation (aws-load-balancer-controller) ${NC}" + kubectl get sa aws-load-balancer-controller -n kube-system -oyaml + + # Install AWS Load Balancer Controller using Helm + echo -e "${YELLOW}Installing AWS Load Balancer Controller using Helm...${NC}" + + # Check if repo already exists before adding it + if ! helm repo list | grep -q "eks"; then + helm repo add eks https://aws.github.io/eks-charts + else + echo -e "${YELLOW}Helm repository eks already exists, skipping add...${NC}" + fi + + helm repo update + + # Check if the release already exists + if helm list -n kube-system | grep -q "aws-load-balancer-controller"; then + echo -e "${YELLOW}AWS Load Balancer Controller already exists, upgrading...${NC}" + helm upgrade aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=$EKS_CLUSTER_NAME \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller \ + --set region=$AWS_REGION \ + --set vpcId=$VPC_ID + else + echo -e "${YELLOW}Installing AWS Load Balancer Controller...${NC}" + helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ + -n kube-system \ + --set clusterName=$EKS_CLUSTER_NAME \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller \ + --set region=$AWS_REGION \ + --set vpcId=$VPC_ID + fi + + # Verify installation + echo -e "${YELLOW}Verifying AWS Load Balancer Controller installation...${NC}" + kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller + + # Clean up the policy file + rm -f iam_policy.json + + echo -e "${GREEN}✅ AWS Load Balancer Controller installation completed${NC}" + echo +} + +install_slinky_prerequisites() { + echo -e "${BLUE}=== Installing Slinky Prerequisites ===${NC}" + + # Add Helm repositories + echo -e "${YELLOW}Adding Helm repositories...${NC}" + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add metrics-server https://kubernetes-sigs.github.io/metrics-server/ + helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add jetstack https://charts.jetstack.io + + helm repo update + + MAX_RETRIES=30 + RETRY_INTERVAL=5 + READY=false + ATTEMPT=1 + + while [[ "$READY" == "false" ]] && [[ $ATTEMPT -le $MAX_RETRIES ]]; do + # Check if deployment is available + AVAILABLE=$(kubectl get deployment aws-load-balancer-controller -n kube-system -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null) + if [[ "$AVAILABLE" == "True" ]]; then + kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller + echo -e "${GREEN}✅ AWS Load Balancer Controller is available${NC}" + READY=true + break + fi + echo -e "${YELLOW}Waiting for AWS Load Balancer Controller to be available (attempt $ATTEMPT/$MAX_RETRIES)...${NC}" + sleep $RETRY_INTERVAL + ((ATTEMPT++)) + done + + if [[ "$READY" == "false" ]]; then + echo -e "${YELLOW}⚠️ AWS Load Balancer Controller not ready after waiting. Temporarily disabling webhook...${NC}" + kubectl delete -A ValidatingWebhookConfiguration aws-load-balancer-webhook --ignore-not-found=true + fi + + #V4 + if [[ "$READY" == "true" ]]; then + echo -e "${YELLOW}Installing cert-manager...${NC}" + if ! helm list -n cert-manager | grep -q "cert-manager"; then + echo -e "${YELLOW}Starting cert-manager installation...${NC}" + + # Install without waiting for the startup check to complete + helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --set crds.enabled=true \ + --set startupapicheck.enabled=false \ + --timeout 10m || true + + echo -e "${YELLOW}Waiting for cert-manager pods to be ready...${NC}" + kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=5m || true + + # Check if installation succeeded + if helm list -n cert-manager | grep -q "cert-manager"; then + echo -e "${GREEN}✅ Cert-manager installed successfully${NC}" + kubectl get pods -n cert-manager + else + echo -e "${RED}⚠️ Cert-manager installation may have issues${NC}" + fi + else + echo -e "${YELLOW}cert-manager already exists, skipping installation...${NC}" + fi + else + echo -e "${RED}Cannot install cert-manager as Load Balancer Controller is not ready${NC}" + fi + + + + # Install Prometheus + echo -e "${YELLOW}Installing Prometheus...${NC}" + if ! helm list -n prometheus | grep -q "prometheus"; then + helm install prometheus prometheus-community/kube-prometheus-stack \ + --namespace prometheus --create-namespace --set installCRDs=true + else + echo -e "${YELLOW}Prometheus already exists, skipping installation...${NC}" + fi + + # Verify installations + echo -e "${YELLOW}Verifying prerequisite installations...${NC}" + kubectl get all -n cert-manager + kubectl get all -n prometheus + + # Install Slurm Operator + echo -e "${BLUE}=== Installing Slurm Operator ===${NC}" + + # Download values file + echo -e "${YELLOW}Downloading Slurm Operator values file...${NC}" + curl -L https://raw.githubusercontent.com/SlinkyProject/slurm-operator/refs/tags/v0.3.0/helm/slurm-operator/values.yaml \ + -o values-operator.yaml + + # Delete any stale CRDs + echo -e "${YELLOW}Cleaning up any stale CRDs...${NC}" + kubectl delete crd clusters.slinky.slurm.net 2>/dev/null || true + kubectl delete crd nodesets.slinky.slurm.net 2>/dev/null || true + + # Install Slurm Operator + echo -e "${YELLOW}Installing Slurm Operator...${NC}" + if helm list -n slinky | grep -q "slurm-operator"; then + echo -e "${YELLOW}Existing Slurm Operator found. Uninstalling...${NC}" + helm uninstall slurm-operator -n slinky + # Wait for the resources to be cleaned up + sleep 10 + fi + + echo -e "${YELLOW}Installing Slurm Operator...${NC}" + helm install slurm-operator oci://ghcr.io/slinkyproject/charts/slurm-operator \ + --values=values-operator.yaml \ + --version=0.3.0 \ + --namespace=slinky \ + --create-namespace \ + --set installCRDs=true \ + --set webhook.installCRDs=true + + # Verify Slurm Operator installation + echo -e "${YELLOW}Verifying Slurm Operator installation...${NC}" + kubectl get all -n slinky + + # Clean up values file + rm -f values-operator.yaml + + echo -e "${GREEN}✅ Slinky prerequisites installation completed${NC}" +} + + + +# Function to create cluster +set_slurm_values() { + echo -e "${BLUE}=== Setting Slurm Cluster Values ===${NC}" + + # Use the environment variables + echo -e "${YELLOW}Using environment variables:${NC}" + echo -e "Accelerated instance type: ${GREEN}$ACCEL_INSTANCE_TYPE${NC}" + echo -e "Accelerated instance count: ${GREEN}$ACCEL_INSTANCE_COUNT${NC}" + echo -e "General purpose instance type: ${GREEN}$GEN_INSTANCE_TYPE${NC}" + + # Download base values file (using g5 as a template) + echo -e "${YELLOW}Downloading base values file...${NC}" + export VALUES_FILE="custom-values.yaml" + curl -L https://github.com/aws-samples/awsome-distributed-training/raw/refs/heads/feature/slinkly-slurm-hyperpod-eks/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/g5/g5-values.yaml -o $VALUES_FILE + if [[ $? -ne 0 ]]; then + echo -e "${BLUE}Failed to download base values file.${NC}" + exit 1 + fi + + # Verify general purpose nodes + echo -e "${YELLOW}Verifying general purpose nodes with instance type: $GEN_INSTANCE_TYPE${NC}" + kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE + + # Verify compute nodes + echo -e "${YELLOW}Verifying compute nodes with instance type: $ACCEL_INSTANCE_TYPE${NC}" + kubectl get nodes -l node.kubernetes.io/instance-type=$ACCEL_INSTANCE_TYPE + + # Automatically detect available dlc-slurmd image from ECR + echo -e "${YELLOW}Detecting available dlc-slurmd image from ECR...${NC}" + + # Get the latest image tag from ECR repository + AVAILABLE_TAG=$(aws ecr list-images --repository-name dlc-slurmd --region $AWS_REGION --query 'imageIds[0].imageTag' --output text 2>/dev/null) + + if [[ "$AVAILABLE_TAG" == "None" ]] || [[ -z "$AVAILABLE_TAG" ]]; then + echo -e "${RED}No dlc-slurmd images found in ECR repository${NC}" + echo -e "${YELLOW}Falling back to public image: ghcr.io/slinkyproject/slurmd:25.05-ubuntu24.04${NC}" + CONTAINER_IMAGE="ghcr.io/slinkyproject/slurmd:25.05-ubuntu24.04" + CONTAINER_REPO="ghcr.io/slinkyproject/slurmd" + else + echo -e "${GREEN}Found image tag: $AVAILABLE_TAG${NC}" + CONTAINER_IMAGE="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd:$AVAILABLE_TAG" + CONTAINER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dlc-slurmd" + fi + + echo -e "${YELLOW}Using container image: ${GREEN}$CONTAINER_IMAGE${NC}" + + # Generate SSH key if needed + echo -e "${YELLOW}Checking for SSH key...${NC}" + if [[ ! -f ~/.ssh/id_rsa.pub ]]; then + echo -e "${YELLOW}No SSH key found. Generating new SSH key...${NC}" + ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa + else + echo -e "${GREEN}Using existing SSH key at ~/.ssh/id_rsa.pub${NC}" + fi + + # Get SSH public key + SSH_PUBLIC_KEY=$(cat ~/.ssh/id_rsa.pub) + + # Update values file with user's configuration + echo -e "${YELLOW}Customizing values file with your configuration...${NC}" + + # Update common affinity for non-compute components to use general purpose instance type + yq eval ".commonAffinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0] = \"$GEN_INSTANCE_TYPE\"" -i $VALUES_FILE + + # Update compute node configuration + echo -e "${YELLOW}Updating compute node configuration...${NC}" + + # Update container image - repository + yq eval ".compute.nodesets[0].image.repository = \"$CONTAINER_REPO\"" -i $VALUES_FILE + + # Update image tag if using ECR + if [[ "$CONTAINER_REPO" == *"ecr"* ]]; then + yq eval ".compute.nodesets[0].image.tag = \"$AVAILABLE_TAG\"" -i $VALUES_FILE + fi + + # Update SSH public key + yq eval ".login.rootSshAuthorizedKeys[0] = \"$SSH_PUBLIC_KEY\"" -i $VALUES_FILE + + # Update node count to match the accelerated instance count + yq eval ".compute.nodesets[0].replicas = $ACCEL_INSTANCE_COUNT" -i $VALUES_FILE + + # Update node selector to match the accelerated instance type + yq eval ".compute.nodesets[0].nodeSelector.\"node.kubernetes.io/instance-type\" = \"$ACCEL_INSTANCE_TYPE\"" -i $VALUES_FILE + + # Remove OpenZFS configurations + yq eval 'del(.login.extraVolumeMounts[] | select(.name == "fsx-openzfs"))' -i $VALUES_FILE + yq eval 'del(.login.extraVolumes[] | select(.name == "fsx-openzfs"))' -i $VALUES_FILE + yq eval 'del(.compute.nodesets[].extraVolumeMounts[] | select(.name == "fsx-openzfs"))' -i $VALUES_FILE + yq eval 'del(.compute.nodesets[].extraVolumes[] | select(.name == "fsx-openzfs"))' -i $VALUES_FILE + + # Check if general purpose node has capacity, if not remove restrictive affinity + GEN_NODE_CAPACITY=$(kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE -o jsonpath='{.items[0].status.allocatable.pods}' 2>/dev/null || echo "0") + GEN_NODE_USED=$(kubectl get pods --all-namespaces --field-selector spec.nodeName=$(kubectl get nodes -l node.kubernetes.io/instance-type=$GEN_INSTANCE_TYPE -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) 2>/dev/null | wc -l || echo "0") + + if [[ $GEN_NODE_USED -ge $GEN_NODE_CAPACITY ]] && [[ $GEN_NODE_CAPACITY -gt 0 ]]; then + echo -e "${YELLOW}General purpose node is at capacity ($GEN_NODE_USED/$GEN_NODE_CAPACITY), removing restrictive affinity...${NC}" + yq eval 'del(.commonAffinity)' -i $VALUES_FILE + else + echo -e "${GREEN}General purpose node has capacity, keeping affinity rules${NC}" + fi + + dynamic_pods_allocation #this function is not tested + + echo -e "\n${BLUE}=== Configuration Parameters ===${NC}" + echo -e "${YELLOW}Please review the following configuration:${NC}" + echo "----------------------------------------" + yq eval '... comments=""' custom-values.yaml + echo "----------------------------------------" + + echo -e "\n${YELLOW}Please verify if these values look correct.${NC}" + read + echo -e "${GREEN}✓ Slurm values have been successfully configured!${NC}" +} + +get_gpu_info() { + local ec2_type=${ACCEL_INSTANCE_TYPE#ml.} + local gpu_count=$(aws ec2 describe-instance-types --instance-types "${ec2_type}" \ + --query 'InstanceTypes[0].GpuInfo.Gpus[0].Count' --output text) + echo "${gpu_count}" +} + +has_efa_support() { + local ec2_type=${ACCEL_INSTANCE_TYPE#ml.} + local efa_supported=$(aws ec2 describe-instance-types --instance-types "${ec2_type}" \ + --query 'InstanceTypes[0].NetworkInfo.EfaSupported' --output text) + echo "${efa_supported}" +} + +calculate_pods_per_node() { + local gpu_count=$1 + if [ ${gpu_count} -le 4 ]; then + echo 1 # One pod for nodes with less than 4 GPUs + elif [ ${gpu_count} -ge 8 ]; then + echo 2 # Two pods for nodes with 8 or more GPUs + else + echo 1 + fi +} + +calculate_gpus_per_pod() { + local gpu_count=$1 + local pods_per_node=$2 + echo $(( gpu_count / pods_per_node )) +} + +dynamic_pods_allocation(){ + #dynamic allocation of pods per node based on the instance type + local gpu_count=get_gpu_info + + local efa_info=$(get_efa_info) + local efa_supported=$(echo "${efa_info}" | jq -r '.EfaSupported') + local max_efa=$(echo "${efa_info}" | jq -r '.MaxEfa') + + local pods_per_node=$(calculate_pods_per_node ${gpu_count}) #returns 1 or 2 based on the gpu count + local gpus_per_pod=$(calculate_gpus_per_pod ${gpu_count} ${pods_per_node}) # diveds the total number of gpus in in an instance by the number of pods + + local total_rep = $(( ACCEL_INSTANCE_COUNT * pods_per_node )) + local resources_path=".compute.nodesets[0]" + + #number of replicas = the total number of pods + # Update replicas to match pods per node + yq eval "${resources_path}.replicas = ${total_rep}" -i "${values_file}" + + # Clear existing resources configuration + yq eval "${resources_path}.resources = {}" -i "${values_file}" + + # Set GPU resources + yq eval "${resources_path}.resources.limits.\"nvidia.com/gpu\" = ${gpus_per_pod}" -i "${values_file}" + yq eval "${resources_path}.resources.requests.\"nvidia.com/gpu\" = ${gpus_per_pod}" -i "${values_file}" + + # Add EFA configuration for p5 instances + if [ "${efa_supported}" == "true" ] && [ "${max_efa}" -gt 0 ]; then + local efa_per_pod=$(( max_efa / pods_per_node )) + yq eval "${resources_path}.resources.limits.\"vpc.amazonaws.com/efa\" = ${efa_per_pod}" -i "${values_file}" + yq eval "${resources_path}.resources.requests.\"vpc.amazonaws.com/efa\" = ${efa_per_pod}" -i "${values_file}" + fi + echo "Configuration set for ${instance_type}:" + echo "- GPUs per node: ${gpu_count}" + echo "- Pods per node: ${pods_per_node}" + echo "- GPUs per pod: ${gpus_per_pod}" + # +} + +create_and_verify_fsx_pvc() { + local namespace="slurm" + local pvc_name="fsx-claim" + local max_retries=30 + local retry_interval=10 + + echo "Creating FSx for Lustre PVC in ${namespace} namespace..." + + # Create namespace if it doesn't exist + if ! kubectl get namespace ${namespace} >/dev/null 2>&1; then + echo "Creating namespace: ${namespace}" + kubectl create ns ${namespace} + if [ $? -ne 0 ]; then + echo "Failed to create namespace ${namespace}" + return 1 + fi + fi + + local yaml_file="lustre-pvc-slurm.yaml" + local yaml_url="https://github.com/aws-samples/awsome-distributed-training/raw/refs/heads/feature/slinkly-slurm-hyperpod-eks/1.architectures/7.sagemaker-hyperpod-eks/slinky-slurm/lustre-pvc-slurm.yaml" + + if [ ! -f "${yaml_file}" ]; then + echo "PVC YAML file not found. Downloading from repository..." + if ! curl -s -L -o "${yaml_file}" "${yaml_url}"; then + echo "Failed to download ${yaml_file}" + return 1 + fi + echo "Successfully downloaded ${yaml_file}" + else + echo "Using existing ${yaml_file}" + fi + + # Apply the PVC configuration + echo "Creating PVC ${pvc_name}..." + kubectl apply -f "${yaml_file}" + if [ $? -ne 0 ]; then + echo "Failed to apply PVC configuration" + return 1 + fi + + # Wait for PVC to be bound + echo "Waiting for PVC to be bound..." + + seconds=0 + timeout=600 # 10 minutes + retry_interval=60 # 1 minute + + while [ $seconds -lt $timeout ]; do + status=$(kubectl get pvc ${pvc_name} -n ${namespace} -ojson | jq -r .status.phase) + + if [ "$status" == "Bound" ]; then + echo "PVC successfully bound!" + break + fi + + remaining=$((timeout - seconds)) + echo "Current status: ${status}, waiting ${retry_interval} seconds... (${remaining} seconds remaining)" + sleep ${retry_interval} + seconds=$((seconds + retry_interval)) + done + + if [ $seconds -ge $timeout ]; then + echo "Timeout of ${timeout} seconds reached waiting for PVC to be bound." + return 1 + fi + + # Get and display PVC details + echo "PVC Details:" + kubectl get pvc -n ${namespace} + + # Get volume ID + volume_name=$(kubectl get pvc ${pvc_name} -n ${namespace} -ojson | jq -r .spec.volumeName) + if [ -n "$volume_name" ]; then + volume_id=$(kubectl get pv ${volume_name} -ojson | jq -r .spec.csi.volumeHandle) + echo "Volume ID: ${volume_id}" + else + echo "Failed to get volume name" + return 1 + fi + + return 0 +} + + +# Function to deploy Slurm cluster +deploy_slurm_cluster() { + local namespace="${1:-slurm}" + local values_file="${2:-custom-values.yaml}" + local version="${3:-0.3.0}" + local dry_run="${4:-false}" + local configure_nlb="${5:-false}" + + echo -e "${BLUE}=== Deploying Slurm Cluster ===${NC}" + + # Verify the values file exists + if [[ ! -f "$values_file" ]]; then + echo -e "${RED}Error: Values file $values_file not found${NC}" + return 1 + fi + + # Perform dry run if requested + if [[ "$dry_run" == "true" ]]; then + echo -e "${YELLOW}Performing dry run installation...${NC}" + helm install --dry-run slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values="$values_file" --version="$version" --namespace="$namespace" + + # Check if dry run was successful + if [[ $? -ne 0 ]]; then + echo -e "${RED}Dry run failed. Please check the values file and try again.${NC}" + return 1 + fi + echo -e "${GREEN}Dry run completed successfully.${NC}" + + # Don't proceed further if this is just a dry run + if [[ "$dry_run" == "true" ]]; then + return 0 + fi + fi + + # Create namespace if it doesn't exist + if ! kubectl get namespace "$namespace" &>/dev/null; then + echo -e "${YELLOW}Creating namespace $namespace...${NC}" + kubectl create namespace "$namespace" + fi + + # Perform actual installation + echo -e "${YELLOW}Installing Slurm cluster...${NC}" + helm install slurm oci://ghcr.io/slinkyproject/charts/slurm \ + --values="$values_file" --version="$version" --namespace="$namespace" + + # Check if installation was successful + if [[ $? -ne 0 ]]; then + echo -e "${RED}Installation failed. Please check the error messages above.${NC}" + return 1 + fi + + echo -e "${GREEN}✅ Slurm cluster installation initiated${NC}" + + # Watch the deployment status + echo -e "${YELLOW}Watching deployment status...${NC}" + kubectl -n "$namespace" get pods -l app.kubernetes.io/instance=slurm --watch & + watch_pid=$! + # Allow user to stop watching after a while + sleep 15 + kill $watch_pid 2>/dev/null + echo -e "\n${YELLOW}Continuing with deployment...${NC}" + # Verify the deployment status of all components + echo -e "${YELLOW}Verifying deployment status of all components...${NC}" + kubectl get all -n "$namespace" + + echo -e "${GREEN}✅ Slurm cluster deployment completed${NC}" + echo -e "${YELLOW}Note: It may take a few minutes for all components to start up${NC}" + + # Configure NLB if requested + if [[ "$configure_nlb" == "true" ]]; then + echo -e "${YELLOW}Configuring Network Load Balancer for login access...${NC}" + # Wait a bit for the service to be created + sleep 10 + configure_login_nlb "$namespace" "slurm-login" + fi + + return 0 +} + +# Function to configure a Login Network Load Balancer +configure_login_nlb() { + local namespace="${1:-slurm}" + local service_name="${2:-slurm-login}" + + echo -e "${BLUE}=== Configuring Login Network Load Balancer ===${NC}" + + # Identify public subnets in the VPC + echo -e "${YELLOW}Identifying public subnets in VPC...${NC}" + export PUBLIC_SUBNET_ID_1=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=map-public-ip-on-launch,Values=true" --query "Subnets[0].SubnetId" --output text) + export PUBLIC_SUBNET_ID_2=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=${VPC_ID}" "Name=map-public-ip-on-launch,Values=true" --query "Subnets[1].SubnetId" --output text) + + # Verify subnets were found + if [[ -z "$PUBLIC_SUBNET_ID_1" || "$PUBLIC_SUBNET_ID_1" == "None" || -z "$PUBLIC_SUBNET_ID_2" || "$PUBLIC_SUBNET_ID_2" == "None" ]]; then + echo -e "${RED}Error: Could not find two public subnets in VPC ${VPC_ID}${NC}" + return 1 + fi + + echo -e "${GREEN}Found public subnets: ${PUBLIC_SUBNET_ID_1}, ${PUBLIC_SUBNET_ID_2}${NC}" + + # Add annotations to the service to make it internet facing + echo -e "${YELLOW}Adding annotations to ${service_name} service...${NC}" + kubectl annotate service ${service_name} -n ${namespace} \ + service.beta.kubernetes.io/aws-load-balancer-type="nlb" \ + service.beta.kubernetes.io/aws-load-balancer-scheme="internet-facing" \ + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type="ip" \ + service.beta.kubernetes.io/aws-load-balancer-subnets="${PUBLIC_SUBNET_ID_1},${PUBLIC_SUBNET_ID_2}" \ + service.beta.kubernetes.io/aws-load-balancer-healthcheck-port="22" \ + --overwrite + + # Verify the service configuration + echo -e "${YELLOW}Verifying service configuration...${NC}" + kubectl describe service ${service_name} -n ${namespace} + + # Get the NLB DNS name + NLB_DNS=$(kubectl get service ${service_name} -n ${namespace} -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') + + if [[ -n "$NLB_DNS" ]]; then + echo -e "${GREEN}✅ Login NLB configured successfully${NC}" + echo -e "${GREEN}✅ You can access the Slurm login node using:${NC}" + echo -e "${YELLOW}ssh -i ~/.ssh/id_rsa @${NLB_DNS}${NC}" + else + echo -e "${YELLOW}NLB DNS name not yet available. It may take a few minutes to provision.${NC}" + echo -e "${YELLOW}Run the following command later to get the DNS name:${NC}" + echo -e "${YELLOW}kubectl get service ${service_name} -n ${namespace} -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'${NC}" + fi + + return 0 +} + + +#===Main Script=== +main() { + print_header "🚀 Welcome to the SageMaker HyperPod Slurm Cluster Creation Script! 🚀" + + # Prerequisites + display_important_prereqs + + # Checking AWS Account ID + echo -e "\n${BLUE}🔍 AWS Account Verification${NC}" + echo -e "Your AWS Account ID is: ${GREEN}$AWS_ACCOUNT_ID${NC}" + echo "Press Enter to confirm ✅ or Ctrl+C to exit❌..." + read + + # Checking Git installation + check_git + + # Checking AWS CLI version and installation + echo -e "\n${BLUE}📦 1a: AWS CLI Installation and Verification${NC}" + check_and_install_aws_cli + + # Checking Region + echo -e "\n${BLUE}🌎 AWS Region Configuration${NC}" + region_check + # Cluster Configuration + echo -e "${BLUE} Generating cluster configuration...${NC}" + create_config + create_fsx_lustre_storage_class + install_aws_load_balancer_controller + install_slinky_prerequisites + set_slurm_values + create_and_verify_fsx_pvc + deploy_slurm_cluster "slurm" "custom-values.yaml" "0.3.0" "false" "true" + goodbye +} +# Execute the main function +main \ No newline at end of file