[HuggingFace][Neuronx] Training - Optimum Neuron 0.1.0 - Neuron sdk 2.21.1 - Transformers to 4.48.3

dlc_developer_config.toml

@@ -9,7 +9,7 @@ ei_mode = false
 neuron_mode = false
 # Please only set it to true if you are preparing a NEURONX related PR
 # Do remember to revert it back to false before merging any PR (including NEURONX dedicated PR)
-neuronx_mode = false
+neuronx_mode = true
 # Please only set it to true if you are preparing a GRAVITON related PR
 # Do remember to revert it back to false before merging any PR (including GRAVITON dedicated PR)
 graviton_mode = false
@@ -37,12 +37,12 @@ deep_canary_mode = false
 [build]
 # Add in frameworks you would like to build. By default, builds are disabled unless you specify building an image.
 # available frameworks - ["autogluon", "huggingface_tensorflow", "huggingface_pytorch", "huggingface_tensorflow_trcomp", "huggingface_pytorch_trcomp", "pytorch_trcomp", "tensorflow", "pytorch", "stabilityai_pytorch"]
-build_frameworks = []
+build_frameworks = ["huggingface_pytorch"]
 
 
 # By default we build both training and inference containers. Set true/false values to determine which to build.
 build_training = true
-build_inference = true
+build_inference = false
 
 # Set do_build to "false" to skip builds and test the latest image built by this PR
 # Note: at least one build is required to set do_build to "false"

huggingface/pytorch/training/buildspec-neuronx.yml

@@ -2,8 +2,8 @@ account_id: &ACCOUNT_ID <set-$ACCOUNT_ID-in-environment>
 region: &REGION <set-$REGION-in-environment>
 base_framework: &BASE_FRAMEWORK pytorch
 framework: &FRAMEWORK !join [ "huggingface_", *BASE_FRAMEWORK]
-version: &VERSION 2.1.2
-short_version: &SHORT_VERSION "2.1"
+version: &VERSION "2.5.1"
+short_version: &SHORT_VERSION "2.5"
 contributor: huggingface
 arch_type: x86
 
@@ -30,13 +30,13 @@ images:
   BuildNeuronHFPytorchPy310TrainingDockerImage:
     <<: *TRAINING_REPOSITORY
     build: &HUGGINGFACE_PYTORCH_INF_TRAINING_PY3 false
-    image_size_baseline: 20000
+    image_size_baseline: 28000
     device_type: &DEVICE_TYPE neuronx
     python_version: &DOCKER_PYTHON_VERSION py3
     tag_python_version: &TAG_PYTHON_VERSION py310
-    neuron_sdk_version: &NEURON_SDK_VERSION sdk2.20.0
-    os_version: &OS_VERSION ubuntu20.04
-    transformers_version: &TRANSFORMERS_VERSION 4.48.1
+    neuron_sdk_version: &NEURON_SDK_VERSION sdk2.22.0
+    os_version: &OS_VERSION ubuntu22.04
+    transformers_version: &TRANSFORMERS_VERSION 4.49.0
     datasets_version: &DATASETS_VERSION 2.18.0
     tag: !join [ *VERSION, '-', 'transformers', *TRANSFORMERS_VERSION, '-', *DEVICE_TYPE, '-', *TAG_PYTHON_VERSION,"-", *NEURON_SDK_VERSION, '-', *OS_VERSION ]
     docker_file: !join [ docker/, *SHORT_VERSION, /, *DOCKER_PYTHON_VERSION, /, *NEURON_SDK_VERSION, /Dockerfile., *DEVICE_TYPE ]

huggingface/pytorch/training/docker/2.5/py3/sdk2.22.0/Dockerfile.neuronx

@@ -0,0 +1,94 @@
+# https://github.com/aws/deep-learning-containers/blob/master/available_images.md
+# refer to the above page to pull latest PyTorch Neuronx image
+
+# Docker image region us-west-2
+# FROM 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuronx:2.5.1-neuronx-py310-sdk2.21.0-ubuntu22.04
+FROM 763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuronx:2.5.1-neuronx-py310-sdk2.22.0-ubuntu22.04
+
+# The pytorch-training-neuronx container has the `apex_setup.py` fils in the root directory, which makes the
+# test/dlc_tests/sanity/test_pre_release.py:test_stray_files test fail.
+# We are removing the apex directory as well out of consistency.
+RUN rm -rf /root/apex && \
+    rm  /root/apex_setup.py
+
+LABEL maintainer="Amazon AI"
+LABEL dlc_major_version="2"
+
+# Version args
+ARG OPTIMUM_NEURON_VERSION=0.1.0
+ARG TRANSFORMERS_VERSION
+ARG DATASETS_VERSION
+ARG GEVENT_VERSION=24.10.3
+ARG PYTHON=python3
+
+RUN apt-get remove -y --purge emacs && \
+apt-get autoremove -y
+
+RUN pip install --upgrade pip
+
+# We need to set this environment variable to avoid the following error when building KenLM:
+# https://github.com/kpu/kenlm/issues/462
+ENV CMAKE_POLICY_VERSION_MINIMUM=3.5
+
+# Install Hugging Face libraries and its dependencies
+# Install optimum-neuron with this exta starting from next release. \
+# "optimum-neuron[training]"==${OPTIMUM_NEURON_VERSION} \
+RUN pip install --no-cache-dir \
+	"sagemaker==2.232.2" \
+	evaluate \
+	transformers[sklearn,sentencepiece,audio,vision]==${TRANSFORMERS_VERSION} \
+	datasets==${DATASETS_VERSION} \
+        optimum-neuron==${OPTIMUM_NEURON_VERSION} \
+        "peft==0.14.0" \
+	gevent==${GEVENT_VERSION}
+
+# Pin numpy to version required by neuronx-cc
+# Update Pillow, urllib, wandb versions to fix high and critical vulnerabilities
+RUN pip install -U \
+	"tensorboard>=2.11.0" \
+	"numpy>=1.24.3,<=1.25.2" \
+	"numba==0.58.1" \
+	"Pillow==10.3.0" \
+	"requests<2.32.0" \
+        wandb \
+        pytorch-lightning \
+	Jinja2 \
+	mlflow \
+	tornado \
+	"awscli<2" \
+	boto3 \
+	botocore \
+	google-auth \
+	"urllib3>=1.26.17,<1.27"
+
+# pytorch-lightning has critical vulnerabilities and is not needed in the container.
+RUN rm /requirements.txt && \
+    pip uninstall pytorch-lightning -y
+
+# We have this error from `pip check`:
+# neuronx-cc 2.16.345.0+69131dd3 has requirement networkx~=2.6, but you have networkx 3.4.2.
+# To fix that, we are downgrading networkx to 2.6.3
+run pip install -U "networkx==2.6.3"
+
+
+RUN apt-get update \
+ && apt install -y --no-install-recommends \
+    git-lfs \
+	libgssapi-krb5-2 \
+	libexpat1 \
+	expat \
+	libarchive13 \
+	libgstreamer1.0-0 \
+	libgstreamer-plugins-base1.0-0 \
+ && apt-get upgrade -y apparmor \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN HOME_DIR=/root \
+ && curl -o ${HOME_DIR}/oss_compliance.zip https://aws-dlinfra-utilities.s3.amazonaws.com/oss_compliance.zip \
+ && unzip ${HOME_DIR}/oss_compliance.zip -d ${HOME_DIR}/ \
+ && cp ${HOME_DIR}/oss_compliance/test/testOSSCompliance /usr/local/bin/testOSSCompliance \
+ && chmod +x /usr/local/bin/testOSSCompliance \
+ && chmod +x ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh \
+ && ${HOME_DIR}/oss_compliance/generate_oss_compliance.sh ${HOME_DIR} ${PYTHON} \
+ && rm -rf ${HOME_DIR}/oss_compliance*