From 1f72c0aff1b61f18bb0c61a1610254fd1280f0d3 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 11:53:42 -0700 Subject: [PATCH 01/18] Ecosystem library pypi promotion --- .../workflows/pypi_release_domain_library.yml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/pypi_release_domain_library.yml diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml new file mode 100644 index 0000000000..b8954f6148 --- /dev/null +++ b/.github/workflows/pypi_release_domain_library.yml @@ -0,0 +1,58 @@ +name: Prepare binary for pypi + +on: + workflow_dispatch: + inputs: + domain: + description: "What domain to prepare and remease" + required: false + type: choice + default: torchao + options: + - torchao + - executorch + +permissions: + id-token: write + contents: read + +jobs: + trigger: + runs-on: ubuntu-latest + environment: pytorchbot-env + timeout-minutes: 60 + container: + image: continuumio/miniconda3:23.10.0-1 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: configure aws credentials + id: aws_creds + uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3.0.2 + with: + role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_s3_update + aws-region: us-east-1 + - name: Stage torchao binaries for release + if: ${{ inputs.domain == 'torchao' }} + shell: bash + run: | + set -ex + cd release/pypi + # Install requirements + LINUX_VERSION_SUFFIX="%2Bcu128" + DRY_RUN=enabled + TORCHAO_VERSION=1.12.0 + + upload_pypi_to_staging() { + local package_name + package_name=$1 + local promote_version + promote_version=$2 + echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi staging' =-=-=-=" + ( + set -x + PACKAGE_VERSION="${promote_version}" PACKAGE_NAME="${package_name}" DRY_RUN="${DRY_RUN}" bash ${DIR}/upload_pypi_to_staging.sh + ) + echo + } + PLATFORM="none-any" VERSION_SUFFIX="${CPU_VERSION_SUFFIX}" upload_pypi_to_staging torchao "${TORCHAO_VERSION}" + PLATFORM="manylinux_2_28_x86_64" VERSION_SUFFIX="${LINUX_VERSION_SUFFIX}" ARCH="cu128" upload_pypi_to_staging torchao "${TORCHAO_VERSION}" From 633af5697f1a614fbf490d9cb91dd56b79436b05 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 11:56:29 -0700 Subject: [PATCH 02/18] test --- .github/workflows/pypi_release_domain_library.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index b8954f6148..e8efc5c710 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -1,6 +1,9 @@ name: Prepare binary for pypi on: + pull_request: + paths: + - .github/workflows/pypi_release_domain_library.yml workflow_dispatch: inputs: domain: From 8ea4e8f91c1c5084e1640b8f31f5febfbb71b603 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 11:57:49 -0700 Subject: [PATCH 03/18] fix --- .github/workflows/pypi_release_domain_library.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index e8efc5c710..dae2b567b3 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -15,10 +15,6 @@ on: - torchao - executorch -permissions: - id-token: write - contents: read - jobs: trigger: runs-on: ubuntu-latest @@ -28,12 +24,6 @@ jobs: image: continuumio/miniconda3:23.10.0-1 steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: configure aws credentials - id: aws_creds - uses: aws-actions/configure-aws-credentials@50ac8dd1e1b10d09dac7b8727528b91bed831ac0 # v3.0.2 - with: - role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_s3_update - aws-region: us-east-1 - name: Stage torchao binaries for release if: ${{ inputs.domain == 'torchao' }} shell: bash From 3e1f620478cad4c7235a11f761b05376b3905bf3 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 11:59:01 -0700 Subject: [PATCH 04/18] fix --- .github/workflows/pypi_release_domain_library.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index dae2b567b3..acc2253ae8 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -18,7 +18,6 @@ on: jobs: trigger: runs-on: ubuntu-latest - environment: pytorchbot-env timeout-minutes: 60 container: image: continuumio/miniconda3:23.10.0-1 From 0b64d2b68eeeb676187688f5728878a822cddb4d Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:00:04 -0700 Subject: [PATCH 05/18] morefixes --- .github/workflows/pypi_release_domain_library.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index acc2253ae8..385a4ac0d7 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -24,7 +24,6 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Stage torchao binaries for release - if: ${{ inputs.domain == 'torchao' }} shell: bash run: | set -ex From fabd768af3407ead28016bcc3dc4191ca930033f Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:01:17 -0700 Subject: [PATCH 06/18] morefixes --- .github/workflows/pypi_release_domain_library.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index 385a4ac0d7..0846d46304 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -33,6 +33,7 @@ jobs: DRY_RUN=enabled TORCHAO_VERSION=1.12.0 + pwd upload_pypi_to_staging() { local package_name package_name=$1 From fc5cc455ea76826194682a33fc774e26e1456403 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:03:15 -0700 Subject: [PATCH 07/18] stage_binaries_ecosystem --- release/pypi/promote_pypi_to_production.sh | 0 release/pypi/promote_pypi_to_staging.sh | 0 release/pypi/upload_pypi_to_staging.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 release/pypi/promote_pypi_to_production.sh mode change 100644 => 100755 release/pypi/promote_pypi_to_staging.sh mode change 100644 => 100755 release/pypi/upload_pypi_to_staging.sh diff --git a/release/pypi/promote_pypi_to_production.sh b/release/pypi/promote_pypi_to_production.sh old mode 100644 new mode 100755 diff --git a/release/pypi/promote_pypi_to_staging.sh b/release/pypi/promote_pypi_to_staging.sh old mode 100644 new mode 100755 diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh old mode 100644 new mode 100755 From 3967b35d750d1467429134528086c82a9080dcf6 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:06:20 -0700 Subject: [PATCH 08/18] fix --- .github/workflows/pypi_release_domain_library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index 0846d46304..2234090e7a 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -42,7 +42,7 @@ jobs: echo "=-=-=-= Promoting ${package_name}'s v${promote_version} to pypi staging' =-=-=-=" ( set -x - PACKAGE_VERSION="${promote_version}" PACKAGE_NAME="${package_name}" DRY_RUN="${DRY_RUN}" bash ${DIR}/upload_pypi_to_staging.sh + PACKAGE_VERSION="${promote_version}" PACKAGE_NAME="${package_name}" DRY_RUN="${DRY_RUN}" bash ./upload_pypi_to_staging.sh ) echo } From 56b273d298bfd05ef907688b4d4e08269f67e20b Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:10:54 -0700 Subject: [PATCH 09/18] test --- .github/workflows/pypi_release_domain_library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index 2234090e7a..ff0aaa52a8 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 container: - image: continuumio/miniconda3:23.10.0-1 + image: pytorch/almalinux-builder:cpu steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Stage torchao binaries for release From 790f38a8463333d361f84bf9d624e4b957f57675 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:21:04 -0700 Subject: [PATCH 10/18] fix --- release/pypi/upload_pypi_to_staging.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh index 319f11aee3..30f954f8b9 100755 --- a/release/pypi/upload_pypi_to_staging.sh +++ b/release/pypi/upload_pypi_to_staging.sh @@ -23,6 +23,7 @@ ARCH=${ARCH:-cpu} # This extract links to packages from the index.html # We strip all extra characters including final sha256 char +echo "Retrieving packages for promotion for ${PACKAGE_NAME} ${PACKAGE_VERSION}" pkgs_to_promote=$(\ curl -fsSL "https://download.pytorch.org/whl/test/${ARCH}/${PACKAGE_NAME}/index.html" \ | grep "${PACKAGE_NAME}-${PACKAGE_VERSION}${VERSION_SUFFIX}-" \ From 5f4ddc330a08194189788b330fd2cd6038d9b11c Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:24:39 -0700 Subject: [PATCH 11/18] fixfix --- .github/workflows/pypi_release_domain_library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index ff0aaa52a8..67605162ec 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -31,7 +31,7 @@ jobs: # Install requirements LINUX_VERSION_SUFFIX="%2Bcu128" DRY_RUN=enabled - TORCHAO_VERSION=1.12.0 + TORCHAO_VERSION=1.13.0 pwd upload_pypi_to_staging() { From 505aae011029ce841beb0e89d2760f3f762f103c Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:32:37 -0700 Subject: [PATCH 12/18] fix --- release/pypi/upload_pypi_to_staging.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh index 30f954f8b9..01409230e0 100755 --- a/release/pypi/upload_pypi_to_staging.sh +++ b/release/pypi/upload_pypi_to_staging.sh @@ -31,6 +31,7 @@ pkgs_to_promote=$(\ | cut -d '"' -f2 \ | cut -d "#" -f1 ) +echo "Found ${pkgs_to_promote}" tmp_dir="$(mktemp -d)" output_tmp_dir="$(mktemp -d)" @@ -54,8 +55,10 @@ for pkg in ${pkgs_to_promote}; do curl -fSL -o "${orig_pkg}" "https://download.pytorch.org${pkg}" ) + if [[ -n "${VERSION_SUFFIX}" ]]; then - OUTPUT_DIR="${output_tmp_dir}" python "${DIR}/prep_binary_for_pypi.py" "${orig_pkg}" --output-dir . + echo "Calling prep binary for pypi on ${orig_pkg}" + OUTPUT_DIR="${output_tmp_dir}" python "./prep_binary_for_pypi.py" "${orig_pkg}" --output-dir . else mv "${orig_pkg}" "${output_tmp_dir}/" fi From 3dc5c0bc4ee8f1dfffbcf7e5757074f9b145de49 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 12:42:21 -0700 Subject: [PATCH 13/18] fix --- .github/workflows/pypi_release_domain_library.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index 67605162ec..d3b3928727 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -30,6 +30,7 @@ jobs: cd release/pypi # Install requirements LINUX_VERSION_SUFFIX="%2Bcu128" + CPU_VERSION_SUFFIX="%2Bcpu" DRY_RUN=enabled TORCHAO_VERSION=1.13.0 From 87f3835f9c0b3dd3f5909548c1133e6e0a1c7a1b Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 13:21:24 -0700 Subject: [PATCH 14/18] fix --- .github/workflows/pypi_release_domain_library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index d3b3928727..5b1c8189cb 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -32,7 +32,7 @@ jobs: LINUX_VERSION_SUFFIX="%2Bcu128" CPU_VERSION_SUFFIX="%2Bcpu" DRY_RUN=enabled - TORCHAO_VERSION=1.13.0 + TORCHAO_VERSION=0.13.0 pwd upload_pypi_to_staging() { From 17599c4d76f83707703e425d7fe80a4b2da33808 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 1 Sep 2025 16:32:26 -0700 Subject: [PATCH 15/18] fix --- release/pypi/upload_pypi_to_staging.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release/pypi/upload_pypi_to_staging.sh b/release/pypi/upload_pypi_to_staging.sh index 01409230e0..2f4b7345d8 100755 --- a/release/pypi/upload_pypi_to_staging.sh +++ b/release/pypi/upload_pypi_to_staging.sh @@ -57,8 +57,7 @@ for pkg in ${pkgs_to_promote}; do if [[ -n "${VERSION_SUFFIX}" ]]; then - echo "Calling prep binary for pypi on ${orig_pkg}" - OUTPUT_DIR="${output_tmp_dir}" python "./prep_binary_for_pypi.py" "${orig_pkg}" --output-dir . + OUTPUT_DIR="${output_tmp_dir}" python "${DIR}/prep_binary_for_pypi.py" "${orig_pkg}" --output-dir . else mv "${orig_pkg}" "${output_tmp_dir}/" fi From 04b014a4bc93d91494928ea34dc45778f5527241 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 2 Sep 2025 06:01:52 -0700 Subject: [PATCH 16/18] test --- .github/workflows/pypi_release_domain_library.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index 5b1c8189cb..dca48e7ac9 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -29,6 +29,7 @@ jobs: set -ex cd release/pypi # Install requirements + pip install -y auditwheel LINUX_VERSION_SUFFIX="%2Bcu128" CPU_VERSION_SUFFIX="%2Bcpu" DRY_RUN=enabled From cac76a50d2aab9dce475befca2086947f2b9c708 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 2 Sep 2025 06:40:06 -0700 Subject: [PATCH 17/18] audit --- .github/workflows/pypi_release_domain_library.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index dca48e7ac9..ec62e3fd97 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -29,7 +29,7 @@ jobs: set -ex cd release/pypi # Install requirements - pip install -y auditwheel + pip install auditwheel LINUX_VERSION_SUFFIX="%2Bcu128" CPU_VERSION_SUFFIX="%2Bcpu" DRY_RUN=enabled From b34a2bd0aa1e2ae712f77c94a0c988ab18334484 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 2 Sep 2025 07:28:35 -0700 Subject: [PATCH 18/18] aws_cli --- .github/workflows/pypi_release_domain_library.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi_release_domain_library.yml b/.github/workflows/pypi_release_domain_library.yml index ec62e3fd97..90a0ddfa70 100644 --- a/.github/workflows/pypi_release_domain_library.yml +++ b/.github/workflows/pypi_release_domain_library.yml @@ -1,4 +1,4 @@ -name: Prepare binary for pypi +name: Prepare binary and release binary to pypi on: pull_request: @@ -7,13 +7,12 @@ on: workflow_dispatch: inputs: domain: - description: "What domain to prepare and remease" + description: "Domain to prepare and release" required: false type: choice default: torchao options: - torchao - - executorch jobs: trigger: @@ -29,7 +28,7 @@ jobs: set -ex cd release/pypi # Install requirements - pip install auditwheel + pip install auditwheel awscli LINUX_VERSION_SUFFIX="%2Bcu128" CPU_VERSION_SUFFIX="%2Bcpu" DRY_RUN=enabled