From 5eda2af8ef4757325ce4de4c93d2e7d21429fba7 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Mon, 11 Nov 2024 16:48:37 -0800 Subject: [PATCH 1/4] [WIP] pypi_deploy.sh. Currently downloads some wheels given a specified version. Has code to edit release versions and upload, taken from other similar scripts, but not tested e2e. --- build_tools/pypi_deploy.sh | 84 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 build_tools/pypi_deploy.sh diff --git a/build_tools/pypi_deploy.sh b/build_tools/pypi_deploy.sh new file mode 100755 index 00000000..e7dd8c28 --- /dev/null +++ b/build_tools/pypi_deploy.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright 2024 Advanced Micro Devices, Inc. +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# This script promotes Python packages from nightly releases to PyPI. +# +# Prerequisites: +# * You will need to have PyPI credentials set up. See +# https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives +# * Install requirements, e.g. in a Python virtual environment (venv): +# `pip install -r requirements-pypi-deploy.txt` +# * Choose a release candidate to promote from +# https://github.com/nod-ai/SHARK-Platform/releases/tag/dev-wheels +# +# Usage: +# ./pypi_deploy.sh 2.9.0rc20241108 + +set -euo pipefail + +RELEASE="$1" + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")"; +TMPDIR="$(mktemp --directory --tmpdir shark_platform_pypi_wheels.XXXXX)" +ASSETS_PAGE="https://github.com/nod-ai/SHARK-Platform/releases/expanded_assets/dev-wheels" + +# TODO: rewrite in Python? + +function download_wheels() { + echo "" + echo "Downloading wheels for '${RELEASE}'..." + + # sharktank + python -m pip download sharktank==${RELEASE} \ + --no-deps --python-version 3.11 -f ${ASSETS_PAGE} + + # shortfin + python -m pip download shortfin==${RELEASE} \ + --no-deps --python-version 3.11 -f ${ASSETS_PAGE} + python -m pip download shortfin==${RELEASE} \ + --no-deps --python-version 3.12 -f ${ASSETS_PAGE} + python -m pip download shortfin==${RELEASE} \ + --no-deps --python-version 3.13 -f ${ASSETS_PAGE} + python -m pip download shortfin==${RELEASE} \ + --no-deps --python-version 3.13 -f ${ASSETS_PAGE} + # TODO: 3.13t somehow + + echo "" + echo "Downloaded wheels:" + ls +} + +function edit_release_versions() { + echo "" + echo "Editing release versions..." + for file in * + do + ${SCRIPT_DIR}/promote_whl_from_rc_to_final.py ${file} --delete-old-wheel + done + + echo "Edited wheels:" + ls +} + +function upload_wheels() { + echo "" + echo "Uploading wheels..." + twine upload --verbose * +} + +function main() { + echo "Changing into ${TMPDIR}" + cd "${TMPDIR}" + # TODO: check_requirements (using pip) + + download_wheels + # edit_release_versions + # upload_wheels +} + +main From 3492d217fadd590778c31805acf3c6a78a0abd33 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Thu, 14 Nov 2024 11:24:19 -0800 Subject: [PATCH 2/4] Organize scripts, build meta package, add download for 3.13t. --- .../python_deploy/compute_common_version.py | 8 ++++-- .../python_deploy/compute_local_version.py | 0 .../promote_whl_from_rc_to_final.py | 0 .../{ => python_deploy}/pypi_deploy.sh | 28 +++++++++++++++++-- .../requirements-pypi-deploy.txt | 0 .../python_deploy/write_requirements.py | 0 shark-ai/build_tools/build_linux_package.sh | 25 +++++++++++++++++ 7 files changed, 56 insertions(+), 5 deletions(-) mode change 100644 => 100755 build_tools/python_deploy/compute_common_version.py mode change 100644 => 100755 build_tools/python_deploy/compute_local_version.py rename build_tools/{ => python_deploy}/promote_whl_from_rc_to_final.py (100%) rename build_tools/{ => python_deploy}/pypi_deploy.sh (69%) rename build_tools/{ => python_deploy}/requirements-pypi-deploy.txt (100%) mode change 100644 => 100755 build_tools/python_deploy/write_requirements.py create mode 100755 shark-ai/build_tools/build_linux_package.sh diff --git a/build_tools/python_deploy/compute_common_version.py b/build_tools/python_deploy/compute_common_version.py old mode 100644 new mode 100755 index 6aea7f25..ed6f8c70 --- a/build_tools/python_deploy/compute_common_version.py +++ b/build_tools/python_deploy/compute_common_version.py @@ -6,8 +6,12 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # This scripts grabs the `X.Y.Z[.dev]` version identifier from the -# sharktank and shortfin version files and computes the version -# for the meta package. +# 'sharktank' and 'shortfin' version files and computes the version +# for the meta 'shark-ai' package. +# +# Usage: +# ./compute_common_version.py --stable-release --write-json +# cat ../../shark-ai/version_local.json import argparse from pathlib import Path diff --git a/build_tools/python_deploy/compute_local_version.py b/build_tools/python_deploy/compute_local_version.py old mode 100644 new mode 100755 diff --git a/build_tools/promote_whl_from_rc_to_final.py b/build_tools/python_deploy/promote_whl_from_rc_to_final.py similarity index 100% rename from build_tools/promote_whl_from_rc_to_final.py rename to build_tools/python_deploy/promote_whl_from_rc_to_final.py diff --git a/build_tools/pypi_deploy.sh b/build_tools/python_deploy/pypi_deploy.sh similarity index 69% rename from build_tools/pypi_deploy.sh rename to build_tools/python_deploy/pypi_deploy.sh index e7dd8c28..bd07ed57 100755 --- a/build_tools/pypi_deploy.sh +++ b/build_tools/python_deploy/pypi_deploy.sh @@ -13,6 +13,13 @@ # https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives # * Install requirements, e.g. in a Python virtual environment (venv): # `pip install -r requirements-pypi-deploy.txt` +# * Install python3.13t and install pip. On Ubuntu: +# ```bash +# sudo add-apt-repository ppa:deadsnakes +# sudo apt-get update +# sudo apt-get install python3.13-nogil +# python3.13t -m ensurepip --upgrade +# ``` # * Choose a release candidate to promote from # https://github.com/nod-ai/SHARK-Platform/releases/tag/dev-wheels # @@ -24,6 +31,7 @@ set -euo pipefail RELEASE="$1" SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")"; +REPO_ROOT="$(cd "$SCRIPT_DIR"/../../ && pwd)" TMPDIR="$(mktemp --directory --tmpdir shark_platform_pypi_wheels.XXXXX)" ASSETS_PAGE="https://github.com/nod-ai/SHARK-Platform/releases/expanded_assets/dev-wheels" @@ -46,7 +54,13 @@ function download_wheels() { --no-deps --python-version 3.13 -f ${ASSETS_PAGE} python -m pip download shortfin==${RELEASE} \ --no-deps --python-version 3.13 -f ${ASSETS_PAGE} - # TODO: 3.13t somehow + # TODO: fetch 3.13t using the same `python` somehow + # * https://pip.pypa.io/en/stable/cli/pip_download/ + # * https://py-free-threading.github.io/installing_cpython/ + # * https://pip.pypa.io/en/stable/installation/ + python3.13t -m pip download shortfin==${RELEASE} --no-deps -f ${ASSETS_PAGE} + + # TODO: shark-ai meta package when it exists echo "" echo "Downloaded wheels:" @@ -71,14 +85,22 @@ function upload_wheels() { twine upload --verbose * } +function build_shark_ai_meta_package() { + echo "" + echo "Computing common version for shark-ai meta package..." + ${SCRIPT_DIR}/compute_common_version.py --stable-release --write-json + ${REPO_ROOT}/shark-ai/build_tools/build_linux_package.sh +} + function main() { echo "Changing into ${TMPDIR}" cd "${TMPDIR}" # TODO: check_requirements (using pip) download_wheels - # edit_release_versions - # upload_wheels + edit_release_versions + build_shark_ai_meta_package + upload_wheels } main diff --git a/build_tools/requirements-pypi-deploy.txt b/build_tools/python_deploy/requirements-pypi-deploy.txt similarity index 100% rename from build_tools/requirements-pypi-deploy.txt rename to build_tools/python_deploy/requirements-pypi-deploy.txt diff --git a/build_tools/python_deploy/write_requirements.py b/build_tools/python_deploy/write_requirements.py old mode 100644 new mode 100755 diff --git a/shark-ai/build_tools/build_linux_package.sh b/shark-ai/build_tools/build_linux_package.sh new file mode 100755 index 00000000..d16f339b --- /dev/null +++ b/shark-ai/build_tools/build_linux_package.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2024 Advanced Micro Devices, Inc. +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# build_linux_package.sh +# +# Builds shark-ai Python package for Linux. +# +# Usage: +# ./build_tools/build_linux_package.sh + +set -xeu -o errtrace + +THIS_DIR="$(cd $(dirname $0) && pwd)" +REPO_ROOT="$(cd "$THIS_DIR"/../../ && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-${THIS_DIR}/wheelhouse}" + +python -m pip wheel --disable-pip-version-check --no-deps -v -w "${OUTPUT_DIR}" "${REPO_ROOT}/shark-ai" + +wheel_output="$(echo "${OUTPUT_DIR}/shark_ai-"*".whl")" +ls "${wheel_output}" From b016e90a888ea5b0cc879ef92c270bba8174e29c Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Thu, 14 Nov 2024 11:34:04 -0800 Subject: [PATCH 3/4] Add docs. --- build_tools/python_deploy/README.md | 48 ++++++++++++++++++++++++ build_tools/python_deploy/pypi_deploy.sh | 3 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 build_tools/python_deploy/README.md diff --git a/build_tools/python_deploy/README.md b/build_tools/python_deploy/README.md new file mode 100644 index 00000000..d36545a9 --- /dev/null +++ b/build_tools/python_deploy/README.md @@ -0,0 +1,48 @@ +# Python Deployment + +These scripts assist with building Python packages and pushing them to +[PyPI (the Python Package Index)](https://pypi.org/). See also + +* The Python Packaging User Guide: + +## Overview + +See comments in scripts for canonical usage. This page includes additional +notes. + +### Package building + +These scripts build packages: + +* [`/shark-ai/build_tools/build_linux_package.sh`](/shark-ai/build_tools/build_linux_package.sh) +* [`/sharktank/build_tools/build_linux_package.sh`](/sharktank/build_tools/build_linux_package.sh) +* [`/shortfin/build_tools/build_linux_package.sh`](/shortfin/build_tools/build_linux_package.sh) + +### Version management + +These scripts handle versioning across packages, including considerations like +major, minor, and patch levels (`X.Y.Z`), as well as suffixes like +`rc20241107`: + +* [`compute_common_version.py`](./compute_common_version.py) +* [`compute_local_version.py`](./compute_local_version.py) +* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py) +* [`write_requirements.py`](./write_requirements.py) + +### PyPI deployment + +These scripts handle promoting nightly releases packages to stable and pushing +to PyPI: + +* [`promote_whl_from_rc_to_final.py`](./promote_whl_from_rc_to_final.py) +* [`pypi_deploy.sh`](./pypi_deploy.sh) + +Both of these scripts expect to have the dependencies from +[`requirements-pypi-deploy.txt`](./requirements-pypi-deploy.txt) installed. +This can be easily managed by using a Python virtual environment: + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install -r ./requirements-pypi-deploy.txt +``` diff --git a/build_tools/python_deploy/pypi_deploy.sh b/build_tools/python_deploy/pypi_deploy.sh index bd07ed57..9caa108a 100755 --- a/build_tools/python_deploy/pypi_deploy.sh +++ b/build_tools/python_deploy/pypi_deploy.sh @@ -60,7 +60,7 @@ function download_wheels() { # * https://pip.pypa.io/en/stable/installation/ python3.13t -m pip download shortfin==${RELEASE} --no-deps -f ${ASSETS_PAGE} - # TODO: shark-ai meta package when it exists + # TODO: shark-ai meta package when it is published to nightlies echo "" echo "Downloaded wheels:" @@ -86,6 +86,7 @@ function upload_wheels() { } function build_shark_ai_meta_package() { + # TODO: download meta package from nightly releases instead of this echo "" echo "Computing common version for shark-ai meta package..." ${SCRIPT_DIR}/compute_common_version.py --stable-release --write-json From 6a03eb435e6073e29a0f724699e63bac2d5d44cd Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Thu, 14 Nov 2024 11:59:53 -0800 Subject: [PATCH 4/4] Compute local versions, write requirements, move meta wheel into publish dir. --- build_tools/python_deploy/pypi_deploy.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/build_tools/python_deploy/pypi_deploy.sh b/build_tools/python_deploy/pypi_deploy.sh index 9caa108a..c5f1128c 100755 --- a/build_tools/python_deploy/pypi_deploy.sh +++ b/build_tools/python_deploy/pypi_deploy.sh @@ -80,17 +80,34 @@ function edit_release_versions() { } function upload_wheels() { + # TODO: list packages that would be uploaded, pause, prompt to continue echo "" - echo "Uploading wheels..." + echo "Uploading wheels:" + ls twine upload --verbose * } function build_shark_ai_meta_package() { # TODO: download meta package from nightly releases instead of this echo "" + + # TODO: rework `write_requirements.py` to use the versions from the downloaded whls? + echo "Computing local versions for sharktank and shortfin..." + ${SCRIPT_DIR}/compute_local_version.py ${REPO_ROOT}/sharktank + ${SCRIPT_DIR}/compute_local_version.py ${REPO_ROOT}/shortfin + echo "Computing common version for shark-ai meta package..." ${SCRIPT_DIR}/compute_common_version.py --stable-release --write-json + + echo "Writing requirements for shark-ai meta package..." + ${SCRIPT_DIR}/write_requirements.py + + echo "Building shark-ai meta package..." ${REPO_ROOT}/shark-ai/build_tools/build_linux_package.sh + + # TODO: This is error-prone. We only want to publish the whl for this release. + # copy instead? specify exact file name? clear directory before building? + mv ${REPO_ROOT}/shark-ai/build_tools/wheelhouse/* . } function main() {