From b7d9b309e639442a2ecdf193e00ffdb2f862057d Mon Sep 17 00:00:00 2001 From: Hang Date: Thu, 24 Oct 2024 16:04:05 +0800 Subject: [PATCH] [Packaging] Bump embedded Python version to 3.12.7 (#29887) --- .azure-pipelines/breaking-change-tests.yml | 4 +- azure-pipelines-full-tests.yml | 45 +------- azure-pipelines.yml | 107 ++++++++---------- build_scripts/windows/scripts/build.cmd | 28 +++-- doc/extensions/authoring.md | 2 +- scripts/ci/test_extensions.sh | 6 +- .../extension_regression_test.yml | 15 +-- scripts/regression_test/regression_test.yml | 8 +- scripts/release/debian/build.sh | 2 +- .../homebrew/docker/formula_generate.py | 11 +- scripts/release/homebrew/docker/run.sh | 4 +- .../components/records_collection.py | 4 +- 12 files changed, 105 insertions(+), 131 deletions(-) diff --git a/.azure-pipelines/breaking-change-tests.yml b/.azure-pipelines/breaking-change-tests.yml index 80091f7e703..62b655c9a58 100644 --- a/.azure-pipelines/breaking-change-tests.yml +++ b/.azure-pipelines/breaking-change-tests.yml @@ -22,9 +22,9 @@ jobs: fetchTags: true persistCredentials: true - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/azdev_setup.yml - bash: | set -ev diff --git a/azure-pipelines-full-tests.yml b/azure-pipelines-full-tests.yml index d3f8a3e0d96..772c66fc694 100644 --- a/azure-pipelines-full-tests.yml +++ b/azure-pipelines-full-tests.yml @@ -25,8 +25,6 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' Python312: python.version: '3.12' steps: @@ -46,8 +44,6 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' Python312: python.version: '3.12' steps: @@ -67,8 +63,6 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' Python312: python.version: '3.12' steps: @@ -113,40 +107,6 @@ jobs: fullTest: true jobName: 'FullTest' -- job: AutomationFullTestPython311ProfileLatest - displayName: Automation Full Test Python311 Profile Latest - timeoutInMinutes: 9999 - strategy: - maxParallel: 8 - matrix: - instance1: - Instance_idx: 1 - instance2: - Instance_idx: 2 - instance3: - Instance_idx: 3 - instance4: - Instance_idx: 4 - instance5: - Instance_idx: 5 - instance6: - Instance_idx: 6 - instance7: - Instance_idx: 7 - instance8: - Instance_idx: 8 - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '3.11' - profile: 'latest' - instance_cnt: '8' - instance_idx: '$(Instance_idx)' - fullTest: true - jobName: 'FullTest' - - job: AutomationFullTestPython312ProfileLatest displayName: Automation Full Test Python312 Profile Latest timeoutInMinutes: 9999 @@ -187,16 +147,15 @@ jobs: - AutomationTest20190301 - AutomationTest20180301 - AutomationFullTestPython39ProfileLatest - - AutomationFullTestPython311ProfileLatest condition: and(failed(), in(variables['Build.Reason'], 'BatchedCI')) displayName: Notify CI Errors pool: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - task: AzureCLI@2 inputs: azureSubscription: 'Azure CLI' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d02692c5717..5928a14132d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -129,9 +129,9 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - bash: ./scripts/ci/dependency_check.sh displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' @@ -144,9 +144,9 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - bash: ./scripts/ci/dependency_check.sh displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' @@ -159,9 +159,9 @@ jobs: steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - task: BatchScript@1 inputs: @@ -176,9 +176,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -437,9 +437,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - task: PipAuthenticate@1 condition: eq(variables['Build.SourceBranch'], 'refs/heads/release') @@ -469,7 +469,12 @@ jobs: - job: TestPythonWheel displayName: Test Python Wheels - + strategy: + matrix: + Python39: + python.version: '3.9' + Python312: + python.version: '3.12' dependsOn: BuildPythonWheel condition: succeeded() pool: @@ -480,35 +485,23 @@ jobs: inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' artifactName: metadata - - - task: DownloadPipelineArtifact@1 displayName: 'Download PyPI Packages' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' artifactName: pypi - - + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' - bash: | #!/usr/bin/env bash - - # Verify the pip wheels - set -ex CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - PYPI_FILES=$(cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi; pwd) - - echo "== Testing pip install on Python 3.9 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi mcr.microsoft.com/mirror/docker/library/python:3.9-slim \ - /bin/bash -c "cd /mnt/pypi && ls && pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5" - - echo "== Testing pip install on Python 3.11 ==" - docker run \ - --rm -v $PYPI_FILES:/mnt/pypi mcr.microsoft.com/mirror/docker/library/python:3.11-slim \ - /bin/bash -c "cd /mnt/pypi && ls && pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5" - + echo "== Testing pip install on $PYTHON_VERSION ==" + cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi + pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5 displayName: 'Test pip Install' - job: TestCore @@ -520,8 +513,8 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' + Python312: + python.version: '3.12' steps: - template: .azure-pipelines/templates/automation_test.yml parameters: @@ -537,8 +530,8 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' + Python312: + python.version: '3.12' steps: - template: .azure-pipelines/templates/automation_test.yml parameters: @@ -557,14 +550,14 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' + Python312: + python.version: '3.12' steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel + - bash: pip install --upgrade pip wheel setuptools displayName: 'Install pip and wheel' - bash: ./scripts/ci/test_profile_integration.sh displayName: 'Run Integration Test against Profiles' @@ -578,15 +571,15 @@ jobs: name: ${{ variables.ubuntu_pool }} strategy: matrix: - Python311: - python.version: '3.11' + Python312: + python.version: '3.12' steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' inputs: versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel - displayName: 'Install pip and wheel' + - bash: pip install --upgrade pip wheel setuptools + displayName: 'Install pip and wheel setuptools' - bash: ./scripts/ci/test_extensions.sh displayName: 'Load extensions' @@ -623,7 +616,7 @@ jobs: -e CLI_VERSION=$CLI_VERSION \ -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ --name azurecli \ - mcr.microsoft.com/mirror/docker/library/python:3.11-slim \ + mcr.microsoft.com/azurelinux/base/python:3 \ /mnt/scripts/run.sh # clean up @@ -670,7 +663,7 @@ jobs: set -ev # Force relink python@3.xx in Homebrew to resolve the conflict with pre-installed python 3.xx on macOS-12 image # See: https://github.com/Azure/azure-cli/issues/29054 - python_version=3.11 + python_version=3.12 brew unlink python@$python_version && brew link --overwrite python@$python_version echo == Remove pre-installed azure-cli == @@ -1050,9 +1043,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -1066,9 +1059,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -1083,8 +1076,8 @@ jobs: matrix: Python39: python.version: '3.9' - Python311: - python.version: '3.11' + Python312: + python.version: '3.12' pool: name: ${{ variables.ubuntu_pool }} steps: @@ -1112,9 +1105,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -1135,9 +1128,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -1163,9 +1156,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - template: .azure-pipelines/templates/azdev_setup.yml - bash: | set -ev @@ -1253,9 +1246,9 @@ jobs: name: ${{ variables.ubuntu_pool }} steps: - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' + displayName: 'Use Python 3.12' inputs: - versionSpec: 3.11 + versionSpec: 3.12 - task: AzureCLI@2 inputs: azureSubscription: 'Azure CLI' diff --git a/build_scripts/windows/scripts/build.cmd b/build_scripts/windows/scripts/build.cmd index c5674d3fd3b..96599d31a85 100644 --- a/build_scripts/windows/scripts/build.cmd +++ b/build_scripts/windows/scripts/build.cmd @@ -30,7 +30,7 @@ if "%ARCH%"=="x86" ( echo Please set ARCH to "x86" or "x64" goto ERROR ) -set PYTHON_VERSION=3.11.8 +set PYTHON_VERSION=3.12.7 set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip" set PYTHON_DOWNLOAD_URL="https://www.python.org/ftp/python/%PYTHON_VERSION%/python-%PYTHON_VERSION%-embed-%PYTHON_ARCH%.zip" @@ -121,32 +121,42 @@ if not exist %PYTHON_DIR% ( del get-pip.py echo Pip set up successful - dir . + REM setuptools is not installed by default in Python 3.12, but it is required by some dependencys + REM See https://github.com/Azure/azure-cli/pull/27196 + REM Install wheel to force pip install azure-cli in legacy mode + REM see https://github.com/Azure/azure-cli/pull/29887 + echo Installing setuptools wheel + %PYTHON_DIR%\python.exe -Im pip install setuptools wheel + popd ) set PYTHON_EXE=%PYTHON_DIR%\python.exe + robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL set CLI_SRC=%REPO_ROOT%\src for %%a in (%CLI_SRC%\azure-cli %CLI_SRC%\azure-cli-core %CLI_SRC%\azure-cli-telemetry) do ( pushd %%a - %BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --no-cache-dir --no-deps . + %BUILDING_DIR%\python.exe -Im pip install --no-warn-script-location --no-cache-dir --no-deps . + if %errorlevel% neq 0 goto ERROR popd ) -%BUILDING_DIR%\python.exe -m pip install --no-warn-script-location --requirement %CLI_SRC%\azure-cli\requirements.py3.windows.txt -REM Check azure.cli can be executed. This also prints the Python version. -%BUILDING_DIR%\python.exe -m azure.cli --version +%BUILDING_DIR%\python.exe -Im pip install --no-warn-script-location --requirement %CLI_SRC%\azure-cli\requirements.py3.windows.txt +if %errorlevel% neq 0 goto ERROR +REM Check azure.cli can be executed. This also prints the Python version. +%BUILDING_DIR%\python.exe -Im azure.cli --version if %errorlevel% neq 0 goto ERROR + pushd %BUILDING_DIR% -%BUILDING_DIR%\python.exe %REPO_ROOT%\scripts\compact_aaz.py +%BUILDING_DIR%\python.exe -I %REPO_ROOT%\scripts\compact_aaz.py if %errorlevel% neq 0 goto ERROR -%BUILDING_DIR%\python.exe %~dp0\patch_models_v2.py +%BUILDING_DIR%\python.exe -I %~dp0\patch_models_v2.py if %errorlevel% neq 0 goto ERROR -%BUILDING_DIR%\python.exe %REPO_ROOT%\scripts\trim_sdk.py +%BUILDING_DIR%\python.exe -I %REPO_ROOT%\scripts\trim_sdk.py if %errorlevel% neq 0 goto ERROR popd diff --git a/doc/extensions/authoring.md b/doc/extensions/authoring.md index a4a7f219af5..e259d1d6761 100644 --- a/doc/extensions/authoring.md +++ b/doc/extensions/authoring.md @@ -130,7 +130,7 @@ See [Extension Metadata](metadata.md) for more information. ### Test your extension on Python 3 -- The Azure CLI supports Python 3.8 ~ 3.11 so verify that your extension does the same. +- The Azure CLI supports Python 3.8 ~ 3.12 so verify that your extension does the same. - You can create virtual environments for different versions and run your extension in them. - e.g. `python3.8 -m venv env38` and `python3.10 -m venv env310`. diff --git a/scripts/ci/test_extensions.sh b/scripts/ci/test_extensions.sh index df2c15ad51f..d2f48c74a5b 100755 --- a/scripts/ci/test_extensions.sh +++ b/scripts/ci/test_extensions.sh @@ -24,7 +24,11 @@ exit_code=0 # Disable k8s-extension temporarily: https://github.com/Azure/azure-cli-extensions/pull/6702 # Disable alias temporarily: https://github.com/Azure/azure-cli/pull/27717 # hybridaks is going to be deprecated: https://github.com/Azure/azure-cli/pull/29838 -ignore_list='azure-cli-ml fzf arcappliance arcdata connectedk8s k8s-extension alias hybridaks' +# db-up is going to be deprecated: https://github.com/Azure/azure-cli/pull/29887 +ignore_list='azure-cli-ml fzf arcappliance arcdata connectedk8s k8s-extension alias hybridaks db-up' + +# Does not exit if az extension add fails until all extensions have been tested +set +e for ext in $output; do echo diff --git a/scripts/regression_test/extension_regression_test.yml b/scripts/regression_test/extension_regression_test.yml index fa93aef2bab..64607c26a4c 100644 --- a/scripts/regression_test/extension_regression_test.yml +++ b/scripts/regression_test/extension_regression_test.yml @@ -19,8 +19,8 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.10' - displayName: "Use Python 3.10" + versionSpec: '3.12' + displayName: "Use Python 3.12" - task: AzureCLI@1 displayName: 'checkout branch' inputs: @@ -76,8 +76,8 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.10' - displayName: "Use Python 3.10" + versionSpec: '3.12' + displayName: "Use Python 3.12" - task: AzureCLI@1 displayName: 'checkout cli and extension repo' inputs: @@ -113,13 +113,14 @@ jobs: scriptLocation: inlineScript inlineScript: | set -ev - + source env/bin/activate cd azure-cli-extensions - + az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590 - + python ../scripts/ci/automation_full_test.py "12" "$(Instance_idx)" "latest" "" "True" "extension" + - bash: | publishErrorModules='false' if [[ -f '/$(HOME)/.azdev/env_config/mnt/vss/_work/1/s/env/test_results_error_modules_$(Instance_idx).txt' ]]; then diff --git a/scripts/regression_test/regression_test.yml b/scripts/regression_test/regression_test.yml index 27890a81d98..4881e28b626 100644 --- a/scripts/regression_test/regression_test.yml +++ b/scripts/regression_test/regression_test.yml @@ -20,8 +20,8 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.11' - displayName: "Use Python 3.11" + versionSpec: '3.12' + displayName: "Use Python 3.12" - task: AzureCLI@1 displayName: 'update version' inputs: @@ -88,8 +88,8 @@ jobs: steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.11' - displayName: "Use Python 3.11" + versionSpec: '3.12' + displayName: "Use Python 3.12" - task: AzureCLI@1 displayName: 'Checkout Target Branch' inputs: diff --git a/scripts/release/debian/build.sh b/scripts/release/debian/build.sh index d66d7026de5..c5625bb9aef 100755 --- a/scripts/release/debian/build.sh +++ b/scripts/release/debian/build.sh @@ -15,7 +15,7 @@ set -exv ls -Rl /mnt/artifacts WORKDIR=`cd $(dirname $0); cd ../../../; pwd` -PYTHON_VERSION="3.11.8" +PYTHON_VERSION="3.12.7" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # Update APT packages diff --git a/scripts/release/homebrew/docker/formula_generate.py b/scripts/release/homebrew/docker/formula_generate.py index 8c2ab96ada9..b35a2bed9db 100644 --- a/scripts/release/homebrew/docker/formula_generate.py +++ b/scripts/release/homebrew/docker/formula_generate.py @@ -10,7 +10,7 @@ import requests import jinja2 -from poet.poet import make_graph, RESOURCE_TEMPLATE +from poet.poet import make_graph, RESOURCE_TEMPLATE, research_package from collections import OrderedDict import bisect import argparse @@ -19,7 +19,7 @@ CLI_VERSION = os.environ['CLI_VERSION'] HOMEBREW_UPSTREAM_URL = os.environ['HOMEBREW_UPSTREAM_URL'] HOMEBREW_FORMULAR_LATEST = "https://raw.githubusercontent.com/Homebrew/homebrew-core/master/Formula/a/azure-cli.rb" -PYTHON_VERSION = '3.11' +PYTHON_VERSION = '3.12' def main(): @@ -85,6 +85,13 @@ def collect_resources() -> str: def collect_resources_dict() -> dict: nodes = make_graph('azure-cli') + + # Homebrew does not install pip and setuptools after Python 3.12 + # see https://github.com/Azure/azure-cli/pull/29887 + extra_dependencies = ['pip', 'setuptools'] + for dependency in extra_dependencies: + nodes[dependency] = research_package(dependency) + filtered_nodes = {nodes[node_name]['name']: nodes[node_name] for node_name in sorted(nodes) if resource_filter(node_name)} return filtered_nodes diff --git a/scripts/release/homebrew/docker/run.sh b/scripts/release/homebrew/docker/run.sh index c529dbce638..65a5caf11de 100755 --- a/scripts/release/homebrew/docker/run.sh +++ b/scripts/release/homebrew/docker/run.sh @@ -2,6 +2,8 @@ root=$(cd $(dirname $0); pwd) +tdnf install -y ca-certificates + pip install wheel pip install -U pip pip install -r $root/requirements.txt @@ -12,4 +14,4 @@ pip list # default option is update_existing to build from homebrew master branch, # append '-b use_template' to build from formula_template.txt -python $root/formula_generate.py +python3 $root/formula_generate.py diff --git a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py index bc95c2c7d67..3419075cca7 100644 --- a/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py +++ b/src/azure-cli-telemetry/azure/cli/telemetry/components/records_collection.py @@ -40,9 +40,7 @@ def snapshot_and_read(self): for each in os.listdir(self._cache_dir): self._read_file(os.path.join(self._cache_dir, each)) - shutil.rmtree(self._cache_dir, - ignore_errors=True, - onerror=lambda _, p, tr: self._logger.error('Fail to remove file %s', p)) + shutil.rmtree(self._cache_dir, ignore_errors=True) self._logger.info('Remove directory %s', self._cache_dir) def _read_file(self, path):