ci(github-actions): Upgrade uv and ruff versions to fix CI

(cherry picked from commit 26c93bc)

Author: Lee-W

.github/actions/breeze/action.yml

@@ -24,7 +24,7 @@ inputs:
     default: "3.10"
   uv-version:
     description: 'uv version to use'
-    default: "0.9.4"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.9.7"  # Keep this comment to allow automatic replacement of uv version
 outputs:
   host-python-version:
     description: Python version used in host

.github/actions/install-prek/action.yml

@@ -24,10 +24,10 @@ inputs:
     default: "3.10"
   uv-version:
     description: 'uv version to use'
-    default: "0.9.4"  # Keep this comment to allow automatic replacement of uv version
+    default: "0.9.7"  # Keep this comment to allow automatic replacement of uv version
   prek-version:
     description: 'prek version to use'
-    default: "0.2.10"  # Keep this comment to allow automatic replacement of prek version
+    default: "0.2.12"  # Keep this comment to allow automatic replacement of prek version
   save-cache:
     description: "Whether to save prek cache"
     required: true

.github/workflows/basic-tests.yml

@@ -66,7 +66,7 @@ on:  # yamllint disable-line rule:truthy
         type: string
       uv-version:
         description: 'uv version to use'
-        default: "0.9.4"  # Keep this comment to allow automatic replacement of uv version
+        default: "0.9.7"  # Keep this comment to allow automatic replacement of uv version
         type: string
       platform:
         description: 'Platform for the build - linux/amd64 or linux/arm64'

.github/workflows/release_dockerhub_image.yml

@@ -58,7 +58,7 @@ jobs:
       AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
       AMD_ONLY: ${{ github.event.inputs.amdOnly }}
       LIMIT_PYTHON_VERSIONS: ${{ github.event.inputs.limitPythonVersions }}
-      UV_VERSION: "0.9.4"  # Keep this comment to allow automatic replacement of uv version
+      UV_VERSION: "0.9.7"  # Keep this comment to allow automatic replacement of uv version
     if: contains(fromJSON('[
       "ashb",
       "eladkal",

.pre-commit-config.yaml

@@ -46,17 +46,17 @@ ARG AIRFLOW_UID="50000"
 ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 
 # latest released version here
-ARG AIRFLOW_VERSION="3.0.6"
+ARG AIRFLOW_VERSION="3.1.1"
 
 ARG BASE_IMAGE="debian:bookworm-slim"
 ARG AIRFLOW_PYTHON_VERSION="3.12.12"
 
 # You can swap comments between those two args to test pip from the main version
 # When you attempt to test if the version of `pip` from specified branch works for our builds
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
-ARG AIRFLOW_PIP_VERSION=25.2
+ARG AIRFLOW_PIP_VERSION=25.3
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.9.4
+ARG AIRFLOW_UV_VERSION=0.9.7
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -443,12 +443,41 @@ set -euo pipefail
 common::get_colors
 declare -a packages
 
-readonly MYSQL_LTS_VERSION="8.0"
 readonly MARIADB_LTS_VERSION="10.11"
 
 : "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
 : "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
 
+if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then
+    echo
+    echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}"
+    echo
+    exit 1
+fi
+
+if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then
+    echo
+    echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}"
+    echo
+    exit 1
+fi
+
+if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
+    echo
+    echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}"
+    echo
+    echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys,"
+    echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client."
+    echo ""
+    echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling"
+    echo "with fixes and re-signing the packages which lasts few days"
+    echo "See https://bugs.mysql.com/bug.php?id=113432 for more details."
+    echo "As a community we are not able to support this broken packaging practice from Oracle"
+    echo "Feel free however to install MySQL drivers on your own as extension of the image."
+    echo
+    exit 1
+fi
+
 retry() {
     local retries=3
     local count=0
@@ -467,44 +496,6 @@ retry() {
     done
 }
 
-install_mysql_client() {
-    if [[ "${1}" == "dev" ]]; then
-        packages=("libmysqlclient-dev" "mysql-client")
-    elif [[ "${1}" == "prod" ]]; then
-        # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example
-        # 18 - MySQL 5.6.48
-        # 20 - MySQL 5.7.42
-        # 21 - MySQL 8.0.34
-        # 22 - MySQL 8.1
-        packages=("libmysqlclient21" "mysql-client")
-    else
-        echo
-        echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
-        echo
-        exit 1
-    fi
-
-    common::import_trusted_gpg "B7B3B788A8D3785C" "mysql"
-
-    echo
-    echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
-    echo
-
-    echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \
-        /etc/apt/sources.list.d/mysql.list
-    retry apt-get update
-    retry apt-get install --no-install-recommends -y "${packages[@]}"
-    apt-get autoremove -yqq --purge
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
-    # Remove mysql repository from sources.list.d as MySQL repos have a basic flaw that they put expiry
-    # date on their GPG signing keys and they sign their repo with those keys. This means that after a
-    # certain date, the GPG key becomes invalid and if you have the repository added in your sources.list
-    # then you will not be able to install anything from any other repository. This id unlike any other
-    # repository we have seen (for example Postgres, MariaDB, MsSQL - all have non-expiring signing keys)
-    rm /etc/apt/sources.list.d/mysql.list
-}
-
 install_mariadb_client() {
     # List of compatible package Oracle MySQL -> MariaDB:
     # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
@@ -544,23 +535,7 @@ install_mariadb_client() {
 }
 
 if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
-    if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
-        INSTALL_MYSQL_CLIENT_TYPE="mariadb"
-        echo
-        echo "${COLOR_YELLOW}Client forced to mariadb for ARM${COLOR_RESET}"
-        echo
-    fi
-
-    if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
-        install_mysql_client "${@}"
-    elif [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mariadb" ]]; then
-        install_mariadb_client "${@}"
-    else
-        echo
-        echo "${COLOR_RED}Specify either mysql or mariadb, got ${INSTALL_MYSQL_CLIENT_TYPE}${COLOR_RESET}"
-        echo
-        exit 1
-    fi
+    install_mariadb_client "${@}"
 fi
 EOF
 
@@ -1116,6 +1091,8 @@ function install_from_sources() {
               --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \
               --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \
               --editable ./task-sdk-tests \
+              --editable ./airflow-ctl-tests \
+              --editable ./airflow-e2e-tests \
               --editable ./devel-common[all] --editable ./dev \
               --group dev --group docs --group docs-gen --group leveldb"
         local -a projects_with_devel_dependencies
@@ -1877,7 +1854,7 @@ COPY --from=scripts install_from_docker_context_files.sh install_airflow_when_bu
 # an incorrect architecture.
 ARG TARGETARCH
 # Value to be able to easily change cache id and therefore use a bare new cache
-ARG DEPENDENCY_CACHE_EPOCH="10"
+ARG DEPENDENCY_CACHE_EPOCH="11"
 
 # hadolint ignore=SC2086, SC2010, DL3042
 RUN --mount=type=cache,id=prod-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/tmp/.cache/,uid=${AIRFLOW_UID} \

Dockerfile

@@ -383,12 +383,41 @@ set -euo pipefail
 common::get_colors
 declare -a packages
 
-readonly MYSQL_LTS_VERSION="8.0"
 readonly MARIADB_LTS_VERSION="10.11"
 
 : "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
 : "${INSTALL_MYSQL_CLIENT_TYPE:-mariadb}"
 
+if [[ "${INSTALL_MYSQL_CLIENT}" != "true" && "${INSTALL_MYSQL_CLIENT}" != "false" ]]; then
+    echo
+    echo "${COLOR_RED}INSTALL_MYSQL_CLIENT must be either true or false${COLOR_RESET}"
+    echo
+    exit 1
+fi
+
+if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" != "mysql" && "${INSTALL_MYSQL_CLIENT_TYPE}" != "mariadb" ]]; then
+    echo
+    echo "${COLOR_RED}INSTALL_MYSQL_CLIENT_TYPE must be either mysql or mariadb${COLOR_RESET}"
+    echo
+    exit 1
+fi
+
+if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
+    echo
+    echo "${COLOR_RED}The 'mysql' client type is not supported any more. Use 'mariadb' instead.${COLOR_RESET}"
+    echo
+    echo "The MySQL drivers are wrongly packaged and released by Oracle with an expiration date on their GPG keys,"
+    echo "which causes builds to fail after the expiration date. MariaDB client is protocol-compatible with MySQL client."
+    echo ""
+    echo "Every two years the MySQL packages fail and Oracle team is always surprised and struggling"
+    echo "with fixes and re-signing the packages which lasts few days"
+    echo "See https://bugs.mysql.com/bug.php?id=113432 for more details."
+    echo "As a community we are not able to support this broken packaging practice from Oracle"
+    echo "Feel free however to install MySQL drivers on your own as extension of the image."
+    echo
+    exit 1
+fi
+
 retry() {
     local retries=3
     local count=0
@@ -407,44 +436,6 @@ retry() {
     done
 }
 
-install_mysql_client() {
-    if [[ "${1}" == "dev" ]]; then
-        packages=("libmysqlclient-dev" "mysql-client")
-    elif [[ "${1}" == "prod" ]]; then
-        # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example
-        # 18 - MySQL 5.6.48
-        # 20 - MySQL 5.7.42
-        # 21 - MySQL 8.0.34
-        # 22 - MySQL 8.1
-        packages=("libmysqlclient21" "mysql-client")
-    else
-        echo
-        echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
-        echo
-        exit 1
-    fi
-
-    common::import_trusted_gpg "B7B3B788A8D3785C" "mysql"
-
-    echo
-    echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
-    echo
-
-    echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \
-        /etc/apt/sources.list.d/mysql.list
-    retry apt-get update
-    retry apt-get install --no-install-recommends -y "${packages[@]}"
-    apt-get autoremove -yqq --purge
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
-    # Remove mysql repository from sources.list.d as MySQL repos have a basic flaw that they put expiry
-    # date on their GPG signing keys and they sign their repo with those keys. This means that after a
-    # certain date, the GPG key becomes invalid and if you have the repository added in your sources.list
-    # then you will not be able to install anything from any other repository. This id unlike any other
-    # repository we have seen (for example Postgres, MariaDB, MsSQL - all have non-expiring signing keys)
-    rm /etc/apt/sources.list.d/mysql.list
-}
-
 install_mariadb_client() {
     # List of compatible package Oracle MySQL -> MariaDB:
     # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
@@ -484,23 +475,7 @@ install_mariadb_client() {
 }
 
 if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
-    if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
-        INSTALL_MYSQL_CLIENT_TYPE="mariadb"
-        echo
-        echo "${COLOR_YELLOW}Client forced to mariadb for ARM${COLOR_RESET}"
-        echo
-    fi
-
-    if [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mysql" ]]; then
-        install_mysql_client "${@}"
-    elif [[ "${INSTALL_MYSQL_CLIENT_TYPE}" == "mariadb" ]]; then
-        install_mariadb_client "${@}"
-    else
-        echo
-        echo "${COLOR_RED}Specify either mysql or mariadb, got ${INSTALL_MYSQL_CLIENT_TYPE}${COLOR_RESET}"
-        echo
-        exit 1
-    fi
+    install_mariadb_client "${@}"
 fi
 EOF
 
@@ -870,6 +845,8 @@ function install_from_sources() {
               --editable ./airflow-core --editable ./task-sdk --editable ./airflow-ctl \
               --editable ./kubernetes-tests --editable ./docker-tests --editable ./helm-tests \
               --editable ./task-sdk-tests \
+              --editable ./airflow-ctl-tests \
+              --editable ./airflow-e2e-tests \
               --editable ./devel-common[all] --editable ./dev \
               --group dev --group docs --group docs-gen --group leveldb"
         local -a projects_with_devel_dependencies
@@ -1207,6 +1184,27 @@ function environment_initialization() {
         ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
     fi
 
+    if [[ ${INTEGRATION_LOCALSTACK:-"false"} == "true" ]]; then
+        echo
+        echo "${COLOR_BLUE}Configuring LocalStack integration${COLOR_RESET}"
+        echo
+
+        # Define LocalStack AWS configuration
+        declare -A localstack_config=(
+            ["AWS_ENDPOINT_URL"]="http://localstack:4566"
+            ["AWS_ACCESS_KEY_ID"]="test"
+            ["AWS_SECRET_ACCESS_KEY"]="test"
+            ["AWS_DEFAULT_REGION"]="us-east-1"
+        )
+
+        # Export each configuration variable and log it
+        for key in "${!localstack_config[@]}"; do
+            export "$key"="${localstack_config[$key]}"
+            echo "  * ${COLOR_BLUE}${key}:${COLOR_RESET} ${localstack_config[$key]}"
+        done
+        echo
+    fi
+
     cd "${AIRFLOW_SOURCES}"
 
     # Temporarily add /opt/airflow/providers/standard/tests to PYTHONPATH in order to see example dags
@@ -1653,10 +1651,10 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe
 # You can swap comments between those two args to test pip from the main version
 # When you attempt to test if the version of `pip` from specified branch works for our builds
 # Also use `force pip` label on your PR to swap all places we use `uv` to `pip`
-ARG AIRFLOW_PIP_VERSION=25.2
+ARG AIRFLOW_PIP_VERSION=25.3
 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main"
-ARG AIRFLOW_UV_VERSION=0.9.4
-ARG AIRFLOW_PREK_VERSION="0.2.10"
+ARG AIRFLOW_UV_VERSION=0.9.7
+ARG AIRFLOW_PREK_VERSION="0.2.12"
 
 # UV_LINK_MODE=copy is needed since we are using cache mounted from the host
 ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
@@ -1670,7 +1668,7 @@ ENV PATH="/usr/python/bin:/root/.local/bin:/root/.cargo/bin:${PATH}"
 # an incorrect architecture.
 ARG TARGETARCH
 # Value to be able to easily change cache id and therefore use a bare new cache
-ARG DEPENDENCY_CACHE_EPOCH="1"
+ARG DEPENDENCY_CACHE_EPOCH="2"
 
 # Install useful command line tools in their own virtualenv so that they do not clash with
 # dependencies installed in Airflow also reinstall PIP and UV to make sure they are installed

Dockerfile.ci

@@ -310,7 +310,7 @@ Installing and Using ruff
 
    .. code-block:: bash
 
-      pip install "ruff>=0.14.1"
+      pip install "ruff>=0.14.3"
 
 2. **Running ruff**: Execute ``ruff`` to check your Dags for potential issues:
 
@@ -725,7 +725,7 @@ Unit tests ensure that there is no incorrect code in your Dag. You can write uni
 
     import pytest
 
-    from airflow.models import DagBag
+    from airflow.dag_processing.dagbag import DagBag
 
 
     @pytest.fixture()

airflow-core/docs/best-practices.rst

@@ -442,9 +442,9 @@ can be used for CI images:
 | `DEV_APT_DEPS`                    |                             | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) |
 | `ADDITIONAL_DEV_APT_DEPS`         |                             | Additional apt dev dependencies installed in the first part of the image                                          |
 | `ADDITIONAL_DEV_APT_ENV`          |                             | Additional env variables defined when installing dev deps                                                         |
-| `AIRFLOW_PIP_VERSION`             | `25.2`                      | `pip` version used.                                                                                               |
-| `AIRFLOW_UV_VERSION`              | `0.9.4`                     | `uv` version used.                                                                                                |
-| `AIRFLOW_PREK_VERSION`            | `0.2.10`                    | `prek` version used.                                                                                              |
+| `AIRFLOW_PIP_VERSION`             | `25.3`                      | `pip` version used.                                                                                               |
+| `AIRFLOW_UV_VERSION`              | `0.9.7`                     | `uv` version used.                                                                                                |
+| `AIRFLOW_PREK_VERSION`            | `0.2.12`                    | `prek` version used.                                                                                              |
 | `AIRFLOW_USE_UV`                  | `true`                      | Whether to use UV for installation.                                                                               |
 | `PIP_PROGRESS_BAR`                | `on`                        | Progress bar for PIP installation                                                                                 |