From 17d30f0368a2f499ce8ed9a89b35247f0a197007 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 7 Jul 2025 10:23:08 +0800 Subject: [PATCH 1/2] HADOOP-19605. Upgrade Protobuf 3.25.5 for docker images --- BUILDING.txt | 53 ++++++++++--------- dev-support/docker/Dockerfile | 2 +- dev-support/docker/Dockerfile_aarch64 | 2 +- dev-support/docker/Dockerfile_centos_7 | 2 +- dev-support/docker/Dockerfile_centos_8 | 2 +- dev-support/docker/Dockerfile_debian_10 | 2 +- dev-support/docker/Dockerfile_ubuntu_24 | 2 +- .../docker/pkg-resolver/install-protobuf.sh | 20 ++++--- dev-support/docker/vcpkg/vcpkg.json | 2 +- 9 files changed, 48 insertions(+), 39 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index 4dfb48fb089eb..f5749b9ec182e 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -7,7 +7,7 @@ Requirements: * JDK 1.8 * Maven 3.3 or later * Boost 1.86.0 (if compiling native code) -* Protocol Buffers 3.21.12 (if compiling native code) +* Protocol Buffers 3.25.5 (if compiling native code) * CMake 3.19 or newer (if compiling native code) * Zlib devel (if compiling native code) * Cyrus SASL devel (if compiling native code) @@ -78,13 +78,15 @@ Refer to dev-support/docker/Dockerfile): $ ./bootstrap $ make -j$(nproc) $ sudo make install -* Protocol Buffers 3.21.12 (required to build native code) - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure - $ make -j$(nproc) - $ sudo make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ sudo cmake --install build * Boost $ curl -L https://sourceforge.net/projects/boost/files/boost/1.86.0/boost_1_86_0.tar.bz2/download > boost_1_86_0.tar.bz2 $ tar --bzip2 -xf boost_1_86_0.tar.bz2 && cd boost_1_86_0 @@ -434,14 +436,15 @@ Installing required dependencies for clean install of macOS 10.14: * Install native libraries, only openssl is required to compile native code, you may optionally install zlib, lz4, etc. $ brew install openssl -* Protocol Buffers 3.21.12 (required to compile native code) - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure - $ make - $ make check - $ make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ cmake --install build $ protoc --version Note that building Hadoop 3.1.1/3.1.2/3.2.0 native code from source is broken @@ -473,13 +476,15 @@ Building on CentOS 8 * Install python2 for building documentation. $ sudo dnf install python2 -* Install Protocol Buffers v3.21.12. - $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz > protobuf-3.21.12.tar.gz - $ tar -zxvf protobuf-3.21.12.tar.gz && cd protobuf-3.21.12 - $ ./autogen.sh - $ ./configure --prefix=/usr/local - $ make - $ sudo make install +* Protocol Buffers 3.25.5 (required to build native code) + $ curl -L https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz > protobuf-3.25.5.tar.gz + $ curl -L https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz > abseil-cpp-20230802.1.tar.gz + $ tar -zxvf protobuf-3.25.5.tar.gz + $ tar -zxvf abseil-cpp-20230802.1.tar.gz --strip-components 1 -C protobuf-3.25.5/third_party/abseil-cpp + $ cd protobuf-3.25.5 + $ cmake -S . -B build -Dprotobuf_BUILD_TESTS=OFF + $ cmake --build build --parallel $(nproc) + $ sudo cmake --install build --prefix /usr/local $ cd .. * Install libraries provided by CentOS 8. @@ -531,7 +536,7 @@ Requirements: * JDK 1.8 * Maven 3.0 or later (maven.apache.org) * Boost 1.86.0 (boost.org) -* Protocol Buffers 3.21.12 (https://github.com/protocolbuffers/protobuf/tags) +* Protocol Buffers 3.25.5 (https://github.com/protocolbuffers/protobuf/tags) * CMake 3.19 or newer (cmake.org) * Visual Studio 2019 (visualstudio.com) * Windows SDK 8.1 (optional, if building CPU rate control for the container executor. Get this from diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 6beef860a9b39..91634c815a5eb 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -71,7 +71,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index b9249f5aec9d3..ead6821a41087 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -72,7 +72,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_centos_7 b/dev-support/docker/Dockerfile_centos_7 index b97e59969a760..82445b8353a44 100644 --- a/dev-support/docker/Dockerfile_centos_7 +++ b/dev-support/docker/Dockerfile_centos_7 @@ -88,7 +88,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_centos_8 b/dev-support/docker/Dockerfile_centos_8 index ee0c8e88f74e4..f4e9b380886b6 100644 --- a/dev-support/docker/Dockerfile_centos_8 +++ b/dev-support/docker/Dockerfile_centos_8 @@ -101,7 +101,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-1.8.0 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_debian_10 b/dev-support/docker/Dockerfile_debian_10 index 71446b27f686b..1f6b296ee49f8 100644 --- a/dev-support/docker/Dockerfile_debian_10 +++ b/dev-support/docker/Dockerfile_debian_10 @@ -66,7 +66,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/Dockerfile_ubuntu_24 b/dev-support/docker/Dockerfile_ubuntu_24 index 28ab04c3e09fe..b605a626a260e 100644 --- a/dev-support/docker/Dockerfile_ubuntu_24 +++ b/dev-support/docker/Dockerfile_ubuntu_24 @@ -71,7 +71,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-17-openjdk-amd64 ENV SPOTBUGS_HOME /opt/spotbugs ####### -# Set env vars for Google Protobuf 3.21.12 +# Set env vars for Google Protobuf 3.25.5 ####### ENV PROTOBUF_HOME /opt/protobuf ENV PATH "${PATH}:/opt/protobuf/bin" diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh index f8319f6acea60..a40104ddd3d92 100644 --- a/dev-support/docker/pkg-resolver/install-protobuf.sh +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -27,29 +27,33 @@ if [ $? -eq 1 ]; then exit 1 fi -default_version="3.21.12" +default_version="3.25.5" version_to_install=$default_version if [ -n "$2" ]; then version_to_install="$2" fi -if [ "$version_to_install" != "3.21.12" ]; then +if [ "$version_to_install" != "3.25.5" ]; then echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead" version_to_install=$default_version fi -if [ "$version_to_install" == "3.21.12" ]; then +if [ "$version_to_install" == "3.25.5" ]; then # hadolint ignore=DL3003 mkdir -p /opt/protobuf-src && curl -L -s -S \ - https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.12.tar.gz \ + https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.25.5.tar.gz \ -o /opt/protobuf.tar.gz && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src && + mkdir -p /opt/abseil-cpp-src && + curl -L -s -S \ + https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz \ + -o /opt/abseil-cpp.tar.gz && + tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C protobuf-src/third_party/abseil-cpp && cd /opt/protobuf-src && - ./autogen.sh && - ./configure --prefix=/opt/protobuf && - make "-j$(nproc)" && - make install && + cmake -S . -B build -Dprotobuf_BUILD_TESTS=OFF && + cmake --build build --parallel $(nproc) && + cmake --install build --prefix /opt/protobuf && cd /root && rm -rf /opt/protobuf-src else diff --git a/dev-support/docker/vcpkg/vcpkg.json b/dev-support/docker/vcpkg/vcpkg.json index 91c4776ce0df6..335d1405a296b 100644 --- a/dev-support/docker/vcpkg/vcpkg.json +++ b/dev-support/docker/vcpkg/vcpkg.json @@ -10,7 +10,7 @@ "overrides": [ { "name": "protobuf", - "version": "3.21.12" + "version": "3.25.5" } ] } From 1be14176bf2aad02d947ae744c5ea155baf9a475 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 7 Jul 2025 10:44:09 +0800 Subject: [PATCH 2/2] fix --- dev-support/docker/pkg-resolver/install-protobuf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-support/docker/pkg-resolver/install-protobuf.sh b/dev-support/docker/pkg-resolver/install-protobuf.sh index a40104ddd3d92..c52d26c8cc6d0 100644 --- a/dev-support/docker/pkg-resolver/install-protobuf.sh +++ b/dev-support/docker/pkg-resolver/install-protobuf.sh @@ -49,7 +49,7 @@ if [ "$version_to_install" == "3.25.5" ]; then curl -L -s -S \ https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.tar.gz \ -o /opt/abseil-cpp.tar.gz && - tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C protobuf-src/third_party/abseil-cpp && + tar xzf /opt/abseil-cpp.tar.gz --strip-components 1 -C /opt/protobuf-src/third_party/abseil-cpp && cd /opt/protobuf-src && cmake -S . -B build -Dprotobuf_BUILD_TESTS=OFF && cmake --build build --parallel $(nproc) &&