Merge branch 'stable-23.10' into MCOL-4628-SET-DOUBLE_COL-TO-DECIMAL_COL-RETURNS-ERROR

Author: AestheticAkhmad

.drone.jsonnet

@@ -15,10 +15,16 @@ local platforms_arm = {
   "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"],
 };
 
+local rewrite_ubuntu_mirror = @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list || true; " +
+                              @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list.d/ubuntu.sources || true; " +
+                              "cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list | grep archive || true; ";
+
 local customEnvCommandsMap = {
   // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'],
   "clang-20": [
-    "apt install -y wget curl lsb-release software-properties-common gnupg",
+    rewrite_ubuntu_mirror,
+    "apt-get clean && apt-get update",
+    "apt-get install -y wget curl lsb-release software-properties-common gnupg",
     "wget https://apt.llvm.org/llvm.sh",
     "bash llvm.sh 20",
     "export CC=/usr/bin/clang",
@@ -114,7 +120,7 @@ local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " +
 
 local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel";
 
-local deb_deps = "apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev";
+local deb_deps = rewrite_ubuntu_mirror + "apt-get clean && apt-get update && apt-get install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev";
 
 local testPreparation(platform) =
   local platform_map = {
@@ -127,6 +133,10 @@ local testPreparation(platform) =
   };
   platform_map[platform];
 
+local make_clickable_link(link) = "echo -e '\\e]8;;" +  link + "\\e\\\\" +  link + "\\e]8;;\\e\\\\'";
+local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}",
+      make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")];
+
 local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = {
   local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb",
   local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd",
@@ -154,10 +164,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
   local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server",
 
   local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64",
-  local get_sccache = "echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) " +
-                      "&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz " +
-                      "&& tar xzf sccache.tar.gz " +
-                      "&& install sccache*/sccache /usr/local/bin/ && echo sccache installed",
+  local get_sccache = ["echo getting sccache...",
+                      rewrite_ubuntu_mirror,
+                      "(apt-get clean && apt-get update -y && apt-get install -y curl || yum install -y curl || true)",
+                      "curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz &&",
+                      "tar xzf sccache.tar.gz",
+                      "install sccache*/sccache /usr/local/bin/ && echo sccache installed"],
 
   local pipeline = self,
 
@@ -182,7 +194,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
 
       "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
       'echo "Data uploaded to: ' + publish_pkg_url + '"',
-
+      make_clickable_link(publish_pkg_url),
       "rm -rf " + result + "/*",
     ],
   },
@@ -231,7 +243,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
   local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
 
   local installCmapi(containerName, pkg_format) =
-    if (pkg_format == "deb") then execInnerDocker('bash -c "apt update -y && apt install -y mariadb-columnstore-cmapi"', containerName)
+    if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName)
     else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName),
 
   local prepareTestStage(containerName, result, do_setup) =
@@ -606,8 +618,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}",
                SERVER_SHA: "${SERVER_SHA:-" + server + "}",
              },
-             commands: [
-               "echo $$SERVER_REF",
+             commands: echo_running_on +
+             [
+              "echo $$SERVER_REF",
                "echo $$SERVER_REMOTE",
                "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir,
                'git config --global url."https://github.com/".insteadOf [email protected]:',
@@ -626,9 +639,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              volumes: [pipeline._volumes.mdb],
              environment: {
                DEBIAN_FRONTEND: "noninteractive",
-               DEB_BUILD_OPTIONS: "parallel=4",
-               DH_BUILD_DDEBS: "1",
-               BUILDPACKAGE_FLAGS: "-b",  // Save time and produce only binary packages, not source
                AWS_ACCESS_KEY_ID: {
                  from_secret: "aws_access_key_id",
                },
@@ -644,14 +654,14 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              },
              commands: [
                          "mkdir /mdb/" + builddir + "/" + result,
-                         get_sccache,
                        ]
+                       + get_sccache
                        + customEnvCommands(customBuildEnvCommandsMapKey, builddir) +
                        [
-                         'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
+                        'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
                          "--build-type RelWithDebInfo " +
                          "--distro " + platform + " " +
-                         "--build-packages --sccache " +
+                         "--build-packages --install-deps --sccache " +
                          " " + customBootstrapParams +
                          " " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
                          "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +
@@ -728,19 +738,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
          ] +
          [pipeline.publish("cmapi build")] +
          [pipeline.publish()] +
-         [
-           {
-             name: "publish pkg url",
-             depends_on: ["publish pkg"],
-             image: "alpine/git",
-             commands: [
-               "echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'",
-               "echo 'for installation run:'",
-               "echo 'export OS=" + result + "'",
-               "echo 'export PACKAGES_URL=" + packages_url + "'",
-             ],
-           },
-         ] +
          (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) +
          [pipeline.smoke] +
          [pipeline.smokelog] +

.gitignore

@@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
 oam/install_scripts/mcs-writeengineserver.service
 oam/install_scripts/mcs-loadbrm.py
 oam/install_scripts/mcs-savebrm.py
+oam/install_scripts/mcssavebrm.py
 oam/install_scripts/mcs-storagemanager.service
 oam/install_scripts/mcs_module_installer.sh
 oam/install_scripts/slave-rep-columnstore.sh
@@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
 obj/
 build/build
 
-

build/bootstrap_mcs.sh

@@ -19,6 +19,9 @@ CONFIG_DIR=$RPM_CONFIG_DIR
 
 SCRIPT_LOCATION=$(dirname "$0")
 MDB_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../../../..)
+COLUMSNTORE_SOURCE_PATH=$(realpath "$SCRIPT_LOCATION"/../)
+
+DEFAULT_MARIA_BUILD_PATH=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
 
 BUILD_TYPE_OPTIONS=("Debug" "RelWithDebInfo")
 DISTRO_OPTIONS=("ubuntu:20.04" "ubuntu:22.04" "ubuntu:24.04" "debian:11" "debian:12" "rockylinux:8" "rockylinux:9")
@@ -28,21 +31,10 @@ MDB_CMAKE_FLAGS=()
 
 source "$SCRIPT_LOCATION"/utils.sh
 
-if [ "$EUID" -ne 0 ]; then
-    error "Please run script as root to install MariaDb to system paths"
-    exit 1
-fi
-
-cd $SCRIPT_LOCATION
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-message "Columnstore will be built from $color_yellow$CURRENT_BRANCH$color_cyan branch"
-cd - >/dev/null
-
 echo "Arguments received: $@"
-message "Building Mariadb Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
 
 optparse.define short=A long=asan desc="Build with ASAN" variable=ASAN default=false value=true
-optparse.define short=a long=build-path variable=MARIA_BUILD_PATH default=$(realpath "$MDB_SOURCE_PATH"/../BuildOf_$(basename "$MDB_SOURCE_PATH"))
+optparse.define short=a long=build-path desc="Path for build output" variable=MARIA_BUILD_PATH default=$DEFAULT_MARIA_BUILD_PATH
 optparse.define short=B long=run-microbench desc="Compile and run microbenchmarks " variable=RUN_BENCHMARKS default=false value=true
 optparse.define short=c long=cloud desc="Enable cloud storage" variable=CLOUD_STORAGE_ENABLED default=false value=true
 optparse.define short=C long=force-cmake-reconfig desc="Force cmake reconfigure" variable=FORCE_CMAKE_CONFIG default=false value=true
@@ -68,10 +60,13 @@ optparse.define short=T long=tsan desc="Build with TSAN" variable=TSAN default=f
 optparse.define short=u long=skip-unit-tests desc="Skip UnitTests" variable=SKIP_UNIT_TESTS default=false value=true
 optparse.define short=U long=ubsan desc="Build with UBSAN" variable=UBSAN default=false value=true
 optparse.define short=v long=verbose desc="Verbose makefile commands" variable=MAKEFILE_VERBOSE default=false value=true
+optparse.define short=V long=add-branch-name-to-outdir desc="Add branch name to build output directory" variable=BRANCH_NAME_TO_OUTDIR default=false value=true
 optparse.define short=W long=without-core-dumps desc="Do not produce core dumps" variable=WITHOUT_COREDUMPS default=false value=true
 
 source $(optparse.build)
 
+message "Building MariaDB Server from $color_yellow$MDB_SOURCE_PATH$color_normal"
+
 if [[ ! " ${BUILD_TYPE_OPTIONS[*]} " =~ " ${MCS_BUILD_TYPE} " ]]; then
     getChoice -q "Select your Build Type" -o BUILD_TYPE_OPTIONS
     MCS_BUILD_TYPE=$selectedChoice
@@ -87,24 +82,19 @@ if [[ "$OS" == *"rocky"* ]]; then
     pkg_format="rpm"
 fi
 
-disable_git_restore_frozen_revision() {
-    cd $MDB_SOURCE_PATH
-    git config submodule.storage/columnstore/columnstore.update none
-    cd - >/dev/null
-}
-
-DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
-
 install_deps() {
+    if [[ $INSTALL_DEPS = false ]]; then
+        return
+    fi
     message_split
-
+    prereq=""
     RPM_BUILD_DEPS="dnf install -y lz4 lz4-devel systemd-devel git make libaio-devel openssl-devel boost-devel bison \
       snappy-devel flex libcurl-devel libxml2-devel ncurses-devel automake libtool policycoreutils-devel \
       rpm-build lsof iproute pam-devel perl-DBI cracklib-devel expect createrepo python3 checkpolicy \
       cppunit-devel cmake3 libxcrypt-devel xz-devel zlib-devel libzstd-devel glibc-devel"
 
     DEB_BUILD_DEPS="apt-get -y update && apt-get -y install build-essential automake libboost-all-dev \
-      bison cmake libncurses5-dev libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
+      bison cmake libncurses5-dev python3 libaio-dev libsystemd-dev libpcre2-dev libperl-dev libssl-dev libxml2-dev \
       libkrb5-dev flex libpam-dev git libsnappy-dev libcurl4-openssl-dev libgtest-dev libcppunit-dev googletest \
       libjemalloc-dev liblz-dev liblzo2-dev liblzma-dev liblz4-dev libbz2-dev libbenchmark-dev libdistro-info-perl \
       graphviz devscripts ccache equivs eatmydata curl"
@@ -113,22 +103,54 @@ install_deps() {
         command="dnf install -y curl 'dnf-command(config-manager)' && dnf config-manager --set-enabled powertools && \
       dnf install -y gcc-toolset-${GCC_VERSION} libarchive cmake && . /opt/rh/gcc-toolset-${GCC_VERSION}/enable && \
       ${RPM_BUILD_DEPS}"
-
-    elif [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]; then
+    elif
+        [[ "$OS" == "rockylinux:9"* || "$OS" == "rocky:9"* ]]
+    then
         command="dnf install -y 'dnf-command(config-manager)' && dnf config-manager --set-enabled crb && \
       dnf install -y pcre2-devel gcc gcc-c++ curl-minimal && ${RPM_BUILD_DEPS}"
 
     elif [[ "$OS" == "debian:11"* ]] || [[ "$OS" == "debian:12"* ]] || [[ "$OS" == "ubuntu:20.04"* ]] || [[ "$OS" == "ubuntu:22.04"* ]] || [[ "$OS" == "ubuntu:24.04"* ]]; then
         command="${DEB_BUILD_DEPS}"
-
     else
         echo "Unsupported OS: $OS"
         exit 17
     fi
 
+    if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
+        if [ -f /.dockerenv ]; then
+            change_ubuntu_mirror us
+        fi
+        command="${command} lto-disabled-list"
+    fi
+
+    eval "$prereq"
     message "Installing dependencies for $OS"
-    eval "$command"
+    retry_eval 5 "$command"
 }
+
+install_deps
+
+cd $COLUMSNTORE_SOURCE_PATH
+COLUMNSTORE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "Columnstore will be built from $color_yellow$COLUMNSTORE_BRANCH$color_cyan branch"
+
+cd $MDB_SOURCE_PATH
+MARIADB_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+message "MariaDB will be built from $color_yellow$MARIADB_BRANCH$color_cyan branch"
+cd - >/dev/null
+
+if [[ ${BRANCH_NAME_TO_OUTDIR} = true ]]; then
+    MARIA_BUILD_PATH="${MARIA_BUILD_PATH}_${MARIADB_BRANCH}_${COLUMNSTORE_BRANCH}"
+fi
+
+disable_git_restore_frozen_revision() {
+    cd $MDB_SOURCE_PATH
+    git config submodule.storage/columnstore/columnstore.update none
+    cd - >/dev/null
+}
+
+DEP_GRAPH_PATH="$MARIA_BUILD_PATH/dependency_graph/mariadb.dot"
+
 stop_service() {
     message_split
     message "Stopping MariaDB services"
@@ -198,10 +220,9 @@ modify_packaging() {
 
     #disable LTO for 22.04 for now
     if [[ $OS == 'ubuntu:22.04' || $OS == 'ubuntu:24.04' ]]; then
-        apt install -y lto-disabled-list &&
-            for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
-                echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
-            done &&
+        for i in mariadb-plugin-columnstore mariadb-server mariadb-server-core mariadb mariadb-10.6; do
+            echo "$i any" >>/usr/share/lto-disabled-list/lto-disabled-list
+        done &&
             grep mariadb /usr/share/lto-disabled-list/lto-disabled-list
     fi
 
@@ -259,8 +280,9 @@ construct_cmake_flags() {
         -DWITH_WSREP=NO
     )
 
-    if [[ MAINTAINER_MODE = true ]]; then
+    if [[ $MAINTAINER_MODE = true ]]; then
         MDB_CMAKE_FLAGS+=(-DCOLUMNSTORE_MAINTAINER=YES)
+        message "Columnstore maintainer mode on"
     else
         warn "Maintainer mode is disabled, be careful, alien"
     fi
@@ -368,6 +390,11 @@ construct_cmake_flags() {
         MDB_CMAKE_FLAGS+=(-DDEB=${CODENAME})
     fi
 
+    if [[ "$NO_CLEAN" == true ]]; then
+        message "Skipping Columnstore.etc config installation"
+        MDB_CMAKE_FLAGS+=(-DSKIP_CONFIG_INSTALLATION=ON)
+    fi
+
     MDB_CMAKE_FLAGS+=($CUSTOM_CMAKE_FLAGS)
 
     message "Building with flags"
@@ -406,9 +433,15 @@ generate_svgs() {
 }
 
 build_package() {
+    cd $MDB_SOURCE_PATH
+
     if [[ $pkg_format == "rpm" ]]; then
         command="cmake ${MDB_CMAKE_FLAGS[@]} && make -j\$(nproc) package"
     else
+        export DEBIAN_FRONTEND="noninteractive"
+        export DEB_BUILD_OPTIONS="parallel=$(nproc)"
+        export DH_BUILD_DDEBS="1"
+        export BUILDPACKAGE_FLAGS="-b"
         command="mk-build-deps debian/control -t 'apt-get -y -o Debug::pkgProblemResolver=yes --no-install-recommends' -r -i && \
        CMAKEFLAGS=\"${MDB_CMAKE_FLAGS[@]}\" debian/autobake-deb.sh"
     fi
@@ -420,6 +453,14 @@ build_package() {
     check_errorcode
 }
 
+check_debian_install_file() {
+    message "checking debian/mariadb-plugin-columnstore.install"
+    message_split
+    python3 $COLUMSNTORE_SOURCE_PATH/build/debian_install_file_compare.py \
+        ${COLUMSNTORE_SOURCE_PATH}/debian/mariadb-plugin-columnstore.install \
+        $MARIA_BUILD_PATH/mariadb-plugin-columnstore.install.generated
+}
+
 build_binary() {
     MARIA_BUILD_PATH=$(realpath "$MARIA_BUILD_PATH")
     message_split
@@ -438,7 +479,7 @@ build_binary() {
     message "Configuring cmake silently"
     ${CMAKE_BIN_NAME} "${MDB_CMAKE_FLAGS[@]}" -S"$MDB_SOURCE_PATH" -B"$MARIA_BUILD_PATH" | spinner
     message_split
-
+    check_debian_install_file
     generate_svgs
 
     ${CMAKE_BIN_NAME} --build "$MARIA_BUILD_PATH" -j "$CPUS" | onelinearizator &&
@@ -553,6 +594,12 @@ make_dir() {
 
 install() {
     if [[ $RECOMPILE_ONLY = false ]]; then
+
+        if [ "$EUID" -ne 0 ]; then
+            error "Please run script as root to install MariaDb to system paths"
+            exit 1
+        fi
+
         message_split
         message "Installing MariaDB"
         disable_plugins_for_bootstrap
@@ -577,8 +624,10 @@ install() {
 
         make_dir /etc/columnstore
 
-        cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
-        cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
+        if [[ "$NO_CLEAN" == false ]]; then
+            cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/etc/Columnstore.xml /etc/columnstore/Columnstore.xml
+            cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/storage-manager/storagemanager.cnf /etc/columnstore/storagemanager.cnf
+        fi
 
         cp "$MDB_SOURCE_PATH"/storage/columnstore/columnstore/oam/install_scripts/*.service /lib/systemd/system/
 
@@ -636,10 +685,6 @@ smoke() {
     fi
 }
 
-if [[ $INSTALL_DEPS = true || $BUILD_PACKAGES = true ]]; then
-    install_deps
-fi
-
 if [[ $DO_NOT_FREEZE_REVISION = false ]]; then
     disable_git_restore_frozen_revision
 fi