From 80eea831794cba17ce66bbbd481f07b2916f7bb1 Mon Sep 17 00:00:00 2001 From: Gabor Pali Date: Tue, 25 Nov 2025 16:15:25 +0100 Subject: [PATCH 1/4] chore(`clouseau`): bump to 3.0.0 Chase the release of the brand new 3.0.0 version of Clouseau aka. ZIOSE (ZIO "Search Engine") and make it the default onwards. While this is a drop-in replacement for 2.x, note that it is based on completely different foundation, libraries and build system, as well as Java 21 and G1 as the default garbage collector -- update the `dev/run` script to make it work with both 2.x and 3.x in response to that. This warrants that Clouseau 2.x will keep working from both binary and source, albeit it has now been deprecated. --- README-DEV.rst | 48 +++++----- build-aux/Jenkinsfile | 4 +- configure | 36 ++++--- configure.ps1 | 38 +++++--- dev/run | 218 ++++++++++++++++++++++++++++++------------ 5 files changed, 233 insertions(+), 111 deletions(-) diff --git a/README-DEV.rst b/README-DEV.rst index d6dcc4a5588..a2eec4b44f3 100644 --- a/README-DEV.rst +++ b/README-DEV.rst @@ -278,31 +278,32 @@ suites, and let Clouseau automatically manage them. Although the ``./configure`` and the ``./dev/run`` scripts try to take care of the details of the Clouseau deployment, it is still the responsibility of the user to provide a suitable Java environment for -running. Clouseau can run with JRE 1.7 and 1.8 only. Also, when -Nouveau is in use, which uses a more recent Java environment, the old -JDK has to be installed separately and the ``CLOUSEAU_JAVA_HOME`` -environment variable has to be set to point its location. +running. Depending on the version, Clouseau can run with either JRE +1.7 and 1.8 only, when 2.x is selected, or JRE 21 for 3.x. Also, when +Nouveau is installed, which might require a different Java +environment, the one specific for Clouseau has to be installed +separately and the ``CLOUSEAU_JAVA_HOME`` environment variable has to +be set to point its location. Fortunately, the ```asdf`` tool ` provides a -convenient way to install old versions of JDK through its ```java`` -plugin `:: +convenient way to install different versions of JDK through its +```java`` plugin `:: asdf plugin add java -Then use ``asdf`` to install it:: +Then use ``asdf`` to install it, for example:: - asdf install java zulu-jre-8.74.0.17 + asdf install java openjdk-21 Finally, use ``asdf`` to set the ``CLOUSEAU_JAVA_HOME`` environment variable:: - export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-jre-8.74.0.17) + export CLOUSEAU_JAVA_HOME=$(asdf where java openjdk-21) -If the use of ``asdf`` is not an option, `the Zulu site -` could be used directly to get the -distribution package for the appropriate JRE version. But this is -just one of the possibilities to access installers for old Java -environments. +If the use of ``asdf`` is not an option, any standard Java installer +would do it, and the Zulu site ` could +be used directly to get the distribution package for the older JRE and +JDK versions, for example. Once both Clouseau and the corresponding Java environment are set, they are not put in use automatically. In order to do so, the @@ -333,15 +334,16 @@ respective test targets, e.g. ``mango-test``:: This can even be done if there was no local Clouseau deployment configured previously. Mind that this will require building Clouseau from source, which causes the nodes start up somewhat slower. It also -requires JDK 1.7 and Apache Maven 3.8 to be present, that is why it is -important the set the ``CLOUSEAU_JAVA_HOME`` and the -``CLOUSEAU_M2_HOME`` environment variables accordingly, for instance:: - - asdf install java zulu-7.56.0.11 - asdf plugin add maven - asdf install maven 3.8.8 - export CLOUSEAU_JAVA_HOME=$(asdf where java zulu-7.56.0.11) - export CLOUSEAU_M2_HOME=$(asdf where maven 3.8.8) +requires JDK 11 (or later) and SBT 1.10 (or later) to be present. +That is why it is important the set the ``CLOUSEAU_JAVA_HOME`` and the +``CLOUSEAU_SBT_HOME`` environment variables accordingly, for +instance:: + + asdf install java openjdk-21 + asdf plugin add sbt + asdf install sbt 1.11.7 + export CLOUSEAU_JAVA_HOME=$(asdf where java openjdk-21) + export CLOUSEAU_SBT_HOME=$(asdf where sbt 1.11.7) Static Code Analysis ~~~~~~~~~~~~~~~~~~~~ diff --git a/build-aux/Jenkinsfile b/build-aux/Jenkinsfile index 27c2cee1e44..8a5a3330521 100644 --- a/build-aux/Jenkinsfile +++ b/build-aux/Jenkinsfile @@ -173,7 +173,7 @@ meta = [ name: 'FreeBSD x86_64', spidermonkey_vsn: '91', with_clouseau: true, - clouseau_java_home: '/usr/local/openjdk8-jre', + clouseau_java_home: '/usr/local/openjdk21', quickjs_test262: false, gnu_make: 'gmake' ], @@ -184,7 +184,7 @@ meta = [ name: 'FreeBSD ARM64 QuickJS', disable_spidermonkey: true, with_clouseau: true, - clouseau_java_home: '/usr/local/openjdk8-jre', + clouseau_java_home: '/usr/local/openjdk21', quickjs_test262: false, gnu_make: 'gmake' ], diff --git a/configure b/configure index 30ad40c24a7..f0433f5153b 100755 --- a/configure +++ b/configure @@ -30,6 +30,7 @@ WITH_FAUXTON="true" WITH_DOCS="true" WITH_NOUVEAU="false" WITH_CLOUSEAU="false" +WITH_ZIOSE="true" ERLANG_MD5="false" SKIP_DEPS="false" WITH_SPIDERMONKEY="true" @@ -43,7 +44,7 @@ JS_ENGINE=${JS_ENGINE:-"spidermonkey"} SM_VSN=${SM_VSN:-"91"} CLOUSEAU_MTH=${CLOUSEAU_MTH:-"dist"} CLOUSEAU_URI=${CLOUSEAU_URI:-"https://github.com/cloudant-labs/clouseau/releases/download/%s/clouseau-%s-dist.zip"} -CLOUSEAU_VSN=${CLOUSEAU_VSN:-"2.25.0"} +CLOUSEAU_VSN=${CLOUSEAU_VSN:-"3.0.0"} CLOUSEAU_DIR="$(pwd)"/clouseau ARCH="$(uname -m)" MULTIARCH_NAME="$(command -v dpkg-architecture > /dev/null && dpkg-architecture -q DEB_HOST_MULTIARCH || true)" @@ -314,6 +315,10 @@ parse_opts() { parse_opts $@ +if [ "$(echo "$CLOUSEAU_VSN" | cut -d '.' -f 1)" -lt "3" ]; then + WITH_ZIOSE="false" +fi + if [ "${WITH_SPIDERMONKEY}" = "true" ] && [ "${ARCH}" = "aarch64" ] && [ "${SM_VSN}" = "60" ]; then echo "ERROR: SpiderMonkey 60 is known broken on ARM 64 (aarch64). Use another version instead." exit 1 @@ -488,14 +493,17 @@ install_local_clouseau() { case "$CLOUSEAU_MTH" in dist) _DIST_URL=$(printf "$CLOUSEAU_URI" "$CLOUSEAU_VSN" "$CLOUSEAU_VSN") - _MAVEN_BASE_URI=https://repo1.maven.org/maven2 - _LOGBACK_DIST_URL="$_MAVEN_BASE_URI"/ch/qos/logback - : "${LOGBACK_VERSION:=1.2.13}" - _LOGBACK_CORE_JAR=logback-core-"$LOGBACK_VERSION".jar - _LOGBACK_CORE_URL="$_LOGBACK_DIST_URL"/logback-core/"$LOGBACK_VERSION"/"$_LOGBACK_CORE_JAR" - _LOGBACK_CLASSIC_JAR=logback-classic-"$LOGBACK_VERSION".jar - _LOGBACK_CLASSIC_URL="$_LOGBACK_DIST_URL"/logback-classic/"$LOGBACK_VERSION"/"$_LOGBACK_CLASSIC_JAR" + if [ "${WITH_ZIOSE}" = "false" ]; then + _MAVEN_BASE_URI=https://repo1.maven.org/maven2 + _LOGBACK_DIST_URL="$_MAVEN_BASE_URI"/ch/qos/logback + + : "${LOGBACK_VERSION:=1.2.13}" + _LOGBACK_CORE_JAR=logback-core-"$LOGBACK_VERSION".jar + _LOGBACK_CORE_URL="$_LOGBACK_DIST_URL"/logback-core/"$LOGBACK_VERSION"/"$_LOGBACK_CORE_JAR" + _LOGBACK_CLASSIC_JAR=logback-classic-"$LOGBACK_VERSION".jar + _LOGBACK_CLASSIC_URL="$_LOGBACK_DIST_URL"/logback-classic/"$LOGBACK_VERSION"/"$_LOGBACK_CLASSIC_JAR" + fi check_local_clouseau_dir "$CLOUSEAU_DIR" mkdir -p "$CLOUSEAU_DIR" @@ -511,10 +519,16 @@ install_local_clouseau() { rm clouseau.zip - echo "Fetching Logback $LOGBACK_VERSION from $_LOGBACK_DIST_URL..." + if [ "${WITH_ZIOSE}" = "false" ]; then + echo "Fetching Logback $LOGBACK_VERSION from $_LOGBACK_DIST_URL..." - fetch_file "$CLOUSEAU_DIR"/"$_LOGBACK_CORE_JAR" "$_LOGBACK_CORE_URL" - fetch_file "$CLOUSEAU_DIR"/"$_LOGBACK_CLASSIC_JAR" "$_LOGBACK_CLASSIC_URL" + fetch_file "$CLOUSEAU_DIR"/"$_LOGBACK_CORE_JAR" "$_LOGBACK_CORE_URL" + fetch_file "$CLOUSEAU_DIR"/"$_LOGBACK_CLASSIC_JAR" "$_LOGBACK_CLASSIC_URL" + fi + + if [ "${WITH_ZIOSE}" = "true" ]; then + touch "$CLOUSEAU_DIR"/.ziose + fi ;; git) diff --git a/configure.ps1 b/configure.ps1 index c7c730c70d3..886136071f6 100644 --- a/configure.ps1 +++ b/configure.ps1 @@ -66,7 +66,7 @@ Param( [ValidateNotNullOrEmpty()] [string]$ClouseauMethod = "dist", # method for Clouseau to deploy: git or dist (default dist) [ValidateNotNullOrEmpty()] - [string]$ClouseauVersion = "2.25.0", # select the version of Clouseau to use (default 2.25.0) + [string]$ClouseauVersion = "3.0.0", # select the version of Clouseau to use (default 3.0.0) [ValidateNotNullOrEmpty()] [string]$ClouseauUri = "https://github.com/cloudant-labs/clouseau/releases/download/{0}/clouseau-{0}-dist.zip", # location for retrieving Clouseau (default https://github.com/cloudant-labs/clouseau/releases/download/2.25.0/clouseau-2.25.0-dist.zip) [ValidateNotNullOrEmpty()] @@ -150,6 +150,7 @@ $BuildDocs = (-not $DisableDocs).ToString().ToLower() $WithProper = (-not $DisableProper).ToString().ToLower() $ErlangMD5 = ($EnableErlangMD5).ToString().ToLower() $WithSpiderMonkey = (-not $DisableSpiderMonkey).ToString().ToLower() +$WithZIOSE = [int]($ClouseauVersion.Split('.')[0]) -ge 3 if ($JSEngine -eq "quickjs") { $WithSpiderMonkey = "false" @@ -333,12 +334,15 @@ if ($WithClouseau) New-Item -Path $ClouseauDir -ItemType Directory | Out-Null - $LogbackVersion = "1.2.13" $ClouseauDistUrl = $ClouseauUri -f $ClouseauVersion - $LogbackCoreJar = "logback-core-$LogbackVersion.jar" - $LogbackCoreJarUrl = "https://repo1.maven.org/maven2/ch/qos/logback/logback-core/$LogbackVersion/$LogbackCoreJar" - $LogbackClassicJar = "logback-classic-$LogbackVersion.jar" - $LogbackClassicJarUrl = "https://repo1.maven.org/maven2/ch/qos/logback/logback-classic/$LogbackVersion/$LogbackClassicJar" + + If (-not $WithZIOSE) { + $LogbackVersion = "1.2.13" + $LogbackCoreJar = "logback-core-$LogbackVersion.jar" + $LogbackCoreJarUrl = "https://repo1.maven.org/maven2/ch/qos/logback/logback-core/$LogbackVersion/$LogbackCoreJar" + $LogbackClassicJar = "logback-classic-$LogbackVersion.jar" + $LogbackClassicJarUrl = "https://repo1.maven.org/maven2/ch/qos/logback/logback-classic/$LogbackVersion/$LogbackClassicJar" + } Set-Variable ProgressPreference SilentlyContinue Invoke-WebRequest -MaximumRedirection 1 -OutFile clouseau.zip $ClouseauDistUrl @@ -356,16 +360,22 @@ if ($WithClouseau) Remove-Item "$ClouseauDir\clouseau-$ClouseauVersion" Remove-Item clouseau.zip - Invoke-WebRequest -MaximumRedirection 1 -OutFile "$ClouseauDir\$LogbackCoreJar" $LogbackCoreJarUrl - If ($LASTEXITCODE -ne 0) { - Write-Output "ERROR: $LogbackCoreJarUrl could not be downloaded." - exit 1 + If (-not $WithZIOSE) { + Invoke-WebRequest -MaximumRedirection 1 -OutFile "$ClouseauDir\$LogbackCoreJar" $LogbackCoreJarUrl + If ($LASTEXITCODE -ne 0) { + Write-Output "ERROR: $LogbackCoreJarUrl could not be downloaded." + exit 1 + } + + Invoke-WebRequest -MaximumRedirection 1 -OutFile "$ClouseauDir\$LogbackClassicJar" $LogbackClassicJarUrl + If ($LASTEXITCODE -ne 0) { + Write-Output "ERROR: $LogbackClassicJarUrl could not be downloaded." + exit 1 + } } - Invoke-WebRequest -MaximumRedirection 1 -OutFile "$ClouseauDir\$LogbackClassicJar" $LogbackClassicJarUrl - If ($LASTEXITCODE -ne 0) { - Write-Output "ERROR: $LogbackClassicJarUrl could not be downloaded." - exit 1 + If ($WithZIOSE) { + New-Item -ItemType File -Path "$ClouseauDir\.ziose" } } elseif ($ClouseauMethod -eq "git") { diff --git a/dev/run b/dev/run index 478fb2ae9a0..640d15d22ff 100755 --- a/dev/run +++ b/dev/run @@ -583,13 +583,42 @@ class StartupError(Exception): super().__init__(self.message) -def generate_default_clouseau_ini(path): - print("(generate default clouseau.ini)") - with open(path, "w") as handle: - handle.write( - """[clouseau] +DEFAULT_CLOUSEAU_INI = """[clouseau] """ - ) + +# Currently it does not support more than 3 concurrent nodes +DEFAULT_CLOUSEAU_CONF = """logger: { + format: Raw + output: Stdout + level: debug +} +config: [ + { + node: { + name: clouseau1 + domain: 127.0.0.1 + } + } + { + node: { + name: clouseau2 + domain: 127.0.0.1 + } + } + { + node: { + name: clouseau3 + domain: 127.0.0.1 + } + } +] +""" + + +def generate_default_clouseau_config(path, config_file, body): + print(f"(generate default {config_file})") + with open(path, "w") as handle: + handle.write(body) def generate_default_logback_properties(path): @@ -635,23 +664,41 @@ def boot_clouseau(ctx, idx): clouseau_java_home + "/bin/java" if clouseau_java_home else shutil.which("java") ) java_version = get_java_version(java) - - pom_file = os.path.join(clouseau_dir, "pom.xml") - method = "src" if os.path.isfile(pom_file) else "dist" + file_exists = lambda f: os.path.isfile(os.path.join(clouseau_dir, f)) + + if file_exists("build.sbt"): + is_ziose = True + method = "src" + elif file_exists("pom.xml"): + is_ziose = False + method = "src" + else: + is_ziose = file_exists(".ziose") + method = "dist" if not java_version: print( "Warning: Java version could not be determined, Clouseau may not be able to run" ) else: - if method == "src" and (java_version < 1.7 or 1.7 < java_version): - raise StartupError( - "Java is not suitable to run Clouseau. Please use JDK 1.7 and configure its (root) path in `CLOUSEAU_JAVA_HOME`" - ) - elif java_version < 1.7 or 1.8 < java_version: - raise StartupError( - "Java is not suitable to run Clouseau. Please use JRE 1.7 or 1.8 and configure its (root) path in `CLOUSEAU_JAVA_HOME`" - ) + if is_ziose: + if method == "src" and (java_version < 11): + raise StartupError( + "Java is not suitable to build and run Clouseau. Please use JDK 11 or higher and configure its (root) path in `CLOUSEAU_JAVA_HOME`" + ) + elif method == "dist" and (java_version < 21): + raise StartupError( + "Java is not suitable to run Clouseau. Please use JRE 21 or higher configure its (root) path in `CLOUSEAU_JAVA_HOME`" + ) + else: + if method == "src" and (java_version < 1.7 or 1.7 < java_version): + raise StartupError( + "Java is not suitable to run Clouseau. Please use JDK 1.7 and configure its (root) path in `CLOUSEAU_JAVA_HOME`" + ) + elif method == "dist" and (java_version < 1.7 or 1.8 < java_version): + raise StartupError( + "Java is not suitable to run Clouseau. Please use JRE 1.7 or 1.8 and configure its (root) path in `CLOUSEAU_JAVA_HOME`" + ) logfname = os.path.join(log_dir(ctx), "clouseau{}.log".format(idx)) log = open(logfname, "w") @@ -666,30 +713,53 @@ def boot_clouseau(ctx, idx): env = os.environ.copy() if clouseau_java_home: env["JAVA_HOME"] = clouseau_java_home + env["PATH"] = f"{clouseau_java_home}/bin:{env['PATH']}" + + if is_ziose: + clouseau_sbt_home = os.environ.get( + "CLOUSEAU_SBT_HOME" + ) or os.environ.get("SBT_HOME") + sbt = ( + clouseau_sbt_home + "/bin/sbt" + if clouseau_sbt_home + else shutil.which("sbt") + ) - clouseau_mvn_home = os.environ.get("CLOUSEAU_M2_HOME") or os.environ.get( - "M2_HOME" - ) - mvn = ( - clouseau_mvn_home + "/bin/mvn" - if clouseau_mvn_home - else shutil.which("mvn") - ) + if not sbt: + raise StartupError( + "Scala Built Tool (SBT) could not be found. Please install SBT 1.10 or later and configure its (root) path in `CLOUSEAU_SBT_HOME`" + ) - if not mvn: - raise StartupError( - "Maven could not be found. Please install Maven 3.8 and configure its (root) path in `CLOUSEAU_M2_HOME`" + sbt_java_home = ( + ["-java-home", clouseau_java_home] if clouseau_java_home else [] + ) + cmd = [sbt] + sbt_java_home + ["assembly"] + else: + clouseau_mvn_home = os.environ.get( + "CLOUSEAU_M2_HOME" + ) or os.environ.get("M2_HOME") + mvn = ( + clouseau_mvn_home + "/bin/mvn" + if clouseau_mvn_home + else shutil.which("mvn") ) + if not mvn: + raise StartupError( + "Maven could not be found. Please install Maven 3.8 and configure its (root) path in `CLOUSEAU_M2_HOME`" + ) + + cmd = [ + mvn, + "-B", + "compile", + "dependency:build-classpath", + "-Dmdep.outputFile={}".format(cp_file), + ] + try: sp.check_call( - [ - mvn, - "-B", - "compile", - "dependency:build-classpath", - "-Dmdep.outputFile={}".format(cp_file), - ], + cmd, cwd=clouseau_dir, env=env, stdin=sp.PIPE, @@ -700,42 +770,70 @@ def boot_clouseau(ctx, idx): log.write(str(exc)) raise StartupError("Could not build Clouseau from sources.") - with open(cp_file, "r") as handle: - classpath_deps = handle.read().rstrip() + if is_ziose: + clouseau_jars = glob.glob( + os.path.join(clouseau_dir, "clouseau", "target", "**", "*.jar"), + recursive=True, + ) - targets = os.path.join(clouseau_dir, "target", "classes") - classpath = separator.join([classpath_deps, targets]) + if not clouseau_jars: + raise StartupError("No JAR files could be built for Clouseau") + + classpath = separator.join(clouseau_jars) + else: + with open(cp_file, "r") as handle: + classpath_deps = handle.read().rstrip() + + targets = os.path.join(clouseau_dir, "target", "classes") + classpath = separator.join([classpath_deps, targets]) else: - clouseau_jars = [ - os.path.join(clouseau_dir, fname) - for fname in os.listdir(clouseau_dir) - if fname.endswith(".jar") - ] + clouseau_jars = glob.glob(os.path.join(clouseau_dir, "*.jar")) if not clouseau_jars: raise StartupError("Clouseau has no JAR files") classpath = separator.join(clouseau_jars) - clouseau_ini = os.path.join(clouseau_dir, "clouseau.ini") - if not os.path.exists(clouseau_ini): - generate_default_clouseau_ini(clouseau_ini) + if is_ziose: + name_property = "node" + cookie_property = "cookie" + clouseau_name_format = "clouseau{}" + gc_config = [ + "-XX:+HeapDumpOnOutOfMemoryError", + "-XX:+UseG1GC", + "-XX:+ParallelRefProcEnabled", + ] + main_class = "com.cloudant.ziose.clouseau.Main" + config_file = "clouseau.conf" + config_body = DEFAULT_CLOUSEAU_CONF + else: + name_property = "clouseau.name" + cookie_property = "clouseau.cookie" + clouseau_name_format = "clouseau{}@127.0.0.1" + gc_config = ["-XX:+UseConcMarkSweepGC", "-XX:+CMSParallelRemarkEnabled"] + main_class = "com.cloudant.clouseau.Main" + config_file = "clouseau.ini" + config_body = DEFAULT_CLOUSEAU_INI - logback_properties = os.path.join(clouseau_dir, "logback.xml") - if not os.path.exists(logback_properties): - generate_default_logback_properties(logback_properties) + config_path = os.path.join(clouseau_dir, config_file) + if not os.path.exists(config_path): + generate_default_clouseau_config(config_path, config_file, config_body) - classpath = separator.join([classpath, clouseau_dir]) # for logback.xml + if not is_ziose: + logback_properties = os.path.join(clouseau_dir, "logback.xml") + if not os.path.exists(logback_properties): + generate_default_logback_properties(logback_properties) + + classpath = separator.join([classpath, clouseau_dir]) # for logback.xml - clouseau_name = "clouseau{}@127.0.0.1".format(idx) clouseau_indexes_dir = os.path.join(ctx["devdir"], "clouseau{}".format(idx), "data") if ctx["erlang_cookie"]: - clouseau_cookie = ["-Dclouseau.cookie={}".format(ctx["erlang_cookie"])] + clouseau_cookie = ["-D{}={}".format(cookie_property, ctx["erlang_cookie"])] else: clouseau_cookie = [] - # `HOME` must be set for Scalang to find the Erlang cookie + # `HOME` must be set for finding the Erlang cookie (when not specified) if not os.environ.get("HOME"): # This is usually `USERPROFILE` on Windows alternativeHome = os.environ.get("USERPROFILE") @@ -750,18 +848,16 @@ def boot_clouseau(ctx, idx): "-Dsun.net.inetaddr.ttl=30", "-Dsun.net.inetaddr.negative.ttl=30", '-XX:OnOutOfMemoryError="kill -9 %p"', - "-XX:+UseConcMarkSweepGC", - "-XX:+CMSParallelRemarkEnabled", + ] + + gc_config + + [ "-cp", classpath, - "-Dclouseau.name={}".format(clouseau_name), + "-D{}={}".format(name_property, clouseau_name_format.format(idx)), "-Dclouseau.dir={}".format(clouseau_indexes_dir), ] + clouseau_cookie - + [ - "com.cloudant.clouseau.Main", - clouseau_ini, - ] + + [main_class, config_path] ) try: From 16f483f3d144cc873401889c76e7b772401f87b5 Mon Sep 17 00:00:00 2001 From: Gabor Pali Date: Wed, 26 Nov 2025 21:52:17 +0100 Subject: [PATCH 2/4] Make `CLOUSEAU_JAVA_HOME` configurable explicitly Even though the build environments implictly configure the Java location that is specific to Clouseau via environment variables, it makes sense to make it possible for the developers to redefine the respective paths ad hoc. When upgrading to 3.x, it can even help with rerolling the container images only after the switch and avoid breaking the CI that may still run with Clouseau 2.x. Later this may become equally beneficial if the required Java versions are different from what other dependencies, e.g. Nouveau assume. --- build-aux/Jenkinsfile | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/build-aux/Jenkinsfile b/build-aux/Jenkinsfile index 8a5a3330521..162e10b32ca 100644 --- a/build-aux/Jenkinsfile +++ b/build-aux/Jenkinsfile @@ -40,6 +40,7 @@ meta = [ spidermonkey_vsn: '60', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/usr', quickjs_test262: true, image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" ], @@ -49,6 +50,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/usr', quickjs_test262: true, image: "apache/couchdbci-centos:9-erlang-${ERLANG_VERSION}" ], @@ -58,6 +60,7 @@ meta = [ spidermonkey_vsn: '91', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-ubuntu:jammy-erlang-${ERLANG_VERSION}" ], @@ -67,6 +70,7 @@ meta = [ spidermonkey_vsn: '115', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-ubuntu:noble-erlang-${ERLANG_VERSION}" ], @@ -76,6 +80,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" ], @@ -118,6 +123,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', // Test this in in the bookworm-quickjs variant quickjs_test262: false, image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" @@ -128,6 +134,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: false, image: "${DOCKER_IMAGE_BASE}-${MAXIMUM_ERLANG_VERSION}" ], @@ -137,6 +144,7 @@ meta = [ disable_spidermonkey: true, with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}" ], @@ -154,6 +162,7 @@ meta = [ spidermonkey_vsn: '78', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', // Test this in in the bookworm-quickjs variant quickjs_test262: false, image: "${DOCKER_IMAGE_BASE}-${ERLANG_VERSION}", @@ -165,6 +174,7 @@ meta = [ spidermonkey_vsn: '128', with_nouveau: true, with_clouseau: true, + clouseau_java_home: '/opt/java/openjdk', quickjs_test262: true, image: "apache/couchdbci-debian:trixie-erlang-${ERLANG_VERSION}" ], @@ -205,6 +215,7 @@ meta = [ name: 'Windows 2022', spidermonkey_vsn: '128', with_clouseau: true, + clouseau_java_home: /C:\tools\zulu21.46.19-ca-jdk21.0.9-win_x64/, quickjs_test262: false, node_label: 'win' ] @@ -302,7 +313,12 @@ def generateNativeStage(platform) { powershell( script: "New-Item -ItemType Directory -Path '${platform}/build' -Force", label: 'Create build directories' ) powershell( script: "tar -xf (Get-Item apache-couchdb-*.tar.gz) -C '${platform}/build' --strip-components=1", label: 'Unpack release' ) dir( "${platform}/build" ) { - withClouseau = meta[platform].with_clouseau ? '-WithClouseau' : '' + def withClouseau = meta[platform].with_clouseau ? '-WithClouseau' : '' + def setClouseauJavaHome = '' + + if (meta[platform].clouseau_java_home) { + setClouseauJavaHome = "Set-Item -Path env:CLOUSEAU_JAVA_HOME -Value '${meta[platform].clouseau_java_home}'" + } powershell( script: """ .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 @@ -317,12 +333,14 @@ def generateNativeStage(platform) { powershell( script: """ .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + ${setClouseauJavaHome} make -f Makefile.win elixir-search ERLANG_COOKIE=crumbles """, label: 'Clouseau tests') powershell( script: """ .\\..\\..\\couchdb-glazier\\bin\\shell.ps1 Set-Item -Path env:GRADLE_OPTS -Value '-Dorg.gradle.daemon=false' + ${setClouseauJavaHome} make -f Makefile.win mango-test ERLANG_COOKIE=crumbles """, label: 'Mango tests') @@ -382,14 +400,20 @@ def generateContainerStage(platform) { sh( script: "mkdir -p ${platform}/build", label: 'Create build directories' ) sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) quickjs_tests262 = meta[platform].quickjs_test262 + def setClouseauJavaHome = '' + + if (meta[platform].clouseau_java_home) { + setClouseauJavaHome = "env CLOUSEAU_JAVA_HOME='${meta[platform].clouseau_java_home}' " + } + dir( "${platform}/build" ) { sh "${configure(meta[platform])}" sh 'make' retry(3) {sh 'make eunit'} if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry(3) {sh 'make elixir'} - retry(3) {sh 'make elixir-search'} - retry(3) {sh 'make mango-test'} + retry(3) {sh "${setClouseauJavaHome}make elixir-search"} + retry(3) {sh "${setClouseauJavaHome}make mango-test"} retry(3) {sh 'make weatherreport-test'} retry(3) {sh 'make nouveau-test'} } From fcb2f74a4d01e61de8a57cfe58d468b0b1e11c74 Mon Sep 17 00:00:00 2001 From: Gabor Pali Date: Thu, 27 Nov 2025 13:52:40 +0100 Subject: [PATCH 3/4] Set Erlang cookie explicitly for running Clouseau The Erlang cookie shared by the CouchDB and Clouseau nodes goes out of sync somehow mysteriously which makes the corresponding tests fail immediately. Until the exact causes are known, work around this issue by configuring both sides with the same string (via the `ERLANG_COOKIE` make variable) and do not rely on other implicit methods of generating and obtaining the cookie from other sources. --- build-aux/Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build-aux/Jenkinsfile b/build-aux/Jenkinsfile index 162e10b32ca..67c80cbd38c 100644 --- a/build-aux/Jenkinsfile +++ b/build-aux/Jenkinsfile @@ -276,8 +276,8 @@ def generateNativeStage(platform) { retry (3) {sh '$MAKE eunit'} if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry (3) {sh '$MAKE elixir'} - retry (3) {sh '$MAKE elixir-search'} - retry (3) {sh '$MAKE mango-test'} + retry (3) {sh '$MAKE elixir-search ERLANG_COOKIE=crumbles'} + retry (3) {sh '$MAKE mango-test ERLANG_COOKIE=crumbles'} retry (3) {sh '$MAKE weatherreport-test'} retry (3) {sh '$MAKE nouveau-test'} } @@ -412,8 +412,8 @@ def generateContainerStage(platform) { retry(3) {sh 'make eunit'} if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry(3) {sh 'make elixir'} - retry(3) {sh "${setClouseauJavaHome}make elixir-search"} - retry(3) {sh "${setClouseauJavaHome}make mango-test"} + retry(3) {sh "${setClouseauJavaHome}make elixir-search ERLANG_COOKIE=crumbles"} + retry(3) {sh "${setClouseauJavaHome}make mango-test ERLANG_COOKIE=crumbles"} retry(3) {sh 'make weatherreport-test'} retry(3) {sh 'make nouveau-test'} } From f3e3928245b71c2d73a91e4c7209489e95aa1a9e Mon Sep 17 00:00:00 2001 From: Gabor Pali Date: Fri, 28 Nov 2025 09:11:18 +0100 Subject: [PATCH 4/4] Set timeout for Clouseau tests on containerized builders For some reason, the local cluster to run the integration tests that may require Clouseau (Search) does not always start up properly which makes the execution hang indefinitely when run in a Docker container. Work around the issue by setting a reasonable time limit (5 minutes) for each of those and combined with the Jenkins retry, it can help with recovering the build in such unfortunate scenarios. --- build-aux/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-aux/Jenkinsfile b/build-aux/Jenkinsfile index 67c80cbd38c..9bea62863bc 100644 --- a/build-aux/Jenkinsfile +++ b/build-aux/Jenkinsfile @@ -412,8 +412,8 @@ def generateContainerStage(platform) { retry(3) {sh 'make eunit'} if (meta[platform].quickjs_test262) {retry(3) {sh 'make quickjs-test262'}} retry(3) {sh 'make elixir'} - retry(3) {sh "${setClouseauJavaHome}make elixir-search ERLANG_COOKIE=crumbles"} - retry(3) {sh "${setClouseauJavaHome}make mango-test ERLANG_COOKIE=crumbles"} + retry(3) {sh "${setClouseauJavaHome}timeout 5m make elixir-search ERLANG_COOKIE=crumbles"} + retry(3) {sh "${setClouseauJavaHome}timeout 5m make mango-test ERLANG_COOKIE=crumbles"} retry(3) {sh 'make weatherreport-test'} retry(3) {sh 'make nouveau-test'} }