fix(ci): Exit with subprocess return code in run_ci.py (#2560)

CI was not reflecting failures in shell script fallbacks (we are
migrating to python)

* Use sys.exit() to propagate the return code from subprocess.call().
* This ensures that the CI pipeline correctly reflects the success or
failure of the executed scripts.

You might want to merge #2561 first, to reduce the number of errors.

---------

Co-authored-by: chaokunyang <shawn.ck.yang@gmail.com>

Author: esafak

.github/pull_request_template.md

@@ -10,6 +10,12 @@ Contribution Checklist
     - Fory has a strong focus on performance. If the PR you submit will have an impact on performance, please benchmark it first and provide the benchmark result here.
 -->
 
+## Why?
+
+## What does this PR do?
+
+<!-- Describe the purpose of this PR. -->
+
 ## What does this PR do?
 
 <!-- Describe the purpose of this PR. -->

.github/workflows/ci.yml

@@ -62,6 +62,8 @@ jobs:
         run: python ./ci/run_ci.py cpp --install-deps-only
       - name: Install python dependencies
         run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U
+      - name: Install pyfory for xlang tests
+        run: pip install -e python/
       - name: Run CI with Maven
         run: python ./ci/run_ci.py java --version ${{ matrix.java-version }}
       - name: Upload Test Report
@@ -93,6 +95,8 @@ jobs:
           python-version: 3.8
       - name: Install bazel
         run: python ./ci/run_ci.py cpp --install-deps-only
+      - name: Install pyfory for xlang tests
+        run: pip install -e python/
       - name: Install python dependencies
         run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U
       - name: Run CI with Maven
@@ -190,10 +194,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - name: Set up JDK8
+      - name: Set up JDK11
         uses: actions/setup-java@v4
         with:
-          java-version: 8
+          java-version: 11
           distribution: "temurin"
       - name: Set up Python 3.8
         uses: actions/setup-python@v5
@@ -272,9 +276,14 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install bazel
+      - name: Install bazel (Unix)
+        if: runner.os != 'Windows'
         shell: bash
         run: python ./ci/run_ci.py cpp --install-deps-only
+      - name: Install bazel (Windows)
+        if: runner.os == 'Windows'
+        shell: bash
+        run: ./ci/run_ci.sh install_bazel_windows
       - name: Run Python CI
         shell: bash
         run: python ./ci/run_ci.py python
@@ -301,7 +310,7 @@ jobs:
         run: python ./ci/run_ci.py cpp --install-deps-only
       - name: Install python dependencies
         run: pip install pyarrow==15.0.0 cython wheel pytest setuptools -U
-      - name: Install pyfory
+      - name: Install pyfory for xlang tests
         run: pip install -e python/
       - name: Run Golang CI
         run: python ./ci/run_ci.py go

ci/run_ci.py

@@ -84,7 +84,7 @@ def run_shell_script(command, *args):
             cmd = [bash_path, script_path, command]
             cmd.extend(args)
             logging.info(f"Falling back to shell script with bash: {' '.join(cmd)}")
-            return subprocess.call(cmd)
+            sys.exit(subprocess.call(cmd))
         else:
             logging.error(
                 "Bash is not available on this Windows system. Cannot run shell script."
@@ -101,7 +101,7 @@ def run_shell_script(command, *args):
         cmd = [script_path, command]
         cmd.extend(args)
         logging.info(f"Falling back to shell script: {' '.join(cmd)}")
-        return subprocess.call(cmd)
+        sys.exit(subprocess.call(cmd))
 
 
 def parse_args():
@@ -237,14 +237,13 @@ def parse_args():
         if USE_PYTHON_JAVA:
             func(**arg_dict)
         else:
-            
             if not arg_dict.get("version"):
                 func(**arg_dict)
                 return
             # Map Python version argument to shell script command
             version = arg_dict.get("version", "17")
             release = arg_dict.get("release", False)
-            
+
             if release:
                 logging.info("Release mode requested - using Python implementation")
                 func(**arg_dict)

ci/run_ci.sh

@@ -141,7 +141,7 @@ install_jdks() {
 
 graalvm_test() {
   cd "$ROOT"/java
-  mvn -T10 -B --no-transfer-progress clean install -DskipTests
+  mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'
   echo "Start to build graalvm native image"
   cd "$ROOT"/integration_tests/graalvm_tests
   mvn -DskipTests=true --no-transfer-progress -Pnative package
@@ -231,7 +231,7 @@ case $1 in
       echo "Executing fory java tests"
       cd "$ROOT/java"
       set +e
-      mvn -T16 --batch-mode --no-transfer-progress test
+      mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'
       testcode=$?
       if [[ $testcode -ne 0 ]]; then
         exit $testcode

ci/tasks/java.py

@@ -17,7 +17,6 @@
 
 import logging
 import os
-import sys
 import subprocess
 import re
 from . import common
@@ -26,7 +25,7 @@
 def get_jdk_major_version():
     try:
         # Run the 'java -version' command
-        result = subprocess.run(['java', '-version'], capture_output=True, text=True)
+        result = subprocess.run(["java", "-version"], capture_output=True, text=True)
         output = result.stderr  # java -version outputs to stderr
 
         # Use regex to find the version string
@@ -37,8 +36,8 @@ def get_jdk_major_version():
         version_string = match.group(1)
 
         # Parse the version string
-        version_parts = version_string.split('.')
-        if version_parts[0] == '1':
+        version_parts = version_string.split(".")
+        if version_parts[0] == "1":
             # Java 8 or earlier
             return int(version_parts[1])
         else:
@@ -81,54 +80,56 @@ def create_toolchains_xml(jdk_mappings):
     import os
     import xml.etree.ElementTree as ET
     from xml.dom import minidom
-    
+
     # Create ~/.m2 directory if it doesn't exist
     m2_dir = os.path.expanduser("~/.m2")
     os.makedirs(m2_dir, exist_ok=True)
-    
+
     # Create the root element
     toolchains = ET.Element("toolchains")
-    
+
     for version, jdk_name in jdk_mappings.items():
         toolchain = ET.SubElement(toolchains, "toolchain")
-        
+
         # Set type
         type_elem = ET.SubElement(toolchain, "type")
         type_elem.text = "jdk"
-        
+
         # Set provides
         provides = ET.SubElement(toolchain, "provides")
         version_elem = ET.SubElement(provides, "version")
         version_elem.text = version
         vendor_elem = ET.SubElement(provides, "vendor")
         vendor_elem.text = "azul"
-        
+
         # Set configuration
         configuration = ET.SubElement(toolchain, "configuration")
         jdk_home = ET.SubElement(configuration, "jdkHome")
         jdk_home.text = os.path.abspath(os.path.join(common.PROJECT_ROOT_DIR, jdk_name))
-    
+
     # Create pretty XML string
-    rough_string = ET.tostring(toolchains, 'unicode')
+    rough_string = ET.tostring(toolchains, "unicode")
     reparsed = minidom.parseString(rough_string)
     pretty_xml = reparsed.toprettyxml(indent="  ")
-    
+
     # Add proper XML header with encoding
     xml_header = '<?xml version="1.0" encoding="UTF8"?>\n'
-    pretty_xml = xml_header + pretty_xml.split('\n', 1)[1]  # Remove the default header and add our custom one
-    
+    pretty_xml = (
+        xml_header + pretty_xml.split("\n", 1)[1]
+    )  # Remove the default header and add our custom one
+
     # Write to ~/.m2/toolchains.xml
     toolchains_path = os.path.join(m2_dir, "toolchains.xml")
-    with open(toolchains_path, 'w', encoding='utf-8') as f:
+    with open(toolchains_path, "w", encoding="utf-8") as f:
         f.write(pretty_xml)
-    
+
     logging.info(f"Created toolchains.xml at {toolchains_path}")
     logging.info("Toolchains configuration:")
     for version, jdk_name in jdk_mappings.items():
         jdk_path = os.path.join(common.PROJECT_ROOT_DIR, jdk_name)
         logging.info(f"  JDK {version}: {jdk_path}")
     # print toolchains.xml
-    with open(toolchains_path, 'r', encoding='utf-8') as f:
+    with open(toolchains_path, "r", encoding="utf-8") as f:
         logging.info(f.read())
 
 
@@ -149,7 +150,9 @@ def run_java8():
     logging.info("Executing fory java tests with Java 8")
     install_jdks()
     common.cd_project_subdir("java")
-    common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!fory-format'")
+    common.exec_cmd(
+        "mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'"
+    )
     logging.info("Executing fory java tests succeeds")
 
 
@@ -208,7 +211,9 @@ def run_integration_tests():
     logging.info("Executing fory integration tests")
 
     common.cd_project_subdir("java")
-    common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests")
+    common.exec_cmd(
+        "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'"
+    )
 
     logging.info("benchmark tests")
     common.cd_project_subdir("java/benchmark")
@@ -270,7 +275,9 @@ def run_graalvm_test():
     logging.info("Start GraalVM tests")
 
     common.cd_project_subdir("java")
-    common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests")
+    common.exec_cmd(
+        "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'"
+    )
 
     logging.info("Start to build graalvm native image")
     common.cd_project_subdir("integration_tests/graalvm_tests")
@@ -285,17 +292,19 @@ def run_graalvm_test():
 
 def run_release():
     """Release to Maven Central."""
-    logging.info(f"Starting release to Maven Central with Java")
+    logging.info("Starting release to Maven Central with Java")
     common.cd_project_subdir("java")
-    
+
     # Clean and install without tests first
     logging.info("Cleaning and installing dependencies")
     common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests")
-    
+
     # Deploy to Maven Central
     logging.info("Deploying to Maven Central")
-    common.exec_cmd("mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release")
-    
+    common.exec_cmd(
+        "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release"
+    )
+
     logging.info("Release to Maven Central completed successfully")
 
 

integration_tests/graalvm_tests/pom.xml

@@ -173,7 +173,6 @@
                 <!-- for fast build -->
                 <!-- <buildArg>-Ob</buildArg> -->
                 <buildArg>-H:+UnlockExperimentalVMOptions</buildArg>
-                <buildArg>-H:DynamicProxyConfigurationFiles=src/main/resources/META-INF/native-image/proxy-config.json</buildArg>
               </buildArgs>
             </configuration>
           </plugin>

integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java

@@ -17,42 +17,42 @@
  * under the License.
  */
 
- package org.apache.fory.graalvm;
-
-import org.apache.fory.Fory;
+package org.apache.fory.graalvm;
 
 import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import org.apache.fory.Fory;
 
 public class ObjectStreamExample extends AbstractMap<Integer, Integer> {
-  private static final Fory FORY = Fory.builder()
-      .withName(ObjectStreamExample.class.getName())
-      .registerGuavaTypes(false)
-      .build();
+  private static final Fory FORY =
+      Fory.builder()
+          .withName(ObjectStreamExample.class.getName())
+          .registerGuavaTypes(false)
+          .build();
 
   static {
-      FORY.register(ObjectStreamExample.class, true);
-      FORY.ensureSerializersCompiled();
+    FORY.register(ObjectStreamExample.class, true);
+    FORY.ensureSerializersCompiled();
   }
 
   final int[] ints = new int[10];
 
   public static void main(String[] args) {
-      FORY.reset();
-      byte[] bytes = FORY.serialize(new ObjectStreamExample());
-      FORY.reset();
-      ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes);
-      System.out.println(Arrays.toString(o.ints));
+    FORY.reset();
+    byte[] bytes = FORY.serialize(new ObjectStreamExample());
+    FORY.reset();
+    ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes);
+    System.out.println(Arrays.toString(o.ints));
   }
 
   @Override
   public Set<Entry<Integer, Integer>> entrySet() {
-      HashSet<Entry<Integer, Integer>> set = new HashSet<>();
-      for (int i = 0; i < ints.length; i++) {
-          set.add(new AbstractMap.SimpleEntry<>(i, ints[i]));
-      }
-      return set;
+    HashSet<Entry<Integer, Integer>> set = new HashSet<>();
+    for (int i = 0; i < ints.length; i++) {
+      set.add(new AbstractMap.SimpleEntry<>(i, ints[i]));
+    }
+    return set;
   }
-}
+}

integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java

@@ -50,6 +50,7 @@ private static Fory createFory() {
             .build();
     // register and generate serializer code.
     fory.register(TestInvocationHandler.class, true);
+    fory.ensureSerializersCompiled();
     return fory;
   }
 

integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json

@@ -0,0 +1,10 @@
+[
+  {
+    "interfaces": [
+      "java.util.function.Function"
+    ],
+    "condition": {
+      "typeReachable": "org.apache.fory.graalvm.ProxyExample$TestInvocationHandler"
+    }
+  }
+]