From 92341ba9be8a83067bac09d39c7f5009ec89f58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 12:44:51 -0400 Subject: [PATCH 01/44] fix(ci): Exit with subprocess return code in run_ci.py * Use sys.exit() to propagate the return code from subprocess.call(). * This ensures that the CI pipeline correctly reflects the success or failure of the executed scripts. Fixes #2555 --- ci/run_ci.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_ci.py b/ci/run_ci.py index e637cc94f7..b68d4d2a32 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -84,7 +84,7 @@ def run_shell_script(command, *args): cmd = [bash_path, script_path, command] cmd.extend(args) logging.info(f"Falling back to shell script with bash: {' '.join(cmd)}") - return subprocess.call(cmd) + sys.exit(subprocess.call(cmd)) else: logging.error( "Bash is not available on this Windows system. Cannot run shell script." @@ -101,7 +101,7 @@ def run_shell_script(command, *args): cmd = [script_path, command] cmd.extend(args) logging.info(f"Falling back to shell script: {' '.join(cmd)}") - return subprocess.call(cmd) + sys.exit(subprocess.call(cmd)) def parse_args(): From f676a533de5493b973ba09577672ee3635f8a283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 21:21:38 -0400 Subject: [PATCH 02/44] Ensure subprocess is called with correct working directory * Modify `setup.py` to include `cwd=project_dir` in `subprocess.check_call`. * This ensures that the bazel build command is executed in the project's root directory, preventing potential issues with relative path resolution. --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 87123efa8d..aee86ab8f1 100644 --- a/python/setup.py +++ b/python/setup.py @@ -50,7 +50,7 @@ def __init__(self, attrs=None): elif arch in ("aarch64", "arm64"): bazel_args += ["--copt=-fsigned-char"] bazel_args += ["//:cp_fory_so"] - subprocess.check_call(bazel_args) + subprocess.check_call(bazel_args, cwd=project_dir) def has_ext_modules(self): return True From 0046a37a2cd68f409d2bd57e1b73f49524d5b124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 21:54:36 -0400 Subject: [PATCH 03/44] Fix formatting, install pyfory in openj9 and Java tests for xlang --- .github/workflows/ci.yml | 6 +++++- ci/run_ci.py | 3 +-- python/setup.py | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f47cf79e..b08fc96d77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: run: python ./ci/run_ci.py cpp --install-deps-only - name: Install python dependencies run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U + - name: Install pyfory for xlang tests + run: pip install -e python/ - name: Run CI with Maven run: python ./ci/run_ci.py java --version ${{ matrix.java-version }} - name: Upload Test Report @@ -93,6 +95,8 @@ jobs: python-version: 3.8 - name: Install bazel run: python ./ci/run_ci.py cpp --install-deps-only + - name: Install pyfory for xlang tests + run: pip install -e python/ - name: Install python dependencies run: pip install pyarrow==15.0.0 Cython wheel pytest setuptools -U - name: Run CI with Maven @@ -303,7 +307,7 @@ jobs: run: python ./ci/run_ci.py cpp --install-deps-only - name: Install python dependencies run: pip install pyarrow==15.0.0 cython wheel pytest setuptools -U - - name: Install pyfory + - name: Install pyfory for xlang tests run: pip install -e python/ - name: Run Golang CI run: python ./ci/run_ci.py go diff --git a/ci/run_ci.py b/ci/run_ci.py index b68d4d2a32..ed1a3f59b9 100644 --- a/ci/run_ci.py +++ b/ci/run_ci.py @@ -237,14 +237,13 @@ def parse_args(): if USE_PYTHON_JAVA: func(**arg_dict) else: - if not arg_dict.get("version"): func(**arg_dict) return # Map Python version argument to shell script command version = arg_dict.get("version", "17") release = arg_dict.get("release", False) - + if release: logging.info("Release mode requested - using Python implementation") func(**arg_dict) diff --git a/python/setup.py b/python/setup.py index aee86ab8f1..a0fbc52d05 100644 --- a/python/setup.py +++ b/python/setup.py @@ -39,6 +39,7 @@ print(f"setup_dir: {setup_dir}") print(f"fory_cpp_src_dir: {fory_cpp_src_dir}") + class BinaryDistribution(Distribution): def __init__(self, attrs=None): super().__init__(attrs=attrs) From 7638af3c6a6c9732e46554c7b695898d3e57c585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 22:01:50 -0400 Subject: [PATCH 04/44] Fix formatting --- .github/pull_request_template.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2b4a80c576..10c6205459 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,6 +9,7 @@ Contribution Checklist - Fory has a strong focus on performance. If the PR you submit will have an impact on performance, please benchmark it first and provide the benchmark result here. --> + ## Why? From c7a1a990cef24ad6cadaa325f48424a82f6abbba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 22:38:23 -0400 Subject: [PATCH 05/44] Attempt to fix type_id=None bug, remove Java 8 since fory-format requires 11 --- .github/workflows/ci.yml | 2 +- python/pyfory/_registry.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08fc96d77..3c31591bc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: MY_VAR: "PATH" strategy: matrix: - java-version: ["8", "11", "17", "21", "24"] + java-version: ["11", "17", "21", "24"] steps: - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py index 05715a579d..be76af0d8d 100644 --- a/python/pyfory/_registry.py +++ b/python/pyfory/_registry.py @@ -379,6 +379,9 @@ def _register_pytype( serializer: Serializer = None, internal: bool = False, ): + # Set default type_id when None, similar to _register_xtype + if type_id is None and typename is not None: + type_id = self._next_type_id() return self.__register_type( cls, type_id=type_id, @@ -398,7 +401,7 @@ def __register_type( serializer: Serializer = None, internal: bool = False, ): - dynamic_type = type_id < 0 + dynamic_type = type_id is not None and type_id < 0 if not internal and serializer is None: serializer = self._create_serializer(cls) if typename is None: From 7e505592886ba114e6061e71f88e42ec1787e23c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:38:40 +0800 Subject: [PATCH 06/44] fix java8 ut --- ci/tasks/java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 7bb73e50ae..5734493466 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -149,7 +149,7 @@ def run_java8(): logging.info("Executing fory java tests with Java 8") install_jdks() common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!fory-format'") + common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format'") logging.info("Executing fory java tests succeeds") From 1b9e7c5cbff83bcf6f5b01b0c710098327492e9e Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:40:58 +0800 Subject: [PATCH 07/44] fix graavlm build --- .../org.apache.fory/fory-core/native-image.properties | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 50f599329b..3aed1c9146 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -226,6 +226,8 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.logging.LoggerFactory,\ org.apache.fory.memory.BoundsChecking,\ org.apache.fory.memory.MemoryBuffer,\ + org.apache.fory.memory.MemoryAllocator,\ + org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\ org.apache.fory.memory.MemoryUtils,\ org.apache.fory.memory.Platform,\ org.apache.fory.meta.ClassDef$ArrayFieldType,\ From 57c278b7cde55fafb87e55e4913ee7fd8a917e4d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:43:43 +0800 Subject: [PATCH 08/44] skip fory format install for integration tests --- ci/tasks/java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 5734493466..db6f2372a9 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -208,7 +208,7 @@ def run_integration_tests(): logging.info("Executing fory integration tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format'") logging.info("benchmark tests") common.cd_project_subdir("java/benchmark") From d6f2d6abb2b0695c819891654638e56ffcd9f89a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:44:41 +0800 Subject: [PATCH 09/44] lint java code --- .../src/main/java/org/apache/fory/codegen/Expression.java | 3 ++- .../main/java/org/apache/fory/resolver/AllowListChecker.java | 2 +- .../java/org/apache/fory/format/encoder/RowEncoderBuilder.java | 3 --- .../org/apache/fory/format/encoder/ImplementInterfaceTest.java | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java index f5135bcb67..a6a751e8c5 100644 --- a/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java +++ b/java/fory-core/src/main/java/org/apache/fory/codegen/Expression.java @@ -2432,7 +2432,8 @@ public ExprCode doGenCode(CodegenContext ctx) { action.apply( new Reference(i), new Reference(leftElemValue, leftElemType, true), - // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable is false. + // elemValue nullability check uses isNullAt inside action, so elemValueRef's nullable + // is false. new Reference(rightElemValue, rightElemType, false)); ExprCode elementExprCode = elemExpr.genCode(ctx); diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java index b49ded2e4f..9b280cce5f 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/AllowListChecker.java @@ -239,7 +239,7 @@ public void addListener(ClassResolver classResolver) { lock.writeLock().lock(); listeners.put(classResolver, true); } finally { - lock.writeLock().unlock(); + lock.writeLock().unlock(); } } diff --git a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java index 49ec2e134f..f24da862d3 100644 --- a/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java +++ b/java/fory-format/src/main/java/org/apache/fory/format/encoder/RowEncoderBuilder.java @@ -27,9 +27,6 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.OptionalDouble; -import java.util.OptionalInt; -import java.util.OptionalLong; import java.util.SortedMap; import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; diff --git a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java index 724acb773f..c054edeab4 100644 --- a/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java +++ b/java/fory-format/src/test/java/org/apache/fory/format/encoder/ImplementInterfaceTest.java @@ -26,7 +26,6 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeSet; - import lombok.Data; import org.apache.arrow.vector.types.pojo.Field; import org.apache.fory.annotation.ForyField; From 452864753d956a532f5493e46b1b6c49b04b4455 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:47:20 +0800 Subject: [PATCH 10/44] add java8 back --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c31591bc3..b08fc96d77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: MY_VAR: "PATH" strategy: matrix: - java-version: ["11", "17", "21", "24"] + java-version: ["8", "11", "17", "21", "24"] steps: - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} From 9654208e5ac4d8332a506d88683bc6ed505e506d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 10:56:10 +0800 Subject: [PATCH 11/44] fix skip fory-format on java8 --- ci/run_ci.sh | 6 +++--- ci/tasks/java.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 63a1330d14..c81ca4d120 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -141,7 +141,7 @@ install_jdks() { graalvm_test() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format' echo "Start to build graalvm native image" cd "$ROOT"/integration_tests/graalvm_tests mvn -DskipTests=true --no-transfer-progress -Pnative package @@ -153,7 +153,7 @@ graalvm_test() { integration_tests() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format' echo "benchmark tests" cd "$ROOT"/java/benchmark mvn -T10 -B --no-transfer-progress clean test install -Pjmh @@ -231,7 +231,7 @@ case $1 in echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T16 --batch-mode --no-transfer-progress test + mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format' testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode diff --git a/ci/tasks/java.py b/ci/tasks/java.py index db6f2372a9..8f190591b4 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -270,7 +270,7 @@ def run_graalvm_test(): logging.info("Start GraalVM tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format'") logging.info("Start to build graalvm native image") common.cd_project_subdir("integration_tests/graalvm_tests") From 9296ca4f84d72aa92c2baaaa575e12317b6a1a9b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 11:02:46 +0800 Subject: [PATCH 12/44] fix skip fory-testsuite --- ci/run_ci.sh | 6 +++--- ci/tasks/java.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/run_ci.sh b/ci/run_ci.sh index c81ca4d120..b2cabddbcd 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -141,7 +141,7 @@ install_jdks() { graalvm_test() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format' + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite' echo "Start to build graalvm native image" cd "$ROOT"/integration_tests/graalvm_tests mvn -DskipTests=true --no-transfer-progress -Pnative package @@ -153,7 +153,7 @@ graalvm_test() { integration_tests() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format' + mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite' echo "benchmark tests" cd "$ROOT"/java/benchmark mvn -T10 -B --no-transfer-progress clean test install -Pjmh @@ -231,7 +231,7 @@ case $1 in echo "Executing fory java tests" cd "$ROOT/java" set +e - mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format' + mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite' testcode=$? if [[ $testcode -ne 0 ]]; then exit $testcode diff --git a/ci/tasks/java.py b/ci/tasks/java.py index 8f190591b4..a705082fb8 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -149,7 +149,7 @@ def run_java8(): logging.info("Executing fory java tests with Java 8") install_jdks() common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format'") + common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'") logging.info("Executing fory java tests succeeds") @@ -208,7 +208,7 @@ def run_integration_tests(): logging.info("Executing fory integration tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format'") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'") logging.info("benchmark tests") common.cd_project_subdir("java/benchmark") @@ -270,7 +270,7 @@ def run_graalvm_test(): logging.info("Start GraalVM tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format'") + common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'") logging.info("Start to build graalvm native image") common.cd_project_subdir("integration_tests/graalvm_tests") From b84f4963e88c31eaf0caa720720a076f2bd26bce Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 11:04:40 +0800 Subject: [PATCH 13/44] fix native-image.properties --- .../org.apache.fory/fory-core/native-image.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 3aed1c9146..1ae683c105 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -227,7 +227,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.memory.BoundsChecking,\ org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.memory.MemoryAllocator,\ - org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\ + org.apache.fory.memory.MemoryBuffer$DefaultMemoryAllocator,\ org.apache.fory.memory.MemoryUtils,\ org.apache.fory.memory.Platform,\ org.apache.fory.meta.ClassDef$ArrayFieldType,\ From 6fdb74d2f91609a91bf071f9579465fbea0c8357 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 11:08:58 +0800 Subject: [PATCH 14/44] fix javadoc --- .../src/main/java/org/apache/fory/type/TypeUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java index afee63b3db..dde3fca6ab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java @@ -657,8 +657,8 @@ public static boolean isBean(TypeRef typeRef, TypeResolutionContext ctx) { } /** - * Check if a class is one of {@link Optional), {@link OptionalInt}, - * {@link OptionaLong}, or {@link OptionalDouble}. + * Check if a class is one of {@link Optional}, {@link OptionalInt}, {@link OptionalLong}, or + * {@link OptionalDouble}. */ public static boolean isOptionalType(Class type) { return type == Optional.class From c86c5455e6acca9a7231499b05f73132e2b2e180 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 11:12:13 +0800 Subject: [PATCH 15/44] use jdk 11 for integration tests --- .github/workflows/ci.yml | 4 ++-- ci/run_ci.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b08fc96d77..6a6071ac54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,10 +194,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - name: Set up JDK8 + - name: Set up JDK11 uses: actions/setup-java@v4 with: - java-version: 8 + java-version: 11 distribution: "temurin" - name: Set up Python 3.8 uses: actions/setup-python@v5 diff --git a/ci/run_ci.sh b/ci/run_ci.sh index b2cabddbcd..789f78cdad 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -153,7 +153,7 @@ graalvm_test() { integration_tests() { cd "$ROOT"/java - mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite' + mvn -T10 -B --no-transfer-progress clean install -DskipTests echo "benchmark tests" cd "$ROOT"/java/benchmark mvn -T10 -B --no-transfer-progress clean test install -Pjmh From c53a4c60bac16fb2610bf8cb8d37f4987c9a51ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Mon, 1 Sep 2025 23:46:51 -0400 Subject: [PATCH 16/44] attempt to fix python CI bazel error --- python/setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index a0fbc52d05..110c4346a0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -37,6 +37,7 @@ fory_cpp_src_dir = abspath(pjoin(setup_dir, "../src/")) print(f"setup_dir: {setup_dir}") +print(f"project_dir: {project_dir}") print(f"fory_cpp_src_dir: {fory_cpp_src_dir}") @@ -51,7 +52,10 @@ def __init__(self, attrs=None): elif arch in ("aarch64", "arm64"): bazel_args += ["--copt=-fsigned-char"] bazel_args += ["//:cp_fory_so"] - subprocess.check_call(bazel_args, cwd=project_dir) + # Ensure Windows path compatibility + cwd_path = os.path.normpath(project_dir) + print(f"Running bazel with cwd={cwd_path}: {' '.join(bazel_args)}") + subprocess.check_call(bazel_args, cwd=cwd_path) def has_ext_modules(self): return True From f421ce0015bf226edbb46ad1656fa7667056d34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 00:02:27 -0400 Subject: [PATCH 17/44] run bazel sync --configure first --- python/setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 110c4346a0..ee63be0862 100644 --- a/python/setup.py +++ b/python/setup.py @@ -54,7 +54,12 @@ def __init__(self, attrs=None): bazel_args += ["//:cp_fory_so"] # Ensure Windows path compatibility cwd_path = os.path.normpath(project_dir) - print(f"Running bazel with cwd={cwd_path}: {' '.join(bazel_args)}") + # Force Bazel to run a sync first to ensure repositories are loaded + sync_args = ["bazel", "sync", "--configure"] + print(f"Running bazel sync with cwd={cwd_path}: {' '.join(sync_args)}") + subprocess.check_call(sync_args, cwd=cwd_path) + + print(f"Running bazel build with cwd={cwd_path}: {' '.join(bazel_args)}") subprocess.check_call(bazel_args, cwd=cwd_path) def has_ext_modules(self): From 95d55fdfc630d72c2fdd5981fc0f4277b0ac4c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 00:06:16 -0400 Subject: [PATCH 18/44] run bazel sync --configure first --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index ee63be0862..2cefa8b6c7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -55,7 +55,7 @@ def __init__(self, attrs=None): # Ensure Windows path compatibility cwd_path = os.path.normpath(project_dir) # Force Bazel to run a sync first to ensure repositories are loaded - sync_args = ["bazel", "sync", "--configure"] + sync_args = ["bazel", "sync", "--configure", "--enable_workspace"] print(f"Running bazel sync with cwd={cwd_path}: {' '.join(sync_args)}") subprocess.check_call(sync_args, cwd=cwd_path) From ca76174996db44c128c661133a8d582eacff0dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 00:25:46 -0400 Subject: [PATCH 19/44] run bazel fetch --- python/setup.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python/setup.py b/python/setup.py index 2cefa8b6c7..99163d2b56 100644 --- a/python/setup.py +++ b/python/setup.py @@ -54,11 +54,9 @@ def __init__(self, attrs=None): bazel_args += ["//:cp_fory_so"] # Ensure Windows path compatibility cwd_path = os.path.normpath(project_dir) - # Force Bazel to run a sync first to ensure repositories are loaded - sync_args = ["bazel", "sync", "--configure", "--enable_workspace"] - print(f"Running bazel sync with cwd={cwd_path}: {' '.join(sync_args)}") - subprocess.check_call(sync_args, cwd=cwd_path) - + # Force repository fetching by running fetch first + print(f"Running bazel fetch to load repositories with cwd={cwd_path}") + subprocess.check_call(["bazel", "fetch", "//..."], cwd=cwd_path) print(f"Running bazel build with cwd={cwd_path}: {' '.join(bazel_args)}") subprocess.check_call(bazel_args, cwd=cwd_path) From a1298bb43b2bce48a0c47c10c9a2228ccefbee3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 00:32:06 -0400 Subject: [PATCH 20/44] restrict bazel fetch scope --- python/setup.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/python/setup.py b/python/setup.py index 99163d2b56..758a6735ad 100644 --- a/python/setup.py +++ b/python/setup.py @@ -54,9 +54,14 @@ def __init__(self, attrs=None): bazel_args += ["//:cp_fory_so"] # Ensure Windows path compatibility cwd_path = os.path.normpath(project_dir) - # Force repository fetching by running fetch first - print(f"Running bazel fetch to load repositories with cwd={cwd_path}") - subprocess.check_call(["bazel", "fetch", "//..."], cwd=cwd_path) + # Force repository fetching by running fetch for our specific target + fetch_args = ["bazel", "fetch", "//:cp_fory_so"] + try: + print(f"Running bazel fetch to load repositories with cwd={cwd_path}") + subprocess.check_call(fetch_args, cwd=cwd_path) + except subprocess.CalledProcessError: + # If fetch fails, continue anyway as build might still work + pass print(f"Running bazel build with cwd={cwd_path}: {' '.join(bazel_args)}") subprocess.check_call(bazel_args, cwd=cwd_path) From 1ec9d2ee115899f301625fa14496c40572bb6adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 00:48:11 -0400 Subject: [PATCH 21/44] Use FunctionSerializer for functions * Introduce FunctionSerializer for proper serialization of functions and lambdas. * Update _registry.py to use FunctionSerializer for function types. * Modify test_serializer.py to reflect the new support for functions, ensuring they are no longer treated as unsupported. * Add tests in test_function.py to cover various scenarios of function serialization, including simple, complex, and nested functions. * Adjust error messages in _fory.py for clarity regarding unregistered types. --- python/pyfory/_fory.py | 10 +++++----- python/pyfory/_registry.py | 6 ++++-- python/pyfory/tests/test_function.py | 8 ++++---- python/pyfory/tests/test_serializer.py | 16 +++++++++++++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 35ab78d36b..831e24d3bc 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -166,7 +166,7 @@ def __init__( self._unsupported_callback = None self._unsupported_objects = None self._peer_language = None - + def register( self, cls: Union[type, TypeVar], @@ -544,8 +544,8 @@ class _PicklerStub: def dump(self, o): raise ValueError( f"Type {type(o)} is not registered, " - f"pickle is not allowed when type registration enabled, Please register" - f"the type or pass unsupported_callback" + f"pickle is not allowed when type registration enabled, " + f"Please register the type or pass unsupported_callback" ) def clear_memo(self): @@ -555,6 +555,6 @@ def clear_memo(self): class _UnpicklerStub: def load(self): raise ValueError( - "pickle is not allowed when type registration enabled, Please register" - "the type or pass unsupported_callback" + "pickle is not allowed when type registration enabled, " + "Please register the type or pass unsupported_callback" ) diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py index be76af0d8d..2333ff7824 100644 --- a/python/pyfory/_registry.py +++ b/python/pyfory/_registry.py @@ -474,6 +474,8 @@ def get_typeinfo(self, cls, create=True): type_id = TypeId.NAMED_ENUM elif type(serializer) is PickleSerializer: type_id = PickleSerializer.PICKLE_TYPE_ID + elif isinstance(serializer, FunctionSerializer): + type_id = TypeId.NAMED_EXT elif isinstance(serializer, (ObjectSerializer, StatefulSerializer, ReduceSerializer)): type_id = TypeId.NAMED_EXT if not self.require_registration: @@ -497,8 +499,8 @@ def _create_serializer(self, cls): break else: if cls is types.FunctionType: - # Use PickleSerializer for function types (including lambdas) - serializer = PickleSerializer(self.fory, cls) + # Use FunctionSerializer for function types (including lambdas) + serializer = FunctionSerializer(self.fory, cls) elif dataclasses.is_dataclass(cls): serializer = DataClassSerializer(self.fory, cls) elif issubclass(cls, enum.Enum): diff --git a/python/pyfory/tests/test_function.py b/python/pyfory/tests/test_function.py index ad2e03c069..738ef69d62 100644 --- a/python/pyfory/tests/test_function.py +++ b/python/pyfory/tests/test_function.py @@ -26,7 +26,7 @@ def test_lambda_functions_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer # Simple lambda simple_lambda = lambda x: x * 2 # noqa: E731 @@ -64,7 +64,7 @@ def complex_function(a, b, c=10): # Register the necessary types for complex functions fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer # Test complex function serialized = fory.serialize(complex_function) @@ -79,7 +79,7 @@ def test_nested_functions_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer def outer_function(x): def inner_function(y): @@ -104,7 +104,7 @@ def test_local_class_serialization(): # Register the necessary types fory.register_type(tuple) fory.register_type(list) - fory.register_type(dict) + # dict is already registered by default with MapSerializer def create_local_class(): from dataclasses import dataclass diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 3f6cbcb498..1c7f739617 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -469,6 +469,8 @@ def test_pickle_fallback(): def test_unsupported_callback(): fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False) + # Test with functions that now have proper serialization support + # Functions should no longer be treated as unsupported def f1(x): return x @@ -478,10 +480,18 @@ def f2(x): obj1 = [1, True, f1, f2, {1: 2}] unsupported_objects = [] binary1 = fory.serialize(obj1, unsupported_callback=unsupported_objects.append) - assert len(unsupported_objects) == 2 - assert unsupported_objects == [f1, f2] + # Functions are now properly supported, so unsupported_objects should be empty + assert len(unsupported_objects) == 0 new_obj1 = fory.deserialize(binary1, unsupported_objects=unsupported_objects) - assert new_obj1 == obj1 + # Functions should roundtrip correctly + assert len(new_obj1) == len(obj1) + assert new_obj1[0] == obj1[0] # 1 + assert new_obj1[1] == obj1[1] # True + assert new_obj1[2](5) == f1(5) # Test f1 functionality + assert new_obj1[3](5) == f2(5) # Test f2 functionality + assert new_obj1[4] == obj1[4] # {1: 2} + # Don't check full equality since functions are new objects after deserialization + # The functionality test above already confirmed they work correctly def test_slice(): From ff42f2d1e14662e4ed42b7946bbf68bb80299d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 01:17:37 -0400 Subject: [PATCH 22/44] Restore install_bazel_windows --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a6071ac54..eeb6ea48ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -278,9 +278,14 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install bazel + - name: Install bazel (Unix) + if: runner.os != 'Windows' shell: bash run: python ./ci/run_ci.py cpp --install-deps-only + - name: Install bazel (Windows) + if: runner.os == 'Windows' + shell: bash + run: ./ci/run_ci.sh install_bazel_windows - name: Run Python CI shell: bash run: python ./ci/run_ci.py python From e5713c35c6ef98ec61578db3436dd31f0d3a5869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 01:24:24 -0400 Subject: [PATCH 23/44] E721: Use isinstance for type check --- python/pyfory/_fory.py | 2 +- python/setup.py | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index 831e24d3bc..c467a65ed1 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -345,7 +345,7 @@ def _deserialize( buffers: Iterable = None, unsupported_objects: Iterable = None, ): - if type(buffer) == bytes: + if isinstance(buffer, bytes): buffer = Buffer(buffer) if unsupported_objects is not None: self._unsupported_objects = iter(unsupported_objects) diff --git a/python/setup.py b/python/setup.py index 758a6735ad..d31f5544fc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -54,15 +54,6 @@ def __init__(self, attrs=None): bazel_args += ["//:cp_fory_so"] # Ensure Windows path compatibility cwd_path = os.path.normpath(project_dir) - # Force repository fetching by running fetch for our specific target - fetch_args = ["bazel", "fetch", "//:cp_fory_so"] - try: - print(f"Running bazel fetch to load repositories with cwd={cwd_path}") - subprocess.check_call(fetch_args, cwd=cwd_path) - except subprocess.CalledProcessError: - # If fetch fails, continue anyway as build might still work - pass - print(f"Running bazel build with cwd={cwd_path}: {' '.join(bazel_args)}") subprocess.check_call(bazel_args, cwd=cwd_path) def has_ext_modules(self): From a6231106fdb05f6eb6f625f383d715e09d3fa069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 01:29:26 -0400 Subject: [PATCH 24/44] fix formatting, integration test pom.xml --- integration_tests/jdk_compatibility_tests/pom.xml | 15 +++++++++++++++ python/pyfory/tests/test_serializer.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/integration_tests/jdk_compatibility_tests/pom.xml b/integration_tests/jdk_compatibility_tests/pom.xml index a2eb0f6b88..2464b080eb 100644 --- a/integration_tests/jdk_compatibility_tests/pom.xml +++ b/integration_tests/jdk_compatibility_tests/pom.xml @@ -38,11 +38,26 @@ + + org.apache.fory + fory-core + ${project.version} + + + org.apache.fory + fory-format + ${project.version} + org.apache.fory benchmark ${project.version} + + org.testng + testng + test + diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py index 1c7f739617..6df8875b4c 100644 --- a/python/pyfory/tests/test_serializer.py +++ b/python/pyfory/tests/test_serializer.py @@ -486,9 +486,9 @@ def f2(x): # Functions should roundtrip correctly assert len(new_obj1) == len(obj1) assert new_obj1[0] == obj1[0] # 1 - assert new_obj1[1] == obj1[1] # True + assert new_obj1[1] == obj1[1] # True assert new_obj1[2](5) == f1(5) # Test f1 functionality - assert new_obj1[3](5) == f2(5) # Test f2 functionality + assert new_obj1[3](5) == f2(5) # Test f2 functionality assert new_obj1[4] == obj1[4] # {1: 2} # Don't check full equality since functions are new objects after deserialization # The functionality test above already confirmed they work correctly From dc44a39177efa2c9908113795fafa663cd36deb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 01:35:55 -0400 Subject: [PATCH 25/44] reformat two files --- ci/tasks/java.py | 61 ++++++++++++++++++++++++------------------ python/pyfory/_fory.py | 31 +++++---------------- 2 files changed, 42 insertions(+), 50 deletions(-) diff --git a/ci/tasks/java.py b/ci/tasks/java.py index a705082fb8..6fa0754ed5 100644 --- a/ci/tasks/java.py +++ b/ci/tasks/java.py @@ -17,7 +17,6 @@ import logging import os -import sys import subprocess import re from . import common @@ -26,7 +25,7 @@ def get_jdk_major_version(): try: # Run the 'java -version' command - result = subprocess.run(['java', '-version'], capture_output=True, text=True) + result = subprocess.run(["java", "-version"], capture_output=True, text=True) output = result.stderr # java -version outputs to stderr # Use regex to find the version string @@ -37,8 +36,8 @@ def get_jdk_major_version(): version_string = match.group(1) # Parse the version string - version_parts = version_string.split('.') - if version_parts[0] == '1': + version_parts = version_string.split(".") + if version_parts[0] == "1": # Java 8 or earlier return int(version_parts[1]) else: @@ -81,54 +80,56 @@ def create_toolchains_xml(jdk_mappings): import os import xml.etree.ElementTree as ET from xml.dom import minidom - + # Create ~/.m2 directory if it doesn't exist m2_dir = os.path.expanduser("~/.m2") os.makedirs(m2_dir, exist_ok=True) - + # Create the root element toolchains = ET.Element("toolchains") - + for version, jdk_name in jdk_mappings.items(): toolchain = ET.SubElement(toolchains, "toolchain") - + # Set type type_elem = ET.SubElement(toolchain, "type") type_elem.text = "jdk" - + # Set provides provides = ET.SubElement(toolchain, "provides") version_elem = ET.SubElement(provides, "version") version_elem.text = version vendor_elem = ET.SubElement(provides, "vendor") vendor_elem.text = "azul" - + # Set configuration configuration = ET.SubElement(toolchain, "configuration") jdk_home = ET.SubElement(configuration, "jdkHome") jdk_home.text = os.path.abspath(os.path.join(common.PROJECT_ROOT_DIR, jdk_name)) - + # Create pretty XML string - rough_string = ET.tostring(toolchains, 'unicode') + rough_string = ET.tostring(toolchains, "unicode") reparsed = minidom.parseString(rough_string) pretty_xml = reparsed.toprettyxml(indent=" ") - + # Add proper XML header with encoding xml_header = '\n' - pretty_xml = xml_header + pretty_xml.split('\n', 1)[1] # Remove the default header and add our custom one - + pretty_xml = ( + xml_header + pretty_xml.split("\n", 1)[1] + ) # Remove the default header and add our custom one + # Write to ~/.m2/toolchains.xml toolchains_path = os.path.join(m2_dir, "toolchains.xml") - with open(toolchains_path, 'w', encoding='utf-8') as f: + with open(toolchains_path, "w", encoding="utf-8") as f: f.write(pretty_xml) - + logging.info(f"Created toolchains.xml at {toolchains_path}") logging.info("Toolchains configuration:") for version, jdk_name in jdk_mappings.items(): jdk_path = os.path.join(common.PROJECT_ROOT_DIR, jdk_name) logging.info(f" JDK {version}: {jdk_path}") # print toolchains.xml - with open(toolchains_path, 'r', encoding='utf-8') as f: + with open(toolchains_path, "r", encoding="utf-8") as f: logging.info(f.read()) @@ -149,7 +150,9 @@ def run_java8(): logging.info("Executing fory java tests with Java 8") install_jdks() common.cd_project_subdir("java") - common.exec_cmd("mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'") + common.exec_cmd( + "mvn -T16 --batch-mode --no-transfer-progress test -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("Executing fory java tests succeeds") @@ -208,7 +211,9 @@ def run_integration_tests(): logging.info("Executing fory integration tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("benchmark tests") common.cd_project_subdir("java/benchmark") @@ -270,7 +275,9 @@ def run_graalvm_test(): logging.info("Start GraalVM tests") common.cd_project_subdir("java") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'") + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean install -DskipTests -pl '!:fory-format,!:fory-testsuite'" + ) logging.info("Start to build graalvm native image") common.cd_project_subdir("integration_tests/graalvm_tests") @@ -285,17 +292,19 @@ def run_graalvm_test(): def run_release(): """Release to Maven Central.""" - logging.info(f"Starting release to Maven Central with Java") + logging.info("Starting release to Maven Central with Java") common.cd_project_subdir("java") - + # Clean and install without tests first logging.info("Cleaning and installing dependencies") common.exec_cmd("mvn -T10 -B --no-transfer-progress clean install -DskipTests") - + # Deploy to Maven Central logging.info("Deploying to Maven Central") - common.exec_cmd("mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release") - + common.exec_cmd( + "mvn -T10 -B --no-transfer-progress clean deploy -Dgpg.skip -DskipTests -Papache-release" + ) + logging.info("Release to Maven Central completed successfully") diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py index c467a65ed1..6bc87ba1a9 100644 --- a/python/pyfory/_fory.py +++ b/python/pyfory/_fory.py @@ -133,9 +133,7 @@ def __init__( """ self.language = language self.is_py = language == Language.PYTHON - self.require_type_registration = ( - _ENABLE_TYPE_REGISTRATION_FORCIBLY or require_type_registration - ) + self.require_type_registration = _ENABLE_TYPE_REGISTRATION_FORCIBLY or require_type_registration self.ref_tracking = ref_tracking if self.ref_tracking: self.ref_resolver = MapRefResolver() @@ -151,8 +149,7 @@ def __init__( self.buffer = Buffer.allocate(32) if not require_type_registration: warnings.warn( - "Type registration is disabled, unknown types can be deserialized " - "which may be insecure.", + "Type registration is disabled, unknown types can be deserialized which may be insecure.", RuntimeWarning, stacklevel=2, ) @@ -360,10 +357,7 @@ def _deserialize( if get_bit(buffer, reader_index, 0): return None is_little_endian_ = get_bit(buffer, reader_index, 1) - assert is_little_endian_, ( - "Big endian is not supported for now, " - "please ensure peer machine is little endian." - ) + assert is_little_endian_, "Big endian is not supported for now, please ensure peer machine is little endian." is_target_x_lang = get_bit(buffer, reader_index, 2) if is_target_x_lang: self._peer_language = Language(buffer.read_int8()) @@ -371,16 +365,10 @@ def _deserialize( self._peer_language = Language.PYTHON is_out_of_band_serialization_enabled = get_bit(buffer, reader_index, 3) if is_out_of_band_serialization_enabled: - assert buffers is not None, ( - "buffers shouldn't be null when the serialized stream is " - "produced with buffer_callback not null." - ) + assert buffers is not None, "buffers shouldn't be null when the serialized stream is produced with buffer_callback not null." self._buffers = iter(buffers) else: - assert buffers is None, ( - "buffers should be null when the serialized stream is " - "produced with buffer_callback null." - ) + assert buffers is None, "buffers should be null when the serialized stream is produced with buffer_callback null." if is_target_x_lang: obj = self.xdeserialize_ref(buffer) else: @@ -532,9 +520,7 @@ def reset(self): self.objects.clear() -_ENABLE_TYPE_REGISTRATION_FORCIBLY = os.getenv( - "ENABLE_TYPE_REGISTRATION_FORCIBLY", "0" -) in { +_ENABLE_TYPE_REGISTRATION_FORCIBLY = os.getenv("ENABLE_TYPE_REGISTRATION_FORCIBLY", "0") in { "1", "true", } @@ -554,7 +540,4 @@ def clear_memo(self): class _UnpicklerStub: def load(self): - raise ValueError( - "pickle is not allowed when type registration enabled, " - "Please register the type or pass unsupported_callback" - ) + raise ValueError("pickle is not allowed when type registration enabled, Please register the type or pass unsupported_callback") From fc4e557db11e8f1369abd6e259ec804f0194ce50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emre=20=C5=9Eafak?= <3928300+esafak@users.noreply.github.com> Date: Tue, 2 Sep 2025 01:48:17 -0400 Subject: [PATCH 26/44] reformat one file --- .../fory/graalvm/ObjectStreamExample.java | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java index a73c90bdf5..227ce126d4 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ObjectStreamExample.java @@ -17,42 +17,42 @@ * under the License. */ - package org.apache.fory.graalvm; - -import org.apache.fory.Fory; +package org.apache.fory.graalvm; import java.util.AbstractMap; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import org.apache.fory.Fory; public class ObjectStreamExample extends AbstractMap { - private static final Fory FORY = Fory.builder() - .withName(ObjectStreamExample.class.getName()) - .registerGuavaTypes(false) - .build(); + private static final Fory FORY = + Fory.builder() + .withName(ObjectStreamExample.class.getName()) + .registerGuavaTypes(false) + .build(); static { - FORY.register(ObjectStreamExample.class, true); - FORY.ensureSerializersCompiled(); + FORY.register(ObjectStreamExample.class, true); + FORY.ensureSerializersCompiled(); } final int[] ints = new int[10]; public static void main(String[] args) { - FORY.reset(); - byte[] bytes = FORY.serialize(new ObjectStreamExample()); - FORY.reset(); - ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes); - System.out.println(Arrays.toString(o.ints)); + FORY.reset(); + byte[] bytes = FORY.serialize(new ObjectStreamExample()); + FORY.reset(); + ObjectStreamExample o = (ObjectStreamExample) FORY.deserialize(bytes); + System.out.println(Arrays.toString(o.ints)); } @Override public Set> entrySet() { - HashSet> set = new HashSet<>(); - for (int i = 0; i < ints.length; i++) { - set.add(new AbstractMap.SimpleEntry<>(i, ints[i])); - } - return set; + HashSet> set = new HashSet<>(); + for (int i = 0; i < ints.length; i++) { + set.add(new AbstractMap.SimpleEntry<>(i, ints[i])); + } + return set; } -} \ No newline at end of file +} From 6af6b236735a01eda5e6ca0e75fa4a86c3ba785d Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 14:33:28 +0800 Subject: [PATCH 27/44] fix graalvm init error --- .../org/apache/fory/builder/CodecBuilder.java | 4 ++ .../apache/fory/resolver/ClassResolver.java | 45 ++++++++++++++----- .../serializer/ObjectStreamSerializer.java | 4 +- .../apache/fory/serializer/Serializers.java | 27 +++++++++++ .../org/apache/fory/util/GraalvmSupport.java | 17 +++++++ .../fory-core/native-image.properties | 4 ++ 6 files changed, 89 insertions(+), 12 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java index cf37d8ff7a..eefb7212b7 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/CodecBuilder.java @@ -257,6 +257,10 @@ private Expression getRecordFieldValue(Expression inputBeanExpr, Descriptor desc if (ref == null) { Class funcInterface = methodInfo.f0; TypeRef getterType = TypeRef.of(funcInterface); + if (GraalvmSupport.isGraalBuildtime()) { + // generate getter ahead at native image build time. + Functions.makeGetterFunction(beanClass, fieldName); + } Expression getter = new StaticInvoke( Functions.class, diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 54895c5929..ab22c7a411 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -87,7 +87,8 @@ import org.apache.fory.annotation.CodegenInvoke; import org.apache.fory.annotation.Internal; import org.apache.fory.builder.CodecUtils; -import org.apache.fory.builder.Generated; +import org.apache.fory.builder.Generated.GeneratedMetaSharedSerializer; +import org.apache.fory.builder.Generated.GeneratedObjectSerializer; import org.apache.fory.builder.JITContext; import org.apache.fory.codegen.CodeGenerator; import org.apache.fory.codegen.Expression; @@ -163,6 +164,7 @@ import org.apache.fory.type.TypeUtils; import org.apache.fory.type.Types; import org.apache.fory.util.GraalvmSupport; +import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.Functions; @@ -1423,7 +1425,7 @@ private void createSerializer0(Class cls) { if (deserializationClassInfo != null && GraalvmSupport.isGraalBuildtime()) { getGraalvmClassRegistry() .deserializerClassMap - .put(classDef.getId(), deserializationClassInfo.serializer.getClass()); + .put(classDef.getId(), getGraalvmSerializerClass(deserializationClassInfo.serializer)); Tuple2 classDefTuple = extRegistry.classIdToDef.get(classDef.getId()); // empty serializer for graalvm build time classDefTuple.f1.serializer = null; @@ -1432,7 +1434,9 @@ private void createSerializer0(Class cls) { } if (GraalvmSupport.isGraalBuildtime()) { // Instance for generated class should be hold at graalvm runtime only. - getGraalvmClassRegistry().serializerClassMap.put(cls, classInfo.serializer.getClass()); + getGraalvmClassRegistry() + .serializerClassMap + .put(cls, getGraalvmSerializerClass(classInfo.serializer)); classInfo.serializer = null; } } @@ -1560,13 +1564,21 @@ private ClassDef buildClassDef(ClassInfo classInfo) { } boolean needToWriteClassDef(Serializer serializer) { - return fory.getConfig().getCompatibleMode() == CompatibleMode.COMPATIBLE - && (serializer instanceof Generated.GeneratedObjectSerializer - // May already switched to MetaSharedSerializer when update class info cache. - || serializer instanceof Generated.GeneratedMetaSharedSerializer - || serializer instanceof LazyInitBeanSerializer - || serializer instanceof ObjectSerializer - || serializer instanceof MetaSharedSerializer); + if (fory.getConfig().getCompatibleMode() != CompatibleMode.COMPATIBLE) { + return false; + } + if (GraalvmSupport.isGraalBuildtime() && serializer instanceof GraalvmSerializerHolder) { + Class serializerClass = + ((GraalvmSerializerHolder) serializer).getSerializerClass(); + return GeneratedObjectSerializer.class.isAssignableFrom(serializerClass) + || GeneratedMetaSharedSerializer.class.isAssignableFrom(serializerClass); + } + return (serializer instanceof GeneratedObjectSerializer + // May already switched to MetaSharedSerializer when update class info cache. + || serializer instanceof GeneratedMetaSharedSerializer + || serializer instanceof LazyInitBeanSerializer + || serializer instanceof ObjectSerializer + || serializer instanceof MetaSharedSerializer); } private ClassInfo readClassInfoWithMetaShare(MemoryBuffer buffer, MetaContext metaContext) { @@ -2212,6 +2224,7 @@ public Fory getFory() { */ public void ensureSerializersCompiled() { try { + fory.getJITContext().lock(); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { @@ -2262,6 +2275,13 @@ private GraalvmClassRegistry getGraalvmClassRegistry() { fory.getConfig().getConfigHash(), k -> new GraalvmClassRegistry()); } + private Class getGraalvmSerializerClass(Serializer serializer) { + if (serializer instanceof GraalvmSerializerHolder) { + return ((GraalvmSerializerHolder) serializer).getSerializerClass(); + } + return serializer.getClass(); + } + private Class getSerializerClassFromGraalvmRegistry(Class cls) { GraalvmClassRegistry registry = getGraalvmClassRegistry(); List classResolvers = registry.resolvers; @@ -2307,7 +2327,10 @@ private Class getMetaSharedDeserializerClassFromGraalvmReg if (Functions.isLambda(cls) || ReflectionUtils.isJdkProxy(cls)) { return null; } - throw new RuntimeException(String.format("Class %s is not registered", cls)); + throw new RuntimeException( + String.format( + "Class %s is not registered, registered classes: %s", + cls, registry.deserializerClassMap)); } return null; } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java index b976046fca..ef672d9b73 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java @@ -61,6 +61,7 @@ import org.apache.fory.resolver.FieldResolver; import org.apache.fory.resolver.FieldResolver.ClassField; import org.apache.fory.util.ExceptionUtils; +import org.apache.fory.util.GraalvmSupport; import org.apache.fory.util.Preconditions; import org.apache.fory.util.unsafe._JDKAccess; @@ -354,7 +355,8 @@ public SlotsInfo(Fory fory, Class type) { this.slotsSerializer = (CompatibleSerializerBase) Serializers.newSerializer(fory, type, c)); } - if (sc == CompatibleSerializer.class) { + if (sc == CompatibleSerializer.class || GraalvmSupport.isGraalBuildtime()) { + // skip init generated serializer at graalvm build time this.slotsSerializer = new CompatibleSerializer(fory, type, fieldResolver); } else { this.slotsSerializer = (CompatibleSerializerBase) Serializers.newSerializer(fory, type, sc); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java index ce936d7405..c1719b64bb 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializers.java @@ -43,6 +43,7 @@ import java.util.function.ToIntFunction; import java.util.regex.Pattern; import org.apache.fory.Fory; +import org.apache.fory.builder.Generated; import org.apache.fory.collection.Tuple2; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; @@ -50,6 +51,7 @@ import org.apache.fory.resolver.ClassResolver; import org.apache.fory.util.ExceptionUtils; import org.apache.fory.util.GraalvmSupport; +import org.apache.fory.util.GraalvmSupport.GraalvmSerializerHolder; import org.apache.fory.util.StringUtils; import org.apache.fory.util.unsafe._JDKAccess; @@ -88,6 +90,11 @@ public static Serializer newSerializer( } Tuple2 ctrInfo = CTR_MAP.getIfPresent(serializerClass); if (ctrInfo != null) { + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } MethodType sig = ctrInfo.f0; MethodHandle handle = ctrInfo.f1; if (sig.equals(SIG1)) { @@ -125,6 +132,11 @@ private static Serializer createSerializer( try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG1); CTR_MAP.put(serializerClass, Tuple2.of(SIG1, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer) ctr.invoke(fory, type); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); @@ -132,6 +144,11 @@ private static Serializer createSerializer( try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG2); CTR_MAP.put(serializerClass, Tuple2.of(SIG2, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer) ctr.invoke(fory); } catch (NoSuchMethodException e) { ExceptionUtils.ignore(e); @@ -139,10 +156,20 @@ private static Serializer createSerializer( try { MethodHandle ctr = lookup.findConstructor(serializerClass, SIG3); CTR_MAP.put(serializerClass, Tuple2.of(SIG3, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer) ctr.invoke(type); } catch (NoSuchMethodException e) { MethodHandle ctr = ReflectionUtils.getCtrHandle(serializerClass); CTR_MAP.put(serializerClass, Tuple2.of(SIG4, ctr)); + if (GraalvmSupport.isGraalBuildtime()) { + if (Generated.class.isAssignableFrom(serializerClass)) { + return new GraalvmSerializerHolder(fory, type, serializerClass); + } + } return (Serializer) ctr.invoke(); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java index 55d0239812..edf00db991 100644 --- a/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java +++ b/java/fory-core/src/main/java/org/apache/fory/util/GraalvmSupport.java @@ -19,6 +19,10 @@ package org.apache.fory.util; +import java.util.Objects; +import org.apache.fory.Fory; +import org.apache.fory.serializer.Serializer; + /** A helper for Graalvm native image support. */ public class GraalvmSupport { // https://github.com/oracle/graal/blob/master/sdk/src/org.graalvm.nativeimage/src/org/graalvm/nativeimage/ImageInfo.java @@ -46,4 +50,17 @@ public static boolean isGraalRuntime() { return IN_GRAALVM_NATIVE_IMAGE && GRAAL_IMAGE_RUNTIME.equals(System.getProperty(GRAAL_IMAGE_CODE_KEY)); } + + public static class GraalvmSerializerHolder extends Serializer { + private final Class serializerClass; + + public GraalvmSerializerHolder(Fory fory, Class type, Class serializerClass) { + super(fory, type); + this.serializerClass = Objects.requireNonNull(serializerClass); + } + + public Class getSerializerClass() { + return serializerClass; + } + } } diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 1ae683c105..c63aabfa62 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -435,6 +435,10 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.LazySerializer$LazyObjectSerializer,\ org.apache.fory.serializer.shim.ShimDispatcher,\ org.apache.fory.serializer.shim.ProtobufDispatcher,\ + org.apache.fory.serializer.ObjectStreamSerializer,\ + org.apache.fory.serializer.ObjectStreamSerializer$StreamClassInfo,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ + org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ org.apache.fory.shaded.org.codehaus.janino.IClass$1,\ org.apache.fory.type.Descriptor$1,\ org.apache.fory.type.Descriptor,\ From d80532491acabf60e359ec65612a3a0dcc10954c Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 14:37:59 +0800 Subject: [PATCH 28/44] fix ObjectStreamSerializer$1 --- .../org.apache.fory/fory-core/native-image.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index c63aabfa62..27f55c6a68 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -436,6 +436,7 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.shim.ShimDispatcher,\ org.apache.fory.serializer.shim.ProtobufDispatcher,\ org.apache.fory.serializer.ObjectStreamSerializer,\ + org.apache.fory.serializer.ObjectStreamSerializer$1,\ org.apache.fory.serializer.ObjectStreamSerializer$StreamClassInfo,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildCollectionSerializer,\ org.apache.fory.serializer.collection.ChildContainerSerializers$ChildMapSerializer,\ From c9fcb508865a14952e161d493357c56aa9cfdfe1 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 14:53:33 +0800 Subject: [PATCH 29/44] fix lambda init --- .../src/main/java/org/apache/fory/resolver/ClassResolver.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index ab22c7a411..4fc14c5416 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -2225,6 +2225,8 @@ public Fory getFory() { public void ensureSerializersCompiled() { try { fory.getJITContext().lock(); + Function lambda = x -> x * 2; + Serializers.newSerializer(fory, lambda.getClass(), LambdaSerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { From b79b206a36cdd5984a4619b2c1ee0e245a1180ac Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 14:58:57 +0800 Subject: [PATCH 30/44] use serializable func --- .../src/main/java/org/apache/fory/resolver/ClassResolver.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 4fc14c5416..1c717efaab 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -168,6 +168,7 @@ import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.Functions; +import org.apache.fory.util.function.SerializableFunction; /** * Class registry for types of serializing objects, responsible for reading/writing types, setting @@ -2225,7 +2226,7 @@ public Fory getFory() { public void ensureSerializersCompiled() { try { fory.getJITContext().lock(); - Function lambda = x -> x * 2; + SerializableFunction lambda = x -> x * 2; Serializers.newSerializer(fory, lambda.getClass(), LambdaSerializer.class); classInfoMap.forEach( (cls, classInfo) -> { From 9cb0a299dd0a984db25c653896afed0c2ef308d8 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 15:27:21 +0800 Subject: [PATCH 31/44] fix lmabda lazy init --- .../apache/fory/resolver/ClassResolver.java | 4 +- .../fory/serializer/LambdaSerializer.java | 70 ++++++++++++++----- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 1c717efaab..4e7183679d 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -168,7 +168,6 @@ import org.apache.fory.util.Preconditions; import org.apache.fory.util.StringUtils; import org.apache.fory.util.function.Functions; -import org.apache.fory.util.function.SerializableFunction; /** * Class registry for types of serializing objects, responsible for reading/writing types, setting @@ -2226,8 +2225,7 @@ public Fory getFory() { public void ensureSerializersCompiled() { try { fory.getJITContext().lock(); - SerializableFunction lambda = x -> x * 2; - Serializers.newSerializer(fory, lambda.getClass(), LambdaSerializer.class); + Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS, LambdaSerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java index 2edfafeac4..835c0ce6d1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java @@ -21,13 +21,18 @@ import java.io.ObjectStreamClass; import java.io.Serializable; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; import java.lang.invoke.SerializedLambda; -import java.lang.reflect.Method; import java.util.Objects; import org.apache.fory.Fory; +import org.apache.fory.collection.ClassValueCache; +import org.apache.fory.exception.ForyException; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.reflect.ReflectionUtils; import org.apache.fory.util.Preconditions; +import org.apache.fory.util.function.SerializableFunction; +import org.apache.fory.util.unsafe._JDKAccess; /** * Serializer for java serializable lambda. Use fory to serialize java lambda instead of JDK @@ -36,20 +41,34 @@ */ @SuppressWarnings({"unchecked", "rawtypes"}) public class LambdaSerializer extends Serializer { + public static Class STUB_LAMBDA_CLASS = + ((SerializableFunction) (x -> x * 2)).getClass(); private static final Class SERIALIZED_LAMBDA = SerializedLambda.class; - private static final Method READ_RESOLVE_METHOD = - (Method) - Objects.requireNonNull( - ReflectionUtils.getObjectFieldValue( - ObjectStreamClass.lookup(SERIALIZED_LAMBDA), "readResolveMethod")); + private static final MethodHandle READ_RESOLVE_HANDLE; private static final boolean SERIALIZED_LAMBDA_HAS_JDK_WRITE = JavaSerializer.getWriteObjectMethod(SERIALIZED_LAMBDA) != null; private static final boolean SERIALIZED_LAMBDA_HAS_JDK_READ = JavaSerializer.getReadObjectMethod(SERIALIZED_LAMBDA) != null; - private final Method writeReplaceMethod; + private static final ClassValueCache writeReplaceMethodCache = + ClassValueCache.newClassKeySoftCache(32); + + private final MethodHandle writeReplaceHandle; private Serializer dataSerializer; - public LambdaSerializer(Fory fory, Class cls) { + static { + try { + // Initialize READ_RESOLVE_HANDLE + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(SERIALIZED_LAMBDA); + Object readResolveMethod = + ReflectionUtils.getObjectFieldValue( + ObjectStreamClass.lookup(SERIALIZED_LAMBDA), "readResolveMethod"); + READ_RESOLVE_HANDLE = lookup.unreflect((java.lang.reflect.Method) readResolveMethod); + } catch (IllegalAccessException e) { + throw new ForyException(e); + } + } + + public LambdaSerializer(Fory fory, Class cls) { super(fory, cls); if (cls != ReplaceStub.class) { if (!Serializable.class.isAssignableFrom(cls)) { @@ -59,12 +78,27 @@ public LambdaSerializer(Fory fory, Class cls) { cls, Serializable.class.getName()); throw new UnsupportedOperationException(msg); } - writeReplaceMethod = - (Method) + MethodHandle methodHandle = writeReplaceMethodCache.getIfPresent(cls); + if (methodHandle == null) { + try { + MethodHandles.Lookup lookup = _JDKAccess._trustedLookup(cls); + Object writeReplaceMethod = ReflectionUtils.getObjectFieldValue( ObjectStreamClass.lookup(cls), "writeReplaceMethod"); + methodHandle = + lookup.unreflect( + (java.lang.reflect.Method) Objects.requireNonNull(writeReplaceMethod)); + writeReplaceMethodCache.put(cls, methodHandle); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to create writeReplace MethodHandle", e); + } + } + writeReplaceHandle = methodHandle; } else { - writeReplaceMethod = null; + writeReplaceHandle = null; + } + if (cls == STUB_LAMBDA_CLASS) { + getDataSerializer(); } } @@ -72,10 +106,10 @@ public LambdaSerializer(Fory fory, Class cls) { public void write(MemoryBuffer buffer, Object value) { assert value.getClass() != ReplaceStub.class; try { - Object replacement = writeReplaceMethod.invoke(value); + Object replacement = writeReplaceHandle.invoke(value); Preconditions.checkArgument(SERIALIZED_LAMBDA.isInstance(replacement)); getDataSerializer().write(buffer, replacement); - } catch (Exception e) { + } catch (Throwable e) { throw new RuntimeException("Can't serialize lambda " + value, e); } } @@ -83,10 +117,10 @@ public void write(MemoryBuffer buffer, Object value) { @Override public Object copy(Object value) { try { - Object replacement = writeReplaceMethod.invoke(value); + Object replacement = writeReplaceHandle.invoke(value); Object newReplacement = getDataSerializer().copy(replacement); - return READ_RESOLVE_METHOD.invoke(newReplacement); - } catch (Exception e) { + return READ_RESOLVE_HANDLE.invoke(newReplacement); + } catch (Throwable e) { throw new RuntimeException("Can't copy lambda " + value, e); } } @@ -95,8 +129,8 @@ public Object copy(Object value) { public Object read(MemoryBuffer buffer) { try { Object replacement = getDataSerializer().read(buffer); - return READ_RESOLVE_METHOD.invoke(replacement); - } catch (Exception e) { + return READ_RESOLVE_HANDLE.invoke(replacement); + } catch (Throwable e) { throw new RuntimeException("Can't deserialize lambda", e); } } From fb2da8f48b7e40a69e2dd70b6dd676f67fc8e8ff Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 15:34:01 +0800 Subject: [PATCH 32/44] register lamdba --- .../src/main/java/org/apache/fory/resolver/ClassResolver.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 4e7183679d..03a83b4616 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -37,6 +37,7 @@ import java.io.Externalizable; import java.io.IOException; import java.io.Serializable; +import java.lang.invoke.SerializedLambda; import java.lang.reflect.Field; import java.lang.reflect.Member; import java.lang.reflect.Type; @@ -422,6 +423,7 @@ private void registerDefaultClasses() { register(AtomicReference.class); register(EnumSet.allOf(Language.class).getClass()); register(EnumSet.of(Language.JAVA).getClass()); + register(SerializedLambda.class); register(Throwable.class, StackTraceElement.class, Exception.class, RuntimeException.class); register(NullPointerException.class); register(IOException.class); From dd1afa63d48e3c71d5ddf3595a4a56a174b980c9 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 15:39:51 +0800 Subject: [PATCH 33/44] ensureSerializersCompiled for proxy --- .../src/main/java/org/apache/fory/graalvm/ProxyExample.java | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java index 990faaa2b3..2fd1956751 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java @@ -50,6 +50,7 @@ private static Fory createFory() { .build(); // register and generate serializer code. fory.register(TestInvocationHandler.class, true); + fory.ensureSerializersCompiled(); return fory; } From 7e452c2f19a9c23ad1d1f677afb0f082acf33283 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 15:49:44 +0800 Subject: [PATCH 34/44] refine exception msg for lambda serialzier --- .../java/org/apache/fory/serializer/LambdaSerializer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java index 835c0ce6d1..efa80b7724 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/LambdaSerializer.java @@ -89,8 +89,9 @@ public LambdaSerializer(Fory fory, Class cls) { lookup.unreflect( (java.lang.reflect.Method) Objects.requireNonNull(writeReplaceMethod)); writeReplaceMethodCache.put(cls, methodHandle); - } catch (IllegalAccessException e) { - throw new RuntimeException("Failed to create writeReplace MethodHandle", e); + } catch (Throwable e) { + throw new RuntimeException( + String.format("Failed to create writeReplace MethodHandle for %s", cls), e); } } writeReplaceHandle = methodHandle; From 5bb7521cd1bf6ae29ea1e6cc4a92c44f9edccb3b Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 16:31:40 +0800 Subject: [PATCH 35/44] init proxy serializer --- .../main/java/org/apache/fory/resolver/ClassResolver.java | 2 ++ .../java/org/apache/fory/serializer/JdkProxySerializer.java | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 03a83b4616..ab24d49498 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -2228,6 +2228,8 @@ public void ensureSerializersCompiled() { try { fory.getJITContext().lock(); Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS, LambdaSerializer.class); + Serializers.newSerializer( + fory, JdkProxySerializer.SUBT_PROXY.getClass(), JdkProxySerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index ac1dd80c5b..b7abf08c4a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -22,6 +22,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Proxy; +import java.util.function.Function; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; @@ -32,7 +33,6 @@ /** Serializer for jdk {@link Proxy}. */ @SuppressWarnings({"rawtypes", "unchecked"}) public class JdkProxySerializer extends Serializer { - // Make offset compatible with graalvm native image. private static final Field FIELD; private static final long PROXY_HANDLER_FIELD_OFFSET; @@ -46,6 +46,9 @@ public class JdkProxySerializer extends Serializer { (proxy, method, args) -> { throw new IllegalStateException("Deserialization stub handler still active"); }; + public static Object SUBT_PROXY = + Proxy.newProxyInstance( + Serializer.class.getClassLoader(), new Class[] {Function.class}, STUB_HANDLER); public JdkProxySerializer(Fory fory, Class cls) { super(fory, cls); From cc7c9245da233da7a2f44eb3a354be14fcd3a151 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 16:40:28 +0800 Subject: [PATCH 36/44] move proxy at static --- .../java/org/apache/fory/graalvm/ProxyExample.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java index 2fd1956751..67370ac6f3 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java @@ -36,6 +36,11 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl } } + static Function function = + (Function) + Proxy.newProxyInstance( + TestInvocationHandler.class.getClassLoader(), new Class[] {Function.class}, new TestInvocationHandler()); + static Fory fory; static { @@ -55,10 +60,7 @@ private static Fory createFory() { } public static void main(String[] args) { - Function function = - (Function) - Proxy.newProxyInstance( - fory.getClassLoader(), new Class[] {Function.class}, new TestInvocationHandler()); + Function deserializedFunction = (Function) fory.deserialize(fory.serialize(function)); Preconditions.checkArgument(deserializedFunction.apply(null).equals(1)); fory = createFory(); From f3f27595975b0461c760d371e028fae476cc8ef4 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 16:44:17 +0800 Subject: [PATCH 37/44] add TestInvocationHandler for build time init --- .../org.apache.fory/graalvm_tests/native-image.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties index 35d26127a2..a172028d52 100644 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties @@ -27,6 +27,7 @@ Args=-H:+ReportExceptionStackTraces \ org.apache.fory.graalvm.ThreadSafeExample,\ org.apache.fory.graalvm.CompatibleThreadSafeExample,\ org.apache.fory.graalvm.ProxyExample,\ + org.apache.fory.graalvm.ProxyExample$TestInvocationHandler,\ org.apache.fory.graalvm.ObjectStreamExample,\ org.apache.fory.graalvm.CollectionExample,\ org.apache.fory.graalvm.Benchmark From 610ca017e1a08bb744a197a7edc8404a25055617 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 16:55:45 +0800 Subject: [PATCH 38/44] fix proxy --- .../java/org/apache/fory/graalvm/ProxyExample.java | 10 ++++------ .../graalvm_tests/native-image.properties | 1 - .../java/org/apache/fory/resolver/ClassResolver.java | 2 +- .../org/apache/fory/serializer/JdkProxySerializer.java | 5 ++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java index 67370ac6f3..2fd1956751 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ProxyExample.java @@ -36,11 +36,6 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl } } - static Function function = - (Function) - Proxy.newProxyInstance( - TestInvocationHandler.class.getClassLoader(), new Class[] {Function.class}, new TestInvocationHandler()); - static Fory fory; static { @@ -60,7 +55,10 @@ private static Fory createFory() { } public static void main(String[] args) { - + Function function = + (Function) + Proxy.newProxyInstance( + fory.getClassLoader(), new Class[] {Function.class}, new TestInvocationHandler()); Function deserializedFunction = (Function) fory.deserialize(fory.serialize(function)); Preconditions.checkArgument(deserializedFunction.apply(null).equals(1)); fory = createFory(); diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties index a172028d52..35d26127a2 100644 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/native-image.properties @@ -27,7 +27,6 @@ Args=-H:+ReportExceptionStackTraces \ org.apache.fory.graalvm.ThreadSafeExample,\ org.apache.fory.graalvm.CompatibleThreadSafeExample,\ org.apache.fory.graalvm.ProxyExample,\ - org.apache.fory.graalvm.ProxyExample$TestInvocationHandler,\ org.apache.fory.graalvm.ObjectStreamExample,\ org.apache.fory.graalvm.CollectionExample,\ org.apache.fory.graalvm.Benchmark diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index ab24d49498..26962c6e67 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -2229,7 +2229,7 @@ public void ensureSerializersCompiled() { fory.getJITContext().lock(); Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS, LambdaSerializer.class); Serializers.newSerializer( - fory, JdkProxySerializer.SUBT_PROXY.getClass(), JdkProxySerializer.class); + fory, JdkProxySerializer.SUBT_PROXY_CLASS, JdkProxySerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index b7abf08c4a..972a7f6f8c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -46,9 +46,8 @@ public class JdkProxySerializer extends Serializer { (proxy, method, args) -> { throw new IllegalStateException("Deserialization stub handler still active"); }; - public static Object SUBT_PROXY = - Proxy.newProxyInstance( - Serializer.class.getClassLoader(), new Class[] {Function.class}, STUB_HANDLER); + public static Class SUBT_PROXY_CLASS = + Proxy.getProxyClass(Serializer.class.getClassLoader(), Function.class); public JdkProxySerializer(Fory fory, Class cls) { super(fory, cls); From ea85b81d04b4cc2aa048b9adb3663afd770bca37 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 16:59:55 +0800 Subject: [PATCH 39/44] fix proxy serializer init --- .../apache/fory/resolver/ClassResolver.java | 2 +- .../fory/serializer/JdkProxySerializer.java | 24 +++++++++++++------ .../fory-core/native-image.properties | 2 ++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index 26962c6e67..ab24d49498 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -2229,7 +2229,7 @@ public void ensureSerializersCompiled() { fory.getJITContext().lock(); Serializers.newSerializer(fory, LambdaSerializer.STUB_LAMBDA_CLASS, LambdaSerializer.class); Serializers.newSerializer( - fory, JdkProxySerializer.SUBT_PROXY_CLASS, JdkProxySerializer.class); + fory, JdkProxySerializer.SUBT_PROXY.getClass(), JdkProxySerializer.class); classInfoMap.forEach( (cls, classInfo) -> { if (classInfo.serializer == null) { diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index 972a7f6f8c..c792d3cbdc 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -21,8 +21,8 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.util.function.Function; import org.apache.fory.Fory; import org.apache.fory.memory.MemoryBuffer; import org.apache.fory.memory.Platform; @@ -42,12 +42,22 @@ public class JdkProxySerializer extends Serializer { PROXY_HANDLER_FIELD_OFFSET = Platform.objectFieldOffset(FIELD); } - private static final InvocationHandler STUB_HANDLER = - (proxy, method, args) -> { - throw new IllegalStateException("Deserialization stub handler still active"); - }; - public static Class SUBT_PROXY_CLASS = - Proxy.getProxyClass(Serializer.class.getClassLoader(), Function.class); + private static class StubInvocationHandler implements InvocationHandler { + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + throw new IllegalStateException("Deserialization stub handler still active"); + } + } + + private static final InvocationHandler STUB_HANDLER = new StubInvocationHandler(); + + private interface StubInterface { + int f(); + } + + public static Object SUBT_PROXY = + Proxy.newProxyInstance( + Serializer.class.getClassLoader(), new Class[] {StubInterface.class}, STUB_HANDLER); public JdkProxySerializer(Fory fory, Class cls) { super(fory, cls); diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties index 27f55c6a68..c9dc1b45d7 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/native-image.properties @@ -310,6 +310,8 @@ Args=--initialize-at-build-time=org.apache.fory.memory.MemoryBuffer,\ org.apache.fory.serializer.JavaSerializer$4,\ org.apache.fory.serializer.JavaSerializer,\ org.apache.fory.serializer.JdkProxySerializer$ReplaceStub,\ + org.apache.fory.serializer.JdkProxySerializer$StubInterface,\ + org.apache.fory.serializer.JdkProxySerializer$StubInvocationHandler,\ org.apache.fory.serializer.JdkProxySerializer,\ org.apache.fory.serializer.LambdaSerializer$ReplaceStub,\ org.apache.fory.serializer.LambdaSerializer,\ From 1cfbb479d5aa818cd4760fdf3e3c2730f7a16f32 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 17:59:57 +0800 Subject: [PATCH 40/44] add serialization-config.json --- .../fory-core/serialization-config.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json new file mode 100644 index 0000000000..be37412a48 --- /dev/null +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json @@ -0,0 +1,11 @@ +{ + "types": [ + { + "name": "java.lang.reflect.Proxy", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true + } + ] +} From ae937d9a417946882c16ba8fdf9858883909f922 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 18:27:49 +0800 Subject: [PATCH 41/44] fix config --- .../fory/serializer/JdkProxySerializer.java | 2 +- .../fory-core/reflection-config.json | 9 ++++++++ .../fory-core/serialization-config.json | 22 +++++++++---------- 3 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java index c792d3cbdc..9c2bc7418c 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/JdkProxySerializer.java @@ -52,7 +52,7 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl private static final InvocationHandler STUB_HANDLER = new StubInvocationHandler(); private interface StubInterface { - int f(); + int apply(); } public static Object SUBT_PROXY = diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json new file mode 100644 index 0000000000..505934bf94 --- /dev/null +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/reflection-config.json @@ -0,0 +1,9 @@ +[ + { + "name": "java.lang.reflect.Proxy", + "allDeclaredConstructors": true, + "allPublicConstructors": true, + "allDeclaredMethods": true, + "allPublicMethods": true + } +] diff --git a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json index be37412a48..154ba0da21 100644 --- a/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json +++ b/java/fory-core/src/main/resources/META-INF/native-image/org.apache.fory/fory-core/serialization-config.json @@ -1,11 +1,11 @@ -{ - "types": [ - { - "name": "java.lang.reflect.Proxy", - "allDeclaredConstructors": true, - "allPublicConstructors": true, - "allDeclaredMethods": true, - "allPublicMethods": true - } - ] -} +[ + { + "name": "java.lang.reflect.Proxy", + "fields": [], + "methods": [ + { "name": "" }, + { "name": "writeReplace" } + ], + "customTargetConstructorClass": "java.lang.Object" + } +] From 07b37ded6a2e1423902cb09e87cf800d225b8f85 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 22:34:01 +0800 Subject: [PATCH 42/44] move proxy-config.json --- .../main/resources/META-INF/native-image/proxy-config.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json deleted file mode 100644 index 305ed56fea..0000000000 --- a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/proxy-config.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "interfaces": [ - "java.util.function.Function" - ] - } -] From 32ac09795c5928e9aba176f91ce1d17d88f12d14 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 22:38:33 +0800 Subject: [PATCH 43/44] add proxy-config.json --- integration_tests/graalvm_tests/pom.xml | 1 - .../org.apache.fory/graalvm_tests/proxy-config.json | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json diff --git a/integration_tests/graalvm_tests/pom.xml b/integration_tests/graalvm_tests/pom.xml index 84da032e4d..94a0d83d32 100644 --- a/integration_tests/graalvm_tests/pom.xml +++ b/integration_tests/graalvm_tests/pom.xml @@ -173,7 +173,6 @@ -H:+UnlockExperimentalVMOptions - -H:DynamicProxyConfigurationFiles=src/main/resources/META-INF/native-image/proxy-config.json diff --git a/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json new file mode 100644 index 0000000000..549b812dd1 --- /dev/null +++ b/integration_tests/graalvm_tests/src/main/resources/META-INF/native-image/org.apache.fory/graalvm_tests/proxy-config.json @@ -0,0 +1,10 @@ +[ + { + "interfaces": [ + "java.util.function.Function" + ], + "condition": { + "typeReachable": "org.apache.fory.graalvm.ProxyExample$TestInvocationHandler" + } + } +] \ No newline at end of file From 62d1ff0862f1f52dc205afb683d806cdababef88 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 2 Sep 2025 22:51:08 +0800 Subject: [PATCH 44/44] fix proxy as lambda --- .../src/main/java/org/apache/fory/resolver/ClassResolver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java index ab24d49498..3f2f339cd8 100644 --- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java +++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java @@ -972,14 +972,14 @@ public Class getSerializerClass(Class cls, boolean code return CollectionSerializers.EnumSetSerializer.class; } else if (Charset.class.isAssignableFrom(cls)) { return Serializers.CharsetSerializer.class; - } else if (Functions.isLambda(cls)) { - return LambdaSerializer.class; } else if (ReflectionUtils.isJdkProxy(cls)) { if (JavaSerializer.getWriteReplaceMethod(cls) != null) { return ReplaceResolveSerializer.class; } else { return JdkProxySerializer.class; } + } else if (Functions.isLambda(cls)) { + return LambdaSerializer.class; } else if (Calendar.class.isAssignableFrom(cls)) { return TimeSerializers.CalendarSerializer.class; } else if (ZoneId.class.isAssignableFrom(cls)) {