From e0dc1609a95cd6d4e5b1c9d40a97817d6089334a Mon Sep 17 00:00:00 2001 From: Ingo Kegel Date: Mon, 31 Aug 2026 18:10:43 +0200 Subject: [PATCH 1/4] feat(json)!: encode byte arrays as base64 JSON strings by default JSON has no binary type, and the ecosystem standard for byte arrays in JSON is a base64 string (RFC 7493, the protobuf JSON mapping, Jackson, Gson, Moshi and kotlinx.serialization). The previous default wrote byte[] as a JSON array of decimal numbers, which is several times larger on the wire and much slower to write and parse. The @JsonBase64 annotation already provided the base64 encoding as an opt-in. This change makes it the default for byte[] properties, so the annotation is no longer needed for that. Base64 reads now reserve the decoded array in the graph memory budget, like other array reads. BREAKING CHANGE: byte[] values are written as base64 JSON strings instead of decimal number arrays. JSON written with earlier versions cannot be read back with the default codec. --- .../apache/fory/json/codec/ArrayCodec.java | 12 +++++----- .../fory/json/codec/Base64ByteArrayCodec.java | 2 ++ .../apache/fory/json/reader/JsonReader.java | 5 +++- .../apache/fory/json/JsonContainerTest.java | 24 ++++++++++++++++--- .../fory/json/JsonGraphMemoryBudgetTest.java | 2 +- .../org/apache/fory/json/JsonMixinTest.java | 2 +- .../org/apache/fory/json/JsonScalarTest.java | 2 +- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java index 86c2a02445..4f62a78b31 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java @@ -58,7 +58,7 @@ public abstract class ArrayCodec implements JsonValueCodec { this.componentType = componentType; } - public static ArrayCodec create( + public static JsonValueCodec create( Class arrayType, TypeRef arrayTypeRef, JsonTypeResolver resolver) { if (!arrayType.isArray()) { throw new ForyJsonException("Unsupported JSON array type " + arrayType); @@ -70,7 +70,7 @@ public static ArrayCodec create( } @Internal - public static ArrayCodec create(Class arrayType, JsonTypeInfo componentTypeInfo) { + public static JsonValueCodec create(Class arrayType, JsonTypeInfo componentTypeInfo) { if (!arrayType.isArray()) { throw new ForyJsonException("Unsupported JSON array type " + arrayType); } @@ -90,7 +90,7 @@ public static ArrayCodec create(Class arrayType, JsonTypeInfo componen && componentCodec == ScalarCodecs.ShortCodec.PRIMITIVE) { return bind(ShortArrayCodec.INSTANCE); } else if (componentType == byte.class && componentCodec == ScalarCodecs.ByteCodec.PRIMITIVE) { - return bind(ByteArrayCodec.INSTANCE); + return bind(Base64ByteArrayCodec.INSTANCE); } else if (componentType == char.class && componentCodec == ScalarCodecs.CharCodec.PRIMITIVE) { return bind(CharArrayCodec.INSTANCE); } else if (componentType == float.class @@ -161,7 +161,7 @@ public static ArrayCodec create(Class arrayType, JsonTypeInfo componen /** Returns the exact unsigned primitive-array specialization for one semantic array id. */ @Internal - public static ArrayCodec createUnsignedPrimitive( + public static JsonValueCodec createUnsignedPrimitive( Class arrayType, int typeId, boolean writeLongAsString) { if (arrayType == byte[].class && typeId == Types.UINT8_ARRAY) { return bind(ByteArrayCodec.UNSIGNED); @@ -183,9 +183,9 @@ public static ArrayCodec createUnsignedPrimitive( } @SuppressWarnings("unchecked") - private static ArrayCodec bind(ArrayCodec codec) { + private static JsonValueCodec bind(JsonValueCodec codec) { // The factory has matched the runtime array class to this exact singleton implementation. - return (ArrayCodec) codec; + return (JsonValueCodec) codec; } // Package visibility lets Java 8 nested codecs call these helpers without synthetic accessors. diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/Base64ByteArrayCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/Base64ByteArrayCodec.java index 9fa6786af8..32f1f77efa 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/Base64ByteArrayCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/Base64ByteArrayCodec.java @@ -27,6 +27,8 @@ /** A complete {@code byte[]} codec using a quoted standard Base64 JSON string. */ public final class Base64ByteArrayCodec implements JsonValueCodec { + public static final Base64ByteArrayCodec INSTANCE = new Base64ByteArrayCodec(); + public Base64ByteArrayCodec() {} @Override diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java index 1438f0143c..02e30d1deb 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java @@ -45,6 +45,7 @@ import org.apache.fory.json.meta.JsonSubtypeScanInfo; import org.apache.fory.json.resolver.JsonTypeResolver; import org.apache.fory.memory.NativeByteOrder; +import org.apache.fory.serializer.GraphMemoryEstimates; /** * Representation-neutral JSON cursor and common scalar parsing owner. @@ -511,7 +512,9 @@ public final byte[] readBase64() { } int end = position; int padding = (int) (shape & 3); - byte[] decoded = new byte[(encodedLength >>> 2) * 3 - padding]; + int decodedLength = (encodedLength >>> 2) * 3 - padding; + reserveGraphMemory(GraphMemoryEstimates.objectArrayBytes() + decodedLength); + byte[] decoded = new byte[decodedLength]; position = bodyStart; decodeBase64(decoded, encodedLength); position = end; diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java index 07ee115fdb..3f68dbc2cf 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonContainerTest.java @@ -25,6 +25,7 @@ import static org.apache.fory.json.JsonTestSupport.newUtf8Reader; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertNull; import static org.testng.Assert.assertThrows; import static org.testng.Assert.assertTrue; @@ -68,6 +69,7 @@ import java.util.concurrent.atomic.AtomicLongArray; import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.fory.json.codec.ArrayCodec; +import org.apache.fory.json.codec.JsonValueCodec; import org.apache.fory.json.codec.MapCodec; import org.apache.fory.json.codec.MapKeyCodec; import org.apache.fory.json.data.FastContainers; @@ -128,9 +130,9 @@ public Object readName(JsonReader reader) { @Test public void unsignedArrayOverflow() { byte[] input = "[4294967295]".getBytes(StandardCharsets.UTF_8); - ArrayCodec uint8 = + JsonValueCodec uint8 = ArrayCodec.createUnsignedPrimitive(byte[].class, Types.UINT8_ARRAY, false); - ArrayCodec uint16 = + JsonValueCodec uint16 = ArrayCodec.createUnsignedPrimitive(short[].class, Types.UINT16_ARRAY, false); assertThrows(ForyJsonException.class, () -> uint8.readUtf8(newUtf8Reader(input))); @@ -606,11 +608,27 @@ public void readPrimitiveArrayRoots() { assertEquals( json.fromJson("[true,false]".getBytes(StandardCharsets.UTF_8), boolean[].class), new boolean[] {true, false}); - assertEquals(json.fromJson("[1,-2,3]", byte[].class), new byte[] {1, -2, 3}); + assertEquals(json.fromJson("\"Af4D\"", byte[].class), new byte[] {1, -2, 3}); assertEquals(json.fromJson("[\"a\",\"你\"]", char[].class), new char[] {'a', '你'}); assertThrows(ForyJsonException.class, () -> json.fromJson("[1,null]", int[].class)); } + @Test + public void byteArrayDefaultsToBase64() { + ForyJson json = newJson(); + assertEquals(json.toJson(new byte[] {1, -2, 3}), "\"Af4D\""); + assertEquals( + new String(json.toJsonBytes(new byte[] {1, -2, 3}), StandardCharsets.UTF_8), "\"Af4D\""); + assertEquals(json.fromJson("\"Af4D\"", byte[].class), new byte[] {1, -2, 3}); + assertEquals( + json.fromJson("\"Af4D\"".getBytes(StandardCharsets.UTF_8), byte[].class), + new byte[] {1, -2, 3}); + assertEquals(json.toJson(new byte[0]), "\"\""); + assertEquals(json.toJson(null, byte[].class), "null"); + assertNull(json.fromJson("null", byte[].class)); + assertThrows(ForyJsonException.class, () -> json.fromJson("[1,-2,3]", byte[].class)); + } + @Test public void readStringArrays() { ForyJson json = newJson(); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java index 687b425add..5446a847b0 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java @@ -174,7 +174,7 @@ public void primitiveArrayOwners() { assertEquals( assertClassBudget("[true]", boolean[].class, headerBytes + Byte.BYTES), new boolean[] {true}); - assertEquals(assertClassBudget("[1]", byte[].class, headerBytes + Byte.BYTES), new byte[] {1}); + assertEquals(assertClassBudget("\"AQ==\"", byte[].class, headerBytes + Byte.BYTES), new byte[] {1}); assertEquals( assertClassBudget("[2]", short[].class, headerBytes + Short.BYTES), new short[] {2}); assertEquals(assertClassBudget("[3]", int[].class, headerBytes + Integer.BYTES), new int[] {3}); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java index 04016bf198..a48304b886 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java @@ -267,7 +267,7 @@ public void replacementAndRemoval() { newJsonBuilder().registerMixin(RepresentationRemoveMixin.class).build(); assertEquals( representation.toJson(new RepresentationRemoveTarget()), - "{\"name\":\"name\",\"raw\":\"1\",\"bytes\":[1]," + "{\"name\":\"name\",\"raw\":\"1\",\"bytes\":\"AQ==\"," + "\"child\":{\"label\":\"kid\"},\"hidden\":7}"); ForyJson anyField = newJsonBuilder().registerMixin(AnyFieldRemoveMixin.class).build(); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java index 14c422c4bc..77bf099165 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonScalarTest.java @@ -1193,7 +1193,7 @@ public void readQuotedScalarContainers(boolean codegen) { assertEquals( json.fromJson("[\"true\",\"false\"]".getBytes(StandardCharsets.UTF_8), boolean[].class), new boolean[] {true, false}); - assertEquals(json.fromJson("[\"2\",\"3\"]", byte[].class), new byte[] {2, 3}); + assertEquals(json.fromJson("\"AgM=\"", byte[].class), new byte[] {2, 3}); assertEquals(json.fromJson("[\"4\",\"5\"]", short[].class), new short[] {4, 5}); assertEquals(json.fromJson("[\"6\",\"7\"]", int[].class), new int[] {6, 7}); assertEquals( From fa460bea141f0e294820b5c7e9ed6c60e0bb3278 Mon Sep 17 00:00:00 2001 From: Ingo Kegel Date: Mon, 31 Aug 2026 18:12:14 +0200 Subject: [PATCH 2/4] perf(json): speed up base64 decoding Base64 decoding did a branch-heavy validation scan and then a second pass that re-read every character with per-character escape handling and a four-way branch per digit. On binary-heavy payloads this dominated the deserialization time. The validation now runs as a single table-driven scan, and the decode pass uses the same digit table without re-checking escapes. Bodies with escaped characters fall back to the previous validating two-pass path. --- .../apache/fory/json/reader/JsonReader.java | 100 +++++++++++++++--- 1 file changed, 83 insertions(+), 17 deletions(-) diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java index 02e30d1deb..32942663b3 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java @@ -503,6 +503,45 @@ public final byte[] readBase64() { throw error("Expected Base64 JSON string"); } int bodyStart = position; + int end = -1; + int padding = 0; + while (position < length()) { + char ch = charAt(position++); + if (ch == '"') { + end = position - 1; + break; + } + if (ch == '\\') { + // Rare: an escaped character in the body, fall back to the validating two-pass path + position = bodyStart; + return readBase64Escaped(bodyStart); + } + if (ch == '=') { + if (++padding > 2) { + throw error("Invalid Base64 JSON string padding"); + } + } else if (padding != 0 || base64Digit(ch) < 0) { + throw error("Invalid Base64 JSON string"); + } + } + if (end < 0) { + throw error("Unterminated Base64 JSON string"); + } + int bodyLength = end - bodyStart; + if (bodyLength == 0) { + return EMPTY_BYTES; + } + if ((bodyLength & 3) != 0) { + throw error("Invalid Base64 JSON string length"); + } + int decodedLength = (bodyLength >>> 2) * 3 - padding; + reserveGraphMemory(GraphMemoryEstimates.objectArrayBytes() + decodedLength); + byte[] decoded = new byte[decodedLength]; + decodeBase64(decoded, bodyStart, end); + return decoded; + } + + private byte[] readBase64Escaped(int bodyStart) { // Validate and consume the complete encoded text before allocating, so untrusted input can // only request decoded storage proportional to code units already proven readable. long shape = scanBase64Shape(); @@ -516,7 +555,7 @@ public final byte[] readBase64() { reserveGraphMemory(GraphMemoryEstimates.objectArrayBytes() + decodedLength); byte[] decoded = new byte[decodedLength]; position = bodyStart; - decodeBase64(decoded, encodedLength); + decodeBase64Escaped(decoded, encodedLength); position = end; return decoded; } @@ -542,7 +581,7 @@ private long scanBase64Shape() { throw error("Invalid Base64 JSON string padding"); } } else { - if (padding != 0 || decodeBase64Digit(ch) < 0) { + if (padding != 0 || base64Digit(ch) < 0) { throw error("Invalid Base64 JSON string"); } } @@ -551,18 +590,40 @@ private long scanBase64Shape() { throw error("Unterminated Base64 JSON string"); } - private void decodeBase64(byte[] decoded, int encodedLength) { + private void decodeBase64(byte[] decoded, int start, int end) { + int output = 0; + for (int index = start; index < end; index += 4) { + int bits = (base64Digit(charAt(index)) << 18) | (base64Digit(charAt(index + 1)) << 12); + char third = charAt(index + 2); + char fourth = charAt(index + 3); + if (third != '=') { + bits |= base64Digit(third) << 6; + } + if (fourth != '=') { + bits |= base64Digit(fourth); + } + decoded[output++] = (byte) (bits >>> 16); + if (output < decoded.length) { + decoded[output++] = (byte) (bits >>> 8); + if (output < decoded.length) { + decoded[output++] = (byte) bits; + } + } + } + } + + private void decodeBase64Escaped(byte[] decoded, int encodedLength) { int output = 0; for (int index = 0; index < encodedLength; index += 4) { int bits = - (decodeBase64Digit(readBase64Char()) << 18) | (decodeBase64Digit(readBase64Char()) << 12); + (base64Digit(readBase64Char()) << 18) | (base64Digit(readBase64Char()) << 12); char third = readBase64Char(); char fourth = readBase64Char(); if (third != '=') { - bits |= decodeBase64Digit(third) << 6; + bits |= base64Digit(third) << 6; } if (fourth != '=') { - bits |= decodeBase64Digit(fourth); + bits |= base64Digit(fourth); } decoded[output++] = (byte) (bits >>> 16); if (output < decoded.length) { @@ -579,20 +640,25 @@ private char readBase64Char() { return ch == '\\' ? readEscapedFieldNameChar() : ch; } - private static int decodeBase64Digit(char ch) { - if (ch >= 'A' && ch <= 'Z') { - return ch - 'A'; - } - if (ch >= 'a' && ch <= 'z') { - return ch - 'a' + 26; + private static final byte[] BASE64_DIGIT_VALUES = new byte[128]; + + static { + java.util.Arrays.fill(BASE64_DIGIT_VALUES, (byte) -1); + for (char c = 'A'; c <= 'Z'; c++) { + BASE64_DIGIT_VALUES[c] = (byte) (c - 'A'); } - if (ch >= '0' && ch <= '9') { - return ch - '0' + 52; + for (char c = 'a'; c <= 'z'; c++) { + BASE64_DIGIT_VALUES[c] = (byte) (c - 'a' + 26); } - if (ch == '+') { - return 62; + for (char c = '0'; c <= '9'; c++) { + BASE64_DIGIT_VALUES[c] = (byte) (c - '0' + 52); } - return ch == '/' ? 63 : -1; + BASE64_DIGIT_VALUES['+'] = 62; + BASE64_DIGIT_VALUES['/'] = 63; + } + + private static int base64Digit(char ch) { + return ch < 128 ? BASE64_DIGIT_VALUES[ch] : -1; } public String readCharSequence() { From 9df3ced6f6bb657251385022c2d0d2f1779d9758 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 1 Sep 2026 10:09:19 +0800 Subject: [PATCH 3/4] feat(json): add byte array format annotation --- .agents/languages/java.md | 5 + docs/json/android.md | 6 +- docs/json/annotations.md | 36 +++-- docs/json/graalvm.md | 6 +- docs/json/kotlin.md | 2 +- .../apache/fory/graalvm/ForyJsonExample.java | 25 +++- .../json/corpus/PlatformCorpusChecks.kt | 18 +++ .../kotlin/json/corpus/PlatformModels.kt | 8 ++ .../json/corpus/KspRetentionResourceTest.kt | 8 ++ .../processing/JsonMixinAnnotations.java | 4 +- .../processing/JsonTypeProcessor.java | 13 +- .../processing/JsonTypeProcessorTest.java | 19 ++- .../{JsonBase64.java => JsonByteArray.java} | 23 +++- .../fory/json/annotation/JsonMixin.java | 2 +- .../apache/fory/json/codec/ArrayCodec.java | 6 +- .../fory/json/codec/ObjectCodecBuilder.java | 62 +++++---- .../apache/fory/json/reader/JsonReader.java | 7 +- .../json/resolver/JsonMixinAnnotations.java | 4 +- .../json/resolver/JsonValueDeclaration.java | 6 +- .../fory/json/ForyJsonGraalVMFeature.java | 11 +- .../fory/json/JsonAndroidRuntimeTest.java | 6 +- ....java => JsonByteArrayAnnotationTest.java} | 124 +++++++++++++++--- .../fory/json/JsonGraphMemoryBudgetTest.java | 33 ++++- .../org/apache/fory/json/JsonMixinTest.java | 14 +- .../apache/fory/json/JsonUnwrappedTest.java | 6 +- .../fory/json/kotlin/ksp/KspModelBuilder.kt | 11 +- 26 files changed, 347 insertions(+), 118 deletions(-) rename java/fory-json/src/main/java/org/apache/fory/json/annotation/{JsonBase64.java => JsonByteArray.java} (59%) rename java/fory-json/src/test/java/org/apache/fory/json/{JsonBase64AnnotationTest.java => JsonByteArrayAnnotationTest.java} (72%) diff --git a/.agents/languages/java.md b/.agents/languages/java.md index a96d04b7f0..9367fd1ddf 100644 --- a/.agents/languages/java.md +++ b/.agents/languages/java.md @@ -91,6 +91,11 @@ Load this file when changing anything under `java/` or when Java drives a cross- and locale types, `Float16`, `BFloat16`, and user-defined types remain registerable. Field/type `@JsonCodec`, `@JsonFormat`, and semantic metadata remain separate from exact registry mutation and are fixed by the target class or effective Mixin. +- Fory JSON `byte[]` defaults to a Base64 string. `@JsonByteArray` selects required + `Format.BASE64` or `Format.ARRAY` for an exact byte-array field or getter in both directions. + Keep selection in the existing property codec path, including Mixin, Java processor, Kotlin + KSP, and GraalVM handling. Numeric arrays use signed-byte semantics and graph-memory accounting; + Base64 values remain binary leaves outside that budget. - Fory JSON `ObjectCodec` instances are resolver-owned and must not be registered directly. A language module that supplies a custom object model must use a `JsonCodecFactory`. A configurable factory's stable key must cover every option that can change its created codec class, object diff --git a/docs/json/android.md b/docs/json/android.md index 96db8a1c52..c9b706dfa4 100644 --- a/docs/json/android.md +++ b/docs/json/android.md @@ -130,12 +130,12 @@ on the `ForyJson` builder that should use it: ```java import org.apache.fory.json.ForyJson; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonMixin; @JsonMixin(target = ThirdPartyInvoice.class) public abstract class ThirdPartyInvoiceMixin { - @JsonBase64 byte[] signature; + @JsonByteArray(JsonByteArray.Format.BASE64) byte[] signature; } ForyJson json = @@ -191,7 +191,7 @@ This reflection-based section applies to Java models. Kotlin models use the Kotl a minified Android build, apply KSP instead of writing broad package keep rules. Java `@JsonType` models support effective `JsonValidator`, `JsonValue`, `JsonRawValue`, -`JsonBase64`, and `JsonFormat` annotations. Without `@JsonType`, those annotations still work +`JsonByteArray`, and `JsonFormat` annotations. Without `@JsonType`, those annotations still work through reflection, but a release-minified application must keep the exact annotated members, annotation attributes, and codec constructor itself. A `JsonValue` method may use a non-JavaBean name, so its manual rule must name that method explicitly. diff --git a/docs/json/annotations.md b/docs/json/annotations.md index 46353322b0..d06a0c72b3 100644 --- a/docs/json/annotations.md +++ b/docs/json/annotations.md @@ -21,7 +21,7 @@ license: | Fory JSON provides these mapping and validation annotations in `org.apache.fory.json.annotation`: -`JsonAnyGetter`, `JsonAnyProperty`, `JsonAnySetter`, `JsonBase64`, `JsonCodec`, `JsonCreator`, `JsonFormat`, +`JsonAnyGetter`, `JsonAnyProperty`, `JsonAnySetter`, `JsonByteArray`, `JsonCodec`, `JsonCreator`, `JsonFormat`, `JsonIgnore`, `JsonProperty`, `JsonPropertyOrder`, `JsonRawValue`, `JsonSubTypes`, `JsonUnwrapped`, `JsonValidator`, and `JsonValue`. `JsonType` is a separate build-time model marker. They are Fory JSON APIs, not Jackson, Gson, or Fory binary-protocol compatibility annotations. @@ -356,28 +356,36 @@ Any-property features are independent. as a trusted raw root value. That combination is serialization-only: the ordinary one-String `JsonCreator` cannot turn an input object or array into a String. -## `JsonBase64` +## `JsonByteArray` -`JsonBase64` selects a quoted standard Base64 JSON string for one exact `byte[]` field or getter: +Unannotated `byte[]` values use quoted standard Base64 JSON strings. `JsonByteArray` selects +`BASE64` or `ARRAY` for one exact `byte[]` field or getter, in both reading and writing: ```java -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; public final class Attachment { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] numbers; + + @JsonByteArray(JsonByteArray.Format.BASE64) public byte[] content; } ``` -Bytes `{1, 2, 3}` are written as `{"content":"AQID"}` and decoded back to the original array. -Fory writes the Base64 characters directly to the JSON output and decodes directly from the JSON -input without creating an intermediate String. Standard Base64 padding is preserved. Java null -follows the property's normal inclusion rule and reads from JSON null as null. +For bytes `{1, -2, 3}`, `numbers` is written as `[1,-2,3]` and `content` as `"Af4D"`. +`ARRAY` reads JSON arrays using the signed byte range `[-128, 127]`; `BASE64` reads standard +Base64 strings and preserves padding when writing. Each representation also accepts JSON null, +and null output follows the property's normal inclusion rule. The default Base64 codec does not +accept numeric-array input; select `ARRAY` for a property that uses that format. + +The format is required when the annotation is present. It applies only to the annotated byte-array +property, not to container elements or map values. Mixin declarations can select or remove it. +It cannot share a logical property with `JsonRawValue`, an occurrence `JsonCodec`, `JsonFormat`, +or an Any declaration. Conflicting formats on the field and getter of one property are rejected. -The annotation is not a type-use annotation and does not change ordinary unannotated `byte[]` -properties, container elements, or Map values. It cannot share a logical property with -`JsonRawValue`, an occurrence `JsonCodec`, `JsonFormat`, or an Any declaration. The equivalent explicit codec is -`@JsonCodec(Base64ByteArrayCodec.class)`. +Base64 values are binary leaves excluded from the graph-memory budget. Numeric arrays count their +array storage against that budget; see [Security](security.md#depth-and-graph-memory-limits). ## `JsonFormat` @@ -439,7 +447,7 @@ unwrapped values are intentionally rejected. Types with ambiguous formatting sem legacy and SQL date types, `Duration`, `Period`, `TimeZone`, `ZoneId`, and `ZoneOffset`, are not supported. A wrapper with a complete registered, annotation-selected, polymorphic, or `JsonValue` representation is also rejected because that representation owns the whole wrapper. -`JsonFormat` cannot share a field with `JsonCodec`, `JsonBase64`, `JsonRawValue`, `JsonAnyProperty`, +`JsonFormat` cannot share a field with `JsonCodec`, `JsonByteArray`, `JsonRawValue`, `JsonAnyProperty`, `JsonUnwrapped`, or `JsonValue`. ## `JsonUnwrapped` diff --git a/docs/json/graalvm.md b/docs/json/graalvm.md index 08295063c4..22d48b3f97 100644 --- a/docs/json/graalvm.md +++ b/docs/json/graalvm.md @@ -209,10 +209,10 @@ JVM and Android. `JsonValue` fields and effective public zero-argument methods are supported, including matching one-String `JsonCreator` constructors and public static factories. Fixed `JsonRawValue` fields and -getters support trusted raw String values, and fixed `JsonBase64` fields and getters support Base64 -`byte[]` values as on the JVM. `JsonFormat` date/time fields use the same direct-field, +getters support trusted raw String values, and `JsonByteArray` fields and getters select Base64 strings or numeric +byte arrays as on the JVM. `JsonFormat` date/time fields use the same direct-field, one-wrapper-level, and `timezone` behavior as on the JVM. For direct target annotations, annotate -each reachable owning model with `JsonType` so Native Image retains these members and the Base64 +each reachable owning model with `JsonType` so Native Image retains these members and the selected byte-array codec constructor. A directly annotated `JsonValue` Record uses its generated component accessor and canonical constructor operations. An effective declaration supplied by a Mixin uses the Mixin workflow above diff --git a/docs/json/kotlin.md b/docs/json/kotlin.md index 4b6bc18220..64ef261a49 100644 --- a/docs/json/kotlin.md +++ b/docs/json/kotlin.md @@ -280,7 +280,7 @@ their normal Fory JSON representation when used from Kotlin: | text | `String`, exact `CharSequence`, `StringBuilder`, and `StringBuffer` use String shapes | | arbitrary/reduced-precision number | `BigInteger`, `BigDecimal`, Fory `Float16`, and `BFloat16` use their core numeric shapes and limits | | enum | quoted enum constant name | -| Java/Kotlin arrays | normal JSON arrays; `ByteArray` is numeric unless `JsonBase64` selects binary; unsigned semantic arrays are listed below | +| Java/Kotlin arrays | normal JSON arrays except `ByteArray`, which uses Base64 strings by default; `@field:JsonByteArray(JsonByteArray.Format.ARRAY)` selects numeric arrays; unsigned semantic arrays are listed below | | Optional and atomic | `Optional`, primitive Optionals, atomic scalars/references, and atomic arrays keep their transparent core shapes subject to the nullability rules above | | quoted JDK values | `Currency`, `File`, `URI`, `Path`, `Pattern`, `UUID`, `Locale`, `Charset`, and `TimeZone` keep their core String shapes | | legacy date/time | `Date`, `Calendar`, and available `java.sql.Date`, `Time`, and `Timestamp` keep their epoch-millisecond shapes | diff --git a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ForyJsonExample.java b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ForyJsonExample.java index ff5055edf6..4d848b5690 100644 --- a/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ForyJsonExample.java +++ b/integration_tests/graalvm_tests/src/main/java/org/apache/fory/graalvm/ForyJsonExample.java @@ -49,7 +49,7 @@ import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; import org.apache.fory.json.annotation.JsonAnySetter; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonFormat; @@ -489,6 +489,20 @@ private static void testValueAnnotations() { new String(json.toJsonBytes(raw), StandardCharsets.UTF_8).equals("{\"body\":{\"id\":1}}")); Preconditions.checkArgument( json.fromJson("{\"body\":\"text\"}", RawValue.class).body.equals("text")); + ArrayBytes arrayBytes = new ArrayBytes(); + arrayBytes.value = new byte[] {1, -2, 3}; + Preconditions.checkArgument(json.toJson(arrayBytes).equals("{\"value\":[1,-2,3]}")); + Preconditions.checkArgument( + new String(json.toJsonBytes(arrayBytes), StandardCharsets.UTF_8) + .equals("{\"value\":[1,-2,3]}")); + Preconditions.checkArgument( + Arrays.equals( + json.fromJson("{\"value\":[1,-2,3]}", ArrayBytes.class).value, arrayBytes.value)); + Preconditions.checkArgument( + Arrays.equals( + json.fromJson("{\"value\":[1,-2,3]}".getBytes(StandardCharsets.UTF_8), ArrayBytes.class) + .value, + arrayBytes.value)); Base64Bytes base64Bytes = new Base64Bytes(); base64Bytes.value = new byte[] {1, 2, 3}; Preconditions.checkArgument(json.toJson(base64Bytes).equals("{\"value\":\"AQID\"}")); @@ -1273,9 +1287,16 @@ public static final class RawValue { @JsonRawValue public String body; } + @JsonType + public static final class ArrayBytes { + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] value; + } + @JsonType public static final class Base64Bytes { - @JsonBase64 public byte[] value; + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] value; } @JsonType diff --git a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt index 7e8b4cf2ed..bed2e24309 100644 --- a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt +++ b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt @@ -20,6 +20,7 @@ package org.apache.fory.integration.kotlin.json.corpus import org.apache.fory.json.ForyJson +import org.apache.fory.json.kotlin.jsonTypeRef /** Executes the same representative round trip on the JVM, Android, and Native Image. */ public object PlatformCorpusChecks { @@ -32,6 +33,23 @@ public object PlatformCorpusChecks { check(text.contains("\"display_label\":\"mixin\"")) verifyRoot(json.fromJson(text, type)) verifyRoot(json.fromJson(json.toJsonBytes(decoded, type), type)) + verifyByteArrays(json) + } + + private fun verifyByteArrays(json: ForyJson) { + val type = jsonTypeRef() + val bytes = byteArrayOf(1, -2, 3) + val value = PlatformByteArrays(bytes, bytes, bytes) + val text = json.toJson(value, type) + check(text.contains("\"numbers\":[1,-2,3]")) + check(text.contains("\"binary\":\"Af4D\"")) + check(text.contains("\"defaultBytes\":\"Af4D\"")) + for (decoded in + listOf(json.fromJson(text, type), json.fromJson(json.toJsonBytes(value, type), type))) { + check(decoded.numbers.contentEquals(bytes)) + check(decoded.binary.contentEquals(bytes)) + check(decoded.defaultBytes.contentEquals(bytes)) + } } private fun verifyRoot(actual: PlatformRoot) { diff --git a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt index 865ad9ab08..9c90552698 100644 --- a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt +++ b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt @@ -20,6 +20,7 @@ package org.apache.fory.integration.kotlin.json.corpus import kotlin.jvm.JvmInline +import org.apache.fory.json.annotation.JsonByteArray import org.apache.fory.json.annotation.JsonCodec import org.apache.fory.json.annotation.JsonMixin import org.apache.fory.json.annotation.JsonSubTypes @@ -116,3 +117,10 @@ internal fun platformRootValue(): PlatformRoot = token = PlatformToken("custom"), box = PlatformBox("generic"), ) + +@JsonType +public data class PlatformByteArrays( + @field:JsonByteArray(JsonByteArray.Format.ARRAY) public val numbers: ByteArray, + @get:JsonByteArray(JsonByteArray.Format.BASE64) public val binary: ByteArray, + public val defaultBytes: ByteArray, +) diff --git a/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt b/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt index acc1f1f8a7..e6a408a628 100644 --- a/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt +++ b/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt @@ -48,6 +48,13 @@ public class KspRetentionResourceTest { assertTrue(sealed.contains("class $PACKAGE.PlatformOpen"), sealed) assertFalse(sealed.contains("class $PACKAGE.PlatformOpenDescendant"), sealed) assertConstructor(rules("PlatformRoot"), "$PACKAGE.PlatformTokenCodec") + val byteArrays = rules("PlatformByteArrays") + assertTrue( + byteArrays.contains("@interface org.apache.fory.json.annotation.JsonByteArray"), + byteArrays + ) + assertConstructor(byteArrays, "org.apache.fory.json.codec.Base64ByteArrayCodec") + assertConstructor(byteArrays, "org.apache.fory.json.codec.ArrayCodec\$SignedByteArrayCodec") assertConstructor( rules("PlatformDirectOverride"), "$PACKAGE.PlatformDirectOverrideCodec", @@ -98,6 +105,7 @@ public class KspRetentionResourceTest { setOf( "META-INF/proguard/fory-json-$PACKAGE.PlatformAccount.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformBox.pro", + "META-INF/proguard/fory-json-$PACKAGE.PlatformByteArrays.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformCircle.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformDirectOverride.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformId.pro", diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonMixinAnnotations.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonMixinAnnotations.java index 0434aaec3e..c22b2868c8 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonMixinAnnotations.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonMixinAnnotations.java @@ -53,7 +53,7 @@ final class JsonMixinAnnotations { private static final String JSON_ANY_GETTER = JSON_PACKAGE + ".annotation.JsonAnyGetter"; private static final String JSON_ANY_PROPERTY = JSON_PACKAGE + ".annotation.JsonAnyProperty"; private static final String JSON_ANY_SETTER = JSON_PACKAGE + ".annotation.JsonAnySetter"; - private static final String JSON_BASE64 = JSON_PACKAGE + ".annotation.JsonBase64"; + private static final String JSON_BYTE_ARRAY = JSON_PACKAGE + ".annotation.JsonByteArray"; private static final String JSON_CODEC = JSON_PACKAGE + ".annotation.JsonCodec"; private static final String JSON_CREATOR = JSON_PACKAGE + ".annotation.JsonCreator"; private static final String JSON_FORMAT = JSON_PACKAGE + ".annotation.JsonFormat"; @@ -73,7 +73,7 @@ final class JsonMixinAnnotations { JSON_ANY_GETTER, JSON_ANY_PROPERTY, JSON_ANY_SETTER, - JSON_BASE64, + JSON_BYTE_ARRAY, JSON_CODEC, JSON_CREATOR, JSON_FORMAT, diff --git a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonTypeProcessor.java b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonTypeProcessor.java index eab6c6e540..083cb22cd9 100644 --- a/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonTypeProcessor.java +++ b/java/fory-annotation-processor/src/main/java/org/apache/fory/annotation/processing/JsonTypeProcessor.java @@ -71,7 +71,7 @@ final class JsonTypeProcessor { private static final String JSON_PROPERTY = JSON_PACKAGE + ".annotation.JsonProperty"; private static final String JSON_VALUE = JSON_PACKAGE + ".annotation.JsonValue"; private static final String JSON_RAW_VALUE = JSON_PACKAGE + ".annotation.JsonRawValue"; - private static final String JSON_BASE64 = JSON_PACKAGE + ".annotation.JsonBase64"; + private static final String JSON_BYTE_ARRAY = JSON_PACKAGE + ".annotation.JsonByteArray"; private static final String JSON_UNWRAPPED = JSON_PACKAGE + ".annotation.JsonUnwrapped"; private static final String JSON_VALIDATOR = JSON_PACKAGE + ".annotation.JsonValidator"; private static final String BASE64_CODEC = JSON_PACKAGE + ".codec.Base64ByteArrayCodec"; @@ -569,8 +569,13 @@ private void collectMixinTargets(JsonMixinAnnotations annotations, Model model) private void collectOccurrenceCodec( JsonMixinAnnotations annotations, Element element, Model model) { collectCodecAnnotation(annotationMirror(annotations, element, JSON_CODEC), model); - if (hasAnnotation(annotations, element, JSON_BASE64)) { - model.codecTypes.add(BASE64_CODEC); + AnnotationMirror byteArray = annotationMirror(annotations, element, JSON_BYTE_ARRAY); + if (byteArray != null) { + VariableElement format = (VariableElement) annotationValue(byteArray, "value").getValue(); + model.codecTypes.add( + format.getSimpleName().contentEquals("ARRAY") + ? JSON_PACKAGE + ".codec.ArrayCodec$SignedByteArrayCodec" + : BASE64_CODEC); } } @@ -1134,7 +1139,7 @@ private boolean isJsonMethod( || hasAnnotation(annotations, method, JSON_CODEC) || hasAnnotation(annotations, method, JSON_VALUE) || hasAnnotation(annotations, method, JSON_RAW_VALUE) - || hasAnnotation(annotations, method, JSON_BASE64) + || hasAnnotation(annotations, method, JSON_BYTE_ARRAY) || hasAnnotation(annotations, method, JSON_VALIDATOR) || hasJsonAnnotations(annotations, method.getParameters())) { return true; diff --git a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/JsonTypeProcessorTest.java b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/JsonTypeProcessorTest.java index 8aac74e26f..6e7659b693 100644 --- a/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/JsonTypeProcessorTest.java +++ b/java/fory-annotation-processor/src/test/java/org/apache/fory/annotation/processing/JsonTypeProcessorTest.java @@ -877,7 +877,7 @@ public void encodedRecordPipeline() throws Exception { "package test;\n" + "import org.apache.fory.json.annotation.*;\n" + "@JsonType public record EncodedRecord(\n" - + " @JsonRawValue String raw, @JsonBase64 byte[] bytes) {}\n"); + + " @JsonRawValue String raw, @JsonByteArray(JsonByteArray.Format.ARRAY) byte[] bytes) {}\n"); assertTrue(result.success, result.diagnostics()); ClassLoader loader = result.classLoader(); Class type = loader.loadClass("test.EncodedRecord"); @@ -885,8 +885,8 @@ public void encodedRecordPipeline() throws Exception { type.getConstructor(String.class, byte[].class) .newInstance("{\"id\":1}", new byte[] {1, 2, 3}); for (ForyJson json : jsonRuntimes(loader)) { - assertEquals(json.toJson(value), "{\"raw\":{\"id\":1},\"bytes\":\"AQID\"}"); - Object decoded = json.fromJson("{\"raw\":\"text\",\"bytes\":\"AQI=\"}", type); + assertEquals(json.toJson(value), "{\"raw\":{\"id\":1},\"bytes\":[1,2,3]}"); + Object decoded = json.fromJson("{\"raw\":\"text\",\"bytes\":[1,2]}", type); assertEquals(type.getMethod("raw").invoke(decoded), "text"); assertTrue( Arrays.equals((byte[]) type.getMethod("bytes").invoke(decoded), new byte[] {1, 2})); @@ -902,7 +902,7 @@ public void encodedCreatorPipeline() throws Exception { + "import java.util.Arrays;\n" + "import org.apache.fory.json.annotation.*;\n" + "@JsonType public final class EncodedCreator {\n" - + " @JsonBase64 public final byte[] bytes;\n" + + " @JsonByteArray(JsonByteArray.Format.BASE64) public final byte[] bytes;\n" + " @JsonCreator({\"bytes\"}) public EncodedCreator(byte[] bytes) {\n" + " this.bytes = bytes;\n" + " }\n" @@ -1858,7 +1858,7 @@ public void deterministicRules() throws Exception { } @Test - public void valueRawAndBase64Rules() throws Exception { + public void valueRawAndByteArrayRules() throws Exception { Map sources = new LinkedHashMap<>(); sources.put( "test.ValueModel", @@ -1875,7 +1875,8 @@ public void valueRawAndBase64Rules() throws Exception { + "import org.apache.fory.json.annotation.*;\n" + "@JsonType public final class RawModel {\n" + " @JsonRawValue public String body;\n" - + " @JsonBase64 public byte[] bytes;\n" + + " @JsonByteArray(JsonByteArray.Format.BASE64) public byte[] bytes;\n" + + " @JsonByteArray(JsonByteArray.Format.ARRAY) public byte[] numbers;\n" + " private String other;\n" + " @JsonRawValue public String getOther() { return other; }\n" + " public void setOther(String other) { this.other = other; }\n" @@ -1893,6 +1894,10 @@ public void valueRawAndBase64Rules() throws Exception { valueRules.contains("@interface org.apache.fory.json.annotation.JsonRawValue"), valueRules); String rawRules = result.generatedResource(RULE_PREFIX + "test.RawModel.pro"); + assertTrue( + rawRules.contains( + "class org.apache.fory.json.codec.ArrayCodec$SignedByteArrayCodec { public (); }"), + rawRules); assertTrue(result.hasGeneratedSource("test/RawModel_ForyJsonCodec.java")); assertTrue(rawRules.contains("java.lang.String body;"), rawRules); assertTrue(rawRules.contains("byte[] bytes;"), rawRules); @@ -1900,7 +1905,7 @@ public void valueRawAndBase64Rules() throws Exception { assertTrue( rawRules.contains("@interface org.apache.fory.json.annotation.JsonRawValue"), rawRules); assertTrue( - rawRules.contains("@interface org.apache.fory.json.annotation.JsonBase64"), rawRules); + rawRules.contains("@interface org.apache.fory.json.annotation.JsonByteArray"), rawRules); assertFalse( rawRules.contains("@interface org.apache.fory.json.annotation.JsonCodec"), rawRules); assertTrue( diff --git a/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonBase64.java b/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonByteArray.java similarity index 59% rename from java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonBase64.java rename to java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonByteArray.java index 1046bd346c..b221093bd6 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonBase64.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonByteArray.java @@ -26,14 +26,25 @@ import java.lang.annotation.Target; /** - * Selects a quoted standard Base64 JSON string as the representation of one exact {@code byte[]} - * field or getter. + * Selects the JSON representation of one exact {@code byte[]} field or getter for both reading and + * writing. Unannotated byte arrays use a quoted standard Base64 string. * - *

Writing encodes the bytes without an intermediate String, and reading decodes the JSON string - * directly into bytes. Null inclusion and omission follow the property's normal configuration, and - * an included null is written as JSON {@code null}. + *

Null inclusion and omission follow the property's normal configuration, and an included null + * is written as JSON {@code null}. This annotation cannot be combined with {@link JsonCodec} on the + * same logical property. */ @Documented @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD, ElementType.METHOD}) -public @interface JsonBase64 {} +public @interface JsonByteArray { + /** Returns the representation used when reading and writing this property. */ + Format value(); + + /** The supported JSON representations of a byte array. */ + enum Format { + /** A quoted standard Base64 string with padding. */ + BASE64, + /** A JSON array of signed byte values in the range {@code [-128, 127]}. */ + ARRAY + } +} diff --git a/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonMixin.java b/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonMixin.java index fb73e516b8..d62e61c992 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonMixin.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/annotation/JsonMixin.java @@ -37,7 +37,7 @@ * access, invocation, and value. * *

A Mixin may contribute {@link JsonAnyGetter}, {@link JsonAnyProperty}, {@link JsonAnySetter}, - * {@link JsonBase64}, {@link JsonCodec}, {@link JsonCreator}, {@link JsonFormat}, {@link + * {@link JsonByteArray}, {@link JsonCodec}, {@link JsonCreator}, {@link JsonFormat}, {@link * JsonIgnore}, {@link JsonProperty}, {@link JsonPropertyOrder}, {@link JsonRawValue}, {@link * JsonSubTypes}, {@link JsonUnwrapped}, {@link JsonValidator}, and {@link JsonValue}. {@link * JsonType} remains a marker declared directly on a model and cannot be contributed or removed by a diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java index 4f62a78b31..6372628f05 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/ArrayCodec.java @@ -1262,7 +1262,6 @@ short readElement(Utf8JsonReader reader) { } public abstract static class ByteArrayCodec extends ArrayCodec { - private static final ByteArrayCodec INSTANCE = new SignedByteArrayCodec(); private static final ByteArrayCodec UNSIGNED = new UnsignedByteArrayCodec(); private ByteArrayCodec() { @@ -1394,7 +1393,10 @@ public byte[] readUtf8(Utf8JsonReader reader) { } } - private static final class SignedByteArrayCodec extends ByteArrayCodec { + /** A complete {@code byte[]} codec using a JSON array of signed byte values. */ + public static final class SignedByteArrayCodec extends ByteArrayCodec { + public SignedByteArrayCodec() {} + @Override void writeElement(StringJsonWriter writer, byte value) { writer.writeInt(value); diff --git a/java/fory-json/src/main/java/org/apache/fory/json/codec/ObjectCodecBuilder.java b/java/fory-json/src/main/java/org/apache/fory/json/codec/ObjectCodecBuilder.java index 0a8120e6d9..eb8b70d1df 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/codec/ObjectCodecBuilder.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/codec/ObjectCodecBuilder.java @@ -42,7 +42,7 @@ import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; import org.apache.fory.json.annotation.JsonAnySetter; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonFormat; @@ -1079,7 +1079,7 @@ static boolean usesJsonMetadata(Method method, boolean record) { || method.isAnnotationPresent(JsonAnySetter.class) || method.isAnnotationPresent(JsonValue.class) || method.isAnnotationPresent(JsonRawValue.class) - || method.isAnnotationPresent(JsonBase64.class) + || method.isAnnotationPresent(JsonByteArray.class) || method.isAnnotationPresent(JsonValidator.class)) { return true; } @@ -1094,7 +1094,7 @@ static boolean usesJsonReturn(Method method) { return method.isAnnotationPresent(JsonAnyGetter.class) || method.isAnnotationPresent(JsonValue.class) || method.isAnnotationPresent(JsonRawValue.class) - || method.isAnnotationPresent(JsonBase64.class) + || method.isAnnotationPresent(JsonByteArray.class) || getterPropertyName(method) != null; } @@ -1882,8 +1882,8 @@ private static boolean validateMemberAnnotations( if (annotations.has(field, JsonFormat.class)) { validateFormatField(field, annotations); } - if (annotations.has(field, JsonBase64.class)) { - validateBase64Field(field, annotations); + if (annotations.has(field, JsonByteArray.class)) { + validateByteArrayField(field, annotations); } if (annotations.has(field, JsonRawValue.class)) { validateRawField(field, annotations); @@ -1940,8 +1940,8 @@ private static boolean validateMemberAnnotations( validateRawMethod( type, method, propertyDiscoveryEnabled, record, generatedCodec, annotations); } - if (annotations.has(method, JsonBase64.class)) { - validateBase64Method( + if (annotations.has(method, JsonByteArray.class)) { + validateByteArrayMethod( type, method, propertyDiscoveryEnabled, record, generatedCodec, annotations); } if (annotations.has(method, JsonUnwrapped.class)) { @@ -2008,8 +2008,8 @@ private static boolean validateMemberAnnotations( validateRawMethod( type, method, propertyDiscoveryEnabled, record, generatedCodec, annotations); } - if (annotations.has(method, JsonBase64.class)) { - validateBase64Method( + if (annotations.has(method, JsonByteArray.class)) { + validateByteArrayMethod( type, method, propertyDiscoveryEnabled, record, generatedCodec, annotations); } if (annotations.has(method, JsonUnwrapped.class)) { @@ -2307,7 +2307,7 @@ private static void validateRawField(Field field, Annotations annotations) { throw new ForyJsonException("Invalid @JsonRawValue field " + field); } if (annotations.has(field, JsonCodec.class) - || annotations.has(field, JsonBase64.class) + || annotations.has(field, JsonByteArray.class) || annotations.has(field, JsonAnyProperty.class)) { throw new ForyJsonException("Conflicting JSON annotations on @JsonRawValue field " + field); } @@ -2339,24 +2339,24 @@ && isPropagatedRecordAnnotation( throw new ForyJsonException("Invalid @JsonRawValue method " + method); } if (annotations.has(method, JsonCodec.class) - || annotations.has(method, JsonBase64.class) + || annotations.has(method, JsonByteArray.class) || annotations.has(method, JsonAnyGetter.class)) { throw new ForyJsonException("Conflicting JSON annotations on @JsonRawValue method " + method); } } - private static void validateBase64Field(Field field, Annotations annotations) { + private static void validateByteArrayField(Field field, Annotations annotations) { if (!isEligibleField(field) || field.getType() != byte[].class) { - throw new ForyJsonException("Invalid @JsonBase64 field " + field); + throw new ForyJsonException("Invalid @JsonByteArray field " + field); } if (annotations.has(field, JsonCodec.class) || annotations.has(field, JsonRawValue.class) || annotations.has(field, JsonAnyProperty.class)) { - throw new ForyJsonException("Conflicting JSON annotations on @JsonBase64 field " + field); + throw new ForyJsonException("Conflicting JSON annotations on @JsonByteArray field " + field); } JsonIgnore ignore = annotations.get(field, JsonIgnore.class); if (ignore != null && ignore.ignoreRead() && ignore.ignoreWrite()) { - throw new ForyJsonException("@JsonBase64 has no JSON read or write direction: " + field); + throw new ForyJsonException("@JsonByteArray has no JSON read or write direction: " + field); } } @@ -2365,7 +2365,7 @@ private static void validateFormatField(Field field, Annotations annotations) { throw new ForyJsonException("Invalid @JsonFormat field " + field); } if (annotations.has(field, JsonCodec.class) - || annotations.has(field, JsonBase64.class) + || annotations.has(field, JsonByteArray.class) || annotations.has(field, JsonRawValue.class) || annotations.has(field, JsonAnyProperty.class) || annotations.has(field, JsonUnwrapped.class) @@ -2378,7 +2378,7 @@ private static void validateFormatField(Field field, Annotations annotations) { } } - private static void validateBase64Method( + private static void validateByteArrayMethod( Class type, Method method, boolean propertyDiscoveryEnabled, @@ -2388,7 +2388,7 @@ private static void validateBase64Method( if ((!propertyDiscoveryEnabled && !(record && isPropagatedRecordAnnotation( - type, method, JsonBase64.class, generatedCodec, annotations))) + type, method, JsonByteArray.class, generatedCodec, annotations))) || !isEligibleAccessor(method) || method.isVarArgs() || method.getTypeParameters().length != 0 @@ -2396,12 +2396,13 @@ && isPropagatedRecordAnnotation( || method.getReturnType() != byte[].class || ((!record && getterPropertyName(method) == null) || (record && !isRecordAccessor(type, method, generatedCodec)))) { - throw new ForyJsonException("Invalid @JsonBase64 method " + method); + throw new ForyJsonException("Invalid @JsonByteArray method " + method); } if (annotations.has(method, JsonCodec.class) || annotations.has(method, JsonRawValue.class) || annotations.has(method, JsonAnyGetter.class)) { - throw new ForyJsonException("Conflicting JSON annotations on @JsonBase64 method " + method); + throw new ForyJsonException( + "Conflicting JSON annotations on @JsonByteArray method " + method); } } @@ -3393,18 +3394,25 @@ private void mergeUnwrapped(AnnotatedElement source) { private void mergeCodec(AnnotatedElement source) { JsonCodec declared = annotations.get(source, JsonCodec.class); - if (annotations.has(source, JsonBase64.class)) { + JsonByteArray byteArray = annotations.get(source, JsonByteArray.class); + if (byteArray != null) { if (formatAnnotation != null) { - throw formatConflict(source, "@JsonBase64"); + throw formatConflict(source, "@JsonByteArray"); } if (declared != null || codecAnnotation != null) { throw new ForyJsonException( - "@JsonBase64 cannot coexist with @JsonCodec for property " + name); + "@JsonByteArray cannot coexist with @JsonCodec for property " + name); } - if (valueCodecClass == null) { - valueCodecClass = Base64ByteArrayCodec.class; - codecSource = source; + Class> codecClass = + byteArray.value() == JsonByteArray.Format.ARRAY + ? ArrayCodec.SignedByteArrayCodec.class + : Base64ByteArrayCodec.class; + if (valueCodecClass != null && valueCodecClass != codecClass) { + throw new ForyJsonException( + "Conflicting @JsonByteArray declarations for property " + name); } + valueCodecClass = codecClass; + codecSource = source; return; } if (declared != null && formatAnnotation != null) { @@ -3412,7 +3420,7 @@ private void mergeCodec(AnnotatedElement source) { } if (declared != null && valueCodecClass != null) { throw new ForyJsonException( - "@JsonBase64 cannot coexist with @JsonCodec for property " + name); + "@JsonByteArray cannot coexist with @JsonCodec for property " + name); } if (declared == null) { return; diff --git a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java index 32942663b3..aa66788184 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/reader/JsonReader.java @@ -45,7 +45,6 @@ import org.apache.fory.json.meta.JsonSubtypeScanInfo; import org.apache.fory.json.resolver.JsonTypeResolver; import org.apache.fory.memory.NativeByteOrder; -import org.apache.fory.serializer.GraphMemoryEstimates; /** * Representation-neutral JSON cursor and common scalar parsing owner. @@ -535,7 +534,7 @@ public final byte[] readBase64() { throw error("Invalid Base64 JSON string length"); } int decodedLength = (bodyLength >>> 2) * 3 - padding; - reserveGraphMemory(GraphMemoryEstimates.objectArrayBytes() + decodedLength); + // Base64 is a binary leaf: validated input bounds its storage, not the graph memory budget. byte[] decoded = new byte[decodedLength]; decodeBase64(decoded, bodyStart, end); return decoded; @@ -552,7 +551,6 @@ private byte[] readBase64Escaped(int bodyStart) { int end = position; int padding = (int) (shape & 3); int decodedLength = (encodedLength >>> 2) * 3 - padding; - reserveGraphMemory(GraphMemoryEstimates.objectArrayBytes() + decodedLength); byte[] decoded = new byte[decodedLength]; position = bodyStart; decodeBase64Escaped(decoded, encodedLength); @@ -615,8 +613,7 @@ private void decodeBase64(byte[] decoded, int start, int end) { private void decodeBase64Escaped(byte[] decoded, int encodedLength) { int output = 0; for (int index = 0; index < encodedLength; index += 4) { - int bits = - (base64Digit(readBase64Char()) << 18) | (base64Digit(readBase64Char()) << 12); + int bits = (base64Digit(readBase64Char()) << 18) | (base64Digit(readBase64Char()) << 12); char third = readBase64Char(); char fourth = readBase64Char(); if (third != '=') { diff --git a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonMixinAnnotations.java b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonMixinAnnotations.java index 2c1e992073..5752525b9d 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonMixinAnnotations.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonMixinAnnotations.java @@ -45,7 +45,7 @@ import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; import org.apache.fory.json.annotation.JsonAnySetter; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonFormat; @@ -69,7 +69,7 @@ final class JsonMixinAnnotations { JsonAnyGetter.class, JsonAnyProperty.class, JsonAnySetter.class, - JsonBase64.class, + JsonByteArray.class, JsonCodec.class, JsonCreator.class, JsonFormat.class, diff --git a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonValueDeclaration.java b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonValueDeclaration.java index 6193ac5b84..992a8e56ee 100644 --- a/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonValueDeclaration.java +++ b/java/fory-json/src/main/java/org/apache/fory/json/resolver/JsonValueDeclaration.java @@ -31,7 +31,7 @@ import org.apache.fory.json.ForyJsonException; import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonFormat; import org.apache.fory.json.annotation.JsonIgnore; @@ -187,7 +187,7 @@ private static void validateField(Class type, Field field, JsonSharedRegistry throw new ForyJsonException("Invalid @JsonValue field " + field); } if (registry.annotation(type, field, JsonCodec.class) != null - || registry.annotation(type, field, JsonBase64.class) != null + || registry.annotation(type, field, JsonByteArray.class) != null || registry.annotation(type, field, JsonFormat.class) != null || registry.annotation(type, field, JsonAnyProperty.class) != null || registry.annotation(type, field, JsonUnwrapped.class) != null @@ -209,7 +209,7 @@ private static void validateMethod(Class type, Method method, JsonSharedRegis throw new ForyJsonException("Invalid @JsonValue method " + method); } if (registry.annotation(type, method, JsonCodec.class) != null - || registry.annotation(type, method, JsonBase64.class) != null + || registry.annotation(type, method, JsonByteArray.class) != null || registry.annotation(type, method, JsonAnyGetter.class) != null || registry.annotation(type, method, JsonUnwrapped.class) != null || registry.annotation(type, method, JsonIgnore.class) != null) { diff --git a/java/fory-json/src/main/java17/org/apache/fory/json/ForyJsonGraalVMFeature.java b/java/fory-json/src/main/java17/org/apache/fory/json/ForyJsonGraalVMFeature.java index d5de194e4b..d5f9868993 100644 --- a/java/fory-json/src/main/java17/org/apache/fory/json/ForyJsonGraalVMFeature.java +++ b/java/fory-json/src/main/java17/org/apache/fory/json/ForyJsonGraalVMFeature.java @@ -51,7 +51,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.apache.fory.json.annotation.ForyJsonProvider; import org.apache.fory.json.annotation.JsonAnySetter; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonMixin; @@ -60,6 +60,7 @@ import org.apache.fory.json.annotation.JsonUnwrapped; import org.apache.fory.json.annotation.JsonValidator; import org.apache.fory.json.annotation.JsonValue; +import org.apache.fory.json.codec.ArrayCodec; import org.apache.fory.json.codec.Base64ByteArrayCodec; import org.apache.fory.json.codec.JsonUnwrappedInfo; import org.apache.fory.json.codec.ObjectCodec; @@ -999,8 +1000,12 @@ private void registerParameterCodecs(JsonMixinView annotations, Parameter[] para private void registerOccurrenceCodecs(JsonMixinView annotations, AnnotatedElement element) { registerCodecs(annotation(annotations, element, JsonCodec.class)); - if (annotation(annotations, element, JsonBase64.class) != null) { - registerCodec(Base64ByteArrayCodec.class); + JsonByteArray byteArray = annotation(annotations, element, JsonByteArray.class); + if (byteArray != null) { + registerCodec( + byteArray.value() == JsonByteArray.Format.ARRAY + ? ArrayCodec.SignedByteArrayCodec.class + : Base64ByteArrayCodec.class); } } diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonAndroidRuntimeTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonAndroidRuntimeTest.java index dc78da6723..b2e87a27bf 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonAndroidRuntimeTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonAndroidRuntimeTest.java @@ -34,7 +34,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonFormat; @@ -257,7 +257,9 @@ public String value() { public static final class AndroidRaw { @JsonRawValue public String body; - @JsonBase64 public byte[] bytes; + + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] bytes; } public static final class AndroidFormat { diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonBase64AnnotationTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonByteArrayAnnotationTest.java similarity index 72% rename from java/fory-json/src/test/java/org/apache/fory/json/JsonBase64AnnotationTest.java rename to java/fory-json/src/test/java/org/apache/fory/json/JsonByteArrayAnnotationTest.java index 025c2a66c7..08c1150725 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonBase64AnnotationTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonByteArrayAnnotationTest.java @@ -27,7 +27,7 @@ import java.util.Map; import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonIgnore; @@ -40,12 +40,81 @@ import org.testng.annotations.Factory; import org.testng.annotations.Test; -public class JsonBase64AnnotationTest extends ForyJsonTestModels { +public class JsonByteArrayAnnotationTest extends ForyJsonTestModels { @Factory(dataProvider = "enableCodegen") - public JsonBase64AnnotationTest(boolean codegen) { + public JsonByteArrayAnnotationTest(boolean codegen) { super(codegen); } + @Test + public void arrayRoundTrip() { + ForyJson json = newJson(); + ArrayField value = new ArrayField(); + byte[][] values = {new byte[0], {-128}, {1, -2, 127}}; + String[] encoded = {"[]", "[-128]", "[1,-2,127]"}; + for (int i = 0; i < values.length; i++) { + value.bytes = values[i]; + String text = "{\"bytes\":" + encoded[i] + "}"; + assertEquals(json.toJson(value), text); + assertEquals(new String(json.toJsonBytes(value), StandardCharsets.UTF_8), text); + assertEquals(json.fromJson(text, ArrayField.class).bytes, values[i]); + assertEquals( + json.fromJson(text.getBytes(StandardCharsets.UTF_8), ArrayField.class).bytes, values[i]); + assertEquals( + json.fromJson("{\"ignored\":\"汉\",\"bytes\":" + encoded[i] + "}", ArrayField.class).bytes, + values[i]); + } + assertNull(json.fromJson("{\"bytes\":null}", ArrayField.class).bytes); + for (String encodedValue : new String[] {"[128]", "[-129]", "[null]", "\"AQ==\""}) { + assertThrows( + ForyJsonException.class, + () -> json.fromJson("{\"bytes\":" + encodedValue + "}", ArrayField.class)); + } + assertGeneratedWhenSupported(json, ArrayField.class, codegenEnabled()); + } + + @Test + public void arrayGetter() { + ForyJson json = newJson(); + ArrayGetter value = new ArrayGetter(); + value.bytes = new byte[] {1, -2}; + assertEquals(json.toJson(value), "{\"bytes\":[1,-2]}"); + assertEquals(json.fromJson("{\"bytes\":[1,-2]}", ArrayGetter.class).bytes, value.bytes); + } + + @Test + public void conflictingFormats() { + assertThrows(ForyJsonException.class, () -> newJson().toJson(new ConflictingFormat())); + } + + public static final class ArrayField { + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] bytes; + } + + public static final class ArrayGetter { + private byte[] bytes; + + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] getBytes() { + return bytes; + } + + public void setBytes(byte[] bytes) { + this.bytes = bytes; + } + } + + public static final class ConflictingFormat { + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] bytes = {1}; + + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] getBytes() { + return bytes; + } + } + @Test public void fieldRoundTrip() { ForyJson json = newJson(); @@ -151,10 +220,10 @@ public void recordRoundTrip() throws Exception { } Class type = compileRecordClass( - "JsonBase64Record", + "JsonByteArrayRecord", "package org.apache.fory.json.records;\n" - + "import org.apache.fory.json.annotation.JsonBase64;\n" - + "public record JsonBase64Record(@JsonBase64 byte[] bytes) {}\n"); + + "import org.apache.fory.json.annotation.JsonByteArray;\n" + + "public record JsonByteArrayRecord(@JsonByteArray(JsonByteArray.Format.BASE64) byte[] bytes) {}\n"); Object value = type.getConstructor(byte[].class).newInstance((Object) new byte[] {1, 2, 3}); for (ForyJson json : new ForyJson[] {newJson(), newJsonBuilder().withFieldMode(true).build()}) { assertEquals(json.toJson(value), "{\"bytes\":\"AQID\"}"); @@ -199,13 +268,16 @@ public void rejectInvalidDeclarations() { } public static final class Base64Field { - @JsonBase64 public byte[] bytes; + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] bytes; } @JsonPropertyOrder({"text", "bytes"}) public static final class UnicodeBase64 { public String text; - @JsonBase64 public byte[] bytes; + + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] bytes; } public static final class Base64Getter { @@ -217,7 +289,7 @@ public Base64Getter(byte[] bytes) { this.bytes = bytes; } - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) public byte[] getBytes() { return bytes; } @@ -228,19 +300,20 @@ public void setBytes(byte[] bytes) { } public static final class Base64ReadOnly { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) @JsonIgnore(ignoreRead = false, ignoreWrite = true) public byte[] bytes; } public static final class Base64WriteOnly { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) @JsonIgnore(ignoreRead = true, ignoreWrite = false) public byte[] bytes; } public static final class PropertyListBase64 { - @JsonBase64 public final byte[] bytes; + @JsonByteArray(JsonByteArray.Format.BASE64) + public final byte[] bytes; @JsonCreator({"bytes"}) public PropertyListBase64(byte[] bytes) { @@ -249,7 +322,8 @@ public PropertyListBase64(byte[] bytes) { } public static final class ParameterLocalBase64 { - @JsonBase64 public final byte[] bytes; + @JsonByteArray(JsonByteArray.Format.BASE64) + public final byte[] bytes; @JsonCreator public ParameterLocalBase64(@JsonProperty("bytes") byte[] bytes) { @@ -258,7 +332,7 @@ public ParameterLocalBase64(@JsonProperty("bytes") byte[] bytes) { } public static final class Base64Always { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) @JsonProperty(include = JsonProperty.Include.ALWAYS) public byte[] bytes; } @@ -269,33 +343,41 @@ public static final class DirectCodecBase64 { } public static final class NonBinaryBase64 { - @JsonBase64 public String value = "x"; + @JsonByteArray(JsonByteArray.Format.BASE64) + public String value = "x"; } public static final class StaticBase64 { - @JsonBase64 public static byte[] value = {1}; + @JsonByteArray(JsonByteArray.Format.BASE64) + public static byte[] value = {1}; } public static final class CodecBase64 { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) @JsonCodec(Base64ByteArrayCodec.class) public byte[] value = {1}; } public static final class RawBase64 { - @JsonBase64 @JsonRawValue public byte[] value = {1}; + @JsonByteArray(JsonByteArray.Format.BASE64) + @JsonRawValue + public byte[] value = {1}; } public static final class IgnoredBase64 { - @JsonBase64 @JsonIgnore public byte[] value = {1}; + @JsonByteArray(JsonByteArray.Format.BASE64) + @JsonIgnore + public byte[] value = {1}; } public static final class AnyFieldBase64 { - @JsonBase64 @JsonAnyProperty public Map values; + @JsonByteArray(JsonByteArray.Format.BASE64) + @JsonAnyProperty + public Map values; } public static final class AnyGetterBase64 { - @JsonBase64 + @JsonByteArray(JsonByteArray.Format.BASE64) @JsonAnyGetter public Map getValues() { return null; diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java index 5446a847b0..6144f108a4 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonGraphMemoryBudgetTest.java @@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import org.apache.fory.json.annotation.JsonAnyProperty; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonSubTypes; import org.apache.fory.json.annotation.JsonUnwrapped; @@ -174,7 +175,6 @@ public void primitiveArrayOwners() { assertEquals( assertClassBudget("[true]", boolean[].class, headerBytes + Byte.BYTES), new boolean[] {true}); - assertEquals(assertClassBudget("\"AQ==\"", byte[].class, headerBytes + Byte.BYTES), new byte[] {1}); assertEquals( assertClassBudget("[2]", short[].class, headerBytes + Short.BYTES), new short[] {2}); assertEquals(assertClassBudget("[3]", int[].class, headerBytes + Integer.BYTES), new int[] {3}); @@ -193,6 +193,37 @@ public void primitiveArrayOwners() { new int[] {7}); } + @Test + public void byteArrayRepresentations() { + long arrayBytes = shallow(ArrayBytes.class) + GraphMemoryEstimates.objectArrayBytes() + 1; + assertEquals( + assertClassBudget("{\"bytes\":[1]}", ArrayBytes.class, arrayBytes).bytes, new byte[] {1}); + assertEquals( + assertClassBytesBudget( + "{\"bytes\":[1]}".getBytes(StandardCharsets.UTF_8), ArrayBytes.class, arrayBytes) + .bytes, + new byte[] {1}); + ForyJson binaryJson = jsonWithBudget(shallow(BinaryBytes.class)); + for (String encoded : new String[] {"AQ==", "A\\u0051=="}) { + String input = "{\"bytes\":\"" + encoded + "\"}"; + assertEquals(binaryJson.fromJson(input, BinaryBytes.class).bytes, new byte[] {1}); + assertEquals( + binaryJson.fromJson(input.getBytes(StandardCharsets.UTF_8), BinaryBytes.class).bytes, + new byte[] {1}); + assertEquals(jsonWithBudget(1).fromJson("\"" + encoded + "\"", byte[].class), new byte[] {1}); + } + } + + public static final class ArrayBytes { + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] bytes; + } + + public static final class BinaryBytes { + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] bytes; + } + @Test public void primitiveArrayBatches() { int headerBytes = GraphMemoryEstimates.objectArrayBytes(); diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java index a48304b886..6442962cd9 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonMixinTest.java @@ -37,7 +37,7 @@ import org.apache.fory.json.annotation.JsonAnyGetter; import org.apache.fory.json.annotation.JsonAnyProperty; import org.apache.fory.json.annotation.JsonAnySetter; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonIgnore; @@ -437,7 +437,7 @@ public void recordSelectors() throws Exception { + mixinName + " {\n" + " @JsonProperty(\"display_name\") String name;\n" - + " @JsonBase64 byte[] bytes;\n" + + " @JsonByteArray(JsonByteArray.Format.BASE64) byte[] bytes;\n" + " " + mixinName + "(\n" @@ -511,7 +511,9 @@ public abstract static class BasicMixin { String name; @JsonRawValue String body; - @JsonBase64 byte[] bytes; + + @JsonByteArray(JsonByteArray.Format.BASE64) + byte[] bytes; @JsonUnwrapped(prefix = "child_") BasicChild child; @@ -830,7 +832,9 @@ public static final class RepresentationRemoveTarget { public String name = "name"; @JsonRawValue public String raw = "1"; - @JsonBase64 public byte[] bytes = new byte[] {1}; + + @JsonByteArray(JsonByteArray.Format.ARRAY) + public byte[] bytes = new byte[] {1}; @JsonUnwrapped(prefix = "child_") public BasicChild child = new BasicChild("kid"); @@ -846,7 +850,7 @@ public abstract static class RepresentationRemoveMixin { @JsonMixinRemove(JsonRawValue.class) String raw; - @JsonMixinRemove(JsonBase64.class) + @JsonMixinRemove(JsonByteArray.class) byte[] bytes; @JsonMixinRemove(JsonUnwrapped.class) diff --git a/java/fory-json/src/test/java/org/apache/fory/json/JsonUnwrappedTest.java b/java/fory-json/src/test/java/org/apache/fory/json/JsonUnwrappedTest.java index 94f345b61e..2e1a629cfe 100644 --- a/java/fory-json/src/test/java/org/apache/fory/json/JsonUnwrappedTest.java +++ b/java/fory-json/src/test/java/org/apache/fory/json/JsonUnwrappedTest.java @@ -31,7 +31,7 @@ import java.util.LinkedHashMap; import java.util.Map; import org.apache.fory.json.annotation.JsonAnyProperty; -import org.apache.fory.json.annotation.JsonBase64; +import org.apache.fory.json.annotation.JsonByteArray; import org.apache.fory.json.annotation.JsonCodec; import org.apache.fory.json.annotation.JsonCreator; import org.apache.fory.json.annotation.JsonIgnore; @@ -801,7 +801,9 @@ public static class ValueRepresentationParent { public static class ValueRepresentationChild { @JsonRawValue public String raw; - @JsonBase64 public byte[] bytes; + + @JsonByteArray(JsonByteArray.Format.BASE64) + public byte[] bytes; } public static class ValueObjectParent { diff --git a/kotlin/fory-json-kotlin-ksp/src/main/kotlin/org/apache/fory/json/kotlin/ksp/KspModelBuilder.kt b/kotlin/fory-json-kotlin-ksp/src/main/kotlin/org/apache/fory/json/kotlin/ksp/KspModelBuilder.kt index 6e89bb7c71..2a1b03c415 100644 --- a/kotlin/fory-json-kotlin-ksp/src/main/kotlin/org/apache/fory/json/kotlin/ksp/KspModelBuilder.kt +++ b/kotlin/fory-json-kotlin-ksp/src/main/kotlin/org/apache/fory/json/kotlin/ksp/KspModelBuilder.kt @@ -47,7 +47,7 @@ internal const val JSON_MIXIN: String = "org.apache.fory.json.annotation.JsonMix internal const val JSON_SUB_TYPES: String = "org.apache.fory.json.annotation.JsonSubTypes" private const val JSON_SUB_TYPE = "org.apache.fory.json.annotation.JsonSubTypes.Type" private const val JSON_CODEC = "org.apache.fory.json.annotation.JsonCodec" -private const val JSON_BASE64 = "org.apache.fory.json.annotation.JsonBase64" +private const val JSON_BYTE_ARRAY = "org.apache.fory.json.annotation.JsonByteArray" private const val JSON_ANY_SETTER = "org.apache.fory.json.annotation.JsonAnySetter" private const val JSON_VALIDATOR = "org.apache.fory.json.annotation.JsonValidator" private const val JSON_CREATOR = "org.apache.fory.json.annotation.JsonCreator" @@ -1147,7 +1147,14 @@ internal class KspModelBuilder( result.annotations += name when (name) { JSON_CODEC -> collectCodecAnnotation(annotation, result.codecs) - JSON_BASE64 -> result.codecs += BASE64_CODEC + JSON_BYTE_ARRAY -> { + val format = + annotation.arguments.first { it.name?.asString() == "value" }.value as KSClassDeclaration + result.codecs += + if (format.simpleName.asString() == "ARRAY") + "org.apache.fory.json.codec.ArrayCodec\$SignedByteArrayCodec" + else BASE64_CODEC + } JSON_SUB_TYPES -> collectSubtypeTypes(annotation, result.types) else -> annotation.arguments.forEach { argument -> collectTypeValue(argument.value, result.types) } From 22109ef6990ba2cb547219c60953dcc30452e8fd Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Tue, 1 Sep 2026 10:14:57 +0800 Subject: [PATCH 4/4] fix(kotlin): keep corpus consumer rules stable --- .../json/corpus/PlatformCorpusChecks.kt | 22 +++++-------------- .../kotlin/json/corpus/PlatformModels.kt | 12 +++++----- .../json/corpus/KspRetentionResourceTest.kt | 13 +++++------ 3 files changed, 16 insertions(+), 31 deletions(-) diff --git a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt index bed2e24309..754d82f3ff 100644 --- a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt +++ b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformCorpusChecks.kt @@ -20,7 +20,6 @@ package org.apache.fory.integration.kotlin.json.corpus import org.apache.fory.json.ForyJson -import org.apache.fory.json.kotlin.jsonTypeRef /** Executes the same representative round trip on the JVM, Android, and Native Image. */ public object PlatformCorpusChecks { @@ -31,25 +30,11 @@ public object PlatformCorpusChecks { verifyRoot(decoded) val text = json.toJson(decoded, type) check(text.contains("\"display_label\":\"mixin\"")) - verifyRoot(json.fromJson(text, type)) - verifyRoot(json.fromJson(json.toJsonBytes(decoded, type), type)) - verifyByteArrays(json) - } - - private fun verifyByteArrays(json: ForyJson) { - val type = jsonTypeRef() - val bytes = byteArrayOf(1, -2, 3) - val value = PlatformByteArrays(bytes, bytes, bytes) - val text = json.toJson(value, type) check(text.contains("\"numbers\":[1,-2,3]")) check(text.contains("\"binary\":\"Af4D\"")) check(text.contains("\"defaultBytes\":\"Af4D\"")) - for (decoded in - listOf(json.fromJson(text, type), json.fromJson(json.toJsonBytes(value, type), type))) { - check(decoded.numbers.contentEquals(bytes)) - check(decoded.binary.contentEquals(bytes)) - check(decoded.defaultBytes.contentEquals(bytes)) - } + verifyRoot(json.fromJson(text, type)) + verifyRoot(json.fromJson(json.toJsonBytes(decoded, type), type)) } private fun verifyRoot(actual: PlatformRoot) { @@ -61,5 +46,8 @@ public object PlatformCorpusChecks { check(actual.profile.label == expected.profile.label) check(actual.token == expected.token) check(actual.box == expected.box) + check(actual.numbers.contentEquals(expected.numbers)) + check(actual.binary.contentEquals(expected.binary)) + check(actual.defaultBytes.contentEquals(expected.defaultBytes)) } } diff --git a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt index 9c90552698..41ed8b6cca 100644 --- a/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt +++ b/integration_tests/kotlin_json_corpus/src/main/kotlin/org/apache/fory/integration/kotlin/json/corpus/PlatformModels.kt @@ -105,6 +105,11 @@ public data class PlatformRoot( public val profile: PlatformJavaProfile, @field:JsonCodec(PlatformTokenCodec::class) public val token: PlatformToken, public val box: PlatformBox, + @field:JsonByteArray(JsonByteArray.Format.ARRAY) + public val numbers: ByteArray = byteArrayOf(1, -2, 3), + @get:JsonByteArray(JsonByteArray.Format.BASE64) + public val binary: ByteArray = byteArrayOf(1, -2, 3), + public val defaultBytes: ByteArray = byteArrayOf(1, -2, 3), ) internal fun platformRootValue(): PlatformRoot = @@ -117,10 +122,3 @@ internal fun platformRootValue(): PlatformRoot = token = PlatformToken("custom"), box = PlatformBox("generic"), ) - -@JsonType -public data class PlatformByteArrays( - @field:JsonByteArray(JsonByteArray.Format.ARRAY) public val numbers: ByteArray, - @get:JsonByteArray(JsonByteArray.Format.BASE64) public val binary: ByteArray, - public val defaultBytes: ByteArray, -) diff --git a/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt b/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt index e6a408a628..fcd4a5d82b 100644 --- a/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt +++ b/integration_tests/kotlin_json_corpus/src/test/kotlin/org/apache/fory/integration/kotlin/json/corpus/KspRetentionResourceTest.kt @@ -47,14 +47,14 @@ public class KspRetentionResourceTest { assertTrue(sealed.contains("class $PACKAGE.PlatformSquare"), sealed) assertTrue(sealed.contains("class $PACKAGE.PlatformOpen"), sealed) assertFalse(sealed.contains("class $PACKAGE.PlatformOpenDescendant"), sealed) - assertConstructor(rules("PlatformRoot"), "$PACKAGE.PlatformTokenCodec") - val byteArrays = rules("PlatformByteArrays") + val root = rules("PlatformRoot") + assertConstructor(root, "$PACKAGE.PlatformTokenCodec") assertTrue( - byteArrays.contains("@interface org.apache.fory.json.annotation.JsonByteArray"), - byteArrays + root.contains("@interface org.apache.fory.json.annotation.JsonByteArray"), + root, ) - assertConstructor(byteArrays, "org.apache.fory.json.codec.Base64ByteArrayCodec") - assertConstructor(byteArrays, "org.apache.fory.json.codec.ArrayCodec\$SignedByteArrayCodec") + assertConstructor(root, "org.apache.fory.json.codec.Base64ByteArrayCodec") + assertConstructor(root, "org.apache.fory.json.codec.ArrayCodec\$SignedByteArrayCodec") assertConstructor( rules("PlatformDirectOverride"), "$PACKAGE.PlatformDirectOverrideCodec", @@ -105,7 +105,6 @@ public class KspRetentionResourceTest { setOf( "META-INF/proguard/fory-json-$PACKAGE.PlatformAccount.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformBox.pro", - "META-INF/proguard/fory-json-$PACKAGE.PlatformByteArrays.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformCircle.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformDirectOverride.pro", "META-INF/proguard/fory-json-$PACKAGE.PlatformId.pro",