Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
Expand Down Expand Up @@ -559,7 +559,7 @@
~/.swiftpm
~/Library/Caches/org.swift.swiftpm
swift/.build
key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**') }}
key: ${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-${{ hashFiles('swift/Package.resolved', 'swift/Package.swift', 'swift/Sources/**', 'swift/Tests/ForyXlangTests/**') }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-swiftpm-xlang-release-${{ steps.swift-package-cache.outputs.toolchain }}-
- name: Prebuild Swift xlang peer for cache reuse
Expand Down
35 changes: 35 additions & 0 deletions cpp/fory/serialization/struct_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,20 @@ struct MaximumFieldTagStruct {
FORY_STRUCT(MaximumFieldTagStruct, (value, fory::F(536870911)));
};

struct HighTagCompatibleWriter {
std::string removed;
int32_t shared;

FORY_STRUCT(HighTagCompatibleWriter, (removed, fory::F(65551)),
(shared, fory::F(536870911)));
};

struct HighTagCompatibleReader {
int32_t shared = 0;

FORY_STRUCT(HighTagCompatibleReader, (shared, fory::F(536870911)));
};

struct SignedToUnsignedWriter {
int32_t value;

Expand Down Expand Up @@ -1403,6 +1417,12 @@ TEST(StructComprehensiveTest, FieldTagRange) {
invalid.field_id = 536870912;
EXPECT_FALSE(invalid.to_bytes().ok());

Buffer invalid_wire;
invalid_wire.write_uint8(static_cast<uint8_t>((3u << 6) | (15u << 2)));
invalid_wire.write_var_uint32(536870912u - 15u);
invalid_wire.write_uint8(static_cast<uint8_t>(TypeId::INT32));
EXPECT_FALSE(FieldInfo::from_bytes(invalid_wire).ok());

TypeMeta duplicate;
duplicate.type_id = static_cast<uint32_t>(TypeId::COMPATIBLE_STRUCT);
duplicate.user_type_id = 632;
Expand All @@ -1412,6 +1432,21 @@ TEST(StructComprehensiveTest, FieldTagRange) {
EXPECT_FALSE(duplicate.to_bytes().ok());
}

TEST(StructComprehensiveTest, CompatibleHighTagsRead) {
auto writer =
Fory::builder().xlang(true).compatible(true).track_ref(false).build();
auto reader =
Fory::builder().xlang(true).compatible(true).track_ref(false).build();
ASSERT_TRUE(writer.register_struct<HighTagCompatibleWriter>(633).ok());
ASSERT_TRUE(reader.register_struct<HighTagCompatibleReader>(633).ok());

auto encoded = writer.serialize(HighTagCompatibleWriter{"skip", 42});
ASSERT_TRUE(encoded.ok()) << encoded.error().to_string();
auto decoded = reader.deserialize<HighTagCompatibleReader>(*encoded);
ASSERT_TRUE(decoded.ok()) << decoded.error().to_string();
EXPECT_EQ(decoded->shared, 42);
}

TEST(StructComprehensiveTest, NonPrimitiveFieldsSortByFieldIdentifier) {
auto fields = TypeMeta::sort_field_infos({
make_test_field_info("string_value", 20,
Expand Down
29 changes: 29 additions & 0 deletions cpp/fory/serialization/xlang_test_main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,17 @@ struct SimpleStruct {
(f7, fory::F(65551)), f8, (last, fory::F(536870911)));
};

struct ExactFieldTags {
int32_t first;
int32_t second;
int32_t last;
bool operator==(const ExactFieldTags &other) const {
return first == other.first && second == other.second && last == other.last;
}
FORY_STRUCT(ExactFieldTags, (first, fory::F(15)), (second, fory::F(65551)),
(last, fory::F(536870911)));
};

struct EvolvingOverrideStruct {
std::string f1;
bool operator==(const EvolvingOverrideStruct &other) const {
Expand Down Expand Up @@ -938,6 +949,7 @@ namespace {
void run_test_buffer(const std::string &data_file);
void run_test_buffer_var(const std::string &data_file);
void run_test_murmur_hash3(const std::string &data_file);
void run_test_exact_field_tags(const std::string &data_file);
void run_test_string_serializer(const std::string &data_file);
void run_test_cross_language_serializer(const std::string &data_file);
void run_test_simple_struct(const std::string &data_file);
Expand Down Expand Up @@ -1030,6 +1042,8 @@ int main(int argc, char **argv) {
run_test_cross_language_serializer(data_file);
} else if (case_name == "test_simple_struct") {
run_test_simple_struct(data_file);
} else if (case_name == "test_exact_field_tags") {
run_test_exact_field_tags(data_file);
} else if (case_name == "test_named_simple_struct") {
run_test_simple_named_struct(data_file);
} else if (case_name == "test_struct_evolving_override") {
Expand Down Expand Up @@ -1570,6 +1584,21 @@ void run_test_simple_struct(const std::string &data_file) {
write_file(data_file, out);
}

void run_test_exact_field_tags(const std::string &data_file) {
auto fory = build_fory(false, true);
ensure_ok(fory.register_struct<ExactFieldTags>(104),
"register ExactFieldTags");
auto bytes = read_file(data_file);
Buffer buffer = make_buffer(bytes);
auto value = read_next<ExactFieldTags>(fory, buffer);
if (!(value == ExactFieldTags{39, 40, 41})) {
fail("ExactFieldTags mismatch");
}
std::vector<uint8_t> out;
append_serialized(fory, value, out);
write_file(data_file, out);
}

void run_test_simple_named_struct(const std::string &data_file) {
auto bytes = read_file(data_file);
auto fory = build_fory(true, true);
Expand Down
19 changes: 19 additions & 0 deletions csharp/tests/Fory.XlangPeer/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ private static byte[] ExecuteCase(string caseName, byte[] input)
"test_string_serializer" => CaseStringSerializer(input),
"test_cross_language_serializer" => CaseCrossLanguageSerializer(input),
"test_simple_struct" => CaseSimpleStruct(input),
"test_exact_field_tags" => CaseExactFieldTags(input),
"test_named_simple_struct" => CaseNamedSimpleStruct(input),
"test_struct_evolving_override" => CaseStructEvolvingOverride(input),
"test_list" => CaseList(input),
Expand Down Expand Up @@ -461,6 +462,13 @@ private static byte[] CaseSimpleStruct(byte[] input)
return RoundTripSingle<SimpleStruct>(input, fory);
}

private static byte[] CaseExactFieldTags(byte[] input)
{
ForyRuntime fory = BuildFory(compatible: false);
fory.Register<ExactFieldTags>(104);
return RoundTripSingle<ExactFieldTags>(input, fory);
}

private static byte[] CaseNamedSimpleStruct(byte[] input)
{
ForyRuntime fory = BuildFory(compatible: true);
Expand Down Expand Up @@ -1282,6 +1290,17 @@ public sealed class SimpleStruct
public int Last { get; set; }
}

[ForyStruct]
public sealed class ExactFieldTags
{
[ForyField(15)]
public int First { get; set; }
[ForyField(65551)]
public int Second { get; set; }
[ForyField(536870911)]
public int Last { get; set; }
}

[ForyStruct]
public sealed class EvolvingOverrideStruct
{
Expand Down
14 changes: 14 additions & 0 deletions dart/packages/fory-test/lib/entity/xlang_test_models.dart
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,20 @@ class SimpleStruct {
int last = 0;
}

@ForyStruct()
class ExactFieldTags {
ExactFieldTags();

@ForyField(id: 15, type: Int32Type())
int first = 0;

@ForyField(id: 65551, type: Int32Type())
int second = 0;

@ForyField(id: 536870911, type: Int32Type())
int last = 0;
}

@ForyStruct()
class EvolvingOverrideStruct {
EvolvingOverrideStruct();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,11 @@ void _runCase(String caseName) {
_registerSimpleById(fory);
_roundTripFory(fory);
return;
case 'test_exact_field_tags':
final fory = _newFory();
registerXlangType(fory, ExactFieldTags, id: 104);
_roundTripFory(fory);
return;
case 'test_named_simple_struct':
final fory = _newFory(compatible: true);
_registerSimpleByName(fory);
Expand Down
17 changes: 11 additions & 6 deletions docs/compiler/schema-idl.md
Original file line number Diff line number Diff line change
Expand Up @@ -976,6 +976,11 @@ Fields define the properties of a message.
field_type field_name = field_number;
```

`field_number` is the field tag ID. It must be unique within the message and
satisfy `0 <= field_number < 2^29` (`0` through `536870911`). Keep
assigned field numbers stable and reserve removed numbers instead of reusing
them for different fields.

### With Modifiers

```protobuf
Expand Down Expand Up @@ -1149,22 +1154,22 @@ Use `ref(thread_safe=false)` in Fory IDL (or

## Field Numbers

Each field must have a unique positive integer identifier:
Each field must have a unique tag ID in the protocol range:

```protobuf
message Example {
string first = 1;
string second = 2;
string third = 3;
string first = 0;
string second = 1;
string third = 2;
}
```

**Rules and best practices:**

- Numbers must be unique within a message.
- Numbers must be positive integers.
- Numbers must satisfy `0 <= field_number < 2^29` (`0` through `536870911`).
- Gaps are allowed and are useful when fields are removed.
- Prefer sequential numbering from `1`.
- Prefer sequential numbering.
- Never reuse a removed field number for a different field.

## Type System
Expand Down
5 changes: 3 additions & 2 deletions docs/object-serialization/cpp/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@ FORY_STRUCT(DataV2, id, (timestamp, fory::F().tagged()), version);
FORY_STRUCT(Counter, FORY_PROPERTY(value, fory::F().varint()));
```

`fory::F(id)` uses explicit id-based field identity. IDs must be
non-negative:
`fory::F(id)` uses explicit id-based field identity. Configured IDs must be
unique within the complete struct schema and satisfy `0 <= id < 2^29` (`0`
through `536870911`):

```cpp
FORY_STRUCT(DataV2, (id, fory::F(0)), (timestamp, fory::F(1).tagged()),
Expand Down
3 changes: 2 additions & 1 deletion docs/object-serialization/csharp/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ This page covers schema metadata for C# generated serializers.

## `[ForyStruct]` and `[ForyField]`

Use `[ForyStruct]` to enable source-generated serializers. Use `[ForyField]` to assign an optional stable non-negative field id or to override the Fory schema type used for a field.
Use `[ForyStruct]` to enable source-generated serializers. Use `[ForyField]` to assign an optional stable field ID or to override the Fory schema type used for a field. Configured IDs must be unique within the complete struct schema and satisfy `0 <= id < 2^29` (`0` through `536870911`).

External-type serialization puts `Target` on a local abstract serializer
declaration. Its properties own the field names, IDs, schema descriptors,
Expand Down Expand Up @@ -77,6 +77,7 @@ public sealed class Metrics
```

`Id` is optional. When it is omitted, compatible mode still matches the field by name.
Once assigned, keep an ID stable and do not reuse it for a different field.

```csharp
using Apache.Fory;
Expand Down
1 change: 1 addition & 0 deletions docs/object-serialization/dart/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ String name = '';
```

Once a payload is shared across services, never reuse an `id` for a different field.
Configured IDs must satisfy `0 <= id < 2^29` (`0` through `536870911`).

An ordinary child has one flattened field namespace. IDs must therefore be
unique across all fields included from its child, superclass, and
Expand Down
4 changes: 4 additions & 0 deletions docs/object-serialization/go/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ type User struct {
}
```

Configured IDs must be unique within the struct schema and satisfy
`0 <= id < 2^29` (`0` through `536870911`). Keep assigned IDs stable and do
not reuse them for different fields.

**Benefits**:

- Smaller serialized size (numeric IDs vs field names)
Expand Down
10 changes: 8 additions & 2 deletions docs/object-serialization/go/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,8 @@ f2 := fory.New(fory.WithTrackRef(true)) // Must match!

**Common causes**:

1. **Invalid tag ID**: ID must be non-negative
1. **Invalid tag ID**: ID must satisfy `0 <= id < 2^29` (`0` through
`536870911`)

```go
// Wrong: negative ID
Expand All @@ -218,6 +219,11 @@ type Bad struct {
type Good struct {
Field int `fory:"id=0"`
}

// Wrong: ID reaches the exclusive upper bound
type TooLarge struct {
Field int `fory:"id=536870912"`
}
```

2. **Duplicate tag IDs**: Each field must have a unique ID within the struct
Expand Down Expand Up @@ -256,7 +262,7 @@ type User struct {
}
```

2. **Use field IDs for consistent ordering**: Field IDs (non-negative integers) act as aliases for field names, used for both sorting and field matching during deserialization:
2. **Use field IDs for consistent ordering**: Field IDs in the protocol range act as aliases for field names, used for both sorting and field matching during deserialization:

```go
type User struct {
Expand Down
15 changes: 8 additions & 7 deletions docs/object-serialization/java/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,10 @@ public class User {

### Parameters

| Parameter | Type | Default | Description |
| --------- | --------- | ------- | -------------------------------------- |
| `id` | `int` | `-1` | Non-negative field tag ID, or no ID |
| `dynamic` | `Dynamic` | `AUTO` | Control polymorphism for struct fields |
| Parameter | Type | Default | Description |
| --------- | --------- | ------- | -------------------------------------------- |
| `id` | `int` | `-1` | Field tag ID, or the internal no-ID sentinel |
| `dynamic` | `Dynamic` | `AUTO` | Control polymorphism for struct fields |

Use `@Nullable` on the field type or nested type position for nullable schema
metadata and `@Ref` for reference tracking. `@ForyField` does not carry either
Expand Down Expand Up @@ -123,10 +123,11 @@ public class User {

**Notes**:

- IDs must be unique within a class
- IDs must be >= 0 when configured
- If not specified, the annotation default `-1` is ignored and field name is used in metadata
- Configured IDs must satisfy `0 <= id < 2^29` (`0` through `536870911`)
- IDs must be unique within the complete struct schema, including inherited fields
- If not specified, the annotation default `-1` is the internal no-ID sentinel and the field name is used in metadata
(larger overhead)
- Once assigned, keep an ID stable and do not reuse it for a different field

**Without field IDs** (field names used in metadata):

Expand Down
18 changes: 18 additions & 0 deletions docs/object-serialization/javascript/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,24 @@ const byName = Type.struct(

Use `.` inside `typeName` to add a namespace prefix.

## Field IDs

Call `setId(id)` on a field type to assign a stable numeric field identity:

```ts
const userType = Type.struct(
{ typeId: 1001 },
{
id: Type.int64().setId(0),
name: Type.string().setId(1),
},
);
```

Configured IDs must be unique within the struct schema and satisfy
`0 <= id < 2^29` (`0` through `536870911`). Keep assigned IDs stable and do
not reuse them for different fields. Fields without an ID use their names.

## Decorator Metadata

Decorators keep the schema next to a TypeScript class declaration:
Expand Down
4 changes: 4 additions & 0 deletions docs/object-serialization/kotlin/schema-metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ Use `@ForyField(id = 1)` on constructor properties. `@field:ForyField(id = 1)` i
for field-backed properties. Do not use `@get:ForyField` or `@set:ForyField`; accessors are not
schema fields and the processor rejects them.

Configured field IDs must be unique within the struct schema and satisfy
`0 <= id < 2^29` (`0` through `536870911`). Keep assigned IDs stable and do
not reuse them for different fields. If `id` is omitted, the field name is used.

## Nullability

Use Kotlin `?` to describe nullable schema positions. Nullability is preserved inside collections
Expand Down
Loading
Loading