chore: Fury Header adds language field

Signed-off-by: LiangliangSui <[email protected]>

.github/workflows/ci.yml

@@ -21,6 +21,7 @@ on:
   push:
     branches:
       - main
+      - improve_doc
       - 'releases/**'
       - 'deploy/**'
       - 'test*'

docs/specification/xlang_serialization_spec.md

@@ -139,16 +139,17 @@ Fury will write the byte order for that object into the data instead of converti
 Fury header consists starts one byte:
 
 ```
-|    2 bytes   |     4 bits    | 1 bit | 1 bit | 1 bit  | 1 bit |          optional 4 bytes          |
-+--------------+---------------+-------+-------+--------+-------+------------------------------------+
-| magic number | reserved bits |  oob  | xlang | endian | null  | unsigned int for meta start offset |
+|    2 bytes   |     4 bits    | 1 bit | 1 bit | 1 bit  | 1 bit |         1 byte           |          optional 4 bytes          |
++--------------+---------------+-------+-------+--------+-------+--------------------------+------------------------------------+
+| magic number | reserved bits |  oob  | xlang | endian | null  |  serialization language  | unsigned int for meta start offset |
 ```
 
 - magic number: used to identify fury serialization protocol, current version use `0x62d4`.
 - null flag: 1 when object is null, 0 otherwise. If an object is null, other bits won't be set.
 - endian flag: 1 when data is encoded by little endian, 0 for big endian.
 - xlang flag: 1 when serialization uses xlang format, 0 when serialization uses Fury java format.
 - oob flag: 1 when passed `BufferCallback` is not null, 0 otherwise.
+- serialization language: indicates the language used for serialization, such as JAVA, PYTHON, GO, etc.
 
 If meta share mode is enabled, an uncompressed unsigned int is appended to indicate the start offset of metadata.
 

go/fury/fury.go

@@ -80,7 +80,10 @@ const (
 	XLANG Language = iota
 	JAVA
 	PYTHON
+	CPP
 	GO
+	JAVASCRIPT
+	RUST
 )
 
 const (

java/fury-core/src/main/java/org/apache/fury/config/Language.java

@@ -26,4 +26,6 @@ public enum Language {
   PYTHON,
   CPP,
   GO,
+  JAVASCRIPT,
+  RUST,
 }

javascript/packages/fury/lib/fury.ts

@@ -111,7 +111,7 @@ export default class {
     bitmap |= ConfigFlags.isCrossLanguageFlag;
     this.binaryWriter.int16(MAGIC_NUMBER);
     this.binaryWriter.uint8(bitmap);
-    this.binaryWriter.uint8(Language.XLANG);
+    this.binaryWriter.uint8(Language.JAVASCRIPT);
     const cursor = this.binaryWriter.getCursor();
     this.binaryWriter.skip(4); // preserve 4-byte for nativeObjects start offsets.
     this.binaryWriter.uint32(0); // nativeObjects length.

javascript/packages/fury/lib/type.ts

@@ -115,6 +115,8 @@ export enum Language {
   PYTHON = 2,
   CPP = 3,
   GO = 4,
+  JAVASCRIPT = 5,
+  RUST = 6,
 }
 
 export const MAGIC_NUMBER = 0x62D4;

python/pyfury/_fury.py

@@ -586,6 +586,8 @@ class Language(enum.Enum):
     PYTHON = 2
     CPP = 3
     GO = 4
+    JAVA_SCRIPT = 5
+    RUST = 6
 
 
 @dataclass

rust/fury/src/deserializer.rs

@@ -241,10 +241,7 @@ impl<'de, 'bf: 'de> DeserializerState<'de, 'bf> {
 
     fn head(&mut self) -> Result<(), Error> {
         let _bitmap = self.reader.u8();
-        let language: Language = self.reader.u8().try_into()?;
-        if Language::XLANG != language {
-            return Err(Error::UnsupportLanguage { language });
-        }
+        let _language: Language = self.reader.u8().try_into()?;
         self.reader.skip(8); // native offset and size
         Ok(())
     }

rust/fury/src/serializer.rs

@@ -347,7 +347,7 @@ impl<'de> SerializerState<'de> {
         bitmap |= config_flags::IS_LITTLE_ENDIAN_FLAG;
         bitmap |= config_flags::IS_CROSS_LANGUAGE_FLAG;
         self.writer.u8(bitmap);
-        self.writer.u8(Language::XLANG as u8);
+        self.writer.u8(Language::RUST as u8);
         self.writer.skip(4); // native offset
         self.writer.skip(4); // native size
         self