feat(Rust): support basic type se/de aligned with java

.github/workflows/ci.yml

@@ -248,6 +248,11 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: 3.11
+      - name: Set up Java 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: 11
+          distribution: temurin
       - name: Run Rust CI
         run: python ./ci/run_ci.py rust
 

ci/tasks/rust.py

@@ -16,6 +16,8 @@
 # under the License.
 
 import logging
+import os
+import subprocess
 from . import common
 
 
@@ -76,4 +78,12 @@ def run():
     )
     for cmd in cmds:
         common.exec_cmd(cmd)
-    logging.info("Executing fory rust tests succeeds")
+
+    logging.info("Executing Rust <-> Java11 cross-language tests")
+    os.environ["FORY_RUST_JAVA_CI"] = "1"
+    java_dir = os.path.join("..", "java")
+    subprocess.check_call(["mvn", "clean", "install", "-DskipTests"], cwd=java_dir)
+    subprocess.check_call(
+        ["mvn", "test", "-Dtest=org.apache.fory.RustXlangTest"],
+        cwd=java_dir,
+    )

java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java

@@ -44,10 +44,12 @@
 import org.apache.fory.test.TestUtils;
 import org.apache.fory.util.MurmurHash3;
 import org.testng.Assert;
+import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-/** Tests in this class need fory python/rust installed. */
+/** Tests in this class need fory rust installed. */
+// cd java/fory-core && mvn test -Dtest=org.apache.fory.RustXlangTest
 @Test
 public class RustXlangTest extends ForyTestBase {
   private static final Logger LOG = LoggerFactory.getLogger(RustXlangTest.class);
@@ -77,11 +79,27 @@ public class RustXlangTest extends ForyTestBase {
   private static final int RUST_TESTCASE_INDEX = 4;
 
   @BeforeClass
-  public void isPyforyInstalled() {
-    // TestUtils.verifyPyforyInstalled();
+  public void isRustJavaCIEnabled() {
+    String enabled = System.getenv("FORY_RUST_JAVA_CI");
+    if (enabled == null || !enabled.equals("1")) {
+      throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1");
+    }
+    boolean rustInstalled = true;
+    try {
+      Process process = new ProcessBuilder("rustc", "--version").start();
+      int exitCode = process.waitFor();
+      if (exitCode != 0) {
+        rustInstalled = false;
+      }
+    } catch (IOException | InterruptedException e) {
+      rustInstalled = false;
+    }
+    if (!rustInstalled) {
+      throw new SkipException("Skipping RustXlangTest: rust not installed");
+    }
   }
 
-  @Test(enabled = false)
+  @Test
   public void testRust() throws Exception {
     List<String> command = rustBaseCommand;
     command.set(RUST_TESTCASE_INDEX, "test_buffer");

rust/fory-core/src/serializer/string.rs

@@ -30,77 +30,16 @@ enum StrEncoding {
     Utf8 = 2,
 }
 
-fn best_coder(s: &str) -> StrEncoding {
-    let chars: Vec<char> = s.chars().collect();
-    let num_chars = chars.len();
-    if num_chars == 0 {
-        return StrEncoding::Latin1;
-    }
-
-    let sample_num = num_chars.min(64);
-    let vectorized_len = sample_num / 4;
-    let vectorized_chars = vectorized_len * 4;
-
-    let mut ascii_count = 0;
-    let mut latin1_count = 0;
-
-    for i in 0..vectorized_len {
-        let base = i * 4;
-        for j in 0..4 {
-            let c = chars[base + j] as u32;
-            if c <= 0x7F {
-                ascii_count += 1;
-                latin1_count += 1;
-            } else if c <= 0xFF {
-                latin1_count += 1;
-            }
-        }
-    }
-
-    for &c in chars.iter().take(sample_num).skip(vectorized_chars) {
-        let c = c as u32;
-        if c <= 0x7F {
-            ascii_count += 1;
-            latin1_count += 1;
-        } else if c <= 0xFF {
-            latin1_count += 1;
-        }
-    }
-
-    if latin1_count == num_chars || latin1_count == sample_num {
-        StrEncoding::Latin1
-    } else if (ascii_count as f64) >= sample_num as f64 * 0.5 {
-        StrEncoding::Utf8
-    } else {
-        StrEncoding::Utf16
-    }
-}
-
 impl Serializer for String {
     fn reserved_space() -> usize {
         mem::size_of::<i32>()
     }
 
     fn write(&self, context: &mut WriteContext) {
-        let encoding = best_coder(self);
         let mut buf = Writer::default();
-        match encoding {
-            StrEncoding::Latin1 => {
-                let len = buf.latin1_string(self);
-                let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64;
-                context.writer.var_uint36_small(bitor);
-            }
-            StrEncoding::Utf16 => {
-                let len = buf.utf16_string(self);
-                let bitor = (len as u64) << 2 | StrEncoding::Utf16 as u64;
-                context.writer.var_uint36_small(bitor);
-            }
-            StrEncoding::Utf8 => {
-                let len = buf.utf8_string(self);
-                let bitor = (len as u64) << 2 | StrEncoding::Utf8 as u64;
-                context.writer.var_uint36_small(bitor);
-            }
-        }
+        let len = buf.utf8_string(self);
+        let bitor = (len as u64) << 2 | StrEncoding::Utf8 as u64;
+        context.writer.var_uint36_small(bitor);
         context.writer.bytes(buf.dump().as_slice());
     }