fix: base64 encode meta data in order to support unicode

Author: cablehead

docs/src/content/docs/reference/store-api.mdx

@@ -39,7 +39,7 @@ Append frame to topic
 
 ```sh
 curl --unix-socket ./store/sock \
-    -H "xs-meta: {\"key\":\"value\"}" \
+    -H "xs-meta: $(echo -n '{\"key\":\"value\"}' | base64)" \
     -X POST --data "content" \
     "http://localhost/topic?ttl=forever"
 ```
@@ -54,7 +54,7 @@ Query Parameters:
 
 Headers:
 
-- `xs-meta` - Optional JSON metadata
+- `xs-meta` - Optional Base64-encoded JSON metadata. Must be encoded using standard Base64 to support Unicode characters.
 
 Response: Frame JSON
 

src/api.rs

@@ -4,6 +4,8 @@ use std::str::FromStr;
 
 use scru128::Scru128Id;
 
+use base64::Engine;
+
 use tokio::io::AsyncWriteExt;
 use tokio_stream::wrappers::ReceiverStream;
 use tokio_stream::StreamExt;
@@ -282,7 +284,22 @@ async fn handle_stream_append(
         .map(|x| x.to_str())
         .transpose()
         .unwrap()
-        .map(|s| serde_json::from_str(s).map_err(|_| format!("xs-meta isn't valid JSON: {}", s)))
+        .map(|s| {
+            // First decode the Base64-encoded string
+            base64::prelude::BASE64_STANDARD
+                .decode(s)
+                .map_err(|e| format!("xs-meta isn't valid Base64: {}", e))
+                .and_then(|decoded| {
+                    // Then parse the decoded bytes as UTF-8 string
+                    String::from_utf8(decoded)
+                        .map_err(|e| format!("xs-meta isn't valid UTF-8: {}", e))
+                        .and_then(|json_str| {
+                            // Finally parse the UTF-8 string as JSON
+                            serde_json::from_str(&json_str)
+                                .map_err(|e| format!("xs-meta isn't valid JSON: {}", e))
+                        })
+                })
+        })
         .transpose()
     {
         Ok(meta) => meta,

src/client/commands.rs

@@ -1,13 +1,15 @@
 use futures::StreamExt;
+
+use base64::Engine;
 use ssri::Integrity;
+use url::form_urlencoded;
 
 use http_body_util::{combinators::BoxBody, BodyExt, Empty, StreamBody};
 use hyper::body::Bytes;
 use hyper::Method;
 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
 use tokio::sync::mpsc::Receiver;
 use tokio_util::io::ReaderStream;
-use url::form_urlencoded;
 
 use super::request;
 use crate::store::{ReadOptions, TTL};
@@ -100,10 +102,9 @@ where
     let body = StreamBody::new(mapped_stream);
 
     let headers = meta.map(|meta_value| {
-        vec![(
-            "xs-meta".to_string(),
-            serde_json::to_string(meta_value).unwrap(),
-        )]
+        let json_string = serde_json::to_string(meta_value).unwrap();
+        let encoded = base64::prelude::BASE64_STANDARD.encode(json_string);
+        vec![("xs-meta".to_string(), encoded)]
     });
 
     let res = request::request(addr, Method::POST, topic, query.as_deref(), body, headers).await?;

src/main.rs

@@ -189,7 +189,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
         Command::Version(args) => version(args).await,
     };
     if let Err(err) = res {
-        eprintln!("{}", err);
+        eprintln!("command error: {:?}", err);
         std::process::exit(1);
     }
     Ok(())

tests/integration.rs

@@ -145,6 +145,38 @@ async fn test_integration() {
     assert_eq!(frame.topic, "note");
     assert_eq!(frame.context_id.to_string(), context_id);
 
+    // assert unicode support
+    let unicode_output = cmd!(
+        cargo_bin("xs"),
+        "append",
+        store_path,
+        "unicode",
+        "--meta",
+        r#"{"name": "Información"}"#
+    )
+    .stdin_bytes("contenido en español".as_bytes())
+    .read()
+    .unwrap();
+
+    let unicode_frame: Frame = serde_json::from_str(&unicode_output).unwrap();
+
+    // Verify it can be retrieved correctly
+    let retrieved = cmd!(
+        cargo_bin("xs"),
+        "get",
+        store_path,
+        &unicode_frame.id.to_string()
+    )
+    .read()
+    .unwrap();
+    let retrieved_frame: Frame = serde_json::from_str(&retrieved).unwrap();
+
+    assert_eq!(retrieved_frame.topic, "unicode");
+    assert_eq!(
+        retrieved_frame.meta.unwrap().get("name").unwrap(),
+        "Información"
+    );
+
     // Clean up
     child.kill().await.unwrap();
 }