Merge pull request #42 from simdjson/add_rust_serd

fix Rust through CMake

Author: FranciscoThiesen

Dockerfile

@@ -27,7 +27,7 @@ ARG GROUP_ID
 LABEL maintainer "LLVM Developers"
 # Install packages for minimal useful image.
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends build-essential  ca-certificates libcurl4-openssl-dev cmake make wget python3 python3-dev sudo curl ninja-build vim git binutils && \
+    apt-get install -y --no-install-recommends build-essential  ca-certificates rust-all libcurl4-openssl-dev cmake make wget python3 python3-dev sudo curl ninja-build vim git binutils && \
     rm -rf /var/lib/apt/lists/*
 # Copy build results of stage 1 to /usr/local.
 COPY --from=builder /tmp/clang-install/ /usr/local/

benchmarks/CMakeLists.txt

@@ -7,5 +7,40 @@ CPMAddPackage(
   PATCHES reflect-cpp.patch
   GIT_TAG v0.13.0
 )
+
+
+
+if(NOT WIN32)
+# We want the check whether Rust is available before trying to build a crate.
+CPMAddPackage(
+  NAME corrosion
+  GITHUB_REPOSITORY corrosion-rs/corrosion
+  VERSION 0.4.4
+  DOWNLOAD_ONLY ON
+  OPTIONS "Rust_FIND_QUIETLY OFF"
+)
+include("${corrosion_SOURCE_DIR}/cmake/FindRust.cmake")
+endif()
+
+if(RUST_FOUND)
+  message(STATUS "Rust found: " ${Rust_VERSION} )
+  add_subdirectory("${corrosion_SOURCE_DIR}" "${PROJECT_BINARY_DIR}/_deps/corrosion" EXCLUDE_FROM_ALL)
+  # Important: we want to build in release mode!
+  corrosion_import_crate(MANIFEST_PATH "competitors/serde-benchmark/Cargo.toml" NO_LINKER_OVERRIDE PROFILE release)
+else()
+  message(STATUS "Rust/Cargo is unavailable." )
+  message(STATUS "We will not benchmark serde-benchmark." )
+  if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    message(STATUS "Under macOS, you may be able to install rust with")
+    message(STATUS "curl https://sh.rustup.rs -sSf | sh")
+  elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    message(STATUS "Under Linux, you may be able to install rust with a command such as")
+    message(STATUS "apt-get install cargo" )
+    message(STATUS "or" )
+    message(STATUS "curl https://sh.rustup.rs -sSf | sh")
+  endif()
+endif()
+
+
 add_subdirectory(simpleparser)
 add_subdirectory(src)

benchmarks/competitors/serde-benchmark/Cargo.lock

@@ -0,0 +1,17 @@
+[package]
+name = "serde-benchmark"
+version = "0.1.0"
+
+[lib]
+path = "lib.rs"
+crate-type = ["cdylib"]
+
+[dependencies]
+serde = { version = "1.0", features = ["derive"] }
+libc = "0.2"
+serde_json = "1.0"
+
+[profile.release]
+opt-level = 3
+debug = false
+lto = true

benchmarks/competitors/serde-benchmark/Cargo.toml

@@ -0,0 +1,18 @@
+## Rust Serde FFI
+
+This folder includes FFI bindings for rust/serde.
+
+### Links
+
+- https://github.com/eqrion/cbindgen/blob/master/docs.md
+- https://gist.github.com/zbraniecki/b251714d77ffebbc73c03447f2b2c69f
+- https://michael-f-bryan.github.io/rust-ffi-guide/setting_up.html
+
+### Building
+
+- Generating cbindgen output
+  - Install dependencies with `brew install cbindgen` or `apt-get install cbindgen` or `cargo install cbindgen` or the equivalent: we used `cargo install --version 0.23.0 cbindgen`.
+  - Go to the directory where this README.md file is located
+  - Generate with `cbindgen --config cbindgen.toml --crate serde-benchmark --output serde_benchmark.h`
+- Building
+  - Run with `cargo build --release`

benchmarks/competitors/serde-benchmark/README.md

@@ -0,0 +1,12 @@
+autogen_warning = "/* Warning, this file is autogenerated by cbindgen. Don't modify this manually. */"
+include_version = true
+braces = "SameLine"
+line_length = 100
+tab_width = 2
+language = "C++"
+namespaces = ["serde_benchmark"]
+include_guard = "serde_benchmark_ffi_h"
+
+[parse]
+parse_deps = true
+include = ["serde_json", "serde"]

benchmarks/competitors/serde-benchmark/cbindgen.toml

@@ -0,0 +1,170 @@
+extern crate serde;
+extern crate serde_json;
+extern crate libc;
+use std::slice;
+
+use libc::{c_char, size_t};
+use serde::{Serialize, Deserialize};
+
+/******************************************************/
+/******************************************************/
+/**
+ * Warning: the C++ code may not generate the same JSON.
+ */
+ /******************************************************/
+ /******************************************************/
+
+// This has no equivalent in C++:
+#[derive(Serialize, Deserialize)]
+pub struct Metadata {
+    result_type: String,
+    iso_language_code: String,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct User {
+    id: i64,
+    id_str: String,
+    name: String,
+    screen_name: String,
+    location: String,
+    description: String,
+    // C++ does not have those:
+    // url: Option<String>,
+    //protected: bool,
+    //listed_count: i64,
+    //created_at: String,
+    //favourites_count: i64,
+    //utc_offset: Option<i64>,
+    //time_zone: Option<String>,
+    //geo_enabled: bool,
+    verified: bool,
+    followers_count: i64,
+    friends_count: i64,
+    statuses_count: i64,
+    // C++ does not have those:
+    //lang: String,
+    //profile_background_color: String,
+    //profile_background_image_url: String,
+    //profile_background_image_url_https: String,
+    //profile_background_tile: bool,
+    //profile_image_url: String,
+    //profile_image_url_https: String,
+    //profile_banner_url: Option<String>,
+    //profile_link_color: String,
+    //profile_sidebar_border_color: String,
+    //profile_sidebar_fill_color: String,
+    //profile_text_color: String,
+    //profile_use_background_image: bool,
+    //default_profile: bool,
+    //default_profile_image: bool,
+    //following: bool,
+    //follow_request_sent: bool,
+    //notifications: bool,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Hashtag {
+    text: String,
+
+    // C++ has those but D. Lemire does not know what they are, they don't appear in the JSON:
+    // int64_t indices_start;
+    // int64_t indices_end;
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Url {
+    url: String,
+    expanded_url: String,
+    display_url: String,
+    // C++ has those but D. Lemire does not know what they are, they don't appear in the JSON:
+    // int64_t indices_start;
+    // int64_t indices_end;
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct UserMention {
+    id: i64,
+    name: String,
+    screen_name: String,
+    // Not in the C++ equivalent:
+    //id_str: String,
+    //indices: Vec<i64>,
+    // C++ has those but D. Lemire does not know what they are, they don't appear in the JSON:
+    // int64_t indices_start;
+    // int64_t indices_end;
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Entities {
+    hashtags: Vec<Hashtag>,
+    urls: Vec<Url>,
+    user_mentions: Vec<UserMention>,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Status {
+    created_at: String,
+    id: i64,
+    text: String,
+    user: User,
+    entities: Entities,
+    retweet_count: i64,
+    favorite_count: i64,
+    favorited: bool,
+    retweeted: bool,
+    // None of these are in the C++ equivalent:
+    /*
+    metadata: Metadata,
+    id_str: String,
+    source: String,
+    truncated: bool,
+    in_reply_to_status_id: Option<i64>,
+    in_reply_to_status_id_str: Option<String>,
+    in_reply_to_user_id: Option<i64>,
+    in_reply_to_user_id_str: Option<String>,
+    in_reply_to_screen_name: Option<String>,
+    geo: Option<String>,
+    coordinates: Option<String>,
+    place: Option<String>,
+    contributors: Option<String>,
+    lang: String,
+    */
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct TwitterData {
+    statuses: Vec<Status>,
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn  twitter_from_str(raw_input: *const c_char, raw_input_length: size_t) -> *mut TwitterData {
+  let input = std::str::from_utf8_unchecked(slice::from_raw_parts(raw_input as *const u8, raw_input_length));
+  match serde_json::from_str(&input) {
+    Ok(result) => Box::into_raw(Box::new(result)),
+    Err(_) => std::ptr::null_mut(),
+  }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn str_from_twitter(raw: *mut TwitterData) -> *const c_char {
+  let twitter_thing = { &*raw };
+  let serialized = serde_json::to_string(&twitter_thing).unwrap();
+  return std::ffi::CString::new(serialized.as_str()).unwrap().into_raw()
+}
+
+
+#[no_mangle]
+pub unsafe extern "C" fn free_twitter(raw: *mut TwitterData) {
+  if raw.is_null() {
+    return;
+  }
+
+  drop(Box::from_raw(raw))
+}
+
+
+#[no_mangle]
+pub unsafe extern fn free_string(ptr: *const c_char) {
+    let _ = std::ffi::CString::from_raw(ptr as *mut _);
+}