Merge pull request #66 from snipsco/release/0.9.0

Release 0.9.0

.appveyor.yml

@@ -19,7 +19,7 @@ install:
   - if defined MSYS2_BITS set PATH=%PATH%;C:\msys64\mingw%MSYS2_BITS%\bin
   - rustc -V
   - cargo -V
-  - ps: (Get-Content python/snips_nlu_utils_py/Cargo.toml) | ForEach-Object { $_ -replace "^snips-nlu-utils = .*$", "snips-nlu-utils = { path = `"../..`" }" } | Set-Content python/snips_nlu_utils_py/Cargo.toml
+  - ps: (Get-Content python/ffi/Cargo.toml) | ForEach-Object { $_ -replace "^snips-nlu-utils = .*$", "snips-nlu-utils = { path = `"../..`" }" } | Set-Content python/ffi/Cargo.toml
   - "%PYTHON%\\python.exe -m pip install -r python/requirements.txt"
 
 build: false

.gitignore

@@ -98,6 +98,7 @@ celerybeat-schedule
 # virtualenv
 .venv
 venv/
+venv2/
 venv3/
 venv36/
 venv37/

.travis/before_install.sh

@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 
 # Install Rust
-if [ -z ${TRAVIS_RUST_VERSION+w} ]; then
+if [[ -z ${TRAVIS_RUST_VERSION+w} ]]; then
   curl https://sh.rustup.rs -sSf | bash -s -- -y
 fi
 
-if [ $TRAVIS_OS_NAME == "osx" ] && [ $PYTHON_TESTS == "true" ]; then
+if [[ ${TRAVIS_OS_NAME} == "osx" ]] && [[ ${PYTHON_TESTS} == "true" ]]; then
   # install pyenv
   git clone --depth 1 https://github.com/pyenv/pyenv ~/.pyenv
   PYENV_ROOT="$HOME/.pyenv"

.travis/test.sh

@@ -1,18 +1,17 @@
 #!/usr/bin/env bash
-source .travis/common.sh
 
 export PATH="/usr/local/bin:$HOME/.cargo/bin:$PATH"
 perl -p -i -e "s/^snips-nlu-utils = .*\$/snips-nlu-utils = { path = \"..\/..\" \}/g" */**/Cargo.toml
 
-if [ "${RUST_TESTS}" == "true" ]; then
+if [[ "${RUST_TESTS}" == "true" ]]; then
   echo "Rust tests..."
-  cargo test --all || die "Rust tests failed"
+  cargo test --all
 fi
 
-if [ "${PYTHON_TESTS}" == "true" ]; then
+if [[ "${PYTHON_TESTS}" == "true" ]]; then
   echo "Python tests..."
   cd python
   python -m pip install tox
-  tox || die "Python tests failed"
+  tox
   cd ..
 fi

CHANGELOG.md

@@ -1,15 +1,22 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-## [0.8.0] - 2018-02-27
+## [0.9.0] - 2019-03-05
+### Added
+- Add `hash_str_to_i32` function
+
+### Changed
+- Symbols are now tokenized separately
+- Refactor FFI: use C API instead of direct Rust to Python binding
 
+## [0.8.0] - 2019-02-27
 ### Added
 - Portuguese support
 
 ### Changed
 - Refactored the FFI
 
-## [0.7.1] - 2018-01-16
+## [0.7.1] - 2019-01-16
 ### Added
 - Python3.7 builds
 
@@ -26,6 +33,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 - Add `get_shape` function
 
+[0.9.0]: https://github.com/snipsco/snips-nlu-utils/compare/0.8.0...0.9.0
 [0.8.0]: https://github.com/snipsco/snips-nlu-utils/compare/0.7.1...0.8.0
 [0.7.1]: https://github.com/snipsco/snips-nlu-utils/compare/0.7.0...0.7.1
 [0.7.0]: https://github.com/snipsco/snips-nlu-utils/compare/0.6.1...0.7.0

Cargo.toml

@@ -1,12 +1,19 @@
 [package]
 name = "snips-nlu-utils"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Adrien Ball <[email protected]>"]
 edition = "2018"
 
+[workspace]
+members = [
+    "python/ffi"
+]
+
 [dependencies]
 itertools = "0.8"
 lazy_static = "1.2"
 libc = "0.2"
 regex = "1.1"
 unicode-normalization = "0.1.5"
+failure = "0.1"
+fnv = "1.0"

post_release.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -ex
+
+VERSION=$1
+
+if [[ -z "$VERSION" ]]
+then
+    echo "Usage: $0 <version>"
+    exit 1
+fi
+
+
+./update_version.sh ${VERSION}
+
+git commit . -m "Set post-release version to $VERSION"

python/MANIFEST.in

@@ -1,6 +1,6 @@
 include snips_nlu_utils/__version__
 include README.rst requirements.txt
 recursive-include snips_nlu_utils *.so
-include snips_nlu_utils_py/Cargo.toml
-recursive-include snips_nlu_utils_py/src *.rs
+include ffi/Cargo.toml
+recursive-include ffi/src *.rs
 global-exclude __pycache__ *.py[cod]

python/ffi/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "snips-nlu-utils-python-ffi"
+version = "0.9.0"
+authors = ["Adrien Ball <[email protected]>"]
+edition = "2018"
+
+[lib]
+name = "snips_nlu_utils_python_ffi"
+crate-type = ["cdylib"]
+
+[dependencies]
+failure = "0.1"
+libc = "0.2"
+ffi-utils = { git = "https://github.com/snipsco/snips-utils-rs", rev = "291ce1d" }
+snips-nlu-utils = { git = "https://github.com/snipsco/snips-nlu-utils", tag = "0.9.0" }

python/ffi/README.md

@@ -0,0 +1,4 @@
+Foreign Function Interface
+--
+
+This package provides ffi bindings to the utils crate.

python/ffi/src/destroy.rs

@@ -0,0 +1,24 @@
+use crate::types::CNgramArray;
+use crate::types::CTokenArray;
+use crate::Result;
+use ffi_utils::{take_back_c_string, take_back_c_string_array, CStringArray, RawPointerConverter};
+
+pub fn destroy_string_c(ptr: *mut libc::c_char) -> Result<()> {
+    take_back_c_string!(ptr);
+    Ok(())
+}
+
+pub fn destroy_string_array_c(ptr: *mut CStringArray) -> Result<()> {
+    take_back_c_string_array!(ptr);
+    Ok(())
+}
+
+pub fn destroy_token_array_c(ptr: *mut CTokenArray) -> Result<()> {
+    let _ = unsafe { CTokenArray::from_raw_pointer(ptr) };
+    Ok(())
+}
+
+pub fn destroy_ngram_array_c(ptr: *mut CNgramArray) -> Result<()> {
+    let _ = unsafe { CNgramArray::from_raw_pointer(ptr) };
+    Ok(())
+}

python/ffi/src/lib.rs

@@ -0,0 +1,96 @@
+mod destroy;
+mod string;
+mod token;
+mod types;
+
+use crate::types::CNgramArray;
+use ffi_utils::*;
+
+type Result<T> = ::std::result::Result<T, ::failure::Error>;
+
+generate_error_handling!(snips_nlu_utils_get_last_error);
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_destroy_string(ptr: *mut libc::c_char) -> SNIPS_RESULT {
+    wrap!(destroy::destroy_string_c(ptr))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_destroy_string_array(
+    ptr: *mut ::ffi_utils::CStringArray,
+) -> SNIPS_RESULT {
+    wrap!(destroy::destroy_string_array_c(ptr))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_destroy_ngram_array(
+    ptr: *mut types::CNgramArray,
+) -> SNIPS_RESULT {
+    wrap!(destroy::destroy_ngram_array_c(ptr))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_destroy_token_array(
+    ptr: *mut types::CTokenArray,
+) -> SNIPS_RESULT {
+    wrap!(destroy::destroy_token_array_c(ptr))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_remove_diacritics(
+    input: *const ::libc::c_char,
+    result: *mut *const ::libc::c_char,
+) -> SNIPS_RESULT {
+    wrap!(string::remove_diacritics_c(input, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_normalize(
+    input: *const ::libc::c_char,
+    result: *mut *const ::libc::c_char,
+) -> SNIPS_RESULT {
+    wrap!(string::normalize_c(input, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_get_shape(
+    input: *const ::libc::c_char,
+    result: *mut *const ::libc::c_char,
+) -> SNIPS_RESULT {
+    wrap!(string::get_shape_c(input, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_hash_str_to_i32(
+    input: *const ::libc::c_char,
+    result: *mut ::libc::c_int,
+) -> SNIPS_RESULT {
+    wrap!(string::hash_str_to_i32_c(input, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_tokenize(
+    input: *const ::libc::c_char,
+    language: *const ::libc::c_char,
+    result: *mut *const types::CTokenArray,
+) -> SNIPS_RESULT {
+    wrap!(token::tokenize_c(input, language, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_tokenize_light(
+    input: *const ::libc::c_char,
+    language: *const ::libc::c_char,
+    result: *mut *const ::ffi_utils::CStringArray,
+) -> SNIPS_RESULT {
+    wrap!(token::tokenize_light_c(input, language, result))
+}
+
+#[no_mangle]
+pub extern "C" fn snips_nlu_utils_compute_all_ngrams(
+    tokens: *const CStringArray,
+    max_ngram_size: libc::c_uint,
+    result: *mut *const CNgramArray,
+) -> SNIPS_RESULT {
+    wrap!(token::compute_all_ngrams_c(tokens, max_ngram_size, result))
+}

python/ffi/src/string.rs

@@ -0,0 +1,38 @@
+use crate::Result;
+use ffi_utils::{convert_to_c_string, CReprOf, RawPointerConverter};
+use snips_nlu_utils::string::*;
+use std::ffi::CStr;
+
+pub fn remove_diacritics_c(
+    input: *const ::libc::c_char,
+    result: *mut *const ::libc::c_char,
+) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let cleaned_string = remove_diacritics(str_input);
+    let cs = convert_to_c_string!(cleaned_string);
+    unsafe { *result = cs };
+    Ok(())
+}
+
+pub fn normalize_c(input: *const ::libc::c_char, result: *mut *const ::libc::c_char) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let cleaned_string = normalize(str_input);
+    let cs = convert_to_c_string!(cleaned_string);
+    unsafe { *result = cs };
+    Ok(())
+}
+
+pub fn get_shape_c(input: *const ::libc::c_char, result: *mut *const ::libc::c_char) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let shape = get_shape(str_input).to_string();
+    let cs = convert_to_c_string!(shape);
+    unsafe { *result = cs };
+    Ok(())
+}
+
+pub fn hash_str_to_i32_c(input: *const ::libc::c_char, result: *mut ::libc::c_int) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let hash = hash_str_to_i32(str_input);
+    unsafe { *result = hash };
+    Ok(())
+}

python/ffi/src/token.rs

@@ -0,0 +1,49 @@
+use crate::types::CNgramArray;
+use crate::types::CTokenArray;
+use crate::Result;
+use ffi_utils::{convert_to_c_string_array, AsRust, CReprOf, CStringArray, RawPointerConverter};
+use snips_nlu_utils::language::Language;
+use snips_nlu_utils::token::*;
+use std::ffi::CStr;
+use std::str::FromStr;
+
+pub fn tokenize_c(
+    input: *const libc::c_char,
+    language: *const libc::c_char,
+    result: *mut *const CTokenArray,
+) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let str_lang = unsafe { CStr::from_ptr(language) }.to_str()?;
+    let language = Language::from_str(str_lang)?;
+    let tokens = tokenize(str_input, language);
+    let c_token_array = CTokenArray::c_repr_of(tokens)?.into_raw_pointer();
+    unsafe { *result = c_token_array };
+    Ok(())
+}
+
+pub fn tokenize_light_c(
+    input: *const libc::c_char,
+    language: *const libc::c_char,
+    result: *mut *const CStringArray,
+) -> Result<()> {
+    let str_input = unsafe { CStr::from_ptr(input) }.to_str()?;
+    let str_lang = unsafe { CStr::from_ptr(language) }.to_str()?;
+    let language = Language::from_str(str_lang)?;
+    let tokens = tokenize_light(str_input, language);
+    let cs = convert_to_c_string_array!(tokens);
+    unsafe { *result = cs };
+    Ok(())
+}
+
+pub fn compute_all_ngrams_c(
+    tokens: *const CStringArray,
+    max_ngram_size: libc::c_uint,
+    result: *mut *const CNgramArray,
+) -> Result<()> {
+    let tokens_vec = unsafe { (*tokens).as_rust()? };
+    let tokens_slice: Vec<&str> = tokens_vec.iter().map(|token| &**token).collect();
+    let ngrams = compute_all_ngrams(&tokens_slice, max_ngram_size as usize);
+    let c_ngrams = CNgramArray::c_repr_of(ngrams)?.into_raw_pointer();
+    unsafe { *result = c_ngrams };
+    Ok(())
+}