Switch to use ICU4X

Author: zbraniecki

Cargo.toml

@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 members = [
     "fluent-syntax",
     "fluent-bundle",
@@ -16,16 +17,17 @@ exclude = [
 
 [workspace.dependencies]
 criterion = "0.3"
-fluent-langneg = "0.13"
+fluent-langneg = "0.14"
 futures = "0.3"
 iai = "0.1"
-intl_pluralrules = "7.0.1"
 rustc-hash = "1"
 serde = "1.0"
 serde_json = "1.0"
 thiserror = "1.0"
 tokio = "1.0"
-unic-langid = "0.9"
+icu_locid = "1.4"
+icu_plurals = { version = "1.4", features = ["experimental"] }
+icu_provider = "1.4"
 
 fluent-bundle = { path = "fluent-bundle" }
 fluent-fallback = { path = "fluent-fallback" }

fluent-bundle/Cargo.toml

@@ -26,11 +26,13 @@ include = [
 ]
 
 [dependencies]
-fluent-langneg.workspace = true
+fluent-langneg = "0.14"
 fluent-syntax.workspace = true
-intl_pluralrules.workspace = true
 rustc-hash.workspace = true
-unic-langid.workspace = true
+icu_locid.workspace = true
+icu_plurals.workspace = true
+icu_provider.workspace = true
+fixed_decimal = { version = "0.5.5", features = ["ryu"] }
 intl-memoizer = { path = "../intl-memoizer" }
 self_cell = "0.10"
 smallvec = "1"
@@ -39,12 +41,12 @@ smallvec = "1"
 criterion.workspace = true
 iai.workspace = true
 serde = { workspace = true, features = ["derive"]}
-unic-langid = { workspace = true, features = ["macros"] }
 rand = "0.8"
 serde_yaml = "0.8"
 
 [features]
-default = []
+default = ["icu_provider/sync"]
+sync = ["icu_provider/sync"]
 all-benchmarks = []
 
 [[bench]]

fluent-bundle/README.md

@@ -23,7 +23,7 @@ Usage
 
 ```rust
 use fluent_bundle::{FluentBundle, FluentResource};
-use unic_langid::langid;
+use icu_locid::langid;
 
 fn main() {
     let ftl_string = "hello-world = Hello, world!".to_owned();

fluent-bundle/benches/resolver.rs

@@ -10,7 +10,7 @@ use std::rc::Rc;
 
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue};
 use fluent_syntax::ast;
-use unic_langid::langid;
+use icu_locid::langid;
 
 fn read_file(path: &str) -> Result<String, io::Error> {
     let mut f = File::open(path)?;

fluent-bundle/benches/resolver_iai.rs

@@ -1,6 +1,6 @@
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue};
 use fluent_syntax::ast;
-use unic_langid::{langid, LanguageIdentifier};
+use icu_locid::{langid, LanguageIdentifier};
 
 const LANG_EN: LanguageIdentifier = langid!("en");
 

fluent-bundle/examples/custom_formatter.rs

@@ -2,7 +2,7 @@
 // to format selected types of values.
 //
 // This allows users to plug their own number formatter to Fluent.
-use unic_langid::LanguageIdentifier;
+use icu_locid::LanguageIdentifier;
 
 use fluent_bundle::memoizer::MemoizerKind;
 use fluent_bundle::types::{FluentNumber, FluentNumberOptions};

fluent-bundle/examples/custom_type.rs

@@ -9,8 +9,8 @@
 // Lastly, we'll also create a new formatter which will be memoizable.
 //
 // The type and its options are modelled after ECMA402 Intl.DateTimeFormat.
+use icu_locid::LanguageIdentifier;
 use intl_memoizer::Memoizable;
-use unic_langid::LanguageIdentifier;
 
 use fluent_bundle::types::FluentType;
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue};

fluent-bundle/examples/external_arguments.rs

@@ -1,5 +1,5 @@
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue};
-use unic_langid::langid;
+use icu_locid::langid;
 
 fn main() {
     let ftl_string = String::from(

fluent-bundle/examples/functions.rs

@@ -1,5 +1,5 @@
 use fluent_bundle::{FluentBundle, FluentResource, FluentValue};
-use unic_langid::langid;
+use icu_locid::langid;
 
 fn main() {
     // We define the resources here so that they outlive

fluent-bundle/examples/simple-app.rs

@@ -19,14 +19,14 @@
 //! default one.
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue};
 use fluent_langneg::{negotiate_languages, NegotiationStrategy};
+use icu_locid::{langid, LanguageIdentifier};
 use std::env;
 use std::fs;
 use std::fs::File;
 use std::io;
 use std::io::prelude::*;
 use std::path::Path;
 use std::str::FromStr;
-use unic_langid::{langid, LanguageIdentifier};
 
 /// We need a generic file read helper function to
 /// read the localization resource file.

fluent-bundle/src/bundle.rs

@@ -12,8 +12,8 @@ use std::default::Default;
 use std::fmt;
 
 use fluent_syntax::ast;
+use icu_locid::LanguageIdentifier;
 use intl_memoizer::IntlLangMemoizer;
-use unic_langid::LanguageIdentifier;
 
 use crate::args::FluentArgs;
 use crate::entry::Entry;
@@ -32,7 +32,7 @@ use crate::types::FluentValue;
 ///
 /// ```
 /// use fluent_bundle::{FluentBundle, FluentResource, FluentValue, FluentArgs};
-/// use unic_langid::langid;
+/// use icu_locid::langid;
 ///
 /// // 1. Create a FluentResource
 ///
@@ -163,7 +163,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("
     /// hello = Hi!
@@ -253,7 +253,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("
     /// hello = Hi!
@@ -359,7 +359,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("hello = Hi!");
     /// let resource = FluentResource::try_new(ftl_string)
@@ -384,7 +384,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("hello-world = Hello World!");
     /// let resource = FluentResource::try_new(ftl_string)
@@ -412,7 +412,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("hello-world = Hello World!");
     /// let resource = FluentResource::try_new(ftl_string)
@@ -459,7 +459,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("hello-world = Hello World!");
     /// let resource = FluentResource::try_new(ftl_string)
@@ -508,7 +508,7 @@ impl<R, M> FluentBundle<R, M> {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource, FluentValue};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("length = { STRLEN(\"12345\") }");
     /// let resource = FluentResource::try_new(ftl_string)
@@ -567,7 +567,7 @@ impl<R> FluentBundle<R, IntlLangMemoizer> {
     /// ```
     /// use fluent_bundle::FluentBundle;
     /// use fluent_bundle::FluentResource;
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let langid_en = langid!("en-US");
     /// let mut bundle: FluentBundle<FluentResource> = FluentBundle::new(vec![langid_en]);

fluent-bundle/src/concurrent.rs

@@ -1,6 +1,6 @@
+use icu_locid::LanguageIdentifier;
 use intl_memoizer::{concurrent::IntlLangMemoizer, Memoizable};
 use rustc_hash::FxHashMap;
-use unic_langid::LanguageIdentifier;
 
 use crate::memoizer::MemoizerKind;
 use crate::types::FluentType;
@@ -23,7 +23,7 @@ impl<R> FluentBundle<R> {
     /// ```
     /// use fluent_bundle::concurrent::FluentBundle;
     /// use fluent_bundle::FluentResource;
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let langid_en = langid!("en-US");
     /// let mut bundle: FluentBundle<FluentResource> =

fluent-bundle/src/errors.rs

@@ -33,7 +33,7 @@ pub enum FluentError {
     ///
     /// ```
     /// use fluent_bundle::{FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
     /// let ftl_string = String::from("intro = Welcome, { $name }.");
     /// let res1 = FluentResource::try_new(ftl_string)

fluent-bundle/src/lib.rs

@@ -18,7 +18,7 @@
 //! ```
 //! use fluent_bundle::{FluentBundle, FluentValue, FluentResource, FluentArgs};
 //! // Used to provide a locale for the bundle.
-//! use unic_langid::langid;
+//! use icu_locid::langid;
 //!
 //! // 1. Crate a FluentResource
 //!

fluent-bundle/src/memoizer.rs

@@ -1,6 +1,6 @@
 use crate::types::FluentType;
+use icu_locid::LanguageIdentifier;
 use intl_memoizer::Memoizable;
-use unic_langid::LanguageIdentifier;
 
 /// This trait contains thread-safe methods which extend [intl_memoizer::IntlLangMemoizer].
 /// It is used as the generic bound in this crate when a memoizer is needed.

fluent-bundle/src/types/mod.rs

@@ -22,7 +22,7 @@ use std::borrow::{Borrow, Cow};
 use std::fmt;
 use std::str::FromStr;
 
-use intl_pluralrules::{PluralCategory, PluralRuleType};
+use icu_plurals::{PluralCategory, PluralRuleType};
 
 use crate::memoizer::MemoizerKind;
 use crate::resolver::Scope;
@@ -157,10 +157,10 @@ impl<'source> FluentValue<'source> {
     /// ```
     /// use fluent_bundle::resolver::Scope;
     /// use fluent_bundle::{types::FluentValue, FluentBundle, FluentResource};
-    /// use unic_langid::langid;
+    /// use icu_locid::langid;
     ///
-    /// let langid_ars = langid!("en");
-    /// let bundle: FluentBundle<FluentResource> = FluentBundle::new(vec![langid_ars]);
+    /// let langid_en = langid!("en");
+    /// let bundle: FluentBundle<FluentResource> = FluentBundle::new(vec![langid_en]);
     /// let scope = Scope::new(&bundle, None, None);
     ///
     /// // Matching examples:
@@ -189,12 +189,12 @@ impl<'source> FluentValue<'source> {
             (&FluentValue::Number(ref a), &FluentValue::Number(ref b)) => a == b,
             (&FluentValue::String(ref a), &FluentValue::Number(ref b)) => {
                 let cat = match a.as_ref() {
-                    "zero" => PluralCategory::ZERO,
-                    "one" => PluralCategory::ONE,
-                    "two" => PluralCategory::TWO,
-                    "few" => PluralCategory::FEW,
-                    "many" => PluralCategory::MANY,
-                    "other" => PluralCategory::OTHER,
+                    "zero" => PluralCategory::Zero,
+                    "one" => PluralCategory::One,
+                    "two" => PluralCategory::Two,
+                    "few" => PluralCategory::Few,
+                    "many" => PluralCategory::Many,
+                    "other" => PluralCategory::Other,
                     _ => return false,
                 };
                 // This string matches a plural rule keyword. Check if the number
@@ -203,8 +203,8 @@ impl<'source> FluentValue<'source> {
                     .bundle
                     .intls
                     .with_try_get_threadsafe::<PluralRules, _, _>(
-                        (PluralRuleType::CARDINAL,),
-                        |pr| pr.0.select(b) == Ok(cat),
+                        (PluralRuleType::Cardinal,),
+                        |pr| pr.0.category_for(b) == cat,
                     )
                     .unwrap()
             }

fluent-bundle/src/types/number.rs

@@ -1,9 +1,8 @@
 use std::borrow::Cow;
-use std::convert::TryInto;
 use std::default::Default;
 use std::str::FromStr;
 
-use intl_pluralrules::operands::PluralOperands;
+use icu_plurals::PluralOperands;
 
 use crate::args::FluentArgs;
 use crate::types::FluentValue;
@@ -219,18 +218,12 @@ macro_rules! from_num {
 
 impl From<&FluentNumber> for PluralOperands {
     fn from(input: &FluentNumber) -> Self {
-        let mut operands: Self = input
-            .value
-            .try_into()
-            .expect("Failed to generate operands out of FluentNumber");
+        use fixed_decimal::{FixedDecimal, FloatPrecision};
+        let mut fd = FixedDecimal::try_from_f64(input.value, FloatPrecision::Floating).unwrap();
         if let Some(mfd) = input.options.minimum_fraction_digits {
-            if mfd > operands.v {
-                operands.f *= 10_u64.pow(mfd as u32 - operands.v as u32);
-                operands.v = mfd;
-            }
+            fd.pad_end(-(mfd as i16));
         }
-        // XXX: Add support for other options.
-        operands
+        (&fd).into()
     }
 }
 

fluent-bundle/src/types/plural.rs

@@ -1,22 +1,18 @@
-use fluent_langneg::{negotiate_languages, NegotiationStrategy};
+use icu_locid::LanguageIdentifier;
+use icu_plurals::{PluralRuleType, PluralRules as IntlPluralRules};
 use intl_memoizer::Memoizable;
-use intl_pluralrules::{PluralRuleType, PluralRules as IntlPluralRules};
-use unic_langid::LanguageIdentifier;
 
 pub struct PluralRules(pub IntlPluralRules);
 
 impl Memoizable for PluralRules {
     type Args = (PluralRuleType,);
     type Error = &'static str;
     fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> {
-        let default_lang: LanguageIdentifier = "en".parse().unwrap();
-        let pr_lang = negotiate_languages(
-            &[lang],
-            &IntlPluralRules::get_locales(args.0),
-            Some(&default_lang),
-            NegotiationStrategy::Lookup,
-        )[0]
-        .clone();
-        Ok(Self(IntlPluralRules::create(pr_lang, args.0)?))
+        let inner = match args.0 {
+            PluralRuleType::Cardinal => IntlPluralRules::try_new_cardinal(&lang.into()),
+            PluralRuleType::Ordinal => IntlPluralRules::try_new_ordinal(&lang.into()),
+            _ => todo!(),
+        };
+        Ok(Self(inner.unwrap()))
     }
 }

fluent-bundle/tests/bundle.rs

@@ -1,6 +1,6 @@
 use fluent_bundle::{FluentArgs, FluentBundle, FluentResource};
+use icu_locid::langid;
 use std::borrow::Cow;
-use unic_langid::langid;
 
 #[test]
 fn add_resource_override() {