Merge branch 'main' into restructure

Author: robertbastian

.gitignore

@@ -25,7 +25,7 @@ dhat-heap.json
 # Do not check-in binary file tree test data
 provider/testdata/data/bincode
 provider/testdata/data/postcard/*
-!provider/testdata/data/postcard/fingerprints.txt
+!provider/testdata/data/postcard/fingerprints.csv
 
 # Ignore irrelevant files that get generated on macOS
 **/.DS_Store

Cargo.lock

@@ -26,7 +26,6 @@ members = [
     "ffi/capi_cdylib",
     "ffi/diplomat",
     "ffi/capi_staticlib",
-    "ffi/ecma402",
     "ffi/freertos",
     "provider/adapters",
     "provider/blob",
@@ -55,7 +54,11 @@ members = [
     "utils/zerovec",
     "utils/zerovec/derive",
 ]
-exclude = ["provider/testdata/data/const"]
+exclude = [
+    # This crate uses datagen which requires downloading ZIP files during build,
+    # so we don't want to build it as part of the workspace.
+    "ffi/ecma402",
+]
 
 # LTO is needed for WASM and other size-optimized builds,
 # and it improve the performance of benchmarks

Cargo.toml

@@ -262,14 +262,14 @@ mod datagen {
         pub fn make_conditional(
             &mut self,
             pattern: &str,
-            regex: &str,
+            regex: &StringMatcher<'static>,
             alternative_pattern: &str,
         ) -> Result<(), DataError> {
             let old = ListJoinerPattern::from_str(pattern, true, true)?;
             for i in 0..12 {
                 if self.0[i].default == old {
                     self.0[i].special_case = Some(SpecialCasePattern {
-                        condition: StringMatcher::new(regex)?,
+                        condition: regex.clone(),
                         pattern: ListJoinerPattern::from_str(
                             alternative_pattern,
                             i % 4 == 0 || i % 4 == 3, // allow_prefix = start or pair
@@ -363,7 +363,7 @@ pub(crate) mod test {
         ])
         .unwrap();
         patterns
-            .make_conditional("{0}. {1}", "A", "{0} :o {1}")
+            .make_conditional("{0}. {1}", &StringMatcher::new("A").unwrap(), "{0} :o {1}")
             .unwrap();
         patterns
     }

components/list/src/provider.rs

@@ -28,15 +28,13 @@ all-features = true
 
 [dependencies]
 ecma402_traits = { version = "0.2.0" }
-icu = { version = "0.6", path = "../../components/icu", default-features = false }
+icu = { version = "0.6", path = "../../components/icu", features = ["std"] }
 icu_provider = { version = "0.6", path = "../../provider/core" }
-icu_plurals = { version = "0.6", path = "../../components/plurals", features = ["std"] }
 
+# Bake dependencies
+icu_plurals = { version = "0.6", path = "../../components/plurals" }
+zerovec = { version = "0.7", path = "../../utils/zerovec" }
 
-[dev-dependencies]
-criterion = "0.3"
-serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] }
-serde_json = {version = "1.0" }
-icu_locid = { version = "0.6", path = "../../components/locid" }
-icu_provider = { version = "0.6", path = "../../provider/core" }
-icu_testdata = { version = "0.6", path = "../../provider/testdata" }
+[build-dependencies]
+icu = { version = "0.6", path = "../../components/icu", features = ["std"] }
+icu_datagen = { version = "0.6", path = "../../provider/datagen" }

ffi/ecma402/Cargo.toml

@@ -0,0 +1,22 @@
+// This file is part of ICU4X. For terms of use, please see the file
+// called LICENSE at the top level of the ICU4X source tree
+// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+
+    icu_datagen::datagen(
+        None,
+        &icu_datagen::keys(&["plurals/cardinal@1", "plurals/ordinal@1"]),
+        &icu_datagen::SourceData::default()
+            .with_cldr_latest(icu_datagen::CldrLocaleSubset::Full)
+            .unwrap(),
+        vec![icu_datagen::Out::Module {
+            mod_directory: std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap())
+                .join("baked"),
+            pretty: false,
+            insert_feature_gates: false,
+        }],
+    )
+    .unwrap();
+}

ffi/ecma402/build.rs

@@ -19,7 +19,7 @@ use icu::locid::LanguageIdentifier;
 
 /// Implements ECMA-402 [`Intl.PluralRules`][link].
 ///
-/// [link]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/PluralRulres/PluralRules
+/// [link]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/PluralRules
 pub mod pluralrules;
 
 /// An adapter between [`icu::locid`] and [`ecma402_traits`].
@@ -46,3 +46,9 @@ impl std::fmt::Display for crate::Locale {
         }
     }
 }
+
+mod provider {
+    include!(concat!(env!("OUT_DIR"), "/baked/mod.rs"));
+}
+
+pub(crate) use provider::BakedDataProvider;

ffi/ecma402/src/lib.rs

@@ -241,9 +241,7 @@ impl ecma402_traits::pluralrules::PluralRules for PluralRules {
         L: ecma402_traits::Locale,
         Self: Sized,
     {
-        // TODO: introduce a global data provider here.
-        let dp = icu_provider::inv::InvariantDataProvider;
-        Self::try_new_with_provider(l, opts, &dp)
+        Self::try_new_with_provider(l, opts, &crate::BakedDataProvider)
     }
 
     fn select<W>(&self, number: f64, writer: &mut W) -> std::fmt::Result
@@ -330,13 +328,9 @@ mod testing {
                 expected: vec!["other", "other", "other", "other", "other", "other"],
             },
         ];
-        let provider = icu_testdata::get_provider();
         for (i, test) in tests.into_iter().enumerate() {
-            let plr = super::PluralRules::try_new_with_provider(
-                crate::Locale::FromLocale(test.locale),
-                test.opts,
-                &provider,
-            )?;
+            let plr =
+                super::PluralRules::try_new(crate::Locale::FromLocale(test.locale), test.opts)?;
             assert_eq!(
                 test.numbers
                     .iter()

ffi/ecma402/src/pluralrules.rs

@@ -3,25 +3,181 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use super::*;
+use crate::helpers::result_is_err_missing_resource_options;
 
-pub trait LocaleFallbackProvider {
-    fn load_fallback_metadata_for_key(
+/// A data provider wrapper that performs locale fallback. This enables arbitrary locales to be
+/// handled at runtime.
+///
+/// # Examples
+///
+/// ```
+/// use icu_locid::locale;
+/// use icu_provider::prelude::*;
+/// use icu_provider::hello_world::*;
+/// use icu_provider_adapters::fallback::LocaleFallbackProvider;
+///
+/// let provider = icu_testdata::get_provider();
+///
+/// let req = DataRequest {
+///     options: locale!("ja-JP").into(),
+///     metadata: Default::default(),
+/// };
+///
+/// // The provider does not have data for "ja-JP":
+/// let result: Result<DataResponse<HelloWorldV1Marker>, DataError> =
+///     provider.load_resource(&req);
+/// assert!(matches!(result, Err(_)));
+///
+/// // But if we wrap the provider in a fallback provider...
+/// let provider = LocaleFallbackProvider::try_new(provider)
+///     .expect("Fallback data present");
+///
+/// // ...then we can load "ja-JP" based on "ja" data
+/// let result: Result<DataResponse<HelloWorldV1Marker>, DataError> =
+///     provider.load_resource(&req);
+/// assert!(matches!(result, Ok(_)));
+///
+/// assert_eq!(
+///     "こんにちは世界",
+///     result.unwrap().take_payload().unwrap().get().message
+/// );
+/// ```
+pub struct LocaleFallbackProvider<P> {
+    pub inner: P,
+    fallbacker: LocaleFallbacker,
+}
+
+impl<P> LocaleFallbackProvider<P>
+where
+    P: ResourceProvider<LocaleFallbackLikelySubtagsV1Marker>
+        + ResourceProvider<LocaleFallbackParentsV1Marker>,
+{
+    /// Create a [`LocaleFallbackProvider`] by wrapping another data provider and then loading
+    /// fallback data from it.
+    ///
+    /// If the data provider being wrapped does not contain fallback data, use
+    /// [`LocaleFallbackProvider::new_with_fallbacker`].
+    pub fn try_new(provider: P) -> Result<Self, DataError> {
+        let fallbacker = LocaleFallbacker::try_new(&provider)?;
+        Ok(Self {
+            inner: provider,
+            fallbacker,
+        })
+    }
+}
+
+impl<P> LocaleFallbackProvider<P> {
+    /// Wrap a provider with an arbitrary fallback engine.
+    ///
+    /// This relaxes the requirement that the wrapped provider contains its own fallback data.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use icu_locid::locale;
+    /// use icu_provider::hello_world::*;
+    /// use icu_provider::prelude::*;
+    /// use icu_provider_adapters::fallback::{LocaleFallbacker, LocaleFallbackProvider};
+    ///
+    /// let provider = HelloWorldProvider;
+    ///
+    /// let req = DataRequest {
+    ///     options: locale!("de-CH").into(),
+    ///     metadata: Default::default(),
+    /// };
+    ///
+    /// // There is no "de-CH" data in the `HelloWorldProvider`
+    /// ResourceProvider::<HelloWorldV1Marker>::load_resource(&provider, &req)
+    ///     .expect_err("No data for de-CH");
+    ///
+    /// // `HelloWorldProvider` does not contain fallback data,
+    /// // but we can fetch it from `icu_testdata`, and then
+    /// // use it to create the fallbacking data provider.
+    /// let fallbacker = LocaleFallbacker::try_new(&icu_testdata::get_provider())
+    ///     .expect("Fallback data present");
+    /// let provider = LocaleFallbackProvider::new_with_fallbacker(provider, fallbacker);
+    ///
+    /// // Now we can load the "de-CH" data via fallback to "de".
+    /// let german_hello_world: DataPayload<HelloWorldV1Marker> = provider
+    ///     .load_resource(&req)
+    ///     .expect("Loading should succeed")
+    ///     .take_payload()
+    ///     .expect("Data should be present");
+    ///
+    /// assert_eq!("Hallo Welt", german_hello_world.get().message);
+    /// ```
+    pub fn new_with_fallbacker(provider: P, fallbacker: LocaleFallbacker) -> Self {
+        Self {
+            inner: provider,
+            fallbacker,
+        }
+    }
+
+    fn run_fallback<F, R>(
         &self,
         key: ResourceKey,
-    ) -> Result<LocaleFallbackKeyMetadata, DataError>;
+        base_req: &DataRequest,
+        mut f: F,
+    ) -> Result<R, DataError>
+    where
+        F: FnMut(&DataRequest) -> Result<R, DataError>,
+    {
+        let key_fallbacker = self.fallbacker.for_key(key);
+        let mut fallback_iterator = key_fallbacker.fallback_for(base_req.clone());
+        while !fallback_iterator.get().options.is_empty() {
+            let result = f(fallback_iterator.get());
+            if !result_is_err_missing_resource_options(&result) {
+                // Log the original request rather than the fallback request
+                return result.map_err(|e| e.with_req(key, base_req));
+            }
+            fallback_iterator.step();
+        }
+        Err(DataErrorKind::MissingResourceOptions.with_req(key, base_req))
+    }
 }
 
-pub struct LocaleFallbackAdapter<P>(pub P);
+impl<P> AnyProvider for LocaleFallbackProvider<P>
+where
+    P: AnyProvider,
+{
+    fn load_any(&self, key: ResourceKey, base_req: &DataRequest) -> Result<AnyResponse, DataError> {
+        self.run_fallback(key, base_req, |req| self.inner.load_any(key, req))
+    }
+}
 
-impl<P> BufferProvider for LocaleFallbackAdapter<P>
+impl<P> BufferProvider for LocaleFallbackProvider<P>
 where
-    P: LocaleFallbackProvider + BufferProvider,
+    P: BufferProvider,
 {
     fn load_buffer(
         &self,
-        _key: ResourceKey,
-        _req: &DataRequest,
+        key: ResourceKey,
+        base_req: &DataRequest,
     ) -> Result<DataResponse<BufferMarker>, DataError> {
-        todo!()
+        self.run_fallback(key, base_req, |req| self.inner.load_buffer(key, req))
+    }
+}
+
+impl<P, M> DynProvider<M> for LocaleFallbackProvider<P>
+where
+    P: DynProvider<M>,
+    M: ResourceMarker,
+{
+    fn load_payload(
+        &self,
+        key: ResourceKey,
+        base_req: &DataRequest,
+    ) -> Result<DataResponse<M>, DataError> {
+        self.run_fallback(key, base_req, |req| self.inner.load_payload(key, req))
+    }
+}
+
+impl<P, M> ResourceProvider<M> for LocaleFallbackProvider<P>
+where
+    P: ResourceProvider<M>,
+    M: ResourceMarker,
+{
+    fn load_resource(&self, base_req: &DataRequest) -> Result<DataResponse<M>, DataError> {
+        self.run_fallback(M::KEY, base_req, |req| self.inner.load_resource(req))
     }
 }