Baked data: use VarULE to store data when specified

components/list/src/provider/mod.rs

@@ -69,6 +69,8 @@ data_marker!(
     ListFormatterPatterns<'static>,
 );
 
+icu_provider::marker::does_not_deref_to_varule!(ListFormatterPatterns<'_>);
+
 /// Symbols and metadata required for [`ListFormatter`](crate::ListFormatter).
 ///
 /// <div class="stab unstable">

components/properties/src/provider.rs

@@ -476,6 +476,8 @@ pub enum PropertyCodePointMap<'data, T: TrieValue> {
     // https://docs.rs/serde/latest/serde/trait.Serializer.html#tymethod.serialize_unit_variant
 }
 
+icu_provider::marker::does_not_deref_to_varule!(<T: TrieValue> PropertyCodePointMap<'_, T>);
+
 macro_rules! data_struct_generic {
     ($(marker($marker:ident, $ty:ident, $path:literal),)+) => {
         $(

provider/baked/Cargo.toml

@@ -20,6 +20,7 @@ include.workspace = true
 icu_provider = { workspace = true }
 writeable = { workspace = true }
 zerotrie = { workspace = true, features = ["alloc"] }
+zerovec = { workspace = true }
 
 crlify = { workspace = true, optional = true }
 databake = { workspace = true, optional = true}

provider/baked/src/export.rs

@@ -105,6 +105,8 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Mutex;
 
+use crate::zerotrie::BakedValue;
+
 // TokenStream isn't Send/Sync
 type SyncTokenStream = String;
 
@@ -565,20 +567,27 @@ impl DataExporter for BakedExporter {
                     .any(|(_, ids)| ids.iter().any(|id| !id.locale.is_default()));
 
             let mut baked_values = deduplicated_values
-                .into_iter()
+                .iter()
                 .map(|(payload, ids)| {
                     stats.structs_count += 1;
                     stats.identifiers_count += ids.len();
                     stats.structs_total_size += payload.baked_size();
 
-                    (payload.tokenize(&self.dependencies), ids)
+                    let baked_value = payload
+                        .maybe_as_varule_bytes()
+                        .map(BakedValue::VarULE)
+                        .unwrap_or_else(|| {
+                            BakedValue::Struct(payload.tokenize(&self.dependencies))
+                        });
+                    (baked_value, ids)
                 })
                 .collect::<Vec<_>>();
 
             // Stability
             baked_values.sort_by(|a, b| a.1.first().cmp(&b.1.first()));
 
-            let (data, lookup_struct_size) = crate::zerotrie::bake(&marker_bake, baked_values);
+            let (data, lookup_struct_size) =
+                crate::zerotrie::bake(&marker_bake, &baked_values, &self.dependencies);
 
             stats.lookup_struct_size = lookup_struct_size;
 

provider/baked/src/zerotrie.rs

@@ -7,22 +7,28 @@
 // This is a valid separator as `DataLocale` will never produce it.
 const ID_SEPARATOR: u8 = 0x1E;
 
-use icu_provider::prelude::*;
 pub use icu_provider::DynamicDataMarker;
+use icu_provider::{marker::MaybeAsVarULE, prelude::*};
 pub use zerotrie::ZeroTrieSimpleAscii;
+use zerovec::VarZeroSlice;
+
+#[cfg(feature = "export")]
+use zerovec::VarZeroVec;
+
+#[cfg(feature = "export")]
+pub(crate) enum BakedValue<'a> {
+    VarULE(&'a [u8]),
+    Struct(databake::TokenStream),
+}
 
 #[cfg(feature = "export")]
 pub(crate) fn bake(
     marker_bake: &databake::TokenStream,
-    bakes_to_ids: Vec<(
-        databake::TokenStream,
-        std::collections::BTreeSet<DataIdentifierCow>,
-    )>,
+    bakes_to_ids: &[(BakedValue, &std::collections::BTreeSet<DataIdentifierCow>)],
+    ctx: &databake::CrateEnv,
 ) -> (databake::TokenStream, usize) {
     use databake::*;
 
-    let bakes = bakes_to_ids.iter().map(|(bake, _)| bake);
-
     // Safety invariant upheld: the only values being added to the trie are `baked_index`
     // values, which come from `bakes`
     let trie = ZeroTrieSimpleAscii::from_iter(bakes_to_ids.iter().enumerate().flat_map(
@@ -39,15 +45,61 @@ pub(crate) fn bake(
     ));
 
     let baked_trie = trie.as_borrowed_slice().bake(&Default::default());
+    let baked_trie = quote! {
+        const TRIE: icu_provider_baked::zerotrie::ZeroTrieSimpleAscii<&'static [u8]> = icu_provider_baked:: #baked_trie;
+    };
+
+    let use_vzv = matches!(bakes_to_ids.first(), Some((BakedValue::VarULE(_), _)));
+
+    let (baked_values, value_store_ty) = if !use_vzv {
+        let bakes = bakes_to_ids.iter().map(|(bake, _)| match bake {
+            BakedValue::Struct(tokens) => tokens,
+            BakedValue::VarULE(_) => {
+                unreachable!(
+                    "All instances should equivalently return Some or None in MaybeAsVarULE"
+                )
+            }
+        });
+        (
+            quote! {
+                const VALUES: &'static [<#marker_bake as icu_provider_baked::zerotrie::DynamicDataMarker>::DataStruct] = &[#(#bakes,)*];
+            },
+            quote! {
+                icu_provider_baked::zerotrie::Data
+            },
+        )
+    } else {
+        let byteses = bakes_to_ids
+            .iter()
+            .map(|(bake, _)| match bake {
+                BakedValue::VarULE(bytes) => *bytes,
+                BakedValue::Struct(_) => {
+                    unreachable!(
+                        "All instances should equivalently return Some or None in MaybeAsVarULE"
+                    )
+                }
+            })
+            .collect::<Vec<&[u8]>>();
+        let vzv = VarZeroVec::<[u8]>::from(&byteses);
+        let vzv_bytes = vzv.as_bytes().bake(ctx);
+        (
+            quote! {
+                const VALUES: &'static zerovec::VarZeroSlice<<<#marker_bake as icu_provider_baked::zerotrie::DynamicDataMarker>::DataStruct as icu_provider::marker::MaybeAsVarULE>::VarULE> = unsafe { zerovec::VarZeroSlice::from_bytes_unchecked(#vzv_bytes) };
+            },
+            quote! {
+                icu_provider_baked::zerotrie::DataForVarULEs
+            },
+        )
+    };
 
     (
         quote! {
             // Safety invariant upheld: see above
-            icu_provider_baked::zerotrie::Data<#marker_bake> = {
-                const TRIE: icu_provider_baked::zerotrie::ZeroTrieSimpleAscii<&'static [u8]> = icu_provider_baked:: #baked_trie;
-                const VALUES: &'static [<#marker_bake as icu_provider_baked::zerotrie::DynamicDataMarker>::DataStruct] = &[#(#bakes,)*];
+            #value_store_ty<#marker_bake> = {
+                #baked_trie
+                #baked_values
                 unsafe {
-                    icu_provider_baked::zerotrie::Data::from_trie_and_values_unchecked(TRIE, VALUES)
+                    #value_store_ty::from_trie_and_values_unchecked(TRIE, VALUES)
                 }
             }
 
@@ -57,6 +109,51 @@ pub(crate) fn bake(
     )
 }
 
+fn get_index(
+    trie: ZeroTrieSimpleAscii<&'static [u8]>,
+    id: DataIdentifierBorrowed,
+    attributes_prefix_match: bool,
+) -> Option<usize> {
+    use writeable::Writeable;
+    let mut cursor = trie.cursor();
+    let _is_ascii = id.locale.write_to(&mut cursor);
+    if !id.marker_attributes.is_empty() {
+        cursor.step(ID_SEPARATOR);
+        id.marker_attributes.write_to(&mut cursor).ok()?;
+        loop {
+            if let Some(v) = cursor.take_value() {
+                break Some(v);
+            }
+            if !attributes_prefix_match || cursor.probe(0).is_none() {
+                break None;
+            }
+        }
+    } else {
+        cursor.take_value()
+    }
+}
+
+#[cfg(feature = "alloc")]
+#[allow(clippy::type_complexity)]
+fn iter(
+    trie: &'static ZeroTrieSimpleAscii<&'static [u8]>,
+) -> core::iter::FilterMap<
+    zerotrie::ZeroTrieStringIterator<'static>,
+    fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
+> {
+    use alloc::borrow::ToOwned;
+    trie.iter().filter_map(move |(s, _)| {
+        if let Some((locale, attrs)) = s.split_once(ID_SEPARATOR as char) {
+            Some(DataIdentifierCow::from_owned(
+                DataMarkerAttributes::try_from_str(attrs).ok()?.to_owned(),
+                locale.parse().ok()?,
+            ))
+        } else {
+            s.parse().ok().map(DataIdentifierCow::from_locale)
+        }
+    })
+}
+
 pub struct Data<M: DataMarker> {
     // Unsafe invariant: actual values contained MUST be valid indices into `values`
     trie: ZeroTrieSimpleAscii<&'static [u8]>,
@@ -82,26 +179,10 @@ impl<M: DataMarker> super::DataStore<M> for Data<M> {
         id: DataIdentifierBorrowed,
         attributes_prefix_match: bool,
     ) -> Option<DataPayload<M>> {
-        use writeable::Writeable;
-        let mut cursor = self.trie.cursor();
-        let _is_ascii = id.locale.write_to(&mut cursor);
-        if !id.marker_attributes.is_empty() {
-            cursor.step(ID_SEPARATOR);
-            id.marker_attributes.write_to(&mut cursor).ok()?;
-            loop {
-                if let Some(v) = cursor.take_value() {
-                    break Some(v);
-                }
-                if !attributes_prefix_match || cursor.probe(0).is_none() {
-                    break None;
-                }
-            }
-        } else {
-            cursor.take_value()
-        }
-        // Safety: Allowed since `i` came from the trie and the field safety invariant
-        .map(|i| unsafe { self.values.get_unchecked(i) })
-        .map(DataPayload::from_static_ref)
+        get_index(self.trie, id, attributes_prefix_match)
+            // Safety: Allowed since `i` came from the trie and the field safety invariant
+            .map(|i| unsafe { self.values.get_unchecked(i) })
+            .map(DataPayload::from_static_ref)
     }
 
     #[cfg(feature = "alloc")]
@@ -111,17 +192,58 @@ impl<M: DataMarker> super::DataStore<M> for Data<M> {
     >;
     #[cfg(feature = "alloc")]
     fn iter(&'static self) -> Self::IterReturn {
-        #![allow(unused_imports)]
-        use alloc::borrow::ToOwned;
-        self.trie.iter().filter_map(move |(s, _)| {
-            if let Some((locale, attrs)) = s.split_once(ID_SEPARATOR as char) {
-                Some(DataIdentifierCow::from_owned(
-                    DataMarkerAttributes::try_from_str(attrs).ok()?.to_owned(),
-                    locale.parse().ok()?,
-                ))
-            } else {
-                s.parse().ok().map(DataIdentifierCow::from_locale)
-            }
-        })
+        iter(&self.trie)
+    }
+}
+
+pub struct DataForVarULEs<M: DataMarker>
+where
+    M::DataStruct: MaybeAsVarULE,
+{
+    // Unsafe invariant: actual values contained MUST be valid indices into `values`
+    trie: ZeroTrieSimpleAscii<&'static [u8]>,
+    values: &'static VarZeroSlice<<M::DataStruct as MaybeAsVarULE>::VarULE>,
+}
+
+impl<M: DataMarker> DataForVarULEs<M>
+where
+    M::DataStruct: MaybeAsVarULE,
+{
+    /// Construct from a trie and values
+    ///
+    /// # Safety
+    /// The actual values contained in the trie must be valid indices into `values`
+    pub const unsafe fn from_trie_and_values_unchecked(
+        trie: ZeroTrieSimpleAscii<&'static [u8]>,
+        values: &'static VarZeroSlice<<M::DataStruct as MaybeAsVarULE>::VarULE>,
+    ) -> Self {
+        Self { trie, values }
+    }
+}
+
+impl<M: DataMarker> super::DataStore<M> for DataForVarULEs<M>
+where
+    M::DataStruct: MaybeAsVarULE + From<&'static <M::DataStruct as MaybeAsVarULE>::VarULE>,
+{
+    fn get(
+        &self,
+        id: DataIdentifierBorrowed,
+        attributes_prefix_match: bool,
+    ) -> Option<DataPayload<M>> {
+        get_index(self.trie, id, attributes_prefix_match)
+            // Safety: Allowed since `i` came from the trie and the field safety invariant
+            .map(|i| unsafe { self.values.get_unchecked(i) })
+            .map(From::from)
+            .map(DataPayload::from_owned)
+    }
+
+    #[cfg(feature = "alloc")]
+    type IterReturn = core::iter::FilterMap<
+        zerotrie::ZeroTrieStringIterator<'static>,
+        fn((alloc::string::String, usize)) -> Option<DataIdentifierCow<'static>>,
+    >;
+    #[cfg(feature = "alloc")]
+    fn iter(&'static self) -> Self::IterReturn {
+        iter(&self.trie)
     }
 }

provider/baked/tests/data/hello_world_v1.rs.data

@@ -16,10 +16,10 @@ macro_rules! __impl_hello_world_v1 {
         const _: () = <$provider>::MUST_USE_MAKE_PROVIDER_MACRO;
         #[clippy::msrv = "1.81"]
         impl $provider {
-            const DATA_HELLO_WORLD_V1: icu_provider_baked::zerotrie::Data<icu_provider::hello_world::HelloWorldV1> = {
+            const DATA_HELLO_WORLD_V1: icu_provider_baked::zerotrie::DataForVarULEs<icu_provider::hello_world::HelloWorldV1> = {
                 const TRIE: icu_provider_baked::zerotrie::ZeroTrieSimpleAscii<&'static [u8]> = icu_provider_baked::zerotrie::ZeroTrieSimpleAscii { store: b"\xCDbcdefijlprsvz\x02\x04\nCIKX[^fpsn\x80s\x81e\x82-AT\x83\xC3lno\x012\x84\x85\xC2\x1E-\treverse\x90\t\xC301G\x0C\x0F\xC201\x06\xC212\x01\x86\x879\x8842\x89B\x8A-u-sd-gbeng\x8B\x8C\xC2ai\x01\x8D\x8Es\x8Fa\x90\0\x1Ereverse\x90\na\x90\x01t\x90\x02\xC2ou\x02\x90\x03\x90\x04r\x90\x05-Latn\x90\x06i\x90\x07h\x90\x08" };
-                const VALUES: &'static [<icu_provider::hello_world::HelloWorldV1 as icu_provider_baked::zerotrie::DynamicDataMarker>::DataStruct] = &[icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("ওহে বিশ\u{9cd}ব") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Ahoj světe") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hallo Welt") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Servus Welt") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Καλημέρα κόσμε") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello World") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🗺\u{fe0f}") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🌍") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🌎") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🌏") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🇬🇧") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Hello from 🏴\u{e0067}\u{e0062}\u{e0065}\u{e006e}\u{e0067}\u{e007f}") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Saluton, Mondo") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("سلام دنیا\u{200e}") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("hei maailma") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Halló, heimur") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("こんにちは世界") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Ave, munde") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Olá, mundo") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Salut, lume") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Привет, мир") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Поздрав свете") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Pozdrav svete") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Xin chào thế giới") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("你好世界") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("Olleh Dlrow") }, icu_provider::hello_world::HelloWorld { message: alloc::borrow::Cow::Borrowed("界世はちにんこ") }];
-                unsafe { icu_provider_baked::zerotrie::Data::from_trie_and_values_unchecked(TRIE, VALUES) }
+                const VALUES: &'static zerovec::VarZeroSlice<<<icu_provider::hello_world::HelloWorldV1 as icu_provider_baked::zerotrie::DynamicDataMarker>::DataStruct as icu_provider::marker::MaybeAsVarULE>::VarULE> = unsafe { zerovec::VarZeroSlice::from_bytes_unchecked(b"\x1B\0\x19\0$\0.\09\0T\0_\0q\0\x80\0\x8F\0\x9E\0\xB1\0\xD8\0\xE6\0\xFA\0\x05\x01\x13\x01(\x012\x01=\x01H\x01\\\x01u\x01\x82\x01\x98\x01\xA4\x01\xAF\x01\xE0\xA6\x93\xE0\xA6\xB9\xE0\xA7\x87 \xE0\xA6\xAC\xE0\xA6\xBF\xE0\xA6\xB6\xE0\xA7\x8D\xE0\xA6\xACAhoj sv\xC4\x9BteHallo WeltServus Welt\xCE\x9A\xCE\xB1\xCE\xBB\xCE\xB7\xCE\xBC\xCE\xAD\xCF\x81\xCE\xB1 \xCE\xBA\xCF\x8C\xCF\x83\xCE\xBC\xCE\xB5Hello WorldHello from \xF0\x9F\x97\xBA\xEF\xB8\x8FHello from \xF0\x9F\x8C\x8DHello from \xF0\x9F\x8C\x8EHello from \xF0\x9F\x8C\x8FHello from \xF0\x9F\x87\xAC\xF0\x9F\x87\xA7Hello from \xF0\x9F\x8F\xB4\xF3\xA0\x81\xA7\xF3\xA0\x81\xA2\xF3\xA0\x81\xA5\xF3\xA0\x81\xAE\xF3\xA0\x81\xA7\xF3\xA0\x81\xBFSaluton, Mondo\xD8\xB3\xD9\x84\xD8\xA7\xD9\x85 \xD8\xAF\xD9\x86\xDB\x8C\xD8\xA7\xE2\x80\x8Ehei maailmaHall\xC3\xB3, heimur\xE3\x81\x93\xE3\x82\x93\xE3\x81\xAB\xE3\x81\xA1\xE3\x81\xAF\xE4\xB8\x96\xE7\x95\x8CAve, mundeOl\xC3\xA1, mundoSalut, lume\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82, \xD0\xBC\xD0\xB8\xD1\x80\xD0\x9F\xD0\xBE\xD0\xB7\xD0\xB4\xD1\x80\xD0\xB0\xD0\xB2 \xD1\x81\xD0\xB2\xD0\xB5\xD1\x82\xD0\xB5Pozdrav sveteXin ch\xC3\xA0o th\xE1\xBA\xBF gi\xE1\xBB\x9Bi\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8COlleh Dlrow\xE7\x95\x8C\xE4\xB8\x96\xE3\x81\xAF\xE3\x81\xA1\xE3\x81\xAB\xE3\x82\x93\xE3\x81\x93") };
+                unsafe { icu_provider_baked::zerotrie::DataForVarULEs::from_trie_and_values_unchecked(TRIE, VALUES) }
             };
         }
         #[clippy::msrv = "1.81"]

provider/core/macros/src/lib.rs

@@ -274,12 +274,18 @@ fn data_struct_impl(attr: DataStructArgs, input: DeriveInput) -> TokenStream2 {
 
     let name = &input.ident;
 
-    let name_with_lt = if !lifetimes.is_empty() {
+    let name_with_static_lt = if !lifetimes.is_empty() {
         quote!(#name<'static>)
     } else {
         quote!(#name)
     };
 
+    let name_with_implied_lt = if !lifetimes.is_empty() {
+        quote!(#name<'_>)
+    } else {
+        quote!(#name)
+    };
+
     if lifetimes.len() > 1 {
         return syn::Error::new(
             input.generics.span(),
@@ -317,7 +323,7 @@ fn data_struct_impl(attr: DataStructArgs, input: DeriveInput) -> TokenStream2 {
             #[doc = #docs]
             pub struct #marker_name;
             impl icu_provider::DynamicDataMarker for #marker_name {
-                type DataStruct = #name_with_lt;
+                type DataStruct = #name_with_static_lt;
             }
         ));
 
@@ -359,6 +365,11 @@ fn data_struct_impl(attr: DataStructArgs, input: DeriveInput) -> TokenStream2 {
         }
     }
 
+    // TODO: Allow this to be configured
+    result.extend(quote!(
+        icu_provider::marker::does_not_deref_to_varule!(#name_with_implied_lt);
+    ));
+
     result.extend(quote!(
         #[derive(icu_provider::prelude::yoke::Yokeable, icu_provider::prelude::zerofrom::ZeroFrom)]
         #input