icu_provider_baked/
zerotrie.rs

Help
1// This file is part of ICU4X. For terms of use, please see the file
2// called LICENSE at the top level of the ICU4X source tree
3// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
4
5//! Data stored as as [`ZeroTrieSimpleAscii`]
6
7use icu_provider::baked::zerotrie::*;
8use icu_provider::prelude::*;
9pub use zerotrie::ZeroTrieSimpleAscii;
10
11use icu_provider::export::ExportMarker;
12
13pub(crate) fn bake(
14    marker_bake: &databake::TokenStream,
15    bakes_to_ids: &[(
16        &DataPayload<ExportMarker>,
17        &std::collections::BTreeSet<DataIdentifierCow>,
18    )],
19    ctx: &databake::CrateEnv,
20) -> (databake::TokenStream, usize) {
21    use databake::*;
22
23    // Safety invariant upheld: the only values being added to the trie are `baked_index`
24    // values, which come from `bakes`
25    let trie = ZeroTrieSimpleAscii::from_iter(bakes_to_ids.iter().enumerate().flat_map(
26        |(bake_index, (_, ids))| {
27            ids.iter().map(move |id| {
28                let mut encoded = id.locale.to_string().into_bytes();
29                if !id.marker_attributes.is_empty() {
30                    encoded.push(ID_SEPARATOR);
31                    encoded.extend_from_slice(id.marker_attributes.as_bytes());
32                }
33                (encoded, bake_index)
34            })
35        },
36    ));
37
38    let baked_trie = trie.as_borrowed_slice().bake(&Default::default());
39    let baked_trie = quote! {
40        const TRIE: icu_provider::baked::zerotrie::ZeroTrieSimpleAscii<&'static [u8]> = icu_provider::baked:: #baked_trie;
41    };
42
43    let payloads = bakes_to_ids
44        .iter()
45        .map(|(payload, _)| *payload)
46        .collect::<Vec<_>>();
47
48    let maybe_vzv_tokens = DataPayload::tokenize_encoded_seq(&payloads, ctx);
49
50    let (baked_values, value_store_ty) = if let Some(vzv_tokens) = maybe_vzv_tokens {
51        (
52            quote! {
53                const VALUES: &'static zerovec::VarZeroSlice<<<#marker_bake as icu_provider::baked::zerotrie::DynamicDataMarker>::DataStruct as icu_provider::ule::MaybeAsVarULE>::EncodedStruct> = #vzv_tokens;
54            },
55            quote! {
56                icu_provider::baked::zerotrie::DataForVarULEs
57            },
58        )
59    } else {
60        let bakes = payloads.iter().map(|payload| payload.tokenize(ctx));
61        (
62            quote! {
63                const VALUES: &'static [<#marker_bake as icu_provider::baked::zerotrie::DynamicDataMarker>::DataStruct] = &[#(#bakes,)*];
64            },
65            quote! {
66                icu_provider::baked::zerotrie::Data
67            },
68        )
69    };
70
71    (
72        quote! {
73            // Safety invariant upheld: see above
74            #value_store_ty<#marker_bake> = {
75                #baked_trie
76                #baked_values
77                unsafe {
78                    #value_store_ty::from_trie_and_values_unchecked(TRIE, VALUES)
79                }
80            }
81
82        },
83        core::mem::size_of::<Data<icu_provider::hello_world::HelloWorldV1>>()
84            + trie.as_borrowed_slice().borrows_size(),
85    )
86}
icu_provider_baked/zerotrie.rs

icu_provider_baked/
zerotrie.rs
