icu_provider_source/properties/
script.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).

use crate::SourceDataProvider;
use icu::collections::codepointtrie::CodePointTrie;
use icu::properties::props::Script;
use icu::properties::provider::{
    ScriptWithExtensionsPropertyV1, ScriptWithExtensionsPropertyV1Marker,
};
use icu::properties::script::ScriptWithExt;
use icu_provider::prelude::*;
use std::collections::HashSet;
use std::convert::TryFrom;
use zerovec::{VarZeroVec, ZeroSlice, ZeroVec};

// implement data provider
impl DataProvider<ScriptWithExtensionsPropertyV1Marker> for SourceDataProvider {
    fn load(
        &self,
        req: DataRequest,
    ) -> Result<DataResponse<ScriptWithExtensionsPropertyV1Marker>, DataError> {
        self.check_req::<ScriptWithExtensionsPropertyV1Marker>(req)?;
        let scx_data = self
            .icuexport()?
            .read_and_parse_toml::<super::uprops_serde::script_extensions::Main>(&format!(
                "uprops/{}/scx.toml",
                self.trie_type(),
            ))?
            .script_extensions
            .first()
            .ok_or_else(|| DataError::custom("Could not parse Script_Extensions data from TOML"))?;

        let cpt_data = &scx_data.code_point_trie;
        let scx_array_data = &scx_data.script_code_array;

        let trie = CodePointTrie::<ScriptWithExt>::try_from(cpt_data).map_err(|e| {
            DataError::custom("Could not parse CodePointTrie TOML").with_display_context(&e)
        })?;

        // Convert the input from Vec<Vec<u16>> to Vec<ZeroVec<Script>> so that
        // we can go through the VarZeroVec construction process for a desired result
        // type of VZV<ZeroSlice<Script>>
        let ule_scx_array_data: Vec<ZeroVec<Script>> = scx_array_data
            .iter()
            .map(|v| v.iter().map(|i| Script(*i)).collect::<ZeroVec<Script>>())
            .collect::<Vec<ZeroVec<Script>>>();
        let scx_vzv: VarZeroVec<ZeroSlice<Script>> =
            VarZeroVec::from(ule_scx_array_data.as_slice());

        let data_struct = ScriptWithExtensionsPropertyV1 {
            trie,
            extensions: scx_vzv,
        };

        Ok(DataResponse {
            metadata: Default::default(),
            payload: DataPayload::from_owned(data_struct),
        })
    }
}

impl crate::IterableDataProviderCached<ScriptWithExtensionsPropertyV1Marker>
    for SourceDataProvider
{
    fn iter_ids_cached(&self) -> Result<HashSet<DataIdentifierCow<'static>>, DataError> {
        Ok(HashSet::from_iter([Default::default()]))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_script_val_from_script_extensions() {
        let provider = SourceDataProvider::new_testing();

        let swe =
            icu::properties::script::ScriptWithExtensions::try_new_unstable(&provider).unwrap();
        let swe = swe.as_borrowed();

        assert_eq!(swe.get_script_val('𐓐'), Script::Osage); // U+104D0 OSAGE CAPITAL LETTER KHA
        assert_eq!(swe.get_script_val('🥳'), Script::Common); // U+1F973 FACE WITH PARTY HORN AND PARTY HAT
        assert_eq!(swe.get_script_val32(0x200D), Script::Inherited); // ZERO WIDTH JOINER
        assert_eq!(swe.get_script_val('௫'), Script::Tamil); // U+0BEB TAMIL DIGIT FIVE
        assert_eq!(swe.get_script_val32(0x11303), Script::Grantha); // GRANTHA SIGN VISARGA
        assert_eq!(swe.get_script_val32(0x30A0), Script::Common); // U+30A0 KATAKANA-HIRAGANA DOUBLE HYPHEN
    }

    #[test]
    fn test_scx_array_from_script_extensions() {
        let provider = SourceDataProvider::new_testing();

        let swe =
            icu::properties::script::ScriptWithExtensions::try_new_unstable(&provider).unwrap();
        let swe = swe.as_borrowed();

        assert_eq!(
            swe.get_script_extensions_val('𐓐') /* U+104D0 OSAGE CAPITAL LETTER KHA */
                .iter()
                .collect::<Vec<_>>(),
            [Script::Osage]
        );
        assert_eq!(
            swe.get_script_extensions_val('🥳') /* U+1F973 FACE WITH PARTY HORN AND PARTY HAT */
                .iter()
                .collect::<Vec<_>>(),
            [Script::Common]
        );
        assert_eq!(
            swe.get_script_extensions_val32(0x200D) // ZERO WIDTH JOINER
                .iter()
                .collect::<Vec<_>>(),
            [Script::Inherited]
        );
        assert_eq!(
            swe.get_script_extensions_val('௫') // U+0BEB TAMIL DIGIT FIVE
                .iter()
                .collect::<Vec<_>>(),
            [Script::Tamil, Script::Grantha]
        );
        assert_eq!(
            swe.get_script_extensions_val32(0x11303) // GRANTHA SIGN VISARGA
                .iter()
                .collect::<Vec<_>>(),
            [Script::Tamil, Script::Grantha]
        );
        assert_eq!(
            swe.get_script_extensions_val32(0x30A0) // KATAKANA-HIRAGANA DOUBLE HYPHEN
                .iter()
                .collect::<Vec<_>>(),
            [Script::Hiragana, Script::Katakana]
        );

        assert_eq!(
            swe.get_script_extensions_val32(0x200D) // ZERO WIDTH JOINER
                .iter()
                .next(),
            Some(Script::Inherited)
        );

        assert!(swe
            .get_script_extensions_val32(0x11303) // GRANTHA SIGN VISARGA
            .contains(&Script::Grantha));

        assert!(!swe
            .get_script_extensions_val32(0x11303) // GRANTHA SIGN VISARGA
            .contains(&Script::Common));

        // // Invalid code point
        assert_eq!(
            swe.get_script_extensions_val32(0x11_0000) // CODE_POINT_MAX + 1 is invalid
                .iter()
                .collect::<Vec<_>>(),
            [Script::Unknown]
        );
    }

    #[test]
    fn test_has_script() {
        let provider = SourceDataProvider::new_testing();

        let swe =
            icu::properties::script::ScriptWithExtensions::try_new_unstable(&provider).unwrap();
        let swe = swe.as_borrowed();

        assert!(swe.has_script('𐓐', Script::Osage));
        assert!(!swe.has_script('𐓐', Script::Common));
        assert!(!swe.has_script('𐓐', Script::Inherited));

        assert!(swe.has_script('🥳', Script::Common));
        assert!(!swe.has_script('🥳', Script::Inherited));

        assert!(!swe.has_script32(0x200D, Script::Common));
        assert!(swe.has_script32(0x200D, Script::Inherited));

        assert!(swe.has_script('௫', Script::Tamil));
        assert!(swe.has_script('௫', Script::Grantha));
        assert!(!swe.has_script('௫', Script::Common));
        assert!(!swe.has_script('௫', Script::Inherited));

        assert!(swe.has_script32(0x11303, Script::Tamil));
        assert!(swe.has_script32(0x11303, Script::Grantha));
        assert!(!swe.has_script32(0x11303, Script::Common));
        assert!(!swe.has_script32(0x11303, Script::Inherited));

        assert!(swe.has_script32(0x30A0, Script::Hiragana));
        assert!(swe.has_script32(0x30A0, Script::Katakana));
        assert!(!swe.has_script32(0x30A0, Script::Common));
        assert!(!swe.has_script32(0x30A0, Script::Inherited));

        // U+0964 DEVANAGARI DANDA
        assert!(!swe.has_script32(0x0964, Script::Common));
        assert!(swe.has_script32(0x0964, Script::Devanagari));
        assert!(swe.has_script32(0x0964, Script::Bengali));

        // TestHasScript() test cases from ICU4J

        // U+063F ARABIC LETTER FARSI YEH WITH THREE DOTS ABOVE
        assert!(!swe.has_script32(0x063F, Script::Common));
        assert!(swe.has_script32(0x063F, Script::Arabic)); // main Script value
        assert!(!swe.has_script32(0x063F, Script::Syriac));
        assert!(!swe.has_script32(0x063F, Script::Thaana));

        // U+0640 ARABIC TATWEEL
        assert!(!swe.has_script32(0x0640, Script::Common)); // main Script value
        assert!(swe.has_script32(0x0640, Script::Arabic));
        assert!(swe.has_script32(0x0640, Script::Syriac));
        assert!(!swe.has_script32(0x0640, Script::Thaana));

        // U+0650 ARABIC KASRA
        assert!(!swe.has_script32(0x0650, Script::Inherited)); // main Script value
        assert!(swe.has_script32(0x0650, Script::Arabic));
        assert!(swe.has_script32(0x0650, Script::Syriac));
        assert!(!swe.has_script32(0x0650, Script::Thaana));

        // U+0660 ARABIC-INDIC DIGIT ZERO
        assert!(!swe.has_script32(0x0660, Script::Common));
        assert!(swe.has_script32(0x0660, Script::Arabic)); // main Script value
        assert!(!swe.has_script32(0x0660, Script::Syriac));
        assert!(swe.has_script32(0x0660, Script::Thaana));

        // U+FDF2 ARABIC LIGATURE ALLAH ISOLATED FORM
        assert!(!swe.has_script32(0xFDF2, Script::Common));
        assert!(swe.has_script32(0xFDF2, Script::Arabic)); // main Script value
        assert!(!swe.has_script32(0xFDF2, Script::Syriac));
        assert!(swe.has_script32(0xFDF2, Script::Thaana));

        // The ICU4J comment for this test says:
        // An unguarded implementation might go into an infinite loop.
        assert!(!swe.has_script32(0x0640, Script(0xAFFE)));
    }

    #[test]
    fn test_get_script_extensions_set() {
        let provider = SourceDataProvider::new_testing();

        let swe =
            icu::properties::script::ScriptWithExtensions::try_new_unstable(&provider).unwrap();
        let swe = swe.as_borrowed();

        let grantha = swe.get_script_extensions_set(Script::Grantha);
        assert!(!grantha.contains32(0x0BE5)); // unknown with unknown script in Tamil block
        assert!(grantha.contains32(0x0BE6)); // TAMIL DIGIT ZERO
        assert!(grantha.contains32(0x0BEB)); // TAMIL DIGIT FIVE
        assert!(grantha.contains32(0x0BEF)); // TAMIL DIGIT NINE
        assert!(grantha.contains32(0x0BF2)); // TAMIL NUMBER ONE THOUSAND
        assert!(grantha.contains32(0x0BF3)); // TAMIL DAY SIGN
        assert!(!grantha.contains32(0x0BF4)); // TAMIL MONTH SIGN
        assert!(grantha.contains32(0x11300)); // GRANTHA SIGN COMBINING ANUSVARA ABOVE
        assert!(grantha.contains32(0x11301)); // GRANTHA SIGN CANDRABINDU
        assert!(grantha.contains32(0x11302)); // GRANTHA SIGN ANUSVARA
        assert!(grantha.contains32(0x11303)); // GRANTHA SIGN VISARGA
        assert!(!grantha.contains32(0x11304)); // unknown with unknown script in Grantha block
        assert!(grantha.contains32(0x11305)); // GRANTHA LETTER A

        let tamil = swe.get_script_extensions_set(Script::Tamil);
        assert!(!tamil.contains32(0x0BE5)); // unknown with unknown script in Tamil block
        assert!(tamil.contains32(0x0BE6)); // TAMIL DIGIT ZERO
        assert!(tamil.contains32(0x0BEB)); // TAMIL DIGIT FIVE
        assert!(tamil.contains32(0x0BEF)); // TAMIL DIGIT NINE
        assert!(tamil.contains32(0x0BF2)); // TAMIL NUMBER ONE THOUSAND
        assert!(tamil.contains32(0x0BF3)); // TAMIL DAY SIGN
        assert!(tamil.contains32(0x0BF4)); // TAMIL MONTH SIGN
        assert!(!tamil.contains32(0x11300)); // GRANTHA SIGN COMBINING ANUSVARA ABOVE
        assert!(tamil.contains32(0x11301)); // GRANTHA SIGN CANDRABINDU
        assert!(!tamil.contains32(0x11302)); // GRANTHA SIGN ANUSVARA
        assert!(tamil.contains32(0x11303)); // GRANTHA SIGN VISARGA
        assert!(!tamil.contains32(0x11304)); // unknown with unknown script in Grantha block
        assert!(!tamil.contains32(0x11305)); // GRANTHA LETTER A

        let hiragana = swe.get_script_extensions_set(Script::Hiragana);
        assert!(hiragana.contains32(0x3046)); // HIRAGANA LETTER U
        assert!(hiragana.contains32(0x309F)); // HIRAGANA DIGRAPH YORI
        assert!(hiragana.contains32(0x30A0)); // KATAKANA-HIRAGANA DOUBLE HYPHEN
        assert!(!hiragana.contains32(0x30A1)); // KATAKANA LETTER SMALL A
        assert!(hiragana.contains32(0x30FB)); // KATAKANA MIDDLE DOT
        assert!(hiragana.contains32(0x30FC)); // KATAKANA-HIRAGANA PROLONGED SOUND MARK
        assert!(!hiragana.contains32(0x30FD)); // KATAKANA ITERATION MARK

        let katakana = swe.get_script_extensions_set(Script::Katakana);
        assert!(!katakana.contains32(0x3046)); // HIRAGANA LETTER U
        assert!(!katakana.contains32(0x309F)); // HIRAGANA DIGRAPH YORI
        assert!(katakana.contains32(0x30A0)); // KATAKANA-HIRAGANA DOUBLE HYPHEN
        assert!(katakana.contains32(0x30A1)); // KATAKANA LETTER SMALL A
        assert!(katakana.contains32(0x30FB)); // KATAKANA MIDDLE DOT
        assert!(katakana.contains32(0x30FC)); // KATAKANA-HIRAGANA PROLONGED SOUND MARK
        assert!(katakana.contains32(0x30FD)); // KATAKANA ITERATION MARK

        let common = swe.get_script_extensions_set(Script::Common);
        assert!(common.contains('🥳'));
        assert!(!common.contains32(0x200D));
        assert!(!common.contains32(0x30A0));

        let inherited = swe.get_script_extensions_set(Script::Inherited);
        assert!(!inherited.contains('🥳'));
        assert!(inherited.contains32(0x200D));
        assert!(!inherited.contains32(0x30A0));

        // inspired by https://github.com/unicode-org/unicodetools/issues/192

        let bangla = swe.get_script_extensions_set(Script::Bengali);
        assert!(bangla.contains32(0x09E7)); // BENGALI DIGIT ONE
        assert!(!bangla.contains32(0x0963)); // DEVANAGARI VOWEL SIGN VOCALIC LL
        assert!(bangla.contains32(0x0964)); // DEVANAGARI DANDA
        assert!(bangla.contains32(0x0965)); // DEVANAGARI DOUBLE DANDA
        assert!(!bangla.contains32(0x0966)); // DEVANAGARI DIGIT ZERO

        let devanagari = swe.get_script_extensions_set(Script::Devanagari);
        assert!(!devanagari.contains32(0x09E7)); // BENGALI DIGIT ONE
        assert!(devanagari.contains32(0x0963)); // DEVANAGARI VOWEL SIGN VOCALIC LL
        assert!(devanagari.contains32(0x0964)); // DEVANAGARI DANDA
        assert!(devanagari.contains32(0x0965)); // DEVANAGARI DOUBLE DANDA
        assert!(devanagari.contains32(0x0966)); // DEVANAGARI DIGIT ZERO

        assert!(!common.contains32(0x0964)); // DEVANAGARI DANDA
        assert!(!common.contains32(0x0965)); // DEVANAGARI DOUBLE DANDA
    }
}