|
#define | U_HIDE_OBSOLETE_UTF_OLD_H 0 |
| Hides the obsolete definitions in unicode/utf_old.h.
|
|
#define | UTF_SIZE 16 |
| Number of bits in a Unicode string code unit - ICU uses 16-bit Unicode.
|
|
#define | UTF_SAFE |
| The default choice for general Unicode string macros is to use the ..._SAFE macro implementations with strict=false.
|
|
#define | UTF8_ERROR_VALUE_1 0x15 |
| UTF8_ERROR_VALUE_1 and UTF8_ERROR_VALUE_2 are special error values for UTF-8, which need 1 or 2 bytes in UTF-8:
|
|
#define | UTF8_ERROR_VALUE_2 0x9f |
| See documentation on UTF8_ERROR_VALUE_1 for details.
|
|
#define | UTF_ERROR_VALUE 0xffff |
| Error value for all UTFs.
|
|
#define | UTF_IS_ERROR(c) (((c)&0xfffe)==0xfffe || (c)==UTF8_ERROR_VALUE_1 || (c)==UTF8_ERROR_VALUE_2) |
| Is a given 32-bit code an error value as returned by one of the macros for any UTF?
|
|
#define | UTF_IS_VALID(c) |
| This is a combined macro: Is c a valid Unicode value and not an error code?
|
|
#define | UTF_IS_SURROGATE(uchar) (((uchar)&0xfffff800)==0xd800) |
| Is this code unit or code point a surrogate (U+d800..U+dfff)?
|
|
#define | UTF_IS_UNICODE_NONCHAR(c) |
| Is a given 32-bit code point a Unicode noncharacter?
|
|
#define | UTF_IS_UNICODE_CHAR(c) |
| Is a given 32-bit value a Unicode code point value (0..U+10ffff) that can be assigned a character?
|
|
#define | UTF8_COUNT_TRAIL_BYTES(leadByte) (utf8_countTrailBytes[(uint8_t)leadByte]) |
| Count the trail bytes for a UTF-8 lead byte.
|
|
#define | UTF8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1) |
| Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
|
|
#define | UTF8_IS_SINGLE(uchar) (((uchar)&0x80)==0) |
| Is this this code point a single code unit (byte)?
|
|
#define | UTF8_IS_LEAD(uchar) ((uint8_t)((uchar)-0xc0)<0x3e) |
| Is this this code unit the lead code unit (byte) of a code point?
|
|
#define | UTF8_IS_TRAIL(uchar) (((uchar)&0xc0)==0x80) |
| Is this this code unit a trailing code unit (byte) of a code point?
|
|
#define | UTF8_NEED_MULTIPLE_UCHAR(c) ((uint32_t)(c)>0x7f) |
| Does this scalar Unicode value need multiple code units for storage?
|
|
#define | UTF8_CHAR_LENGTH(c) |
| Given the lead character, how many bytes are taken by this code point.
|
|
#define | UTF8_MAX_CHAR_LENGTH 4 |
| The maximum number of bytes per code point.
|
|
#define | UTF8_ARRAY_SIZE(size) ((5*(size))/2) |
| Average number of code units compared to UTF-16.
|
|
#define | UTF8_GET_CHAR_UNSAFE(s, i, c) |
|
#define | UTF8_GET_CHAR_SAFE(s, start, i, length, c, strict) |
|
#define | UTF8_NEXT_CHAR_UNSAFE(s, i, c) |
|
#define | UTF8_APPEND_CHAR_UNSAFE(s, i, c) |
|
#define | UTF8_FWD_1_UNSAFE(s, i) |
|
#define | UTF8_FWD_N_UNSAFE(s, i, n) |
|
#define | UTF8_SET_CHAR_START_UNSAFE(s, i) |
|
#define | UTF8_NEXT_CHAR_SAFE(s, i, length, c, strict) |
|
#define | UTF8_APPEND_CHAR_SAFE(s, i, length, c) |
|
#define | UTF8_FWD_1_SAFE(s, i, length) U8_FWD_1(s, i, length) |
|
#define | UTF8_FWD_N_SAFE(s, i, length, n) U8_FWD_N(s, i, length, n) |
|
#define | UTF8_SET_CHAR_START_SAFE(s, start, i) U8_SET_CP_START(s, start, i) |
|
#define | UTF8_PREV_CHAR_UNSAFE(s, i, c) |
|
#define | UTF8_BACK_1_UNSAFE(s, i) |
|
#define | UTF8_BACK_N_UNSAFE(s, i, n) |
|
#define | UTF8_SET_CHAR_LIMIT_UNSAFE(s, i) |
|
#define | UTF8_PREV_CHAR_SAFE(s, start, i, c, strict) |
|
#define | UTF8_BACK_1_SAFE(s, start, i) U8_BACK_1(s, start, i) |
|
#define | UTF8_BACK_N_SAFE(s, start, i, n) U8_BACK_N(s, start, i, n) |
|
#define | UTF8_SET_CHAR_LIMIT_SAFE(s, start, i, length) U8_SET_CP_LIMIT(s, start, i, length) |
|
#define | UTF_IS_FIRST_SURROGATE(uchar) (((uchar)&0xfffffc00)==0xd800) |
| Is uchar a first/lead surrogate?
|
|
#define | UTF_IS_SECOND_SURROGATE(uchar) (((uchar)&0xfffffc00)==0xdc00) |
| Is uchar a second/trail surrogate?
|
|
#define | UTF_IS_SURROGATE_FIRST(c) (((c)&0x400)==0) |
| Assuming c is a surrogate, is it a first/lead surrogate?
|
|
#define | UTF_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000) |
| Helper constant for UTF16_GET_PAIR_VALUE.
|
|
#define | UTF16_GET_PAIR_VALUE(first, second) (((first)<<10UL)+(second)-UTF_SURROGATE_OFFSET) |
| Get the UTF-32 value from the surrogate code units.
|
|
#define | UTF_FIRST_SURROGATE(supplementary) (UChar)(((supplementary)>>10)+0xd7c0) |
|
#define | UTF_SECOND_SURROGATE(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00) |
|
#define | UTF16_LEAD(supplementary) UTF_FIRST_SURROGATE(supplementary) |
|
#define | UTF16_TRAIL(supplementary) UTF_SECOND_SURROGATE(supplementary) |
|
#define | UTF16_IS_SINGLE(uchar) !UTF_IS_SURROGATE(uchar) |
|
#define | UTF16_IS_LEAD(uchar) UTF_IS_FIRST_SURROGATE(uchar) |
|
#define | UTF16_IS_TRAIL(uchar) UTF_IS_SECOND_SURROGATE(uchar) |
|
#define | UTF16_NEED_MULTIPLE_UCHAR(c) ((uint32_t)(c)>0xffff) |
| Does this scalar Unicode value need multiple code units for storage?
|
|
#define | UTF16_CHAR_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2) |
|
#define | UTF16_MAX_CHAR_LENGTH 2 |
|
#define | UTF16_ARRAY_SIZE(size) (size) |
| Average number of code units compared to UTF-16.
|
|
#define | UTF16_GET_CHAR_UNSAFE(s, i, c) |
| Get a single code point from an offset that points to any of the code units that belong to that code point.
|
|
#define | UTF16_GET_CHAR_SAFE(s, start, i, length, c, strict) |
|
#define | UTF16_NEXT_CHAR_UNSAFE(s, i, c) |
|
#define | UTF16_APPEND_CHAR_UNSAFE(s, i, c) |
|
#define | UTF16_FWD_1_UNSAFE(s, i) |
|
#define | UTF16_FWD_N_UNSAFE(s, i, n) |
|
#define | UTF16_SET_CHAR_START_UNSAFE(s, i) |
|
#define | UTF16_NEXT_CHAR_SAFE(s, i, length, c, strict) |
|
#define | UTF16_APPEND_CHAR_SAFE(s, i, length, c) |
|
#define | UTF16_FWD_1_SAFE(s, i, length) U16_FWD_1(s, i, length) |
|
#define | UTF16_FWD_N_SAFE(s, i, length, n) U16_FWD_N(s, i, length, n) |
|
#define | UTF16_SET_CHAR_START_SAFE(s, start, i) U16_SET_CP_START(s, start, i) |
|
#define | UTF16_PREV_CHAR_UNSAFE(s, i, c) |
|
#define | UTF16_BACK_1_UNSAFE(s, i) |
|
#define | UTF16_BACK_N_UNSAFE(s, i, n) |
|
#define | UTF16_SET_CHAR_LIMIT_UNSAFE(s, i) |
|
#define | UTF16_PREV_CHAR_SAFE(s, start, i, c, strict) |
|
#define | UTF16_BACK_1_SAFE(s, start, i) U16_BACK_1(s, start, i) |
|
#define | UTF16_BACK_N_SAFE(s, start, i, n) U16_BACK_N(s, start, i, n) |
|
#define | UTF16_SET_CHAR_LIMIT_SAFE(s, start, i, length) U16_SET_CP_LIMIT(s, start, i, length) |
|
#define | UTF32_IS_SAFE(c, strict) |
|
#define | UTF32_IS_SINGLE(uchar) 1 |
|
#define | UTF32_IS_LEAD(uchar) 0 |
|
#define | UTF32_IS_TRAIL(uchar) 0 |
|
#define | UTF32_NEED_MULTIPLE_UCHAR(c) 0 |
|
#define | UTF32_CHAR_LENGTH(c) 1 |
|
#define | UTF32_MAX_CHAR_LENGTH 1 |
|
#define | UTF32_ARRAY_SIZE(size) (size) |
|
#define | UTF32_GET_CHAR_UNSAFE(s, i, c) |
|
#define | UTF32_GET_CHAR_SAFE(s, start, i, length, c, strict) |
|
#define | UTF32_NEXT_CHAR_UNSAFE(s, i, c) |
|
#define | UTF32_APPEND_CHAR_UNSAFE(s, i, c) |
|
#define | UTF32_FWD_1_UNSAFE(s, i) |
|
#define | UTF32_FWD_N_UNSAFE(s, i, n) |
|
#define | UTF32_SET_CHAR_START_UNSAFE(s, i) |
|
#define | UTF32_NEXT_CHAR_SAFE(s, i, length, c, strict) |
|
#define | UTF32_APPEND_CHAR_SAFE(s, i, length, c) |
|
#define | UTF32_FWD_1_SAFE(s, i, length) |
|
#define | UTF32_FWD_N_SAFE(s, i, length, n) |
|
#define | UTF32_SET_CHAR_START_SAFE(s, start, i) |
|
#define | UTF32_PREV_CHAR_UNSAFE(s, i, c) |
|
#define | UTF32_BACK_1_UNSAFE(s, i) |
|
#define | UTF32_BACK_N_UNSAFE(s, i, n) |
|
#define | UTF32_SET_CHAR_LIMIT_UNSAFE(s, i) |
|
#define | UTF32_PREV_CHAR_SAFE(s, start, i, c, strict) |
|
#define | UTF32_BACK_1_SAFE(s, start, i) |
|
#define | UTF32_BACK_N_SAFE(s, start, i, n) |
|
#define | UTF32_SET_CHAR_LIMIT_SAFE(s, i, length) |
|
#define | UTF_ARRAY_SIZE(size) UTF16_ARRAY_SIZE(size) |
| Estimate the number of code units for a string based on the number of UTF-16 code units.
|
|
#define | UTF_GET_CHAR_UNSAFE(s, i, c) UTF16_GET_CHAR_UNSAFE(s, i, c) |
|
#define | UTF_GET_CHAR_SAFE(s, start, i, length, c, strict) UTF16_GET_CHAR_SAFE(s, start, i, length, c, strict) |
|
#define | UTF_NEXT_CHAR_UNSAFE(s, i, c) UTF16_NEXT_CHAR_UNSAFE(s, i, c) |
|
#define | UTF_NEXT_CHAR_SAFE(s, i, length, c, strict) UTF16_NEXT_CHAR_SAFE(s, i, length, c, strict) |
|
#define | UTF_APPEND_CHAR_UNSAFE(s, i, c) UTF16_APPEND_CHAR_UNSAFE(s, i, c) |
|
#define | UTF_APPEND_CHAR_SAFE(s, i, length, c) UTF16_APPEND_CHAR_SAFE(s, i, length, c) |
|
#define | UTF_FWD_1_UNSAFE(s, i) UTF16_FWD_1_UNSAFE(s, i) |
|
#define | UTF_FWD_1_SAFE(s, i, length) UTF16_FWD_1_SAFE(s, i, length) |
|
#define | UTF_FWD_N_UNSAFE(s, i, n) UTF16_FWD_N_UNSAFE(s, i, n) |
|
#define | UTF_FWD_N_SAFE(s, i, length, n) UTF16_FWD_N_SAFE(s, i, length, n) |
|
#define | UTF_SET_CHAR_START_UNSAFE(s, i) UTF16_SET_CHAR_START_UNSAFE(s, i) |
|
#define | UTF_SET_CHAR_START_SAFE(s, start, i) UTF16_SET_CHAR_START_SAFE(s, start, i) |
|
#define | UTF_PREV_CHAR_UNSAFE(s, i, c) UTF16_PREV_CHAR_UNSAFE(s, i, c) |
|
#define | UTF_PREV_CHAR_SAFE(s, start, i, c, strict) UTF16_PREV_CHAR_SAFE(s, start, i, c, strict) |
|
#define | UTF_BACK_1_UNSAFE(s, i) UTF16_BACK_1_UNSAFE(s, i) |
|
#define | UTF_BACK_1_SAFE(s, start, i) UTF16_BACK_1_SAFE(s, start, i) |
|
#define | UTF_BACK_N_UNSAFE(s, i, n) UTF16_BACK_N_UNSAFE(s, i, n) |
|
#define | UTF_BACK_N_SAFE(s, start, i, n) UTF16_BACK_N_SAFE(s, start, i, n) |
|
#define | UTF_SET_CHAR_LIMIT_UNSAFE(s, i) UTF16_SET_CHAR_LIMIT_UNSAFE(s, i) |
|
#define | UTF_SET_CHAR_LIMIT_SAFE(s, start, i, length) UTF16_SET_CHAR_LIMIT_SAFE(s, start, i, length) |
|
#define | UTF_IS_SINGLE(uchar) U16_IS_SINGLE(uchar) |
| Does this code unit alone encode a code point (BMP, not a surrogate)? Same as UTF16_IS_SINGLE.
|
|
#define | UTF_IS_LEAD(uchar) U16_IS_LEAD(uchar) |
| Is this code unit the first one of several (a lead surrogate)? Same as UTF16_IS_LEAD.
|
|
#define | UTF_IS_TRAIL(uchar) U16_IS_TRAIL(uchar) |
| Is this code unit one of several but not the first one (a trail surrogate)? Same as UTF16_IS_TRAIL.
|
|
#define | UTF_NEED_MULTIPLE_UCHAR(c) UTF16_NEED_MULTIPLE_UCHAR(c) |
| Does this code point require multiple code units (is it a supplementary code point)? Same as UTF16_NEED_MULTIPLE_UCHAR.
|
|
#define | UTF_CHAR_LENGTH(c) U16_LENGTH(c) |
| How many code units are used to encode this code point (1 or 2)? Same as UTF16_CHAR_LENGTH.
|
|
#define | UTF_MAX_CHAR_LENGTH U16_MAX_LENGTH |
| How many code units are used at most for any Unicode code point (2)? Same as UTF16_MAX_CHAR_LENGTH.
|
|
#define | UTF_GET_CHAR(s, start, i, length, c) U16_GET(s, start, i, length, c) |
| Set c to the code point that contains the code unit i.
|
|
#define | UTF_NEXT_CHAR(s, i, length, c) U16_NEXT(s, i, length, c) |
| Set c to the code point that starts at code unit i and advance i to beyond the code units of this code point (post-increment).
|
|
#define | UTF_APPEND_CHAR(s, i, length, c) UTF16_APPEND_CHAR_SAFE(s, i, length, c) |
| Append the code units of code point c to the string at index i and advance i to beyond the new code units (post-increment).
|
|
#define | UTF_FWD_1(s, i, length) U16_FWD_1(s, i, length) |
| Advance i to beyond the code units of the code point that begins at i.
|
|
#define | UTF_FWD_N(s, i, length, n) U16_FWD_N(s, i, length, n) |
| Advance i to beyond the code units of the n code points where the first one begins at i.
|
|
#define | UTF_SET_CHAR_START(s, start, i) U16_SET_CP_START(s, start, i) |
| Take the random-access index i and adjust it so that it points to the beginning of a code point.
|
|
#define | UTF_PREV_CHAR(s, start, i, c) U16_PREV(s, start, i, c) |
| Set c to the code point that has code units before i and move i backward (towards the beginning of the string) to the first code unit of this code point (pre-increment).
|
|
#define | UTF_BACK_1(s, start, i) U16_BACK_1(s, start, i) |
| Move i backward (towards the beginning of the string) to the first code unit of the code point that has code units before i.
|
|
#define | UTF_BACK_N(s, start, i, n) U16_BACK_N(s, start, i, n) |
| Move i backward (towards the beginning of the string) to the first code unit of the n code points that have code units before i.
|
|
#define | UTF_SET_CHAR_LIMIT(s, start, i, length) U16_SET_CP_LIMIT(s, start, i, length) |
| Take the random-access index i and adjust it so that it points beyond a code point.
|
|
C API: Deprecated macros for Unicode string handling.
The macros in utf_old.h are all deprecated and their use discouraged. Some of the design principles behind the set of UTF macros have changed or proved impractical. Almost all of the old "UTF macros" are at least renamed. If you are looking for a new equivalent to an old macro, please see the comment at the old one.
Brief summary of reasons for deprecation:
- Switch on UTF_SIZE (selection of UTF-8/16/32 default string processing) was impractical.
- Switch on UTF_SAFE etc. (selection of unsafe/safe/strict default string processing) was of little use and impractical.
- Whole classes of macros became obsolete outside of the UTF_SIZE/UTF_SAFE selection framework: UTF32_ macros (all trivial) and UTF_ default and intermediate macros (all aliases).
- The selection framework also caused many macro aliases.
- Change in Unicode standard: "irregular" sequences (3.0) became illegal (3.2).
- Change of language in Unicode standard: Growing distinction between internal x-bit Unicode strings and external UTF-x forms, with the former more lenient. Suggests renaming of UTF16_ macros to U16_.
- The prefix "UTF_" without a width number confused some users.
- "Safe" append macros needed the addition of an error indicator output.
- "Safe" UTF-8 macros used legitimate (if rarely used) code point values to indicate error conditions.
- The use of the "_CHAR" infix for code point operations confused some users.
More details:
Until ICU 2.2, utf.h theoretically allowed to choose among UTF-8/16/32 for string processing, and among unsafe/safe/strict default macros for that.
It proved nearly impossible to write non-trivial, high-performance code that is UTF-generic. Unsafe default macros would be dangerous for default string processing, and the main reason for the "strict" versions disappeared: Between Unicode 3.0 and 3.2 all "irregular" UTF-8 sequences became illegal. The only other conditions that "strict" checked for were non-characters, which are valid during processing. Only during text input/output should they be checked, and at that time other well-formedness checks may be necessary or useful as well. This can still be done by using U16_NEXT and U_IS_UNICODE_NONCHAR or U_IS_UNICODE_CHAR.
The old UTF8_..._SAFE macros also used some normal Unicode code points to indicate malformed sequences. The new UTF8_ macros without suffix use negative values instead.
The entire contents of utf32.h was moved here without replacement because all those macros were trivial and were meaningful only in the framework of choosing the UTF size.
See Jitterbug 2150 and its discussion on the ICU mailing list in September 2002.
Obsolete part of pre-ICU 2.4 utf.h file documentation:
The original concept for these files was for ICU to allow in principle to set which UTF (UTF-8/16/32) is used internally by defining UTF_SIZE to either 8, 16, or 32. utf.h would then define the UChar type accordingly. UTF-16 was the default.
This concept has been abandoned. A lot of the ICU source code assumes UChar strings are in UTF-16. This is especially true for low-level code like conversion, normalization, and collation. The utf.h header enforces the default of UTF-16. The UTF-8 and UTF-32 macros remain for now for completeness and backward compatibility.
Accordingly, utf.h defines UChar to be an unsigned 16-bit integer. If this matches wchar_t, then UChar is defined to be exactly wchar_t, otherwise uint16_t.
UChar32 is defined to be a signed 32-bit integer (int32_t), large enough for a 21-bit Unicode code point (Unicode scalar value, 0..0x10ffff). Before ICU 2.4, the definition of UChar32 was similarly platform-dependent as the definition of UChar. For details see the documentation for UChar32 itself.
utf.h also defines a number of C macros for handling single Unicode code points and for using UTF Unicode strings. It includes utf8.h, utf16.h, and utf32.h for the actual implementations of those macros and then aliases one set of them (for UTF-16) for general use. The UTF-specific macros have the UTF size in the macro name prefixes (UTF16_...), while the general alias macros always begin with UTF_...
Many string operations can be done with or without error checking. Where such a distinction is useful, there are two versions of the macros, "unsafe" and "safe" ones with ..._UNSAFE and ..._SAFE suffixes. The unsafe macros are fast but may cause program failures if the strings are not well-formed. The safe macros have an additional, boolean parameter "strict". If strict is false, then only illegal sequences are detected. Otherwise, irregular sequences and non-characters are detected as well (like single surrogates). Safe macros return special error code points for illegal/irregular sequences: Typically, U+ffff, or values that would result in a code unit sequence of the same length as the erroneous input sequence.
Note that _UNSAFE macros have fewer parameters: They do not have the strictness parameter, and they do not have start/length parameters for boundary checking.
Here, the macros are aliased in two steps: In the first step, the UTF-specific macros with UTF16_ prefix and UNSAFE and _SAFE suffixes are aliased according to the UTF_SIZE to macros with UTF prefix and the same suffixes and signatures. Then, in a second step, the default, general alias macros are set to use either the unsafe or the safe/not strict (default) or the safe/strict macro; these general macros do not have a strictness parameter.
It is possible to change the default choice for the general alias macros to be unsafe, safe/not strict or safe/strict. The default is safe/not strict. It is not recommended to select the unsafe macros as the basis for Unicode string handling in ICU! To select this, define UTF_SAFE, UTF_STRICT, or UTF_UNSAFE.
For general use, one should use the default, general macros with UTF_ prefix and no SAFE/_UNSAFE suffix. Only in some cases it may be necessary to control the choice of macro directly and use a less generic alias. For example, if it can be assumed that a string is well-formed and the index will stay within the bounds, then the _UNSAFE version may be used. If a UTF-8 string is to be processed, then the macros with UTF8 prefixes need to be used.
Deprecated ICU 2.4. Use the macros in utf.h, utf16.h, utf8.h instead.
Definition in file utf_old.h.