1/*
2*******************************************************************************
3* Copyright (c) 1996-2010, International Business Machines Corporation
4* and others. All Rights Reserved.
5*******************************************************************************
6* File unorm.h
7*
8* Created by: Vladimir Weinstein 12052000
9*
10* Modification history :
11*
12* Date Name Description
13* 02/01/01 synwee Added normalization quickcheck enum and method.
14*/
15#ifndef UNORM_H
16#define UNORM_H
17
18#include "unicode/utypes.h"
19
20#if !UCONFIG_NO_NORMALIZATION
21
22#include "unicode/uiter.h"
23#include "unicode/unorm2.h"
24
25/**
26 * \file
27 * \brief C API: Unicode Normalization
28 *
29 * <h2>Unicode normalization API</h2>
30 *
31 * Note: This API has been replaced by the unorm2.h API and is only available
32 * for backward compatibility. The functions here simply delegate to the
33 * unorm2.h functions, for example unorm2_getInstance() and unorm2_normalize().
34 * There is one exception: The new API does not provide a replacement for unorm_compare().
35 *
36 * <code>unorm_normalize</code> transforms Unicode text into an equivalent composed or
37 * decomposed form, allowing for easier sorting and searching of text.
38 * <code>unorm_normalize</code> supports the standard normalization forms described in
39 * <a href="http://www.unicode.org/unicode/reports/tr15/" target="unicode">
40 * Unicode Standard Annex #15: Unicode Normalization Forms</a>.
41 *
42 * Characters with accents or other adornments can be encoded in
43 * several different ways in Unicode. For example, take the character A-acute.
44 * In Unicode, this can be encoded as a single character (the
45 * "composed" form):
46 *
47 * \code
48 * 00C1 LATIN CAPITAL LETTER A WITH ACUTE
49 * \endcode
50 *
51 * or as two separate characters (the "decomposed" form):
52 *
53 * \code
54 * 0041 LATIN CAPITAL LETTER A
55 * 0301 COMBINING ACUTE ACCENT
56 * \endcode
57 *
58 * To a user of your program, however, both of these sequences should be
59 * treated as the same "user-level" character "A with acute accent". When you are searching or
60 * comparing text, you must ensure that these two sequences are treated
61 * equivalently. In addition, you must handle characters with more than one
62 * accent. Sometimes the order of a character's combining accents is
63 * significant, while in other cases accent sequences in different orders are
64 * really equivalent.
65 *
66 * Similarly, the string "ffi" can be encoded as three separate letters:
67 *
68 * \code
69 * 0066 LATIN SMALL LETTER F
70 * 0066 LATIN SMALL LETTER F
71 * 0069 LATIN SMALL LETTER I
72 * \endcode
73 *
74 * or as the single character
75 *
76 * \code
77 * FB03 LATIN SMALL LIGATURE FFI
78 * \endcode
79 *
80 * The ffi ligature is not a distinct semantic character, and strictly speaking
81 * it shouldn't be in Unicode at all, but it was included for compatibility
82 * with existing character sets that already provided it. The Unicode standard
83 * identifies such characters by giving them "compatibility" decompositions
84 * into the corresponding semantic characters. When sorting and searching, you
85 * will often want to use these mappings.
86 *
87 * <code>unorm_normalize</code> helps solve these problems by transforming text into the
88 * canonical composed and decomposed forms as shown in the first example above.
89 * In addition, you can have it perform compatibility decompositions so that
90 * you can treat compatibility characters the same as their equivalents.
91 * Finally, <code>unorm_normalize</code> rearranges accents into the proper canonical
92 * order, so that you do not have to worry about accent rearrangement on your
93 * own.
94 *
95 * Form FCD, "Fast C or D", is also designed for collation.
96 * It allows to work on strings that are not necessarily normalized
97 * with an algorithm (like in collation) that works under "canonical closure", i.e., it treats precomposed
98 * characters and their decomposed equivalents the same.
99 *
100 * It is not a normalization form because it does not provide for uniqueness of representation. Multiple strings
101 * may be canonically equivalent (their NFDs are identical) and may all conform to FCD without being identical
102 * themselves.
103 *
104 * The form is defined such that the "raw decomposition", the recursive canonical decomposition of each character,
105 * results in a string that is canonically ordered. This means that precomposed characters are allowed for as long
106 * as their decompositions do not need canonical reordering.
107 *
108 * Its advantage for a process like collation is that all NFD and most NFC texts - and many unnormalized texts -
109 * already conform to FCD and do not need to be normalized (NFD) for such a process. The FCD quick check will
110 * return UNORM_YES for most strings in practice.
111 *
112 * unorm_normalize(UNORM_FCD) may be implemented with UNORM_NFD.
113 *
114 * For more details on FCD see the collation design document:
115 * http://source.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
116 *
117 * ICU collation performs either NFD or FCD normalization automatically if normalization
118 * is turned on for the collator object.
119 * Beyond collation and string search, normalized strings may be useful for string equivalence comparisons,
120 * transliteration/transcription, unique representations, etc.
121 *
122 * The W3C generally recommends to exchange texts in NFC.
123 * Note also that most legacy character encodings use only precomposed forms and often do not
124 * encode any combining marks by themselves. For conversion to such character encodings the
125 * Unicode text needs to be normalized to NFC.
126 * For more usage examples, see the Unicode Standard Annex.
127 */
128
129/**
130 * Constants for normalization modes.
131 * @stable ICU 2.0
132 */
133typedef enum {
134 /** No decomposition/composition. @stable ICU 2.0 */
135 UNORM_NONE = 1,
136 /** Canonical decomposition. @stable ICU 2.0 */
137 UNORM_NFD = 2,
138 /** Compatibility decomposition. @stable ICU 2.0 */
139 UNORM_NFKD = 3,
140 /** Canonical decomposition followed by canonical composition. @stable ICU 2.0 */
141 UNORM_NFC = 4,
142 /** Default normalization. @stable ICU 2.0 */
143 UNORM_DEFAULT = UNORM_NFC,
144 /** Compatibility decomposition followed by canonical composition. @stable ICU 2.0 */
145 UNORM_NFKC =5,
146 /** "Fast C or D" form. @stable ICU 2.0 */
147 UNORM_FCD = 6,
148
149 /** One more than the highest normalization mode constant. @stable ICU 2.0 */
150 UNORM_MODE_COUNT
151} UNormalizationMode;
152
153/**
154 * Constants for options flags for normalization.
155 * Use 0 for default options,
156 * including normalization according to the Unicode version
157 * that is currently supported by ICU (see u_getUnicodeVersion).
158 * @stable ICU 2.6
159 */
160enum {
161 /**
162 * Options bit set value to select Unicode 3.2 normalization
163 * (except NormalizationCorrections).
164 * At most one Unicode version can be selected at a time.
165 * @stable ICU 2.6
166 */
167 UNORM_UNICODE_3_2=0x20
168};
169
170/**
171 * Lowest-order bit number of unorm_compare() options bits corresponding to
172 * normalization options bits.
173 *
174 * The options parameter for unorm_compare() uses most bits for
175 * itself and for various comparison and folding flags.
176 * The most significant bits, however, are shifted down and passed on
177 * to the normalization implementation.
178 * (That is, from unorm_compare(..., options, ...),
179 * options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT will be passed on to the
180 * internal normalization functions.)
181 *
182 * @see unorm_compare
183 * @stable ICU 2.6
184 */
185#define UNORM_COMPARE_NORM_OPTIONS_SHIFT 20
186
187/**
188 * Normalize a string.
189 * The string will be normalized according the specified normalization mode
190 * and options.
191 * The source and result buffers must not be the same, nor overlap.
192 *
193 * @param source The string to normalize.
194 * @param sourceLength The length of source, or -1 if NUL-terminated.
195 * @param mode The normalization mode; one of UNORM_NONE,
196 * UNORM_NFD, UNORM_NFC, UNORM_NFKC, UNORM_NFKD, UNORM_DEFAULT.
197 * @param options The normalization options, ORed together (0 for no options).
198 * @param result A pointer to a buffer to receive the result string.
199 * The result string is NUL-terminated if possible.
200 * @param resultLength The maximum size of result.
201 * @param status A pointer to a UErrorCode to receive any errors.
202 * @return The total buffer size needed; if greater than resultLength,
203 * the output was truncated, and the error code is set to U_BUFFER_OVERFLOW_ERROR.
204 * @stable ICU 2.0
205 */
206U_STABLE int32_t U_EXPORT2
207unorm_normalize(const UChar *source, int32_t sourceLength,
208 UNormalizationMode mode, int32_t options,
209 UChar *result, int32_t resultLength,
210 UErrorCode *status);
211
212/**
213 * Performing quick check on a string, to quickly determine if the string is
214 * in a particular normalization format.
215 * Three types of result can be returned UNORM_YES, UNORM_NO or
216 * UNORM_MAYBE. Result UNORM_YES indicates that the argument
217 * string is in the desired normalized format, UNORM_NO determines that
218 * argument string is not in the desired normalized format. A
219 * UNORM_MAYBE result indicates that a more thorough check is required,
220 * the user may have to put the string in its normalized form and compare the
221 * results.
222 *
223 * @param source string for determining if it is in a normalized format
224 * @param sourcelength length of source to test, or -1 if NUL-terminated
225 * @param mode which normalization form to test for
226 * @param status a pointer to a UErrorCode to receive any errors
227 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
228 *
229 * @see unorm_isNormalized
230 * @stable ICU 2.0
231 */
232U_STABLE UNormalizationCheckResult U_EXPORT2
233unorm_quickCheck(const UChar *source, int32_t sourcelength,
234 UNormalizationMode mode,
235 UErrorCode *status);
236
237/**
238 * Performing quick check on a string; same as unorm_quickCheck but
239 * takes an extra options parameter like most normalization functions.
240 *
241 * @param src String that is to be tested if it is in a normalization format.
242 * @param srcLength Length of source to test, or -1 if NUL-terminated.
243 * @param mode Which normalization form to test for.
244 * @param options The normalization options, ORed together (0 for no options).
245 * @param pErrorCode ICU error code in/out parameter.
246 * Must fulfill U_SUCCESS before the function call.
247 * @return UNORM_YES, UNORM_NO or UNORM_MAYBE
248 *
249 * @see unorm_quickCheck
250 * @see unorm_isNormalized
251 * @stable ICU 2.6
252 */
253U_STABLE UNormalizationCheckResult U_EXPORT2
254unorm_quickCheckWithOptions(const UChar *src, int32_t srcLength,
255 UNormalizationMode mode, int32_t options,
256 UErrorCode *pErrorCode);
257
258/**
259 * Test if a string is in a given normalization form.
260 * This is semantically equivalent to source.equals(normalize(source, mode)) .
261 *
262 * Unlike unorm_quickCheck(), this function returns a definitive result,
263 * never a "maybe".
264 * For NFD, NFKD, and FCD, both functions work exactly the same.
265 * For NFC and NFKC where quickCheck may return "maybe", this function will
266 * perform further tests to arrive at a TRUE/FALSE result.
267 *
268 * @param src String that is to be tested if it is in a normalization format.
269 * @param srcLength Length of source to test, or -1 if NUL-terminated.
270 * @param mode Which normalization form to test for.
271 * @param pErrorCode ICU error code in/out parameter.
272 * Must fulfill U_SUCCESS before the function call.
273 * @return Boolean value indicating whether the source string is in the
274 * "mode" normalization form.
275 *
276 * @see unorm_quickCheck
277 * @stable ICU 2.2
278 */
279U_STABLE UBool U_EXPORT2
280unorm_isNormalized(const UChar *src, int32_t srcLength,
281 UNormalizationMode mode,
282 UErrorCode *pErrorCode);
283
284/**
285 * Test if a string is in a given normalization form; same as unorm_isNormalized but
286 * takes an extra options parameter like most normalization functions.
287 *
288 * @param src String that is to be tested if it is in a normalization format.
289 * @param srcLength Length of source to test, or -1 if NUL-terminated.
290 * @param mode Which normalization form to test for.
291 * @param options The normalization options, ORed together (0 for no options).
292 * @param pErrorCode ICU error code in/out parameter.
293 * Must fulfill U_SUCCESS before the function call.
294 * @return Boolean value indicating whether the source string is in the
295 * "mode/options" normalization form.
296 *
297 * @see unorm_quickCheck
298 * @see unorm_isNormalized
299 * @stable ICU 2.6
300 */
301U_STABLE UBool U_EXPORT2
302unorm_isNormalizedWithOptions(const UChar *src, int32_t srcLength,
303 UNormalizationMode mode, int32_t options,
304 UErrorCode *pErrorCode);
305
306/**
307 * Iterative normalization forward.
308 * This function (together with unorm_previous) is somewhat
309 * similar to the C++ Normalizer class (see its non-static functions).
310 *
311 * Iterative normalization is useful when only a small portion of a longer
312 * string/text needs to be processed.
313 *
314 * For example, the likelihood may be high that processing the first 10% of some
315 * text will be sufficient to find certain data.
316 * Another example: When one wants to concatenate two normalized strings and get a
317 * normalized result, it is much more efficient to normalize just a small part of
318 * the result around the concatenation place instead of re-normalizing everything.
319 *
320 * The input text is an instance of the C character iteration API UCharIterator.
321 * It may wrap around a simple string, a CharacterIterator, a Replaceable, or any
322 * other kind of text object.
323 *
324 * If a buffer overflow occurs, then the caller needs to reset the iterator to the
325 * old index and call the function again with a larger buffer - if the caller cares
326 * for the actual output.
327 * Regardless of the output buffer, the iterator will always be moved to the next
328 * normalization boundary.
329 *
330 * This function (like unorm_previous) serves two purposes:
331 *
332 * 1) To find the next boundary so that the normalization of the part of the text
333 * from the current position to that boundary does not affect and is not affected
334 * by the part of the text beyond that boundary.
335 *
336 * 2) To normalize the text up to the boundary.
337 *
338 * The second step is optional, per the doNormalize parameter.
339 * It is omitted for operations like string concatenation, where the two adjacent
340 * string ends need to be normalized together.
341 * In such a case, the output buffer will just contain a copy of the text up to the
342 * boundary.
343 *
344 * pNeededToNormalize is an output-only parameter. Its output value is only defined
345 * if normalization was requested (doNormalize) and successful (especially, no
346 * buffer overflow).
347 * It is useful for operations like a normalizing transliterator, where one would
348 * not want to replace a piece of text if it is not modified.
349 *
350 * If doNormalize==TRUE and pNeededToNormalize!=NULL then *pNeeded... is set TRUE
351 * if the normalization was necessary.
352 *
353 * If doNormalize==FALSE then *pNeededToNormalize will be set to FALSE.
354 *
355 * If the buffer overflows, then *pNeededToNormalize will be undefined;
356 * essentially, whenever U_FAILURE is true (like in buffer overflows), this result
357 * will be undefined.
358 *
359 * @param src The input text in the form of a C character iterator.
360 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
361 * @param destCapacity The number of UChars that fit into dest.
362 * @param mode The normalization mode.
363 * @param options The normalization options, ORed together (0 for no options).
364 * @param doNormalize Indicates if the source text up to the next boundary
365 * is to be normalized (TRUE) or just copied (FALSE).
366 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
367 * different text from the input.
368 * Not defined if an error occurs including buffer overflow.
369 * Always FALSE if !doNormalize.
370 * @param pErrorCode ICU error code in/out parameter.
371 * Must fulfill U_SUCCESS before the function call.
372 * @return Length of output (number of UChars) when successful or buffer overflow.
373 *
374 * @see unorm_previous
375 * @see unorm_normalize
376 *
377 * @stable ICU 2.1
378 */
379U_STABLE int32_t U_EXPORT2
380unorm_next(UCharIterator *src,
381 UChar *dest, int32_t destCapacity,
382 UNormalizationMode mode, int32_t options,
383 UBool doNormalize, UBool *pNeededToNormalize,
384 UErrorCode *pErrorCode);
385
386/**
387 * Iterative normalization backward.
388 * This function (together with unorm_next) is somewhat
389 * similar to the C++ Normalizer class (see its non-static functions).
390 * For all details see unorm_next.
391 *
392 * @param src The input text in the form of a C character iterator.
393 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
394 * @param destCapacity The number of UChars that fit into dest.
395 * @param mode The normalization mode.
396 * @param options The normalization options, ORed together (0 for no options).
397 * @param doNormalize Indicates if the source text up to the next boundary
398 * is to be normalized (TRUE) or just copied (FALSE).
399 * @param pNeededToNormalize Output flag indicating if the normalization resulted in
400 * different text from the input.
401 * Not defined if an error occurs including buffer overflow.
402 * Always FALSE if !doNormalize.
403 * @param pErrorCode ICU error code in/out parameter.
404 * Must fulfill U_SUCCESS before the function call.
405 * @return Length of output (number of UChars) when successful or buffer overflow.
406 *
407 * @see unorm_next
408 * @see unorm_normalize
409 *
410 * @stable ICU 2.1
411 */
412U_STABLE int32_t U_EXPORT2
413unorm_previous(UCharIterator *src,
414 UChar *dest, int32_t destCapacity,
415 UNormalizationMode mode, int32_t options,
416 UBool doNormalize, UBool *pNeededToNormalize,
417 UErrorCode *pErrorCode);
418
419/**
420 * Concatenate normalized strings, making sure that the result is normalized as well.
421 *
422 * If both the left and the right strings are in
423 * the normalization form according to "mode/options",
424 * then the result will be
425 *
426 * \code
427 * dest=normalize(left+right, mode, options)
428 * \endcode
429 *
430 * With the input strings already being normalized,
431 * this function will use unorm_next() and unorm_previous()
432 * to find the adjacent end pieces of the input strings.
433 * Only the concatenation of these end pieces will be normalized and
434 * then concatenated with the remaining parts of the input strings.
435 *
436 * It is allowed to have dest==left to avoid copying the entire left string.
437 *
438 * @param left Left source string, may be same as dest.
439 * @param leftLength Length of left source string, or -1 if NUL-terminated.
440 * @param right Right source string. Must not be the same as dest, nor overlap.
441 * @param rightLength Length of right source string, or -1 if NUL-terminated.
442 * @param dest The output buffer; can be NULL if destCapacity==0 for pure preflighting.
443 * @param destCapacity The number of UChars that fit into dest.
444 * @param mode The normalization mode.
445 * @param options The normalization options, ORed together (0 for no options).
446 * @param pErrorCode ICU error code in/out parameter.
447 * Must fulfill U_SUCCESS before the function call.
448 * @return Length of output (number of UChars) when successful or buffer overflow.
449 *
450 * @see unorm_normalize
451 * @see unorm_next
452 * @see unorm_previous
453 *
454 * @stable ICU 2.1
455 */
456U_STABLE int32_t U_EXPORT2
457unorm_concatenate(const UChar *left, int32_t leftLength,
458 const UChar *right, int32_t rightLength,
459 UChar *dest, int32_t destCapacity,
460 UNormalizationMode mode, int32_t options,
461 UErrorCode *pErrorCode);
462
463/**
464 * Option bit for unorm_compare:
465 * Both input strings are assumed to fulfill FCD conditions.
466 * @stable ICU 2.2
467 */
468#define UNORM_INPUT_IS_FCD 0x20000
469
470/**
471 * Option bit for unorm_compare:
472 * Perform case-insensitive comparison.
473 * @stable ICU 2.2
474 */
475#define U_COMPARE_IGNORE_CASE 0x10000
476
477#ifndef U_COMPARE_CODE_POINT_ORDER
478/* see also unistr.h and ustring.h */
479/**
480 * Option bit for u_strCaseCompare, u_strcasecmp, unorm_compare, etc:
481 * Compare strings in code point order instead of code unit order.
482 * @stable ICU 2.2
483 */
484#define U_COMPARE_CODE_POINT_ORDER 0x8000
485#endif
486
487/**
488 * Compare two strings for canonical equivalence.
489 * Further options include case-insensitive comparison and
490 * code point order (as opposed to code unit order).
491 *
492 * Canonical equivalence between two strings is defined as their normalized
493 * forms (NFD or NFC) being identical.
494 * This function compares strings incrementally instead of normalizing
495 * (and optionally case-folding) both strings entirely,
496 * improving performance significantly.
497 *
498 * Bulk normalization is only necessary if the strings do not fulfill the FCD
499 * conditions. Only in this case, and only if the strings are relatively long,
500 * is memory allocated temporarily.
501 * For FCD strings and short non-FCD strings there is no memory allocation.
502 *
503 * Semantically, this is equivalent to
504 * strcmp[CodePointOrder](NFD(foldCase(NFD(s1))), NFD(foldCase(NFD(s2))))
505 * where code point order and foldCase are all optional.
506 *
507 * UAX 21 2.5 Caseless Matching specifies that for a canonical caseless match
508 * the case folding must be performed first, then the normalization.
509 *
510 * @param s1 First source string.
511 * @param length1 Length of first source string, or -1 if NUL-terminated.
512 *
513 * @param s2 Second source string.
514 * @param length2 Length of second source string, or -1 if NUL-terminated.
515 *
516 * @param options A bit set of options:
517 * - U_FOLD_CASE_DEFAULT or 0 is used for default options:
518 * Case-sensitive comparison in code unit order, and the input strings
519 * are quick-checked for FCD.
520 *
521 * - UNORM_INPUT_IS_FCD
522 * Set if the caller knows that both s1 and s2 fulfill the FCD conditions.
523 * If not set, the function will quickCheck for FCD
524 * and normalize if necessary.
525 *
526 * - U_COMPARE_CODE_POINT_ORDER
527 * Set to choose code point order instead of code unit order
528 * (see u_strCompare for details).
529 *
530 * - U_COMPARE_IGNORE_CASE
531 * Set to compare strings case-insensitively using case folding,
532 * instead of case-sensitively.
533 * If set, then the following case folding options are used.
534 *
535 * - Options as used with case-insensitive comparisons, currently:
536 *
537 * - U_FOLD_CASE_EXCLUDE_SPECIAL_I
538 * (see u_strCaseCompare for details)
539 *
540 * - regular normalization options shifted left by UNORM_COMPARE_NORM_OPTIONS_SHIFT
541 *
542 * @param pErrorCode ICU error code in/out parameter.
543 * Must fulfill U_SUCCESS before the function call.
544 * @return <0 or 0 or >0 as usual for string comparisons
545 *
546 * @see unorm_normalize
547 * @see UNORM_FCD
548 * @see u_strCompare
549 * @see u_strCaseCompare
550 *
551 * @stable ICU 2.2
552 */
553U_STABLE int32_t U_EXPORT2
554unorm_compare(const UChar *s1, int32_t length1,
555 const UChar *s2, int32_t length2,
556 uint32_t options,
557 UErrorCode *pErrorCode);
558
559#endif /* #if !UCONFIG_NO_NORMALIZATION */
560
561#endif
562