1 | /*===--- ConvertUTF.h - Universal Character Names conversions ---------------=== |
2 | * |
3 | * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | * See https://llvm.org/LICENSE.txt for license information. |
5 | * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | * |
7 | *==------------------------------------------------------------------------==*/ |
8 | /* |
9 | * Copyright © 1991-2015 Unicode, Inc. All rights reserved. |
10 | * Distributed under the Terms of Use in |
11 | * http://www.unicode.org/copyright.html. |
12 | * |
13 | * Permission is hereby granted, free of charge, to any person obtaining |
14 | * a copy of the Unicode data files and any associated documentation |
15 | * (the "Data Files") or Unicode software and any associated documentation |
16 | * (the "Software") to deal in the Data Files or Software |
17 | * without restriction, including without limitation the rights to use, |
18 | * copy, modify, merge, publish, distribute, and/or sell copies of |
19 | * the Data Files or Software, and to permit persons to whom the Data Files |
20 | * or Software are furnished to do so, provided that |
21 | * (a) this copyright and permission notice appear with all copies |
22 | * of the Data Files or Software, |
23 | * (b) this copyright and permission notice appear in associated |
24 | * documentation, and |
25 | * (c) there is clear notice in each modified Data File or in the Software |
26 | * as well as in the documentation associated with the Data File(s) or |
27 | * Software that the data or software has been modified. |
28 | * |
29 | * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF |
30 | * ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE |
31 | * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
32 | * NONINFRINGEMENT OF THIRD PARTY RIGHTS. |
33 | * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS |
34 | * NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL |
35 | * DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, |
36 | * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER |
37 | * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR |
38 | * PERFORMANCE OF THE DATA FILES OR SOFTWARE. |
39 | * |
40 | * Except as contained in this notice, the name of a copyright holder |
41 | * shall not be used in advertising or otherwise to promote the sale, |
42 | * use or other dealings in these Data Files or Software without prior |
43 | * written authorization of the copyright holder. |
44 | */ |
45 | |
46 | /* --------------------------------------------------------------------- |
47 | |
48 | Conversions between UTF32, UTF-16, and UTF-8. Header file. |
49 | |
50 | Several funtions are included here, forming a complete set of |
51 | conversions between the three formats. UTF-7 is not included |
52 | here, but is handled in a separate source file. |
53 | |
54 | Each of these routines takes pointers to input buffers and output |
55 | buffers. The input buffers are const. |
56 | |
57 | Each routine converts the text between *sourceStart and sourceEnd, |
58 | putting the result into the buffer between *targetStart and |
59 | targetEnd. Note: the end pointers are *after* the last item: e.g. |
60 | *(sourceEnd - 1) is the last item. |
61 | |
62 | The return result indicates whether the conversion was successful, |
63 | and if not, whether the problem was in the source or target buffers. |
64 | (Only the first encountered problem is indicated.) |
65 | |
66 | After the conversion, *sourceStart and *targetStart are both |
67 | updated to point to the end of last text successfully converted in |
68 | the respective buffers. |
69 | |
70 | Input parameters: |
71 | sourceStart - pointer to a pointer to the source buffer. |
72 | The contents of this are modified on return so that |
73 | it points at the next thing to be converted. |
74 | targetStart - similarly, pointer to pointer to the target buffer. |
75 | sourceEnd, targetEnd - respectively pointers to the ends of the |
76 | two buffers, for overflow checking only. |
77 | |
78 | These conversion functions take a ConversionFlags argument. When this |
79 | flag is set to strict, both irregular sequences and isolated surrogates |
80 | will cause an error. When the flag is set to lenient, both irregular |
81 | sequences and isolated surrogates are converted. |
82 | |
83 | Whether the flag is strict or lenient, all illegal sequences will cause |
84 | an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>, |
85 | or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code |
86 | must check for illegal sequences. |
87 | |
88 | When the flag is set to lenient, characters over 0x10FFFF are converted |
89 | to the replacement character; otherwise (when the flag is set to strict) |
90 | they constitute an error. |
91 | |
92 | Output parameters: |
93 | The value "sourceIllegal" is returned from some routines if the input |
94 | sequence is malformed. When "sourceIllegal" is returned, the source |
95 | value will point to the illegal value that caused the problem. E.g., |
96 | in UTF-8 when a sequence is malformed, it points to the start of the |
97 | malformed sequence. |
98 | |
99 | Author: Mark E. Davis, 1994. |
100 | Rev History: Rick McGowan, fixes & updates May 2001. |
101 | Fixes & updates, Sept 2001. |
102 | |
103 | ------------------------------------------------------------------------ */ |
104 | |
105 | #ifndef LLVM_SUPPORT_CONVERTUTF_H |
106 | #define LLVM_SUPPORT_CONVERTUTF_H |
107 | |
108 | #include <cstddef> |
109 | #include <string> |
110 | |
111 | #if defined(_WIN32) |
112 | #include <system_error> |
113 | #endif |
114 | |
115 | // Wrap everything in namespace llvm so that programs can link with llvm and |
116 | // their own version of the unicode libraries. |
117 | |
118 | namespace llvm { |
119 | |
120 | /* --------------------------------------------------------------------- |
121 | The following 4 definitions are compiler-specific. |
122 | The C standard does not guarantee that wchar_t has at least |
123 | 16 bits, so wchar_t is no less portable than unsigned short! |
124 | All should be unsigned values to avoid sign extension during |
125 | bit mask & shift operations. |
126 | ------------------------------------------------------------------------ */ |
127 | |
128 | typedef unsigned int UTF32; /* at least 32 bits */ |
129 | typedef unsigned short UTF16; /* at least 16 bits */ |
130 | typedef unsigned char UTF8; /* typically 8 bits */ |
131 | typedef unsigned char Boolean; /* 0 or 1 */ |
132 | |
133 | /* Some fundamental constants */ |
134 | #define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD |
135 | #define UNI_MAX_BMP (UTF32)0x0000FFFF |
136 | #define UNI_MAX_UTF16 (UTF32)0x0010FFFF |
137 | #define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF |
138 | #define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF |
139 | |
140 | #define UNI_MAX_UTF8_BYTES_PER_CODE_POINT 4 |
141 | |
142 | #define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF |
143 | #define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE |
144 | |
145 | #define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF |
146 | #define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0xFFFE0000 |
147 | |
148 | typedef enum { |
149 | conversionOK, /* conversion successful */ |
150 | sourceExhausted, /* partial character in source, but hit end */ |
151 | targetExhausted, /* insuff. room in target for conversion */ |
152 | sourceIllegal /* source sequence is illegal/malformed */ |
153 | } ConversionResult; |
154 | |
155 | typedef enum { |
156 | strictConversion = 0, |
157 | lenientConversion |
158 | } ConversionFlags; |
159 | |
160 | ConversionResult ConvertUTF8toUTF16 ( |
161 | const UTF8** sourceStart, const UTF8* sourceEnd, |
162 | UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); |
163 | |
164 | /** |
165 | * Convert a partial UTF8 sequence to UTF32. If the sequence ends in an |
166 | * incomplete code unit sequence, returns \c sourceExhausted. |
167 | */ |
168 | ConversionResult ConvertUTF8toUTF32Partial( |
169 | const UTF8** sourceStart, const UTF8* sourceEnd, |
170 | UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); |
171 | |
172 | /** |
173 | * Convert a partial UTF8 sequence to UTF32. If the sequence ends in an |
174 | * incomplete code unit sequence, returns \c sourceIllegal. |
175 | */ |
176 | ConversionResult ConvertUTF8toUTF32( |
177 | const UTF8** sourceStart, const UTF8* sourceEnd, |
178 | UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); |
179 | |
180 | ConversionResult ConvertUTF16toUTF8 ( |
181 | const UTF16** sourceStart, const UTF16* sourceEnd, |
182 | UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); |
183 | |
184 | ConversionResult ConvertUTF32toUTF8 ( |
185 | const UTF32** sourceStart, const UTF32* sourceEnd, |
186 | UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); |
187 | |
188 | ConversionResult ConvertUTF16toUTF32 ( |
189 | const UTF16** sourceStart, const UTF16* sourceEnd, |
190 | UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); |
191 | |
192 | ConversionResult ConvertUTF32toUTF16 ( |
193 | const UTF32** sourceStart, const UTF32* sourceEnd, |
194 | UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); |
195 | |
196 | Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); |
197 | |
198 | Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd); |
199 | |
200 | unsigned getUTF8SequenceSize(const UTF8 *source, const UTF8 *sourceEnd); |
201 | |
202 | unsigned getNumBytesForUTF8(UTF8 firstByte); |
203 | |
204 | /*************************************************************************/ |
205 | /* Below are LLVM-specific wrappers of the functions above. */ |
206 | |
207 | template <typename T> class ArrayRef; |
208 | template <typename T> class SmallVectorImpl; |
209 | class StringRef; |
210 | |
211 | /** |
212 | * Convert an UTF8 StringRef to UTF8, UTF16, or UTF32 depending on |
213 | * WideCharWidth. The converted data is written to ResultPtr, which needs to |
214 | * point to at least WideCharWidth * (Source.Size() + 1) bytes. On success, |
215 | * ResultPtr will point one after the end of the copied string. On failure, |
216 | * ResultPtr will not be changed, and ErrorPtr will be set to the location of |
217 | * the first character which could not be converted. |
218 | * \return true on success. |
219 | */ |
220 | bool ConvertUTF8toWide(unsigned WideCharWidth, llvm::StringRef Source, |
221 | char *&ResultPtr, const UTF8 *&ErrorPtr); |
222 | |
223 | /** |
224 | * Converts a UTF-8 StringRef to a std::wstring. |
225 | * \return true on success. |
226 | */ |
227 | bool ConvertUTF8toWide(llvm::StringRef Source, std::wstring &Result); |
228 | |
229 | /** |
230 | * Converts a UTF-8 C-string to a std::wstring. |
231 | * \return true on success. |
232 | */ |
233 | bool ConvertUTF8toWide(const char *Source, std::wstring &Result); |
234 | |
235 | /** |
236 | * Converts a std::wstring to a UTF-8 encoded std::string. |
237 | * \return true on success. |
238 | */ |
239 | bool convertWideToUTF8(const std::wstring &Source, std::string &Result); |
240 | |
241 | |
242 | /** |
243 | * Convert an Unicode code point to UTF8 sequence. |
244 | * |
245 | * \param Source a Unicode code point. |
246 | * \param [in,out] ResultPtr pointer to the output buffer, needs to be at least |
247 | * \c UNI_MAX_UTF8_BYTES_PER_CODE_POINT bytes. On success \c ResultPtr is |
248 | * updated one past end of the converted sequence. |
249 | * |
250 | * \returns true on success. |
251 | */ |
252 | bool ConvertCodePointToUTF8(unsigned Source, char *&ResultPtr); |
253 | |
254 | /** |
255 | * Convert the first UTF8 sequence in the given source buffer to a UTF32 |
256 | * code point. |
257 | * |
258 | * \param [in,out] source A pointer to the source buffer. If the conversion |
259 | * succeeds, this pointer will be updated to point to the byte just past the |
260 | * end of the converted sequence. |
261 | * \param sourceEnd A pointer just past the end of the source buffer. |
262 | * \param [out] target The converted code |
263 | * \param flags Whether the conversion is strict or lenient. |
264 | * |
265 | * \returns conversionOK on success |
266 | * |
267 | * \sa ConvertUTF8toUTF32 |
268 | */ |
269 | inline ConversionResult convertUTF8Sequence(const UTF8 **source, |
270 | const UTF8 *sourceEnd, |
271 | UTF32 *target, |
272 | ConversionFlags flags) { |
273 | if (*source == sourceEnd) |
274 | return sourceExhausted; |
275 | unsigned size = getNumBytesForUTF8(firstByte: **source); |
276 | if ((ptrdiff_t)size > sourceEnd - *source) |
277 | return sourceExhausted; |
278 | return ConvertUTF8toUTF32(sourceStart: source, sourceEnd: *source + size, targetStart: &target, targetEnd: target + 1, flags); |
279 | } |
280 | |
281 | /** |
282 | * Returns true if a blob of text starts with a UTF-16 big or little endian byte |
283 | * order mark. |
284 | */ |
285 | bool hasUTF16ByteOrderMark(ArrayRef<char> SrcBytes); |
286 | |
287 | /** |
288 | * Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string. |
289 | * |
290 | * \param [in] SrcBytes A buffer of what is assumed to be UTF-16 encoded text. |
291 | * \param [out] Out Converted UTF-8 is stored here on success. |
292 | * \returns true on success |
293 | */ |
294 | bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out); |
295 | |
296 | /** |
297 | * Converts a UTF16 string into a UTF8 std::string. |
298 | * |
299 | * \param [in] Src A buffer of UTF-16 encoded text. |
300 | * \param [out] Out Converted UTF-8 is stored here on success. |
301 | * \returns true on success |
302 | */ |
303 | bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out); |
304 | |
305 | /** |
306 | * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string. |
307 | * |
308 | * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text. |
309 | * \param [out] Out Converted UTF-8 is stored here on success. |
310 | * \returns true on success |
311 | */ |
312 | bool convertUTF32ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out); |
313 | |
314 | /** |
315 | * Converts a UTF32 string into a UTF8 std::string. |
316 | * |
317 | * \param [in] Src A buffer of UTF-32 encoded text. |
318 | * \param [out] Out Converted UTF-8 is stored here on success. |
319 | * \returns true on success |
320 | */ |
321 | bool convertUTF32ToUTF8String(ArrayRef<UTF32> Src, std::string &Out); |
322 | |
323 | /** |
324 | * Converts a UTF-8 string into a UTF-16 string with native endianness. |
325 | * |
326 | * \returns true on success |
327 | */ |
328 | bool convertUTF8ToUTF16String(StringRef SrcUTF8, |
329 | SmallVectorImpl<UTF16> &DstUTF16); |
330 | |
331 | #if defined(_WIN32) |
332 | namespace sys { |
333 | namespace windows { |
334 | std::error_code UTF8ToUTF16(StringRef utf8, SmallVectorImpl<wchar_t> &utf16); |
335 | /// Convert to UTF16 from the current code page used in the system |
336 | std::error_code CurCPToUTF16(StringRef utf8, SmallVectorImpl<wchar_t> &utf16); |
337 | std::error_code UTF16ToUTF8(const wchar_t *utf16, size_t utf16_len, |
338 | SmallVectorImpl<char> &utf8); |
339 | /// Convert from UTF16 to the current code page used in the system |
340 | std::error_code UTF16ToCurCP(const wchar_t *utf16, size_t utf16_len, |
341 | SmallVectorImpl<char> &utf8); |
342 | } // namespace windows |
343 | } // namespace sys |
344 | #endif |
345 | |
346 | } /* end namespace llvm */ |
347 | |
348 | #endif |
349 | |