1	//===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "clang/Format/Format.h"
10
11	#include "FormatTestUtils.h"
12	#include "TestLexer.h"
13	#include "gtest/gtest.h"
14
15	namespace clang {
16	namespace format {
17
18	// Not really the equality, but everything we need.
19	static bool operator==(const FormatToken &LHS,
20	const FormatToken &RHS) noexcept {
21	return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22	LHS.getType() == RHS.getType();
23	}
24
25	namespace {
26
27	class TokenAnnotatorTest : public ::testing::Test {
28	protected:
29	TokenList annotate(llvm::StringRef Code,
30	const FormatStyle &Style = getLLVMStyle()) {
31	return TestLexer(Allocator, Buffers, Style).annotate(Code);
32	}
33	llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34	std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35	};
36
37	#define EXPECT_TOKEN_KIND(FormatTok, Kind) \
38	EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39	#define EXPECT_TOKEN_TYPE(FormatTok, Type) \
40	EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41	#define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec) \
42	EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43	#define EXPECT_BRACE_KIND(FormatTok, Kind) \
44	EXPECT_EQ(FormatTok->getBlockKind(), Kind) << *(FormatTok)
45	#define EXPECT_TOKEN(FormatTok, Kind, Type) \
46	do { \
47	EXPECT_TOKEN_KIND(FormatTok, Kind); \
48	EXPECT_TOKEN_TYPE(FormatTok, Type); \
49	} while (false)
50
51	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
52	auto Tokens = annotate(Code: "auto x = [](const decltype(x) &ptr) {};");
53	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
54	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
55	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
56	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
57	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
58	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
59
60	Tokens = annotate(Code: "auto x = [](const decltype(x) *ptr) {};");
61	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
62	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
63	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
64
65	Tokens = annotate(Code: "#define lambda [](const decltype(x) &ptr) {}");
66	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
67	EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
68	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
69	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
70	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
71	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
72
73	Tokens = annotate(Code: "#define lambda [](const decltype(x) *ptr) {}");
74	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
75	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
76	EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
77
78	Tokens = annotate(Code: "void f() {\n"
79	" while (p < a && *p == 'a')\n"
80	" p++;\n"
81	"}");
82	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
83	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
84	EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
85
86	Tokens = annotate(Code: "case *x:");
87	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
88	EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
89	Tokens = annotate(Code: "case &x:");
90	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
91	EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
92
93	Tokens = annotate(Code: "bool b = 3 == int{3} && true;");
94	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
95	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
96
97	Tokens = annotate(Code: "struct {\n"
98	"} *ptr;");
99	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
100	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
101	Tokens = annotate(Code: "union {\n"
102	"} *ptr;");
103	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
104	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
105	Tokens = annotate(Code: "class {\n"
106	"} *ptr;");
107	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
108	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
109
110	Tokens = annotate(Code: "struct {\n"
111	"} &&ptr = {};");
112	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
113	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
114	Tokens = annotate(Code: "union {\n"
115	"} &&ptr = {};");
116	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
117	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
118	Tokens = annotate(Code: "class {\n"
119	"} &&ptr = {};");
120	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
121	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
122	Tokens = annotate(Code: "int i = int{42} * 2;");
123	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
124	EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
125
126	Tokens = annotate(Code: "delete[] *ptr;");
127	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
128	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
129	Tokens = annotate(Code: "delete[] **ptr;");
130	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
131	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
132	EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
133	Tokens = annotate(Code: "delete[] *(ptr);");
134	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
135	EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
136
137	Tokens = annotate(Code: "void f() { void (*fnptr)(char* foo); }");
138	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
139	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
140	// FIXME: The star of a function pointer probably makes more sense as
141	// TT_PointerOrReference.
142	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
143	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
144
145	Tokens = annotate(Code: "void f() { void (*fnptr)(t* foo); }");
146	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
147	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
148	EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
149	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
150
151	Tokens = annotate(Code: "int f3() { return sizeof(Foo&); }");
152	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
153	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
154
155	Tokens = annotate(Code: "int f4() { return sizeof(Foo&&); }");
156	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
157	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
158
159	Tokens = annotate(Code: "void f5() { int f6(Foo&, Bar&); }");
160	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
161	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
162	EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
163
164	Tokens = annotate(Code: "void f7() { int f8(Foo&&, Bar&&); }");
165	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
166	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
167	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
168
169	Tokens = annotate(Code: "Type1 &val1 = val2;");
170	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
171	EXPECT_TOKEN(Tokens[1], tok::amp, TT_PointerOrReference);
172
173	Tokens = annotate(Code: "Type1 *val1 = &val2;");
174	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
175	EXPECT_TOKEN(Tokens[1], tok::star, TT_PointerOrReference);
176	EXPECT_TOKEN(Tokens[4], tok::amp, TT_UnaryOperator);
177
178	Tokens = annotate(Code: "val1 & val2;");
179	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
180	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
181
182	Tokens = annotate(Code: "val1 & val2.member;");
183	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
184	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
185
186	Tokens = annotate(Code: "val1 & val2.*member;");
187	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
188	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
189
190	Tokens = annotate(Code: "val1.*member & val2;");
191	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
192	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
193
194	Tokens = annotate(Code: "val1 & val2->*member;");
195	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
196	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
197
198	Tokens = annotate(Code: "val1->member & val2;");
199	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
200	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
201
202	Tokens = annotate(Code: "val1 & val2 & val3;");
203	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
204	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
205	EXPECT_TOKEN(Tokens[3], tok::amp, TT_BinaryOperator);
206
207	Tokens = annotate(Code: "val1 & val2 // comment\n"
208	" & val3;");
209	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
210	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
211	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
212
213	Tokens =
214	annotate(Code: "val1 & val2.member & val3.member() & val4 & val5->member;");
215	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
216	EXPECT_TOKEN(Tokens[1], tok::amp, TT_BinaryOperator);
217	EXPECT_TOKEN(Tokens[5], tok::amp, TT_BinaryOperator);
218	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
219	EXPECT_TOKEN(Tokens[13], tok::amp, TT_BinaryOperator);
220
221	Tokens = annotate(Code: "class c {\n"
222	" void func(type &a) { a & member; }\n"
223	" anotherType &member;\n"
224	"}");
225	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
226	EXPECT_TOKEN(Tokens[7], tok::amp, TT_PointerOrReference);
227	EXPECT_TOKEN(Tokens[12], tok::amp, TT_BinaryOperator);
228	EXPECT_TOKEN(Tokens[17], tok::amp, TT_PointerOrReference);
229
230	Tokens = annotate(Code: "struct S {\n"
231	" auto Mem = C & D;\n"
232	"}");
233	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
234	EXPECT_TOKEN(Tokens[7], tok::amp, TT_BinaryOperator);
235
236	Tokens =
237	annotate(Code: "template <typename T> void swap() noexcept(Bar<T> && Foo<T>);");
238	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
239	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
240
241	Tokens = annotate(Code: "template <typename T> struct S {\n"
242	" explicit(Bar<T> && Foo<T>) S(const S &);\n"
243	"};");
244	ASSERT_EQ(Tokens.size(), 30u) << Tokens;
245	EXPECT_TOKEN(Tokens[14], tok::ampamp, TT_BinaryOperator);
246
247	Tokens = annotate(Code: "template <bool B = C && D> struct S {};");
248	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
249	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_BinaryOperator);
250
251	Tokens = annotate(Code: "template <typename T, bool B = C && D> struct S {};");
252	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
253	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
254
255	Tokens = annotate(Code: "template <typename T, typename U = T&&> struct S {};");
256	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
257	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
258
259	Tokens = annotate(Code: "template <typename T = int (*)(int)> struct S {};");
260	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
261	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
262	EXPECT_TOKEN(Tokens[7], tok::star, TT_PointerOrReference);
263
264	Tokens = annotate(Code: "Foo<A && B> a = {};");
265	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
266	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
267
268	Tokens = annotate(Code: "Foo<A &&> a = {};");
269	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
270	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
271
272	Tokens = annotate(Code: "template <enable_if_t<foo && !bar>* = nullptr> void f();");
273	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
274	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_BinaryOperator);
275
276	Tokens =
277	annotate(Code: "auto foo() noexcept(noexcept(bar()) && "
278	"trait<std::decay_t<decltype(bar())>> && noexcept(baz())) {}");
279	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
280	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
281	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
282
283	Tokens = annotate(Code: "foo = *i < *j && *j > *k;");
284	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
285	EXPECT_TOKEN(Tokens[4], tok::less, TT_BinaryOperator);
286	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
287	EXPECT_TOKEN(Tokens[10], tok::greater, TT_BinaryOperator);
288
289	FormatStyle Style = getLLVMStyle();
290	Style.TypeNames.push_back(x: "MYI");
291	Tokens = annotate(Code: "if (MYI *p{nullptr})", Style);
292	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
293	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
294	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
295
296	Style.TypeNames.push_back(x: "Class");
297	Tokens = annotate(Code: "if (Class *obj {getObj()})", Style);
298	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
299	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
300	EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
301
302	Tokens = annotate(Code: "class Foo {\n"
303	" void operator<() {}\n"
304	" Foo &f;\n"
305	"};");
306	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
307	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
308	EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);
309	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
310	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
311	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
312
313	Tokens = annotate(Code: "if (new && num) {\n"
314	" new = 1;\n"
315	"}\n"
316	"if (!delete && num) {\n"
317	" delete = 1;\n"
318	"}");
319	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
320	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_BinaryOperator);
321	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
322	}
323
324	TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
325	auto Tokens = annotate(Code: "x - 0");
326	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
327	EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
328	Tokens = annotate(Code: "0 + 0");
329	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
330	EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
331	Tokens = annotate(Code: "x + +0");
332	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
333	EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
334	Tokens = annotate(Code: "x ? -0 : +0");
335	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
336	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
337	EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
338	Tokens = annotate(Code: "(-0)");
339	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
340	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
341	Tokens = annotate(Code: "0, -0");
342	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
343	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
344	Tokens = annotate(Code: "for (; -1;) {\n}");
345	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
346	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
347	Tokens = annotate(Code: "x = -1;");
348	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
349	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
350	Tokens = annotate(Code: "x[-1]");
351	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
352	EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
353	Tokens = annotate(Code: "x = {-1};");
354	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
355	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
356	Tokens = annotate(Code: "case -x:");
357	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
358	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
359	Tokens = annotate(Code: "co_await -x;");
360	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
361	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
362	Tokens = annotate(Code: "co_return -x;");
363	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
364	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
365	Tokens = annotate(Code: "co_yield -x;");
366	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
367	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
368	Tokens = annotate(Code: "delete -x;");
369	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
370	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
371	Tokens = annotate(Code: "return -x;");
372	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
373	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
374	Tokens = annotate(Code: "throw -x;");
375	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
376	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
377	Tokens = annotate(Code: "sizeof -x");
378	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
379	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
380	Tokens = annotate(Code: "co_await +x;");
381	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
382	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
383	Tokens = annotate(Code: "co_return +x;");
384	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
385	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
386	Tokens = annotate(Code: "co_yield +x;");
387	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
388	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
389	Tokens = annotate(Code: "delete +x;");
390	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
391	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
392	Tokens = annotate(Code: "return +x;");
393	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
394	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
395	Tokens = annotate(Code: "throw +x;");
396	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
397	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
398	Tokens = annotate(Code: "sizeof +x");
399	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
400	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
401	Tokens = annotate(Code: "(int)-x");
402	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
403	EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
404	Tokens = annotate(Code: "(-x)");
405	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
406	EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
407	Tokens = annotate(Code: "!+x");
408	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
409	EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
410	EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
411	}
412
413	TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
414	auto Tokens = annotate(Code: "class C {};");
415	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
416	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
417	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
418
419	Tokens = annotate(Code: "const class C {} c;");
420	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
421	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
422	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_ClassRBrace);
423
424	Tokens = annotate(Code: "const class {} c;");
425	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
426	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
427	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_ClassRBrace);
428
429	Tokens = annotate(Code: "class [[deprecated(\"\")]] C { int i; };");
430	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
431	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
432	EXPECT_TOKEN(Tokens[14], tok::r_brace, TT_ClassRBrace);
433	}
434
435	TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
436	auto Tokens = annotate(Code: "struct S {};");
437	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
438	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
439	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_StructRBrace);
440
441	Tokens = annotate(Code: "struct EXPORT_MACRO [[nodiscard]] C { int i; };");
442	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
443	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
444	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_StructRBrace);
445
446	Tokens = annotate(Code: "struct [[deprecated]] [[nodiscard]] C { int i; };");
447	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
448	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
449	EXPECT_TOKEN(Tokens[16], tok::r_brace, TT_StructRBrace);
450
451	Tokens = annotate(Code: "template <typename T> struct S<const T[N]> {};");
452	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
453	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
454	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
455	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
456	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
457	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
458
459	Tokens = annotate(Code: "template <typename T> struct S<T const[N]> {};");
460	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
461	EXPECT_TOKEN(Tokens[7], tok::less, TT_TemplateOpener);
462	EXPECT_TOKEN(Tokens[10], tok::l_square, TT_ArraySubscriptLSquare);
463	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
464	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_StructLBrace);
465	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_StructRBrace);
466
467	Tokens = annotate(Code: "template <typename T, unsigned n> struct S<T const[n]> {\n"
468	" void f(T const (&a)[n]);\n"
469	"};");
470	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
471	EXPECT_TOKEN(Tokens[10], tok::less, TT_TemplateOpener);
472	EXPECT_TOKEN(Tokens[13], tok::l_square, TT_ArraySubscriptLSquare);
473	EXPECT_TOKEN(Tokens[16], tok::greater, TT_TemplateCloser);
474	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_StructLBrace);
475	EXPECT_TOKEN(Tokens[23], tok::l_paren, TT_FunctionTypeLParen);
476	EXPECT_TOKEN(Tokens[24], tok::amp, TT_UnaryOperator);
477	EXPECT_TOKEN(Tokens[27], tok::l_square, TT_ArraySubscriptLSquare);
478	EXPECT_TOKEN(Tokens[32], tok::r_brace, TT_StructRBrace);
479	}
480
481	TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
482	auto Tokens = annotate(Code: "union U {};");
483	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
484	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
485	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_UnionRBrace);
486
487	Tokens = annotate(Code: "union U { void f() { return; } };");
488	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
489	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
490	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
491	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_UnionRBrace);
492	}
493
494	TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
495	auto Tokens = annotate(Code: "enum E {};");
496	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
497	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
498	EXPECT_TOKEN(Tokens[3], tok::r_brace, TT_EnumRBrace);
499	}
500
501	TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
502	auto Tokens = annotate(Code: "auto operator<=>(const T &) const & = default;");
503	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
504	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
505
506	Tokens = annotate(Code: "template <typename T> void F(T) && = delete;");
507	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
508	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
509	}
510
511	TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
512	auto Tokens =
513	annotate(Code: "inline bool var = is_integral_v<int> && is_signed_v<int>;");
514	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
515	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
516	}
517
518	TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
519	auto Tokens =
520	annotate(Code: "template <typename T> "
521	"inline bool var = is_integral_v<int> && is_signed_v<int>;");
522	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
523	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
524	}
525
526	TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
527	auto Tokens =
528	annotate(Code: "#define FOO(typeName) \\\n"
529	" { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
530	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
531	EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
532	EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
533	EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
534	EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
535	}
536
537	TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
538	auto Tokens = annotate(Code: "if (std::tuple_size_v<T> > 0)");
539	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
540	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
541	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
542	EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
543	}
544
545	TEST_F(TokenAnnotatorTest, UnderstandsNonTemplateAngleBrackets) {
546	auto Tokens = annotate(Code: "return a < b && c > d;");
547	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
548	EXPECT_TOKEN(Tokens[2], tok::less, TT_BinaryOperator);
549	EXPECT_TOKEN(Tokens[6], tok::greater, TT_BinaryOperator);
550
551	Tokens = annotate(Code: "a < 0 ? b : a > 0 ? c : d;");
552	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
553	EXPECT_TOKEN(Tokens[1], tok::less, TT_BinaryOperator);
554	EXPECT_TOKEN(Tokens[7], tok::greater, TT_BinaryOperator);
555
556	Tokens = annotate(Code: "ratio{-1, 2} < ratio{-1, 3} == -1 / 3 > -1 / 2;");
557	ASSERT_EQ(Tokens.size(), 27u) << Tokens;
558	EXPECT_TOKEN(Tokens[7], tok::less, TT_BinaryOperator);
559	EXPECT_TOKEN(Tokens[20], tok::greater, TT_BinaryOperator);
560	}
561
562	TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
563	FormatStyle Style = getLLVMStyle();
564	Style.WhitespaceSensitiveMacros.push_back(x: "FOO");
565
566	auto Tokens = annotate(Code: "FOO(1+2 )", Style);
567	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
568	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
569
570	Tokens = annotate(Code: "FOO(a:b:c)", Style);
571	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
572	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
573	}
574
575	TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
576	auto Tokens = annotate(Code: "delete (void *)p;");
577	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
578	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
579
580	Tokens = annotate(Code: "delete[] (void *)p;");
581	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
582	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
583
584	Tokens = annotate(Code: "delete[] /*comment*/ (void *)p;");
585	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
586	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
587
588	Tokens = annotate(Code: "delete[/*comment*/] (void *)p;");
589	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
590	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
591
592	Tokens = annotate(Code: "delete/*comment*/[] (void *)p;");
593	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
594	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
595	}
596
597	TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
598	auto Tokens = annotate(Code: "(void)p;");
599	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
600	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
601
602	Tokens = annotate(Code: "(uint32_t)&&label;");
603	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
604	EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
605	EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_UnaryOperator);
606	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
607
608	Tokens = annotate(Code: "auto x = (Foo)p;");
609	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
610	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
611
612	Tokens = annotate(Code: "(std::vector<int>)p;");
613	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
614	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
615
616	Tokens = annotate(Code: "return (Foo)p;");
617	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
618	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
619
620	Tokens = annotate(Code: "throw (Foo)p;");
621	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
622	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
623
624	Tokens = annotate(Code: "#define FOO(x) (((uint64_t)(x) * BAR) / 100)");
625	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
626	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
627	EXPECT_TOKEN(Tokens[13], tok::r_paren, TT_Unknown);
628	EXPECT_TOKEN(Tokens[14], tok::star, TT_BinaryOperator);
629
630	Tokens = annotate(Code: "#define foo(i) ((i) - bar)");
631	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
632	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_Unknown);
633	EXPECT_TOKEN(Tokens[10], tok::minus, TT_BinaryOperator);
634
635	Tokens = annotate(Code: "return (Foo) & 10;");
636	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
637	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_Unknown);
638	EXPECT_TOKEN(Tokens[4], tok::amp, TT_BinaryOperator);
639
640	Tokens = annotate(Code: "#define FOO(bar) foo((uint64_t)&bar)");
641	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
642	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
643	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
644
645	Tokens = annotate(Code: "#define FOO(bar) foo((Foo) & bar)");
646	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
647	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_Unknown);
648	EXPECT_TOKEN(Tokens[11], tok::amp, TT_BinaryOperator);
649
650	auto Style = getLLVMStyle();
651	Style.TypeNames.push_back(x: "Foo");
652	Tokens = annotate(Code: "#define FOO(bar) foo((Foo)&bar)", Style);
653	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
654	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_TypeName);
655	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_CastRParen);
656	EXPECT_TOKEN(Tokens[11], tok::amp, TT_UnaryOperator);
657	}
658
659	TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
660	auto Tokens = annotate(Code: "void f() throw(int);");
661	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
662	EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
663	}
664
665	TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
666	auto Tokens = annotate(Code: "void f() &;");
667	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
668	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
669
670	Tokens = annotate(Code: "void operator=(T) &&;");
671	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
672	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
673
674	Tokens = annotate(Code: "template <typename T> void f() &;");
675	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
676	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
677
678	Tokens = annotate(Code: "template <typename T> void operator=(T) &;");
679	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
680	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
681	}
682
683	TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
684	auto Tokens = annotate(Code: "x.operator+()");
685	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
686	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
687	EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
688	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
689	Tokens = annotate(Code: "x.operator=()");
690	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
691	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
692	EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
693	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
694	Tokens = annotate(Code: "x.operator+=()");
695	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
696	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
697	EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
698	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
699	Tokens = annotate(Code: "x.operator,()");
700	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
701	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
702	EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
703	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
704	Tokens = annotate(Code: "x.operator()()");
705	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
706	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
707	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
708	EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
709	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
710	Tokens = annotate(Code: "x.operator[]()");
711	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
712	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
713	// EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
714	// EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
715	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
716	Tokens = annotate(Code: "x.operator\"\"_a()");
717	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
718	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
719	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
720	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
721	Tokens = annotate(Code: "x.operator\"\" _a()");
722	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
723	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
724	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
725	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
726	Tokens = annotate(Code: "x.operator\"\"if()");
727	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
728	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
729	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
730	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
731	Tokens = annotate(Code: "x.operator\"\"s()");
732	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
733	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
734	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
735	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
736	Tokens = annotate(Code: "x.operator\"\" s()");
737	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
738	EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_Unknown);
739	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
740	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
741
742	Tokens = annotate(Code: "int operator+(int);");
743	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
744	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
745	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
746	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
747	Tokens = annotate(Code: "auto operator=(T&) {}");
748	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
749	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
750	EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
751	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
752	Tokens = annotate(Code: "auto operator()() {}");
753	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
754	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
755	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
756	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
757	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
758
759	Tokens = annotate(Code: "class Foo {\n"
760	" int operator+(a* b);\n"
761	"}");
762	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
763	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
764	EXPECT_TOKEN(Tokens[5], tok::plus, TT_OverloadedOperator);
765	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
766	EXPECT_TOKEN(Tokens[8], tok::star, TT_PointerOrReference);
767
768	Tokens = annotate(Code: "class Foo {\n"
769	" int c = operator+(a * b);\n"
770	"}");
771	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
772	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_Unknown);
773	EXPECT_TOKEN(Tokens[7], tok::plus, TT_OverloadedOperator);
774	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_OverloadedOperatorLParen);
775	EXPECT_TOKEN(Tokens[10], tok::star, TT_BinaryOperator);
776
777	Tokens = annotate(Code: "void foo() {\n"
778	" operator+(a * b);\n"
779	"}");
780	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
781	EXPECT_TOKEN(Tokens[5], tok::kw_operator, TT_Unknown);
782	EXPECT_TOKEN(Tokens[6], tok::plus, TT_OverloadedOperator);
783	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
784	EXPECT_TOKEN(Tokens[9], tok::star, TT_BinaryOperator);
785
786	Tokens = annotate(Code: "return operator+(a * b, c & d) + operator+(a && b && c);");
787	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
788	EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_Unknown);
789	EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
790	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
791	EXPECT_TOKEN(Tokens[5], tok::star, TT_BinaryOperator);
792	EXPECT_TOKEN(Tokens[9], tok::amp, TT_BinaryOperator);
793	EXPECT_TOKEN(Tokens[13], tok::kw_operator, TT_Unknown);
794	EXPECT_TOKEN(Tokens[14], tok::plus, TT_OverloadedOperator);
795	EXPECT_TOKEN(Tokens[15], tok::l_paren, TT_OverloadedOperatorLParen);
796	EXPECT_TOKEN(Tokens[17], tok::ampamp, TT_BinaryOperator);
797	EXPECT_TOKEN(Tokens[19], tok::ampamp, TT_BinaryOperator);
798
799	Tokens = annotate(Code: "class Foo {\n"
800	" void foo() {\n"
801	" operator+(a * b);\n"
802	" }\n"
803	"}");
804	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
805	EXPECT_TOKEN(Tokens[8], tok::kw_operator, TT_Unknown);
806	EXPECT_TOKEN(Tokens[9], tok::plus, TT_OverloadedOperator);
807	EXPECT_TOKEN(Tokens[10], tok::l_paren, TT_OverloadedOperatorLParen);
808	EXPECT_TOKEN(Tokens[12], tok::star, TT_BinaryOperator);
809
810	Tokens = annotate(Code: "std::vector<Foo> operator()(Foo &foo);");
811	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
812	EXPECT_TOKEN(Tokens[3], tok::less, TT_TemplateOpener);
813	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
814	EXPECT_TOKEN(Tokens[6], tok::kw_operator, TT_FunctionDeclarationName);
815	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperator);
816	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_OverloadedOperator);
817	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_OverloadedOperatorLParen);
818	EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
819
820	Tokens = annotate(Code: "decltype(auto) operator()(T &x);");
821	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
822	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_TypeDeclarationParen);
823	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_TypeDeclarationParen);
824	EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
825	EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperator);
826	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_OverloadedOperator);
827	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_OverloadedOperatorLParen);
828	EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
829	}
830
831	TEST_F(TokenAnnotatorTest, OverloadedOperatorInTemplate) {
832	struct {
833	const char *Text;
834	tok::TokenKind Kind;
835	} Operators[] = {{.Text: "+", .Kind: tok::plus},
836	{.Text: "-", .Kind: tok::minus},
837	// FIXME:
838	// {"*", tok::star},
839	{.Text: "/", .Kind: tok::slash},
840	{.Text: "%", .Kind: tok::percent},
841	{.Text: "^", .Kind: tok::caret},
842	// FIXME:
843	// {"&", tok::amp},
844	{.Text: "|", .Kind: tok::pipe},
845	{.Text: "~", .Kind: tok::tilde},
846	{.Text: "!", .Kind: tok::exclaim},
847	{.Text: "=", .Kind: tok::equal},
848	// FIXME:
849	// {"<", tok::less},
850	{.Text: ">", .Kind: tok::greater},
851	{.Text: "+=", .Kind: tok::plusequal},
852	{.Text: "-=", .Kind: tok::minusequal},
853	{.Text: "*=", .Kind: tok::starequal},
854	{.Text: "/=", .Kind: tok::slashequal},
855	{.Text: "%=", .Kind: tok::percentequal},
856	{.Text: "^=", .Kind: tok::caretequal},
857	{.Text: "&=", .Kind: tok::ampequal},
858	{.Text: "|=", .Kind: tok::pipeequal},
859	{.Text: "<<", .Kind: tok::lessless},
860	{.Text: ">>", .Kind: tok::greatergreater},
861	{.Text: ">>=", .Kind: tok::greatergreaterequal},
862	{.Text: "<<=", .Kind: tok::lesslessequal},
863	{.Text: "==", .Kind: tok::equalequal},
864	{.Text: "!=", .Kind: tok::exclaimequal},
865	{.Text: "<=", .Kind: tok::lessequal},
866	{.Text: ">=", .Kind: tok::greaterequal},
867	{.Text: "<=>", .Kind: tok::spaceship},
868	{.Text: "&&", .Kind: tok::ampamp},
869	{.Text: "||", .Kind: tok::pipepipe},
870	{.Text: "++", .Kind: tok::plusplus},
871	{.Text: "--", .Kind: tok::minusminus},
872	{.Text: ",", .Kind: tok::comma},
873	{.Text: "->*", .Kind: tok::arrowstar},
874	{.Text: "->", .Kind: tok::arrow}};
875
876	for (const auto &Operator : Operators) {
877	std::string Input("C<&operator");
878	Input += Operator.Text;
879	Input += " > a;";
880	auto Tokens = annotate(Code: std::string(Input));
881	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
882	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
883	EXPECT_TOKEN(Tokens[4], Operator.Kind, TT_OverloadedOperator);
884	EXPECT_TOKEN(Tokens[5], tok::greater, TT_TemplateCloser);
885	}
886
887	auto Tokens = annotate(Code: "C<&operator< <X>> lt;");
888	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
889	EXPECT_TOKEN(Tokens[1], tok::less, TT_TemplateOpener);
890	EXPECT_TOKEN(Tokens[4], tok::less, TT_OverloadedOperator);
891	EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
892	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
893	EXPECT_TOKEN(Tokens[8], tok::greater, TT_TemplateCloser);
894	}
895
896	TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
897	auto Tokens = annotate(Code: "template <typename T>\n"
898	"concept C = (Foo && Bar) && (Bar && Baz);");
899
900	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
901	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
902	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
903	EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
904
905	Tokens = annotate(Code: "template <typename T>\n"
906	"concept C = Foo && !Bar;");
907
908	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
909	EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
910	EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
911
912	Tokens = annotate(Code: "template <typename T>\n"
913	"concept C = requires(T t) {\n"
914	" { t.foo() };\n"
915	"} && Bar<T> && Baz<T>;");
916	ASSERT_EQ(Tokens.size(), 35u) << Tokens;
917	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
918	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
919	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
920	EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
921	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
922
923	Tokens = annotate(Code: "template<typename T>\n"
924	"requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
925	"struct Foo;");
926	ASSERT_EQ(Tokens.size(), 36u) << Tokens;
927	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
928	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
929	EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
930	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
931	EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
932	EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
933	EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
934	EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
935	EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
936	EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
937
938	Tokens =
939	annotate(Code: "template<typename T>\n"
940	"requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
941	"struct Foo;");
942	ASSERT_EQ(Tokens.size(), 38u) << Tokens;
943	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
944	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
945	EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
946	EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
947	EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
948	EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
949	EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
950	EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
951	EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
952	EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
953	EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
954
955	Tokens = annotate(Code: "template <typename T>\n"
956	"void foo(T) noexcept requires Bar<T>;");
957	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
958	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
959
960	Tokens = annotate(Code: "template <typename T>\n"
961	"requires Bar<T> || Baz<T>\n"
962	"auto foo(T) -> int;");
963	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
964	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
965	EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
966	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
967	EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
968
969	Tokens = annotate(Code: "template <typename T>\n"
970	"requires Bar<T>\n"
971	"bool foo(T) { return false; }");
972	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
973	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
974	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
975	EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
976
977	Tokens = annotate(Code: "template <typename T>\n"
978	"requires Bar<T>\n"
979	"decltype(auto) foo(T) { return false; }");
980	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
981	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
982	EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
983	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
984
985	Tokens = annotate(Code: "template <typename T>\n"
986	"struct S {\n"
987	" void foo() const requires Bar<T>;\n"
988	" void bar() const & requires Baz<T>;\n"
989	" void bar() && requires Baz2<T>;\n"
990	" void baz() const & noexcept requires Baz<T>;\n"
991	" void baz() && noexcept requires Baz2<T>;\n"
992	"};\n"
993	"\n"
994	"void S::bar() const & requires Baz<T> { }");
995	ASSERT_EQ(Tokens.size(), 85u) << Tokens;
996	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
997	EXPECT_TOKEN(Tokens[24], tok::amp, TT_PointerOrReference);
998	EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
999	EXPECT_TOKEN(Tokens[35], tok::ampamp, TT_PointerOrReference);
1000	EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
1001	EXPECT_TOKEN(Tokens[47], tok::amp, TT_PointerOrReference);
1002	EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
1003	EXPECT_TOKEN(Tokens[59], tok::ampamp, TT_PointerOrReference);
1004	EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
1005	EXPECT_TOKEN(Tokens[76], tok::amp, TT_PointerOrReference);
1006	EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
1007
1008	Tokens = annotate(Code: "void Class::member() && requires(Constant) {}");
1009	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1010	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1011
1012	Tokens = annotate(Code: "void Class::member() && requires(Constant<T>) {}");
1013	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1014	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1015
1016	Tokens =
1017	annotate(Code: "void Class::member() && requires(Namespace::Constant<T>) {}");
1018	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1019	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1020
1021	Tokens = annotate(Code: "void Class::member() && requires(typename "
1022	"Namespace::Outer<T>::Inner::Constant) {}");
1023	ASSERT_EQ(Tokens.size(), 24u) << Tokens;
1024	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1025
1026	Tokens = annotate(Code: "struct [[nodiscard]] zero_t {\n"
1027	" template<class T>\n"
1028	" requires requires { number_zero_v<T>; }\n"
1029	" [[nodiscard]] constexpr operator T() const { "
1030	"return number_zero_v<T>; }\n"
1031	"};");
1032	ASSERT_EQ(Tokens.size(), 44u) << Tokens;
1033	EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
1034	EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
1035	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
1036	EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
1037	EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
1038	EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
1039
1040	Tokens =
1041	annotate(Code: "template <class A, class B> concept C ="
1042	"std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
1043	ASSERT_EQ(Tokens.size(), 31u) << Tokens;
1044	EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
1045	EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
1046	EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
1047	EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
1048	EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
1049	EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
1050	EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
1051
1052	Tokens = annotate(Code: "auto bar() -> int requires(is_integral_v<T>) {}");
1053	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1054	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1055
1056	Tokens = annotate(Code: "auto bar() -> void requires(is_integral_v<T>) {}");
1057	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1058	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1059
1060	Tokens = annotate(Code: "auto bar() -> MyType requires(is_integral_v<T>) {}");
1061	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1062	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1063
1064	Tokens =
1065	annotate(Code: "auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
1066	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1067	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1068
1069	Tokens =
1070	annotate(Code: "auto bar() -> qualified::type requires(is_integral_v<T>) {}");
1071	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1072	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
1073
1074	Tokens =
1075	annotate(Code: "auto bar() -> Template<type> requires(is_integral_v<T>) {}");
1076	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1077	EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
1078
1079	Tokens = annotate(Code: "void foo() requires((A<T>) && C) {}");
1080	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1081	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1082	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_BinaryOperator);
1083
1084	Tokens = annotate(Code: "void foo() requires(((A<T>) && C)) {}");
1085	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1086	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1087	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
1088
1089	Tokens = annotate(Code: "void foo() requires([](T&&){}(t)) {}");
1090	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1091	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1092	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1093
1094	Tokens = annotate(Code: "void foo() requires([](T&& u){}(t)) {}");
1095	ASSERT_EQ(Tokens.size(), 22u) << Tokens;
1096	EXPECT_TOKEN(Tokens[4], tok::kw_requires, TT_RequiresClause);
1097	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1098
1099	Tokens = annotate(Code: "void f() & requires(true) {}");
1100	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1101	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1102	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1103
1104	Tokens = annotate(Code: "void f() & requires(C<true, true>) {}");
1105	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1106	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1107	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1108
1109	Tokens = annotate(Code: "template <typename T>\n"
1110	"concept C = (!Foo<T>) && Bar;");
1111	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1112	EXPECT_TOKEN(Tokens[15], tok::ampamp, TT_BinaryOperator);
1113
1114	Tokens = annotate(Code: "void f() & requires(C<decltype(x)>) {}");
1115	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1116	EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
1117	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
1118
1119	Tokens = annotate(Code: "auto f() -> int& requires(C<decltype(x)>) {}");
1120	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1121	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1122	EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
1123
1124	Tokens = annotate(Code: "bool x = t && requires(decltype(t) x) { x.foo(); };");
1125	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1126	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1127
1128	Tokens = annotate(Code: "bool x = t && requires(Foo<decltype(t)> x) { x.foo(); };");
1129	ASSERT_EQ(Tokens.size(), 26u) << Tokens;
1130	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1131
1132	Tokens = annotate(Code: "bool x = t && requires(Foo<C1 || C2> x) { x.foo(); };");
1133	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1134	EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresExpression);
1135	}
1136
1137	TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
1138	auto Tokens = annotate(Code: "bool b = requires(int i) { i + 5; };");
1139	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1140	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1141	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1142	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1143
1144	Tokens = annotate(Code: "if (requires(int i) { i + 5; }) return;");
1145	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1146	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1147	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1148	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
1149
1150	Tokens = annotate(Code: "if (func() && requires(int i) { i + 5; }) return;");
1151	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1152	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
1153	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
1154	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
1155
1156	Tokens = annotate(Code: "foo(requires(const T t) {});");
1157	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1158	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1159	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1160	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1161
1162	Tokens = annotate(Code: "foo(requires(const int t) {});");
1163	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1164	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1165	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1166	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1167
1168	Tokens = annotate(Code: "foo(requires(const T t) {});");
1169	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1170	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1171	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1172	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1173
1174	Tokens = annotate(Code: "foo(requires(int const* volatile t) {});");
1175	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1176	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1177	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1178	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1179	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1180
1181	Tokens = annotate(Code: "foo(requires(T const* volatile t) {});");
1182	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1183	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1184	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1185	EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
1186	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
1187
1188	Tokens = annotate(Code: "foo(requires(T& t) {});");
1189	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1190	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1191	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1192	EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
1193	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1194
1195	Tokens = annotate(Code: "foo(requires(T&& t) {});");
1196	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1197	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1198	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1199	EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
1200	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
1201
1202	Tokens = annotate(Code: "bool foo = requires(T& t) {};");
1203	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1204	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1205	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1206	EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
1207	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1208
1209	Tokens = annotate(Code: "bool foo = requires(T&& t) {};");
1210	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1211	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1212	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1213	EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
1214	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
1215
1216	Tokens =
1217	annotate(Code: "foo(requires(const typename Outer<T>::Inner * const t) {});");
1218	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1219	EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
1220	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
1221	EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
1222	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
1223
1224	Tokens = annotate(Code: "template <typename T>\n"
1225	"concept C = requires(T T) {\n"
1226	" requires Bar<T> && Foo<T>;\n"
1227	"};");
1228	ASSERT_EQ(Tokens.size(), 28u) << Tokens;
1229	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1230	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1231	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1232	EXPECT_TOKEN(Tokens[14], tok::kw_requires,
1233	TT_RequiresClauseInARequiresExpression);
1234
1235	Tokens = annotate(Code: "template <typename T>\n"
1236	"concept C = requires(T T) {\n"
1237	" { t.func() } -> std::same_as<int>;"
1238	" requires Bar<T> && Foo<T>;\n"
1239	"};");
1240	ASSERT_EQ(Tokens.size(), 43u) << Tokens;
1241	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
1242	EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
1243	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1244	EXPECT_TOKEN(Tokens[29], tok::kw_requires,
1245	TT_RequiresClauseInARequiresExpression);
1246
1247	// Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
1248	Tokens = annotate(Code: "bool r10 = requires (struct new_struct { int x; } s) { "
1249	"requires true; };");
1250	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1251	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1252	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1253	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
1254
1255	Tokens = annotate(Code: "bool foo = requires(C<true, true> c) {\n"
1256	" { c.foo(); }\n"
1257	"};");
1258	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1259	EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
1260	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
1261	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
1262	}
1263
1264	TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
1265	// Everything after #pragma region should be ImplicitStringLiteral
1266	auto Tokens = annotate(Code: "#pragma region Foo(Bar: Hello)");
1267	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1268	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
1269	EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
1270	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
1271
1272	// Make sure it's annotated correctly inside a function as well
1273	Tokens = annotate(Code: "void test(){\n#pragma region Foo(Bar: Hello)\n}");
1274	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1275	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
1276	EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
1277	EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
1278	}
1279
1280	TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
1281	const char *BaseCode = nullptr;
1282	const char *ConstrainedCode = nullptr;
1283	auto BaseTokenCount = 0u;
1284	auto RequiresTokenCount = 0u;
1285	auto PrefixTokenCount = 0u;
1286
1287	auto TestRequires = [&](int Line) {
1288	const auto BaseTokens = annotate(Code: BaseCode);
1289	const auto ConstrainedTokens = annotate(Code: ConstrainedCode);
1290
1291	#define LINE " (Line " << Line << ')'
1292
1293	ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
1294	ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
1295	<< LINE;
1296
1297	for (auto I = 0u; I < BaseTokenCount; ++I) {
1298	EXPECT_EQ(
1299	*BaseTokens[I],
1300	*ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
1301	<< I << LINE;
1302	}
1303
1304	#undef LINE
1305	};
1306
1307	BaseCode = "template<typename T>\n"
1308	"T Pi = 3.14;";
1309	ConstrainedCode = "template<typename T>\n"
1310	" requires Foo<T>\n"
1311	"T Pi = 3.14;";
1312	BaseTokenCount = 11;
1313	RequiresTokenCount = 5;
1314	PrefixTokenCount = 5;
1315	TestRequires(__LINE__);
1316
1317	BaseCode = "template<typename T>\n"
1318	"struct Bar;";
1319	ConstrainedCode = "template<typename T>\n"
1320	" requires Foo<T>\n"
1321	"struct Bar;";
1322	BaseTokenCount = 9;
1323	TestRequires(__LINE__);
1324
1325	BaseCode = "template<typename T>\n"
1326	"struct Bar {\n"
1327	" T foo();\n"
1328	" T bar();\n"
1329	"};";
1330	ConstrainedCode = "template<typename T>\n"
1331	" requires Foo<T>\n"
1332	"struct Bar {\n"
1333	" T foo();\n"
1334	" T bar();\n"
1335	"};";
1336	BaseTokenCount = 21;
1337	TestRequires(__LINE__);
1338
1339	BaseCode = "template<typename T>\n"
1340	"Bar(T) -> Bar<T>;";
1341	ConstrainedCode = "template<typename T>\n"
1342	" requires Foo<T>\n"
1343	"Bar(T) -> Bar<T>;";
1344	BaseTokenCount = 16;
1345	TestRequires(__LINE__);
1346
1347	BaseCode = "template<typename T>\n"
1348	"T foo();";
1349	ConstrainedCode = "template<typename T>\n"
1350	" requires Foo<T>\n"
1351	"T foo();";
1352	BaseTokenCount = 11;
1353	TestRequires(__LINE__);
1354
1355	BaseCode = "template<typename T>\n"
1356	"T foo() {\n"
1357	" auto bar = baz();\n"
1358	" return bar + T{};\n"
1359	"}";
1360	ConstrainedCode = "template<typename T>\n"
1361	" requires Foo<T>\n"
1362	"T foo() {\n"
1363	" auto bar = baz();\n"
1364	" return bar + T{};\n"
1365	"}";
1366	BaseTokenCount = 26;
1367	TestRequires(__LINE__);
1368
1369	BaseCode = "template<typename T>\n"
1370	"T foo();";
1371	ConstrainedCode = "template<typename T>\n"
1372	"T foo() requires Foo<T>;";
1373	BaseTokenCount = 11;
1374	PrefixTokenCount = 9;
1375	TestRequires(__LINE__);
1376
1377	BaseCode = "template<typename T>\n"
1378	"T foo() {\n"
1379	" auto bar = baz();\n"
1380	" return bar + T{};\n"
1381	"}";
1382	ConstrainedCode = "template<typename T>\n"
1383	"T foo() requires Foo<T> {\n"
1384	" auto bar = baz();\n"
1385	" return bar + T{};\n"
1386	"}";
1387	BaseTokenCount = 26;
1388	TestRequires(__LINE__);
1389
1390	BaseCode = "template<typename T>\n"
1391	"T foo();";
1392	ConstrainedCode = "template<typename T>\n"
1393	" requires(Foo<T>)\n"
1394	"T foo();";
1395	BaseTokenCount = 11;
1396	RequiresTokenCount = 7;
1397	PrefixTokenCount = 5;
1398	TestRequires(__LINE__);
1399
1400	BaseCode = "template<typename T>\n"
1401	"Bar(T) -> Bar<typename T::I>;";
1402	ConstrainedCode = "template<typename T>\n"
1403	" requires requires(T &&t) {\n"
1404	" typename T::I;\n"
1405	" }\n"
1406	"Bar(T) -> Bar<typename T::I>;";
1407	BaseTokenCount = 19;
1408	RequiresTokenCount = 14;
1409	PrefixTokenCount = 5;
1410	TestRequires(__LINE__);
1411
1412	BaseCode = "struct [[nodiscard]] zero_t {\n"
1413	" template<class T>\n"
1414	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1415	"};";
1416	ConstrainedCode =
1417	"struct [[nodiscard]] zero_t {\n"
1418	" template<class T>\n"
1419	" requires requires { v<T>; }\n"
1420	" [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
1421	"};";
1422	BaseTokenCount = 35;
1423	RequiresTokenCount = 9;
1424	PrefixTokenCount = 13;
1425	TestRequires(__LINE__);
1426
1427	BaseCode = "constexpr Foo(Foo const &other)\n"
1428	" : value{other.value} {\n"
1429	" do_magic();\n"
1430	" do_more_magic();\n"
1431	"}";
1432	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1433	" requires std::is_copy_constructible<T>\n"
1434	" : value{other.value} {\n"
1435	" do_magic();\n"
1436	" do_more_magic();\n"
1437	"}";
1438	BaseTokenCount = 26;
1439	RequiresTokenCount = 7;
1440	PrefixTokenCount = 8;
1441	TestRequires(__LINE__);
1442
1443	BaseCode = "constexpr Foo(Foo const &other)\n"
1444	" : value{other.value} {\n"
1445	" do_magic();\n"
1446	" do_more_magic();\n"
1447	"}";
1448	ConstrainedCode = "constexpr Foo(Foo const &other)\n"
1449	" requires (std::is_copy_constructible<T>)\n"
1450	" : value{other.value} {\n"
1451	" do_magic();\n"
1452	" do_more_magic();\n"
1453	"}";
1454	RequiresTokenCount = 9;
1455	TestRequires(__LINE__);
1456
1457	BaseCode = "template<typename T>\n"
1458	"ANNOTATE(\"S\"\n"
1459	" \"S\")\n"
1460	"void foo();";
1461	ConstrainedCode = "template<typename T>\n"
1462	" requires(true)\n"
1463	"ANNOTATE(\"S\"\n"
1464	" \"S\")\n"
1465	"void foo();";
1466	BaseTokenCount = 16;
1467	RequiresTokenCount = 4;
1468	PrefixTokenCount = 5;
1469	TestRequires(__LINE__);
1470	}
1471
1472	TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1473	auto Tokens = annotate(Code: "__asm{\n"
1474	"a:\n"
1475	"};");
1476	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1477	EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1478	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1479	EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace);
1480	}
1481
1482	TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1483	auto Tokens = annotate(Code: "int (^)() = ^ ()\n"
1484	" external_source_symbol() { //\n"
1485	" return 1;\n"
1486	"};");
1487	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1488	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1489	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1490
1491	Tokens = annotate(Code: "int *p = ^int*(){ //\n"
1492	" return nullptr;\n"
1493	"}();");
1494	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1495	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1496	}
1497
1498	TEST_F(TokenAnnotatorTest, UnderstandsObjCMethodExpr) {
1499	auto Tokens = annotate(Code: "void f() {\n"
1500	" //\n"
1501	" BOOL a = [b.c n] > 1;\n"
1502	"}");
1503	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1504	EXPECT_TOKEN(Tokens[9], tok::l_square, TT_ObjCMethodExpr);
1505	EXPECT_TOKEN(Tokens[15], tok::greater, TT_BinaryOperator);
1506	}
1507
1508	TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1509	auto Tokens = annotate(Code: "[]() constexpr {}");
1510	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1511	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1512	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1513
1514	Tokens = annotate(Code: "[]() consteval {}");
1515	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1516	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1517	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1518
1519	Tokens = annotate(Code: "[]() mutable {}");
1520	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1521	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1522	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1523
1524	Tokens = annotate(Code: "[]() static {}");
1525	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1526	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1527	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1528
1529	Tokens = annotate(Code: "[]() -> auto {}");
1530	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1531	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1532	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1533	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1534
1535	Tokens = annotate(Code: "[]() -> auto & {}");
1536	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1537	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1538	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1539	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1540
1541	Tokens = annotate(Code: "[]() -> auto * {}");
1542	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1543	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1544	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1545	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1546
1547	Tokens = annotate(Code: "[] {}");
1548	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1549	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1550	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1551
1552	Tokens = annotate(Code: "[] noexcept {}");
1553	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1554	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1555	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
1556
1557	Tokens = annotate(Code: "[] -> auto {}");
1558	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1559	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1560	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_TrailingReturnArrow);
1561	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
1562
1563	Tokens = annotate(Code: "[] <typename T> () {}");
1564	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1565	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1566	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1567	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
1568
1569	Tokens = annotate(Code: "[] <typename T> {}");
1570	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1571	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1572	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1573	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1574
1575	Tokens = annotate(Code: "[] <typename... T> () {}");
1576	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1577	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1578	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1579	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1580
1581	Tokens = annotate(Code: "[] <typename... T> {}");
1582	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1583	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1584	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1585	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1586
1587	Tokens = annotate(Code: "[] <int... T> () {}");
1588	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1589	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1590	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1591	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1592
1593	Tokens = annotate(Code: "[] <int... T> {}");
1594	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1595	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1596	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1597	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1598
1599	Tokens = annotate(Code: "[] <Foo... T> () {}");
1600	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1601	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1602	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1603	EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1604
1605	Tokens = annotate(Code: "[] <Foo... T> {}");
1606	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1607	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1608	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1609	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1610
1611	// Lambdas with a requires-clause
1612	Tokens = annotate(Code: "[] <typename T> (T t) requires Bar<T> {}");
1613	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1614	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1615	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1616	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1617	EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
1618	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1619
1620	Tokens = annotate(Code: "[] <typename T> (T &&t) requires Bar<T> {}");
1621	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1622	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1623	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1624	EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_PointerOrReference);
1625	EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
1626	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1627	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1628
1629	Tokens = annotate(Code: "[] <typename T> (T t) requires Foo<T> || Bar<T> {}");
1630	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1631	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1632	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1633	EXPECT_TOKEN(Tokens[10], tok::kw_requires, TT_RequiresClause);
1634	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
1635	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1636
1637	Tokens = annotate(Code: "[] <typename T> (T t) -> T requires Bar<T> {}");
1638	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1639	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1640	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1641	EXPECT_TOKEN(Tokens[10], tok::arrow, TT_TrailingReturnArrow);
1642	EXPECT_TOKEN(Tokens[12], tok::kw_requires, TT_RequiresClause);
1643	EXPECT_TRUE(Tokens[16]->ClosesRequiresClause);
1644	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1645
1646	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T t) {}");
1647	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1648	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1649	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1650	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1651	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1652	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1653
1654	Tokens = annotate(Code: "[] <typename T> requires Bar<T> (T &&t) {}");
1655	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1656	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1657	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1658	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1659	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1660	EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_PointerOrReference);
1661	EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_LambdaLBrace);
1662
1663	Tokens = annotate(Code: "[] <typename T> requires Foo<T> || Bar<T> (T t) {}");
1664	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1665	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1666	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1667	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1668	EXPECT_TRUE(Tokens[15]->ClosesRequiresClause);
1669	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1670
1671	Tokens = annotate(Code: "[] <typename T> requires true (T&& t) {}");
1672	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1673	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1674	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1675	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1676	EXPECT_TRUE(Tokens[7]->ClosesRequiresClause);
1677	EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
1678	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
1679
1680	Tokens = annotate(Code: "[] <typename T> requires Bar<T> {}");
1681	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1682	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1683	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1684	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1685	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1686	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_LambdaLBrace);
1687
1688	Tokens = annotate(Code: "[] <typename T> requires Bar<T> noexcept {}");
1689	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1690	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1691	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1692	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1693	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1694	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1695
1696	Tokens = annotate(Code: "[] <typename T> requires Bar<T> -> T {}");
1697	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1698	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1699	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1700	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1701	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1702	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_TrailingReturnArrow);
1703	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_LambdaLBrace);
1704
1705	Tokens = annotate(Code: "[] <typename T> requires Foo<T> (T t) requires Bar<T> {}");
1706	ASSERT_EQ(Tokens.size(), 23u) << Tokens;
1707	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1708	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1709	EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
1710	EXPECT_TRUE(Tokens[10]->ClosesRequiresClause);
1711	EXPECT_TOKEN(Tokens[15], tok::kw_requires, TT_RequiresClause);
1712	EXPECT_TRUE(Tokens[19]->ClosesRequiresClause);
1713	EXPECT_TOKEN(Tokens[20], tok::l_brace, TT_LambdaLBrace);
1714
1715	Tokens = annotate(Code: "[] <typename T = int> (T t) {}");
1716	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1717	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1718	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1719	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1720	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1721
1722	Tokens = annotate(Code: "[] <int I = 0> (T t) {}");
1723	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1724	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1725	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1726	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1727	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1728
1729	Tokens = annotate(Code: "[] <bool b = false> (T t) {}");
1730	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1731	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1732	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1733	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1734	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_LambdaLBrace);
1735
1736	Tokens = annotate(Code: "[] <bool b = true && false> (T&& t) {}");
1737	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1738	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1739	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1740	EXPECT_TOKEN(Tokens[7], tok::ampamp, TT_BinaryOperator);
1741	EXPECT_TOKEN(Tokens[9], tok::greater, TT_TemplateCloser);
1742	EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
1743	EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_LambdaLBrace);
1744
1745	Tokens = annotate(Code: "[] <typename T = int> requires Foo<T> (T t) {}");
1746	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1747	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1748	EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1749	EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
1750	EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
1751	EXPECT_TOKEN(Tokens[17], tok::l_brace, TT_LambdaLBrace);
1752	}
1753
1754	TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
1755	auto Tokens = annotate(Code: "template <typename T>\n"
1756	"DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
1757	"string OldFunction(const string &parameter) {}");
1758	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1759	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
1760
1761	Tokens = annotate(Code: "template <typename T>\n"
1762	"A(T) noexcept;");
1763	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1764	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1765	}
1766
1767	TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
1768	auto Tokens = annotate(Code: "void f [[noreturn]] ();");
1769	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1770	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1771
1772	Tokens = annotate(Code: "void f [[noreturn]] () {}");
1773	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1774	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1775
1776	Tokens = annotate(Code: "#define FOO Foo::\n"
1777	"FOO Foo();");
1778	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1779	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_FunctionDeclarationName);
1780
1781	Tokens = annotate(Code: "struct Foo {\n"
1782	" Bar (*func)();\n"
1783	"};");
1784	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1785	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
1786	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_FunctionTypeLParen);
1787
1788	Tokens = annotate(Code: "void instanceof();");
1789	ASSERT_EQ(Tokens.size(), 6u);
1790	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1791
1792	Tokens = annotate(Code: "int iso_time(time_t);");
1793	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1794	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1795
1796	auto Style = getLLVMStyle();
1797	Style.TypeNames.push_back(x: "MyType");
1798	Tokens = annotate(Code: "int iso_time(MyType);", Style);
1799	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1800	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1801	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_TypeName);
1802	}
1803
1804	TEST_F(TokenAnnotatorTest, UnderstandsCtorAndDtorDeclNames) {
1805	auto Tokens = annotate(Code: "class Foo { public: Foo(); };");
1806	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1807	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_CtorDtorDeclName);
1808
1809	Tokens = annotate(Code: "class Foo { public: ~Foo(); };");
1810	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1811	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_CtorDtorDeclName);
1812
1813	Tokens = annotate(Code: "struct Foo { [[deprecated]] Foo() {} };");
1814	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1815	EXPECT_TOKEN(Tokens[8], tok::identifier, TT_CtorDtorDeclName);
1816	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
1817
1818	Tokens = annotate(Code: "struct Foo { [[deprecated]] ~Foo() {} };");
1819	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1820	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
1821	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
1822
1823	Tokens = annotate(Code: "struct Foo { Foo() [[deprecated]] {} };");
1824	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1825	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
1826	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_FunctionLBrace);
1827
1828	Tokens = annotate(Code: "struct Foo { ~Foo() [[deprecated]] {} };");
1829	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1830	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_CtorDtorDeclName);
1831	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
1832
1833	Tokens = annotate(Code: "struct Foo { [[deprecated]] explicit Foo() {} };");
1834	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
1835	EXPECT_TOKEN(Tokens[9], tok::identifier, TT_CtorDtorDeclName);
1836	EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_FunctionLBrace);
1837
1838	Tokens = annotate(Code: "struct Foo { virtual [[deprecated]] ~Foo() {} };");
1839	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1840	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
1841	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
1842
1843	Tokens = annotate(Code: "Foo::Foo() {}");
1844	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1845	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_CtorDtorDeclName);
1846	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
1847
1848	Tokens = annotate(Code: "Foo::~Foo() {}");
1849	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1850	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
1851	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
1852
1853	Tokens = annotate(Code: "struct Test {\n"
1854	" Test()\n"
1855	" : l([] {\n"
1856	" Short::foo();\n"
1857	" }) {}\n"
1858	"};");
1859	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
1860	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
1861	EXPECT_TOKEN(Tokens[14], tok::identifier, TT_Unknown);
1862	}
1863
1864	TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
1865	auto Tokens = annotate(Code: "_Generic(x, int: 1, default: 0)");
1866	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1867	EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
1868	EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
1869	EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
1870	}
1871
1872	TEST_F(TokenAnnotatorTest, UnderstandsTrailingReturnArrow) {
1873	auto Tokens = annotate(Code: "auto f() -> int;");
1874	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1875	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1876
1877	Tokens = annotate(Code: "auto operator->() -> int;");
1878	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1879	EXPECT_TOKEN(Tokens[2], tok::arrow, TT_OverloadedOperator);
1880	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
1881
1882	Tokens = annotate(Code: "auto operator++(int) -> int;");
1883	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1884	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
1885
1886	Tokens = annotate(Code: "auto operator=() -> int;");
1887	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1888	EXPECT_TOKEN(Tokens[5], tok::arrow, TT_TrailingReturnArrow);
1889
1890	Tokens = annotate(Code: "auto operator=(int) -> int;");
1891	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1892	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_TrailingReturnArrow);
1893
1894	Tokens = annotate(Code: "auto foo() -> auto { return Val; }");
1895	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1896	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1897
1898	Tokens = annotate(Code: "struct S { auto bar() const -> int; };");
1899	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1900	EXPECT_TOKEN(Tokens[8], tok::arrow, TT_TrailingReturnArrow);
1901
1902	// Not trailing return arrows
1903	Tokens = annotate(Code: "auto a = b->c;");
1904	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1905	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
1906
1907	Tokens = annotate(Code: "auto a = (b)->c;");
1908	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1909	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
1910
1911	Tokens = annotate(Code: "auto a = b()->c;");
1912	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1913	EXPECT_TOKEN(Tokens[6], tok::arrow, TT_Unknown);
1914
1915	Tokens = annotate(Code: "auto a = b->c();");
1916	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1917	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_Unknown);
1918
1919	Tokens = annotate(Code: "decltype(auto) a = b()->c;");
1920	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1921	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
1922
1923	Tokens = annotate(Code: "void f() { auto a = b->c(); }");
1924	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1925	EXPECT_TOKEN(Tokens[9], tok::arrow, TT_Unknown);
1926
1927	Tokens = annotate(Code: "void f() { auto a = b()->c; }");
1928	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1929	EXPECT_TOKEN(Tokens[11], tok::arrow, TT_Unknown);
1930
1931	Tokens = annotate(Code: "#define P(ptr) auto p = (ptr)->p");
1932	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
1933	EXPECT_TOKEN(Tokens[12], tok::arrow, TT_Unknown);
1934
1935	Tokens = annotate(Code: "void f() FOO(foo->bar);");
1936	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1937	EXPECT_TOKEN(Tokens[7], tok::arrow, TT_Unknown);
1938
1939	Tokens = annotate(Code: "__attribute__((cold)) C() : Base(obj->func()) {}");
1940	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1941	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
1942
1943	// Mixed
1944	Tokens = annotate(Code: "auto f() -> int { auto a = b()->c; }");
1945	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
1946	EXPECT_TOKEN(Tokens[4], tok::arrow, TT_TrailingReturnArrow);
1947	EXPECT_TOKEN(Tokens[13], tok::arrow, TT_Unknown);
1948	}
1949
1950	TEST_F(TokenAnnotatorTest, UnderstandHashInMacro) {
1951	auto Tokens = annotate(Code: "#define Foo(Bar) \\\n"
1952	" { \\\n"
1953	" #Bar \\\n"
1954	" }");
1955	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1956	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
1957	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
1958
1959	Tokens = annotate(Code: "#define Foo(Bar) \\\n"
1960	" { #Bar }");
1961	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1962	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
1963	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
1964
1965	Tokens = annotate(Code: "#define FOO(typeName, realClass) \\\n"
1966	" {#typeName, foo<Foo>(new foo<realClass>(#typeName))}");
1967	ASSERT_EQ(Tokens.size(), 29u) << Tokens;
1968	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
1969	EXPECT_BRACE_KIND(Tokens[27], BK_BracedInit);
1970	}
1971
1972	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacros) {
1973	// '__attribute__' has special handling.
1974	auto Tokens = annotate(Code: "__attribute__(X) void Foo(void);");
1975	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1976	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
1977	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
1978	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
1979
1980	// Generic macro has no special handling in this location.
1981	Tokens = annotate(Code: "A(X) void Foo(void);");
1982	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1983	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
1984	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_Unknown);
1985
1986	// Add a custom AttributeMacro. Test that it has the same behavior.
1987	FormatStyle Style = getLLVMStyle();
1988	Style.AttributeMacros.push_back(x: "A");
1989
1990	// An "AttributeMacro" gets annotated like '__attribute__'.
1991	Tokens = annotate(Code: "A(X) void Foo(void);", Style);
1992	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1993	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
1994	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
1995	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
1996	}
1997
1998	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCDecl) {
1999	// '__attribute__' has special handling.
2000	auto Tokens = annotate(Code: "__attribute__(X) @interface Foo");
2001	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2002	EXPECT_TOKEN(Tokens[0], tok::kw___attribute, TT_Unknown);
2003	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2004	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2005
2006	// Generic macro has no special handling in this location.
2007	Tokens = annotate(Code: "A(X) @interface Foo");
2008	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2009	// Note: Don't check token-type as a random token in this position is hard to
2010	// reason about.
2011	EXPECT_TOKEN_KIND(Tokens[0], tok::identifier);
2012	EXPECT_TOKEN_KIND(Tokens[1], tok::l_paren);
2013
2014	// Add a custom AttributeMacro. Test that it has the same behavior.
2015	FormatStyle Style = getLLVMStyle();
2016	Style.AttributeMacros.push_back(x: "A");
2017
2018	// An "AttributeMacro" gets annotated like '__attribute__'.
2019	Tokens = annotate(Code: "A(X) @interface Foo", Style);
2020	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2021	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_AttributeMacro);
2022	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_AttributeLParen);
2023	EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_AttributeRParen);
2024	}
2025
2026	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCMethodDecl) {
2027	// '__attribute__' has special handling.
2028	auto Tokens = annotate(Code: "- (id)init __attribute__(X);");
2029	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2030	EXPECT_TOKEN(Tokens[5], tok::kw___attribute, TT_Unknown);
2031	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2032	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
2033
2034	// Generic macro has no special handling in this location.
2035	Tokens = annotate(Code: "- (id)init A(X);");
2036	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2037	// Note: Don't check token-type as a random token in this position is hard to
2038	// reason about.
2039	EXPECT_TOKEN_KIND(Tokens[5], tok::identifier);
2040	EXPECT_TOKEN_KIND(Tokens[6], tok::l_paren);
2041
2042	// Add a custom AttributeMacro. Test that it has the same behavior.
2043	FormatStyle Style = getLLVMStyle();
2044	Style.AttributeMacros.push_back(x: "A");
2045
2046	// An "AttributeMacro" gets annotated like '__attribute__'.
2047	Tokens = annotate(Code: "- (id)init A(X);", Style);
2048	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2049	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_AttributeMacro);
2050	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_AttributeLParen);
2051	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_AttributeRParen);
2052	}
2053
2054	TEST_F(TokenAnnotatorTest, UnderstandsAttributeMacrosOnObjCProperty) {
2055	// '__attribute__' has special handling.
2056	auto Tokens = annotate(Code: "@property(weak) id delegate __attribute__(X);");
2057	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2058	EXPECT_TOKEN(Tokens[7], tok::kw___attribute, TT_Unknown);
2059	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2060	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2061
2062	// Generic macro has no special handling in this location.
2063	Tokens = annotate(Code: "@property(weak) id delegate A(X);");
2064	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2065	// Note: Don't check token-type as a random token in this position is hard to
2066	// reason about.
2067	EXPECT_TOKEN_KIND(Tokens[7], tok::identifier);
2068	EXPECT_TOKEN_KIND(Tokens[8], tok::l_paren);
2069
2070	// Add a custom AttributeMacro. Test that it has the same behavior.
2071	FormatStyle Style = getLLVMStyle();
2072	Style.AttributeMacros.push_back(x: "A");
2073
2074	// An "AttributeMacro" gets annotated like '__attribute__'.
2075	Tokens = annotate(Code: "@property(weak) id delegate A(X);", Style);
2076	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2077	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_AttributeMacro);
2078	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_AttributeLParen);
2079	EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_AttributeRParen);
2080	}
2081
2082	TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
2083	auto Annotate = [this](llvm::StringRef Code) {
2084	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_Verilog));
2085	};
2086	// Test that unary operators get labeled as such and that operators like '++'
2087	// don't get split.
2088	tok::TokenKind Unary[] = {tok::plus, tok::minus, tok::exclaim,
2089	tok::tilde, tok::amp, tok::pipe,
2090	tok::caret, tok::plusplus, tok::minusminus};
2091	for (auto Kind : Unary) {
2092	auto Tokens =
2093	Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
2094	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2095	EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
2096	}
2097	// Operators formed by joining two operators like '^~'. For some of these
2098	// joined operators, we don't have a separate type, so we only test for their
2099	// precedence.
2100	std::pair<prec::Level, std::string> JoinedBinary[] = {
2101	{prec::Comma, "->"}, {prec::Comma, "<->"},
2102	{prec::Assignment, "+="}, {prec::Assignment, "-="},
2103	{prec::Assignment, "*="}, {prec::Assignment, "/="},
2104	{prec::Assignment, "%="}, {prec::Assignment, "&="},
2105	{prec::Assignment, "^="}, {prec::Assignment, "<<="},
2106	{prec::Assignment, ">>="}, {prec::Assignment, "<<<="},
2107	{prec::Assignment, ">>>="}, {prec::LogicalOr, "||"},
2108	{prec::LogicalAnd, "&&"}, {prec::Equality, "=="},
2109	{prec::Equality, "!="}, {prec::Equality, "==="},
2110	{prec::Equality, "!=="}, {prec::Equality, "==?"},
2111	{prec::Equality, "!=?"}, {prec::ExclusiveOr, "~^"},
2112	{prec::ExclusiveOr, "^~"},
2113	};
2114	for (auto Operator : JoinedBinary) {
2115	auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
2116	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2117	EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
2118	EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
2119	}
2120	// '~^' and '^~' can be unary as well as binary operators.
2121	auto Tokens = Annotate("x = ~^x;");
2122	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2123	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2124	Tokens = Annotate("x = ^~x;");
2125	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2126	EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
2127	// The unary operators '~&' and '~|' can only be unary operators. The current
2128	// implementation treats each of them as separate unary '~' and '&' or '|'
2129	// operators, which is enough for formatting purposes. In FormatTestVerilog,
2130	// there is a test that there is no space in between. And even if a new line
2131	// is inserted between the '~' and '|', the semantic meaning is the same as
2132	// the joined operator, so the CanBreakBefore property doesn't need to be
2133	// false for the second operator.
2134	Tokens = Annotate("x = ~&x;");
2135	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2136	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2137	EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
2138	Tokens = Annotate("x = ~|x;");
2139	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2140	EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
2141	EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
2142	// Test for block label colons.
2143	Tokens = Annotate("begin : x\n"
2144	"end : x");
2145	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2146	EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
2147	EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
2148	// Test that the dimension colon is annotated correctly.
2149	Tokens = Annotate("var [1 : 0] x;");
2150	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2151	EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
2152	Tokens = Annotate("extern function [1 : 0] x;");
2153	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2154	EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
2155	Tokens = Annotate("module test\n"
2156	" (input wire [7 : 0] a[7 : 0]);\n"
2157	"endmodule");
2158	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2159	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_VerilogDimensionedTypeName);
2160	EXPECT_TOKEN(Tokens[7], tok::colon, TT_BitFieldColon);
2161	EXPECT_TOKEN(Tokens[13], tok::colon, TT_BitFieldColon);
2162	// Test case labels and ternary operators.
2163	Tokens = Annotate("case (x)\n"
2164	" x:\n"
2165	" x;\n"
2166	"endcase");
2167	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2168	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CaseLabelColon);
2169	Tokens = Annotate("case (x)\n"
2170	" x ? x : x:\n"
2171	" x;\n"
2172	"endcase");
2173	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2174	EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
2175	EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
2176	EXPECT_TOKEN(Tokens[9], tok::colon, TT_CaseLabelColon);
2177	// Non-blocking assignments.
2178	Tokens = Annotate("a <= b;");
2179	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2180	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2181	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2182	Tokens = Annotate("if (a <= b) break;");
2183	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2184	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2185	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2186	Tokens = Annotate("a <= b <= a;");
2187	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2188	EXPECT_TOKEN(Tokens[1], tok::lessequal, TT_BinaryOperator);
2189	EXPECT_TOKEN_PRECEDENCE(Tokens[1], prec::Assignment);
2190	EXPECT_TOKEN(Tokens[3], tok::lessequal, TT_BinaryOperator);
2191	EXPECT_TOKEN_PRECEDENCE(Tokens[3], prec::Relational);
2192
2193	// Port lists in module instantiation.
2194	Tokens = Annotate("module_x instance_1(port_1), instance_2(port_2);");
2195	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2196	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2197	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_VerilogInstancePortLParen);
2198	Tokens = Annotate("module_x #(parameter) instance_1(port_1), "
2199	"instance_2(port_2);");
2200	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2201	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_VerilogInstancePortLParen);
2202	EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_VerilogInstancePortLParen);
2203	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_VerilogInstancePortLParen);
2204
2205	// Condition parentheses.
2206	Tokens = Annotate("assert (x);");
2207	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2208	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2209	Tokens = Annotate("assert #0 (x);");
2210	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2211	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_ConditionLParen);
2212	Tokens = Annotate("assert final (x);");
2213	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2214	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2215	Tokens = Annotate("foreach (x[x]) continue;");
2216	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2217	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2218	Tokens = Annotate("repeat (x[x]) continue;");
2219	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2220	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2221	Tokens = Annotate("case (x) endcase;");
2222	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2223	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2224
2225	// Sensitivity list. The TT_Unknown type is clearly not binding for the
2226	// future, please adapt if those tokens get annotated. This test is only here
2227	// to prevent the comma from being annotated as TT_VerilogInstancePortComma.
2228	Tokens = Annotate("always @(posedge x, posedge y);");
2229	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2230	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_Unknown);
2231	EXPECT_TOKEN(Tokens[5], tok::comma, TT_Unknown);
2232	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2233
2234	// String literals in concatenation.
2235	Tokens = Annotate("x = {\"\"};");
2236	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2237	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2238	Tokens = Annotate("x = {\"\", \"\"};");
2239	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2240	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_StringInConcatenation);
2241	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2242	Tokens = Annotate("x = '{{\"\"}};");
2243	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2244	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_StringInConcatenation);
2245	// Cases where the string should not be annotated that type. Fix the
2246	// `TT_Unknown` if needed in the future.
2247	Tokens = Annotate("x = {\"\" == \"\"};");
2248	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2249	EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_Unknown);
2250	EXPECT_TOKEN(Tokens[5], tok::string_literal, TT_Unknown);
2251	Tokens = Annotate("x = {(\"\")};");
2252	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2253	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2254	Tokens = Annotate("x = '{\"\"};");
2255	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2256	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_Unknown);
2257	}
2258
2259	TEST_F(TokenAnnotatorTest, UnderstandTableGenTokens) {
2260	auto Style = getLLVMStyle(Language: FormatStyle::LK_TableGen);
2261	ASSERT_TRUE(Style.isTableGen());
2262
2263	TestLexer Lexer(Allocator, Buffers, Style);
2264	AdditionalKeywords Keywords(Lexer.IdentTable);
2265	auto Annotate = [&Lexer](llvm::StringRef Code) {
2266	return Lexer.annotate(Code);
2267	};
2268
2269	// Additional keywords representation test.
2270	auto Tokens = Annotate("def foo : Bar<1>;");
2271	ASSERT_TRUE(Keywords.isTableGenKeyword(*Tokens[0]));
2272	ASSERT_TRUE(Keywords.isTableGenDefinition(*Tokens[0]));
2273	ASSERT_TRUE(Tokens[0]->is(Keywords.kw_def));
2274	ASSERT_TRUE(Tokens[1]->is(TT_StartOfName));
2275
2276	// Code, the multiline string token.
2277	Tokens = Annotate("[{ code is multiline string }]");
2278	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2279	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2280	EXPECT_FALSE(Tokens[0]->IsMultiline);
2281	// Case with multiple lines.
2282	Tokens = Annotate("[{ It can break\n"
2283	" across lines and the line breaks\n"
2284	" are retained in \n"
2285	" the string. }]");
2286	ASSERT_EQ(Tokens.size(), 2u) << Tokens;
2287	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2288	EXPECT_EQ(Tokens[0]->ColumnWidth, sizeof("[{ It can break\n") - 1);
2289	EXPECT_TRUE(Tokens[0]->IsMultiline);
2290	EXPECT_EQ(Tokens[0]->LastLineColumnWidth, sizeof(" the string. }]") - 1);
2291
2292	// Numeric literals.
2293	Tokens = Annotate("1234");
2294	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2295	Tokens = Annotate("-1");
2296	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2297	Tokens = Annotate("+1234");
2298	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2299	Tokens = Annotate("0b0110");
2300	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2301	Tokens = Annotate("0x1abC");
2302	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2303
2304	// Identifier tokens. In TableGen, identifiers can begin with a number.
2305	// In ambiguous cases, the lexer tries to lex it as a number.
2306	// Even if the try fails, it does not fall back to identifier lexing and
2307	// regard as an error.
2308	// The ambiguity is not documented. The result of those tests are based on the
2309	// implementation of llvm::TGLexer::LexToken.
2310	// This is invalid syntax of number, but not an identifier.
2311	Tokens = Annotate("0x1234x");
2312	EXPECT_TOKEN(Tokens[0], tok::numeric_constant, TT_Unknown);
2313	Tokens = Annotate("identifier");
2314	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2315	// Identifier beginning with a number.
2316	Tokens = Annotate("0x");
2317	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2318	Tokens = Annotate("2dVector");
2319	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2320	Tokens = Annotate("01234Vector");
2321	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_Unknown);
2322
2323	// Structured statements.
2324	Tokens = Annotate("class Foo {}");
2325	EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_FunctionLBrace);
2326	Tokens = Annotate("def Def: Foo {}");
2327	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2328	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2329	Tokens = Annotate("if cond then {} else {}");
2330	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ControlStatementLBrace);
2331	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_ElseLBrace);
2332	Tokens = Annotate("defset Foo Def2 = {}");
2333	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2334
2335	// Bang Operators.
2336	Tokens = Annotate("!foreach");
2337	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2338	Tokens = Annotate("!if");
2339	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenBangOperator);
2340	Tokens = Annotate("!cond");
2341	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2342
2343	auto AnnotateValue = [this, &Style](llvm::StringRef Code) {
2344	// Values are annotated only in specific context.
2345	auto Result = annotate(Code: ("def X { let V = " + Code + "; }").str(), Style);
2346	return decltype(Result){Result.begin() + 6, Result.end() - 3};
2347	};
2348	// Both of bang/cond operators.
2349	Tokens = AnnotateValue("!cond(!eq(x, 0): 1, true: x)");
2350	ASSERT_EQ(Tokens.size(), 15u) << Tokens;
2351	EXPECT_TOKEN(Tokens[0], tok::identifier, TT_TableGenCondOperator);
2352	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TableGenBangOperator);
2353	EXPECT_TOKEN(Tokens[8], tok::colon, TT_TableGenCondOperatorColon);
2354	EXPECT_TOKEN(Tokens[10], tok::comma, TT_TableGenCondOperatorComma);
2355	EXPECT_TOKEN(Tokens[12], tok::colon, TT_TableGenCondOperatorColon);
2356	// DAGArg values with operator identifier
2357	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2358	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2359	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2360	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2361	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown); // $src1
2362	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2363	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2364	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2365	// List literal
2366	Tokens = AnnotateValue("[1, 2, 3]");
2367	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2368	EXPECT_TOKEN(Tokens[0], tok::l_square, TT_TableGenListOpener);
2369	EXPECT_TOKEN(Tokens[6], tok::r_square, TT_TableGenListCloser);
2370	// Suffixes of values
2371	Tokens = AnnotateValue("valid.field");
2372	ASSERT_EQ(Tokens.size(), 3u) << Tokens;
2373	EXPECT_TOKEN(Tokens[1], tok::period, TT_TableGenValueSuffix);
2374	// Code
2375	Tokens = AnnotateValue("[{ code is multiline string }]");
2376	ASSERT_EQ(Tokens.size(), 1u) << Tokens;
2377	EXPECT_TOKEN(Tokens[0], tok::string_literal, TT_TableGenMultiLineString);
2378
2379	// The definition
2380	Tokens = annotate(Code: "def Def : Parent<Child> {}", Style);
2381	ASSERT_EQ(Tokens.size(), 10u) << Tokens; // This contains eof.
2382	// We use inheritance colon and function brace. They are enough.
2383	EXPECT_TOKEN(Tokens[2], tok::colon, TT_InheritanceColon);
2384	EXPECT_TOKEN(Tokens[4], tok::less, TT_TemplateOpener);
2385	EXPECT_TOKEN(Tokens[6], tok::greater, TT_TemplateCloser);
2386	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
2387
2388	// DAGArg breaking options. They use different token types depending on what
2389	// is specified.
2390	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakElements;
2391
2392	// When TableGenBreakInsideDAGArg is DAS_BreakElements and
2393	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
2394	// elements to have line break.
2395	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2396	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2397	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2398	EXPECT_TOKEN(Tokens[1], tok::identifier,
2399	TT_TableGenDAGArgOperatorID); // ins
2400	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2401	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2402
2403	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2404	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2405	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2406	EXPECT_TOKEN(Tokens[1], tok::identifier,
2407	TT_TableGenDAGArgOperatorID); // other
2408	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2409	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2410
2411	// For non-identifier operators, breaks after the operator.
2412	Tokens = AnnotateValue("(!cast<Type>(\"Name\") type1:$src1, type2:$src2)");
2413	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2414	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2415	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_TableGenDAGArgOperatorToBreak);
2416	EXPECT_TOKEN(Tokens[11], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2417	EXPECT_TOKEN(Tokens[15], tok::r_paren, TT_TableGenDAGArgCloser);
2418
2419	Style.TableGenBreakInsideDAGArg = FormatStyle::DAS_BreakAll;
2420
2421	// When TableGenBreakInsideDAGArg is DAS_BreakAll and
2422	// TableGenBreakingDAGArgOperators is not specified, it makes all the DAGArg
2423	// to have line break inside it.
2424	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2425	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2426	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2427	EXPECT_TOKEN(Tokens[1], tok::identifier,
2428	TT_TableGenDAGArgOperatorToBreak); // ins
2429	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2430	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2431
2432	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2433	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2434	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2435	EXPECT_TOKEN(Tokens[1], tok::identifier,
2436	TT_TableGenDAGArgOperatorToBreak); // other
2437	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2438	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2439
2440	// If TableGenBreakingDAGArgOperators is specified, it is limited to the
2441	// specified operators.
2442	Style.TableGenBreakingDAGArgOperators = {"ins", "outs"};
2443	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2444	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2445	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpenerToBreak);
2446	EXPECT_TOKEN(Tokens[1], tok::identifier,
2447	TT_TableGenDAGArgOperatorToBreak); // ins
2448	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListCommaToBreak);
2449	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2450
2451	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2452	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2453	EXPECT_TOKEN(Tokens[0], tok::l_paren, TT_TableGenDAGArgOpener);
2454	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
2455	EXPECT_TOKEN(Tokens[5], tok::comma, TT_TableGenDAGArgListComma);
2456	EXPECT_TOKEN(Tokens[9], tok::r_paren, TT_TableGenDAGArgCloser);
2457
2458	// If TableGenBreakingDAGArgOperators is enabled, it uses
2459	// TT_TableGenDAGArgListColonToAlign to annotate the colon to align.
2460	Style.AlignConsecutiveTableGenBreakingDAGArgColons.Enabled = true;
2461	Tokens = AnnotateValue("(ins type1:$src1, type2:$src2)");
2462	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2463	EXPECT_TOKEN(Tokens[1], tok::identifier,
2464	TT_TableGenDAGArgOperatorToBreak); // ins
2465	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColonToAlign);
2466	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColonToAlign);
2467
2468	Tokens = AnnotateValue("(other type1:$src1, type2:$src2)");
2469	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2470	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_Unknown); // other
2471	EXPECT_TOKEN(Tokens[3], tok::colon, TT_TableGenDAGArgListColon);
2472	EXPECT_TOKEN(Tokens[7], tok::colon, TT_TableGenDAGArgListColon);
2473	}
2474
2475	TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
2476	auto Tokens = annotate(Code: "Class::Class() : BaseClass(), Member() {}");
2477
2478	// The TT_Unknown is clearly not binding for the future, please adapt if those
2479	// tokens get annotated.
2480	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2481	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
2482	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
2483	EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
2484	EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
2485	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
2486	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
2487	EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
2488	EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
2489	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2490	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2491
2492	Tokens = annotate(Code: "Class::Class() : BaseClass{}, Member{} {}");
2493	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2494	EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
2495	EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
2496	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
2497	EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
2498	EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
2499	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
2500	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
2501	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
2502	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2503	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2504
2505	Tokens = annotate(Code: "class Class {\n"
2506	" Class() : BaseClass() {\n"
2507	"#if 0\n"
2508	" // comment\n"
2509	"#endif\n"
2510	" }\n"
2511	" Class f();\n"
2512	"}");
2513	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2514	EXPECT_TOKEN(Tokens[6], tok::colon, TT_CtorInitializerColon);
2515	EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_FunctionLBrace);
2516	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
2517	}
2518
2519	TEST_F(TokenAnnotatorTest, UnderstandsConditionParens) {
2520	auto Tokens = annotate(Code: "if (x) {}");
2521	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2522	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2523	Tokens = annotate(Code: "if constexpr (x) {}");
2524	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2525	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2526	Tokens = annotate(Code: "if CONSTEXPR (x) {}");
2527	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2528	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_ConditionLParen);
2529	Tokens = annotate(Code: "if (x) {} else if (x) {}");
2530	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2531	EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ConditionLParen);
2532	EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_ConditionLParen);
2533	}
2534
2535	TEST_F(TokenAnnotatorTest, CSharpNullableTypes) {
2536	FormatStyle Style = getGoogleStyle(Language: FormatStyle::LK_CSharp);
2537
2538	auto Tokens = annotate(Code: "int? a;", Style);
2539	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2540	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2541
2542	Tokens = annotate(Code: "int? a = 1;", Style);
2543	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2544	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2545
2546	Tokens = annotate(Code: "int?)", Style);
2547	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
2548	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2549
2550	Tokens = annotate(Code: "int?>", Style);
2551	ASSERT_EQ(Tokens.size(), 4u) << Tokens;
2552	EXPECT_TOKEN(Tokens[1], tok::question, TT_CSharpNullable);
2553
2554	Tokens = annotate(Code: "cond? id : id2", Style);
2555	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2556	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
2557
2558	Tokens = annotate(Code: "cond ? cond2 ? : id1 : id2", Style);
2559	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2560	EXPECT_TOKEN(Tokens[1], tok::question, TT_ConditionalExpr);
2561	}
2562
2563	TEST_F(TokenAnnotatorTest, UnderstandsLabels) {
2564	auto Tokens = annotate(Code: "{ x: break; }");
2565	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2566	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
2567	Tokens = annotate(Code: "{ case x: break; }");
2568	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2569	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
2570	Tokens = annotate(Code: "{ x: { break; } }");
2571	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2572	EXPECT_TOKEN(Tokens[2], tok::colon, TT_GotoLabelColon);
2573	Tokens = annotate(Code: "{ case x: { break; } }");
2574	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2575	EXPECT_TOKEN(Tokens[3], tok::colon, TT_CaseLabelColon);
2576	}
2577
2578	TEST_F(TokenAnnotatorTest, UnderstandsNestedBlocks) {
2579	// The closing braces are not annotated. It doesn't seem to cause a problem.
2580	// So we only test for the opening braces.
2581	auto Tokens = annotate(Code: "{\n"
2582	" {\n"
2583	" { int a = 0; }\n"
2584	" }\n"
2585	" {}\n"
2586	"}");
2587	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2588	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
2589	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
2590	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
2591	EXPECT_BRACE_KIND(Tokens[10], BK_Block);
2592	}
2593
2594	TEST_F(TokenAnnotatorTest, UnderstandDesignatedInitializers) {
2595	auto Tokens = annotate(Code: "SomeStruct { .a = 1 };");
2596	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2597	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2598	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
2599
2600	Tokens = annotate(Code: "SomeStruct { .a = 1, .b = 2 };");
2601	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2602	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2603	EXPECT_TOKEN(Tokens[2], tok::period, TT_DesignatedInitializerPeriod);
2604	EXPECT_TOKEN(Tokens[7], tok::period, TT_DesignatedInitializerPeriod);
2605
2606	Tokens = annotate(Code: "SomeStruct {\n"
2607	"#ifdef FOO\n"
2608	" .a = 1,\n"
2609	"#endif\n"
2610	" .b = 2\n"
2611	"};");
2612	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2613	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2614	EXPECT_TOKEN(Tokens[5], tok::period, TT_DesignatedInitializerPeriod);
2615	EXPECT_TOKEN(Tokens[12], tok::period, TT_DesignatedInitializerPeriod);
2616
2617	Tokens = annotate(Code: "SomeStruct {\n"
2618	"#if defined FOO\n"
2619	" .a = 1,\n"
2620	"#endif\n"
2621	" .b = 2\n"
2622	"};");
2623	ASSERT_EQ(Tokens.size(), 20u) << Tokens;
2624	EXPECT_BRACE_KIND(Tokens[1], BK_BracedInit);
2625	EXPECT_TOKEN(Tokens[6], tok::period, TT_DesignatedInitializerPeriod);
2626	EXPECT_TOKEN(Tokens[13], tok::period, TT_DesignatedInitializerPeriod);
2627
2628	Tokens = annotate(Code: "Foo foo[] = {[0]{}};");
2629	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2630	EXPECT_TOKEN(Tokens[6], tok::l_square, TT_DesignatedInitializerLSquare);
2631	EXPECT_BRACE_KIND(Tokens[9], BK_BracedInit);
2632	}
2633
2634	TEST_F(TokenAnnotatorTest, UnderstandsJavaScript) {
2635	auto Annotate = [this](llvm::StringRef Code) {
2636	return annotate(Code, Style: getLLVMStyle(Language: FormatStyle::LK_JavaScript));
2637	};
2638
2639	// Dictionary.
2640	auto Tokens = Annotate("var x = {'x' : 1, 'y' : 2};");
2641	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2642	EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_DictLiteral);
2643	EXPECT_TOKEN(Tokens[4], tok::string_literal, TT_SelectorName);
2644	EXPECT_TOKEN(Tokens[5], tok::colon, TT_DictLiteral);
2645	EXPECT_TOKEN(Tokens[8], tok::string_literal, TT_SelectorName);
2646	EXPECT_TOKEN(Tokens[9], tok::colon, TT_DictLiteral);
2647	// Change when we need to annotate these.
2648	EXPECT_BRACE_KIND(Tokens[3], BK_Unknown);
2649	EXPECT_BRACE_KIND(Tokens[11], BK_Unknown);
2650	EXPECT_TOKEN(Tokens[11], tok::r_brace, TT_Unknown);
2651	}
2652
2653	TEST_F(TokenAnnotatorTest, UnderstandsAttributes) {
2654	auto Tokens = annotate(Code: "bool foo __attribute__((unused));");
2655	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2656	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_StartOfName);
2657	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
2658	EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_Unknown);
2659	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_Unknown);
2660	EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_AttributeRParen);
2661
2662	Tokens = annotate(Code: "bool foo __declspec(dllimport);");
2663	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2664	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
2665	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2666
2667	Tokens = annotate(Code: "bool __attribute__((unused)) foo;");
2668	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2669	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
2670	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
2671	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
2672	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
2673	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_StartOfName);
2674
2675	Tokens = annotate(Code: "void __attribute__((x)) Foo();");
2676	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2677	EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_AttributeLParen);
2678	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_Unknown);
2679	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_Unknown);
2680	EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_AttributeRParen);
2681	EXPECT_TOKEN(Tokens[7], tok::identifier, TT_FunctionDeclarationName);
2682
2683	FormatStyle Style = getLLVMStyle();
2684	Style.AttributeMacros.push_back(x: "FOO");
2685	Tokens = annotate(Code: "bool foo FOO(unused);", Style);
2686	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2687	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_AttributeMacro);
2688	EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_AttributeLParen);
2689	EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_AttributeRParen);
2690	}
2691
2692	TEST_F(TokenAnnotatorTest, UnderstandsControlStatements) {
2693	auto Tokens = annotate(Code: "while (true) {}");
2694	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2695	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
2696	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
2697
2698	Tokens = annotate(Code: "for (;;) {}");
2699	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2700	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_ControlStatementLBrace);
2701	EXPECT_TOKEN(Tokens[6], tok::r_brace, TT_ControlStatementRBrace);
2702
2703	Tokens = annotate(Code: "do {} while (true);");
2704	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2705	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_ControlStatementLBrace);
2706	EXPECT_TOKEN(Tokens[2], tok::r_brace, TT_ControlStatementRBrace);
2707
2708	Tokens = annotate(Code: "if (true) {} else if (false) {} else {}");
2709	ASSERT_EQ(Tokens.size(), 17u) << Tokens;
2710	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
2711	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
2712	EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_ControlStatementLBrace);
2713	EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_ControlStatementRBrace);
2714	EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_ElseLBrace);
2715	EXPECT_TOKEN(Tokens[15], tok::r_brace, TT_ElseRBrace);
2716
2717	Tokens = annotate(Code: "switch (foo) {}");
2718	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2719	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_ControlStatementLBrace);
2720	EXPECT_TOKEN(Tokens[5], tok::r_brace, TT_ControlStatementRBrace);
2721	}
2722
2723	TEST_F(TokenAnnotatorTest, UnderstandsDoWhile) {
2724	auto Tokens = annotate(Code: "do { ++i; } while ( i > 5 );");
2725	ASSERT_EQ(Tokens.size(), 14u) << Tokens;
2726	EXPECT_TOKEN(Tokens[6], tok::kw_while, TT_DoWhile);
2727
2728	Tokens = annotate(Code: "do ++i; while ( i > 5 );");
2729	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2730	EXPECT_TOKEN(Tokens[4], tok::kw_while, TT_DoWhile);
2731	}
2732
2733	TEST_F(TokenAnnotatorTest, StartOfName) {
2734	auto Tokens = annotate(Code: "#pragma clang diagnostic push");
2735	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2736	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
2737	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2738	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
2739
2740	Tokens = annotate(Code: "#pragma clang diagnostic ignored \"-Wzero-length-array\"");
2741	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2742	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
2743	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2744	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_Unknown);
2745
2746	Tokens = annotate(Code: "#define FOO Foo foo");
2747	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2748	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_Unknown);
2749	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_Unknown);
2750	EXPECT_TOKEN(Tokens[4], tok::identifier, TT_StartOfName);
2751
2752	Tokens = annotate(Code: "@interface NSCoder (TestCoder)");
2753	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2754	EXPECT_TOKEN(Tokens[0], tok::at, TT_ObjCDecl);
2755	EXPECT_TOKEN(Tokens[2], tok::identifier, TT_StartOfName);
2756	}
2757
2758	TEST_F(TokenAnnotatorTest, BraceKind) {
2759	auto Tokens = annotate(Code: "void f() {};");
2760	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2761	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2762	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2763	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2764	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
2765
2766	Tokens = annotate(Code: "class Foo<int> f() {}");
2767	ASSERT_EQ(Tokens.size(), 11u) << Tokens;
2768	EXPECT_TOKEN(Tokens[5], tok::identifier, TT_FunctionDeclarationName);
2769	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
2770	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
2771	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
2772
2773	Tokens = annotate(Code: "template <typename T> class Foo<T> f() {}");
2774	ASSERT_EQ(Tokens.size(), 16u) << Tokens;
2775	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_FunctionDeclarationName);
2776	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2777	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2778	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
2779
2780	Tokens = annotate(Code: "void f() override {};");
2781	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2782	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2783	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_FunctionLBrace);
2784	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
2785	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2786
2787	Tokens = annotate(Code: "void f() noexcept(false) {};");
2788	ASSERT_EQ(Tokens.size(), 12u) << Tokens;
2789	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2790	EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
2791	EXPECT_BRACE_KIND(Tokens[8], BK_Block);
2792	EXPECT_BRACE_KIND(Tokens[9], BK_Block);
2793
2794	Tokens = annotate(Code: "auto f() -> void {};");
2795	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2796	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2797	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
2798	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2799	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
2800
2801	Tokens = annotate(Code: "void f() { /**/ };");
2802	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2803	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2804	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2805	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2806	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2807
2808	Tokens = annotate(Code: "void f() { //\n"
2809	"};");
2810	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2811	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2812	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2813	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2814	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2815
2816	Tokens = annotate(Code: "void f() {\n"
2817	" //\n"
2818	"};");
2819	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2820	EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
2821	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2822	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2823	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2824
2825	Tokens = annotate(Code: "struct Foo {\n"
2826	" Foo() {};\n"
2827	" ~Foo() {};\n"
2828	"};");
2829	ASSERT_EQ(Tokens.size(), 19u) << Tokens;
2830	EXPECT_TOKEN(Tokens[3], tok::identifier, TT_CtorDtorDeclName);
2831	EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_FunctionLBrace);
2832	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2833	EXPECT_BRACE_KIND(Tokens[7], BK_Block);
2834	EXPECT_TOKEN(Tokens[10], tok::identifier, TT_CtorDtorDeclName);
2835	EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
2836	EXPECT_BRACE_KIND(Tokens[13], BK_Block);
2837	EXPECT_BRACE_KIND(Tokens[14], BK_Block);
2838
2839	Tokens = annotate(Code: "{\n"
2840	" char *a[] = {\n"
2841	" /* abc */ \"abc\",\n"
2842	"#if FOO\n"
2843	" /* xyz */ \"xyz\",\n"
2844	"#endif\n"
2845	" /* last */ \"last\"};\n"
2846	"}");
2847	ASSERT_EQ(Tokens.size(), 25u) << Tokens;
2848	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
2849	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
2850	EXPECT_BRACE_KIND(Tokens[21], BK_BracedInit);
2851
2852	Tokens =
2853	annotate(Code: "#define SCOP_STAT(NAME, DESC) \\\n"
2854	" {\"polly\", #NAME, \"Number of rejected regions: \" DESC}");
2855	ASSERT_EQ(Tokens.size(), 18u) << Tokens;
2856	EXPECT_BRACE_KIND(Tokens[8], BK_BracedInit);
2857	EXPECT_BRACE_KIND(Tokens[16], BK_BracedInit);
2858
2859	Tokens = annotate(Code: "struct {};");
2860	ASSERT_EQ(Tokens.size(), 5u) << Tokens;
2861	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
2862	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
2863
2864	Tokens = annotate(Code: "struct : Base {};");
2865	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2866	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
2867	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2868
2869	Tokens = annotate(Code: "struct Foo {};");
2870	ASSERT_EQ(Tokens.size(), 6u) << Tokens;
2871	EXPECT_BRACE_KIND(Tokens[2], BK_Block);
2872	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
2873
2874	Tokens = annotate(Code: "struct ::Foo {};");
2875	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2876	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
2877	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2878
2879	Tokens = annotate(Code: "struct NS::Foo {};");
2880	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2881	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2882	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
2883
2884	Tokens = annotate(Code: "struct Foo<int> {};");
2885	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2886	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
2887	EXPECT_BRACE_KIND(Tokens[6], BK_Block);
2888
2889	Tokens = annotate(Code: "struct Foo final {};");
2890	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2891	EXPECT_BRACE_KIND(Tokens[3], BK_Block);
2892	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2893
2894	Tokens = annotate(Code: "struct [[foo]] [[bar]] Foo final : Base1, Base2 {};");
2895	ASSERT_EQ(Tokens.size(), 21u) << Tokens;
2896	EXPECT_BRACE_KIND(Tokens[17], BK_Block);
2897	EXPECT_BRACE_KIND(Tokens[18], BK_Block);
2898
2899	Tokens = annotate(Code: "struct Foo x{};");
2900	ASSERT_EQ(Tokens.size(), 7u) << Tokens;
2901	EXPECT_BRACE_KIND(Tokens[3], BK_BracedInit);
2902	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
2903
2904	Tokens = annotate(Code: "struct ::Foo x{};");
2905	ASSERT_EQ(Tokens.size(), 8u) << Tokens;
2906	EXPECT_BRACE_KIND(Tokens[4], BK_BracedInit);
2907	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
2908
2909	Tokens = annotate(Code: "struct NS::Foo x{};");
2910	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2911	EXPECT_BRACE_KIND(Tokens[5], BK_BracedInit);
2912	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2913
2914	Tokens = annotate(Code: "struct Foo<int> x{};");
2915	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2916	EXPECT_BRACE_KIND(Tokens[6], BK_BracedInit);
2917	EXPECT_BRACE_KIND(Tokens[7], BK_BracedInit);
2918	}
2919
2920	TEST_F(TokenAnnotatorTest, UnderstandsElaboratedTypeSpecifier) {
2921	auto Tokens = annotate(Code: "auto foo() -> enum En {}");
2922	ASSERT_EQ(Tokens.size(), 10u) << Tokens;
2923	EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
2924	}
2925
2926	TEST_F(TokenAnnotatorTest, BlockLBrace) {
2927	auto Tokens = annotate(Code: "{\n"
2928	" {\n"
2929	" foo();\n"
2930	" }\n"
2931	"}");
2932	ASSERT_EQ(Tokens.size(), 9u) << Tokens;
2933	EXPECT_TOKEN(Tokens[0], tok::l_brace, TT_BlockLBrace);
2934	EXPECT_BRACE_KIND(Tokens[0], BK_Block);
2935	EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_BlockLBrace);
2936	EXPECT_BRACE_KIND(Tokens[1], BK_Block);
2937
2938	Tokens = annotate(Code: "void bar() {\n"
2939	" {\n"
2940	" foo();\n"
2941	" }\n"
2942	"}");
2943	ASSERT_EQ(Tokens.size(), 13u) << Tokens;
2944	EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_FunctionLBrace);
2945	EXPECT_BRACE_KIND(Tokens[4], BK_Block);
2946	EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_BlockLBrace);
2947	EXPECT_BRACE_KIND(Tokens[5], BK_Block);
2948	}
2949
2950	} // namespace
2951	} // namespace format
2952	} // namespace clang
2953