1	/* coroutine-specific state, expansions and tests.
2
3	Copyright (C) 2018-2024 Free Software Foundation, Inc.
4
5	Contributed by Iain Sandoe <iain@sandoe.co.uk> under contract to Facebook.
6
7	This file is part of GCC.
8
9	GCC is free software; you can redistribute it and/or modify it under
10	the terms of the GNU General Public License as published by the Free
11	Software Foundation; either version 3, or (at your option) any later
12	version.
13
14	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15	WARRANTY; without even the implied warranty of MERCHANTABILITY or
16	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17	for more details.
18
19	You should have received a copy of the GNU General Public License
20	along with GCC; see the file COPYING3. If not see
21	<http://www.gnu.org/licenses/>. */
22
23	#include "config.h"
24	#include "system.h"
25	#include "coretypes.h"
26	#include "target.h"
27	#include "cp-tree.h"
28	#include "stringpool.h"
29	#include "stmt.h"
30	#include "stor-layout.h"
31	#include "tree-iterator.h"
32	#include "tree.h"
33	#include "gcc-rich-location.h"
34	#include "hash-map.h"
35
36	static bool coro_promise_type_found_p (tree, location_t);
37
38	/* GCC C++ coroutines implementation.
39
40	The user authors a function that becomes a coroutine (lazily) by
41	making use of any of the co_await, co_yield or co_return keywords.
42
43	Unlike a regular function, where the activation record is placed on the
44	stack, and is destroyed on function exit, a coroutine has some state that
45	persists between calls - the coroutine frame (analogous to a stack frame).
46
47	We transform the user's function into three pieces:
48	1. A so-called ramp function, that establishes the coroutine frame and
49	begins execution of the coroutine.
50	2. An actor function that contains the state machine corresponding to the
51	user's suspend/resume structure.
52	3. A stub function that calls the actor function in 'destroy' mode.
53
54	The actor function is executed:
55	* from "resume point 0" by the ramp.
56	* from resume point N ( > 0 ) for handle.resume() calls.
57	* from the destroy stub for destroy point N for handle.destroy() calls.
58
59	The functions in this file carry out the necessary analysis of, and
60	transforms to, the AST to perform this.
61
62	The C++ coroutine design makes use of some helper functions that are
63	authored in a so-called "promise" class provided by the user.
64
65	At parse time (or post substitution) the type of the coroutine promise
66	will be determined. At that point, we can look up the required promise
67	class methods and issue diagnostics if they are missing or incorrect. To
68	avoid repeating these actions at code-gen time, we make use of temporary
69	'proxy' variables for the coroutine handle and the promise - which will
70	eventually be instantiated in the coroutine frame.
71
72	Each of the keywords will expand to a code sequence (although co_yield is
73	just syntactic sugar for a co_await).
74
75	We defer the analysis and transformation until template expansion is
76	complete so that we have complete types at that time. */
77
78
79	/* The state that we collect during parsing (and template expansion) for
80	a coroutine. */
81
82	struct GTY((for_user)) coroutine_info
83	{
84	tree function_decl; /* The original function decl. */
85	tree actor_decl; /* The synthesized actor function. */
86	tree destroy_decl; /* The synthesized destroy function. */
87	tree promise_type; /* The cached promise type for this function. */
88	tree handle_type; /* The cached coroutine handle for this function. */
89	tree self_h_proxy; /* A handle instance that is used as the proxy for the
90	one that will eventually be allocated in the coroutine
91	frame. */
92	tree promise_proxy; /* Likewise, a proxy promise instance. */
93	tree return_void; /* The expression for p.return_void() if it exists. */
94	location_t first_coro_keyword; /* The location of the keyword that made this
95	function into a coroutine. */
96	/* Flags to avoid repeated errors for per-function issues. */
97	bool coro_ret_type_error_emitted;
98	bool coro_promise_error_emitted;
99	bool coro_co_return_error_emitted;
100	};
101
102	struct coroutine_info_hasher : ggc_ptr_hash<coroutine_info>
103	{
104	typedef tree compare_type; /* We only compare the function decl. */
105	static inline hashval_t hash (coroutine_info *);
106	static inline hashval_t hash (const compare_type &);
107	static inline bool equal (coroutine_info *, coroutine_info *);
108	static inline bool equal (coroutine_info *, const compare_type &);
109	};
110
111	/* This table holds all the collected coroutine state for coroutines in
112	the current translation unit. */
113
114	static GTY (()) hash_table<coroutine_info_hasher> *coroutine_info_table;
115
116	/* We will initialize state lazily. */
117	static bool coro_initialized = false;
118
119	/* Return a hash value for the entry pointed to by INFO.
120	The compare type is a tree, but the only trees we are going use are
121	function decls. We use the DECL_UID as the hash value since that is
122	stable across PCH. */
123
124	hashval_t
125	coroutine_info_hasher::hash (coroutine_info *info)
126	{
127	return DECL_UID (info->function_decl);
128	}
129
130	/* Return a hash value for the compare value COMP. */
131
132	hashval_t
133	coroutine_info_hasher::hash (const compare_type& comp)
134	{
135	return DECL_UID (comp);
136	}
137
138	/* Return true if the entries pointed to by LHS and RHS are for the
139	same coroutine. */
140
141	bool
142	coroutine_info_hasher::equal (coroutine_info *lhs, coroutine_info *rhs)
143	{
144	return lhs->function_decl == rhs->function_decl;
145	}
146
147	bool
148	coroutine_info_hasher::equal (coroutine_info *lhs, const compare_type& rhs)
149	{
150	return lhs->function_decl == rhs;
151	}
152
153	/* Get the existing coroutine_info for FN_DECL, or insert a new one if the
154	entry does not yet exist. */
155
156	coroutine_info *
157	get_or_insert_coroutine_info (tree fn_decl)
158	{
159	gcc_checking_assert (coroutine_info_table != NULL);
160
161	coroutine_info **slot = coroutine_info_table->find_slot_with_hash
162	(comparable: fn_decl, hash: coroutine_info_hasher::hash (comp: fn_decl), insert: INSERT);
163
164	if (*slot == NULL)
165	{
166	*slot = new (ggc_cleared_alloc<coroutine_info> ()) coroutine_info ();
167	(*slot)->function_decl = fn_decl;
168	}
169
170	return *slot;
171	}
172
173	/* Get the existing coroutine_info for FN_DECL, fail if it doesn't exist. */
174
175	coroutine_info *
176	get_coroutine_info (tree fn_decl)
177	{
178	if (coroutine_info_table == NULL)
179	return NULL;
180
181	coroutine_info **slot = coroutine_info_table->find_slot_with_hash
182	(comparable: fn_decl, hash: coroutine_info_hasher::hash (comp: fn_decl), insert: NO_INSERT);
183	if (slot)
184	return *slot;
185	return NULL;
186	}
187
188	/* We will lazily create all the identifiers that are used by coroutines
189	on the first attempt to lookup the traits. */
190
191	/* Identifiers that are used by all coroutines. */
192
193	static GTY(()) tree coro_traits_identifier;
194	static GTY(()) tree coro_handle_identifier;
195	static GTY(()) tree coro_promise_type_identifier;
196
197	/* Required promise method name identifiers. */
198
199	static GTY(()) tree coro_await_transform_identifier;
200	static GTY(()) tree coro_initial_suspend_identifier;
201	static GTY(()) tree coro_final_suspend_identifier;
202	static GTY(()) tree coro_return_void_identifier;
203	static GTY(()) tree coro_return_value_identifier;
204	static GTY(()) tree coro_yield_value_identifier;
205	static GTY(()) tree coro_resume_identifier;
206	static GTY(()) tree coro_address_identifier;
207	static GTY(()) tree coro_from_address_identifier;
208	static GTY(()) tree coro_get_return_object_identifier;
209	static GTY(()) tree coro_gro_on_allocation_fail_identifier;
210	static GTY(()) tree coro_unhandled_exception_identifier;
211
212	/* Awaitable methods. */
213
214	static GTY(()) tree coro_await_ready_identifier;
215	static GTY(()) tree coro_await_suspend_identifier;
216	static GTY(()) tree coro_await_resume_identifier;
217
218	/* Accessors for the coroutine frame state used by the implementation. */
219
220	static GTY(()) tree coro_resume_fn_id;
221	static GTY(()) tree coro_destroy_fn_id;
222	static GTY(()) tree coro_promise_id;
223	static GTY(()) tree coro_frame_needs_free_id;
224	static GTY(()) tree coro_resume_index_id;
225	static GTY(()) tree coro_self_handle_id;
226	static GTY(()) tree coro_actor_continue_id;
227	static GTY(()) tree coro_frame_i_a_r_c_id;
228
229	/* Create the identifiers used by the coroutines library interfaces and
230	the implementation frame state. */
231
232	static void
233	coro_init_identifiers ()
234	{
235	coro_traits_identifier = get_identifier ("coroutine_traits");
236	coro_handle_identifier = get_identifier ("coroutine_handle");
237	coro_promise_type_identifier = get_identifier ("promise_type");
238
239	coro_await_transform_identifier = get_identifier ("await_transform");
240	coro_initial_suspend_identifier = get_identifier ("initial_suspend");
241	coro_final_suspend_identifier = get_identifier ("final_suspend");
242	coro_return_void_identifier = get_identifier ("return_void");
243	coro_return_value_identifier = get_identifier ("return_value");
244	coro_yield_value_identifier = get_identifier ("yield_value");
245	coro_resume_identifier = get_identifier ("resume");
246	coro_address_identifier = get_identifier ("address");
247	coro_from_address_identifier = get_identifier ("from_address");
248	coro_get_return_object_identifier = get_identifier ("get_return_object");
249	coro_gro_on_allocation_fail_identifier =
250	get_identifier ("get_return_object_on_allocation_failure");
251	coro_unhandled_exception_identifier = get_identifier ("unhandled_exception");
252
253	coro_await_ready_identifier = get_identifier ("await_ready");
254	coro_await_suspend_identifier = get_identifier ("await_suspend");
255	coro_await_resume_identifier = get_identifier ("await_resume");
256
257	/* Coroutine state frame field accessors. */
258	coro_resume_fn_id = get_identifier ("_Coro_resume_fn");
259	coro_destroy_fn_id = get_identifier ("_Coro_destroy_fn");
260	coro_promise_id = get_identifier ("_Coro_promise");
261	coro_frame_needs_free_id = get_identifier ("_Coro_frame_needs_free");
262	coro_frame_i_a_r_c_id = get_identifier ("_Coro_initial_await_resume_called");
263	coro_resume_index_id = get_identifier ("_Coro_resume_index");
264	coro_self_handle_id = get_identifier ("_Coro_self_handle");
265	coro_actor_continue_id = get_identifier ("_Coro_actor_continue");
266	}
267
268	/* Trees we only need to set up once. */
269
270	static GTY(()) tree coro_traits_templ;
271	static GTY(()) tree coro_handle_templ;
272	static GTY(()) tree void_coro_handle_type;
273
274	/* ================= Parse, Semantics and Type checking ================= */
275
276	/* This initial set of routines are helper for the parsing and template
277	expansion phases.
278
279	At the completion of this, we will have completed trees for each of the
280	keywords, but making use of proxy variables for the self-handle and the
281	promise class instance. */
282
283	/* [coroutine.traits]
284	Lookup the coroutine_traits template decl. */
285
286	static tree
287	find_coro_traits_template_decl (location_t kw)
288	{
289	/* If we are missing fundamental information, such as the traits, (or the
290	declaration found is not a type template), then don't emit an error for
291	every keyword in a TU, just do it once. */
292	static bool traits_error_emitted = false;
293
294	tree traits_decl = lookup_qualified_name (std_node, name: coro_traits_identifier,
295	LOOK_want::NORMAL,
296	/*complain=*/!traits_error_emitted);
297	if (traits_decl == error_mark_node
298	|| !DECL_TYPE_TEMPLATE_P (traits_decl))
299	{
300	if (!traits_error_emitted)
301	{
302	gcc_rich_location richloc (kw);
303	error_at (&richloc, "coroutines require a traits template; cannot"
304	" find %<%E::%E%>", std_node, coro_traits_identifier);
305	inform (&richloc, "perhaps %<#include <coroutine>%> is missing");
306	traits_error_emitted = true;
307	}
308	return NULL_TREE;
309	}
310	else
311	return traits_decl;
312	}
313
314	/* Instantiate Coroutine traits for the function signature. */
315
316	static tree
317	instantiate_coro_traits (tree fndecl, location_t kw)
318	{
319	/* [coroutine.traits.primary]
320	So now build up a type list for the template <typename _R, typename...>.
321	The types are the function's arg types and _R is the function return
322	type. */
323
324	tree functyp = TREE_TYPE (fndecl);
325	tree arg = DECL_ARGUMENTS (fndecl);
326	tree arg_node = TYPE_ARG_TYPES (functyp);
327	tree argtypes = make_tree_vec (list_length (arg_node)-1);
328	unsigned p = 0;
329
330	while (arg_node != NULL_TREE && !VOID_TYPE_P (TREE_VALUE (arg_node)))
331	{
332	if (is_this_parameter (arg)
333	|| DECL_NAME (arg) == closure_identifier)
334	{
335	/* We pass a reference to *this to the param preview. */
336	tree ct = TREE_TYPE (TREE_TYPE (arg));
337	TREE_VEC_ELT (argtypes, p++) = cp_build_reference_type (ct, false);
338	}
339	else
340	TREE_VEC_ELT (argtypes, p++) = TREE_VALUE (arg_node);
341
342	arg_node = TREE_CHAIN (arg_node);
343	arg = DECL_CHAIN (arg);
344	}
345
346	tree argtypepack = cxx_make_type (TYPE_ARGUMENT_PACK);
347	ARGUMENT_PACK_ARGS (argtypepack) = argtypes;
348
349	tree targ = make_tree_vec (2);
350	TREE_VEC_ELT (targ, 0) = TREE_TYPE (functyp);
351	TREE_VEC_ELT (targ, 1) = argtypepack;
352
353	tree traits_class
354	= lookup_template_class (coro_traits_templ, targ,
355	/*in_decl=*/NULL_TREE, /*context=*/NULL_TREE,
356	/*entering scope=*/false, tf_warning_or_error);
357
358	if (traits_class == error_mark_node)
359	{
360	error_at (kw, "cannot instantiate %<coroutine traits%>");
361	return NULL_TREE;
362	}
363
364	return traits_class;
365	}
366
367	/* [coroutine.handle] */
368
369	static tree
370	find_coro_handle_template_decl (location_t kw)
371	{
372	/* As for the coroutine traits, this error is per TU, so only emit
373	it once. */
374	static bool coro_handle_error_emitted = false;
375	tree handle_decl = lookup_qualified_name (std_node, name: coro_handle_identifier,
376	LOOK_want::NORMAL,
377	!coro_handle_error_emitted);
378	if (handle_decl == error_mark_node
379	|| !DECL_CLASS_TEMPLATE_P (handle_decl))
380	{
381	if (!coro_handle_error_emitted)
382	error_at (kw, "coroutines require a handle class template;"
383	" cannot find %<%E::%E%>", std_node, coro_handle_identifier);
384	coro_handle_error_emitted = true;
385	return NULL_TREE;
386	}
387	else
388	return handle_decl;
389	}
390
391	/* Instantiate the handle template for a given promise type. */
392
393	static tree
394	instantiate_coro_handle_for_promise_type (location_t kw, tree promise_type)
395	{
396	/* So now build up a type list for the template, one entry, the promise. */
397	tree targ = make_tree_vec (1);
398	TREE_VEC_ELT (targ, 0) = promise_type;
399	tree handle_type
400	= lookup_template_class (coro_handle_identifier, targ,
401	/* in_decl=*/NULL_TREE,
402	/* context=*/std_node,
403	/* entering scope=*/false, tf_warning_or_error);
404
405	if (handle_type == error_mark_node)
406	{
407	error_at (kw, "cannot instantiate a %<coroutine handle%> for"
408	" promise type %qT", promise_type);
409	return NULL_TREE;
410	}
411
412	return handle_type;
413	}
414
415	/* Look for the promise_type in the instantiated traits. */
416
417	static tree
418	find_promise_type (tree traits_class)
419	{
420	tree promise_type
421	= lookup_member (traits_class, coro_promise_type_identifier,
422	/* protect=*/1, /*want_type=*/true, tf_warning_or_error);
423
424	if (promise_type)
425	promise_type
426	= complete_type_or_else (TREE_TYPE (promise_type), promise_type);
427
428	/* NULL_TREE on fail. */
429	return promise_type;
430	}
431
432	static bool
433	coro_promise_type_found_p (tree fndecl, location_t loc)
434	{
435	gcc_assert (fndecl != NULL_TREE);
436
437	if (!coro_initialized)
438	{
439	/* Trees we only need to create once.
440	Set up the identifiers we will use. */
441	coro_init_identifiers ();
442
443	/* Coroutine traits template. */
444	coro_traits_templ = find_coro_traits_template_decl (kw: loc);
445	if (coro_traits_templ == NULL_TREE)
446	return false;
447
448	/* coroutine_handle<> template. */
449	coro_handle_templ = find_coro_handle_template_decl (kw: loc);
450	if (coro_handle_templ == NULL_TREE)
451	return false;
452
453	/* We can also instantiate the void coroutine_handle<> */
454	void_coro_handle_type =
455	instantiate_coro_handle_for_promise_type (kw: loc, NULL_TREE);
456	if (void_coro_handle_type == NULL_TREE)
457	return false;
458
459	/* A table to hold the state, per coroutine decl. */
460	gcc_checking_assert (coroutine_info_table == NULL);
461	coroutine_info_table =
462	hash_table<coroutine_info_hasher>::create_ggc (n: 11);
463
464	if (coroutine_info_table == NULL)
465	return false;
466
467	coro_initialized = true;
468	}
469
470	/* Save the coroutine data on the side to avoid the overhead on every
471	function decl tree. */
472
473	coroutine_info *coro_info = get_or_insert_coroutine_info (fn_decl: fndecl);
474	/* Without this, we cannot really proceed. */
475	gcc_checking_assert (coro_info);
476
477	/* If we don't already have a current promise type, try to look it up. */
478	if (coro_info->promise_type == NULL_TREE)
479	{
480	/* Get the coroutine traits template class instance for the function
481	signature we have - coroutine_traits <R, ...> */
482
483	tree templ_class = instantiate_coro_traits (fndecl, kw: loc);
484
485	/* Find the promise type for that. */
486	coro_info->promise_type = find_promise_type (traits_class: templ_class);
487
488	/* If we don't find it, punt on the rest. */
489	if (coro_info->promise_type == NULL_TREE)
490	{
491	if (!coro_info->coro_promise_error_emitted)
492	error_at (loc, "unable to find the promise type for"
493	" this coroutine");
494	coro_info->coro_promise_error_emitted = true;
495	return false;
496	}
497
498	/* Test for errors in the promise type that can be determined now. */
499	tree has_ret_void = lookup_member (coro_info->promise_type,
500	coro_return_void_identifier,
501	/*protect=*/1, /*want_type=*/0,
502	tf_none);
503	tree has_ret_val = lookup_member (coro_info->promise_type,
504	coro_return_value_identifier,
505	/*protect=*/1, /*want_type=*/0,
506	tf_none);
507	if (has_ret_void && has_ret_val)
508	{
509	location_t ploc = DECL_SOURCE_LOCATION (fndecl);
510	if (!coro_info->coro_co_return_error_emitted)
511	error_at (ploc, "the coroutine promise type %qT declares both"
512	" %<return_value%> and %<return_void%>",
513	coro_info->promise_type);
514	inform (DECL_SOURCE_LOCATION (BASELINK_FUNCTIONS (has_ret_void)),
515	"%<return_void%> declared here");
516	has_ret_val = BASELINK_FUNCTIONS (has_ret_val);
517	const char *message = "%<return_value%> declared here";
518	if (TREE_CODE (has_ret_val) == OVERLOAD)
519	{
520	has_ret_val = OVL_FIRST (has_ret_val);
521	message = "%<return_value%> first declared here";
522	}
523	inform (DECL_SOURCE_LOCATION (has_ret_val), message);
524	coro_info->coro_co_return_error_emitted = true;
525	return false;
526	}
527
528	/* Try to find the handle type for the promise. */
529	tree handle_type =
530	instantiate_coro_handle_for_promise_type (kw: loc, promise_type: coro_info->promise_type);
531	if (handle_type == NULL_TREE)
532	return false;
533
534	/* Complete this, we're going to use it. */
535	coro_info->handle_type = complete_type_or_else (handle_type, fndecl);
536
537	/* Diagnostic would be emitted by complete_type_or_else. */
538	if (!coro_info->handle_type)
539	return false;
540
541	/* Build a proxy for a handle to "self" as the param to
542	await_suspend() calls. */
543	coro_info->self_h_proxy
544	= build_lang_decl (VAR_DECL, coro_self_handle_id,
545	coro_info->handle_type);
546
547	/* Build a proxy for the promise so that we can perform lookups. */
548	coro_info->promise_proxy
549	= build_lang_decl (VAR_DECL, coro_promise_id,
550	coro_info->promise_type);
551
552	/* Note where we first saw a coroutine keyword. */
553	coro_info->first_coro_keyword = loc;
554	}
555
556	return true;
557	}
558
559	/* Map from actor or destroyer to ramp. */
560	static GTY(()) hash_map<tree, tree> *to_ramp;
561
562	/* Given a tree that is an actor or destroy, find the ramp function. */
563
564	tree
565	coro_get_ramp_function (tree decl)
566	{
567	if (!to_ramp)
568	return NULL_TREE;
569	tree *p = to_ramp->get (k: decl);
570	if (p)
571	return *p;
572	return NULL_TREE;
573	}
574
575	/* Given the DECL for a ramp function (the user's original declaration) return
576	the actor function if it has been defined. */
577
578	tree
579	coro_get_actor_function (tree decl)
580	{
581	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
582	return info->actor_decl;
583
584	return NULL_TREE;
585	}
586
587	/* Given the DECL for a ramp function (the user's original declaration) return
588	the destroy function if it has been defined. */
589
590	tree
591	coro_get_destroy_function (tree decl)
592	{
593	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
594	return info->destroy_decl;
595
596	return NULL_TREE;
597	}
598
599	/* These functions assumes that the caller has verified that the state for
600	the decl has been initialized, we try to minimize work here. */
601
602	static tree
603	get_coroutine_promise_type (tree decl)
604	{
605	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
606	return info->promise_type;
607
608	return NULL_TREE;
609	}
610
611	static tree
612	get_coroutine_handle_type (tree decl)
613	{
614	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
615	return info->handle_type;
616
617	return NULL_TREE;
618	}
619
620	static tree
621	get_coroutine_self_handle_proxy (tree decl)
622	{
623	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
624	return info->self_h_proxy;
625
626	return NULL_TREE;
627	}
628
629	static tree
630	get_coroutine_promise_proxy (tree decl)
631	{
632	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
633	return info->promise_proxy;
634
635	return NULL_TREE;
636	}
637
638	static tree
639	lookup_promise_method (tree fndecl, tree member_id, location_t loc,
640	bool musthave)
641	{
642	tree promise = get_coroutine_promise_type (decl: fndecl);
643	tree pm_memb
644	= lookup_member (promise, member_id,
645	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
646	if (musthave && pm_memb == NULL_TREE)
647	{
648	error_at (loc, "no member named %qE in %qT", member_id, promise);
649	return error_mark_node;
650	}
651	return pm_memb;
652	}
653
654	/* Build an expression of the form p.method (args) where the p is a promise
655	object for the current coroutine.
656	OBJECT is the promise object instance to use, it may be NULL, in which case
657	we will use the promise_proxy instance for this coroutine.
658	ARGS may be NULL, for empty parm lists. */
659
660	static tree
661	coro_build_promise_expression (tree fn, tree promise_obj, tree member_id,
662	location_t loc, vec<tree, va_gc> **args,
663	bool musthave)
664	{
665	tree meth = lookup_promise_method (fndecl: fn, member_id, loc, musthave);
666	if (meth == error_mark_node)
667	return error_mark_node;
668
669	/* If we don't find it, and it isn't needed, an empty return is OK. */
670	if (!meth)
671	return NULL_TREE;
672
673	tree promise
674	= promise_obj ? promise_obj
675	: get_coroutine_promise_proxy (decl: current_function_decl);
676	tree expr;
677	if (BASELINK_P (meth))
678	expr = build_new_method_call (promise, meth, args, NULL_TREE,
679	LOOKUP_NORMAL, NULL, tf_warning_or_error);
680	else
681	{
682	expr = build_class_member_access_expr (promise, meth, NULL_TREE,
683	true, tf_warning_or_error);
684	vec<tree, va_gc> *real_args;
685	if (!args)
686	real_args = make_tree_vector ();
687	else
688	real_args = *args;
689	expr = build_op_call (expr, &real_args, tf_warning_or_error);
690	}
691	return expr;
692	}
693
694	/* Caching get for the expression p.return_void (). */
695
696	static tree
697	get_coroutine_return_void_expr (tree decl, location_t loc, bool musthave)
698	{
699	if (coroutine_info *info = get_coroutine_info (fn_decl: decl))
700	{
701	/* If we don't have it try to build it. */
702	if (!info->return_void)
703	info->return_void
704	= coro_build_promise_expression (fn: current_function_decl, NULL,
705	member_id: coro_return_void_identifier,
706	loc, NULL, musthave);
707	/* Don't return an error if it's an optional call. */
708	if (!musthave && info->return_void == error_mark_node)
709	return NULL_TREE;
710	return info->return_void;
711	}
712	return musthave ? error_mark_node : NULL_TREE;
713	}
714
715	/* Lookup an Awaitable member, which should be await_ready, await_suspend
716	or await_resume. */
717
718	static tree
719	lookup_awaitable_member (tree await_type, tree member_id, location_t loc)
720	{
721	tree aw_memb
722	= lookup_member (await_type, member_id,
723	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
724	if (aw_memb == NULL_TREE)
725	{
726	error_at (loc, "no member named %qE in %qT", member_id, await_type);
727	return error_mark_node;
728	}
729	return aw_memb;
730	}
731
732	/* Here we check the constraints that are common to all keywords (since the
733	presence of a coroutine keyword makes the function into a coroutine). */
734
735	static bool
736	coro_common_keyword_context_valid_p (tree fndecl, location_t kw_loc,
737	const char *kw_name)
738	{
739	if (fndecl == NULL_TREE)
740	{
741	error_at (kw_loc, "%qs cannot be used outside a function", kw_name);
742	return false;
743	}
744
745	/* This is arranged in order of prohibitions in the std. */
746	if (DECL_MAIN_P (fndecl))
747	{
748	/* [basic.start.main] 3. The function main shall not be a coroutine. */
749	error_at (kw_loc, "%qs cannot be used in the %<main%> function",
750	kw_name);
751	return false;
752	}
753
754	if (DECL_DECLARED_CONSTEXPR_P (fndecl))
755	{
756	cp_function_chain->invalid_constexpr = true;
757	if (!is_instantiation_of_constexpr (fndecl))
758	{
759	/* [dcl.constexpr] 3.3 it shall not be a coroutine. */
760	error_at (kw_loc, "%qs cannot be used in a %<constexpr%> function",
761	kw_name);
762	return false;
763	}
764	}
765
766	if (FNDECL_USED_AUTO (fndecl))
767	{
768	/* [dcl.spec.auto] 15. A function declared with a return type that uses
769	a placeholder type shall not be a coroutine. */
770	error_at (kw_loc,
771	"%qs cannot be used in a function with a deduced return type",
772	kw_name);
773	return false;
774	}
775
776	if (varargs_function_p (fndecl))
777	{
778	/* [dcl.fct.def.coroutine] The parameter-declaration-clause of the
779	coroutine shall not terminate with an ellipsis that is not part
780	of a parameter-declaration. */
781	error_at (kw_loc,
782	"%qs cannot be used in a varargs function", kw_name);
783	return false;
784	}
785
786	if (DECL_CONSTRUCTOR_P (fndecl))
787	{
788	/* [class.ctor] 7. a constructor shall not be a coroutine. */
789	error_at (kw_loc, "%qs cannot be used in a constructor", kw_name);
790	return false;
791	}
792
793	if (DECL_DESTRUCTOR_P (fndecl))
794	{
795	/* [class.dtor] 21. a destructor shall not be a coroutine. */
796	error_at (kw_loc, "%qs cannot be used in a destructor", kw_name);
797	return false;
798	}
799
800	return true;
801	}
802
803	/* Here we check the constraints that are not per keyword. */
804
805	static bool
806	coro_function_valid_p (tree fndecl)
807	{
808	location_t f_loc = DECL_SOURCE_LOCATION (fndecl);
809
810	/* For cases where fundamental information cannot be found, e.g. the
811	coroutine traits are missing, we need to punt early. */
812	if (!coro_promise_type_found_p (fndecl, loc: f_loc))
813	return false;
814
815	/* Since we think the function is a coroutine, that implies we parsed
816	a keyword that triggered this. Keywords check promise validity for
817	their context and thus the promise type should be known at this point. */
818	if (get_coroutine_handle_type (decl: fndecl) == NULL_TREE
819	|| get_coroutine_promise_type (decl: fndecl) == NULL_TREE)
820	return false;
821
822	if (current_function_returns_value || current_function_returns_null)
823	{
824	/* TODO: record or extract positions of returns (and the first coro
825	keyword) so that we can add notes to the diagnostic about where
826	the bad keyword is and what made the function into a coro. */
827	error_at (f_loc, "a %<return%> statement is not allowed in coroutine;"
828	" did you mean %<co_return%>?");
829	return false;
830	}
831
832	return true;
833	}
834
835	enum suspend_point_kind {
836	CO_AWAIT_SUSPEND_POINT = 0,
837	CO_YIELD_SUSPEND_POINT,
838	INITIAL_SUSPEND_POINT,
839	FINAL_SUSPEND_POINT
840	};
841
842	/* Helper function to build a named variable for the temps we use for each
843	await point. The root of the name is determined by SUSPEND_KIND, and
844	the variable is of type V_TYPE. The awaitable number is reset each time
845	we encounter a final suspend. */
846
847	static tree
848	get_awaitable_var (suspend_point_kind suspend_kind, tree v_type)
849	{
850	static int awn = 0;
851	char *buf;
852	switch (suspend_kind)
853	{
854	default: buf = xasprintf ("Aw%d", awn++); break;
855	case CO_YIELD_SUSPEND_POINT: buf = xasprintf ("Yd%d", awn++); break;
856	case INITIAL_SUSPEND_POINT: buf = xasprintf ("Is"); break;
857	case FINAL_SUSPEND_POINT: buf = xasprintf ("Fs"); awn = 0; break;
858	}
859	tree ret = get_identifier (buf);
860	free (ptr: buf);
861	ret = build_lang_decl (VAR_DECL, ret, v_type);
862	DECL_ARTIFICIAL (ret) = true;
863	return ret;
864	}
865
866	/* Helpers to diagnose missing noexcept on final await expressions. */
867
868	static bool
869	coro_diagnose_throwing_fn (tree fndecl)
870	{
871	if (!TYPE_NOTHROW_P (TREE_TYPE (fndecl)))
872	{
873	location_t f_loc = cp_expr_loc_or_loc (t: fndecl,
874	DECL_SOURCE_LOCATION (fndecl));
875	error_at (f_loc, "the expression %qE is required to be non-throwing",
876	fndecl);
877	inform (f_loc, "must be declared with %<noexcept(true)%>");
878	return true;
879	}
880	return false;
881	}
882
883	static bool
884	coro_diagnose_throwing_final_aw_expr (tree expr)
885	{
886	if (TREE_CODE (expr) == TARGET_EXPR)
887	expr = TARGET_EXPR_INITIAL (expr);
888	tree fn = NULL_TREE;
889	if (TREE_CODE (expr) == CALL_EXPR)
890	fn = CALL_EXPR_FN (expr);
891	else if (TREE_CODE (expr) == AGGR_INIT_EXPR)
892	fn = AGGR_INIT_EXPR_FN (expr);
893	else if (TREE_CODE (expr) == CONSTRUCTOR)
894	return false;
895	else
896	{
897	gcc_checking_assert (0 && "unhandled expression type");
898	return false;
899	}
900	fn = TREE_OPERAND (fn, 0);
901	return coro_diagnose_throwing_fn (fndecl: fn);
902	}
903
904	/* This performs [expr.await] bullet 3.3 and validates the interface obtained.
905	It is also used to build the initial and final suspend points.
906
907	'a', 'o' and 'e' are used as per the description in the section noted.
908
909	A, the original yield/await expr, is found at source location LOC.
910
911	We will be constructing a CO_AWAIT_EXPR for a suspend point of one of
912	the four suspend_point_kind kinds. This is indicated by SUSPEND_KIND. */
913
914	static tree
915	build_co_await (location_t loc, tree a, suspend_point_kind suspend_kind)
916	{
917	/* Try and overload of operator co_await, .... */
918	tree o;
919	if (MAYBE_CLASS_TYPE_P (TREE_TYPE (a)))
920	{
921	o = build_new_op (loc, CO_AWAIT_EXPR, LOOKUP_NORMAL, a, NULL_TREE,
922	NULL_TREE, NULL_TREE, NULL, tf_warning_or_error);
923	/* If no viable functions are found, o is a. */
924	if (!o || o == error_mark_node)
925	o = a;
926	else if (flag_exceptions && suspend_kind == FINAL_SUSPEND_POINT)
927	{
928	/* We found an overload for co_await(), diagnose throwing cases. */
929	if (TREE_CODE (o) == TARGET_EXPR
930	&& coro_diagnose_throwing_final_aw_expr (expr: o))
931	return error_mark_node;
932
933	/* We now know that the final suspend object is distinct from the
934	final awaiter, so check for a non-throwing DTOR where needed. */
935	if (tree dummy = cxx_maybe_build_cleanup (a, tf_none))
936	{
937	if (CONVERT_EXPR_P (dummy))
938	dummy = TREE_OPERAND (dummy, 0);
939	dummy = TREE_OPERAND (CALL_EXPR_FN (dummy), 0);
940	if (coro_diagnose_throwing_fn (fndecl: dummy))
941	return error_mark_node;
942	}
943	}
944	}
945	else
946	o = a; /* This is most likely about to fail anyway. */
947
948	tree o_type = TREE_TYPE (o);
949	if (o_type && !VOID_TYPE_P (o_type))
950	o_type = complete_type_or_else (o_type, o);
951
952	if (!o_type)
953	return error_mark_node;
954
955	if (TREE_CODE (o_type) != RECORD_TYPE)
956	{
957	error_at (loc, "awaitable type %qT is not a structure",
958	o_type);
959	return error_mark_node;
960	}
961
962	/* Check for required awaitable members and their types. */
963	tree awrd_meth
964	= lookup_awaitable_member (await_type: o_type, member_id: coro_await_ready_identifier, loc);
965	if (!awrd_meth || awrd_meth == error_mark_node)
966	return error_mark_node;
967	tree awsp_meth
968	= lookup_awaitable_member (await_type: o_type, member_id: coro_await_suspend_identifier, loc);
969	if (!awsp_meth || awsp_meth == error_mark_node)
970	return error_mark_node;
971
972	/* The type of the co_await is the return type of the awaitable's
973	await_resume, so we need to look that up. */
974	tree awrs_meth
975	= lookup_awaitable_member (await_type: o_type, member_id: coro_await_resume_identifier, loc);
976	if (!awrs_meth || awrs_meth == error_mark_node)
977	return error_mark_node;
978
979	/* To complete the lookups, we need an instance of 'e' which is built from
980	'o' according to [expr.await] 3.4.
981
982	If we need to materialize this as a temporary, then that will have to be
983	'promoted' to a coroutine frame var. However, if the awaitable is a
984	user variable, parameter or comes from a scope outside this function,
985	then we must use it directly - or we will see unnecessary copies.
986
987	If o is a variable, find the underlying var. */
988	tree e_proxy = STRIP_NOPS (o);
989	if (INDIRECT_REF_P (e_proxy))
990	e_proxy = TREE_OPERAND (e_proxy, 0);
991	while (TREE_CODE (e_proxy) == COMPONENT_REF)
992	{
993	e_proxy = TREE_OPERAND (e_proxy, 0);
994	if (INDIRECT_REF_P (e_proxy))
995	e_proxy = TREE_OPERAND (e_proxy, 0);
996	if (TREE_CODE (e_proxy) == CALL_EXPR)
997	{
998	/* We could have operator-> here too. */
999	tree op = TREE_OPERAND (CALL_EXPR_FN (e_proxy), 0);
1000	if (DECL_OVERLOADED_OPERATOR_P (op)
1001	&& DECL_OVERLOADED_OPERATOR_IS (op, COMPONENT_REF))
1002	{
1003	e_proxy = CALL_EXPR_ARG (e_proxy, 0);
1004	STRIP_NOPS (e_proxy);
1005	gcc_checking_assert (TREE_CODE (e_proxy) == ADDR_EXPR);
1006	e_proxy = TREE_OPERAND (e_proxy, 0);
1007	}
1008	}
1009	STRIP_NOPS (e_proxy);
1010	}
1011
1012	/* Only build a temporary if we need it. */
1013	STRIP_NOPS (e_proxy);
1014	if (TREE_CODE (e_proxy) == PARM_DECL
1015	|| (VAR_P (e_proxy) && !is_local_temp (e_proxy)))
1016	{
1017	e_proxy = o;
1018	o = NULL_TREE; /* The var is already present. */
1019	}
1020	else
1021	{
1022	tree p_type = o_type;
1023	if (glvalue_p (o))
1024	p_type = cp_build_reference_type (p_type, !lvalue_p (o));
1025	e_proxy = get_awaitable_var (suspend_kind, v_type: p_type);
1026	o = cp_build_modify_expr (loc, e_proxy, INIT_EXPR, o,
1027	tf_warning_or_error);
1028	e_proxy = convert_from_reference (e_proxy);
1029	}
1030
1031	/* I suppose we could check that this is contextually convertible to bool. */
1032	tree awrd_func = NULL_TREE;
1033	tree awrd_call
1034	= build_new_method_call (e_proxy, awrd_meth, NULL, NULL_TREE, LOOKUP_NORMAL,
1035	&awrd_func, tf_warning_or_error);
1036
1037	if (!awrd_func || !awrd_call || awrd_call == error_mark_node)
1038	return error_mark_node;
1039
1040	/* The suspend method may return one of three types:
1041	1. void (no special action needed).
1042	2. bool (if true, we don't need to suspend).
1043	3. a coroutine handle, we execute the handle.resume() call. */
1044	tree awsp_func = NULL_TREE;
1045	tree h_proxy = get_coroutine_self_handle_proxy (decl: current_function_decl);
1046	vec<tree, va_gc> *args = make_tree_vector_single (h_proxy);
1047	tree awsp_call
1048	= build_new_method_call (e_proxy, awsp_meth, &args, NULL_TREE,
1049	LOOKUP_NORMAL, &awsp_func, tf_warning_or_error);
1050
1051	release_tree_vector (args);
1052	if (!awsp_func || !awsp_call || awsp_call == error_mark_node)
1053	return error_mark_node;
1054
1055	bool ok = false;
1056	tree susp_return_type = TREE_TYPE (TREE_TYPE (awsp_func));
1057	if (same_type_p (susp_return_type, void_type_node))
1058	ok = true;
1059	else if (same_type_p (susp_return_type, boolean_type_node))
1060	ok = true;
1061	else if (TREE_CODE (susp_return_type) == RECORD_TYPE
1062	&& CLASS_TYPE_P (susp_return_type)
1063	&& CLASSTYPE_TEMPLATE_INFO (susp_return_type))
1064	{
1065	tree tt = CLASSTYPE_TI_TEMPLATE (susp_return_type);
1066	if (tt == coro_handle_templ)
1067	ok = true;
1068	}
1069
1070	if (!ok)
1071	{
1072	error_at (loc, "%<await_suspend%> must return %<void%>, %<bool%> or"
1073	" a coroutine handle");
1074	return error_mark_node;
1075	}
1076
1077	/* Finally, the type of e.await_resume() is the co_await's type. */
1078	tree awrs_func = NULL_TREE;
1079	tree awrs_call
1080	= build_new_method_call (e_proxy, awrs_meth, NULL, NULL_TREE, LOOKUP_NORMAL,
1081	&awrs_func, tf_warning_or_error);
1082
1083	if (!awrs_func || !awrs_call || awrs_call == error_mark_node)
1084	return error_mark_node;
1085
1086	if (flag_exceptions && suspend_kind == FINAL_SUSPEND_POINT)
1087	{
1088	if (coro_diagnose_throwing_fn (fndecl: awrd_func))
1089	return error_mark_node;
1090	if (coro_diagnose_throwing_fn (fndecl: awsp_func))
1091	return error_mark_node;
1092	if (coro_diagnose_throwing_fn (fndecl: awrs_func))
1093	return error_mark_node;
1094	if (tree dummy = cxx_maybe_build_cleanup (e_proxy, tf_none))
1095	{
1096	if (CONVERT_EXPR_P (dummy))
1097	dummy = TREE_OPERAND (dummy, 0);
1098	dummy = TREE_OPERAND (CALL_EXPR_FN (dummy), 0);
1099	if (coro_diagnose_throwing_fn (fndecl: dummy))
1100	return error_mark_node;
1101	}
1102	}
1103
1104	/* We now have three call expressions, in terms of the promise, handle and
1105	'e' proxies. Save them in the await expression for later expansion. */
1106
1107	tree awaiter_calls = make_tree_vec (3);
1108	TREE_VEC_ELT (awaiter_calls, 0) = awrd_call; /* await_ready(). */
1109	TREE_VEC_ELT (awaiter_calls, 1) = awsp_call; /* await_suspend(). */
1110	tree te = NULL_TREE;
1111	if (TREE_CODE (awrs_call) == TARGET_EXPR)
1112	{
1113	te = awrs_call;
1114	awrs_call = TREE_OPERAND (awrs_call, 1);
1115	}
1116	TREE_VEC_ELT (awaiter_calls, 2) = awrs_call; /* await_resume(). */
1117
1118	if (REFERENCE_REF_P (e_proxy))
1119	e_proxy = TREE_OPERAND (e_proxy, 0);
1120
1121	tree await_expr = build5_loc (loc, code: CO_AWAIT_EXPR,
1122	TREE_TYPE (TREE_TYPE (awrs_func)),
1123	arg0: a, arg1: e_proxy, arg2: o, arg3: awaiter_calls,
1124	arg4: build_int_cst (integer_type_node,
1125	(int) suspend_kind));
1126	TREE_SIDE_EFFECTS (await_expr) = true;
1127	if (te)
1128	{
1129	TREE_OPERAND (te, 1) = await_expr;
1130	TREE_SIDE_EFFECTS (te) = true;
1131	await_expr = te;
1132	}
1133	SET_EXPR_LOCATION (await_expr, loc);
1134	return convert_from_reference (await_expr);
1135	}
1136
1137	tree
1138	finish_co_await_expr (location_t kw, tree expr)
1139	{
1140	if (!expr || error_operand_p (t: expr))
1141	return error_mark_node;
1142
1143	if (!coro_common_keyword_context_valid_p (fndecl: current_function_decl, kw_loc: kw,
1144	kw_name: "co_await"))
1145	return error_mark_node;
1146
1147	/* The current function has now become a coroutine, if it wasn't already. */
1148	DECL_COROUTINE_P (current_function_decl) = 1;
1149
1150	/* This function will appear to have no return statement, even if it
1151	is declared to return non-void (most likely). This is correct - we
1152	synthesize the return for the ramp in the compiler. So suppress any
1153	extraneous warnings during substitution. */
1154	suppress_warning (current_function_decl, OPT_Wreturn_type);
1155
1156	/* Defer expansion when we are processing a template.
1157	FIXME: If the coroutine function's type is not dependent, and the operand
1158	is not dependent, we should determine the type of the co_await expression
1159	using the DEPENDENT_EXPR wrapper machinery. That allows us to determine
1160	the subexpression type, but leave its operand unchanged and then
1161	instantiate it later. */
1162	if (processing_template_decl)
1163	{
1164	tree aw_expr = build5_loc (loc: kw, code: CO_AWAIT_EXPR, unknown_type_node, arg0: expr,
1165	NULL_TREE, NULL_TREE, NULL_TREE,
1166	integer_zero_node);
1167	TREE_SIDE_EFFECTS (aw_expr) = true;
1168	return aw_expr;
1169	}
1170
1171	/* We must be able to look up the "await_transform" method in the scope of
1172	the promise type, and obtain its return type. */
1173	if (!coro_promise_type_found_p (fndecl: current_function_decl, loc: kw))
1174	return error_mark_node;
1175
1176	/* [expr.await] 3.2
1177	The incoming cast expression might be transformed by a promise
1178	'await_transform()'. */
1179	tree at_meth
1180	= lookup_promise_method (fndecl: current_function_decl,
1181	member_id: coro_await_transform_identifier, loc: kw,
1182	/*musthave=*/false);
1183	if (at_meth == error_mark_node)
1184	return error_mark_node;
1185
1186	tree a = expr;
1187	if (at_meth)
1188	{
1189	/* try to build a = p.await_transform (e). */
1190	vec<tree, va_gc> *args = make_tree_vector_single (expr);
1191	a = build_new_method_call (get_coroutine_promise_proxy (
1192	decl: current_function_decl),
1193	at_meth, &args, NULL_TREE, LOOKUP_NORMAL,
1194	NULL, tf_warning_or_error);
1195
1196	/* As I read the section.
1197	We saw an await_transform method, so it's mandatory that we replace
1198	expr with p.await_transform (expr), therefore if the method call fails
1199	(presumably, we don't have suitable arguments) then this part of the
1200	process fails. */
1201	if (a == error_mark_node)
1202	return error_mark_node;
1203	}
1204
1205	/* Now we want to build co_await a. */
1206	return build_co_await (loc: kw, a, suspend_kind: CO_AWAIT_SUSPEND_POINT);
1207	}
1208
1209	/* Take the EXPR given and attempt to build:
1210	co_await p.yield_value (expr);
1211	per [expr.yield] para 1. */
1212
1213	tree
1214	finish_co_yield_expr (location_t kw, tree expr)
1215	{
1216	if (!expr || error_operand_p (t: expr))
1217	return error_mark_node;
1218
1219	/* Check the general requirements and simple syntax errors. */
1220	if (!coro_common_keyword_context_valid_p (fndecl: current_function_decl, kw_loc: kw,
1221	kw_name: "co_yield"))
1222	return error_mark_node;
1223
1224	/* The current function has now become a coroutine, if it wasn't already. */
1225	DECL_COROUTINE_P (current_function_decl) = 1;
1226
1227	/* This function will appear to have no return statement, even if it
1228	is declared to return non-void (most likely). This is correct - we
1229	synthesize the return for the ramp in the compiler. So suppress any
1230	extraneous warnings during substitution. */
1231	suppress_warning (current_function_decl, OPT_Wreturn_type);
1232
1233	/* Defer expansion when we are processing a template; see FIXME in the
1234	co_await code. */
1235	if (processing_template_decl)
1236	return build2_loc (loc: kw, code: CO_YIELD_EXPR, unknown_type_node, arg0: expr, NULL_TREE);
1237
1238	if (!coro_promise_type_found_p (fndecl: current_function_decl, loc: kw))
1239	/* We must be able to look up the "yield_value" method in the scope of
1240	the promise type, and obtain its return type. */
1241	return error_mark_node;
1242
1243	/* [expr.yield] / 1
1244	Let e be the operand of the yield-expression and p be an lvalue naming
1245	the promise object of the enclosing coroutine, then the yield-expression
1246	is equivalent to the expression co_await p.yield_value(e).
1247	build p.yield_value(e): */
1248	vec<tree, va_gc> *args = make_tree_vector_single (expr);
1249	tree yield_call
1250	= coro_build_promise_expression (fn: current_function_decl, NULL,
1251	member_id: coro_yield_value_identifier, loc: kw,
1252	args: &args, /*musthave=*/true);
1253	release_tree_vector (args);
1254
1255	/* Now build co_await p.yield_value (e).
1256	Noting that for co_yield, there is no evaluation of any potential
1257	promise transform_await(), so we call build_co_await directly. */
1258
1259	tree op = build_co_await (loc: kw, a: yield_call, suspend_kind: CO_YIELD_SUSPEND_POINT);
1260	if (op != error_mark_node)
1261	{
1262	if (REFERENCE_REF_P (op))
1263	op = TREE_OPERAND (op, 0);
1264	/* If the await expression is wrapped in a TARGET_EXPR, then transfer
1265	that wrapper to the CO_YIELD_EXPR, since this is just a proxy for
1266	its contained await. Otherwise, just build the CO_YIELD_EXPR. */
1267	if (TREE_CODE (op) == TARGET_EXPR)
1268	{
1269	tree t = TREE_OPERAND (op, 1);
1270	t = build2_loc (loc: kw, code: CO_YIELD_EXPR, TREE_TYPE (t), arg0: expr, arg1: t);
1271	TREE_OPERAND (op, 1) = t;
1272	}
1273	else
1274	op = build2_loc (loc: kw, code: CO_YIELD_EXPR, TREE_TYPE (op), arg0: expr, arg1: op);
1275	TREE_SIDE_EFFECTS (op) = 1;
1276	op = convert_from_reference (op);
1277	}
1278
1279	return op;
1280	}
1281
1282	/* Check and build a co_return statement.
1283	First that it's valid to have a co_return keyword here.
1284	If it is, then check and build the p.return_{void(),value(expr)}.
1285	These are built against a proxy for the promise, which will be filled
1286	in with the actual frame version when the function is transformed. */
1287
1288	tree
1289	finish_co_return_stmt (location_t kw, tree expr)
1290	{
1291	if (expr)
1292	STRIP_ANY_LOCATION_WRAPPER (expr);
1293
1294	if (error_operand_p (t: expr))
1295	return error_mark_node;
1296
1297	/* If it fails the following test, the function is not permitted to be a
1298	coroutine, so the co_return statement is erroneous. */
1299	if (!coro_common_keyword_context_valid_p (fndecl: current_function_decl, kw_loc: kw,
1300	kw_name: "co_return"))
1301	return error_mark_node;
1302
1303	/* The current function has now become a coroutine, if it wasn't
1304	already. */
1305	DECL_COROUTINE_P (current_function_decl) = 1;
1306
1307	/* This function will appear to have no return statement, even if it
1308	is declared to return non-void (most likely). This is correct - we
1309	synthesize the return for the ramp in the compiler. So suppress any
1310	extraneous warnings during substitution. */
1311	suppress_warning (current_function_decl, OPT_Wreturn_type);
1312
1313	if (processing_template_decl
1314	&& check_for_bare_parameter_packs (expr))
1315	return error_mark_node;
1316
1317	/* Defer expansion when we are processing a template; see FIXME in the
1318	co_await code. */
1319	if (processing_template_decl)
1320	{
1321	/* co_return expressions are always void type, regardless of the
1322	expression type. */
1323	expr = build2_loc (loc: kw, code: CO_RETURN_EXPR, void_type_node,
1324	arg0: expr, NULL_TREE);
1325	expr = maybe_cleanup_point_expr_void (expr);
1326	return add_stmt (expr);
1327	}
1328
1329	if (!coro_promise_type_found_p (fndecl: current_function_decl, loc: kw))
1330	return error_mark_node;
1331
1332	/* Suppress -Wreturn-type for co_return, we need to check indirectly
1333	whether the promise type has a suitable return_void/return_value. */
1334	suppress_warning (current_function_decl, OPT_Wreturn_type);
1335
1336	if (!processing_template_decl && warn_sequence_point)
1337	verify_sequence_points (expr);
1338
1339	if (expr)
1340	{
1341	/* If we had an id-expression obfuscated by force_paren_expr, we need
1342	to undo it so we can try to treat it as an rvalue below. */
1343	expr = maybe_undo_parenthesized_ref (expr);
1344
1345	if (error_operand_p (t: expr))
1346	return error_mark_node;
1347	}
1348
1349	/* If the promise object doesn't have the correct return call then
1350	there's a mis-match between the co_return <expr> and this. */
1351	tree co_ret_call = error_mark_node;
1352	if (expr == NULL_TREE || VOID_TYPE_P (TREE_TYPE (expr)))
1353	co_ret_call
1354	= get_coroutine_return_void_expr (decl: current_function_decl, loc: kw, musthave: true);
1355	else
1356	{
1357	/* [class.copy.elision] / 3.
1358	An implicitly movable entity is a variable of automatic storage
1359	duration that is either a non-volatile object or an rvalue reference
1360	to a non-volatile object type. For such objects in the context of
1361	the co_return, the overload resolution should be carried out first
1362	treating the object as an rvalue, if that fails, then we fall back
1363	to regular overload resolution. */
1364
1365	tree arg = expr;
1366	if (tree moved = treat_lvalue_as_rvalue_p (expr, /*return*/true))
1367	arg = moved;
1368
1369	releasing_vec args = make_tree_vector_single (arg);
1370	co_ret_call
1371	= coro_build_promise_expression (fn: current_function_decl, NULL,
1372	member_id: coro_return_value_identifier, loc: kw,
1373	args: &args, /*musthave=*/true);
1374	}
1375
1376	/* Makes no sense for a co-routine really. */
1377	if (TREE_THIS_VOLATILE (current_function_decl))
1378	warning_at (kw, 0,
1379	"function declared %<noreturn%> has a"
1380	" %<co_return%> statement");
1381
1382	expr = build2_loc (loc: kw, code: CO_RETURN_EXPR, void_type_node, arg0: expr, arg1: co_ret_call);
1383	expr = maybe_cleanup_point_expr_void (expr);
1384	return add_stmt (expr);
1385	}
1386
1387	/* We need to validate the arguments to __builtin_coro_promise, since the
1388	second two must be constant, and the builtins machinery doesn't seem to
1389	deal with that properly. */
1390
1391	tree
1392	coro_validate_builtin_call (tree call, tsubst_flags_t)
1393	{
1394	tree fn = TREE_OPERAND (CALL_EXPR_FN (call), 0);
1395
1396	gcc_checking_assert (DECL_BUILT_IN_CLASS (fn) == BUILT_IN_NORMAL);
1397	switch (DECL_FUNCTION_CODE (decl: fn))
1398	{
1399	default:
1400	return call;
1401
1402	case BUILT_IN_CORO_PROMISE:
1403	{
1404	/* Argument 0 is already checked by the normal built-in machinery
1405	Argument 1 must be a constant of size type. It probably makes
1406	little sense if it's not a power of 2, but that isn't specified
1407	formally. */
1408	tree arg = CALL_EXPR_ARG (call, 1);
1409	location_t loc = EXPR_LOCATION (arg);
1410
1411	/* We expect alignof expressions in templates. */
1412	if (TREE_CODE (arg) == ALIGNOF_EXPR)
1413	;
1414	else if (!TREE_CONSTANT (arg))
1415	{
1416	error_at (loc, "the align argument to %<__builtin_coro_promise%>"
1417	" must be a constant");
1418	return error_mark_node;
1419	}
1420	/* Argument 2 is the direction - to / from handle address to promise
1421	address. */
1422	arg = CALL_EXPR_ARG (call, 2);
1423	loc = EXPR_LOCATION (arg);
1424	if (!TREE_CONSTANT (arg))
1425	{
1426	error_at (loc, "the direction argument to"
1427	" %<__builtin_coro_promise%> must be a constant");
1428	return error_mark_node;
1429	}
1430	return call;
1431	break;
1432	}
1433	}
1434	}
1435
1436	/* ================= Morph and Expand. =================
1437
1438	The entry point here is morph_fn_to_coro () which is called from
1439	finish_function () when we have completed any template expansion.
1440
1441	This is preceded by helper functions that implement the phases below.
1442
1443	The process proceeds in four phases.
1444
1445	A Initial framing.
1446	The user's function body is wrapped in the initial and final suspend
1447	points and we begin building the coroutine frame.
1448	We build empty decls for the actor and destroyer functions at this
1449	time too.
1450	When exceptions are enabled, the user's function body will also be
1451	wrapped in a try-catch block with the catch invoking the promise
1452	class 'unhandled_exception' method.
1453
1454	B Analysis.
1455	The user's function body is analyzed to determine the suspend points,
1456	if any, and to capture local variables that might persist across such
1457	suspensions. In most cases, it is not necessary to capture compiler
1458	temporaries, since the tree-lowering nests the suspensions correctly.
1459	However, in the case of a captured reference, there is a lifetime
1460	extension to the end of the full expression - which can mean across a
1461	suspend point in which case it must be promoted to a frame variable.
1462
1463	At the conclusion of analysis, we have a conservative frame layout and
1464	maps of the local variables to their frame entry points.
1465
1466	C Build the ramp function.
1467	Carry out the allocation for the coroutine frame (NOTE; the actual size
1468	computation is deferred until late in the middle end to allow for future
1469	optimizations that will be allowed to elide unused frame entries).
1470	We build the return object.
1471
1472	D Build and expand the actor and destroyer function bodies.
1473	The destroyer is a trivial shim that sets a bit to indicate that the
1474	destroy dispatcher should be used and then calls into the actor.
1475
1476	The actor function is the implementation of the user's state machine.
1477	The current suspend point is noted in an index.
1478	Each suspend point is encoded as a pair of internal functions, one in
1479	the relevant dispatcher, and one representing the suspend point.
1480
1481	During this process, the user's local variables and the proxies for the
1482	self-handle and the promise class instance are re-written to their
1483	coroutine frame equivalents.
1484
1485	The complete bodies for the ramp, actor and destroy function are passed
1486	back to finish_function for folding and gimplification. */
1487
1488	/* Helpers to build EXPR_STMT and void-cast EXPR_STMT, common ops. */
1489
1490	static tree
1491	coro_build_expr_stmt (tree expr, location_t loc)
1492	{
1493	return maybe_cleanup_point_expr_void (build_stmt (loc, EXPR_STMT, expr));
1494	}
1495
1496	static tree
1497	coro_build_cvt_void_expr_stmt (tree expr, location_t loc)
1498	{
1499	tree t = build1 (CONVERT_EXPR, void_type_node, expr);
1500	return coro_build_expr_stmt (expr: t, loc);
1501	}
1502
1503	/* Helpers to build an artificial var, with location LOC, NAME and TYPE, in
1504	CTX, and with initializer INIT. */
1505
1506	static tree
1507	coro_build_artificial_var (location_t loc, tree name, tree type, tree ctx,
1508	tree init)
1509	{
1510	tree res = build_lang_decl (VAR_DECL, name, type);
1511	DECL_SOURCE_LOCATION (res) = loc;
1512	DECL_CONTEXT (res) = ctx;
1513	DECL_ARTIFICIAL (res) = true;
1514	DECL_INITIAL (res) = init;
1515	return res;
1516	}
1517
1518	static tree
1519	coro_build_artificial_var (location_t loc, const char *name, tree type,
1520	tree ctx, tree init)
1521	{
1522	return coro_build_artificial_var (loc, get_identifier (name),
1523	type, ctx, init);
1524	}
1525
1526	/* Helpers for label creation:
1527	1. Create a named label in the specified context. */
1528
1529	static tree
1530	create_anon_label_with_ctx (location_t loc, tree ctx)
1531	{
1532	tree lab = build_decl (loc, LABEL_DECL, NULL_TREE, void_type_node);
1533
1534	DECL_CONTEXT (lab) = ctx;
1535	DECL_ARTIFICIAL (lab) = true;
1536	DECL_IGNORED_P (lab) = true;
1537	TREE_USED (lab) = true;
1538	return lab;
1539	}
1540
1541	/* 2. Create a named label in the specified context. */
1542
1543	static tree
1544	create_named_label_with_ctx (location_t loc, const char *name, tree ctx)
1545	{
1546	tree lab_id = get_identifier (name);
1547	tree lab = define_label (loc, lab_id);
1548	DECL_CONTEXT (lab) = ctx;
1549	DECL_ARTIFICIAL (lab) = true;
1550	TREE_USED (lab) = true;
1551	return lab;
1552	}
1553
1554	struct proxy_replace
1555	{
1556	tree from, to;
1557	};
1558
1559	static tree
1560	replace_proxy (tree *here, int *do_subtree, void *d)
1561	{
1562	proxy_replace *data = (proxy_replace *) d;
1563
1564	if (*here == data->from)
1565	{
1566	*here = data->to;
1567	*do_subtree = 0;
1568	}
1569	else
1570	*do_subtree = 1;
1571	return NULL_TREE;
1572	}
1573
1574	/* Support for expansion of co_await statements. */
1575
1576	struct coro_aw_data
1577	{
1578	tree actor_fn; /* Decl for context. */
1579	tree coro_fp; /* Frame pointer var. */
1580	tree resume_idx; /* This is the index var in the frame. */
1581	tree i_a_r_c; /* initial suspend await_resume() was called if true. */
1582	tree self_h; /* This is a handle to the current coro (frame var). */
1583	tree cleanup; /* This is where to go once we complete local destroy. */
1584	tree cororet; /* This is where to go if we suspend. */
1585	tree corocont; /* This is where to go if we continue. */
1586	tree conthand; /* This is the handle for a continuation. */
1587	unsigned index; /* This is our current resume index. */
1588	};
1589
1590	/* Lightweight search for the first await expression in tree-walk order.
1591	returns:
1592	The first await expression found in STMT.
1593	NULL_TREE if there are none.
1594	So can be used to determine if the statement needs to be processed for
1595	awaits. */
1596
1597	static tree
1598	co_await_find_in_subtree (tree *stmt, int *, void *d)
1599	{
1600	tree **p = (tree **) d;
1601	if (TREE_CODE (*stmt) == CO_AWAIT_EXPR)
1602	{
1603	*p = stmt;
1604	return *stmt;
1605	}
1606	return NULL_TREE;
1607	}
1608
1609	/* Starting with a statement:
1610
1611	stmt => some tree containing one or more await expressions.
1612
1613	We replace the statement with:
1614	<STATEMENT_LIST> {
1615	initialize awaitable
1616	if (!ready)
1617	{
1618	suspension context.
1619	}
1620	resume:
1621	revised statement with one await expression rewritten to its
1622	await_resume() return value.
1623	}
1624
1625	We then recurse into the initializer and the revised statement
1626	repeating this replacement until there are no more await expressions
1627	in either. */
1628
1629	static tree *
1630	expand_one_await_expression (tree *stmt, tree *await_expr, void *d)
1631	{
1632	coro_aw_data *data = (coro_aw_data *) d;
1633
1634	tree saved_statement = *stmt;
1635	tree saved_co_await = *await_expr;
1636
1637	tree actor = data->actor_fn;
1638	location_t loc = EXPR_LOCATION (*stmt);
1639	tree var = TREE_OPERAND (saved_co_await, 1); /* frame slot. */
1640	tree expr = TREE_OPERAND (saved_co_await, 2); /* initializer. */
1641	tree awaiter_calls = TREE_OPERAND (saved_co_await, 3);
1642
1643	tree source = TREE_OPERAND (saved_co_await, 4);
1644	bool is_final = (source
1645	&& TREE_INT_CST_LOW (source) == (int) FINAL_SUSPEND_POINT);
1646	bool needs_dtor = TYPE_HAS_NONTRIVIAL_DESTRUCTOR (TREE_TYPE (var));
1647	int resume_point = data->index;
1648	size_t bufsize = sizeof ("destroy.") + 10;
1649	char *buf = (char *) alloca (bufsize);
1650	snprintf (s: buf, maxlen: bufsize, format: "destroy.%d", resume_point);
1651	tree destroy_label = create_named_label_with_ctx (loc, name: buf, ctx: actor);
1652	snprintf (s: buf, maxlen: bufsize, format: "resume.%d", resume_point);
1653	tree resume_label = create_named_label_with_ctx (loc, name: buf, ctx: actor);
1654	tree empty_list = build_empty_stmt (loc);
1655
1656	tree stmt_list = NULL;
1657	tree r;
1658	tree *await_init = NULL;
1659
1660	if (!expr)
1661	needs_dtor = false; /* No need, the var's lifetime is managed elsewhere. */
1662	else
1663	{
1664	r = coro_build_cvt_void_expr_stmt (expr, loc);
1665	append_to_statement_list_force (r, &stmt_list);
1666	/* We have an initializer, which might itself contain await exprs. */
1667	await_init = tsi_stmt_ptr (i: tsi_last (t: stmt_list));
1668	}
1669
1670	/* Use the await_ready() call to test if we need to suspend. */
1671	tree ready_cond = TREE_VEC_ELT (awaiter_calls, 0); /* await_ready(). */
1672	/* Convert to bool, if necessary. */
1673	if (TREE_CODE (TREE_TYPE (ready_cond)) != BOOLEAN_TYPE)
1674	ready_cond = cp_convert (boolean_type_node, ready_cond,
1675	tf_warning_or_error);
1676	/* Be aggressive in folding here, since there are a significant number of
1677	cases where the ready condition is constant. */
1678	ready_cond = invert_truthvalue_loc (loc, ready_cond);
1679	ready_cond
1680	= build1_loc (loc, code: CLEANUP_POINT_EXPR, boolean_type_node, arg1: ready_cond);
1681
1682	tree body_list = NULL;
1683	tree susp_idx = build_int_cst (short_unsigned_type_node, data->index);
1684	r = build2_loc (loc, code: MODIFY_EXPR, short_unsigned_type_node, arg0: data->resume_idx,
1685	arg1: susp_idx);
1686	r = coro_build_cvt_void_expr_stmt (expr: r, loc);
1687	append_to_statement_list (r, &body_list);
1688
1689	/* Find out what we have to do with the awaiter's suspend method.
1690	[expr.await]
1691	(5.1) If the result of await-ready is false, the coroutine is considered
1692	suspended. Then:
1693	(5.1.1) If the type of await-suspend is std::coroutine_handle<Z>,
1694	await-suspend.resume() is evaluated.
1695	(5.1.2) if the type of await-suspend is bool, await-suspend is evaluated,
1696	and the coroutine is resumed if the result is false.
1697	(5.1.3) Otherwise, await-suspend is evaluated. */
1698
1699	tree suspend = TREE_VEC_ELT (awaiter_calls, 1); /* await_suspend(). */
1700	tree susp_type = TREE_TYPE (suspend);
1701
1702	bool is_cont = false;
1703	/* NOTE: final suspend can't resume; the "resume" label in that case
1704	corresponds to implicit destruction. */
1705	if (VOID_TYPE_P (susp_type))
1706	{
1707	/* We just call await_suspend() and hit the yield. */
1708	suspend = coro_build_cvt_void_expr_stmt (expr: suspend, loc);
1709	append_to_statement_list (suspend, &body_list);
1710	}
1711	else if (TREE_CODE (susp_type) == BOOLEAN_TYPE)
1712	{
1713	/* Boolean return, continue if the call returns false. */
1714	suspend = build1_loc (loc, code: TRUTH_NOT_EXPR, boolean_type_node, arg1: suspend);
1715	suspend
1716	= build1_loc (loc, code: CLEANUP_POINT_EXPR, boolean_type_node, arg1: suspend);
1717	tree go_on = build1_loc (loc, code: GOTO_EXPR, void_type_node, arg1: resume_label);
1718	r = build3_loc (loc, code: COND_EXPR, void_type_node, arg0: suspend, arg1: go_on,
1719	arg2: empty_list);
1720	append_to_statement_list (r, &body_list);
1721	}
1722	else
1723	{
1724	r = suspend;
1725	if (!same_type_ignoring_top_level_qualifiers_p (susp_type,
1726	void_coro_handle_type))
1727	r = build1_loc (loc, code: VIEW_CONVERT_EXPR, type: void_coro_handle_type, arg1: r);
1728	r = cp_build_init_expr (loc, data->conthand, r);
1729	r = build1 (CONVERT_EXPR, void_type_node, r);
1730	append_to_statement_list (r, &body_list);
1731	is_cont = true;
1732	}
1733
1734	tree d_l = build_address (destroy_label);
1735	tree r_l = build_address (resume_label);
1736	tree susp = build_address (data->cororet);
1737	tree cont = build_address (data->corocont);
1738	tree final_susp = build_int_cst (integer_type_node, is_final ? 1 : 0);
1739
1740	susp_idx = build_int_cst (integer_type_node, data->index);
1741
1742	tree sw = begin_switch_stmt ();
1743	tree cond = build_decl (loc, VAR_DECL, NULL_TREE, integer_type_node);
1744	DECL_ARTIFICIAL (cond) = 1;
1745	DECL_IGNORED_P (cond) = 1;
1746	layout_decl (cond, 0);
1747
1748	r = build_call_expr_internal_loc (loc, IFN_CO_YIELD, integer_type_node, 5,
1749	susp_idx, final_susp, r_l, d_l,
1750	data->coro_fp);
1751	r = cp_build_init_expr (t: cond, i: r);
1752	finish_switch_cond (r, sw);
1753	r = build_case_label (integer_zero_node, NULL_TREE,
1754	create_anon_label_with_ctx (loc, ctx: actor));
1755	add_stmt (r); /* case 0: */
1756	/* Implement the suspend, a scope exit without clean ups. */
1757	r = build_call_expr_internal_loc (loc, IFN_CO_SUSPN, void_type_node, 1,
1758	is_cont ? cont : susp);
1759	r = coro_build_cvt_void_expr_stmt (expr: r, loc);
1760	add_stmt (r); /* goto ret; */
1761	r = build_case_label (integer_one_node, NULL_TREE,
1762	create_anon_label_with_ctx (loc, ctx: actor));
1763	add_stmt (r); /* case 1: */
1764	r = build1_loc (loc, code: GOTO_EXPR, void_type_node, arg1: resume_label);
1765	add_stmt (r); /* goto resume; */
1766	r = build_case_label (NULL_TREE, NULL_TREE,
1767	create_anon_label_with_ctx (loc, ctx: actor));
1768	add_stmt (r); /* default:; */
1769	r = build1_loc (loc, code: GOTO_EXPR, void_type_node, arg1: destroy_label);
1770	add_stmt (r); /* goto destroy; */
1771
1772	/* part of finish switch. */
1773	SWITCH_STMT_BODY (sw) = pop_stmt_list (SWITCH_STMT_BODY (sw));
1774	pop_switch ();
1775	tree scope = SWITCH_STMT_SCOPE (sw);
1776	SWITCH_STMT_SCOPE (sw) = NULL;
1777	r = do_poplevel (scope);
1778	append_to_statement_list (r, &body_list);
1779
1780	destroy_label = build_stmt (loc, LABEL_EXPR, destroy_label);
1781	append_to_statement_list (destroy_label, &body_list);
1782	if (needs_dtor)
1783	{
1784	tree dtor = build_cleanup (var);
1785	append_to_statement_list (dtor, &body_list);
1786	}
1787	r = build1_loc (loc, code: GOTO_EXPR, void_type_node, arg1: data->cleanup);
1788	append_to_statement_list (r, &body_list);
1789
1790	r = build3_loc (loc, code: COND_EXPR, void_type_node, arg0: ready_cond, arg1: body_list,
1791	arg2: empty_list);
1792
1793	append_to_statement_list (r, &stmt_list);
1794
1795	/* Resume point. */
1796	resume_label = build_stmt (loc, LABEL_EXPR, resume_label);
1797	append_to_statement_list (resume_label, &stmt_list);
1798
1799	/* This will produce the value (if one is provided) from the co_await
1800	expression. */
1801	tree resume_call = TREE_VEC_ELT (awaiter_calls, 2); /* await_resume(). */
1802	if (REFERENCE_REF_P (resume_call))
1803	/* Sink to await_resume call_expr. */
1804	resume_call = TREE_OPERAND (resume_call, 0);
1805
1806	*await_expr = resume_call; /* Replace the co_await expr with its result. */
1807	append_to_statement_list_force (saved_statement, &stmt_list);
1808	/* Get a pointer to the revised statement. */
1809	tree *revised = tsi_stmt_ptr (i: tsi_last (t: stmt_list));
1810	if (needs_dtor)
1811	{
1812	tree dtor = build_cleanup (var);
1813	append_to_statement_list (dtor, &stmt_list);
1814	}
1815	data->index += 2;
1816
1817	/* Replace the original statement with the expansion. */
1818	*stmt = stmt_list;
1819
1820	/* Now, if the awaitable had an initializer, expand any awaits that might
1821	be embedded in it. */
1822	tree *aw_expr_ptr;
1823	if (await_init &&
1824	cp_walk_tree (await_init, co_await_find_in_subtree, &aw_expr_ptr, NULL))
1825	expand_one_await_expression (stmt: await_init, await_expr: aw_expr_ptr, d);
1826
1827	/* Expand any more await expressions in the original statement. */
1828	if (cp_walk_tree (revised, co_await_find_in_subtree, &aw_expr_ptr, NULL))
1829	expand_one_await_expression (stmt: revised, await_expr: aw_expr_ptr, d);
1830
1831	return NULL;
1832	}
1833
1834	/* Check to see if a statement contains at least one await expression, if
1835	so, then process that. */
1836
1837	static tree
1838	process_one_statement (tree *stmt, void *d)
1839	{
1840	tree *aw_expr_ptr;
1841	if (cp_walk_tree (stmt, co_await_find_in_subtree, &aw_expr_ptr, NULL))
1842	expand_one_await_expression (stmt, await_expr: aw_expr_ptr, d);
1843	return NULL_TREE;
1844	}
1845
1846	static tree
1847	await_statement_expander (tree *stmt, int *do_subtree, void *d)
1848	{
1849	tree res = NULL_TREE;
1850
1851	/* Process a statement at a time. */
1852	if (STATEMENT_CLASS_P (*stmt) || TREE_CODE (*stmt) == BIND_EXPR)
1853	return NULL_TREE; /* Just process the sub-trees. */
1854	else if (TREE_CODE (*stmt) == STATEMENT_LIST)
1855	{
1856	for (tree &s : tsi_range (*stmt))
1857	{
1858	res = cp_walk_tree (&s, await_statement_expander,
1859	d, NULL);
1860	if (res)
1861	return res;
1862	}
1863	*do_subtree = 0; /* Done subtrees. */
1864	}
1865	else if (EXPR_P (*stmt))
1866	{
1867	process_one_statement (stmt, d);
1868	*do_subtree = 0; /* Done subtrees. */
1869	}
1870
1871	/* Continue statement walk, where required. */
1872	return res;
1873	}
1874
1875	/* Suspend point hash_map. */
1876
1877	struct suspend_point_info
1878	{
1879	/* coro frame field type. */
1880	tree awaitable_type;
1881	/* coro frame field name. */
1882	tree await_field_id;
1883	};
1884
1885	static hash_map<tree, suspend_point_info> *suspend_points;
1886
1887	struct await_xform_data
1888	{
1889	tree actor_fn; /* Decl for context. */
1890	tree actor_frame;
1891	};
1892
1893	/* When we built the await expressions, we didn't know the coro frame
1894	layout, therefore no idea where to find the promise or where to put
1895	the awaitables. Now we know these things, fill them in. */
1896
1897	static tree
1898	transform_await_expr (tree await_expr, await_xform_data *xform)
1899	{
1900	suspend_point_info *si = suspend_points->get (k: await_expr);
1901	location_t loc = EXPR_LOCATION (await_expr);
1902	if (!si)
1903	{
1904	error_at (loc, "no suspend point info for %qD", await_expr);
1905	return error_mark_node;
1906	}
1907
1908	/* So, on entry, we have:
1909	in : CO_AWAIT_EXPR (a, e_proxy, o, awr_call_vector, mode)
1910	We no longer need a [it had diagnostic value, maybe?]
1911	We need to replace the e_proxy in the awr_call. */
1912
1913	tree coro_frame_type = TREE_TYPE (xform->actor_frame);
1914
1915	/* If we have a frame var for the awaitable, get a reference to it. */
1916	proxy_replace data;
1917	if (si->await_field_id)
1918	{
1919	tree as_m
1920	= lookup_member (coro_frame_type, si->await_field_id,
1921	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
1922	tree as = build_class_member_access_expr (xform->actor_frame, as_m,
1923	NULL_TREE, true,
1924	tf_warning_or_error);
1925
1926	/* Replace references to the instance proxy with the frame entry now
1927	computed. */
1928	data.from = TREE_OPERAND (await_expr, 1);
1929	data.to = as;
1930	cp_walk_tree (&await_expr, replace_proxy, &data, NULL);
1931
1932	/* .. and replace. */
1933	TREE_OPERAND (await_expr, 1) = as;
1934	}
1935
1936	return await_expr;
1937	}
1938
1939	/* A wrapper for the transform_await_expr function so that it can be a
1940	callback from cp_walk_tree. */
1941
1942	static tree
1943	transform_await_wrapper (tree *stmt, int *do_subtree, void *d)
1944	{
1945	/* Set actor function as new DECL_CONTEXT of label_decl. */
1946	struct await_xform_data *xform = (struct await_xform_data *) d;
1947	if (TREE_CODE (*stmt) == LABEL_DECL
1948	&& DECL_CONTEXT (*stmt) != xform->actor_fn)
1949	DECL_CONTEXT (*stmt) = xform->actor_fn;
1950
1951	/* We should have already lowered co_yields to their co_await. */
1952	gcc_checking_assert (TREE_CODE (*stmt) != CO_YIELD_EXPR);
1953	if (TREE_CODE (*stmt) != CO_AWAIT_EXPR)
1954	return NULL_TREE;
1955
1956	tree await_expr = *stmt;
1957	*stmt = transform_await_expr (await_expr, xform);
1958	if (*stmt == error_mark_node)
1959	*do_subtree = 0;
1960	return NULL_TREE;
1961	}
1962
1963	/* This caches information that we determine about function params,
1964	their uses and copies in the coroutine frame. */
1965
1966	struct param_info
1967	{
1968	tree field_id; /* The name of the copy in the coroutine frame. */
1969	tree copy_var; /* The local var proxy for the frame copy. */
1970	vec<tree *> *body_uses; /* Worklist of uses, void if there are none. */
1971	tree frame_type; /* The type used to represent this parm in the frame. */
1972	tree orig_type; /* The original type of the parm (not as passed). */
1973	tree guard_var; /* If we need a DTOR on exception, this bool guards it. */
1974	tree fr_copy_dtor; /* If we need a DTOR on exception, this is it. */
1975	bool by_ref; /* Was passed by reference. */
1976	bool pt_ref; /* Was a pointer to object. */
1977	bool rv_ref; /* Was an rvalue ref. */
1978	bool trivial_dtor; /* The frame type has a trivial DTOR. */
1979	bool this_ptr; /* Is 'this' */
1980	bool lambda_cobj; /* Lambda capture object */
1981	};
1982
1983	struct local_var_info
1984	{
1985	tree field_id;
1986	tree field_idx;
1987	tree frame_type;
1988	bool is_lambda_capture;
1989	bool is_static;
1990	bool has_value_expr_p;
1991	location_t def_loc;
1992	};
1993
1994	/* For figuring out what local variable usage we have. */
1995	struct local_vars_transform
1996	{
1997	tree context;
1998	tree actor_frame;
1999	tree coro_frame_type;
2000	location_t loc;
2001	hash_map<tree, local_var_info> *local_var_uses;
2002	};
2003
2004	static tree
2005	transform_local_var_uses (tree *stmt, int *do_subtree, void *d)
2006	{
2007	local_vars_transform *lvd = (local_vars_transform *) d;
2008
2009	/* For each var in this bind expr (that has a frame id, which means it was
2010	accessed), build a frame reference and add it as the DECL_VALUE_EXPR. */
2011
2012	if (TREE_CODE (*stmt) == BIND_EXPR)
2013	{
2014	tree lvar;
2015	for (lvar = BIND_EXPR_VARS (*stmt); lvar != NULL;
2016	lvar = DECL_CHAIN (lvar))
2017	{
2018	bool existed;
2019	local_var_info &local_var
2020	= lvd->local_var_uses->get_or_insert (k: lvar, existed: &existed);
2021	gcc_checking_assert (existed);
2022
2023	/* Re-write the variable's context to be in the actor func. */
2024	DECL_CONTEXT (lvar) = lvd->context;
2025
2026	/* For capture proxies, this could include the decl value expr. */
2027	if (local_var.is_lambda_capture || local_var.has_value_expr_p)
2028	continue; /* No frame entry for this. */
2029
2030	/* TODO: implement selective generation of fields when vars are
2031	known not-used. */
2032	if (local_var.field_id == NULL_TREE)
2033	continue; /* Wasn't used. */
2034
2035	tree fld_ref
2036	= lookup_member (lvd->coro_frame_type, local_var.field_id,
2037	/*protect=*/1, /*want_type=*/0,
2038	tf_warning_or_error);
2039	tree fld_idx = build3 (COMPONENT_REF, TREE_TYPE (lvar),
2040	lvd->actor_frame, fld_ref, NULL_TREE);
2041	local_var.field_idx = fld_idx;
2042	SET_DECL_VALUE_EXPR (lvar, fld_idx);
2043	DECL_HAS_VALUE_EXPR_P (lvar) = true;
2044	}
2045	cp_walk_tree (&BIND_EXPR_BODY (*stmt), transform_local_var_uses, d, NULL);
2046	*do_subtree = 0; /* We've done the body already. */
2047	return NULL_TREE;
2048	}
2049	return NULL_TREE;
2050	}
2051
2052	/* A helper to build the frame DTOR.
2053	[dcl.fct.def.coroutine] / 12
2054	The deallocation function’s name is looked up in the scope of the promise
2055	type. If this lookup fails, the deallocation function’s name is looked up
2056	in the global scope. If deallocation function lookup finds both a usual
2057	deallocation function with only a pointer parameter and a usual
2058	deallocation function with both a pointer parameter and a size parameter,
2059	then the selected deallocation function shall be the one with two
2060	parameters. Otherwise, the selected deallocation function shall be the
2061	function with one parameter. If no usual deallocation function is found
2062	the program is ill-formed. The selected deallocation function shall be
2063	called with the address of the block of storage to be reclaimed as its
2064	first argument. If a deallocation function with a parameter of type
2065	std::size_t is used, the size of the block is passed as the corresponding
2066	argument. */
2067
2068	static tree
2069	coro_get_frame_dtor (tree coro_fp, tree orig, tree frame_size,
2070	tree promise_type, location_t loc)
2071	{
2072	tree del_coro_fr = NULL_TREE;
2073	tree frame_arg = build1 (CONVERT_EXPR, ptr_type_node, coro_fp);
2074	tree delname = ovl_op_identifier (isass: false, code: DELETE_EXPR);
2075	tree fns = lookup_promise_method (fndecl: orig, member_id: delname, loc,
2076	/*musthave=*/false);
2077	if (fns && BASELINK_P (fns))
2078	{
2079	/* Look for sized version first, since this takes precedence. */
2080	vec<tree, va_gc> *args = make_tree_vector ();
2081	vec_safe_push (v&: args, obj: frame_arg);
2082	vec_safe_push (v&: args, obj: frame_size);
2083	tree dummy_promise = build_dummy_object (promise_type);
2084
2085	/* It's OK to fail for this one... */
2086	del_coro_fr = build_new_method_call (dummy_promise, fns, &args,
2087	NULL_TREE, LOOKUP_NORMAL, NULL,
2088	tf_none);
2089
2090	if (!del_coro_fr || del_coro_fr == error_mark_node)
2091	{
2092	release_tree_vector (args);
2093	args = make_tree_vector_single (frame_arg);
2094	del_coro_fr = build_new_method_call (dummy_promise, fns, &args,
2095	NULL_TREE, LOOKUP_NORMAL, NULL,
2096	tf_none);
2097	}
2098
2099	/* But one of them must succeed, or the program is ill-formed. */
2100	if (!del_coro_fr || del_coro_fr == error_mark_node)
2101	{
2102	error_at (loc, "%qE is provided by %qT but is not usable with"
2103	" the function signature %qD", delname, promise_type, orig);
2104	del_coro_fr = error_mark_node;
2105	}
2106	}
2107	else
2108	{
2109	del_coro_fr = build_op_delete_call (DELETE_EXPR, frame_arg, frame_size,
2110	/*global_p=*/true, /*placement=*/NULL,
2111	/*alloc_fn=*/NULL,
2112	tf_warning_or_error);
2113	if (!del_coro_fr || del_coro_fr == error_mark_node)
2114	del_coro_fr = error_mark_node;
2115	}
2116	return del_coro_fr;
2117	}
2118
2119	/* The actor transform. */
2120
2121	static void
2122	build_actor_fn (location_t loc, tree coro_frame_type, tree actor, tree fnbody,
2123	tree orig, hash_map<tree, local_var_info> *local_var_uses,
2124	vec<tree, va_gc> *param_dtor_list,
2125	tree resume_idx_var, unsigned body_count, tree frame_size)
2126	{
2127	verify_stmt_tree (fnbody);
2128	/* Some things we inherit from the original function. */
2129	tree handle_type = get_coroutine_handle_type (decl: orig);
2130	tree promise_type = get_coroutine_promise_type (decl: orig);
2131	tree promise_proxy = get_coroutine_promise_proxy (decl: orig);
2132
2133	/* One param, the coro frame pointer. */
2134	tree actor_fp = DECL_ARGUMENTS (actor);
2135
2136	/* We have a definition here. */
2137	TREE_STATIC (actor) = 1;
2138
2139	tree actor_outer = push_stmt_list ();
2140	current_stmt_tree ()->stmts_are_full_exprs_p = 1;
2141	tree stmt = begin_compound_stmt (BCS_FN_BODY);
2142
2143	tree actor_bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
2144	tree top_block = make_node (BLOCK);
2145	BIND_EXPR_BLOCK (actor_bind) = top_block;
2146
2147	tree continuation = coro_build_artificial_var (loc, name: coro_actor_continue_id,
2148	type: void_coro_handle_type, ctx: actor,
2149	NULL_TREE);
2150
2151	BIND_EXPR_VARS (actor_bind) = continuation;
2152	BLOCK_VARS (top_block) = BIND_EXPR_VARS (actor_bind) ;
2153
2154	/* Link in the block associated with the outer scope of the re-written
2155	function body. */
2156	tree first = expr_first (fnbody);
2157	gcc_checking_assert (first && TREE_CODE (first) == BIND_EXPR);
2158	tree block = BIND_EXPR_BLOCK (first);
2159	gcc_checking_assert (BLOCK_SUPERCONTEXT (block) == NULL_TREE);
2160	gcc_checking_assert (BLOCK_CHAIN (block) == NULL_TREE);
2161	BLOCK_SUPERCONTEXT (block) = top_block;
2162	BLOCK_SUBBLOCKS (top_block) = block;
2163
2164	add_stmt (actor_bind);
2165	tree actor_body = push_stmt_list ();
2166
2167	/* The entry point for the actor code from the ramp. */
2168	tree actor_begin_label
2169	= create_named_label_with_ctx (loc, name: "actor.begin", ctx: actor);
2170	tree actor_frame = build1_loc (loc, code: INDIRECT_REF, type: coro_frame_type, arg1: actor_fp);
2171
2172	/* Declare the continuation handle. */
2173	add_decl_expr (continuation);
2174
2175	/* Re-write local vars, similarly. */
2176	local_vars_transform xform_vars_data
2177	= {.context: actor, .actor_frame: actor_frame, .coro_frame_type: coro_frame_type, .loc: loc, .local_var_uses: local_var_uses};
2178	cp_walk_tree (&fnbody, transform_local_var_uses, &xform_vars_data, NULL);
2179
2180	tree rat_field = lookup_member (coro_frame_type, coro_resume_index_id,
2181	1, 0, tf_warning_or_error);
2182	tree rat = build3 (COMPONENT_REF, short_unsigned_type_node, actor_frame,
2183	rat_field, NULL_TREE);
2184
2185	tree ret_label
2186	= create_named_label_with_ctx (loc, name: "actor.suspend.ret", ctx: actor);
2187
2188	tree continue_label
2189	= create_named_label_with_ctx (loc, name: "actor.continue.ret", ctx: actor);
2190
2191	tree lsb_if = begin_if_stmt ();
2192	tree chkb0 = build2 (BIT_AND_EXPR, short_unsigned_type_node, rat,
2193	build_int_cst (short_unsigned_type_node, 1));
2194	chkb0 = build2 (NE_EXPR, short_unsigned_type_node, chkb0,
2195	build_int_cst (short_unsigned_type_node, 0));
2196	finish_if_stmt_cond (chkb0, lsb_if);
2197
2198	tree destroy_dispatcher = begin_switch_stmt ();
2199	finish_switch_cond (rat, destroy_dispatcher);
2200	tree ddeflab = build_case_label (NULL_TREE, NULL_TREE,
2201	create_anon_label_with_ctx (loc, ctx: actor));
2202	add_stmt (ddeflab);
2203	tree b = build_call_expr_loc (loc, builtin_decl_explicit (fncode: BUILT_IN_TRAP), 0);
2204	b = coro_build_cvt_void_expr_stmt (expr: b, loc);
2205	add_stmt (b);
2206
2207	/* The destroy point numbered #1 is special, in that it is reached from a
2208	coroutine that is suspended after re-throwing from unhandled_exception().
2209	This label just invokes the cleanup of promise, param copies and the
2210	frame itself. */
2211	tree del_promise_label
2212	= create_named_label_with_ctx (loc, name: "coro.delete.promise", ctx: actor);
2213	b = build_case_label (build_int_cst (short_unsigned_type_node, 1), NULL_TREE,
2214	create_anon_label_with_ctx (loc, ctx: actor));
2215	add_stmt (b);
2216	add_stmt (build_stmt (loc, GOTO_EXPR, del_promise_label));
2217
2218	short unsigned lab_num = 3;
2219	for (unsigned destr_pt = 0; destr_pt < body_count; destr_pt++)
2220	{
2221	tree l_num = build_int_cst (short_unsigned_type_node, lab_num);
2222	b = build_case_label (l_num, NULL_TREE,
2223	create_anon_label_with_ctx (loc, ctx: actor));
2224	add_stmt (b);
2225	b = build_call_expr_internal_loc (loc, IFN_CO_ACTOR, void_type_node, 1,
2226	l_num);
2227	b = coro_build_cvt_void_expr_stmt (expr: b, loc);
2228	add_stmt (b);
2229	b = build1 (GOTO_EXPR, void_type_node, CASE_LABEL (ddeflab));
2230	add_stmt (b);
2231	lab_num += 2;
2232	}
2233
2234	/* Insert the prototype dispatcher. */
2235	finish_switch_stmt (destroy_dispatcher);
2236
2237	finish_then_clause (lsb_if);
2238	begin_else_clause (lsb_if);
2239
2240	tree dispatcher = begin_switch_stmt ();
2241	finish_switch_cond (rat, dispatcher);
2242	b = build_case_label (build_int_cst (short_unsigned_type_node, 0), NULL_TREE,
2243	create_anon_label_with_ctx (loc, ctx: actor));
2244	add_stmt (b);
2245	b = build1 (GOTO_EXPR, void_type_node, actor_begin_label);
2246	add_stmt (b);
2247
2248	tree rdeflab = build_case_label (NULL_TREE, NULL_TREE,
2249	create_anon_label_with_ctx (loc, ctx: actor));
2250	add_stmt (rdeflab);
2251	b = build_call_expr_loc (loc, builtin_decl_explicit (fncode: BUILT_IN_TRAP), 0);
2252	b = coro_build_cvt_void_expr_stmt (expr: b, loc);
2253	add_stmt (b);
2254
2255	lab_num = 2;
2256	/* The final resume should be made to hit the default (trap, UB) entry
2257	although it will be unreachable via the normal entry point, since that
2258	is set to NULL on reaching final suspend. */
2259	for (unsigned resu_pt = 0; resu_pt < body_count; resu_pt++)
2260	{
2261	tree l_num = build_int_cst (short_unsigned_type_node, lab_num);
2262	b = build_case_label (l_num, NULL_TREE,
2263	create_anon_label_with_ctx (loc, ctx: actor));
2264	add_stmt (b);
2265	b = build_call_expr_internal_loc (loc, IFN_CO_ACTOR, void_type_node, 1,
2266	l_num);
2267	b = coro_build_cvt_void_expr_stmt (expr: b, loc);
2268	add_stmt (b);
2269	b = build1 (GOTO_EXPR, void_type_node, CASE_LABEL (rdeflab));
2270	add_stmt (b);
2271	lab_num += 2;
2272	}
2273
2274	/* Insert the prototype dispatcher. */
2275	finish_switch_stmt (dispatcher);
2276	finish_else_clause (lsb_if);
2277
2278	finish_if_stmt (lsb_if);
2279
2280	tree r = build_stmt (loc, LABEL_EXPR, actor_begin_label);
2281	add_stmt (r);
2282
2283	/* actor's coroutine 'self handle'. */
2284	tree ash_m = lookup_member (coro_frame_type, coro_self_handle_id, 1,
2285	0, tf_warning_or_error);
2286	tree ash = build_class_member_access_expr (actor_frame, ash_m, NULL_TREE,
2287	false, tf_warning_or_error);
2288	/* So construct the self-handle from the frame address. */
2289	tree hfa_m = lookup_member (handle_type, coro_from_address_identifier, 1,
2290	0, tf_warning_or_error);
2291
2292	r = build1 (CONVERT_EXPR, build_pointer_type (void_type_node), actor_fp);
2293	vec<tree, va_gc> *args = make_tree_vector_single (r);
2294	tree hfa = build_new_method_call (ash, hfa_m, &args, NULL_TREE, LOOKUP_NORMAL,
2295	NULL, tf_warning_or_error);
2296	r = cp_build_init_expr (t: ash, i: hfa);
2297	r = coro_build_cvt_void_expr_stmt (expr: r, loc);
2298	add_stmt (r);
2299	release_tree_vector (args);
2300
2301	/* Now we know the real promise, and enough about the frame layout to
2302	decide where to put things. */
2303
2304	await_xform_data xform = {.actor_fn: actor, .actor_frame: actor_frame};
2305
2306	/* Transform the await expressions in the function body. Only do each
2307	await tree once! */
2308	hash_set<tree> pset;
2309	cp_walk_tree (&fnbody, transform_await_wrapper, &xform, &pset);
2310
2311	/* Add in our function body with the co_returns rewritten to final form. */
2312	add_stmt (fnbody);
2313
2314	/* now do the tail of the function. */
2315	r = build_stmt (loc, LABEL_EXPR, del_promise_label);
2316	add_stmt (r);
2317
2318	/* Destructors for the things we built explicitly. */
2319	if (tree c = cxx_maybe_build_cleanup (promise_proxy, tf_warning_or_error))
2320	add_stmt (c);
2321
2322	tree del_frame_label
2323	= create_named_label_with_ctx (loc, name: "coro.delete.frame", ctx: actor);
2324	r = build_stmt (loc, LABEL_EXPR, del_frame_label);
2325	add_stmt (r);
2326
2327	/* Here deallocate the frame (if we allocated it), which we will have at
2328	present. */
2329	tree fnf_m
2330	= lookup_member (coro_frame_type, coro_frame_needs_free_id, 1,
2331	0, tf_warning_or_error);
2332	tree fnf2_x = build_class_member_access_expr (actor_frame, fnf_m, NULL_TREE,
2333	false, tf_warning_or_error);
2334
2335	tree need_free_if = begin_if_stmt ();
2336	fnf2_x = build1 (CONVERT_EXPR, integer_type_node, fnf2_x);
2337	tree cmp = build2 (NE_EXPR, integer_type_node, fnf2_x, integer_zero_node);
2338	finish_if_stmt_cond (cmp, need_free_if);
2339	if (param_dtor_list != NULL)
2340	{
2341	int i;
2342	tree pid;
2343	FOR_EACH_VEC_ELT (*param_dtor_list, i, pid)
2344	{
2345	tree m
2346	= lookup_member (coro_frame_type, pid, 1, 0, tf_warning_or_error);
2347	tree a = build_class_member_access_expr (actor_frame, m, NULL_TREE,
2348	false, tf_warning_or_error);
2349	if (tree dtor = cxx_maybe_build_cleanup (a, tf_warning_or_error))
2350	add_stmt (dtor);
2351	}
2352	}
2353
2354	/* Build the frame DTOR. */
2355	tree del_coro_fr = coro_get_frame_dtor (coro_fp: actor_fp, orig, frame_size,
2356	promise_type, loc);
2357	finish_expr_stmt (del_coro_fr);
2358	finish_then_clause (need_free_if);
2359	tree scope = IF_SCOPE (need_free_if);
2360	IF_SCOPE (need_free_if) = NULL;
2361	r = do_poplevel (scope);
2362	add_stmt (r);
2363
2364	/* done. */
2365	r = build_stmt (loc, RETURN_EXPR, NULL);
2366	suppress_warning (r); /* We don't want a warning about this. */
2367	r = maybe_cleanup_point_expr_void (r);
2368	add_stmt (r);
2369
2370	/* This is the suspend return point. */
2371	r = build_stmt (loc, LABEL_EXPR, ret_label);
2372	add_stmt (r);
2373
2374	r = build_stmt (loc, RETURN_EXPR, NULL);
2375	suppress_warning (r); /* We don't want a warning about this. */
2376	r = maybe_cleanup_point_expr_void (r);
2377	add_stmt (r);
2378
2379	/* This is the 'continuation' return point. For such a case we have a coro
2380	handle (from the await_suspend() call) and we want handle.resume() to
2381	execute as a tailcall allowing arbitrary chaining of coroutines. */
2382	r = build_stmt (loc, LABEL_EXPR, continue_label);
2383	add_stmt (r);
2384
2385	/* We want to force a tail-call even for O0/1, so this expands the resume
2386	call into its underlying implementation. */
2387	tree addr = lookup_member (void_coro_handle_type, coro_address_identifier,
2388	1, 0, tf_warning_or_error);
2389	addr = build_new_method_call (continuation, addr, NULL, NULL_TREE,
2390	LOOKUP_NORMAL, NULL, tf_warning_or_error);
2391	tree resume = build_call_expr_loc
2392	(loc, builtin_decl_explicit (fncode: BUILT_IN_CORO_RESUME), 1, addr);
2393
2394	/* In order to support an arbitrary number of coroutine continuations,
2395	we must tail call them. However, some targets do not support indirect
2396	tail calls to arbitrary callees. See PR94359. */
2397	CALL_EXPR_TAILCALL (resume) = true;
2398	resume = coro_build_cvt_void_expr_stmt (expr: resume, loc);
2399	add_stmt (resume);
2400
2401	r = build_stmt (loc, RETURN_EXPR, NULL);
2402	gcc_checking_assert (maybe_cleanup_point_expr_void (r) == r);
2403	add_stmt (r);
2404
2405	/* We've now rewritten the tree and added the initial and final
2406	co_awaits. Now pass over the tree and expand the co_awaits. */
2407
2408	coro_aw_data data = {.actor_fn: actor, .coro_fp: actor_fp, .resume_idx: resume_idx_var, NULL_TREE,
2409	.self_h: ash, .cleanup: del_promise_label, .cororet: ret_label,
2410	.corocont: continue_label, .conthand: continuation, .index: 2};
2411	cp_walk_tree (&actor_body, await_statement_expander, &data, NULL);
2412
2413	BIND_EXPR_BODY (actor_bind) = pop_stmt_list (actor_body);
2414	TREE_SIDE_EFFECTS (actor_bind) = true;
2415
2416	finish_compound_stmt (stmt);
2417	DECL_SAVED_TREE (actor) = pop_stmt_list (actor_outer);
2418	verify_stmt_tree (DECL_SAVED_TREE (actor));
2419	}
2420
2421	/* The prototype 'destroy' function :
2422	frame->__Coro_resume_index |= 1;
2423	actor (frame); */
2424
2425	static void
2426	build_destroy_fn (location_t loc, tree coro_frame_type, tree destroy,
2427	tree actor)
2428	{
2429	/* One param, the coro frame pointer. */
2430	tree destr_fp = DECL_ARGUMENTS (destroy);
2431
2432	/* We have a definition here. */
2433	TREE_STATIC (destroy) = 1;
2434
2435	tree destr_outer = push_stmt_list ();
2436	current_stmt_tree ()->stmts_are_full_exprs_p = 1;
2437	tree dstr_stmt = begin_compound_stmt (BCS_FN_BODY);
2438
2439	tree destr_frame = build1 (INDIRECT_REF, coro_frame_type, destr_fp);
2440
2441	tree rat_field = lookup_member (coro_frame_type, coro_resume_index_id,
2442	1, 0, tf_warning_or_error);
2443	tree rat = build3 (COMPONENT_REF, short_unsigned_type_node,
2444	destr_frame, rat_field, NULL_TREE);
2445
2446	/* _resume_at |= 1 */
2447	tree dstr_idx = build2 (BIT_IOR_EXPR, short_unsigned_type_node, rat,
2448	build_int_cst (short_unsigned_type_node, 1));
2449	tree r = build2 (MODIFY_EXPR, short_unsigned_type_node, rat, dstr_idx);
2450	r = coro_build_cvt_void_expr_stmt (expr: r, loc);
2451	add_stmt (r);
2452
2453	/* So .. call the actor .. */
2454	r = build_call_expr_loc (loc, actor, 1, destr_fp);
2455	r = coro_build_cvt_void_expr_stmt (expr: r, loc);
2456	add_stmt (r);
2457
2458	/* done. */
2459	r = build_stmt (loc, RETURN_EXPR, NULL);
2460	r = maybe_cleanup_point_expr_void (r);
2461	add_stmt (r);
2462
2463	finish_compound_stmt (dstr_stmt);
2464	DECL_SAVED_TREE (destroy) = pop_stmt_list (destr_outer);
2465	}
2466
2467	/* Helper that returns an identifier for an appended extension to the
2468	current un-mangled function name. */
2469
2470	static tree
2471	get_fn_local_identifier (tree orig, const char *append)
2472	{
2473	/* Figure out the bits we need to generate names for the outlined things
2474	For consistency, this needs to behave the same way as
2475	ASM_FORMAT_PRIVATE_NAME does. */
2476	tree nm = DECL_NAME (orig);
2477	const char *sep, *pfx = "";
2478	#ifndef NO_DOT_IN_LABEL
2479	sep = ".";
2480	#else
2481	#ifndef NO_DOLLAR_IN_LABEL
2482	sep = "$";
2483	#else
2484	sep = "_";
2485	pfx = "__";
2486	#endif
2487	#endif
2488
2489	char *an;
2490	if (DECL_ASSEMBLER_NAME (orig))
2491	an = ACONCAT ((IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (orig)), sep, append,
2492	(char *) 0));
2493	else if (DECL_USE_TEMPLATE (orig) && DECL_TEMPLATE_INFO (orig)
2494	&& DECL_TI_ARGS (orig))
2495	{
2496	tree tpl_args = DECL_TI_ARGS (orig);
2497	an = ACONCAT ((pfx, IDENTIFIER_POINTER (nm), (char *) 0));
2498	for (int i = 0; i < TREE_VEC_LENGTH (tpl_args); ++i)
2499	{
2500	tree typ = DECL_NAME (TYPE_NAME (TREE_VEC_ELT (tpl_args, i)));
2501	an = ACONCAT ((an, sep, IDENTIFIER_POINTER (typ), (char *) 0));
2502	}
2503	an = ACONCAT ((an, sep, append, (char *) 0));
2504	}
2505	else
2506	an = ACONCAT ((pfx, IDENTIFIER_POINTER (nm), sep, append, (char *) 0));
2507
2508	return get_identifier (an);
2509	}
2510
2511	/* Build an initial or final await initialized from the promise
2512	initial_suspend or final_suspend expression. */
2513
2514	static tree
2515	build_init_or_final_await (location_t loc, bool is_final)
2516	{
2517	tree suspend_alt = is_final ? coro_final_suspend_identifier
2518	: coro_initial_suspend_identifier;
2519
2520	tree setup_call
2521	= coro_build_promise_expression (fn: current_function_decl, NULL, member_id: suspend_alt,
2522	loc, NULL, /*musthave=*/true);
2523
2524	/* Check for noexcept on the final_suspend call. */
2525	if (flag_exceptions && is_final && setup_call != error_mark_node
2526	&& coro_diagnose_throwing_final_aw_expr (expr: setup_call))
2527	return error_mark_node;
2528
2529	/* So build the co_await for this */
2530	/* For initial/final suspends the call is "a" per [expr.await] 3.2. */
2531	return build_co_await (loc, a: setup_call, suspend_kind: (is_final ? FINAL_SUSPEND_POINT
2532	: INITIAL_SUSPEND_POINT));
2533	}
2534
2535	/* Callback to record the essential data for each await point found in the
2536	function. */
2537
2538	static bool
2539	register_await_info (tree await_expr, tree aw_type, tree aw_nam)
2540	{
2541	bool seen;
2542	suspend_point_info &s
2543	= suspend_points->get_or_insert (k: await_expr, existed: &seen);
2544	if (seen)
2545	{
2546	warning_at (EXPR_LOCATION (await_expr), 0, "duplicate info for %qE",
2547	await_expr);
2548	return false;
2549	}
2550	s.awaitable_type = aw_type;
2551	s.await_field_id = aw_nam;
2552	return true;
2553	}
2554
2555	/* This data set is used when analyzing statements for await expressions. */
2556
2557	struct susp_frame_data
2558	{
2559	/* Function-wide. */
2560	tree *field_list; /* The current coroutine frame field list. */
2561	tree handle_type; /* The self-handle type for this coroutine. */
2562	tree fs_label; /* The destination for co_returns. */
2563	vec<tree, va_gc> *block_stack; /* Track block scopes. */
2564	vec<tree, va_gc> *bind_stack; /* Track current bind expr. */
2565	unsigned await_number; /* Which await in the function. */
2566	unsigned cond_number; /* Which replaced condition in the fn. */
2567	/* Temporary values for one statement or expression being analyzed. */
2568	hash_set<tree> captured_temps; /* The suspend captured these temps. */
2569	vec<tree, va_gc> *to_replace; /* The VAR decls to replace. */
2570	hash_set<tree> *truth_aoif_to_expand; /* The set of TRUTH exprs to expand. */
2571	unsigned saw_awaits; /* Count of awaits in this statement */
2572	bool captures_temporary; /* This expr captures temps by ref. */
2573	bool needs_truth_if_exp; /* We must expand a truth_if expression. */
2574	bool has_awaiter_init; /* We must handle initializing an awaiter. */
2575	};
2576
2577	/* If this is an await expression, then count it (both uniquely within the
2578	function and locally within a single statement). */
2579
2580	static tree
2581	register_awaits (tree *stmt, int *, void *d)
2582	{
2583	tree aw_expr = *stmt;
2584
2585	/* We should have already lowered co_yields to their co_await. */
2586	gcc_checking_assert (TREE_CODE (aw_expr) != CO_YIELD_EXPR);
2587
2588	if (TREE_CODE (aw_expr) != CO_AWAIT_EXPR)
2589	return NULL_TREE;
2590
2591	/* Count how many awaits the current expression contains. */
2592	susp_frame_data *data = (susp_frame_data *) d;
2593	data->saw_awaits++;
2594	/* Each await suspend context is unique, this is a function-wide value. */
2595	data->await_number++;
2596
2597	/* Awaitables should either be user-locals or promoted to coroutine frame
2598	entries at this point, and their initializers should have been broken
2599	out. */
2600	tree aw = TREE_OPERAND (aw_expr, 1);
2601	gcc_checking_assert (!TREE_OPERAND (aw_expr, 2));
2602
2603	tree aw_field_type = TREE_TYPE (aw);
2604	tree aw_field_nam = NULL_TREE;
2605	register_await_info (await_expr: aw_expr, aw_type: aw_field_type, aw_nam: aw_field_nam);
2606
2607	/* Rewrite target expressions on the await_suspend () to remove extraneous
2608	cleanups for the awaitables, which are now promoted to frame vars and
2609	managed via that. */
2610	tree v = TREE_OPERAND (aw_expr, 3);
2611	tree o = TREE_VEC_ELT (v, 1);
2612	if (TREE_CODE (o) == TARGET_EXPR)
2613	TREE_VEC_ELT (v, 1) = get_target_expr (TREE_OPERAND (o, 1));
2614	return NULL_TREE;
2615	}
2616
2617	/* There are cases where any await expression is relevant. */
2618	static tree
2619	find_any_await (tree *stmt, int *dosub, void *d)
2620	{
2621	if (TREE_CODE (*stmt) == CO_AWAIT_EXPR)
2622	{
2623	*dosub = 0; /* We don't need to consider this any further. */
2624	tree **p = (tree **) d;
2625	*p = stmt;
2626	return *stmt;
2627	}
2628	return NULL_TREE;
2629	}
2630
2631	static bool
2632	tmp_target_expr_p (tree t)
2633	{
2634	if (TREE_CODE (t) != TARGET_EXPR)
2635	return false;
2636	tree v = TREE_OPERAND (t, 0);
2637	if (!DECL_ARTIFICIAL (v))
2638	return false;
2639	if (DECL_NAME (v))
2640	return false;
2641	return true;
2642	}
2643
2644	/* Structure to record sub-expressions that need to be handled by the
2645	statement flattener. */
2646
2647	struct coro_interesting_subtree
2648	{
2649	tree* entry;
2650	hash_set<tree> *temps_used;
2651	};
2652
2653	/* tree-walk callback that returns the first encountered sub-expression of
2654	a kind that needs to be handled specifically by the statement flattener. */
2655
2656	static tree
2657	find_interesting_subtree (tree *expr_p, int *dosub, void *d)
2658	{
2659	tree expr = *expr_p;
2660	coro_interesting_subtree *p = (coro_interesting_subtree *)d;
2661	if (TREE_CODE (expr) == CO_AWAIT_EXPR)
2662	{
2663	*dosub = 0; /* We don't need to consider this any further. */
2664	if (TREE_OPERAND (expr, 2))
2665	{
2666	p->entry = expr_p;
2667	return expr;
2668	}
2669	}
2670	else if (tmp_target_expr_p (t: expr)
2671	&& !TARGET_EXPR_ELIDING_P (expr)
2672	&& !p->temps_used->contains (k: expr))
2673	{
2674	p->entry = expr_p;
2675	return expr;
2676	}
2677
2678	return NULL_TREE;
2679	}
2680
2681	/* Node for a doubly-linked list of promoted variables and their
2682	initializers. When the initializer is a conditional expression
2683	the 'then' and 'else' clauses are represented by a linked list
2684	attached to then_cl and else_cl respectively. */
2685
2686	struct var_nest_node
2687	{
2688	var_nest_node () = default;
2689	var_nest_node (tree v, tree i, var_nest_node *p, var_nest_node *n)
2690	: var(v), init(i), prev(p), next(n), then_cl (NULL), else_cl (NULL)
2691	{
2692	if (p)
2693	p->next = this;
2694	if (n)
2695	n->prev = this;
2696	}
2697	tree var;
2698	tree init;
2699	var_nest_node *prev;
2700	var_nest_node *next;
2701	var_nest_node *then_cl;
2702	var_nest_node *else_cl;
2703	};
2704
2705	/* This is called for single statements from the co-await statement walker.
2706	It checks to see if the statement contains any initializers for awaitables
2707	and if any of these capture items by reference. */
2708
2709	static void
2710	flatten_await_stmt (var_nest_node *n, hash_set<tree> *promoted,
2711	hash_set<tree> *temps_used, tree *replace_in)
2712	{
2713	bool init_expr = false;
2714	switch (TREE_CODE (n->init))
2715	{
2716	default: break;
2717	/* Compound expressions must be flattened specifically. */
2718	case COMPOUND_EXPR:
2719	{
2720	tree first = TREE_OPERAND (n->init, 0);
2721	n->init = TREE_OPERAND (n->init, 1);
2722	var_nest_node *ins
2723	= new var_nest_node(NULL_TREE, first, n->prev, n);
2724	/* The compiler (but not the user) can generate temporaries with
2725	uses in the second arm of a compound expr. */
2726	flatten_await_stmt (n: ins, promoted, temps_used, replace_in: &n->init);
2727	flatten_await_stmt (n, promoted, temps_used, NULL);
2728	/* The two arms have been processed separately. */
2729	return;
2730	}
2731	break;
2732	/* Handle conditional expressions. */
2733	case INIT_EXPR:
2734	init_expr = true;
2735	/* FALLTHROUGH */
2736	case MODIFY_EXPR:
2737	{
2738	tree old_expr = TREE_OPERAND (n->init, 1);
2739	if (TREE_CODE (old_expr) == COMPOUND_EXPR)
2740	{
2741	tree first = TREE_OPERAND (old_expr, 0);
2742	TREE_OPERAND (n->init, 1) = TREE_OPERAND (old_expr, 1);
2743	var_nest_node *ins
2744	= new var_nest_node(NULL_TREE, first, n->prev, n);
2745	flatten_await_stmt (n: ins, promoted, temps_used,
2746	replace_in: &TREE_OPERAND (n->init, 1));
2747	flatten_await_stmt (n, promoted, temps_used, NULL);
2748	return;
2749	}
2750	if (TREE_CODE (old_expr) != COND_EXPR)
2751	break;
2752	/* Reconstruct x = t ? y : z;
2753	as (void) t ? x = y : x = z; */
2754	tree var = TREE_OPERAND (n->init, 0);
2755	tree var_type = TREE_TYPE (var);
2756	tree cond = COND_EXPR_COND (old_expr);
2757	/* We are allowed a void type throw in one or both of the cond
2758	expr arms. */
2759	tree then_cl = COND_EXPR_THEN (old_expr);
2760	if (!VOID_TYPE_P (TREE_TYPE (then_cl)))
2761	{
2762	gcc_checking_assert (TREE_CODE (then_cl) != STATEMENT_LIST);
2763	if (init_expr)
2764	then_cl = cp_build_init_expr (t: var, i: then_cl);
2765	else
2766	then_cl = build2 (MODIFY_EXPR, var_type, var, then_cl);
2767	}
2768	tree else_cl = COND_EXPR_ELSE (old_expr);
2769	if (!VOID_TYPE_P (TREE_TYPE (else_cl)))
2770	{
2771	gcc_checking_assert (TREE_CODE (else_cl) != STATEMENT_LIST);
2772	if (init_expr)
2773	else_cl = cp_build_init_expr (t: var, i: else_cl);
2774	else
2775	else_cl = build2 (MODIFY_EXPR, var_type, var, else_cl);
2776	}
2777	n->init = build3 (COND_EXPR, var_type, cond, then_cl, else_cl);
2778	}
2779	/* FALLTHROUGH */
2780	case COND_EXPR:
2781	{
2782	tree *found;
2783	tree cond = COND_EXPR_COND (n->init);
2784	/* If the condition contains an await expression, then we need to
2785	set that first and use a separate var. */
2786	if (cp_walk_tree (&cond, find_any_await, &found, NULL))
2787	{
2788	tree cond_type = TREE_TYPE (cond);
2789	tree cond_var = build_lang_decl (VAR_DECL, NULL_TREE, cond_type);
2790	DECL_ARTIFICIAL (cond_var) = true;
2791	layout_decl (cond_var, 0);
2792	gcc_checking_assert (!TYPE_NEEDS_CONSTRUCTING (cond_type));
2793	cond = cp_build_init_expr (t: cond_var, i: cond);
2794	var_nest_node *ins
2795	= new var_nest_node (cond_var, cond, n->prev, n);
2796	COND_EXPR_COND (n->init) = cond_var;
2797	flatten_await_stmt (n: ins, promoted, temps_used, NULL);
2798	}
2799
2800	n->then_cl
2801	= new var_nest_node (n->var, COND_EXPR_THEN (n->init), NULL, NULL);
2802	n->else_cl
2803	= new var_nest_node (n->var, COND_EXPR_ELSE (n->init), NULL, NULL);
2804	flatten_await_stmt (n: n->then_cl, promoted, temps_used, NULL);
2805	/* Point to the start of the flattened code. */
2806	while (n->then_cl->prev)
2807	n->then_cl = n->then_cl->prev;
2808	flatten_await_stmt (n: n->else_cl, promoted, temps_used, NULL);
2809	while (n->else_cl->prev)
2810	n->else_cl = n->else_cl->prev;
2811	return;
2812	}
2813	break;
2814	}
2815	coro_interesting_subtree v = { NULL, .temps_used: temps_used };
2816	tree t = cp_walk_tree (&n->init, find_interesting_subtree, (void *)&v, NULL);
2817	if (!t)
2818	return;
2819	switch (TREE_CODE (t))
2820	{
2821	default: break;
2822	case CO_AWAIT_EXPR:
2823	{
2824	/* Await expressions with initializers have a compiler-temporary
2825	as the awaitable. 'promote' this. */
2826	tree var = TREE_OPERAND (t, 1);
2827	bool already_present = promoted->add (k: var);
2828	gcc_checking_assert (!already_present);
2829	tree init = TREE_OPERAND (t, 2);
2830	switch (TREE_CODE (init))
2831	{
2832	default: break;
2833	case INIT_EXPR:
2834	case MODIFY_EXPR:
2835	{
2836	tree inner = TREE_OPERAND (init, 1);
2837	/* We can have non-lvalue-expressions here, but when we see
2838	a target expression, mark it as already used. */
2839	if (TREE_CODE (inner) == TARGET_EXPR)
2840	{
2841	temps_used->add (k: inner);
2842	gcc_checking_assert
2843	(TREE_CODE (TREE_OPERAND (inner, 1)) != COND_EXPR);
2844	}
2845	}
2846	break;
2847	case CALL_EXPR:
2848	/* If this is a call and not a CTOR, then we didn't expect it. */
2849	gcc_checking_assert
2850	(DECL_CONSTRUCTOR_P (TREE_OPERAND (CALL_EXPR_FN (init), 0)));
2851	break;
2852	}
2853	var_nest_node *ins = new var_nest_node (var, init, n->prev, n);
2854	TREE_OPERAND (t, 2) = NULL_TREE;
2855	flatten_await_stmt (n: ins, promoted, temps_used, NULL);
2856	flatten_await_stmt (n, promoted, temps_used, NULL);
2857	return;
2858	}
2859	break;
2860	case TARGET_EXPR:
2861	{
2862	/* We have a temporary; promote it, but allow for the idiom in code
2863	generated by the compiler like
2864	a = (target_expr produces temp, op uses temp). */
2865	tree init = t;
2866	temps_used->add (k: init);
2867	tree var_type = TREE_TYPE (init);
2868	char *buf = xasprintf ("T%03u", (unsigned) temps_used->elements ());
2869	tree var = build_lang_decl (VAR_DECL, get_identifier (buf), var_type);
2870	DECL_ARTIFICIAL (var) = true;
2871	free (ptr: buf);
2872	bool already_present = promoted->add (k: var);
2873	gcc_checking_assert (!already_present);
2874	tree inner = TREE_OPERAND (init, 1);
2875	gcc_checking_assert (TREE_CODE (inner) != COND_EXPR);
2876	init = cp_build_modify_expr (input_location, var, INIT_EXPR, init,
2877	tf_warning_or_error);
2878	/* Simplify for the case that we have an init containing the temp
2879	alone. */
2880	if (t == n->init && n->var == NULL_TREE)
2881	{
2882	n->var = var;
2883	proxy_replace pr = {TREE_OPERAND (t, 0), .to: var};
2884	cp_walk_tree (&init, replace_proxy, &pr, NULL);
2885	n->init = init;
2886	if (replace_in)
2887	cp_walk_tree (replace_in, replace_proxy, &pr, NULL);
2888	flatten_await_stmt (n, promoted, temps_used, NULL);
2889	}
2890	else
2891	{
2892	var_nest_node *ins
2893	= new var_nest_node (var, init, n->prev, n);
2894	/* We have to replace the target expr... */
2895	*v.entry = var;
2896	/* ... and any uses of its var. */
2897	proxy_replace pr = {TREE_OPERAND (t, 0), .to: var};
2898	cp_walk_tree (&n->init, replace_proxy, &pr, NULL);
2899	/* Compiler-generated temporaries can also have uses in
2900	following arms of compound expressions, which will be listed
2901	in 'replace_in' if present. */
2902	if (replace_in)
2903	cp_walk_tree (replace_in, replace_proxy, &pr, NULL);
2904	flatten_await_stmt (n: ins, promoted, temps_used, NULL);
2905	flatten_await_stmt (n, promoted, temps_used, NULL);
2906	}
2907	return;
2908	}
2909	break;
2910	}
2911	}
2912
2913	/* Helper for 'process_conditional' that handles recursion into nested
2914	conditionals. */
2915
2916	static void
2917	handle_nested_conditionals (var_nest_node *n, vec<tree>& list,
2918	hash_map<tree, tree>& map)
2919	{
2920	do
2921	{
2922	if (n->var && DECL_NAME (n->var))
2923	{
2924	list.safe_push (obj: n->var);
2925	if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (TREE_TYPE (n->var)))
2926	{
2927	bool existed;
2928	tree& flag = map.get_or_insert (k: n->var, existed: &existed);
2929	if (!existed)
2930	{
2931	/* We didn't see this var before and it needs a DTOR, so
2932	build a guard variable for it. */
2933	char *nam
2934	= xasprintf ("%s_guard",
2935	IDENTIFIER_POINTER (DECL_NAME (n->var)));
2936	flag = build_lang_decl (VAR_DECL, get_identifier (nam),
2937	boolean_type_node);
2938	free (ptr: nam);
2939	DECL_ARTIFICIAL (flag) = true;
2940	}
2941
2942	/* The initializer for this variable is replaced by a compound
2943	expression that performs the init and then records that the
2944	variable is live (and the DTOR should be run at the scope
2945	exit. */
2946	tree set_flag = cp_build_init_expr (t: flag, boolean_true_node);
2947	n->init
2948	= build2 (COMPOUND_EXPR, boolean_type_node, n->init, set_flag);
2949	}
2950	}
2951	if (TREE_CODE (n->init) == COND_EXPR)
2952	{
2953	tree new_then = push_stmt_list ();
2954	handle_nested_conditionals (n: n->then_cl, list, map);
2955	new_then = pop_stmt_list (new_then);
2956	tree new_else = push_stmt_list ();
2957	handle_nested_conditionals (n: n->else_cl, list, map);
2958	new_else = pop_stmt_list (new_else);
2959	tree new_if
2960	= build4 (IF_STMT, void_type_node, COND_EXPR_COND (n->init),
2961	new_then, new_else, NULL_TREE);
2962	add_stmt (new_if);
2963	}
2964	else
2965	finish_expr_stmt (n->init);
2966	n = n->next;
2967	} while (n);
2968	}
2969
2970	/* helper for 'maybe_promote_temps'.
2971
2972	When we have a conditional expression which might embed await expressions
2973	and/or promoted variables, we need to handle it appropriately.
2974
2975	The linked lists for the 'then' and 'else' clauses in a conditional node
2976	identify the promoted variables (but these cannot be wrapped in a regular
2977	cleanup).
2978
2979	So recurse through the lists and build up a composite list of captured vars.
2980	Declare these and any guard variables needed to decide if a DTOR should be
2981	run. Then embed the conditional into a try-finally expression that handles
2982	running each DTOR conditionally on its guard variable. */
2983
2984	static void
2985	process_conditional (var_nest_node *n, tree& vlist)
2986	{
2987	tree init = n->init;
2988	hash_map<tree, tree> var_flags;
2989	auto_vec<tree> var_list;
2990	tree new_then = push_stmt_list ();
2991	handle_nested_conditionals (n: n->then_cl, list&: var_list, map&: var_flags);
2992	new_then = pop_stmt_list (new_then);
2993	tree new_else = push_stmt_list ();
2994	handle_nested_conditionals (n: n->else_cl, list&: var_list, map&: var_flags);
2995	new_else = pop_stmt_list (new_else);
2996	/* Declare the vars. There are two loops so that the boolean flags are
2997	grouped in the frame. */
2998	for (unsigned i = 0; i < var_list.length(); i++)
2999	{
3000	tree var = var_list[i];
3001	DECL_CHAIN (var) = vlist;
3002	vlist = var;
3003	add_decl_expr (var);
3004	}
3005	/* Define the guard flags for variables that need a DTOR. */
3006	for (unsigned i = 0; i < var_list.length(); i++)
3007	{
3008	tree *flag = var_flags.get (k: var_list[i]);
3009	if (flag)
3010	{
3011	DECL_INITIAL (*flag) = boolean_false_node;
3012	DECL_CHAIN (*flag) = vlist;
3013	vlist = *flag;
3014	add_decl_expr (*flag);
3015	}
3016	}
3017	tree new_if
3018	= build4 (IF_STMT, void_type_node, COND_EXPR_COND (init),
3019	new_then, new_else, NULL_TREE);
3020	/* Build a set of conditional DTORs. */
3021	tree final_actions = push_stmt_list ();
3022	while (!var_list.is_empty())
3023	{
3024	tree var = var_list.pop ();
3025	tree *flag = var_flags.get (k: var);
3026	if (!flag)
3027	continue;
3028	if (tree cleanup = cxx_maybe_build_cleanup (var, tf_warning_or_error))
3029	{
3030	tree cond_cleanup = begin_if_stmt ();
3031	finish_if_stmt_cond (*flag, cond_cleanup);
3032	finish_expr_stmt (cleanup);
3033	finish_then_clause (cond_cleanup);
3034	finish_if_stmt (cond_cleanup);
3035	}
3036	}
3037	final_actions = pop_stmt_list (final_actions);
3038	tree try_finally
3039	= build2 (TRY_FINALLY_EXPR, void_type_node, new_if, final_actions);
3040	add_stmt (try_finally);
3041	}
3042
3043	/* Given *STMT, that contains at least one await expression.
3044
3045	The full expression represented in the original source code will contain
3046	suspension points, but it is still required that the lifetime of temporary
3047	values extends to the end of the expression.
3048
3049	We already have a mechanism to 'promote' user-authored local variables
3050	to a coroutine frame counterpart (which allows explicit management of the
3051	lifetime across suspensions). The transform here re-writes STMT into
3052	a bind expression, promotes temporary values into local variables in that
3053	and flattens the statement into a series of cleanups.
3054
3055	Conditional expressions are re-written to regular 'if' statements.
3056	The cleanups for variables initialized inside a conditional (including
3057	nested cases) are wrapped in a try-finally clause, with guard variables
3058	to determine which DTORs need to be run. */
3059
3060	static tree
3061	maybe_promote_temps (tree *stmt, void *d)
3062	{
3063	susp_frame_data *awpts = (susp_frame_data *) d;
3064
3065	location_t sloc = EXPR_LOCATION (*stmt);
3066	tree expr = *stmt;
3067	/* Strip off uninteresting wrappers. */
3068	if (TREE_CODE (expr) == CLEANUP_POINT_EXPR)
3069	expr = TREE_OPERAND (expr, 0);
3070	if (TREE_CODE (expr) == EXPR_STMT)
3071	expr = EXPR_STMT_EXPR (expr);
3072	if (TREE_CODE (expr) == CONVERT_EXPR
3073	&& VOID_TYPE_P (TREE_TYPE (expr)))
3074	expr = TREE_OPERAND (expr, 0);
3075	STRIP_NOPS (expr);
3076
3077	/* We walk the statement trees, flattening it into an ordered list of
3078	variables with initializers and fragments corresponding to compound
3079	expressions, truth or/and if and ternary conditionals. Conditional
3080	expressions carry a nested list of fragments for the then and else
3081	clauses. We anchor to the 'bottom' of the fragment list; we will write
3082	a cleanup nest with one shell for each variable initialized. */
3083	var_nest_node *root = new var_nest_node (NULL_TREE, expr, NULL, NULL);
3084	/* Check to see we didn't promote one twice. */
3085	hash_set<tree> promoted_vars;
3086	hash_set<tree> used_temps;
3087	flatten_await_stmt (n: root, promoted: &promoted_vars, temps_used: &used_temps, NULL);
3088
3089	gcc_checking_assert (root->next == NULL);
3090	tree vlist = NULL_TREE;
3091	var_nest_node *t = root;
3092	/* We build the bind scope expression from the bottom-up.
3093	EXPR_LIST holds the inner expression nest at the current cleanup
3094	level (becoming the final expression list when we've exhausted the
3095	number of sub-expression fragments). */
3096	tree expr_list = NULL_TREE;
3097	do
3098	{
3099	tree new_list = push_stmt_list ();
3100	/* When we have a promoted variable, then add that to the bind scope
3101	and initialize it. When there's no promoted variable, we just need
3102	to run the initializer.
3103	If the initializer is a conditional expression, we need to collect
3104	and declare any promoted variables nested within it. DTORs for such
3105	variables must be run conditionally too. */
3106	if (t->var)
3107	{
3108	tree var = t->var;
3109	DECL_CHAIN (var) = vlist;
3110	vlist = var;
3111	add_decl_expr (var);
3112	if (TREE_CODE (t->init) == COND_EXPR)
3113	process_conditional (n: t, vlist);
3114	else
3115	finish_expr_stmt (t->init);
3116	if (tree cleanup = cxx_maybe_build_cleanup (var, tf_warning_or_error))
3117	{
3118	tree cl = build_stmt (sloc, CLEANUP_STMT, expr_list, cleanup, var);
3119	add_stmt (cl); /* push this onto the level above. */
3120	}
3121	else if (expr_list)
3122	{
3123	if (TREE_CODE (expr_list) != STATEMENT_LIST)
3124	add_stmt (expr_list);
3125	else if (!tsi_end_p (i: tsi_start (t: expr_list)))
3126	add_stmt (expr_list);
3127	}
3128	}
3129	else
3130	{
3131	if (TREE_CODE (t->init) == COND_EXPR)
3132	process_conditional (n: t, vlist);
3133	else
3134	finish_expr_stmt (t->init);
3135	if (expr_list)
3136	{
3137	if (TREE_CODE (expr_list) != STATEMENT_LIST)
3138	add_stmt (expr_list);
3139	else if (!tsi_end_p (i: tsi_start (t: expr_list)))
3140	add_stmt (expr_list);
3141	}
3142	}
3143	expr_list = pop_stmt_list (new_list);
3144	var_nest_node *old = t;
3145	t = t->prev;
3146	delete old;
3147	} while (t);
3148
3149	/* Now produce the bind expression containing the 'promoted' temporaries
3150	as its variable list, and the cleanup nest as the statement. */
3151	tree await_bind = build3_loc (loc: sloc, code: BIND_EXPR, void_type_node,
3152	NULL, NULL, NULL);
3153	BIND_EXPR_BODY (await_bind) = expr_list;
3154	BIND_EXPR_VARS (await_bind) = nreverse (vlist);
3155	tree b_block = make_node (BLOCK);
3156	if (!awpts->block_stack->is_empty ())
3157	{
3158	tree s_block = awpts->block_stack->last ();
3159	if (s_block)
3160	{
3161	BLOCK_SUPERCONTEXT (b_block) = s_block;
3162	BLOCK_CHAIN (b_block) = BLOCK_SUBBLOCKS (s_block);
3163	BLOCK_SUBBLOCKS (s_block) = b_block;
3164	}
3165	}
3166	BLOCK_VARS (b_block) = BIND_EXPR_VARS (await_bind) ;
3167	BIND_EXPR_BLOCK (await_bind) = b_block;
3168	TREE_SIDE_EFFECTS (await_bind) = TREE_SIDE_EFFECTS (BIND_EXPR_BODY (await_bind));
3169	*stmt = await_bind;
3170	hash_set<tree> visited;
3171	return cp_walk_tree (stmt, register_awaits, d, &visited);
3172	}
3173
3174	/* Lightweight callback to determine two key factors:
3175	1) If the statement/expression contains any await expressions.
3176	2) If the statement/expression potentially requires a re-write to handle
3177	TRUTH_{AND,OR}IF_EXPRs since, in most cases, they will need expansion
3178	so that the await expressions are not processed in the case of the
3179	short-circuit arm.
3180
3181	CO_YIELD expressions are re-written to their underlying co_await. */
3182
3183	static tree
3184	analyze_expression_awaits (tree *stmt, int *do_subtree, void *d)
3185	{
3186	susp_frame_data *awpts = (susp_frame_data *) d;
3187
3188	switch (TREE_CODE (*stmt))
3189	{
3190	default: return NULL_TREE;
3191	case CO_YIELD_EXPR:
3192	/* co_yield is syntactic sugar, re-write it to co_await. */
3193	*stmt = TREE_OPERAND (*stmt, 1);
3194	/* FALLTHROUGH */
3195	case CO_AWAIT_EXPR:
3196	awpts->saw_awaits++;
3197	/* A non-null initializer for the awaiter means we need to expand. */
3198	if (TREE_OPERAND (*stmt, 2))
3199	awpts->has_awaiter_init = true;
3200	break;
3201	case TRUTH_ANDIF_EXPR:
3202	case TRUTH_ORIF_EXPR:
3203	{
3204	/* We don't need special action for awaits in the always-executed
3205	arm of a TRUTH_IF. */
3206	if (tree res = cp_walk_tree (&TREE_OPERAND (*stmt, 0),
3207	analyze_expression_awaits, d, NULL))
3208	return res;
3209	/* However, if there are await expressions on the conditionally
3210	executed branch, we must expand the TRUTH_IF to ensure that the
3211	expanded await expression control-flow is fully contained in the
3212	conditionally executed code. */
3213	unsigned aw_count = awpts->saw_awaits;
3214	if (tree res = cp_walk_tree (&TREE_OPERAND (*stmt, 1),
3215	analyze_expression_awaits, d, NULL))
3216	return res;
3217	if (awpts->saw_awaits > aw_count)
3218	{
3219	awpts->truth_aoif_to_expand->add (k: *stmt);
3220	awpts->needs_truth_if_exp = true;
3221	}
3222	/* We've done the sub-trees here. */
3223	*do_subtree = 0;
3224	}
3225	break;
3226	}
3227
3228	return NULL_TREE; /* Recurse until done. */
3229	}
3230
3231	/* Given *EXPR
3232	If EXPR contains a TRUTH_{AND,OR}IF_EXPR, TAOIE with an await expr on
3233	the conditionally executed branch, change this in a ternary operator.
3234
3235	bool not_expr = TAOIE == TRUTH_ORIF_EXPR ? NOT : NOP;
3236	not_expr (always-exec expr) ? conditionally-exec expr : not_expr;
3237
3238	Apply this recursively to the condition and the conditionally-exec
3239	branch. */
3240
3241	struct truth_if_transform {
3242	tree *orig_stmt;
3243	tree scratch_var;
3244	hash_set<tree> *truth_aoif_to_expand;
3245	};
3246
3247	static tree
3248	expand_one_truth_if (tree *expr, int *do_subtree, void *d)
3249	{
3250	truth_if_transform *xform = (truth_if_transform *) d;
3251
3252	bool needs_not = false;
3253	switch (TREE_CODE (*expr))
3254	{
3255	default: break;
3256	case TRUTH_ORIF_EXPR:
3257	needs_not = true;
3258	/* FALLTHROUGH */
3259	case TRUTH_ANDIF_EXPR:
3260	{
3261	if (!xform->truth_aoif_to_expand->contains (k: *expr))
3262	break;
3263
3264	location_t sloc = EXPR_LOCATION (*expr);
3265	/* Transform truth expression into a cond expression with
3266	* the always-executed arm as the condition.
3267	* the conditionally-executed arm as the then clause.
3268	* the 'else' clause is fixed: 'true' for ||,'false' for &&. */
3269	tree cond = TREE_OPERAND (*expr, 0);
3270	tree test1 = TREE_OPERAND (*expr, 1);
3271	tree fixed = needs_not ? boolean_true_node : boolean_false_node;
3272	if (needs_not)
3273	cond = build1 (TRUTH_NOT_EXPR, boolean_type_node, cond);
3274	tree cond_expr
3275	= build3_loc (loc: sloc, code: COND_EXPR, boolean_type_node,
3276	arg0: cond, arg1: test1, arg2: fixed);
3277	*expr = cond_expr;
3278	if (tree res = cp_walk_tree (&COND_EXPR_COND (*expr),
3279	expand_one_truth_if, d, NULL))
3280	return res;
3281	if (tree res = cp_walk_tree (&COND_EXPR_THEN (*expr),
3282	expand_one_truth_if, d, NULL))
3283	return res;
3284	/* We've manually processed necessary sub-trees here. */
3285	*do_subtree = 0;
3286	}
3287	break;
3288	}
3289	return NULL_TREE;
3290	}
3291
3292	/* Helper that adds a new variable of VAR_TYPE to a bind scope BIND, the
3293	name is made up from NAM_ROOT, NAM_VERS. */
3294
3295	static tree
3296	add_var_to_bind (tree& bind, tree var_type,
3297	const char *nam_root, unsigned nam_vers)
3298	{
3299	tree b_vars = BIND_EXPR_VARS (bind);
3300	/* Build a variable to hold the condition, this will be included in the
3301	frame as a local var. */
3302	char *nam = xasprintf ("__%s_%d", nam_root, nam_vers);
3303	tree newvar = build_lang_decl (VAR_DECL, get_identifier (nam), var_type);
3304	free (ptr: nam);
3305	DECL_CHAIN (newvar) = b_vars;
3306	BIND_EXPR_VARS (bind) = newvar;
3307	return newvar;
3308	}
3309
3310	/* Helper to build and add if (!cond) break; */
3311
3312	static void
3313	coro_build_add_if_not_cond_break (tree cond)
3314	{
3315	tree if_stmt = begin_if_stmt ();
3316	tree invert = build1 (TRUTH_NOT_EXPR, boolean_type_node, cond);
3317	finish_if_stmt_cond (invert, if_stmt);
3318	finish_break_stmt ();
3319	finish_then_clause (if_stmt);
3320	finish_if_stmt (if_stmt);
3321	}
3322
3323	/* Tree walk callback to replace continue statements with goto label. */
3324	static tree
3325	replace_continue (tree *stmt, int *do_subtree, void *d)
3326	{
3327	tree expr = *stmt;
3328	if (TREE_CODE (expr) == CLEANUP_POINT_EXPR)
3329	expr = TREE_OPERAND (expr, 0);
3330	if (CONVERT_EXPR_P (expr) && VOID_TYPE_P (expr))
3331	expr = TREE_OPERAND (expr, 0);
3332	STRIP_NOPS (expr);
3333	if (!STATEMENT_CLASS_P (expr))
3334	return NULL_TREE;
3335
3336	switch (TREE_CODE (expr))
3337	{
3338	/* Unless it's a special case, just walk the subtrees as usual. */
3339	default: return NULL_TREE;
3340
3341	case CONTINUE_STMT:
3342	{
3343	tree *label = (tree *)d;
3344	location_t loc = EXPR_LOCATION (expr);
3345	/* re-write a continue to goto label. */
3346	*stmt = build_stmt (loc, GOTO_EXPR, *label);
3347	*do_subtree = 0;
3348	return NULL_TREE;
3349	}
3350
3351	/* Statements that do not require recursion. */
3352	case DECL_EXPR:
3353	case BREAK_STMT:
3354	case GOTO_EXPR:
3355	case LABEL_EXPR:
3356	case CASE_LABEL_EXPR:
3357	case ASM_EXPR:
3358	/* These must break recursion. */
3359	case FOR_STMT:
3360	case WHILE_STMT:
3361	case DO_STMT:
3362	*do_subtree = 0;
3363	return NULL_TREE;
3364	}
3365	}
3366
3367	/* Tree walk callback to analyze, register and pre-process statements that
3368	contain await expressions. */
3369
3370	static tree
3371	await_statement_walker (tree *stmt, int *do_subtree, void *d)
3372	{
3373	tree res = NULL_TREE;
3374	susp_frame_data *awpts = (susp_frame_data *) d;
3375
3376	/* Process a statement at a time. */
3377	if (TREE_CODE (*stmt) == BIND_EXPR)
3378	{
3379	/* For conditional expressions, we might wish to add an artificial var
3380	to their containing bind expr. */
3381	vec_safe_push (v&: awpts->bind_stack, obj: *stmt);
3382	/* We might need to insert a new bind expression, and want to link it
3383	into the correct scope, so keep a note of the current block scope. */
3384	tree blk = BIND_EXPR_BLOCK (*stmt);
3385	vec_safe_push (v&: awpts->block_stack, obj: blk);
3386	res = cp_walk_tree (&BIND_EXPR_BODY (*stmt), await_statement_walker,
3387	d, NULL);
3388	awpts->block_stack->pop ();
3389	awpts->bind_stack->pop ();
3390	*do_subtree = 0; /* Done subtrees. */
3391	return res;
3392	}
3393	else if (TREE_CODE (*stmt) == STATEMENT_LIST)
3394	{
3395	for (tree &s : tsi_range (*stmt))
3396	{
3397	res = cp_walk_tree (&s, await_statement_walker,
3398	d, NULL);
3399	if (res)
3400	return res;
3401	}
3402	*do_subtree = 0; /* Done subtrees. */
3403	return NULL_TREE;
3404	}
3405
3406	/* We have something to be handled as a single statement. We have to handle
3407	a few statements specially where await statements have to be moved out of
3408	constructs. */
3409	tree expr = *stmt;
3410	if (TREE_CODE (*stmt) == CLEANUP_POINT_EXPR)
3411	expr = TREE_OPERAND (expr, 0);
3412	STRIP_NOPS (expr);
3413
3414	if (STATEMENT_CLASS_P (expr))
3415	switch (TREE_CODE (expr))
3416	{
3417	/* Unless it's a special case, just walk the subtrees as usual. */
3418	default: return NULL_TREE;
3419
3420	/* When we have a conditional expression, which contains one or more
3421	await expressions, we have to break the condition out into a
3422	regular statement so that the control flow introduced by the await
3423	transforms can be implemented. */
3424	case IF_STMT:
3425	{
3426	tree *await_ptr;
3427	hash_set<tree> visited;
3428	/* Transform 'if (cond with awaits) then stmt1 else stmt2' into
3429	bool cond = cond with awaits.
3430	if (cond) then stmt1 else stmt2. */
3431	tree if_stmt = *stmt;
3432	/* We treat the condition as if it was a stand-alone statement,
3433	to see if there are any await expressions which will be analyzed
3434	and registered. */
3435	if (!(cp_walk_tree (&IF_COND (if_stmt),
3436	find_any_await, &await_ptr, &visited)))
3437	return NULL_TREE; /* Nothing special to do here. */
3438
3439	gcc_checking_assert (!awpts->bind_stack->is_empty());
3440	tree& bind_expr = awpts->bind_stack->last ();
3441	tree newvar = add_var_to_bind (bind&: bind_expr, boolean_type_node,
3442	nam_root: "ifcd", nam_vers: awpts->cond_number++);
3443	tree insert_list = push_stmt_list ();
3444	tree cond_inner = IF_COND (if_stmt);
3445	if (TREE_CODE (cond_inner) == CLEANUP_POINT_EXPR)
3446	cond_inner = TREE_OPERAND (cond_inner, 0);
3447	add_decl_expr (newvar);
3448	location_t sloc = EXPR_LOCATION (IF_COND (if_stmt));
3449	/* We want to initialize the new variable with the expression
3450	that contains the await(s) and potentially also needs to
3451	have truth_if expressions expanded. */
3452	tree new_s = cp_build_init_expr (sloc, newvar, cond_inner);
3453	finish_expr_stmt (new_s);
3454	IF_COND (if_stmt) = newvar;
3455	add_stmt (if_stmt);
3456	*stmt = pop_stmt_list (insert_list);
3457	/* So now walk the new statement list. */
3458	res = cp_walk_tree (stmt, await_statement_walker, d, NULL);
3459	*do_subtree = 0; /* Done subtrees. */
3460	return res;
3461	}
3462	break;
3463	case FOR_STMT:
3464	{
3465	tree *await_ptr;
3466	hash_set<tree> visited;
3467	/* for loops only need special treatment if the condition or the
3468	iteration expression contain a co_await. */
3469	tree for_stmt = *stmt;
3470	/* At present, the FE always generates a separate initializer for
3471	the FOR_INIT_STMT, when the expression has an await. Check that
3472	this assumption holds in the future. */
3473	gcc_checking_assert
3474	(!(cp_walk_tree (&FOR_INIT_STMT (for_stmt), find_any_await,
3475	&await_ptr, &visited)));
3476
3477	visited.empty ();
3478	bool for_cond_await
3479	= cp_walk_tree (&FOR_COND (for_stmt), find_any_await,
3480	&await_ptr, &visited);
3481
3482	visited.empty ();
3483	bool for_expr_await
3484	= cp_walk_tree (&FOR_EXPR (for_stmt), find_any_await,
3485	&await_ptr, &visited);
3486
3487	/* If the condition has an await, then we will need to rewrite the
3488	loop as
3489	for (init expression;true;iteration expression) {
3490	condition = await expression;
3491	if (condition)
3492	break;
3493	...
3494	}
3495	*/
3496	if (for_cond_await)
3497	{
3498	tree insert_list = push_stmt_list ();
3499	/* This will be expanded when the revised body is handled. */
3500	coro_build_add_if_not_cond_break (FOR_COND (for_stmt));
3501	/* .. add the original for body. */
3502	add_stmt (FOR_BODY (for_stmt));
3503	/* To make the new for body. */
3504	FOR_BODY (for_stmt) = pop_stmt_list (insert_list);
3505	FOR_COND (for_stmt) = boolean_true_node;
3506	}
3507	/* If the iteration expression has an await, it's a bit more
3508	tricky.
3509	for (init expression;condition;) {
3510	...
3511	iteration_expr_label:
3512	iteration expression with await;
3513	}
3514	but, then we will need to re-write any continue statements into
3515	'goto iteration_expr_label:'.
3516	*/
3517	if (for_expr_await)
3518	{
3519	location_t sloc = EXPR_LOCATION (FOR_EXPR (for_stmt));
3520	tree insert_list = push_stmt_list ();
3521	/* The original for body. */
3522	add_stmt (FOR_BODY (for_stmt));
3523	char *buf = xasprintf ("for.iter.expr.%u", awpts->cond_number++);
3524	tree it_expr_label
3525	= create_named_label_with_ctx (loc: sloc, name: buf, NULL_TREE);
3526	free (ptr: buf);
3527	add_stmt (build_stmt (sloc, LABEL_EXPR, it_expr_label));
3528	tree for_expr = FOR_EXPR (for_stmt);
3529	/* Present the iteration expression as a statement. */
3530	if (TREE_CODE (for_expr) == CLEANUP_POINT_EXPR)
3531	for_expr = TREE_OPERAND (for_expr, 0);
3532	STRIP_NOPS (for_expr);
3533	finish_expr_stmt (for_expr);
3534	FOR_EXPR (for_stmt) = NULL_TREE;
3535	FOR_BODY (for_stmt) = pop_stmt_list (insert_list);
3536	/* rewrite continue statements to goto label. */
3537	hash_set<tree> visited_continue;
3538	if ((res = cp_walk_tree (&FOR_BODY (for_stmt),
3539	replace_continue, &it_expr_label, &visited_continue)))
3540	return res;
3541	}
3542
3543	/* So now walk the body statement (list), if there were no await
3544	expressions, then this handles the original body - and either
3545	way we will have finished with this statement. */
3546	res = cp_walk_tree (&FOR_BODY (for_stmt),
3547	await_statement_walker, d, NULL);
3548	*do_subtree = 0; /* Done subtrees. */
3549	return res;
3550	}
3551	break;
3552	case WHILE_STMT:
3553	{
3554	/* We turn 'while (cond with awaits) stmt' into
3555	while (true) {
3556	if (!(cond with awaits))
3557	break;
3558	stmt..
3559	} */
3560	tree *await_ptr;
3561	hash_set<tree> visited;
3562	tree while_stmt = *stmt;
3563	if (!(cp_walk_tree (&WHILE_COND (while_stmt),
3564	find_any_await, &await_ptr, &visited)))
3565	return NULL_TREE; /* Nothing special to do here. */
3566
3567	tree insert_list = push_stmt_list ();
3568	coro_build_add_if_not_cond_break (WHILE_COND (while_stmt));
3569	/* The original while body. */
3570	add_stmt (WHILE_BODY (while_stmt));
3571	/* The new while body. */
3572	WHILE_BODY (while_stmt) = pop_stmt_list (insert_list);
3573	WHILE_COND (while_stmt) = boolean_true_node;
3574	/* So now walk the new statement list. */
3575	res = cp_walk_tree (&WHILE_BODY (while_stmt),
3576	await_statement_walker, d, NULL);
3577	*do_subtree = 0; /* Done subtrees. */
3578	return res;
3579	}
3580	break;
3581	case DO_STMT:
3582	{
3583	/* We turn do stmt while (cond with awaits) into:
3584	do {
3585	stmt..
3586	if (!(cond with awaits))
3587	break;
3588	} while (true); */
3589	tree do_stmt = *stmt;
3590	tree *await_ptr;
3591	hash_set<tree> visited;
3592	if (!(cp_walk_tree (&DO_COND (do_stmt),
3593	find_any_await, &await_ptr, &visited)))
3594	return NULL_TREE; /* Nothing special to do here. */
3595
3596	tree insert_list = push_stmt_list ();
3597	/* The original do stmt body. */
3598	add_stmt (DO_BODY (do_stmt));
3599	coro_build_add_if_not_cond_break (DO_COND (do_stmt));
3600	/* The new while body. */
3601	DO_BODY (do_stmt) = pop_stmt_list (insert_list);
3602	DO_COND (do_stmt) = boolean_true_node;
3603	/* So now walk the new statement list. */
3604	res = cp_walk_tree (&DO_BODY (do_stmt), await_statement_walker,
3605	d, NULL);
3606	*do_subtree = 0; /* Done subtrees. */
3607	return res;
3608	}
3609	break;
3610	case SWITCH_STMT:
3611	{
3612	/* We turn 'switch (cond with awaits) stmt' into
3613	switch_type cond = cond with awaits
3614	switch (cond) stmt. */
3615	tree sw_stmt = *stmt;
3616	tree *await_ptr;
3617	hash_set<tree> visited;
3618	if (!(cp_walk_tree (&SWITCH_STMT_COND (sw_stmt),
3619	find_any_await, &await_ptr, &visited)))
3620	return NULL_TREE; /* Nothing special to do here. */
3621
3622	gcc_checking_assert (!awpts->bind_stack->is_empty());
3623	/* Build a variable to hold the condition, this will be
3624	included in the frame as a local var. */
3625	tree& bind_expr = awpts->bind_stack->last ();
3626	tree sw_type = SWITCH_STMT_TYPE (sw_stmt);
3627	tree newvar = add_var_to_bind (bind&: bind_expr, var_type: sw_type, nam_root: "swch",
3628	nam_vers: awpts->cond_number++);
3629	tree insert_list = push_stmt_list ();
3630	add_decl_expr (newvar);
3631
3632	tree cond_inner = SWITCH_STMT_COND (sw_stmt);
3633	if (TREE_CODE (cond_inner) == CLEANUP_POINT_EXPR)
3634	cond_inner = TREE_OPERAND (cond_inner, 0);
3635	location_t sloc = EXPR_LOCATION (SWITCH_STMT_COND (sw_stmt));
3636	tree new_s = cp_build_init_expr (sloc, newvar,
3637	cond_inner);
3638	finish_expr_stmt (new_s);
3639	SWITCH_STMT_COND (sw_stmt) = newvar;
3640	/* Now add the switch statement with the condition re-
3641	written to use the local var. */
3642	add_stmt (sw_stmt);
3643	*stmt = pop_stmt_list (insert_list);
3644	/* Process the expanded list. */
3645	res = cp_walk_tree (stmt, await_statement_walker,
3646	d, NULL);
3647	*do_subtree = 0; /* Done subtrees. */
3648	return res;
3649	}
3650	break;
3651	case CO_RETURN_EXPR:
3652	{
3653	/* Expand the co_return as per [stmt.return.coroutine]
3654	- for co_return;
3655	{ p.return_void (); goto final_suspend; }
3656	- for co_return [void expr];
3657	{ expr; p.return_void(); goto final_suspend;}
3658	- for co_return [non void expr];
3659	{ p.return_value(expr); goto final_suspend; } */
3660	location_t loc = EXPR_LOCATION (expr);
3661	tree call = TREE_OPERAND (expr, 1);
3662	expr = TREE_OPERAND (expr, 0);
3663	tree ret_list = push_stmt_list ();
3664	/* [stmt.return.coroutine], 2.2
3665	If expr is present and void, it is placed immediately before
3666	the call for return_void; */
3667	if (expr && VOID_TYPE_P (TREE_TYPE (expr)))
3668	finish_expr_stmt (expr);
3669	/* Insert p.return_{void,value(expr)}. */
3670	finish_expr_stmt (call);
3671	TREE_USED (awpts->fs_label) = 1;
3672	add_stmt (build_stmt (loc, GOTO_EXPR, awpts->fs_label));
3673	*stmt = pop_stmt_list (ret_list);
3674	res = cp_walk_tree (stmt, await_statement_walker, d, NULL);
3675	/* Once this is complete, we will have processed subtrees. */
3676	*do_subtree = 0;
3677	return res;
3678	}
3679	break;
3680	case HANDLER:
3681	{
3682	/* [expr.await] An await-expression shall appear only in a
3683	potentially-evaluated expression within the compound-statement
3684	of a function-body outside of a handler. */
3685	tree *await_ptr;
3686	hash_set<tree> visited;
3687	if (!(cp_walk_tree (&HANDLER_BODY (expr), find_any_await,
3688	&await_ptr, &visited)))
3689	return NULL_TREE; /* All OK. */
3690	location_t loc = EXPR_LOCATION (*await_ptr);
3691	error_at (loc, "await expressions are not permitted in handlers");
3692	return NULL_TREE; /* This is going to fail later anyway. */
3693	}
3694	break;
3695	}
3696	else if (EXPR_P (expr))
3697	{
3698	hash_set<tree> visited;
3699	tree *await_ptr;
3700	if (!(cp_walk_tree (stmt, find_any_await, &await_ptr, &visited)))
3701	return NULL_TREE; /* Nothing special to do here. */
3702
3703	visited.empty ();
3704	awpts->saw_awaits = 0;
3705	hash_set<tree> truth_aoif_to_expand;
3706	awpts->truth_aoif_to_expand = &truth_aoif_to_expand;
3707	awpts->needs_truth_if_exp = false;
3708	awpts->has_awaiter_init = false;
3709	if ((res = cp_walk_tree (stmt, analyze_expression_awaits, d, &visited)))
3710	return res;
3711	*do_subtree = 0; /* Done subtrees. */
3712	if (!awpts->saw_awaits)
3713	return NULL_TREE; /* Nothing special to do here. */
3714
3715	if (awpts->needs_truth_if_exp)
3716	{
3717	/* If a truth-and/or-if expression has an await expression in the
3718	conditionally-taken branch, then it must be rewritten into a
3719	regular conditional. */
3720	truth_if_transform xf = {.orig_stmt: stmt, NULL_TREE, .truth_aoif_to_expand: &truth_aoif_to_expand};
3721	if ((res = cp_walk_tree (stmt, expand_one_truth_if, &xf, NULL)))
3722	return res;
3723	}
3724	/* Process this statement, which contains at least one await expression
3725	to 'promote' temporary values to a coroutine frame slot. */
3726	return maybe_promote_temps (stmt, d);
3727	}
3728	/* Continue recursion, if needed. */
3729	return res;
3730	}
3731
3732	/* For figuring out what param usage we have. */
3733
3734	struct param_frame_data
3735	{
3736	tree *field_list;
3737	hash_map<tree, param_info> *param_uses;
3738	hash_set<tree *> *visited;
3739	location_t loc;
3740	bool param_seen;
3741	};
3742
3743	/* A tree walk callback that rewrites each parm use to the local variable
3744	that represents its copy in the frame. */
3745
3746	static tree
3747	rewrite_param_uses (tree *stmt, int *do_subtree ATTRIBUTE_UNUSED, void *d)
3748	{
3749	param_frame_data *data = (param_frame_data *) d;
3750
3751	/* For lambda closure content, we have to look specifically. */
3752	if (VAR_P (*stmt) && DECL_HAS_VALUE_EXPR_P (*stmt))
3753	{
3754	tree t = DECL_VALUE_EXPR (*stmt);
3755	return cp_walk_tree (&t, rewrite_param_uses, d, NULL);
3756	}
3757
3758	if (TREE_CODE (*stmt) != PARM_DECL)
3759	return NULL_TREE;
3760
3761	/* If we already saw the containing expression, then we're done. */
3762	if (data->visited->add (k: stmt))
3763	return NULL_TREE;
3764
3765	bool existed;
3766	param_info &parm = data->param_uses->get_or_insert (k: *stmt, existed: &existed);
3767	gcc_checking_assert (existed);
3768
3769	*stmt = parm.copy_var;
3770	return NULL_TREE;
3771	}
3772
3773	/* Build up a set of info that determines how each param copy will be
3774	handled. */
3775
3776	static hash_map<tree, param_info> *
3777	analyze_fn_parms (tree orig)
3778	{
3779	if (!DECL_ARGUMENTS (orig))
3780	return NULL;
3781
3782	hash_map<tree, param_info> *param_uses = new hash_map<tree, param_info>;
3783
3784	/* Build a hash map with an entry for each param.
3785	The key is the param tree.
3786	Then we have an entry for the frame field name.
3787	Then a cache for the field ref when we come to use it.
3788	Then a tree list of the uses.
3789	The second two entries start out empty - and only get populated
3790	when we see uses. */
3791	bool lambda_p = LAMBDA_FUNCTION_P (orig);
3792
3793	unsigned no_name_parm = 0;
3794	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL; arg = DECL_CHAIN (arg))
3795	{
3796	bool existed;
3797	param_info &parm = param_uses->get_or_insert (k: arg, existed: &existed);
3798	gcc_checking_assert (!existed);
3799	parm.body_uses = NULL;
3800	tree actual_type = TREE_TYPE (arg);
3801	actual_type = complete_type_or_else (actual_type, orig);
3802	if (actual_type == NULL_TREE)
3803	actual_type = error_mark_node;
3804	parm.orig_type = actual_type;
3805	parm.by_ref = parm.pt_ref = parm.rv_ref = false;
3806	if (TREE_CODE (actual_type) == REFERENCE_TYPE)
3807	{
3808	/* If the user passes by reference, then we will save the
3809	pointer to the original. As noted in
3810	[dcl.fct.def.coroutine] / 13, if the lifetime of the
3811	referenced item ends and then the coroutine is resumed,
3812	we have UB; well, the user asked for it. */
3813	if (TYPE_REF_IS_RVALUE (actual_type))
3814	parm.rv_ref = true;
3815	else
3816	parm.pt_ref = true;
3817	}
3818	else if (TYPE_REF_P (DECL_ARG_TYPE (arg)))
3819	parm.by_ref = true;
3820
3821	parm.frame_type = actual_type;
3822
3823	parm.this_ptr = is_this_parameter (arg);
3824	parm.lambda_cobj = lambda_p && DECL_NAME (arg) == closure_identifier;
3825
3826	tree name = DECL_NAME (arg);
3827	if (!name)
3828	{
3829	char *buf = xasprintf ("_Coro_unnamed_parm_%d", no_name_parm++);
3830	name = get_identifier (buf);
3831	free (ptr: buf);
3832	}
3833	parm.field_id = name;
3834
3835	if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (parm.frame_type))
3836	{
3837	char *buf = xasprintf ("%s%s_live", DECL_NAME (arg) ? "_Coro_" : "",
3838	IDENTIFIER_POINTER (name));
3839	parm.guard_var
3840	= coro_build_artificial_var (UNKNOWN_LOCATION, get_identifier (buf),
3841	boolean_type_node, ctx: orig,
3842	boolean_false_node);
3843	free (ptr: buf);
3844	parm.trivial_dtor = false;
3845	}
3846	else
3847	parm.trivial_dtor = true;
3848	}
3849
3850	return param_uses;
3851	}
3852
3853	/* Small helper for the repetitive task of adding a new field to the coro
3854	frame type. */
3855
3856	static tree
3857	coro_make_frame_entry (tree *field_list, const char *name, tree fld_type,
3858	location_t loc)
3859	{
3860	tree id = get_identifier (name);
3861	tree decl = build_decl (loc, FIELD_DECL, id, fld_type);
3862	DECL_CHAIN (decl) = *field_list;
3863	*field_list = decl;
3864	return id;
3865	}
3866
3867	/* For recording local variable usage. */
3868
3869	struct local_vars_frame_data
3870	{
3871	tree *field_list;
3872	hash_map<tree, local_var_info> *local_var_uses;
3873	unsigned int nest_depth, bind_indx;
3874	location_t loc;
3875	bool saw_capture;
3876	bool local_var_seen;
3877	};
3878
3879	/* A tree-walk callback that processes one bind expression noting local
3880	variables, and making a coroutine frame slot available for those that
3881	need it, so that they can be 'promoted' across suspension points. */
3882
3883	static tree
3884	register_local_var_uses (tree *stmt, int *do_subtree, void *d)
3885	{
3886	local_vars_frame_data *lvd = (local_vars_frame_data *) d;
3887
3888	/* As we enter a bind expression - record the vars there and then recurse.
3889	As we exit drop the nest depth.
3890	The bind index is a growing count of how many bind indices we've seen.
3891	We build a space in the frame for each local var. */
3892
3893	if (TREE_CODE (*stmt) == BIND_EXPR)
3894	{
3895	tree lvar;
3896	unsigned serial = 0;
3897	for (lvar = BIND_EXPR_VARS (*stmt); lvar != NULL;
3898	lvar = DECL_CHAIN (lvar))
3899	{
3900	bool existed;
3901	local_var_info &local_var
3902	= lvd->local_var_uses->get_or_insert (k: lvar, existed: &existed);
3903	gcc_checking_assert (!existed);
3904	local_var.def_loc = DECL_SOURCE_LOCATION (lvar);
3905	tree lvtype = TREE_TYPE (lvar);
3906	local_var.frame_type = lvtype;
3907	local_var.field_idx = local_var.field_id = NULL_TREE;
3908
3909	/* Make sure that we only present vars to the tests below. */
3910	if (TREE_CODE (lvar) == TYPE_DECL
3911	|| TREE_CODE (lvar) == NAMESPACE_DECL)
3912	continue;
3913
3914	/* We don't move static vars into the frame. */
3915	local_var.is_static = TREE_STATIC (lvar);
3916	if (local_var.is_static)
3917	continue;
3918
3919	poly_uint64 size;
3920	if (TREE_CODE (lvtype) == ARRAY_TYPE
3921	&& !poly_int_tree_p (DECL_SIZE_UNIT (lvar), value: &size))
3922	{
3923	sorry_at (local_var.def_loc, "variable length arrays are not"
3924	" yet supported in coroutines");
3925	/* Ignore it, this is broken anyway. */
3926	continue;
3927	}
3928
3929	lvd->local_var_seen = true;
3930	/* If this var is a lambda capture proxy, we want to leave it alone,
3931	and later rewrite the DECL_VALUE_EXPR to indirect through the
3932	frame copy of the pointer to the lambda closure object. */
3933	local_var.is_lambda_capture = is_capture_proxy (lvar);
3934	if (local_var.is_lambda_capture)
3935	continue;
3936
3937	/* If a variable has a value expression, then that's what needs
3938	to be processed. */
3939	local_var.has_value_expr_p = DECL_HAS_VALUE_EXPR_P (lvar);
3940	if (local_var.has_value_expr_p)
3941	continue;
3942
3943	/* Make names depth+index unique, so that we can support nested
3944	scopes with identically named locals and still be able to
3945	identify them in the coroutine frame. */
3946	tree lvname = DECL_NAME (lvar);
3947	char *buf = NULL;
3948
3949	/* The outermost bind scope contains the artificial variables that
3950	we inject to implement the coro state machine. We want to be able
3951	to inspect these in debugging. */
3952	if (lvname != NULL_TREE && lvd->nest_depth == 0)
3953	buf = xasprintf ("%s", IDENTIFIER_POINTER (lvname));
3954	else if (lvname != NULL_TREE)
3955	buf = xasprintf ("%s_%u_%u", IDENTIFIER_POINTER (lvname),
3956	lvd->nest_depth, lvd->bind_indx);
3957	else
3958	buf = xasprintf ("_D%u_%u_%u", lvd->nest_depth, lvd->bind_indx,
3959	serial++);
3960
3961	/* TODO: Figure out if we should build a local type that has any
3962	excess alignment or size from the original decl. */
3963	local_var.field_id = coro_make_frame_entry (field_list: lvd->field_list, name: buf,
3964	fld_type: lvtype, loc: lvd->loc);
3965	free (ptr: buf);
3966	/* We don't walk any of the local var sub-trees, they won't contain
3967	any bind exprs. */
3968	}
3969	lvd->bind_indx++;
3970	lvd->nest_depth++;
3971	cp_walk_tree (&BIND_EXPR_BODY (*stmt), register_local_var_uses, d, NULL);
3972	*do_subtree = 0; /* We've done this. */
3973	lvd->nest_depth--;
3974	}
3975	return NULL_TREE;
3976	}
3977
3978	/* Build, return FUNCTION_DECL node based on ORIG with a type FN_TYPE which has
3979	a single argument of type CORO_FRAME_PTR. Build the actor function if
3980	ACTOR_P is true, otherwise the destroy. */
3981
3982	static tree
3983	coro_build_actor_or_destroy_function (tree orig, tree fn_type,
3984	tree coro_frame_ptr, bool actor_p)
3985	{
3986	location_t loc = DECL_SOURCE_LOCATION (orig);
3987	tree fn
3988	= build_lang_decl (FUNCTION_DECL, copy_node (DECL_NAME (orig)), fn_type);
3989
3990	/* Allow for locating the ramp (original) function from this one. */
3991	if (!to_ramp)
3992	to_ramp = hash_map<tree, tree>::create_ggc (size: 10);
3993	to_ramp->put (k: fn, v: orig);
3994
3995	DECL_CONTEXT (fn) = DECL_CONTEXT (orig);
3996	DECL_SOURCE_LOCATION (fn) = loc;
3997	DECL_ARTIFICIAL (fn) = true;
3998	DECL_INITIAL (fn) = error_mark_node;
3999
4000	tree id = get_identifier ("frame_ptr");
4001	tree fp = build_lang_decl (PARM_DECL, id, coro_frame_ptr);
4002	DECL_CONTEXT (fp) = fn;
4003	DECL_ARG_TYPE (fp) = type_passed_as (coro_frame_ptr);
4004	DECL_ARGUMENTS (fn) = fp;
4005
4006	/* Copy selected attributes from the original function. */
4007	TREE_USED (fn) = TREE_USED (orig);
4008	if (DECL_SECTION_NAME (orig))
4009	set_decl_section_name (fn, orig);
4010	/* Copy any alignment that the FE added. */
4011	if (DECL_ALIGN (orig))
4012	SET_DECL_ALIGN (fn, DECL_ALIGN (orig));
4013	/* Copy any alignment the user added. */
4014	DECL_USER_ALIGN (fn) = DECL_USER_ALIGN (orig);
4015	/* Apply attributes from the original fn. */
4016	DECL_ATTRIBUTES (fn) = copy_list (DECL_ATTRIBUTES (orig));
4017
4018	/* A void return. */
4019	tree resdecl = build_decl (loc, RESULT_DECL, 0, void_type_node);
4020	DECL_CONTEXT (resdecl) = fn;
4021	DECL_ARTIFICIAL (resdecl) = 1;
4022	DECL_IGNORED_P (resdecl) = 1;
4023	DECL_RESULT (fn) = resdecl;
4024
4025	/* This is a coroutine component. */
4026	DECL_COROUTINE_P (fn) = 1;
4027
4028	/* Set up a means to find out if a decl is one of the helpers and, if so,
4029	which one. */
4030	if (coroutine_info *info = get_coroutine_info (fn_decl: orig))
4031	{
4032	gcc_checking_assert ((actor_p && info->actor_decl == NULL_TREE)
4033	|| info->destroy_decl == NULL_TREE);
4034	if (actor_p)
4035	info->actor_decl = fn;
4036	else
4037	info->destroy_decl = fn;
4038	}
4039	return fn;
4040	}
4041
4042	/* Re-write the body as per [dcl.fct.def.coroutine] / 5. */
4043
4044	static tree
4045	coro_rewrite_function_body (location_t fn_start, tree fnbody, tree orig,
4046	hash_map<tree, param_info> *param_uses,
4047	tree resume_fn_ptr_type,
4048	tree& resume_idx_var, tree& fs_label)
4049	{
4050	/* This will be our new outer scope. */
4051	tree update_body = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4052	tree top_block = make_node (BLOCK);
4053	BIND_EXPR_BLOCK (update_body) = top_block;
4054	BIND_EXPR_BODY (update_body) = push_stmt_list ();
4055
4056	/* If the function has a top level bind expression, then connect that
4057	after first making sure we give it a new block. */
4058	tree first = expr_first (fnbody);
4059	if (first && TREE_CODE (first) == BIND_EXPR)
4060	{
4061	tree block = BIND_EXPR_BLOCK (first);
4062	gcc_checking_assert (block);
4063	gcc_checking_assert (BLOCK_SUPERCONTEXT (block) == NULL_TREE);
4064	gcc_checking_assert (BLOCK_CHAIN (block) == NULL_TREE);
4065	/* Replace the top block to avoid issues with locations for args
4066	appearing to be in a non-existent place. */
4067	tree replace_blk = make_node (BLOCK);
4068	BLOCK_VARS (replace_blk) = BLOCK_VARS (block);
4069	BLOCK_SUBBLOCKS (replace_blk) = BLOCK_SUBBLOCKS (block);
4070	for (tree b = BLOCK_SUBBLOCKS (replace_blk); b; b = BLOCK_CHAIN (b))
4071	BLOCK_SUPERCONTEXT (b) = replace_blk;
4072	BIND_EXPR_BLOCK (first) = replace_blk;
4073	/* The top block has one child, so far, and we have now got a
4074	superblock. */
4075	BLOCK_SUPERCONTEXT (replace_blk) = top_block;
4076	BLOCK_SUBBLOCKS (top_block) = replace_blk;
4077	}
4078	else
4079	{
4080	/* We are missing a top level BIND_EXPR. We need one to ensure that we
4081	don't shuffle around the coroutine frame and corrupt it. */
4082	tree bind_wrap = build3_loc (loc: fn_start, code: BIND_EXPR, void_type_node,
4083	NULL, NULL, NULL);
4084	BIND_EXPR_BODY (bind_wrap) = fnbody;
4085	/* Ensure we have a block to connect up the scopes. */
4086	tree new_blk = make_node (BLOCK);
4087	BIND_EXPR_BLOCK (bind_wrap) = new_blk;
4088	BLOCK_SUBBLOCKS (top_block) = new_blk;
4089	fnbody = bind_wrap;
4090	}
4091
4092	/* Wrap the function body in a try {} catch (...) {} block, if exceptions
4093	are enabled. */
4094	tree var_list = NULL_TREE;
4095	tree initial_await = build_init_or_final_await (loc: fn_start, is_final: false);
4096
4097	/* [stmt.return.coroutine] / 3
4098	If p.return_void() is a valid expression, flowing off the end of a
4099	coroutine is equivalent to a co_return with no operand; otherwise
4100	flowing off the end of a coroutine results in undefined behavior. */
4101	tree return_void
4102	= get_coroutine_return_void_expr (decl: current_function_decl, loc: fn_start, musthave: false);
4103
4104	/* The pointer to the resume function. */
4105	tree resume_fn_ptr
4106	= coro_build_artificial_var (loc: fn_start, name: coro_resume_fn_id,
4107	type: resume_fn_ptr_type, ctx: orig, NULL_TREE);
4108	DECL_CHAIN (resume_fn_ptr) = var_list;
4109	var_list = resume_fn_ptr;
4110	add_decl_expr (resume_fn_ptr);
4111
4112	/* We will need to be able to set the resume function pointer to nullptr
4113	to signal that the coroutine is 'done'. */
4114	tree zero_resume
4115	= build1 (CONVERT_EXPR, resume_fn_ptr_type, nullptr_node);
4116
4117	/* The pointer to the destroy function. */
4118	tree var = coro_build_artificial_var (loc: fn_start, name: coro_destroy_fn_id,
4119	type: resume_fn_ptr_type, ctx: orig, NULL_TREE);
4120	DECL_CHAIN (var) = var_list;
4121	var_list = var;
4122	add_decl_expr (var);
4123
4124	/* The promise was created on demand when parsing we now link it into
4125	our scope. */
4126	tree promise = get_coroutine_promise_proxy (decl: orig);
4127	DECL_CONTEXT (promise) = orig;
4128	DECL_SOURCE_LOCATION (promise) = fn_start;
4129	DECL_CHAIN (promise) = var_list;
4130	var_list = promise;
4131	add_decl_expr (promise);
4132
4133	/* We need a handle to this coroutine, which is passed to every
4134	await_suspend(). This was created on demand when parsing we now link it
4135	into our scope. */
4136	var = get_coroutine_self_handle_proxy (decl: orig);
4137	DECL_CONTEXT (var) = orig;
4138	DECL_SOURCE_LOCATION (var) = fn_start;
4139	DECL_CHAIN (var) = var_list;
4140	var_list = var;
4141	add_decl_expr (var);
4142
4143	/* If we have function parms, then these will be copied to the coroutine
4144	frame. Create a local (proxy) variable for each parm, since the original
4145	parms will be out of scope once the ramp has finished. The proxy vars will
4146	get DECL_VALUE_EXPRs pointing to the frame copies, so that we can interact
4147	with them in the debugger. */
4148	if (param_uses)
4149	{
4150	gcc_checking_assert (DECL_ARGUMENTS (orig));
4151	/* Add a local var for each parm. */
4152	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
4153	arg = DECL_CHAIN (arg))
4154	{
4155	param_info *parm_i = param_uses->get (k: arg);
4156	gcc_checking_assert (parm_i);
4157	parm_i->copy_var
4158	= build_lang_decl (VAR_DECL, parm_i->field_id, TREE_TYPE (arg));
4159	DECL_SOURCE_LOCATION (parm_i->copy_var) = DECL_SOURCE_LOCATION (arg);
4160	DECL_CONTEXT (parm_i->copy_var) = orig;
4161	DECL_ARTIFICIAL (parm_i->copy_var) = true;
4162	DECL_CHAIN (parm_i->copy_var) = var_list;
4163	var_list = parm_i->copy_var;
4164	add_decl_expr (parm_i->copy_var);
4165	}
4166
4167	/* Now replace all uses of the parms in the function body with the proxy
4168	vars. We want to this to apply to every instance of param's use, so
4169	don't include a 'visited' hash_set on the tree walk, however we will
4170	arrange to visit each containing expression only once. */
4171	hash_set<tree *> visited;
4172	param_frame_data param_data = {NULL, .param_uses: param_uses,
4173	.visited: &visited, .loc: fn_start, .param_seen: false};
4174	cp_walk_tree (&fnbody, rewrite_param_uses, &param_data, NULL);
4175	}
4176
4177	/* We create a resume index, this is initialized in the ramp. */
4178	resume_idx_var
4179	= coro_build_artificial_var (loc: fn_start, name: coro_resume_index_id,
4180	short_unsigned_type_node, ctx: orig, NULL_TREE);
4181	DECL_CHAIN (resume_idx_var) = var_list;
4182	var_list = resume_idx_var;
4183	add_decl_expr (resume_idx_var);
4184
4185	/* If the coroutine has a frame that needs to be freed, this will be set by
4186	the ramp. */
4187	var = coro_build_artificial_var (loc: fn_start, name: coro_frame_needs_free_id,
4188	boolean_type_node, ctx: orig, NULL_TREE);
4189	DECL_CHAIN (var) = var_list;
4190	var_list = var;
4191	add_decl_expr (var);
4192
4193	if (flag_exceptions)
4194	{
4195	/* Build promise.unhandled_exception(); */
4196	tree ueh
4197	= coro_build_promise_expression (fn: current_function_decl, promise_obj: promise,
4198	member_id: coro_unhandled_exception_identifier,
4199	loc: fn_start, NULL, /*musthave=*/true);
4200	/* Create and initialize the initial-await-resume-called variable per
4201	[dcl.fct.def.coroutine] / 5.3. */
4202	tree i_a_r_c
4203	= coro_build_artificial_var (loc: fn_start, name: coro_frame_i_a_r_c_id,
4204	boolean_type_node, ctx: orig,
4205	boolean_false_node);
4206	DECL_CHAIN (i_a_r_c) = var_list;
4207	var_list = i_a_r_c;
4208	add_decl_expr (i_a_r_c);
4209	/* Start the try-catch. */
4210	tree tcb = build_stmt (fn_start, TRY_BLOCK, NULL_TREE, NULL_TREE);
4211	add_stmt (tcb);
4212	TRY_STMTS (tcb) = push_stmt_list ();
4213	if (initial_await != error_mark_node)
4214	{
4215	/* Build a compound expression that sets the
4216	initial-await-resume-called variable true and then calls the
4217	initial suspend expression await resume.
4218	In the case that the user decides to make the initial await
4219	await_resume() return a value, we need to discard it and, it is
4220	a reference type, look past the indirection. */
4221	if (INDIRECT_REF_P (initial_await))
4222	initial_await = TREE_OPERAND (initial_await, 0);
4223	tree vec = TREE_OPERAND (initial_await, 3);
4224	tree aw_r = TREE_VEC_ELT (vec, 2);
4225	aw_r = convert_to_void (aw_r, ICV_STATEMENT, tf_warning_or_error);
4226	tree update = build2 (MODIFY_EXPR, boolean_type_node, i_a_r_c,
4227	boolean_true_node);
4228	aw_r = cp_build_compound_expr (update, aw_r, tf_warning_or_error);
4229	TREE_VEC_ELT (vec, 2) = aw_r;
4230	}
4231	/* Add the initial await to the start of the user-authored function. */
4232	finish_expr_stmt (initial_await);
4233	/* Append the original function body. */
4234	add_stmt (fnbody);
4235	if (return_void)
4236	add_stmt (return_void);
4237	TRY_STMTS (tcb) = pop_stmt_list (TRY_STMTS (tcb));
4238	TRY_HANDLERS (tcb) = push_stmt_list ();
4239	/* Mimic what the parser does for the catch. */
4240	tree handler = begin_handler ();
4241	finish_handler_parms (NULL_TREE, handler); /* catch (...) */
4242
4243	/* Get the initial await resume called value. */
4244	tree not_iarc_if = begin_if_stmt ();
4245	tree not_iarc = build1_loc (loc: fn_start, code: TRUTH_NOT_EXPR,
4246	boolean_type_node, arg1: i_a_r_c);
4247	finish_if_stmt_cond (not_iarc, not_iarc_if);
4248	/* If the initial await resume called value is false, rethrow... */
4249	tree rethrow = build_throw (fn_start, NULL_TREE, tf_warning_or_error);
4250	suppress_warning (rethrow);
4251	finish_expr_stmt (rethrow);
4252	finish_then_clause (not_iarc_if);
4253	tree iarc_scope = IF_SCOPE (not_iarc_if);
4254	IF_SCOPE (not_iarc_if) = NULL;
4255	not_iarc_if = do_poplevel (iarc_scope);
4256	add_stmt (not_iarc_if);
4257	/* ... else call the promise unhandled exception method
4258	but first we set done = true and the resume index to 0.
4259	If the unhandled exception method returns, then we continue
4260	to the final await expression (which duplicates the clearing of
4261	the field). */
4262	tree r = build2 (MODIFY_EXPR, resume_fn_ptr_type, resume_fn_ptr,
4263	zero_resume);
4264	finish_expr_stmt (r);
4265	tree short_zero = build_int_cst (short_unsigned_type_node, 0);
4266	r = build2 (MODIFY_EXPR, short_unsigned_type_node, resume_idx_var,
4267	short_zero);
4268	finish_expr_stmt (r);
4269	finish_expr_stmt (ueh);
4270	finish_handler (handler);
4271	TRY_HANDLERS (tcb) = pop_stmt_list (TRY_HANDLERS (tcb));
4272	}
4273	else
4274	{
4275	if (pedantic)
4276	{
4277	/* We still try to look for the promise method and warn if it's not
4278	present. */
4279	tree ueh_meth
4280	= lookup_promise_method (fndecl: orig, member_id: coro_unhandled_exception_identifier,
4281	loc: fn_start, /*musthave=*/false);
4282	if (!ueh_meth || ueh_meth == error_mark_node)
4283	warning_at (fn_start, 0, "no member named %qE in %qT",
4284	coro_unhandled_exception_identifier,
4285	get_coroutine_promise_type (decl: orig));
4286	}
4287	/* Else we don't check and don't care if the method is missing..
4288	just add the initial suspend, function and return. */
4289	finish_expr_stmt (initial_await);
4290	/* Append the original function body. */
4291	add_stmt (fnbody);
4292	if (return_void)
4293	add_stmt (return_void);
4294	}
4295
4296	/* co_return branches to the final_suspend label, so declare that now. */
4297	fs_label
4298	= create_named_label_with_ctx (loc: fn_start, name: "final.suspend", NULL_TREE);
4299	add_stmt (build_stmt (fn_start, LABEL_EXPR, fs_label));
4300
4301	/* Before entering the final suspend point, we signal that this point has
4302	been reached by setting the resume function pointer to zero (this is
4303	what the 'done()' builtin tests) as per the current ABI. */
4304	zero_resume = build2 (MODIFY_EXPR, resume_fn_ptr_type, resume_fn_ptr,
4305	zero_resume);
4306	finish_expr_stmt (zero_resume);
4307	finish_expr_stmt (build_init_or_final_await (loc: fn_start, is_final: true));
4308	BIND_EXPR_BODY (update_body) = pop_stmt_list (BIND_EXPR_BODY (update_body));
4309	BIND_EXPR_VARS (update_body) = nreverse (var_list);
4310	BLOCK_VARS (top_block) = BIND_EXPR_VARS (update_body);
4311
4312	return update_body;
4313	}
4314
4315	/* Here we:
4316	a) Check that the function and promise type are valid for a
4317	coroutine.
4318	b) Carry out the initial morph to create the skeleton of the
4319	coroutine ramp function and the rewritten body.
4320
4321	Assumptions.
4322
4323	1. We only hit this code once all dependencies are resolved.
4324	2. The function body will be either a bind expr or a statement list
4325	3. That cfun and current_function_decl are valid for the case we're
4326	expanding.
4327	4. 'input_location' will be of the final brace for the function.
4328
4329	We do something like this:
4330	declare a dummy coro frame.
4331	struct _R_frame {
4332	using handle_type = coro::coroutine_handle<coro1::promise_type>;
4333	void (*_Coro_resume_fn)(_R_frame *);
4334	void (*_Coro_destroy_fn)(_R_frame *);
4335	coro1::promise_type _Coro_promise;
4336	bool _Coro_frame_needs_free; free the coro frame mem if set.
4337	bool _Coro_i_a_r_c; [dcl.fct.def.coroutine] / 5.3
4338	short _Coro_resume_index;
4339	handle_type _Coro_self_handle;
4340	parameter copies (were required).
4341	local variables saved (including awaitables)
4342	(maybe) trailing space.
4343	}; */
4344
4345	bool
4346	morph_fn_to_coro (tree orig, tree *resumer, tree *destroyer)
4347	{
4348	gcc_checking_assert (orig && TREE_CODE (orig) == FUNCTION_DECL);
4349
4350	*resumer = error_mark_node;
4351	*destroyer = error_mark_node;
4352	if (!coro_function_valid_p (fndecl: orig))
4353	{
4354	/* For early errors, we do not want a diagnostic about the missing
4355	ramp return value, since the user cannot fix this - a 'return' is
4356	not allowed in a coroutine. */
4357	suppress_warning (orig, OPT_Wreturn_type);
4358	/* Discard the body, we can't process it further. */
4359	pop_stmt_list (DECL_SAVED_TREE (orig));
4360	DECL_SAVED_TREE (orig) = push_stmt_list ();
4361	return false;
4362	}
4363
4364	/* We can't validly get here with an empty statement list, since there's no
4365	way for the FE to decide it's a coroutine in the absence of any code. */
4366	tree fnbody = pop_stmt_list (DECL_SAVED_TREE (orig));
4367	gcc_checking_assert (fnbody != NULL_TREE);
4368
4369	/* We don't have the locus of the opening brace - it's filled in later (and
4370	there doesn't really seem to be any easy way to get at it).
4371	The closing brace is assumed to be input_location. */
4372	location_t fn_start = DECL_SOURCE_LOCATION (orig);
4373	gcc_rich_location fn_start_loc (fn_start);
4374
4375	/* Initial processing of the function-body.
4376	If we have no expressions or just an error then punt. */
4377	tree body_start = expr_first (fnbody);
4378	if (body_start == NULL_TREE || body_start == error_mark_node)
4379	{
4380	DECL_SAVED_TREE (orig) = push_stmt_list ();
4381	append_to_statement_list (fnbody, &DECL_SAVED_TREE (orig));
4382	/* Suppress warnings about the missing return value. */
4383	suppress_warning (orig, OPT_Wreturn_type);
4384	return false;
4385	}
4386
4387	/* So, we've tied off the original user-authored body in fn_body.
4388
4389	Start the replacement synthesized ramp body as newbody.
4390	If we encounter a fatal error we might return a now-empty body.
4391
4392	Note, the returned ramp body is not 'popped', to be compatible with
4393	the way that decl.cc handles regular functions, the scope pop is done
4394	in the caller. */
4395
4396	tree newbody = push_stmt_list ();
4397	DECL_SAVED_TREE (orig) = newbody;
4398
4399	/* If our original body is noexcept, then that's what we apply to our
4400	generated ramp, transfer any MUST_NOT_THOW_EXPR to that. */
4401	bool is_noexcept = TREE_CODE (body_start) == MUST_NOT_THROW_EXPR;
4402	if (is_noexcept)
4403	{
4404	/* The function body we will continue with is the single operand to
4405	the must-not-throw. */
4406	fnbody = TREE_OPERAND (body_start, 0);
4407	/* Transfer the must-not-throw to the ramp body. */
4408	add_stmt (body_start);
4409	/* Re-start the ramp as must-not-throw. */
4410	TREE_OPERAND (body_start, 0) = push_stmt_list ();
4411	}
4412
4413	/* If the original function has a return value with a non-trivial DTOR
4414	and the body contains a var with a DTOR that might throw, the decl is
4415	marked "throwing_cleanup".
4416	We do not [in the ramp, which is synthesised here], use any body var
4417	types with DTORs that might throw.
4418	The original body is transformed into the actor function which only
4419	contains void returns, and is also wrapped in a try-catch block.
4420	So (a) the 'throwing_cleanup' is not correct for the ramp and (b) we do
4421	not need to transfer it to the actor which only contains void returns. */
4422	cp_function_chain->throwing_cleanup = false;
4423
4424	/* Create the coro frame type, as far as it can be known at this stage.
4425	1. Types we already know. */
4426
4427	tree fn_return_type = TREE_TYPE (TREE_TYPE (orig));
4428	tree handle_type = get_coroutine_handle_type (decl: orig);
4429	tree promise_type = get_coroutine_promise_type (decl: orig);
4430
4431	/* 2. Types we need to define or look up. */
4432
4433	tree fr_name = get_fn_local_identifier (orig, append: "Frame");
4434	tree coro_frame_type = xref_tag (record_type, fr_name);
4435	DECL_CONTEXT (TYPE_NAME (coro_frame_type)) = current_scope ();
4436	tree coro_frame_ptr = build_pointer_type (coro_frame_type);
4437	tree act_des_fn_type
4438	= build_function_type_list (void_type_node, coro_frame_ptr, NULL_TREE);
4439	tree act_des_fn_ptr = build_pointer_type (act_des_fn_type);
4440
4441	/* Declare the actor and destroyer function. */
4442	tree actor = coro_build_actor_or_destroy_function (orig, fn_type: act_des_fn_type,
4443	coro_frame_ptr, actor_p: true);
4444	tree destroy = coro_build_actor_or_destroy_function (orig, fn_type: act_des_fn_type,
4445	coro_frame_ptr, actor_p: false);
4446
4447	/* Construct the wrapped function body; we will analyze this to determine
4448	the requirements for the coroutine frame. */
4449
4450	tree resume_idx_var = NULL_TREE;
4451	tree fs_label = NULL_TREE;
4452	hash_map<tree, param_info> *param_uses = analyze_fn_parms (orig);
4453
4454	fnbody = coro_rewrite_function_body (fn_start, fnbody, orig, param_uses,
4455	resume_fn_ptr_type: act_des_fn_ptr,
4456	resume_idx_var, fs_label);
4457	/* Build our dummy coro frame layout. */
4458	coro_frame_type = begin_class_definition (coro_frame_type);
4459
4460	/* The fields for the coro frame. */
4461	tree field_list = NULL_TREE;
4462
4463	/* We need to know, and inspect, each suspend point in the function
4464	in several places. It's convenient to place this map out of line
4465	since it's used from tree walk callbacks. */
4466	suspend_points = new hash_map<tree, suspend_point_info>;
4467
4468	/* Now insert the data for any body await points, at this time we also need
4469	to promote any temporaries that are captured by reference (to regular
4470	vars) they will get added to the coro frame along with other locals. */
4471	susp_frame_data body_aw_points
4472	= {.field_list: &field_list, .handle_type: handle_type, .fs_label: fs_label, NULL, NULL, .await_number: 0, .cond_number: 0,
4473	.captured_temps: hash_set<tree> (), NULL, NULL, .saw_awaits: 0, .captures_temporary: false, .needs_truth_if_exp: false, .has_awaiter_init: false};
4474	body_aw_points.block_stack = make_tree_vector ();
4475	body_aw_points.bind_stack = make_tree_vector ();
4476	body_aw_points.to_replace = make_tree_vector ();
4477	cp_walk_tree (&fnbody, await_statement_walker, &body_aw_points, NULL);
4478
4479	/* 4. Now make space for local vars, this is conservative again, and we
4480	would expect to delete unused entries later. */
4481	hash_map<tree, local_var_info> local_var_uses;
4482	local_vars_frame_data local_vars_data
4483	= {.field_list: &field_list, .local_var_uses: &local_var_uses, .nest_depth: 0, .bind_indx: 0, .loc: fn_start, .saw_capture: false, .local_var_seen: false};
4484	cp_walk_tree (&fnbody, register_local_var_uses, &local_vars_data, NULL);
4485
4486	/* Tie off the struct for now, so that we can build offsets to the
4487	known entries. */
4488	TYPE_FIELDS (coro_frame_type) = field_list;
4489	TYPE_BINFO (coro_frame_type) = make_tree_binfo (0);
4490	BINFO_OFFSET (TYPE_BINFO (coro_frame_type)) = size_zero_node;
4491	BINFO_TYPE (TYPE_BINFO (coro_frame_type)) = coro_frame_type;
4492
4493	coro_frame_type = finish_struct (coro_frame_type, NULL_TREE);
4494
4495	/* Ramp: */
4496	/* Now build the ramp function pieces. */
4497	tree ramp_bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4498	add_stmt (ramp_bind);
4499	tree ramp_body = push_stmt_list ();
4500
4501	tree zeroinit = build1_loc (loc: fn_start, code: CONVERT_EXPR,
4502	type: coro_frame_ptr, nullptr_node);
4503	tree coro_fp = coro_build_artificial_var (loc: fn_start, name: "_Coro_frameptr",
4504	type: coro_frame_ptr, ctx: orig, init: zeroinit);
4505	tree varlist = coro_fp;
4506
4507	/* To signal that we need to cleanup copied function args. */
4508	if (flag_exceptions && DECL_ARGUMENTS (orig))
4509	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
4510	arg = DECL_CHAIN (arg))
4511	{
4512	param_info *parm_i = param_uses->get (k: arg);
4513	gcc_checking_assert (parm_i);
4514	if (parm_i->trivial_dtor)
4515	continue;
4516	DECL_CHAIN (parm_i->guard_var) = varlist;
4517	varlist = parm_i->guard_var;
4518	}
4519
4520	/* Signal that we need to clean up the promise object on exception. */
4521	tree coro_promise_live
4522	= coro_build_artificial_var (loc: fn_start, name: "_Coro_promise_live",
4523	boolean_type_node, ctx: orig, boolean_false_node);
4524	DECL_CHAIN (coro_promise_live) = varlist;
4525	varlist = coro_promise_live;
4526
4527	/* When the get-return-object is in the RETURN slot, we need to arrange for
4528	cleanup on exception. */
4529	tree coro_gro_live
4530	= coro_build_artificial_var (loc: fn_start, name: "_Coro_gro_live",
4531	boolean_type_node, ctx: orig, boolean_false_node);
4532
4533	DECL_CHAIN (coro_gro_live) = varlist;
4534	varlist = coro_gro_live;
4535
4536	/* Collected the scope vars we need ... only one for now. */
4537	BIND_EXPR_VARS (ramp_bind) = nreverse (varlist);
4538
4539	/* We're now going to create a new top level scope block for the ramp
4540	function. */
4541	tree top_block = make_node (BLOCK);
4542
4543	BIND_EXPR_BLOCK (ramp_bind) = top_block;
4544	BLOCK_VARS (top_block) = BIND_EXPR_VARS (ramp_bind);
4545	BLOCK_SUBBLOCKS (top_block) = NULL_TREE;
4546	current_binding_level->blocks = top_block;
4547
4548	/* The decl_expr for the coro frame pointer, initialize to zero so that we
4549	can pass it to the IFN_CO_FRAME (since there's no way to pass a type,
4550	directly apparently). This avoids a "used uninitialized" warning. */
4551
4552	add_decl_expr (coro_fp);
4553	if (flag_exceptions && DECL_ARGUMENTS (orig))
4554	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
4555	arg = DECL_CHAIN (arg))
4556	{
4557	param_info *parm_i = param_uses->get (k: arg);
4558	if (parm_i->trivial_dtor)
4559	continue;
4560	add_decl_expr (parm_i->guard_var);;
4561	}
4562	add_decl_expr (coro_promise_live);
4563	add_decl_expr (coro_gro_live);
4564
4565	/* The CO_FRAME internal function is a mechanism to allow the middle end
4566	to adjust the allocation in response to optimizations. We provide the
4567	current conservative estimate of the frame size (as per the current)
4568	computed layout. */
4569	tree frame_size = TYPE_SIZE_UNIT (coro_frame_type);
4570	tree resizeable
4571	= build_call_expr_internal_loc (fn_start, IFN_CO_FRAME, size_type_node, 2,
4572	frame_size, coro_fp);
4573
4574	/* [dcl.fct.def.coroutine] / 10 (part1)
4575	The unqualified-id get_return_object_on_allocation_failure is looked up
4576	in the scope of the promise type by class member access lookup. */
4577
4578	/* We don't require this, so coro_build_promise_expression can return NULL,
4579	but, if the lookup succeeds, then the function must be usable. */
4580	tree dummy_promise = build_dummy_object (get_coroutine_promise_type (decl: orig));
4581	tree grooaf
4582	= coro_build_promise_expression (fn: orig, promise_obj: dummy_promise,
4583	member_id: coro_gro_on_allocation_fail_identifier,
4584	loc: fn_start, NULL, /*musthave=*/false);
4585
4586	/* however, should that fail, returning an error, the later stages can't
4587	handle the erroneous expression, so we reset the call as if it was
4588	absent. */
4589	if (grooaf == error_mark_node)
4590	grooaf = NULL_TREE;
4591
4592	/* Allocate the frame, this has several possibilities:
4593	[dcl.fct.def.coroutine] / 9 (part 1)
4594	The allocation function’s name is looked up in the scope of the promise
4595	type. It's not a failure for it to be absent see part 4, below. */
4596
4597	tree nwname = ovl_op_identifier (isass: false, code: NEW_EXPR);
4598	tree new_fn = NULL_TREE;
4599
4600	if (TYPE_HAS_NEW_OPERATOR (promise_type))
4601	{
4602	tree fns = lookup_promise_method (fndecl: orig, member_id: nwname, loc: fn_start,
4603	/*musthave=*/true);
4604	/* [dcl.fct.def.coroutine] / 9 (part 2)
4605	If the lookup finds an allocation function in the scope of the promise
4606	type, overload resolution is performed on a function call created by
4607	assembling an argument list. The first argument is the amount of space
4608	requested, and has type std::size_t. The lvalues p1...pn are the
4609	succeeding arguments.. */
4610	vec<tree, va_gc> *args = make_tree_vector ();
4611	vec_safe_push (v&: args, obj: resizeable); /* Space needed. */
4612
4613	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
4614	arg = DECL_CHAIN (arg))
4615	{
4616	param_info *parm_i = param_uses->get (k: arg);
4617	gcc_checking_assert (parm_i);
4618	if (parm_i->this_ptr || parm_i->lambda_cobj)
4619	{
4620	/* We pass a reference to *this to the allocator lookup. */
4621	tree tt = TREE_TYPE (TREE_TYPE (arg));
4622	tree this_ref = build1 (INDIRECT_REF, tt, arg);
4623	tt = cp_build_reference_type (tt, false);
4624	this_ref = convert_to_reference (tt, this_ref, CONV_STATIC,
4625	LOOKUP_NORMAL , NULL_TREE,
4626	tf_warning_or_error);
4627	vec_safe_push (v&: args, obj: convert_from_reference (this_ref));
4628	}
4629	else
4630	vec_safe_push (v&: args, obj: convert_from_reference (arg));
4631	}
4632
4633	/* Note the function selected; we test to see if it's NOTHROW. */
4634	tree func;
4635	/* Failure is not an error for this attempt. */
4636	new_fn = build_new_method_call (dummy_promise, fns, &args, NULL,
4637	LOOKUP_NORMAL, &func, tf_none);
4638	release_tree_vector (args);
4639
4640	if (new_fn == error_mark_node)
4641	{
4642	/* [dcl.fct.def.coroutine] / 9 (part 3)
4643	If no viable function is found, overload resolution is performed
4644	again on a function call created by passing just the amount of
4645	space required as an argument of type std::size_t. */
4646	args = make_tree_vector_single (resizeable); /* Space needed. */
4647	new_fn = build_new_method_call (dummy_promise, fns, &args,
4648	NULL_TREE, LOOKUP_NORMAL, &func,
4649	tf_none);
4650	release_tree_vector (args);
4651	}
4652
4653	/* However, if the promise provides an operator new, then one of these
4654	two options must be available. */
4655	if (new_fn == error_mark_node)
4656	{
4657	error_at (fn_start, "%qE is provided by %qT but is not usable with"
4658	" the function signature %qD", nwname, promise_type, orig);
4659	new_fn = error_mark_node;
4660	}
4661	else if (grooaf && !TYPE_NOTHROW_P (TREE_TYPE (func)))
4662	error_at (fn_start, "%qE is provided by %qT but %qE is not marked"
4663	" %<throw()%> or %<noexcept%>", grooaf, promise_type, nwname);
4664	else if (!grooaf && TYPE_NOTHROW_P (TREE_TYPE (func)))
4665	warning_at (fn_start, 0, "%qE is marked %<throw()%> or %<noexcept%> but"
4666	" no usable %<get_return_object_on_allocation_failure%>"
4667	" is provided by %qT", nwname, promise_type);
4668	}
4669	else /* No operator new in the promise. */
4670	{
4671	/* [dcl.fct.def.coroutine] / 9 (part 4)
4672	If this lookup fails, the allocation function’s name is looked up in
4673	the global scope. */
4674
4675	vec<tree, va_gc> *args;
4676	/* build_operator_new_call () will insert size needed as element 0 of
4677	this, and we might need to append the std::nothrow constant. */
4678	vec_alloc (v&: args, nelems: 2);
4679	if (grooaf)
4680	{
4681	/* [dcl.fct.def.coroutine] / 10 (part 2)
4682	If any declarations (of the get return on allocation fail) are
4683	found, then the result of a call to an allocation function used
4684	to obtain storage for the coroutine state is assumed to return
4685	nullptr if it fails to obtain storage and, if a global allocation
4686	function is selected, the ::operator new(size_t, nothrow_t) form
4687	is used. The allocation function used in this case shall have a
4688	non-throwing noexcept-specification. So we need std::nothrow. */
4689	tree std_nt = lookup_qualified_name (std_node,
4690	get_identifier ("nothrow"),
4691	LOOK_want::NORMAL,
4692	/*complain=*/true);
4693	if (!std_nt || std_nt == error_mark_node)
4694	error_at (fn_start, "%qE is provided by %qT but %<std::nothrow%> "
4695	"cannot be found", grooaf, promise_type);
4696	vec_safe_push (v&: args, obj: std_nt);
4697	}
4698
4699	/* If we get to this point, we must succeed in looking up the global
4700	operator new for the params provided. Extract a simplified version
4701	of the machinery from build_operator_new_call. This can update the
4702	frame size. */
4703	tree cookie = NULL;
4704	new_fn = build_operator_new_call (nwname, &args, &frame_size, &cookie,
4705	/*align_arg=*/NULL,
4706	/*size_check=*/NULL, /*fn=*/NULL,
4707	tf_warning_or_error);
4708	resizeable = build_call_expr_internal_loc
4709	(fn_start, IFN_CO_FRAME, size_type_node, 2, frame_size, coro_fp);
4710	/* If the operator call fails for some reason, then don't try to
4711	amend it. */
4712	if (new_fn != error_mark_node)
4713	CALL_EXPR_ARG (new_fn, 0) = resizeable;
4714
4715	release_tree_vector (args);
4716	}
4717
4718	tree allocated = build1 (CONVERT_EXPR, coro_frame_ptr, new_fn);
4719	tree r = cp_build_init_expr (t: coro_fp, i: allocated);
4720	r = coro_build_cvt_void_expr_stmt (expr: r, loc: fn_start);
4721	add_stmt (r);
4722
4723	/* If the user provided a method to return an object on alloc fail, then
4724	check the returned pointer and call the func if it's null.
4725	Otherwise, no check, and we fail for noexcept/fno-exceptions cases. */
4726
4727	if (grooaf)
4728	{
4729	/* [dcl.fct.def.coroutine] / 10 (part 3)
4730	If the allocation function returns nullptr,the coroutine returns
4731	control to the caller of the coroutine and the return value is
4732	obtained by a call to T::get_return_object_on_allocation_failure(),
4733	where T is the promise type. */
4734
4735	gcc_checking_assert (same_type_p (fn_return_type, TREE_TYPE (grooaf)));
4736	tree if_stmt = begin_if_stmt ();
4737	tree cond = build1 (CONVERT_EXPR, coro_frame_ptr, nullptr_node);
4738	cond = build2 (EQ_EXPR, boolean_type_node, coro_fp, cond);
4739	finish_if_stmt_cond (cond, if_stmt);
4740	if (VOID_TYPE_P (fn_return_type))
4741	{
4742	/* Execute the get-return-object-on-alloc-fail call... */
4743	finish_expr_stmt (grooaf);
4744	/* ... but discard the result, since we return void. */
4745	finish_return_stmt (NULL_TREE);
4746	}
4747	else
4748	{
4749	/* Get the fallback return object. */
4750	r = build_cplus_new (fn_return_type, grooaf, tf_warning_or_error);
4751	finish_return_stmt (r);
4752	}
4753	finish_then_clause (if_stmt);
4754	finish_if_stmt (if_stmt);
4755	}
4756
4757	/* Up to now any exception thrown will propagate directly to the caller.
4758	This is OK since the only source of such exceptions would be in allocation
4759	of the coroutine frame, and therefore the ramp will not have initialized
4760	any further state. From here, we will track state that needs explicit
4761	destruction in the case that promise or g.r.o setup fails or an exception
4762	is thrown from the initial suspend expression. */
4763	tree ramp_cleanup = NULL_TREE;
4764	if (flag_exceptions)
4765	{
4766	ramp_cleanup = build_stmt (fn_start, TRY_BLOCK, NULL, NULL);
4767	add_stmt (ramp_cleanup);
4768	TRY_STMTS (ramp_cleanup) = push_stmt_list ();
4769	}
4770
4771	/* deref the frame pointer, to use in member access code. */
4772	tree deref_fp = build_x_arrow (fn_start, coro_fp, tf_warning_or_error);
4773
4774	/* For now, once allocation has succeeded we always assume that this needs
4775	destruction, there's no impl. for frame allocation elision. */
4776	tree fnf_m = lookup_member (coro_frame_type, coro_frame_needs_free_id,
4777	1, 0,tf_warning_or_error);
4778	tree fnf_x = build_class_member_access_expr (deref_fp, fnf_m, NULL_TREE,
4779	false, tf_warning_or_error);
4780	r = cp_build_init_expr (t: fnf_x, boolean_true_node);
4781	r = coro_build_cvt_void_expr_stmt (expr: r, loc: fn_start);
4782	add_stmt (r);
4783
4784	/* Put the resumer and destroyer functions in. */
4785
4786	tree actor_addr = build1 (ADDR_EXPR, act_des_fn_ptr, actor);
4787	tree resume_m
4788	= lookup_member (coro_frame_type, coro_resume_fn_id,
4789	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
4790	tree resume_x = build_class_member_access_expr (deref_fp, resume_m, NULL_TREE,
4791	false, tf_warning_or_error);
4792	r = cp_build_init_expr (fn_start, resume_x, actor_addr);
4793	finish_expr_stmt (r);
4794
4795	tree destroy_addr = build1 (ADDR_EXPR, act_des_fn_ptr, destroy);
4796	tree destroy_m
4797	= lookup_member (coro_frame_type, coro_destroy_fn_id,
4798	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
4799	tree destroy_x
4800	= build_class_member_access_expr (deref_fp, destroy_m, NULL_TREE, false,
4801	tf_warning_or_error);
4802	r = cp_build_init_expr (fn_start, destroy_x, destroy_addr);
4803	finish_expr_stmt (r);
4804
4805	/* [dcl.fct.def.coroutine] /13
4806	When a coroutine is invoked, a copy is created for each coroutine
4807	parameter. Each such copy is an object with automatic storage duration
4808	that is direct-initialized from an lvalue referring to the corresponding
4809	parameter if the parameter is an lvalue reference, and from an xvalue
4810	referring to it otherwise. A reference to a parameter in the function-
4811	body of the coroutine and in the call to the coroutine promise
4812	constructor is replaced by a reference to its copy. */
4813
4814	vec<tree, va_gc> *promise_args = NULL; /* So that we can adjust refs. */
4815
4816	/* The initialization and destruction of each parameter copy occurs in the
4817	context of the called coroutine. Initializations of parameter copies are
4818	sequenced before the call to the coroutine promise constructor and
4819	indeterminately sequenced with respect to each other. The lifetime of
4820	parameter copies ends immediately after the lifetime of the coroutine
4821	promise object ends. */
4822
4823	vec<tree, va_gc> *param_dtor_list = NULL;
4824
4825	if (DECL_ARGUMENTS (orig))
4826	{
4827	promise_args = make_tree_vector ();
4828	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
4829	arg = DECL_CHAIN (arg))
4830	{
4831	bool existed;
4832	param_info &parm = param_uses->get_or_insert (k: arg, existed: &existed);
4833
4834	tree fld_ref = lookup_member (coro_frame_type, parm.field_id,
4835	/*protect=*/1, /*want_type=*/0,
4836	tf_warning_or_error);
4837	tree fld_idx
4838	= build_class_member_access_expr (deref_fp, fld_ref, NULL_TREE,
4839	false, tf_warning_or_error);
4840
4841	/* Add this to the promise CTOR arguments list, accounting for
4842	refs and special handling for method this ptr. */
4843	if (parm.this_ptr || parm.lambda_cobj)
4844	{
4845	/* We pass a reference to *this to the param preview. */
4846	tree tt = TREE_TYPE (arg);
4847	gcc_checking_assert (POINTER_TYPE_P (tt));
4848	tree ct = TREE_TYPE (tt);
4849	tree this_ref = build1 (INDIRECT_REF, ct, arg);
4850	tree rt = cp_build_reference_type (ct, false);
4851	this_ref = convert_to_reference (rt, this_ref, CONV_STATIC,
4852	LOOKUP_NORMAL, NULL_TREE,
4853	tf_warning_or_error);
4854	vec_safe_push (v&: promise_args, obj: this_ref);
4855	}
4856	else if (parm.rv_ref)
4857	vec_safe_push (v&: promise_args, obj: move (fld_idx));
4858	else
4859	vec_safe_push (v&: promise_args, obj: fld_idx);
4860
4861	if (parm.rv_ref || parm.pt_ref)
4862	/* Initialise the frame reference field directly. */
4863	r = cp_build_modify_expr (fn_start, TREE_OPERAND (fld_idx, 0),
4864	INIT_EXPR, arg, tf_warning_or_error);
4865	else
4866	{
4867	r = forward_parm (arg);
4868	r = cp_build_modify_expr (fn_start, fld_idx, INIT_EXPR, r,
4869	tf_warning_or_error);
4870	}
4871	finish_expr_stmt (r);
4872	if (!parm.trivial_dtor)
4873	{
4874	if (param_dtor_list == NULL)
4875	param_dtor_list = make_tree_vector ();
4876	vec_safe_push (v&: param_dtor_list, obj: parm.field_id);
4877	/* Cleanup this frame copy on exception. */
4878	parm.fr_copy_dtor
4879	= cxx_maybe_build_cleanup (fld_idx, tf_warning_or_error);
4880	if (flag_exceptions)
4881	{
4882	/* This var is now live. */
4883	r = build_modify_expr (fn_start, parm.guard_var,
4884	boolean_type_node, INIT_EXPR, fn_start,
4885	boolean_true_node, boolean_type_node);
4886	finish_expr_stmt (r);
4887	}
4888	}
4889	}
4890	}
4891
4892	/* Set up the promise. */
4893	tree promise_m
4894	= lookup_member (coro_frame_type, coro_promise_id,
4895	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
4896
4897	tree p = build_class_member_access_expr (deref_fp, promise_m, NULL_TREE,
4898	false, tf_warning_or_error);
4899
4900	tree promise_dtor = NULL_TREE;
4901	if (type_build_ctor_call (promise_type))
4902	{
4903	/* Do a placement new constructor for the promise type (we never call
4904	the new operator, just the constructor on the object in place in the
4905	frame).
4906
4907	First try to find a constructor with the same parameter list as the
4908	original function (if it has params), failing that find a constructor
4909	with no parameter list. */
4910
4911	if (DECL_ARGUMENTS (orig))
4912	{
4913	r = build_special_member_call (p, complete_ctor_identifier,
4914	&promise_args, promise_type,
4915	LOOKUP_NORMAL, tf_none);
4916	release_tree_vector (promise_args);
4917	}
4918	else
4919	r = NULL_TREE;
4920
4921	if (r == NULL_TREE || r == error_mark_node)
4922	r = build_special_member_call (p, complete_ctor_identifier, NULL,
4923	promise_type, LOOKUP_NORMAL,
4924	tf_warning_or_error);
4925
4926	r = coro_build_cvt_void_expr_stmt (expr: r, loc: fn_start);
4927	finish_expr_stmt (r);
4928
4929	r = build_modify_expr (fn_start, coro_promise_live, boolean_type_node,
4930	INIT_EXPR, fn_start, boolean_true_node,
4931	boolean_type_node);
4932	finish_expr_stmt (r);
4933
4934	promise_dtor = cxx_maybe_build_cleanup (p, tf_warning_or_error);
4935	}
4936
4937	/* Set up a new bind context for the GRO. */
4938	tree gro_context_bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4939	/* Make and connect the scope blocks. */
4940	tree gro_block = make_node (BLOCK);
4941	BLOCK_SUPERCONTEXT (gro_block) = top_block;
4942	BLOCK_SUBBLOCKS (top_block) = gro_block;
4943	BIND_EXPR_BLOCK (gro_context_bind) = gro_block;
4944	add_stmt (gro_context_bind);
4945
4946	tree get_ro
4947	= coro_build_promise_expression (fn: orig, promise_obj: p,
4948	member_id: coro_get_return_object_identifier,
4949	loc: fn_start, NULL, /*musthave=*/true);
4950	/* Without a return object we haven't got much clue what's going on. */
4951	if (get_ro == error_mark_node)
4952	{
4953	BIND_EXPR_BODY (ramp_bind) = pop_stmt_list (ramp_body);
4954	DECL_SAVED_TREE (orig) = newbody;
4955	/* Suppress warnings about the missing return value. */
4956	suppress_warning (orig, OPT_Wreturn_type);
4957	return false;
4958	}
4959
4960	tree gro_context_body = push_stmt_list ();
4961	tree gro_type = TREE_TYPE (get_ro);
4962	bool gro_is_void_p = VOID_TYPE_P (gro_type);
4963
4964	tree gro = NULL_TREE;
4965	tree gro_bind_vars = NULL_TREE;
4966	/* Used for return objects in the RESULT slot. */
4967	tree gro_ret_dtor = NULL_TREE;
4968	tree gro_cleanup_stmt = NULL_TREE;
4969	/* We have to sequence the call to get_return_object before initial
4970	suspend. */
4971	if (gro_is_void_p)
4972	r = get_ro;
4973	else if (same_type_p (gro_type, fn_return_type))
4974	{
4975	/* [dcl.fct.def.coroutine] / 7
4976	The expression promise.get_return_object() is used to initialize the
4977	glvalue result or... (see below)
4978	Construct the return result directly. */
4979	if (type_build_ctor_call (gro_type))
4980	{
4981	vec<tree, va_gc> *arg = make_tree_vector_single (get_ro);
4982	r = build_special_member_call (DECL_RESULT (orig),
4983	complete_ctor_identifier,
4984	&arg, gro_type, LOOKUP_NORMAL,
4985	tf_warning_or_error);
4986	release_tree_vector (arg);
4987	}
4988	else
4989	r = cp_build_init_expr (fn_start, DECL_RESULT (orig), get_ro);
4990
4991	if (TYPE_HAS_NONTRIVIAL_DESTRUCTOR (gro_type))
4992	/* If some part of the initalization code (prior to the await_resume
4993	of the initial suspend expression), then we need to clean up the
4994	return value. */
4995	gro_ret_dtor = cxx_maybe_build_cleanup (DECL_RESULT (orig),
4996	tf_warning_or_error);
4997	}
4998	else
4999	{
5000	/* ... or ... Construct an object that will be used as the single
5001	param to the CTOR for the return object. */
5002	gro = coro_build_artificial_var (loc: fn_start, name: "_Coro_gro", type: gro_type, ctx: orig,
5003	NULL_TREE);
5004	add_decl_expr (gro);
5005	gro_bind_vars = gro;
5006	r = cp_build_modify_expr (input_location, gro, INIT_EXPR, get_ro,
5007	tf_warning_or_error);
5008	/* The constructed object might require a cleanup. */
5009	if (tree cleanup = cxx_maybe_build_cleanup (gro, tf_warning_or_error))
5010	gro_cleanup_stmt = build_stmt (input_location, CLEANUP_STMT, NULL,
5011	cleanup, gro);
5012	}
5013	finish_expr_stmt (r);
5014
5015	if (gro_cleanup_stmt && gro_cleanup_stmt != error_mark_node)
5016	CLEANUP_BODY (gro_cleanup_stmt) = push_stmt_list ();
5017
5018	/* If we have a live g.r.o in the return slot, then signal this for exception
5019	cleanup. */
5020	if (gro_ret_dtor)
5021	{
5022	r = build_modify_expr (fn_start, coro_gro_live, boolean_type_node,
5023	INIT_EXPR, fn_start, boolean_true_node,
5024	boolean_type_node);
5025	finish_expr_stmt (r);
5026	}
5027	/* Initialize the resume_idx_var to 0, meaning "not started". */
5028	tree resume_idx_m
5029	= lookup_member (coro_frame_type, coro_resume_index_id,
5030	/*protect=*/1, /*want_type=*/0, tf_warning_or_error);
5031	tree resume_idx
5032	= build_class_member_access_expr (deref_fp, resume_idx_m, NULL_TREE, false,
5033	tf_warning_or_error);
5034	r = build_int_cst (short_unsigned_type_node, 0);
5035	r = cp_build_init_expr (fn_start, resume_idx, r);
5036	r = coro_build_cvt_void_expr_stmt (expr: r, loc: fn_start);
5037	add_stmt (r);
5038
5039	/* So .. call the actor .. */
5040	r = build_call_expr_loc (fn_start, actor, 1, coro_fp);
5041	r = maybe_cleanup_point_expr_void (r);
5042	add_stmt (r);
5043
5044	/* Switch to using 'input_location' as the loc, since we're now more
5045	logically doing things related to the end of the function. */
5046
5047	/* The ramp is done, we just need the return value.
5048	[dcl.fct.def.coroutine] / 7
5049	The expression promise.get_return_object() is used to initialize the
5050	glvalue result or prvalue result object of a call to a coroutine.
5051
5052	If the 'get return object' is non-void, then we built it before the
5053	promise was constructed. We now supply a reference to that var,
5054	either as the return value (if it's the same type) or to the CTOR
5055	for an object of the return type. */
5056
5057	if (same_type_p (gro_type, fn_return_type))
5058	r = gro_is_void_p ? NULL_TREE : DECL_RESULT (orig);
5059	else if (!gro_is_void_p)
5060	/* check_return_expr will automatically return gro as an rvalue via
5061	treat_lvalue_as_rvalue_p. */
5062	r = gro;
5063	else if (CLASS_TYPE_P (fn_return_type))
5064	{
5065	/* For class type return objects, we can attempt to construct,
5066	even if the gro is void. ??? Citation ??? c++/100476 */
5067	r = build_special_member_call (NULL_TREE,
5068	complete_ctor_identifier, NULL,
5069	fn_return_type, LOOKUP_NORMAL,
5070	tf_warning_or_error);
5071	r = build_cplus_new (fn_return_type, r, tf_warning_or_error);
5072	}
5073	else
5074	{
5075	/* We can't initialize a non-class return value from void. */
5076	error_at (input_location, "cannot initialize a return object of type"
5077	" %qT with an rvalue of type %<void%>", fn_return_type);
5078	r = error_mark_node;
5079	}
5080
5081	finish_return_stmt (r);
5082
5083	if (gro_cleanup_stmt)
5084	{
5085	CLEANUP_BODY (gro_cleanup_stmt)
5086	= pop_stmt_list (CLEANUP_BODY (gro_cleanup_stmt));
5087	add_stmt (gro_cleanup_stmt);
5088	}
5089
5090	/* Finish up the ramp function. */
5091	BIND_EXPR_VARS (gro_context_bind) = gro_bind_vars;
5092	BIND_EXPR_BODY (gro_context_bind) = pop_stmt_list (gro_context_body);
5093	TREE_SIDE_EFFECTS (gro_context_bind) = true;
5094
5095	if (flag_exceptions)
5096	{
5097	TRY_HANDLERS (ramp_cleanup) = push_stmt_list ();
5098	tree handler = begin_handler ();
5099	finish_handler_parms (NULL_TREE, handler); /* catch (...) */
5100
5101	/* If we have a live G.R.O in the return slot, then run its DTOR.
5102	When the return object is constructed from a separate g.r.o, this is
5103	already handled by its regular cleanup. */
5104	if (gro_ret_dtor && gro_ret_dtor != error_mark_node)
5105	{
5106	tree gro_d_if = begin_if_stmt ();
5107	finish_if_stmt_cond (coro_gro_live, gro_d_if);
5108	finish_expr_stmt (gro_ret_dtor);
5109	finish_then_clause (gro_d_if);
5110	tree gro_d_if_scope = IF_SCOPE (gro_d_if);
5111	IF_SCOPE (gro_d_if) = NULL;
5112	gro_d_if = do_poplevel (gro_d_if_scope);
5113	add_stmt (gro_d_if);
5114	}
5115
5116	/* If the promise is live, then run its dtor if that's available. */
5117	if (promise_dtor && promise_dtor != error_mark_node)
5118	{
5119	tree promise_d_if = begin_if_stmt ();
5120	finish_if_stmt_cond (coro_promise_live, promise_d_if);
5121	finish_expr_stmt (promise_dtor);
5122	finish_then_clause (promise_d_if);
5123	tree promise_d_if_scope = IF_SCOPE (promise_d_if);
5124	IF_SCOPE (promise_d_if) = NULL;
5125	promise_d_if = do_poplevel (promise_d_if_scope);
5126	add_stmt (promise_d_if);
5127	}
5128
5129	/* Clean up any frame copies of parms with non-trivial dtors. */
5130	if (DECL_ARGUMENTS (orig))
5131	for (tree arg = DECL_ARGUMENTS (orig); arg != NULL;
5132	arg = DECL_CHAIN (arg))
5133	{
5134	param_info *parm_i = param_uses->get (k: arg);
5135	if (parm_i->trivial_dtor)
5136	continue;
5137	if (parm_i->fr_copy_dtor && parm_i->fr_copy_dtor != error_mark_node)
5138	{
5139	tree dtor_if = begin_if_stmt ();
5140	finish_if_stmt_cond (parm_i->guard_var, dtor_if);
5141	finish_expr_stmt (parm_i->fr_copy_dtor);
5142	finish_then_clause (dtor_if);
5143	tree parm_d_if_scope = IF_SCOPE (dtor_if);
5144	IF_SCOPE (dtor_if) = NULL;
5145	dtor_if = do_poplevel (parm_d_if_scope);
5146	add_stmt (dtor_if);
5147	}
5148	}
5149
5150	/* We always expect to delete the frame. */
5151	tree del_coro_fr = coro_get_frame_dtor (coro_fp, orig, frame_size,
5152	promise_type, loc: fn_start);
5153	finish_expr_stmt (del_coro_fr);
5154	tree rethrow = build_throw (fn_start, NULL_TREE, tf_warning_or_error);
5155	suppress_warning (rethrow);
5156	finish_expr_stmt (rethrow);
5157	finish_handler (handler);
5158	TRY_HANDLERS (ramp_cleanup) = pop_stmt_list (TRY_HANDLERS (ramp_cleanup));
5159	}
5160
5161	BIND_EXPR_BODY (ramp_bind) = pop_stmt_list (ramp_body);
5162	TREE_SIDE_EFFECTS (ramp_bind) = true;
5163
5164	/* Start to build the final functions.
5165
5166	We push_deferring_access_checks to avoid these routines being seen as
5167	nested by the middle end; we are doing the outlining here. */
5168
5169	push_deferring_access_checks (dk_no_check);
5170
5171	/* Build the actor... */
5172	build_actor_fn (loc: fn_start, coro_frame_type, actor, fnbody, orig,
5173	local_var_uses: &local_var_uses, param_dtor_list,
5174	resume_idx_var, body_count: body_aw_points.await_number, frame_size);
5175
5176	/* Destroyer ... */
5177	build_destroy_fn (loc: fn_start, coro_frame_type, destroy, actor);
5178
5179	pop_deferring_access_checks ();
5180
5181	DECL_SAVED_TREE (orig) = newbody;
5182	/* Link our new functions into the list. */
5183	TREE_CHAIN (destroy) = TREE_CHAIN (orig);
5184	TREE_CHAIN (actor) = destroy;
5185	TREE_CHAIN (orig) = actor;
5186
5187	*resumer = actor;
5188	*destroyer = destroy;
5189
5190	delete suspend_points;
5191	suspend_points = NULL;
5192	return true;
5193	}
5194
5195	#include "gt-cp-coroutines.h"
5196
5197