1	/* strub (stack scrubbing) support.
2	Copyright (C) 2021-2024 Free Software Foundation, Inc.
3	Contributed by Alexandre Oliva <oliva@adacore.com>.
4
5	This file is part of GCC.
6
7	GCC is free software; you can redistribute it and/or modify it under
8	the terms of the GNU General Public License as published by the Free
9	Software Foundation; either version 3, or (at your option) any later
10	version.
11
12	GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13	WARRANTY; without even the implied warranty of MERCHANTABILITY or
14	FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15	for more details.
16
17	You should have received a copy of the GNU General Public License
18	along with GCC; see the file COPYING3. If not see
19	<http://www.gnu.org/licenses/>. */
20
21	#include "config.h"
22	#include "system.h"
23	#include "coretypes.h"
24	#include "backend.h"
25	#include "tree.h"
26	#include "gimple.h"
27	#include "gimplify.h"
28	#include "tree-pass.h"
29	#include "ssa.h"
30	#include "gimple-iterator.h"
31	#include "gimplify-me.h"
32	#include "tree-into-ssa.h"
33	#include "tree-ssa.h"
34	#include "tree-cfg.h"
35	#include "cfghooks.h"
36	#include "cfgloop.h"
37	#include "cfgcleanup.h"
38	#include "tree-eh.h"
39	#include "except.h"
40	#include "builtins.h"
41	#include "attribs.h"
42	#include "tree-inline.h"
43	#include "cgraph.h"
44	#include "alloc-pool.h"
45	#include "symbol-summary.h"
46	#include "sreal.h"
47	#include "ipa-cp.h"
48	#include "ipa-prop.h"
49	#include "ipa-fnsummary.h"
50	#include "gimple-fold.h"
51	#include "fold-const.h"
52	#include "gimple-walk.h"
53	#include "tree-dfa.h"
54	#include "langhooks.h"
55	#include "calls.h"
56	#include "vec.h"
57	#include "stor-layout.h"
58	#include "varasm.h"
59	#include "alias.h"
60	#include "diagnostic.h"
61	#include "intl.h"
62	#include "ipa-strub.h"
63	#include "symtab-thunks.h"
64	#include "attr-fnspec.h"
65	#include "target.h"
66
67	/* This file introduces two passes that, together, implement
68	machine-independent stack scrubbing, strub for short. It arranges
69	for stack frames that have strub enabled to be zeroed-out after
70	relinquishing control to a caller, whether by returning or by
71	propagating an exception. This admittedly unusual design decision
72	was driven by exception support (one needs a stack frame to be
73	active to propagate exceptions out of it), and it enabled an
74	implementation that is entirely machine-independent (no custom
75	epilogue code is required).
76
77	Strub modes can be selected for stack frames by attaching attribute
78	strub to functions or to variables (to their types, actually).
79	Different strub modes, with different implementation details, are
80	available, and they can be selected by an argument to the strub
81	attribute. When enabled by strub-enabled variables, whether by
82	accessing (as in reading from) statically-allocated ones, or by
83	introducing (as in declaring) automatically-allocated ones, a
84	suitable mode is selected automatically.
85
86	At-calls mode modifies the interface of a function, adding a stack
87	watermark argument, that callers use to clean up the stack frame of
88	the called function. Because of the interface change, it can only
89	be used when explicitly selected, or when a function is internal to
90	a translation unit. Strub-at-calls function types are distinct
91	from their original types (they're not modified in-place), and they
92	are not interchangeable with other function types.
93
94	Internal mode, in turn, does not modify the type or the interface
95	of a function. It is currently implemented by turning the function
96	into a wrapper, moving the function body to a separate wrapped
97	function, and scrubbing the wrapped body's stack in the wrapper.
98	Internal-strub function types are mostly interface-compatible with
99	other strub modes, namely callable (from strub functions, though
100	not strub-enabled) and disabled (not callable from strub
101	functions).
102
103	Always_inline functions can be strub functions, but they can only
104	be called from other strub functions, because strub functions must
105	never be inlined into non-strub functions. Internal and at-calls
106	modes are indistinguishable when it comes to always_inline
107	functions: they will necessarily be inlined into another strub
108	function, and will thus be integrated into the caller's stack
109	frame, whatever the mode. (Contrast with non-always_inline strub
110	functions: an at-calls function can be called from other strub
111	functions, ensuring no discontinuity in stack erasing, whereas an
112	internal-strub function can only be called from other strub
113	functions if it happens to be inlined, or if -fstrub=relaxed mode
114	is in effect (that's the default). In -fstrub=strict mode,
115	internal-strub functions are not callable from strub functions,
116	because the wrapper itself is not strubbed.
117
118	The implementation involves two simple-IPA passes. The earliest
119	one, strub-mode, assigns strub modes to functions. It needs to run
120	before any inlining, so that we can prevent inlining of strub
121	functions into non-strub functions. It notes explicit strub mode
122	requests, enables strub in response to strub variables and testing
123	options, and flags unsatisfiable requests.
124
125	Three possibilities of unsatisfiable requests come to mind: (a)
126	when a strub mode is explicitly selected, but the function uses
127	features that make it ineligible for that mode (e.g. at-calls rules
128	out calling __builtin_apply_args, because of the interface changes,
129	and internal mode rules out noclone or otherwise non-versionable
130	functions, non-default varargs, non-local or forced labels, and
131	functions with far too many arguments); (b) when some strub mode
132	must be enabled because of a strub variable, but the function is
133	not eligible or not viable for any mode; and (c) when
134	-fstrub=strict is enabled, and calls are found in strub functions
135	to functions that are not callable from strub contexts.
136	compute_strub_mode implements (a) and (b), and verify_strub
137	implements (c).
138
139	The second IPA pass modifies interfaces of at-calls-strub functions
140	and types, introduces strub calls in and around them. and splits
141	internal-strub functions. It is placed after early inlining, so
142	that even internal-strub functions get a chance of being inlined
143	into other strub functions, but before non-early inlining, so that
144	internal-strub wrapper functions still get a chance of inlining
145	after splitting.
146
147	Wrappers avoid duplicating the copying of large arguments again by
148	passing them by reference to the wrapped bodies. This involves
149	occasional SSA rewriting of address computations, because of the
150	additional indirection. Besides these changes, and the
151	introduction of the stack watermark parameter, wrappers and wrapped
152	functions cooperate to handle variable argument lists (performing
153	va_start in the wrapper, passing the list as an argument, and
154	replacing va_start calls in the wrapped body with va_copy), and
155	__builtin_apply_args (also called in the wrapper and passed to the
156	wrapped body as an argument).
157
158	Strub bodies (both internal-mode wrapped bodies, and at-calls
159	functions) always start by adjusting the watermark parameter, by
160	calling __builtin___strub_update. The compiler inserts them in the
161	main strub pass. Allocations of additional stack space for the
162	frame (__builtin_alloca) are also followed by watermark updates.
163	Stack space temporarily allocated to pass arguments to other
164	functions, released right after the call, is not regarded as part
165	of the frame. Around calls to them, i.e., in internal-mode
166	wrappers and at-calls callers (even calls through pointers), calls
167	to __builtin___strub_enter and __builtin___strub_leave are
168	inserted, the latter as a __finally block, so that it runs at
169	regular and exceptional exit paths. strub_enter only initializes
170	the stack watermark, and strub_leave is where the scrubbing takes
171	place, overwriting with zeros the stack space from the top of the
172	stack to the watermark.
173
174	These calls can be optimized in various cases. In
175	pass_ipa_strub::adjust_at_calls_call, for example, we enable
176	tail-calling and other optimized calls from one strub body to
177	another by passing on the watermark parameter. The builtins
178	themselves may undergo inline substitution during expansion,
179	dependign on optimization levels. This involves dealing with stack
180	red zones (when the builtins are called out-of-line, the red zone
181	cannot be used) and other ugly details related with inlining strub
182	bodies into other strub bodies (see expand_builtin_strub_update).
183	expand_builtin_strub_leave may even perform partial inline
184	substitution. */
185
186	/* Const and pure functions that gain a watermark parameter for strub purposes
187	are still regarded as such, which may cause the inline expansions of the
188	__strub builtins to malfunction. Ideally, attribute "fn spec" would enable
189	us to inform the backend about requirements and side effects of the call, but
190	call_fusage building in calls.c:expand_call does not even look at
191	attr_fnspec, so we resort to asm loads and updates to attain an equivalent
192	effect. Once expand_call gains the ability to issue extra memory uses and
193	clobbers based on pure/const function's fnspec, we can define this to 1. */
194	#define ATTR_FNSPEC_DECONST_WATERMARK 0
195
196	enum strub_mode {
197	/* This mode denotes a regular function, that does not require stack
198	scrubbing (strubbing). It may call any other functions, but if
199	it calls AT_CALLS (or WRAPPED) ones, strubbing logic is
200	automatically introduced around those calls (the latter, by
201	inlining INTERNAL wrappers). */
202	STRUB_DISABLED = 0,
203
204	/* This denotes a function whose signature is (to be) modified to
205	take an extra parameter, for stack use annotation, and its
206	callers must initialize and pass that argument, and perform the
207	strubbing. Functions that are explicitly marked with attribute
208	strub must have the mark visible wherever the function is,
209	including aliases, and overriders and overriding methods.
210	Functions that are implicitly marked for strubbing, for accessing
211	variables explicitly marked as such, will only select this
212	strubbing method if they are internal to a translation unit. It
213	can only be inlined into other strubbing functions, i.e.,
214	STRUB_AT_CALLS or STRUB_WRAPPED. */
215	STRUB_AT_CALLS = 1,
216
217	/* This denotes a function that is to perform strubbing internally,
218	without any changes to its interface (the function is turned into
219	a strubbing wrapper, and its original body is moved to a separate
220	STRUB_WRAPPED function, with a modified interface). Functions
221	may be explicitly marked with attribute strub(2), and the
222	attribute must be visible at the point of definition. Functions
223	that are explicitly marked for strubbing, for accessing variables
224	explicitly marked as such, may select this strubbing mode if
225	their interface cannot change, e.g. because its interface is
226	visible to other translation units, directly, by indirection
227	(having its address taken), inheritance, etc. Functions that use
228	this method must not have the noclone attribute, nor the noipa
229	one. Functions marked as always_inline may select this mode, but
230	they are NOT wrapped, they remain unchanged, and are only inlined
231	into strubbed contexts. Once non-always_inline functions are
232	wrapped, the wrapper becomes STRUB_WRAPPER, and the wrapped becomes
233	STRUB_WRAPPED. */
234	STRUB_INTERNAL = 2,
235
236	/* This denotes a function whose stack is not strubbed, but that is
237	nevertheless explicitly or implicitly marked as callable from strubbing
238	functions. Normally, only STRUB_AT_CALLS (and STRUB_INTERNAL ->
239	STRUB_WRAPPED) functions can be called from strubbing contexts (bodies of
240	STRUB_AT_CALLS, STRUB_INTERNAL and STRUB_WRAPPED functions), but attribute
241	strub(3) enables other functions to be (indirectly) called from these
242	contexts. Some builtins and internal functions may be implicitly marked as
243	STRUB_CALLABLE. */
244	STRUB_CALLABLE = 3,
245
246	/* This denotes the function that took over the body of a
247	STRUB_INTERNAL function. At first, it's only called by its
248	wrapper, but the wrapper may be inlined. The wrapped function,
249	in turn, can only be inlined into other functions whose stack
250	frames are strubbed, i.e., that are STRUB_WRAPPED or
251	STRUB_AT_CALLS. */
252	STRUB_WRAPPED = -1,
253
254	/* This denotes the wrapper function that replaced the STRUB_INTERNAL
255	function. This mode overrides the STRUB_INTERNAL mode at the time the
256	internal to-be-wrapped function becomes a wrapper, so that inlining logic
257	can tell one from the other. */
258	STRUB_WRAPPER = -2,
259
260	/* This denotes an always_inline function that requires strubbing. It can
261	only be called from, and inlined into, other strubbing contexts. */
262	STRUB_INLINABLE = -3,
263
264	/* This denotes a function that accesses strub variables, so it would call for
265	internal strubbing (whether or not it's eligible for that), but since
266	at-calls strubbing is viable, that's selected as an optimization. This
267	mode addresses the inconvenience that such functions may have different
268	modes selected depending on optimization flags, and get a different
269	callable status depending on that choice: if we assigned them
270	STRUB_AT_CALLS mode, they would be callable when optimizing, whereas
271	STRUB_INTERNAL would not be callable. */
272	STRUB_AT_CALLS_OPT = -4,
273
274	};
275
276	/* Look up a strub attribute in TYPE, and return it. */
277
278	static tree
279	get_strub_attr_from_type (tree type)
280	{
281	return lookup_attribute (attr_name: "strub", TYPE_ATTRIBUTES (type));
282	}
283
284	/* Look up a strub attribute in DECL or in its type, and return it. */
285
286	static tree
287	get_strub_attr_from_decl (tree decl)
288	{
289	tree ret = lookup_attribute (attr_name: "strub", DECL_ATTRIBUTES (decl));
290	if (ret)
291	return ret;
292	return get_strub_attr_from_type (TREE_TYPE (decl));
293	}
294
295	#define STRUB_ID_COUNT 8
296	#define STRUB_IDENT_COUNT 3
297	#define STRUB_TYPE_COUNT 5
298
299	#define STRUB_ID_BASE 0
300	#define STRUB_IDENT_BASE (STRUB_ID_BASE + STRUB_ID_COUNT)
301	#define STRUB_TYPE_BASE (STRUB_IDENT_BASE + STRUB_IDENT_COUNT)
302	#define STRUB_CACHE_SIZE (STRUB_TYPE_BASE + STRUB_TYPE_COUNT)
303
304	/* Keep the strub mode and temp identifiers and types from being GC'd. */
305	static GTY((deletable)) tree strub_cache[STRUB_CACHE_SIZE];
306
307	/* Define a function to cache identifier ID, to be used as a strub attribute
308	parameter for a strub mode named after NAME. */
309	#define DEF_STRUB_IDS(IDX, NAME, ID) \
310	static inline tree get_strub_mode_id_ ## NAME () { \
311	int idx = STRUB_ID_BASE + IDX; \
312	tree identifier = strub_cache[idx]; \
313	if (!identifier) \
314	strub_cache[idx] = identifier = get_identifier (ID); \
315	return identifier; \
316	}
317	/* Same as DEF_STRUB_IDS, but use the string expansion of NAME as ID. */
318	#define DEF_STRUB_ID(IDX, NAME) \
319	DEF_STRUB_IDS (IDX, NAME, #NAME)
320
321	/* Define functions for each of the strub mode identifiers.
322	Expose dashes rather than underscores. */
323	DEF_STRUB_ID (0, disabled)
324	DEF_STRUB_IDS (1, at_calls, "at-calls")
325	DEF_STRUB_ID (2, internal)
326	DEF_STRUB_ID (3, callable)
327	DEF_STRUB_ID (4, wrapped)
328	DEF_STRUB_ID (5, wrapper)
329	DEF_STRUB_ID (6, inlinable)
330	DEF_STRUB_IDS (7, at_calls_opt, "at-calls-opt")
331
332	/* Release the temporary macro names. */
333	#undef DEF_STRUB_IDS
334	#undef DEF_STRUB_ID
335
336	/* Return the identifier corresponding to strub MODE. */
337
338	static tree
339	get_strub_mode_attr_parm (enum strub_mode mode)
340	{
341	switch (mode)
342	{
343	case STRUB_DISABLED:
344	return get_strub_mode_id_disabled ();
345
346	case STRUB_AT_CALLS:
347	return get_strub_mode_id_at_calls ();
348
349	case STRUB_INTERNAL:
350	return get_strub_mode_id_internal ();
351
352	case STRUB_CALLABLE:
353	return get_strub_mode_id_callable ();
354
355	case STRUB_WRAPPED:
356	return get_strub_mode_id_wrapped ();
357
358	case STRUB_WRAPPER:
359	return get_strub_mode_id_wrapper ();
360
361	case STRUB_INLINABLE:
362	return get_strub_mode_id_inlinable ();
363
364	case STRUB_AT_CALLS_OPT:
365	return get_strub_mode_id_at_calls_opt ();
366
367	default:
368	gcc_unreachable ();
369	}
370	}
371
372	/* Return the parmeters (TREE_VALUE) for a strub attribute of MODE.
373	We know we use a single parameter, so we bypass the creation of a
374	tree list. */
375
376	static tree
377	get_strub_mode_attr_value (enum strub_mode mode)
378	{
379	return get_strub_mode_attr_parm (mode);
380	}
381
382	/* Determine whether ID is a well-formed strub mode-specifying attribute
383	parameter for a function (type). Only user-visible modes are accepted, and
384	ID must be non-NULL.
385
386	For unacceptable parms, return 0, otherwise a nonzero value as below.
387
388	If the parm enables strub, return positive, otherwise negative.
389
390	If the affected type must be a distinct, incompatible type,return an integer
391	of absolute value 2, otherwise 1. */
392
393	int
394	strub_validate_fn_attr_parm (tree id)
395	{
396	int ret;
397	const char *s = NULL;
398	size_t len = 0;
399
400	/* do NOT test for NULL. This is only to be called with non-NULL arguments.
401	We assume that the strub parameter applies to a function, because only
402	functions accept an explicit argument. If we accepted NULL, and we
403	happened to be called to verify the argument for a variable, our return
404	values would be wrong. */
405	if (TREE_CODE (id) == STRING_CST)
406	{
407	s = TREE_STRING_POINTER (id);
408	len = TREE_STRING_LENGTH (id) - 1;
409	}
410	else if (TREE_CODE (id) == IDENTIFIER_NODE)
411	{
412	s = IDENTIFIER_POINTER (id);
413	len = IDENTIFIER_LENGTH (id);
414	}
415	else
416	return 0;
417
418	enum strub_mode mode;
419
420	if (len != 8)
421	return 0;
422
423	switch (s[0])
424	{
425	case 'd':
426	mode = STRUB_DISABLED;
427	ret = -1;
428	break;
429
430	case 'a':
431	mode = STRUB_AT_CALLS;
432	ret = 2;
433	break;
434
435	case 'i':
436	mode = STRUB_INTERNAL;
437	ret = 1;
438	break;
439
440	case 'c':
441	mode = STRUB_CALLABLE;
442	ret = -2;
443	break;
444
445	default:
446	/* Other parms are for internal use only. */
447	return 0;
448	}
449
450	tree mode_id = get_strub_mode_attr_parm (mode);
451
452	if (TREE_CODE (id) == IDENTIFIER_NODE
453	? id != mode_id
454	: strncmp (s1: s, IDENTIFIER_POINTER (mode_id), n: len) != 0)
455	return 0;
456
457	return ret;
458	}
459
460	/* Return the strub mode from STRUB_ATTR. VAR_P should be TRUE if the attribute
461	is taken from a variable, rather than from a function, or a type thereof. */
462
463	static enum strub_mode
464	get_strub_mode_from_attr (tree strub_attr, bool var_p = false)
465	{
466	enum strub_mode mode = STRUB_DISABLED;
467
468	if (strub_attr)
469	{
470	if (!TREE_VALUE (strub_attr))
471	mode = !var_p ? STRUB_AT_CALLS : STRUB_INTERNAL;
472	else
473	{
474	gcc_checking_assert (!var_p);
475	tree id = TREE_VALUE (strub_attr);
476	if (TREE_CODE (id) == TREE_LIST)
477	id = TREE_VALUE (id);
478	const char *s = (TREE_CODE (id) == STRING_CST
479	? TREE_STRING_POINTER (id)
480	: IDENTIFIER_POINTER (id));
481	size_t len = (TREE_CODE (id) == STRING_CST
482	? TREE_STRING_LENGTH (id) - 1
483	: IDENTIFIER_LENGTH (id));
484
485	switch (len)
486	{
487	case 7:
488	switch (s[6])
489	{
490	case 'r':
491	mode = STRUB_WRAPPER;
492	break;
493
494	case 'd':
495	mode = STRUB_WRAPPED;
496	break;
497
498	default:
499	gcc_unreachable ();
500	}
501	break;
502
503	case 8:
504	switch (s[0])
505	{
506	case 'd':
507	mode = STRUB_DISABLED;
508	break;
509
510	case 'a':
511	mode = STRUB_AT_CALLS;
512	break;
513
514	case 'i':
515	mode = STRUB_INTERNAL;
516	break;
517
518	case 'c':
519	mode = STRUB_CALLABLE;
520	break;
521
522	default:
523	gcc_unreachable ();
524	}
525	break;
526
527	case 9:
528	mode = STRUB_INLINABLE;
529	break;
530
531	case 12:
532	mode = STRUB_AT_CALLS_OPT;
533	break;
534
535	default:
536	gcc_unreachable ();
537	}
538
539	gcc_checking_assert (TREE_CODE (id) == IDENTIFIER_NODE
540	? id == get_strub_mode_attr_parm (mode)
541	: strncmp (IDENTIFIER_POINTER
542	(get_strub_mode_attr_parm (mode)),
543	s, len) == 0);
544	}
545	}
546
547	return mode;
548	}
549
550	/* Look up, decode and return the strub mode associated with FNDECL. */
551
552	static enum strub_mode
553	get_strub_mode_from_fndecl (tree fndecl)
554	{
555	return get_strub_mode_from_attr (strub_attr: get_strub_attr_from_decl (decl: fndecl));
556	}
557
558	/* Look up, decode and return the strub mode associated with NODE. */
559
560	static enum strub_mode
561	get_strub_mode (cgraph_node *node)
562	{
563	return get_strub_mode_from_fndecl (fndecl: node->decl);
564	}
565
566	/* Look up, decode and return the strub mode associated with TYPE. */
567
568	static enum strub_mode
569	get_strub_mode_from_type (tree type)
570	{
571	bool var_p = !FUNC_OR_METHOD_TYPE_P (type);
572	tree attr = get_strub_attr_from_type (type);
573
574	if (attr)
575	return get_strub_mode_from_attr (strub_attr: attr, var_p);
576
577	if (flag_strub >= -1 && !var_p)
578	return STRUB_CALLABLE;
579
580	return STRUB_DISABLED;
581	}
582
583
584	/* Return TRUE iff NODE calls builtin va_start. */
585
586	static bool
587	calls_builtin_va_start_p (cgraph_node *node)
588	{
589	bool result = false;
590
591	for (cgraph_edge *e = node->callees; e; e = e->next_callee)
592	{
593	tree cdecl = e->callee->decl;
594	if (fndecl_built_in_p (node: cdecl, name1: BUILT_IN_VA_START))
595	return true;
596	}
597
598	return result;
599	}
600
601	/* Return TRUE iff NODE calls builtin apply_args, and optionally REPORT it. */
602
603	static bool
604	calls_builtin_apply_args_p (cgraph_node *node, bool report = false)
605	{
606	bool result = false;
607
608	for (cgraph_edge *e = node->callees; e; e = e->next_callee)
609	{
610	tree cdecl = e->callee->decl;
611	if (!fndecl_built_in_p (node: cdecl, name1: BUILT_IN_APPLY_ARGS))
612	continue;
613
614	result = true;
615
616	if (!report)
617	break;
618
619	sorry_at (e->call_stmt
620	? gimple_location (g: e->call_stmt)
621	: DECL_SOURCE_LOCATION (node->decl),
622	"at-calls %<strub%> does not support call to %qD",
623	cdecl);
624	}
625
626	return result;
627	}
628
629	/* Return TRUE iff NODE carries the always_inline attribute. */
630
631	static inline bool
632	strub_always_inline_p (cgraph_node *node)
633	{
634	return lookup_attribute (attr_name: "always_inline", DECL_ATTRIBUTES (node->decl));
635	}
636
637	/* Return TRUE iff the target has strub support for T, a function
638	decl, or a type used in an indirect call, and optionally REPORT the
639	reasons for ineligibility. If T is a type and error REPORTing is
640	enabled, the LOCation (of the indirect call) should be provided. */
641	static inline bool
642	strub_target_support_p (tree t, bool report = false,
643	location_t loc = UNKNOWN_LOCATION)
644	{
645	bool result = true;
646
647	if (!targetm.have_strub_support_for (t))
648	{
649	result = false;
650
651	if (!report)
652	return result;
653
654	if (DECL_P (t))
655	sorry_at (DECL_SOURCE_LOCATION (t),
656	"%qD is not eligible for %<strub%>"
657	" on the target system", t);
658	else
659	sorry_at (loc,
660	"unsupported %<strub%> call"
661	" on the target system");
662	}
663
664	return result;
665	}
666
667	/* Return TRUE iff NODE is potentially eligible for any strub-enabled mode, and
668	optionally REPORT the reasons for ineligibility. */
669
670	static inline bool
671	can_strub_p (cgraph_node *node, bool report = false)
672	{
673	bool result = strub_target_support_p (t: node->decl, report);
674
675	if (!report && (!result || strub_always_inline_p (node)))
676	return result;
677
678	if (flag_split_stack)
679	{
680	result = false;
681
682	if (!report)
683	return result;
684
685	sorry_at (DECL_SOURCE_LOCATION (node->decl),
686	"%qD is not eligible for %<strub%>"
687	" because %<-fsplit-stack%> is enabled",
688	node->decl);
689	}
690
691	if (lookup_attribute (attr_name: "noipa", DECL_ATTRIBUTES (node->decl)))
692	{
693	result = false;
694
695	if (!report)
696	return result;
697
698	sorry_at (DECL_SOURCE_LOCATION (node->decl),
699	"%qD is not eligible for %<strub%>"
700	" because of attribute %<noipa%>",
701	node->decl);
702	}
703
704	/* We can't, and don't want to vectorize the watermark and other
705	strub-introduced parms. */
706	if (lookup_attribute (attr_name: "simd", DECL_ATTRIBUTES (node->decl)))
707	{
708	result = false;
709
710	if (!report)
711	return result;
712
713	sorry_at (DECL_SOURCE_LOCATION (node->decl),
714	"%qD is not eligible for %<strub%>"
715	" because of attribute %<simd%>",
716	node->decl);
717	}
718
719	return result;
720	}
721
722	/* Return TRUE iff NODE is eligible for at-calls strub, and optionally REPORT
723	the reasons for ineligibility. Besides general non-eligibility for
724	strub-enabled modes, at-calls rules out calling builtin apply_args. */
725
726	static bool
727	can_strub_at_calls_p (cgraph_node *node, bool report = false)
728	{
729	bool result = !report || can_strub_p (node, report);
730
731	if (!result && !report)
732	return result;
733
734	return !calls_builtin_apply_args_p (node, report);
735	}
736
737	/* Return TRUE iff the called function (pointer or, if available,
738	decl) undergoes a significant type conversion for the call. Strub
739	mode changes between function types, and other non-useless type
740	conversions, are regarded as significant. When the function type
741	is overridden, the effective strub mode for the call is that of the
742	call fntype, rather than that of the pointer or of the decl.
743	Functions called with type overrides cannot undergo type changes;
744	it's as if their address was taken, so they're considered
745	non-viable for implicit at-calls strub mode. */
746
747	static inline bool
748	strub_call_fntype_override_p (const gcall *gs)
749	{
750	if (gimple_call_internal_p (gs))
751	return false;
752	tree fn_type = TREE_TYPE (TREE_TYPE (gimple_call_fn (gs)));
753	if (tree decl = gimple_call_fndecl (gs))
754	fn_type = TREE_TYPE (decl);
755
756	/* We do NOT want to take the mode from the decl here. This
757	function is used to tell whether we can change the strub mode of
758	a function, and whether the effective mode for the call is to be
759	taken from the decl or from an overrider type. When the strub
760	mode is explicitly declared, or overridden with a type cast, the
761	difference will be noticed in function types. However, if the
762	strub mode is implicit due to e.g. strub variables or -fstrub=*
763	command-line flags, we will adjust call types along with function
764	types. In either case, the presence of type or strub mode
765	overriders in calls will prevent a function from having its strub
766	modes changed in ways that would imply type changes, but taking
767	strub modes from decls would defeat this, since we set strub
768	modes and then call this function to tell whether the original
769	type was overridden to decide whether to adjust the call. We
770	need the answer to be about the type, not the decl. */
771	enum strub_mode mode = get_strub_mode_from_type (type: fn_type);
772	return (get_strub_mode_from_type (type: gs->u.fntype) != mode
773	|| !useless_type_conversion_p (gs->u.fntype, fn_type));
774	}
775
776	/* Return TRUE iff NODE is called directly with a type override. */
777
778	static bool
779	called_directly_with_type_override_p (cgraph_node *node, void *)
780	{
781	for (cgraph_edge *e = node->callers; e; e = e->next_caller)
782	if (e->call_stmt && strub_call_fntype_override_p (gs: e->call_stmt))
783	return true;
784
785	return false;
786	}
787
788	/* Return TRUE iff NODE or any other nodes aliased to it are called
789	with type overrides. We can't safely change the type of such
790	functions. */
791
792	static bool
793	called_with_type_override_p (cgraph_node *node)
794	{
795	return (node->call_for_symbol_thunks_and_aliases
796	(callback: called_directly_with_type_override_p, NULL, include_overwritable: true, exclude_virtual_thunks: true));
797	}
798
799	/* Symbolic macro for the max number of arguments that internal strub may add to
800	a function. */
801
802	#define STRUB_INTERNAL_MAX_EXTRA_ARGS 3
803
804	/* We can't perform internal strubbing if the function body involves certain
805	features:
806
807	- a non-default __builtin_va_start (e.g. x86's __builtin_ms_va_start) is
808	currently unsupported because we can't discover the corresponding va_copy and
809	va_end decls in the wrapper, and we don't convey the alternate variable
810	arguments ABI to the modified wrapped function. The default
811	__builtin_va_start is supported by calling va_start/va_end at the wrapper,
812	that takes variable arguments, passing a pointer to the va_list object to the
813	wrapped function, that runs va_copy from it where the original function ran
814	va_start.
815
816	__builtin_next_arg is currently unsupported because the wrapped function
817	won't be a variable argument function. We could process it in the wrapper,
818	that remains a variable argument function, and replace calls in the wrapped
819	body, but we currently don't.
820
821	__builtin_return_address is rejected because it's generally used when the
822	actual caller matters, and introducing a wrapper breaks such uses as those in
823	the unwinder. */
824
825	static bool
826	can_strub_internally_p (cgraph_node *node, bool report = false)
827	{
828	bool result = !report || can_strub_p (node, report);
829
830	if (!result && !report)
831	return result;
832
833	if (!report && strub_always_inline_p (node))
834	return result;
835
836	/* Since we're not changing the function identity proper, just
837	moving its full implementation, we *could* disable
838	fun->cannot_be_copied_reason and/or temporarily drop a noclone
839	attribute, but we'd have to prevent remapping of the labels. */
840	if (lookup_attribute (attr_name: "noclone", DECL_ATTRIBUTES (node->decl)))
841	{
842	result = false;
843
844	if (!report)
845	return result;
846
847	sorry_at (DECL_SOURCE_LOCATION (node->decl),
848	"%qD is not eligible for internal %<strub%>"
849	" because of attribute %<noclone%>",
850	node->decl);
851	}
852
853	if (node->has_gimple_body_p ())
854	{
855	for (cgraph_edge *e = node->callees; e; e = e->next_callee)
856	{
857	tree cdecl = e->callee->decl;
858	if (!((fndecl_built_in_p (node: cdecl, name1: BUILT_IN_VA_START)
859	&& cdecl != builtin_decl_explicit (fncode: BUILT_IN_VA_START))
860	|| fndecl_built_in_p (node: cdecl, name1: BUILT_IN_NEXT_ARG)
861	|| fndecl_built_in_p (node: cdecl, name1: BUILT_IN_RETURN_ADDRESS)))
862	continue;
863
864	result = false;
865
866	if (!report)
867	return result;
868
869	sorry_at (e->call_stmt
870	? gimple_location (g: e->call_stmt)
871	: DECL_SOURCE_LOCATION (node->decl),
872	"%qD is not eligible for internal %<strub%> "
873	"because it calls %qD",
874	node->decl, cdecl);
875	}
876
877	struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
878	if (fun->has_nonlocal_label)
879	{
880	result = false;
881
882	if (!report)
883	return result;
884
885	sorry_at (DECL_SOURCE_LOCATION (node->decl),
886	"%qD is not eligible for internal %<strub%> "
887	"because it contains a non-local goto target",
888	node->decl);
889	}
890
891	if (fun->has_forced_label_in_static)
892	{
893	result = false;
894
895	if (!report)
896	return result;
897
898	sorry_at (DECL_SOURCE_LOCATION (node->decl),
899	"%qD is not eligible for internal %<strub%> "
900	"because the address of a local label escapes",
901	node->decl);
902	}
903
904	/* Catch any other case that would prevent versioning/cloning
905	so as to also have it covered above. */
906	gcc_checking_assert (!result /* || !node->has_gimple_body_p () */
907	|| tree_versionable_function_p (node->decl));
908
909
910	/* Label values references are not preserved when copying. If referenced
911	in nested functions, as in 920415-1.c and 920721-4.c their decls get
912	remapped independently. The exclusion below might be too broad, in
913	that we might be able to support correctly cases in which the labels
914	are only used internally in a function, but disconnecting forced labels
915	from their original declarations is undesirable in general. */
916	basic_block bb;
917	FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
918	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
919	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
920	{
921	glabel *label_stmt = dyn_cast <glabel *> (p: gsi_stmt (i: gsi));
922	tree target;
923
924	if (!label_stmt)
925	break;
926
927	target = gimple_label_label (gs: label_stmt);
928
929	if (!FORCED_LABEL (target))
930	continue;
931
932	result = false;
933
934	if (!report)
935	return result;
936
937	sorry_at (gimple_location (g: label_stmt),
938	"internal %<strub%> does not support forced labels");
939	}
940	}
941
942	if (list_length (TYPE_ARG_TYPES (TREE_TYPE (node->decl)))
943	>= ((HOST_WIDE_INT_1 << IPA_PARAM_MAX_INDEX_BITS)
944	- STRUB_INTERNAL_MAX_EXTRA_ARGS))
945	{
946	result = false;
947
948	if (!report)
949	return result;
950
951	sorry_at (DECL_SOURCE_LOCATION (node->decl),
952	"%qD has too many arguments for internal %<strub%>",
953	node->decl);
954	}
955
956	return result;
957	}
958
959	/* Return TRUE iff NODE has any strub-requiring local variable, or accesses (as
960	in reading) any variable through a strub-requiring type. */
961
962	static bool
963	strub_from_body_p (cgraph_node *node)
964	{
965	if (!node->has_gimple_body_p ())
966	return false;
967
968	/* If any local variable is marked for strub... */
969	unsigned i;
970	tree var;
971	FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (node->decl),
972	i, var)
973	if (get_strub_mode_from_type (TREE_TYPE (var))
974	!= STRUB_DISABLED)
975	return true;
976
977	/* Now scan the body for loads with strub-requiring types.
978	??? Compound types don't propagate the strub requirement to
979	component types. */
980	basic_block bb;
981	FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
982	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
983	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
984	{
985	gimple *stmt = gsi_stmt (i: gsi);
986
987	if (!gimple_assign_load_p (stmt))
988	continue;
989
990	tree rhs = gimple_assign_rhs1 (gs: stmt);
991	if (get_strub_mode_from_type (TREE_TYPE (rhs))
992	!= STRUB_DISABLED)
993	return true;
994	}
995
996	return false;
997	}
998
999	/* Return TRUE iff node is associated with a builtin that should be callable
1000	from strub contexts. */
1001
1002	static inline bool
1003	strub_callable_builtin_p (cgraph_node *node)
1004	{
1005	if (DECL_BUILT_IN_CLASS (node->decl) != BUILT_IN_NORMAL)
1006	return false;
1007
1008	enum built_in_function fcode = DECL_FUNCTION_CODE (decl: node->decl);
1009
1010	switch (fcode)
1011	{
1012	case BUILT_IN_NONE:
1013	gcc_unreachable ();
1014
1015	/* This temporarily allocates stack for the call, and we can't reasonably
1016	update the watermark for that. Besides, we don't check the actual call
1017	target, nor its signature, and it seems to be overkill to as much as
1018	try to do so. */
1019	case BUILT_IN_APPLY:
1020	return false;
1021
1022	/* Conversely, this shouldn't be called from within strub contexts, since
1023	the caller may have had its signature modified. STRUB_INTERNAL is ok,
1024	the call will remain in the STRUB_WRAPPER, and removed from the
1025	STRUB_WRAPPED clone. */
1026	case BUILT_IN_APPLY_ARGS:
1027	return false;
1028
1029	/* ??? Make all other builtins callable. We wish to make any builtin call
1030	the compiler might introduce on its own callable. Anything that is
1031	predictable enough as to be known not to allow stack data that should
1032	be strubbed to unintentionally escape to non-strub contexts can be
1033	allowed, and pretty much every builtin appears to fit this description.
1034	The exceptions to this rule seem to be rare, and only available as
1035	explicit __builtin calls, so let's keep it simple and allow all of
1036	them... */
1037	default:
1038	return true;
1039	}
1040	}
1041
1042	/* Compute the strub mode to be used for NODE. STRUB_ATTR should be the strub
1043	attribute,found for NODE, if any. */
1044
1045	static enum strub_mode
1046	compute_strub_mode (cgraph_node *node, tree strub_attr)
1047	{
1048	enum strub_mode req_mode = get_strub_mode_from_attr (strub_attr);
1049
1050	gcc_checking_assert (flag_strub >= -2 && flag_strub <= 3);
1051
1052	/* Symbolic encodings of the -fstrub-* flags. */
1053	/* Enable strub when explicitly requested through attributes to functions or
1054	variables, reporting errors if the requests cannot be satisfied. */
1055	const bool strub_flag_auto = flag_strub < 0;
1056	/* strub_flag_auto with strub call verification; without this, functions are
1057	implicitly callable. */
1058	const bool strub_flag_strict = flag_strub < -1;
1059	/* Disable strub altogether, ignore attributes entirely. */
1060	const bool strub_flag_disabled = flag_strub == 0;
1061	/* On top of _auto, also enable strub implicitly for functions that can
1062	safely undergo at-calls strubbing. Internal mode will still be used in
1063	functions that request it explicitly with attribute strub(2), or when the
1064	function body requires strubbing and at-calls strubbing is not viable. */
1065	const bool strub_flag_at_calls = flag_strub == 1;
1066	/* On top of default, also enable strub implicitly for functions that can
1067	safely undergo internal strubbing. At-calls mode will still be used in
1068	functions that requiest it explicitly with attribute strub() or strub(1),
1069	or when the function body requires strubbing and internal strubbing is not
1070	viable. */
1071	const bool strub_flag_internal = flag_strub == 2;
1072	/* On top of default, also enable strub implicitly for functions that can
1073	safely undergo strubbing in either mode. When both modes are viable,
1074	at-calls is preferred. */
1075	const bool strub_flag_either = flag_strub == 3;
1076	/* Besides the default behavior, enable strub implicitly for all viable
1077	functions. */
1078	const bool strub_flag_viable = flag_strub > 0;
1079
1080	/* The consider_* variables should be TRUE if selecting the corresponding
1081	strub modes would be consistent with requests from attributes and command
1082	line flags. Attributes associated with functions pretty much mandate a
1083	selection, and should report an error if not satisfied; strub_flag_auto
1084	implicitly enables some viable strub mode if that's required by references
1085	to variables marked for strub; strub_flag_viable enables strub if viable
1086	(even when favoring one mode, body-requested strub can still be satisfied
1087	by either mode), and falls back to callable, silently unless variables
1088	require strubbing. */
1089
1090	const bool consider_at_calls
1091	= (!strub_flag_disabled
1092	&& (strub_attr
1093	? req_mode == STRUB_AT_CALLS
1094	: true));
1095	const bool consider_internal
1096	= (!strub_flag_disabled
1097	&& (strub_attr
1098	? req_mode == STRUB_INTERNAL
1099	: true));
1100
1101	const bool consider_callable
1102	= (!strub_flag_disabled
1103	&& (strub_attr
1104	? req_mode == STRUB_CALLABLE
1105	: (!strub_flag_strict
1106	|| strub_callable_builtin_p (node))));
1107
1108	/* This is a shorthand for either strub-enabled mode. */
1109	const bool consider_strub
1110	= (consider_at_calls || consider_internal);
1111
1112	/* We can cope with always_inline functions even with noipa and noclone,
1113	because we just leave them alone. */
1114	const bool is_always_inline
1115	= strub_always_inline_p (node);
1116
1117	/* Strubbing in general, and each specific strub mode, may have its own set of
1118	requirements. We require noipa for strubbing, either because of cloning
1119	required for internal strub, or because of caller enumeration required for
1120	at-calls strub. We don't consider the at-calls mode eligible if it's not
1121	even considered, it has no further requirements. Internal mode requires
1122	cloning and the absence of certain features in the body and, like at-calls,
1123	it's not eligible if it's not even under consideration.
1124
1125	??? Do we need target hooks for further constraints? E.g., x86's
1126	"interrupt" attribute breaks internal strubbing because the wrapped clone
1127	carries the attribute and thus isn't callable; in this case, we could use a
1128	target hook to adjust the clone instead. */
1129	const bool strub_eligible
1130	= (consider_strub
1131	&& (is_always_inline || can_strub_p (node)));
1132	const bool at_calls_eligible
1133	= (consider_at_calls && strub_eligible
1134	&& can_strub_at_calls_p (node));
1135	const bool internal_eligible
1136	= (consider_internal && strub_eligible
1137	&& (is_always_inline
1138	|| can_strub_internally_p (node)));
1139
1140	/* In addition to the strict eligibility requirements, some additional
1141	constraints are placed on implicit selection of certain modes. These do
1142	not prevent the selection of a mode if explicitly specified as part of a
1143	function interface (the strub attribute), but they may prevent modes from
1144	being selected by the command line or by function bodies. The only actual
1145	constraint is on at-calls mode: since we change the function's exposed
1146	signature, we won't do it implicitly if the function can possibly be used
1147	in ways that do not expect the signature change, e.g., if the function is
1148	available to or interposable by other units, if its address is taken,
1149	etc. */
1150	const bool at_calls_viable
1151	= (at_calls_eligible
1152	&& (strub_attr
1153	|| (node->has_gimple_body_p ()
1154	&& (!node->externally_visible
1155	|| (node->binds_to_current_def_p ()
1156	&& node->can_be_local_p ()))
1157	&& node->only_called_directly_p ()
1158	&& !called_with_type_override_p (node))));
1159	const bool internal_viable
1160	= (internal_eligible);
1161
1162	/* Shorthand. */
1163	const bool strub_viable
1164	= (at_calls_viable || internal_viable);
1165
1166	/* We wish to analyze the body, to look for implicit requests for strub, both
1167	to implicitly enable it when the body calls for it, and to report errors if
1168	the body calls for it but neither mode is viable (even if that follows from
1169	non-eligibility because of the explicit specification of some non-strubbing
1170	mode). We can refrain from scanning the body only in rare circumstances:
1171	when strub is enabled by a function attribute (scanning might be redundant
1172	in telling us to also enable it), and when we are enabling strub implicitly
1173	but there are non-viable modes: we want to know whether strubbing is
1174	required, to fallback to another mode, even if we're only enabling a
1175	certain mode, or, when either mode would do, to report an error if neither
1176	happens to be viable. */
1177	const bool analyze_body
1178	= (strub_attr
1179	? !consider_strub
1180	: (strub_flag_auto
1181	|| (strub_flag_viable && (!at_calls_viable && !internal_viable))
1182	|| (strub_flag_either && !strub_viable)));
1183
1184	/* Cases in which strubbing is enabled or disabled by strub_flag_auto.
1185	Unsatisfiable requests ought to be reported. */
1186	const bool strub_required
1187	= ((strub_attr && consider_strub)
1188	|| (analyze_body && strub_from_body_p (node)));
1189
1190	/* Besides the required cases, we want to abide by the requests to enabling on
1191	an if-viable basis. */
1192	const bool strub_enable
1193	= (strub_required
1194	|| (strub_flag_at_calls && at_calls_viable)
1195	|| (strub_flag_internal && internal_viable)
1196	|| (strub_flag_either && strub_viable));
1197
1198	/* And now we're finally ready to select a mode that abides by the viability
1199	and eligibility constraints, and that satisfies the strubbing requirements
1200	and requests, subject to the constraints. If both modes are viable and
1201	strub is to be enabled, pick STRUB_AT_CALLS unless STRUB_INTERNAL was named
1202	as preferred. */
1203	const enum strub_mode mode
1204	= ((strub_enable && is_always_inline)
1205	? (strub_required ? STRUB_INLINABLE : STRUB_CALLABLE)
1206	: (strub_enable && internal_viable
1207	&& (strub_flag_internal || !at_calls_viable))
1208	? STRUB_INTERNAL
1209	: (strub_enable && at_calls_viable)
1210	? (strub_required && !strub_attr
1211	? STRUB_AT_CALLS_OPT
1212	: STRUB_AT_CALLS)
1213	: consider_callable
1214	? STRUB_CALLABLE
1215	: STRUB_DISABLED);
1216
1217	switch (mode)
1218	{
1219	case STRUB_CALLABLE:
1220	if (is_always_inline)
1221	break;
1222	/* Fall through. */
1223
1224	case STRUB_DISABLED:
1225	if (strub_enable && !strub_attr)
1226	{
1227	gcc_checking_assert (analyze_body);
1228	error_at (DECL_SOURCE_LOCATION (node->decl),
1229	"%qD requires %<strub%>,"
1230	" but no viable %<strub%> mode was found",
1231	node->decl);
1232	break;
1233	}
1234	/* Fall through. */
1235
1236	case STRUB_AT_CALLS:
1237	case STRUB_INTERNAL:
1238	case STRUB_INLINABLE:
1239	/* Differences from an mode requested through a function attribute are
1240	reported in set_strub_mode_to. */
1241	break;
1242
1243	case STRUB_AT_CALLS_OPT:
1244	/* Functions that select this mode do so because of references to strub
1245	variables. Even if we choose at-calls as an optimization, the
1246	requirements for internal strub must still be satisfied. Optimization
1247	options may render implicit at-calls strub not viable (-O0 sets
1248	force_output for static non-inline functions), and it would not be good
1249	if changing optimization options turned a well-formed into an
1250	ill-formed one. */
1251	if (!internal_viable)
1252	can_strub_internally_p (node, report: true);
1253	break;
1254
1255	case STRUB_WRAPPED:
1256	case STRUB_WRAPPER:
1257	default:
1258	gcc_unreachable ();
1259	}
1260
1261	return mode;
1262	}
1263
1264	/* Set FNDT's strub mode to MODE; FNDT may be a function decl or
1265	function type. If OVERRIDE, do not check whether a mode is already
1266	set. */
1267
1268	static void
1269	strub_set_fndt_mode_to (tree fndt, enum strub_mode mode, bool override)
1270	{
1271	gcc_checking_assert (override
1272	|| !(DECL_P (fndt)
1273	? get_strub_attr_from_decl (fndt)
1274	: get_strub_attr_from_type (fndt)));
1275
1276	tree attr = tree_cons (get_identifier ("strub"),
1277	get_strub_mode_attr_value (mode),
1278	NULL_TREE);
1279	tree *attrp = NULL;
1280	if (DECL_P (fndt))
1281	{
1282	gcc_checking_assert (FUNC_OR_METHOD_TYPE_P (TREE_TYPE (fndt)));
1283	attrp = &DECL_ATTRIBUTES (fndt);
1284	}
1285	else if (FUNC_OR_METHOD_TYPE_P (fndt))
1286	attrp = &TYPE_ATTRIBUTES (fndt);
1287	else
1288	gcc_unreachable ();
1289
1290	TREE_CHAIN (attr) = *attrp;
1291	*attrp = attr;
1292	}
1293
1294	/* Set FNDT's strub mode to callable.
1295	FNDT may be a function decl or a function type. */
1296
1297	void
1298	strub_make_callable (tree fndt)
1299	{
1300	strub_set_fndt_mode_to (fndt, mode: STRUB_CALLABLE, override: false);
1301	}
1302
1303	/* Set NODE to strub MODE. Report incompatibilities between MODE and the mode
1304	requested through explicit attributes, and cases of non-eligibility. */
1305
1306	static void
1307	set_strub_mode_to (cgraph_node *node, enum strub_mode mode)
1308	{
1309	tree attr = get_strub_attr_from_decl (decl: node->decl);
1310	enum strub_mode req_mode = get_strub_mode_from_attr (strub_attr: attr);
1311
1312	if (attr)
1313	{
1314	/* Check for and report incompatible mode changes. */
1315	if (mode != req_mode
1316	&& !(req_mode == STRUB_INTERNAL
1317	&& (mode == STRUB_WRAPPED
1318	|| mode == STRUB_WRAPPER))
1319	&& !((req_mode == STRUB_INTERNAL
1320	|| req_mode == STRUB_AT_CALLS
1321	|| req_mode == STRUB_CALLABLE)
1322	&& mode == STRUB_INLINABLE))
1323	{
1324	error_at (DECL_SOURCE_LOCATION (node->decl),
1325	"%<strub%> mode %qE selected for %qD, when %qE was requested",
1326	get_strub_mode_attr_parm (mode),
1327	node->decl,
1328	get_strub_mode_attr_parm (mode: req_mode));
1329	if (node->alias)
1330	{
1331	cgraph_node *target = node->ultimate_alias_target ();
1332	if (target != node)
1333	error_at (DECL_SOURCE_LOCATION (target->decl),
1334	"the incompatible selection was determined"
1335	" by ultimate alias target %qD",
1336	target->decl);
1337	}
1338
1339	/* Report any incompatibilities with explicitly-requested strub. */
1340	switch (req_mode)
1341	{
1342	case STRUB_AT_CALLS:
1343	can_strub_at_calls_p (node, report: true);
1344	break;
1345
1346	case STRUB_INTERNAL:
1347	can_strub_internally_p (node, report: true);
1348	break;
1349
1350	default:
1351	break;
1352	}
1353	}
1354
1355	/* Drop any incompatible strub attributes leading the decl attribute
1356	chain. Return if we find one with the mode we need. */
1357	for (;;)
1358	{
1359	if (mode == req_mode)
1360	return;
1361
1362	if (DECL_ATTRIBUTES (node->decl) != attr)
1363	break;
1364
1365	DECL_ATTRIBUTES (node->decl) = TREE_CHAIN (attr);
1366	attr = get_strub_attr_from_decl (decl: node->decl);
1367	if (!attr)
1368	break;
1369
1370	req_mode = get_strub_mode_from_attr (strub_attr: attr);
1371	}
1372	}
1373	else if (mode == req_mode)
1374	return;
1375
1376	strub_set_fndt_mode_to (fndt: node->decl, mode, override: attr);
1377	}
1378
1379	/* Compute and set NODE's strub mode. */
1380
1381	static void
1382	set_strub_mode (cgraph_node *node)
1383	{
1384	tree attr = get_strub_attr_from_decl (decl: node->decl);
1385
1386	if (attr)
1387	switch (get_strub_mode_from_attr (strub_attr: attr))
1388	{
1389	/* These can't have been requested through user attributes, so we must
1390	have already gone through them. */
1391	case STRUB_WRAPPER:
1392	case STRUB_WRAPPED:
1393	case STRUB_INLINABLE:
1394	case STRUB_AT_CALLS_OPT:
1395	return;
1396
1397	case STRUB_DISABLED:
1398	case STRUB_AT_CALLS:
1399	case STRUB_INTERNAL:
1400	case STRUB_CALLABLE:
1401	break;
1402
1403	default:
1404	gcc_unreachable ();
1405	}
1406
1407	cgraph_node *xnode = node;
1408	if (node->alias)
1409	xnode = node->ultimate_alias_target ();
1410	/* Weakrefs may remain unresolved (the above will return node) if
1411	their targets are not defined, so make sure we compute a strub
1412	mode for them, instead of defaulting to STRUB_DISABLED and
1413	rendering them uncallable. */
1414	enum strub_mode mode = (xnode != node && !xnode->alias
1415	? get_strub_mode (node: xnode)
1416	: compute_strub_mode (node, strub_attr: attr));
1417
1418	set_strub_mode_to (node, mode);
1419	}
1420
1421
1422	/* Non-strub functions shouldn't be called from within strub contexts,
1423	except through callable ones. Always inline strub functions can
1424	only be called from strub functions. */
1425
1426	static bool
1427	strub_callable_from_p (strub_mode caller_mode, strub_mode callee_mode)
1428	{
1429	switch (caller_mode)
1430	{
1431	case STRUB_WRAPPED:
1432	case STRUB_AT_CALLS_OPT:
1433	case STRUB_AT_CALLS:
1434	case STRUB_INTERNAL:
1435	case STRUB_INLINABLE:
1436	break;
1437
1438	case STRUB_WRAPPER:
1439	case STRUB_DISABLED:
1440	case STRUB_CALLABLE:
1441	return callee_mode != STRUB_INLINABLE;
1442
1443	default:
1444	gcc_unreachable ();
1445	}
1446
1447	switch (callee_mode)
1448	{
1449	case STRUB_WRAPPED:
1450	case STRUB_AT_CALLS:
1451	case STRUB_INLINABLE:
1452	break;
1453
1454	case STRUB_AT_CALLS_OPT:
1455	case STRUB_INTERNAL:
1456	case STRUB_WRAPPER:
1457	return (flag_strub >= -1);
1458
1459	case STRUB_DISABLED:
1460	return false;
1461
1462	case STRUB_CALLABLE:
1463	break;
1464
1465	default:
1466	gcc_unreachable ();
1467	}
1468
1469	return true;
1470	}
1471
1472	/* Return TRUE iff CALLEE can be inlined into CALLER. We wish to avoid inlining
1473	WRAPPED functions back into their WRAPPERs. More generally, we wish to avoid
1474	inlining strubbed functions into non-strubbed ones. CALLER doesn't have to
1475	be an immediate caller of CALLEE: the immediate caller may have already been
1476	cloned for inlining, and then CALLER may be further up the original call
1477	chain. ??? It would be nice if our own caller would retry inlining callee
1478	if caller gets inlined. */
1479
1480	bool
1481	strub_inlinable_to_p (cgraph_node *callee, cgraph_node *caller)
1482	{
1483	strub_mode callee_mode = get_strub_mode (node: callee);
1484
1485	switch (callee_mode)
1486	{
1487	case STRUB_WRAPPED:
1488	case STRUB_AT_CALLS:
1489	case STRUB_INTERNAL:
1490	case STRUB_INLINABLE:
1491	case STRUB_AT_CALLS_OPT:
1492	break;
1493
1494	case STRUB_WRAPPER:
1495	case STRUB_DISABLED:
1496	case STRUB_CALLABLE:
1497	/* When we consider inlining, we've already verified callability, so we
1498	can even inline callable and then disabled into a strub context. That
1499	will get strubbed along with the context, so it's hopefully not a
1500	problem. */
1501	return true;
1502
1503	default:
1504	gcc_unreachable ();
1505	}
1506
1507	strub_mode caller_mode = get_strub_mode (node: caller);
1508
1509	switch (caller_mode)
1510	{
1511	case STRUB_WRAPPED:
1512	case STRUB_AT_CALLS:
1513	case STRUB_INTERNAL:
1514	case STRUB_INLINABLE:
1515	case STRUB_AT_CALLS_OPT:
1516	return true;
1517
1518	case STRUB_WRAPPER:
1519	case STRUB_DISABLED:
1520	case STRUB_CALLABLE:
1521	break;
1522
1523	default:
1524	gcc_unreachable ();
1525	}
1526
1527	return false;
1528	}
1529
1530	/* Check that types T1 and T2 are strub-compatible. Return 1 if the strub modes
1531	are the same, 2 if they are interchangeable, and 0 otherwise. */
1532
1533	int
1534	strub_comptypes (tree t1, tree t2)
1535	{
1536	if (TREE_CODE (t1) != TREE_CODE (t2))
1537	return 0;
1538
1539	enum strub_mode m1 = get_strub_mode_from_type (type: t1);
1540	enum strub_mode m2 = get_strub_mode_from_type (type: t2);
1541
1542	if (m1 == m2)
1543	return 1;
1544
1545	/* We're dealing with types, so only strub modes that can be selected by
1546	attributes in the front end matter. If either mode is at-calls (for
1547	functions) or internal (for variables), the conversion is not
1548	compatible. */
1549	bool var_p = !FUNC_OR_METHOD_TYPE_P (t1);
1550	enum strub_mode mr = var_p ? STRUB_INTERNAL : STRUB_AT_CALLS;
1551	if (m1 == mr || m2 == mr)
1552	return 0;
1553
1554	return 2;
1555	}
1556
1557	/* Return the effective strub mode used for CALL, and set *TYPEP to
1558	the effective type used for the call. The effective type and mode
1559	are those of the callee, unless the call involves a typecast. */
1560
1561	static enum strub_mode
1562	effective_strub_mode_for_call (gcall *call, tree *typep)
1563	{
1564	tree type;
1565	enum strub_mode mode;
1566
1567	if (strub_call_fntype_override_p (gs: call))
1568	{
1569	type = gimple_call_fntype (gs: call);
1570	mode = get_strub_mode_from_type (type);
1571	}
1572	else
1573	{
1574	type = TREE_TYPE (TREE_TYPE (gimple_call_fn (call)));
1575	tree decl = gimple_call_fndecl (gs: call);
1576	if (decl)
1577	mode = get_strub_mode_from_fndecl (fndecl: decl);
1578	else
1579	mode = get_strub_mode_from_type (type);
1580	}
1581
1582	if (typep)
1583	*typep = type;
1584
1585	return mode;
1586	}
1587
1588	/* Create a distinct copy of the type of NODE's function, and change
1589	the fntype of all calls to it with the same main type to the new
1590	type. */
1591
1592	static void
1593	distinctify_node_type (cgraph_node *node)
1594	{
1595	tree old_type = TREE_TYPE (node->decl);
1596	tree new_type = build_distinct_type_copy (old_type);
1597	tree new_ptr_type = NULL_TREE;
1598
1599	/* Remap any calls to node->decl that use old_type, or a variant
1600	thereof, to new_type as well. We don't look for aliases, their
1601	declarations will have their types changed independently, and
1602	we'll adjust their fntypes then. */
1603	for (cgraph_edge *e = node->callers; e; e = e->next_caller)
1604	{
1605	if (!e->call_stmt)
1606	continue;
1607	tree fnaddr = gimple_call_fn (gs: e->call_stmt);
1608	gcc_checking_assert (TREE_CODE (fnaddr) == ADDR_EXPR
1609	&& TREE_OPERAND (fnaddr, 0) == node->decl);
1610	if (strub_call_fntype_override_p (gs: e->call_stmt))
1611	continue;
1612	if (!new_ptr_type)
1613	new_ptr_type = build_pointer_type (new_type);
1614	TREE_TYPE (fnaddr) = new_ptr_type;
1615	gimple_call_set_fntype (call_stmt: e->call_stmt, fntype: new_type);
1616	}
1617
1618	TREE_TYPE (node->decl) = new_type;
1619	}
1620
1621	/* Return TRUE iff TYPE and any variants have the same strub mode. */
1622
1623	static bool
1624	same_strub_mode_in_variants_p (tree type)
1625	{
1626	enum strub_mode mode = get_strub_mode_from_type (type);
1627
1628	for (tree other = TYPE_MAIN_VARIANT (type);
1629	other != NULL_TREE; other = TYPE_NEXT_VARIANT (other))
1630	if (type != other && mode != get_strub_mode_from_type (type: other))
1631	return false;
1632
1633	/* Check that the canonical type, if set, either is in the same
1634	variant chain, or has the same strub mode as type. Also check
1635	the variants of the canonical type. */
1636	if (TYPE_CANONICAL (type)
1637	&& (TYPE_MAIN_VARIANT (TYPE_CANONICAL (type))
1638	!= TYPE_MAIN_VARIANT (type)))
1639	{
1640	if (mode != get_strub_mode_from_type (TYPE_CANONICAL (type)))
1641	return false;
1642	else
1643	return same_strub_mode_in_variants_p (TYPE_CANONICAL (type));
1644	}
1645
1646	return true;
1647	}
1648
1649	/* Check that strub functions don't call non-strub functions, and that
1650	always_inline strub functions are only called by strub
1651	functions. */
1652
1653	static void
1654	verify_strub ()
1655	{
1656	cgraph_node *node;
1657
1658	/* It's expected that check strub-wise pointer type compatibility of variables
1659	and of functions is already taken care of by front-ends, on account of the
1660	attribute's being marked as affecting type identity and of the creation of
1661	distinct types. */
1662
1663	/* Check that call targets in strub contexts have strub-callable types. */
1664
1665	FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
1666	{
1667	enum strub_mode caller_mode = get_strub_mode (node);
1668
1669	for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
1670	{
1671	gcc_checking_assert (e->indirect_unknown_callee);
1672
1673	if (!e->call_stmt)
1674	continue;
1675
1676	enum strub_mode callee_mode
1677	= effective_strub_mode_for_call (call: e->call_stmt, NULL);
1678
1679	if (!strub_callable_from_p (caller_mode, callee_mode))
1680	error_at (gimple_location (g: e->call_stmt),
1681	"indirect non-%<strub%> call in %<strub%> context %qD",
1682	node->decl);
1683	}
1684
1685	for (cgraph_edge *e = node->callees; e; e = e->next_callee)
1686	{
1687	gcc_checking_assert (!e->indirect_unknown_callee);
1688
1689	if (!e->call_stmt)
1690	continue;
1691
1692	tree callee_fntype;
1693	enum strub_mode callee_mode
1694	= effective_strub_mode_for_call (call: e->call_stmt, typep: &callee_fntype);
1695
1696	if (!strub_callable_from_p (caller_mode, callee_mode))
1697	{
1698	if (callee_mode == STRUB_INLINABLE)
1699	error_at (gimple_location (g: e->call_stmt),
1700	"calling %<always_inline%> %<strub%> %qD"
1701	" in non-%<strub%> context %qD",
1702	e->callee->decl, node->decl);
1703	else if (fndecl_built_in_p (node: e->callee->decl, name1: BUILT_IN_APPLY_ARGS)
1704	&& caller_mode == STRUB_INTERNAL)
1705	/* This is ok, it will be kept in the STRUB_WRAPPER, and removed
1706	from the STRUB_WRAPPED's strub context. */
1707	continue;
1708	else if (!strub_call_fntype_override_p (gs: e->call_stmt))
1709	error_at (gimple_location (g: e->call_stmt),
1710	"calling non-%<strub%> %qD in %<strub%> context %qD",
1711	e->callee->decl, node->decl);
1712	else
1713	error_at (gimple_location (g: e->call_stmt),
1714	"calling %qD using non-%<strub%> type %qT"
1715	" in %<strub%> context %qD",
1716	e->callee->decl, callee_fntype, node->decl);
1717	}
1718	}
1719	}
1720	}
1721
1722	namespace {
1723
1724	/* Define a pass to compute strub modes. */
1725	const pass_data pass_data_ipa_strub_mode = {
1726	.type: SIMPLE_IPA_PASS,
1727	.name: "strubm",
1728	.optinfo_flags: OPTGROUP_NONE,
1729	.tv_id: TV_NONE,
1730	PROP_cfg, // properties_required
1731	.properties_provided: 0, // properties_provided
1732	.properties_destroyed: 0, // properties_destroyed
1733	.todo_flags_start: 0, // properties_start
1734	.todo_flags_finish: 0, // properties_finish
1735	};
1736
1737	class pass_ipa_strub_mode : public simple_ipa_opt_pass
1738	{
1739	public:
1740	pass_ipa_strub_mode (gcc::context *ctxt)
1741	: simple_ipa_opt_pass (pass_data_ipa_strub_mode, ctxt)
1742	{}
1743	opt_pass *clone () { return new pass_ipa_strub_mode (m_ctxt); }
1744	virtual bool gate (function *) {
1745	/* In relaxed (-3) and strict (-4) settings, that only enable strub at a
1746	function or variable attribute's request, the attribute handler changes
1747	flag_strub to -1 or -2, respectively, if any strub-enabling occurence of
1748	the attribute is found. Therefore, if it remains at -3 or -4, nothing
1749	that would enable strub was found, so we can disable it and avoid the
1750	overhead. */
1751	if (flag_strub < -2)
1752	flag_strub = 0;
1753	return flag_strub;
1754	}
1755	virtual unsigned int execute (function *);
1756	};
1757
1758	/* Define a pass to introduce strub transformations. */
1759	const pass_data pass_data_ipa_strub = {
1760	.type: SIMPLE_IPA_PASS,
1761	.name: "strub",
1762	.optinfo_flags: OPTGROUP_NONE,
1763	.tv_id: TV_NONE,
1764	PROP_cfg | PROP_ssa, // properties_required
1765	.properties_provided: 0, // properties_provided
1766	.properties_destroyed: 0, // properties_destroyed
1767	.todo_flags_start: 0, // properties_start
1768	TODO_update_ssa
1769	| TODO_cleanup_cfg
1770	| TODO_rebuild_cgraph_edges
1771	| TODO_verify_il, // properties_finish
1772	};
1773
1774	class pass_ipa_strub : public simple_ipa_opt_pass
1775	{
1776	public:
1777	pass_ipa_strub (gcc::context *ctxt)
1778	: simple_ipa_opt_pass (pass_data_ipa_strub, ctxt)
1779	{}
1780	opt_pass *clone () { return new pass_ipa_strub (m_ctxt); }
1781	virtual bool gate (function *) { return flag_strub && !seen_error (); }
1782	virtual unsigned int execute (function *);
1783
1784	/* Define on demand and cache some types we use often. */
1785	#define DEF_TYPE(IDX, NAME, INIT) \
1786	static inline tree get_ ## NAME () { \
1787	int idx = STRUB_TYPE_BASE + IDX; \
1788	static tree type = strub_cache[idx]; \
1789	if (!type) \
1790	strub_cache[idx] = type = (INIT); \
1791	return type; \
1792	}
1793
1794	/* Use a distinct ptr_type_node to denote the watermark, so that we can
1795	recognize it in arg lists and avoid modifying types twice. */
1796	DEF_TYPE (0, wmt, build_variant_type_copy (ptr_type_node))
1797
1798	DEF_TYPE (1, pwmt, build_reference_type (get_wmt ()))
1799
1800	DEF_TYPE (2, qpwmt,
1801	build_qualified_type (get_pwmt (),
1802	TYPE_QUAL_RESTRICT
1803	/* | TYPE_QUAL_CONST */))
1804
1805	DEF_TYPE (3, qptr,
1806	build_qualified_type (ptr_type_node,
1807	TYPE_QUAL_RESTRICT
1808	| TYPE_QUAL_CONST))
1809
1810	DEF_TYPE (4, qpvalst,
1811	build_qualified_type (build_reference_type
1812	(va_list_type_node),
1813	TYPE_QUAL_RESTRICT
1814	/* | TYPE_QUAL_CONST */))
1815
1816	#undef DEF_TYPE
1817
1818	/* Define non-strub builtins on demand. */
1819	#define DEF_NM_BUILTIN(NAME, CODE, FNTYPELIST) \
1820	static tree get_ ## NAME () { \
1821	tree decl = builtin_decl_explicit (CODE); \
1822	if (!decl) \
1823	{ \
1824	tree type = build_function_type_list FNTYPELIST; \
1825	decl = add_builtin_function \
1826	("__builtin_" #NAME, \
1827	type, CODE, BUILT_IN_NORMAL, \
1828	NULL, NULL); \
1829	TREE_NOTHROW (decl) = true; \
1830	set_builtin_decl ((CODE), decl, true); \
1831	} \
1832	return decl; \
1833	}
1834
1835	DEF_NM_BUILTIN (stack_address,
1836	BUILT_IN_STACK_ADDRESS,
1837	(ptr_type_node, NULL))
1838
1839	#undef DEF_NM_BUILTIN
1840
1841	/* Define strub builtins on demand. */
1842	#define DEF_SS_BUILTIN(NAME, FNSPEC, CODE, FNTYPELIST) \
1843	static tree get_ ## NAME () { \
1844	tree decl = builtin_decl_explicit (CODE); \
1845	if (!decl) \
1846	{ \
1847	tree type = build_function_type_list FNTYPELIST; \
1848	tree attrs = NULL; \
1849	if (FNSPEC) \
1850	attrs = tree_cons (get_identifier ("fn spec"), \
1851	build_tree_list \
1852	(NULL_TREE, \
1853	build_string (strlen (FNSPEC), \
1854	(FNSPEC))), \
1855	attrs); \
1856	decl = add_builtin_function_ext_scope \
1857	("__builtin___strub_" #NAME, \
1858	type, CODE, BUILT_IN_NORMAL, \
1859	"__strub_" #NAME, attrs); \
1860	TREE_NOTHROW (decl) = true; \
1861	set_builtin_decl ((CODE), decl, true); \
1862	} \
1863	return decl; \
1864	}
1865
1866	DEF_SS_BUILTIN (enter, ". Ot",
1867	BUILT_IN___STRUB_ENTER,
1868	(void_type_node, get_qpwmt (), NULL))
1869	DEF_SS_BUILTIN (update, ". Wt",
1870	BUILT_IN___STRUB_UPDATE,
1871	(void_type_node, get_qpwmt (), NULL))
1872	DEF_SS_BUILTIN (leave, ". w ",
1873	BUILT_IN___STRUB_LEAVE,
1874	(void_type_node, get_qpwmt (), NULL))
1875
1876	#undef DEF_SS_BUILTIN
1877
1878	/* Define strub identifiers on demand. */
1879	#define DEF_IDENT(IDX, NAME) \
1880	static inline tree get_ ## NAME () { \
1881	int idx = STRUB_IDENT_BASE + IDX; \
1882	tree identifier = strub_cache[idx]; \
1883	if (!identifier) \
1884	strub_cache[idx] = identifier = get_identifier (".strub." #NAME); \
1885	return identifier; \
1886	}
1887
1888	DEF_IDENT (0, watermark_ptr)
1889	DEF_IDENT (1, va_list_ptr)
1890	DEF_IDENT (2, apply_args)
1891
1892	#undef DEF_IDENT
1893
1894	static inline int adjust_at_calls_type (tree);
1895	static inline void adjust_at_calls_call (cgraph_edge *, int, tree);
1896	static inline void adjust_at_calls_calls (cgraph_node *);
1897
1898	/* Add to SEQ a call to the strub watermark update builtin, taking NODE's
1899	location if given. Optionally add the corresponding edge from NODE, with
1900	execution frequency COUNT. Return the modified SEQ. */
1901
1902	static inline gimple_seq
1903	call_update_watermark (tree wmptr, cgraph_node *node, profile_count count,
1904	gimple_seq seq = NULL)
1905	{
1906	tree uwm = get_update ();
1907	gcall *update = gimple_build_call (uwm, 1, wmptr);
1908	if (node)
1909	gimple_set_location (g: update, DECL_SOURCE_LOCATION (node->decl));
1910	gimple_seq_add_stmt (&seq, update);
1911	if (node)
1912	node->create_edge (callee: cgraph_node::get_create (uwm), call_stmt: update, count, cloning_p: false);
1913	return seq;
1914	}
1915
1916	};
1917
1918	} // anon namespace
1919
1920	/* Gather with this type a collection of parameters that we're turning into
1921	explicit references. */
1922
1923	typedef hash_set<tree> indirect_parms_t;
1924
1925	/* Dereference OP's incoming turned-into-reference parm if it's an
1926	INDIRECT_PARMS or an ADDR_EXPR thereof. Set *REC and return according to
1927	gimple-walking expectations. */
1928
1929	static tree
1930	maybe_make_indirect (indirect_parms_t &indirect_parms, tree op, int *rec)
1931	{
1932	if (DECL_P (op))
1933	{
1934	*rec = 0;
1935	if (indirect_parms.contains (k: op))
1936	{
1937	tree ret = gimple_fold_indirect_ref (op);
1938	if (!ret)
1939	ret = build2 (MEM_REF,
1940	TREE_TYPE (TREE_TYPE (op)),
1941	op,
1942	build_int_cst (TREE_TYPE (op), 0));
1943	if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (op)))
1944	&& !TREE_THIS_VOLATILE (ret))
1945	TREE_SIDE_EFFECTS (ret) = TREE_THIS_VOLATILE (ret) = 1;
1946	return ret;
1947	}
1948	}
1949	else if (TREE_CODE (op) == ADDR_EXPR
1950	&& DECL_P (TREE_OPERAND (op, 0)))
1951	{
1952	*rec = 0;
1953	if (indirect_parms.contains (TREE_OPERAND (op, 0)))
1954	{
1955	op = TREE_OPERAND (op, 0);
1956	return op;
1957	}
1958	}
1959
1960	return NULL_TREE;
1961	}
1962
1963	/* A gimple-walking function that adds dereferencing to indirect parms. */
1964
1965	static tree
1966	walk_make_indirect (tree *op, int *rec, void *arg)
1967	{
1968	walk_stmt_info *wi = (walk_stmt_info *)arg;
1969	indirect_parms_t &indirect_parms = *(indirect_parms_t *)wi->info;
1970
1971	if (!*op || TYPE_P (*op))
1972	{
1973	*rec = 0;
1974	return NULL_TREE;
1975	}
1976
1977	if (tree repl = maybe_make_indirect (indirect_parms, op: *op, rec))
1978	{
1979	*op = repl;
1980	wi->changed = true;
1981	}
1982
1983	return NULL_TREE;
1984	}
1985
1986	/* A gimple-walking function that turns any non-gimple-val ADDR_EXPRs into a
1987	separate SSA. Though addresses of e.g. parameters, and of members thereof,
1988	are gimple vals, turning parameters into references, with an extra layer of
1989	indirection and thus explicit dereferencing, need to be regimplified. */
1990
1991	static tree
1992	walk_regimplify_addr_expr (tree *op, int *rec, void *arg)
1993	{
1994	walk_stmt_info *wi = (walk_stmt_info *)arg;
1995	gimple_stmt_iterator &gsi = *(gimple_stmt_iterator *)wi->info;
1996
1997	*rec = 0;
1998
1999	if (!*op || TREE_CODE (*op) != ADDR_EXPR)
2000	return NULL_TREE;
2001
2002	if (!is_gimple_val (*op))
2003	{
2004	tree ret = force_gimple_operand_gsi (&gsi, *op, true,
2005	NULL_TREE, true, GSI_SAME_STMT);
2006	gcc_assert (ret != *op);
2007	*op = ret;
2008	wi->changed = true;
2009	}
2010
2011	return NULL_TREE;
2012	}
2013
2014	/* Turn STMT's PHI arg defs into separate SSA defs if they've become
2015	non-gimple_val. Return TRUE if any edge insertions need to be committed. */
2016
2017	static bool
2018	walk_regimplify_phi (gphi *stmt)
2019	{
2020	bool needs_commit = false;
2021
2022	for (unsigned i = 0, n = gimple_phi_num_args (gs: stmt); i < n; i++)
2023	{
2024	tree op = gimple_phi_arg_def (gs: stmt, index: i);
2025	if ((TREE_CODE (op) == ADDR_EXPR
2026	&& !is_gimple_val (op))
2027	/* ??? A PARM_DECL that was addressable in the original function and
2028	had its address in PHI nodes, but that became a reference in the
2029	wrapped clone would NOT be updated by update_ssa in PHI nodes.
2030	Alas, if we were to create a default def for it now, update_ssa
2031	would complain that the symbol that needed rewriting already has
2032	SSA names associated with it. OTOH, leaving the PARM_DECL alone,
2033	it eventually causes errors because it remains unchanged in PHI
2034	nodes, but it gets rewritten as expected if it appears in other
2035	stmts. So we cheat a little here, and force the PARM_DECL out of
2036	the PHI node and into an assignment. It's a little expensive,
2037	because we insert it at the edge, which introduces a basic block
2038	that's entirely unnecessary, but it works, and the block will be
2039	removed as the default def gets propagated back into the PHI node,
2040	so the final optimized code looks just as expected. */
2041	|| (TREE_CODE (op) == PARM_DECL
2042	&& !TREE_ADDRESSABLE (op)))
2043	{
2044	tree temp = make_ssa_name (TREE_TYPE (op), stmt);
2045	if (TREE_CODE (op) == PARM_DECL)
2046	SET_SSA_NAME_VAR_OR_IDENTIFIER (temp, DECL_NAME (op));
2047	SET_PHI_ARG_DEF (stmt, i, temp);
2048
2049	gimple *assign = gimple_build_assign (temp, op);
2050	if (gimple_phi_arg_has_location (phi: stmt, i))
2051	gimple_set_location (g: assign, location: gimple_phi_arg_location (phi: stmt, i));
2052	gsi_insert_on_edge (gimple_phi_arg_edge (phi: stmt, i), assign);
2053	needs_commit = true;
2054	}
2055	}
2056
2057	return needs_commit;
2058	}
2059
2060	/* Create a reference type to use for PARM when turning it into a
2061	reference. */
2062
2063	static tree
2064	build_ref_type_for (tree parm)
2065	{
2066	gcc_checking_assert (TREE_CODE (parm) == PARM_DECL);
2067
2068	tree ref_type = build_reference_type (TREE_TYPE (parm));
2069
2070	return ref_type;
2071	}
2072
2073	/* Add cgraph edges from current_function_decl to callees in SEQ with frequency
2074	COUNT, assuming all calls in SEQ are direct. */
2075
2076	static void
2077	add_call_edges_for_seq (gimple_seq seq, profile_count count)
2078	{
2079	cgraph_node *node = cgraph_node::get_create (current_function_decl);
2080
2081	for (gimple_stmt_iterator gsi = gsi_start (seq);
2082	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
2083	{
2084	gimple *stmt = gsi_stmt (i: gsi);
2085
2086	gcall *call = dyn_cast <gcall *> (p: stmt);
2087	if (!call)
2088	continue;
2089
2090	tree callee = gimple_call_fndecl (gs: call);
2091	gcc_checking_assert (callee);
2092	node->create_edge (callee: cgraph_node::get_create (callee), call_stmt: call, count, cloning_p: false);
2093	}
2094	}
2095
2096	/* Insert SEQ after the call at GSI, as if the call was in a try block with SEQ
2097	as finally, i.e., SEQ will run after the call whether it returns or
2098	propagates an exception. This handles block splitting, EH edge and block
2099	creation, noreturn and nothrow optimizations, and even throwing calls without
2100	preexisting local handlers. */
2101
2102	static void
2103	gsi_insert_finally_seq_after_call (gimple_stmt_iterator gsi, gimple_seq seq)
2104	{
2105	if (!seq)
2106	return;
2107
2108	gimple *stmt = gsi_stmt (i: gsi);
2109
2110	if (gimple_has_location (g: stmt))
2111	annotate_all_with_location (seq, gimple_location (g: stmt));
2112
2113	gcall *call = dyn_cast <gcall *> (p: stmt);
2114	bool noreturn_p = call && gimple_call_noreturn_p (s: call);
2115	int eh_lp = lookup_stmt_eh_lp (stmt);
2116	bool must_not_throw_p = eh_lp < 0;
2117	bool nothrow_p = (must_not_throw_p
2118	|| (call && gimple_call_nothrow_p (s: call))
2119	|| (eh_lp <= 0
2120	&& (TREE_NOTHROW (cfun->decl)
2121	|| !opt_for_fn (cfun->decl, flag_exceptions))));
2122
2123	if (noreturn_p && nothrow_p)
2124	return;
2125
2126	/* Don't expect an EH edge if we're not to throw, or if we're not in an EH
2127	region yet. */
2128	bool no_eh_edge_p = (nothrow_p || !eh_lp);
2129	bool must_end_bb = stmt_ends_bb_p (stmt);
2130
2131	edge eft = NULL, eeh = NULL;
2132	if (must_end_bb && !(noreturn_p && no_eh_edge_p))
2133	{
2134	gcc_checking_assert (gsi_one_before_end_p (gsi));
2135
2136	edge e;
2137	edge_iterator ei;
2138	FOR_EACH_EDGE (e, ei, gsi_bb (gsi)->succs)
2139	{
2140	if ((e->flags & EDGE_EH))
2141	{
2142	gcc_checking_assert (!eeh);
2143	eeh = e;
2144	#if !CHECKING_P
2145	if (eft || noreturn_p)
2146	break;
2147	#endif
2148	}
2149	if ((e->flags & EDGE_FALLTHRU))
2150	{
2151	gcc_checking_assert (!eft);
2152	eft = e;
2153	#if !CHECKING_P
2154	if (eeh || no_eh_edge_p)
2155	break;
2156	#endif
2157	}
2158	}
2159
2160	gcc_checking_assert (!(eft && (eft->flags & EDGE_FALLTHRU))
2161	== noreturn_p);
2162	gcc_checking_assert (!(eeh && (eeh->flags & EDGE_EH))
2163	== no_eh_edge_p);
2164	gcc_checking_assert (eft != eeh);
2165	}
2166
2167	if (!noreturn_p)
2168	{
2169	gimple_seq nseq = nothrow_p ? seq : gimple_seq_copy (seq);
2170
2171	if (must_end_bb)
2172	{
2173	gcc_checking_assert (gsi_one_before_end_p (gsi));
2174	add_call_edges_for_seq (seq: nseq, count: eft->count ());
2175	gsi_insert_seq_on_edge_immediate (eft, nseq);
2176	}
2177	else
2178	{
2179	add_call_edges_for_seq (seq: nseq, count: gsi_bb (i: gsi)->count);
2180	gsi_insert_seq_after (&gsi, nseq, GSI_SAME_STMT);
2181	}
2182	}
2183
2184	if (nothrow_p)
2185	return;
2186
2187	if (eh_lp)
2188	{
2189	add_call_edges_for_seq (seq, count: eeh->count ());
2190	gsi_insert_seq_on_edge_immediate (eeh, seq);
2191	return;
2192	}
2193
2194	/* A throwing call may appear within a basic block in a function that doesn't
2195	have any EH regions. We're going to add a cleanup if so, therefore the
2196	block will have to be split. */
2197	basic_block bb = gsi_bb (i: gsi);
2198	if (!gsi_one_before_end_p (i: gsi))
2199	split_block (bb, stmt);
2200
2201	/* Create a new block for the EH cleanup. */
2202	basic_block bb_eh_cleanup = create_empty_bb (bb);
2203	if (dom_info_available_p (CDI_DOMINATORS))
2204	set_immediate_dominator (CDI_DOMINATORS, bb_eh_cleanup, bb);
2205	if (current_loops)
2206	add_bb_to_loop (bb_eh_cleanup, current_loops->tree_root);
2207
2208	/* Make the new block an EH cleanup for the call. */
2209	eh_region new_r = gen_eh_region_cleanup (NULL);
2210	eh_landing_pad lp = gen_eh_landing_pad (new_r);
2211	tree label = gimple_block_label (bb_eh_cleanup);
2212	lp->post_landing_pad = label;
2213	EH_LANDING_PAD_NR (label) = lp->index;
2214	add_stmt_to_eh_lp (stmt, lp->index);
2215
2216	/* Add the cleanup code to the EH cleanup block. */
2217	gsi = gsi_after_labels (bb: bb_eh_cleanup);
2218	gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2219
2220	/* And then propagate the exception further. */
2221	gresx *resx = gimple_build_resx (new_r->index);
2222	if (gimple_has_location (g: stmt))
2223	gimple_set_location (g: resx, location: gimple_location (g: stmt));
2224	gsi_insert_before (&gsi, resx, GSI_SAME_STMT);
2225
2226	/* Finally, wire the EH cleanup block into the CFG. */
2227	edge neeh = make_eh_edge (stmt);
2228	neeh->probability = profile_probability::never ();
2229	gcc_checking_assert (neeh->dest == bb_eh_cleanup);
2230	gcc_checking_assert (!neeh->dest->count.initialized_p ());
2231	neeh->dest->count = neeh->count ();
2232	add_call_edges_for_seq (seq, count: neeh->dest->count);
2233	}
2234
2235	/* Copy the attribute list at *ATTRS, minus any NAME attributes, leaving
2236	shareable trailing nodes alone. */
2237
2238	static inline void
2239	remove_named_attribute_unsharing (const char *name, tree *attrs)
2240	{
2241	while (tree found = lookup_attribute (attr_name: name, list: *attrs))
2242	{
2243	/* Copy nodes up to the next NAME attribute. */
2244	while (*attrs != found)
2245	{
2246	*attrs = tree_cons (TREE_PURPOSE (*attrs),
2247	TREE_VALUE (*attrs),
2248	TREE_CHAIN (*attrs));
2249	attrs = &TREE_CHAIN (*attrs);
2250	}
2251	/* Then drop it. */
2252	gcc_checking_assert (*attrs == found);
2253	*attrs = TREE_CHAIN (*attrs);
2254	}
2255	}
2256
2257	/* Record the order of the last cgraph entry whose mode we've already set, so
2258	that we can perform mode setting incrementally without duplication. */
2259	static int last_cgraph_order;
2260
2261	/* Set strub modes for functions introduced since the last call. */
2262
2263	static void
2264	ipa_strub_set_mode_for_new_functions ()
2265	{
2266	if (symtab->order == last_cgraph_order)
2267	return;
2268
2269	cgraph_node *node;
2270
2271	/* Go through the functions twice, once over non-aliases, and then over
2272	aliases, so that aliases can reuse the mode computation of their ultimate
2273	targets. */
2274	for (int aliases = 0; aliases <= 1; aliases++)
2275	FOR_EACH_FUNCTION (node)
2276	{
2277	if (!node->alias != !aliases)
2278	continue;
2279
2280	/* Already done. */
2281	if (node->order < last_cgraph_order)
2282	continue;
2283
2284	set_strub_mode (node);
2285	}
2286
2287	last_cgraph_order = symtab->order;
2288	}
2289
2290	/* Return FALSE if NODE is a strub context, and TRUE otherwise. */
2291
2292	bool
2293	strub_splittable_p (cgraph_node *node)
2294	{
2295	switch (get_strub_mode (node))
2296	{
2297	case STRUB_WRAPPED:
2298	case STRUB_AT_CALLS:
2299	case STRUB_AT_CALLS_OPT:
2300	case STRUB_INLINABLE:
2301	case STRUB_INTERNAL:
2302	case STRUB_WRAPPER:
2303	return false;
2304
2305	case STRUB_CALLABLE:
2306	case STRUB_DISABLED:
2307	break;
2308
2309	default:
2310	gcc_unreachable ();
2311	}
2312
2313	return true;
2314	}
2315
2316	/* Return the PARM_DECL of the incoming watermark pointer, if there is one. */
2317
2318	tree
2319	strub_watermark_parm (tree fndecl)
2320	{
2321	switch (get_strub_mode_from_fndecl (fndecl))
2322	{
2323	case STRUB_WRAPPED:
2324	case STRUB_AT_CALLS:
2325	case STRUB_AT_CALLS_OPT:
2326	break;
2327
2328	case STRUB_INTERNAL:
2329	case STRUB_WRAPPER:
2330	case STRUB_CALLABLE:
2331	case STRUB_DISABLED:
2332	case STRUB_INLINABLE:
2333	return NULL_TREE;
2334
2335	default:
2336	gcc_unreachable ();
2337	}
2338
2339	for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
2340	/* The type (variant) compare finds the parameter even in a just-created
2341	clone, before we set its name, but the type-based compare doesn't work
2342	during builtin expansion within the lto compiler, because we'll have
2343	created a separate variant in that run. */
2344	if (TREE_TYPE (parm) == pass_ipa_strub::get_qpwmt ()
2345	|| DECL_NAME (parm) == pass_ipa_strub::get_watermark_ptr ())
2346	return parm;
2347
2348	gcc_unreachable ();
2349	}
2350
2351	/* Adjust a STRUB_AT_CALLS function TYPE, adding a watermark pointer if it
2352	hasn't been added yet. Return the named argument count. */
2353
2354	int
2355	pass_ipa_strub::adjust_at_calls_type (tree type)
2356	{
2357	int named_args = 0;
2358
2359	gcc_checking_assert (same_strub_mode_in_variants_p (type));
2360
2361	if (!TYPE_ARG_TYPES (type))
2362	return named_args;
2363
2364	tree *tlist = &TYPE_ARG_TYPES (type);
2365	tree qpwmptrt = get_qpwmt ();
2366	while (*tlist && TREE_VALUE (*tlist) != void_type_node)
2367	{
2368	/* The type has already been adjusted. */
2369	if (TREE_VALUE (*tlist) == qpwmptrt)
2370	return named_args;
2371	named_args++;
2372	*tlist = tree_cons (TREE_PURPOSE (*tlist),
2373	TREE_VALUE (*tlist),
2374	TREE_CHAIN (*tlist));
2375	tlist = &TREE_CHAIN (*tlist);
2376	}
2377
2378	/* Add the new argument after all named arguments, so as to not mess with
2379	attributes that reference parameters. */
2380	*tlist = tree_cons (NULL_TREE, get_qpwmt (), *tlist);
2381
2382	#if ATTR_FNSPEC_DECONST_WATERMARK
2383	if (!type_already_adjusted)
2384	{
2385	int flags = flags_from_decl_or_type (type);
2386	tree fnspec = lookup_attribute ("fn spec", type);
2387
2388	if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
2389	{
2390	size_t xargs = 1;
2391	size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
2392	auto_vec<char> nspecv (tgtlen);
2393	char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
2394	if (fnspec)
2395	{
2396	tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
2397	curlen = TREE_STRING_LENGTH (fnspecstr);
2398	memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
2399	}
2400	if (!curlen)
2401	{
2402	nspec[curlen++] = '.';
2403	nspec[curlen++] = ((flags & ECF_CONST)
2404	? 'c'
2405	: (flags & ECF_PURE)
2406	? 'p'
2407	: ' ');
2408	}
2409	while (curlen < tgtlen - 2 * xargs)
2410	{
2411	nspec[curlen++] = '.';
2412	nspec[curlen++] = ' ';
2413	}
2414	nspec[curlen++] = 'W';
2415	nspec[curlen++] = 't';
2416
2417	/* The type has already been copied, if needed, before adding
2418	parameters. */
2419	TYPE_ATTRIBUTES (type)
2420	= tree_cons (get_identifier ("fn spec"),
2421	build_tree_list (NULL_TREE,
2422	build_string (tgtlen, nspec)),
2423	TYPE_ATTRIBUTES (type));
2424	}
2425	}
2426	#endif
2427
2428	return named_args;
2429	}
2430
2431	/* Adjust a call to an at-calls call target. Create a watermark local variable
2432	if needed, initialize it before, pass it to the callee according to the
2433	modified at-calls interface, and release the callee's stack space after the
2434	call, if not deferred. If the call is const or pure, arrange for the
2435	watermark to not be assumed unused or unchanged. */
2436
2437	void
2438	pass_ipa_strub::adjust_at_calls_call (cgraph_edge *e, int named_args,
2439	tree callee_fntype)
2440	{
2441	gcc_checking_assert (e->call_stmt);
2442	gcall *ocall = e->call_stmt;
2443	gimple_stmt_iterator gsi = gsi_for_stmt (ocall);
2444
2445	/* Make sure we haven't modified this call yet. */
2446	gcc_checking_assert (!(int (gimple_call_num_args (ocall)) > named_args
2447	&& (TREE_TYPE (gimple_call_arg (ocall, named_args))
2448	== get_pwmt ())));
2449
2450	tree tsup;
2451	if (!(tsup = gimple_call_fndecl (gs: ocall)))
2452	tsup = TREE_TYPE (TREE_TYPE (gimple_call_fn (ocall)));
2453	if (!strub_target_support_p (t: tsup, report: true, loc: gimple_location (g: ocall)))
2454	return;
2455
2456	/* If we're already within a strub context, pass on the incoming watermark
2457	pointer, and omit the enter and leave calls around the modified call, as an
2458	optimization, or as a means to satisfy a tail-call requirement. */
2459	tree swmp = ((opt_for_fn (e->caller->decl, optimize_size)
2460	|| opt_for_fn (e->caller->decl, optimize) > 2
2461	|| gimple_call_must_tail_p (s: ocall)
2462	|| (opt_for_fn (e->caller->decl, optimize) == 2
2463	&& gimple_call_tail_p (s: ocall)))
2464	? strub_watermark_parm (fndecl: e->caller->decl)
2465	: NULL_TREE);
2466	bool omit_own_watermark = swmp;
2467	tree swm = NULL_TREE;
2468	if (!omit_own_watermark)
2469	{
2470	swm = create_tmp_var (get_wmt (), ".strub.watermark");
2471	TREE_ADDRESSABLE (swm) = true;
2472	swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
2473
2474	/* Initialize the watermark before the call. */
2475	tree enter = get_enter ();
2476	gcall *stptr = gimple_build_call (enter, 1,
2477	unshare_expr (swmp));
2478	if (gimple_has_location (g: ocall))
2479	gimple_set_location (g: stptr, location: gimple_location (g: ocall));
2480	gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
2481	e->caller->create_edge (callee: cgraph_node::get_create (enter),
2482	call_stmt: stptr, count: gsi_bb (i: gsi)->count, cloning_p: false);
2483	}
2484
2485
2486	/* Replace the call with one that passes the swmp argument first. */
2487	gcall *wrcall;
2488	{ gcall *stmt = ocall;
2489	// Mostly copied from gimple_call_copy_skip_args.
2490	int i = 0;
2491	int nargs = gimple_call_num_args (gs: stmt);
2492	auto_vec<tree> vargs (MAX (nargs, named_args) + 1);
2493	gcall *new_stmt;
2494
2495	/* pr71109.c calls a prototypeless function, then defines it with
2496	additional arguments. It's ill-formed, but after it's inlined,
2497	it somehow works out. */
2498	for (; i < named_args && i < nargs; i++)
2499	vargs.quick_push (obj: gimple_call_arg (gs: stmt, index: i));
2500	for (; i < named_args; i++)
2501	vargs.quick_push (null_pointer_node);
2502
2503	vargs.quick_push (obj: unshare_expr (swmp));
2504
2505	for (; i < nargs; i++)
2506	vargs.quick_push (obj: gimple_call_arg (gs: stmt, index: i));
2507
2508	if (gimple_call_internal_p (gs: stmt))
2509	gcc_unreachable ();
2510	else
2511	new_stmt = gimple_build_call_vec (gimple_call_fn (gs: stmt), vargs);
2512	gimple_call_set_fntype (call_stmt: new_stmt, fntype: callee_fntype);
2513
2514	if (gimple_call_lhs (gs: stmt))
2515	gimple_call_set_lhs (gs: new_stmt, lhs: gimple_call_lhs (gs: stmt));
2516
2517	gimple_move_vops (new_stmt, stmt);
2518
2519	if (gimple_has_location (g: stmt))
2520	gimple_set_location (g: new_stmt, location: gimple_location (g: stmt));
2521	gimple_call_copy_flags (dest_call: new_stmt, orig_call: stmt);
2522	gimple_call_set_chain (call_stmt: new_stmt, chain: gimple_call_chain (gs: stmt));
2523
2524	gimple_set_modified (s: new_stmt, modifiedp: true);
2525
2526	wrcall = new_stmt;
2527	}
2528
2529	update_stmt (s: wrcall);
2530	gsi_replace (&gsi, wrcall, true);
2531	cgraph_edge::set_call_stmt (e, new_stmt: wrcall, update_speculative: false);
2532
2533	/* Insert the strub code after the call. */
2534	gimple_seq seq = NULL;
2535
2536	#if !ATTR_FNSPEC_DECONST_WATERMARK
2537	/* If the call will be assumed to not modify or even read the
2538	watermark, make it read and modified ourselves. */
2539	if ((gimple_call_flags (wrcall)
2540	& (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
2541	{
2542	if (!swm)
2543	swm = build2 (MEM_REF,
2544	TREE_TYPE (TREE_TYPE (swmp)),
2545	swmp,
2546	build_int_cst (TREE_TYPE (swmp), 0));
2547
2548	vec<tree, va_gc> *inputs = NULL;
2549	vec<tree, va_gc> *outputs = NULL;
2550	vec_safe_push (v&: outputs,
2551	obj: build_tree_list
2552	(build_tree_list
2553	(NULL_TREE, build_string (2, "=m")),
2554	unshare_expr (swm)));
2555	vec_safe_push (v&: inputs,
2556	obj: build_tree_list
2557	(build_tree_list
2558	(NULL_TREE, build_string (1, "m")),
2559	unshare_expr (swm)));
2560	gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
2561	NULL, NULL);
2562	gimple_seq_add_stmt (&seq, forcemod);
2563
2564	/* If the call will be assumed to not even read the watermark,
2565	make sure it is already in memory before the call. */
2566	if ((gimple_call_flags (wrcall) & ECF_CONST))
2567	{
2568	vec<tree, va_gc> *inputs = NULL;
2569	vec_safe_push (v&: inputs,
2570	obj: build_tree_list
2571	(build_tree_list
2572	(NULL_TREE, build_string (1, "m")),
2573	unshare_expr (swm)));
2574	gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
2575	NULL, NULL);
2576	if (gimple_has_location (g: wrcall))
2577	gimple_set_location (g: force_store, location: gimple_location (g: wrcall));
2578	gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
2579	}
2580	}
2581	#endif
2582
2583	if (!omit_own_watermark)
2584	{
2585	gcall *sleave = gimple_build_call (get_leave (), 1,
2586	unshare_expr (swmp));
2587	gimple_seq_add_stmt (&seq, sleave);
2588
2589	gassign *clobber = gimple_build_assign (swm,
2590	build_clobber
2591	(TREE_TYPE (swm)));
2592	gimple_seq_add_stmt (&seq, clobber);
2593	}
2594
2595	gsi_insert_finally_seq_after_call (gsi, seq);
2596	}
2597
2598	/* Adjust all at-calls calls in NODE. */
2599
2600	void
2601	pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node)
2602	{
2603	/* Adjust unknown-callee indirect calls with STRUB_AT_CALLS types within
2604	onode. */
2605	if (node->indirect_calls)
2606	{
2607	push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2608	for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2609	{
2610	gcc_checking_assert (e->indirect_unknown_callee);
2611
2612	if (!e->call_stmt)
2613	continue;
2614
2615	tree callee_fntype;
2616	enum strub_mode callee_mode
2617	= effective_strub_mode_for_call (call: e->call_stmt, typep: &callee_fntype);
2618
2619	if (callee_mode != STRUB_AT_CALLS
2620	&& callee_mode != STRUB_AT_CALLS_OPT)
2621	continue;
2622
2623	int named_args = adjust_at_calls_type (type: callee_fntype);
2624
2625	adjust_at_calls_call (e, named_args, callee_fntype);
2626	}
2627	pop_cfun ();
2628	}
2629
2630	if (node->callees)
2631	{
2632	push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2633	for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2634	{
2635	gcc_checking_assert (!e->indirect_unknown_callee);
2636
2637	if (!e->call_stmt)
2638	continue;
2639
2640	tree callee_fntype;
2641	enum strub_mode callee_mode
2642	= effective_strub_mode_for_call (call: e->call_stmt, typep: &callee_fntype);
2643
2644	if (callee_mode != STRUB_AT_CALLS
2645	&& callee_mode != STRUB_AT_CALLS_OPT)
2646	continue;
2647
2648	int named_args = adjust_at_calls_type (type: callee_fntype);
2649
2650	adjust_at_calls_call (e, named_args, callee_fntype);
2651	}
2652	pop_cfun ();
2653	}
2654	}
2655
2656	/* The strubm (strub mode) pass computes a strub mode for each function in the
2657	call graph, and checks, before any inlining, that strub callability
2658	requirements in effect are satisfied. */
2659
2660	unsigned int
2661	pass_ipa_strub_mode::execute (function *)
2662	{
2663	last_cgraph_order = 0;
2664	ipa_strub_set_mode_for_new_functions ();
2665
2666	/* Verify before any inlining or other transformations. */
2667	verify_strub ();
2668
2669	return 0;
2670	}
2671
2672	/* Create a strub mode pass. */
2673
2674	simple_ipa_opt_pass *
2675	make_pass_ipa_strub_mode (gcc::context *ctxt)
2676	{
2677	return new pass_ipa_strub_mode (ctxt);
2678	}
2679
2680	/* The strub pass proper adjusts types, signatures, and at-calls calls, and
2681	splits internal-strub functions. */
2682
2683	unsigned int
2684	pass_ipa_strub::execute (function *)
2685	{
2686	cgraph_node *onode;
2687
2688	ipa_strub_set_mode_for_new_functions ();
2689
2690	/* First, adjust the signature of at-calls functions. We adjust types of
2691	at-calls functions first, so that we don't modify types in place unless
2692	strub is explicitly requested. */
2693	FOR_EACH_FUNCTION (onode)
2694	{
2695	enum strub_mode mode = get_strub_mode (node: onode);
2696
2697	if (mode == STRUB_AT_CALLS
2698	|| mode == STRUB_AT_CALLS_OPT)
2699	{
2700	/* Create a type variant if strubbing was not explicitly requested in
2701	the function type. */
2702	if (get_strub_mode_from_type (TREE_TYPE (onode->decl)) != mode)
2703	distinctify_node_type (node: onode);
2704
2705	int named_args = adjust_at_calls_type (TREE_TYPE (onode->decl));
2706
2707	/* An external function explicitly declared with strub won't have a
2708	body. Even with implicit at-calls strub, a function may have had its
2709	body removed after we selected the mode, and then we have nothing
2710	further to do. */
2711	if (!onode->has_gimple_body_p ())
2712	continue;
2713
2714	tree *pargs = &DECL_ARGUMENTS (onode->decl);
2715
2716	/* A noninterposable_alias reuses the same parm decl chain, don't add
2717	the parm twice. */
2718	bool aliased_parms = (onode->alias && *pargs
2719	&& DECL_CONTEXT (*pargs) != onode->decl);
2720
2721	if (aliased_parms)
2722	continue;
2723
2724	for (int i = 0; i < named_args; i++)
2725	pargs = &DECL_CHAIN (*pargs);
2726
2727	tree wmptr = build_decl (DECL_SOURCE_LOCATION (onode->decl),
2728	PARM_DECL,
2729	get_watermark_ptr (),
2730	get_qpwmt ());
2731	DECL_ARTIFICIAL (wmptr) = 1;
2732	DECL_ARG_TYPE (wmptr) = get_qpwmt ();
2733	DECL_CONTEXT (wmptr) = onode->decl;
2734	TREE_USED (wmptr) = 1;
2735	DECL_CHAIN (wmptr) = *pargs;
2736	*pargs = wmptr;
2737
2738	if (onode->alias)
2739	continue;
2740
2741	cgraph_node *nnode = onode;
2742	push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
2743
2744	{
2745	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2746	gimple_seq seq = call_update_watermark (wmptr, node: nnode, count: e->src->count);
2747	gsi_insert_seq_on_edge_immediate (e, seq);
2748	}
2749
2750	if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca)
2751	{
2752	basic_block bb;
2753	FOR_EACH_BB_FN (bb, cfun)
2754	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2755	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
2756	{
2757	gimple *stmt = gsi_stmt (i: gsi);
2758
2759	gcall *call = dyn_cast <gcall *> (p: stmt);
2760
2761	if (!call)
2762	continue;
2763
2764	if (gimple_alloca_call_p (call))
2765	{
2766	/* Capture stack growth. */
2767	gimple_seq seq = call_update_watermark (wmptr, NULL,
2768	count: gsi_bb (i: gsi)
2769	->count);
2770	gsi_insert_finally_seq_after_call (gsi, seq);
2771	}
2772	}
2773	}
2774
2775	pop_cfun ();
2776	}
2777	}
2778
2779	FOR_EACH_FUNCTION (onode)
2780	{
2781	if (!onode->has_gimple_body_p ())
2782	continue;
2783
2784	enum strub_mode mode = get_strub_mode (node: onode);
2785
2786	if (mode != STRUB_INTERNAL)
2787	{
2788	adjust_at_calls_calls (node: onode);
2789	continue;
2790	}
2791
2792	bool is_stdarg = calls_builtin_va_start_p (node: onode);;
2793	bool apply_args = calls_builtin_apply_args_p (node: onode);
2794
2795	vec<ipa_adjusted_param, va_gc> *nparms = NULL;
2796	unsigned j = 0;
2797	{
2798	// The following loop copied from ipa-split.c:split_function.
2799	for (tree parm = DECL_ARGUMENTS (onode->decl);
2800	parm; parm = DECL_CHAIN (parm), j++)
2801	{
2802	ipa_adjusted_param adj = {};
2803	adj.op = IPA_PARAM_OP_COPY;
2804	adj.base_index = j;
2805	adj.prev_clone_index = j;
2806	vec_safe_push (v&: nparms, obj: adj);
2807	}
2808
2809	if (apply_args)
2810	{
2811	ipa_adjusted_param aaadj = {};
2812	aaadj.op = IPA_PARAM_OP_NEW;
2813	aaadj.type = get_qptr ();
2814	vec_safe_push (v&: nparms, obj: aaadj);
2815	}
2816
2817	if (is_stdarg)
2818	{
2819	ipa_adjusted_param vladj = {};
2820	vladj.op = IPA_PARAM_OP_NEW;
2821	vladj.type = get_qpvalst ();
2822	vec_safe_push (v&: nparms, obj: vladj);
2823	}
2824
2825	ipa_adjusted_param wmadj = {};
2826	wmadj.op = IPA_PARAM_OP_NEW;
2827	wmadj.type = get_qpwmt ();
2828	vec_safe_push (v&: nparms, obj: wmadj);
2829	}
2830	ipa_param_adjustments adj (nparms, -1, false);
2831
2832	cgraph_node *nnode = onode->create_version_clone_with_body
2833	(redirect_callers: auto_vec<cgraph_edge *> (0),
2834	NULL, param_adjustments: &adj, NULL, NULL, clone_name: "strub", NULL);
2835
2836	if (!nnode)
2837	{
2838	error_at (DECL_SOURCE_LOCATION (onode->decl),
2839	"failed to split %qD for %<strub%>",
2840	onode->decl);
2841	continue;
2842	}
2843
2844	onode->split_part = true;
2845	if (onode->calls_comdat_local)
2846	nnode->add_to_same_comdat_group (old_node: onode);
2847
2848	set_strub_mode_to (node: onode, mode: STRUB_WRAPPER);
2849	set_strub_mode_to (node: nnode, mode: STRUB_WRAPPED);
2850
2851	adjust_at_calls_calls (node: nnode);
2852
2853	/* Decide which of the wrapped function's parms we want to turn into
2854	references to the argument passed to the wrapper. In general, we want to
2855	copy small arguments, and avoid copying large ones. Variable-sized array
2856	lengths given by other arguments, as in 20020210-1.c, would lead to
2857	problems if passed by value, after resetting the original function and
2858	dropping the length computation; passing them by reference works.
2859	DECL_BY_REFERENCE is *not* a substitute for this: it involves copying
2860	anyway, but performed at the caller. */
2861	indirect_parms_t indirect_nparms (3, false);
2862	unsigned adjust_ftype = 0;
2863	unsigned named_args = 0;
2864	for (tree parm = DECL_ARGUMENTS (onode->decl),
2865	nparm = DECL_ARGUMENTS (nnode->decl),
2866	nparmt = TYPE_ARG_TYPES (TREE_TYPE (nnode->decl));
2867	parm;
2868	named_args++,
2869	parm = DECL_CHAIN (parm),
2870	nparm = DECL_CHAIN (nparm),
2871	nparmt = nparmt ? TREE_CHAIN (nparmt) : NULL_TREE)
2872	if (TREE_THIS_VOLATILE (parm)
2873	|| !(0 /* DECL_BY_REFERENCE (narg) */
2874	|| is_gimple_reg_type (TREE_TYPE (nparm))
2875	|| VECTOR_TYPE_P (TREE_TYPE (nparm))
2876	|| TREE_CODE (TREE_TYPE (nparm)) == COMPLEX_TYPE
2877	|| (tree_fits_uhwi_p (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2878	&& (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2879	<= 4 * UNITS_PER_WORD))))
2880	{
2881	/* No point in indirecting pointer types. Presumably they
2882	won't ever pass the size-based test above, but check the
2883	assumption here, because getting this wrong would mess
2884	with attribute access and possibly others. We deal with
2885	fn spec below. */
2886	gcc_checking_assert (!POINTER_TYPE_P (TREE_TYPE (nparm)));
2887
2888	indirect_nparms.add (k: nparm);
2889
2890	/* ??? Is there any case in which it is not safe to suggest the parms
2891	turned indirect don't alias anything else? They are distinct,
2892	unaliased memory in the wrapper, and the wrapped can't possibly
2893	take pointers into them because none of the pointers passed to the
2894	wrapper can alias other incoming parameters passed by value, even
2895	if with transparent reference, and the wrapper doesn't take any
2896	extra parms that could point into wrapper's parms. So we can
2897	probably drop the TREE_ADDRESSABLE and keep the TRUE. */
2898	tree ref_type = build_ref_type_for (parm: nparm);
2899
2900	if (TREE_THIS_VOLATILE (nparm)
2901	&& TYPE_VOLATILE (TREE_TYPE (nparm))
2902	&& !TYPE_VOLATILE (ref_type))
2903	TREE_SIDE_EFFECTS (nparm) = TREE_THIS_VOLATILE (nparm) = 0;
2904	DECL_ARG_TYPE (nparm) = TREE_TYPE (nparm) = ref_type;
2905	relayout_decl (nparm);
2906	TREE_ADDRESSABLE (nparm) = 0;
2907	DECL_BY_REFERENCE (nparm) = 0;
2908	DECL_NOT_GIMPLE_REG_P (nparm) = 0;
2909	/* ??? This avoids mismatches in debug info bind stmts in
2910	e.g. a-chahan . */
2911	DECL_ABSTRACT_ORIGIN (nparm) = NULL;
2912
2913	if (nparmt)
2914	adjust_ftype++;
2915	}
2916
2917	/* Also adjust the wrapped function type, if needed. */
2918	if (adjust_ftype)
2919	{
2920	tree nftype = TREE_TYPE (nnode->decl);
2921
2922	/* We always add at least one argument at the end of the signature, when
2923	cloning the function, so we don't expect to need to duplicate the
2924	type here. */
2925	gcc_checking_assert (TYPE_ARG_TYPES (nftype)
2926	!= TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
2927
2928	/* Check that fnspec still works for the modified function signature,
2929	and drop it otherwise. */
2930	bool drop_fnspec = false;
2931	tree fnspec = lookup_attribute (attr_name: "fn spec", TYPE_ATTRIBUTES (nftype));
2932	attr_fnspec spec = fnspec ? attr_fnspec (fnspec) : attr_fnspec ("");
2933
2934	unsigned retcopy;
2935	if (!(fnspec && spec.returns_arg (arg_no: &retcopy)))
2936	retcopy = (unsigned) -1;
2937
2938	unsigned i = 0;
2939	for (tree nparm = DECL_ARGUMENTS (nnode->decl),
2940	nparmt = TYPE_ARG_TYPES (nftype);
2941	adjust_ftype > 0;
2942	i++, nparm = DECL_CHAIN (nparm), nparmt = TREE_CHAIN (nparmt))
2943	if (indirect_nparms.contains (k: nparm))
2944	{
2945	TREE_VALUE (nparmt) = TREE_TYPE (nparm);
2946	adjust_ftype--;
2947
2948	if (fnspec && !drop_fnspec)
2949	{
2950	if (i == retcopy)
2951	drop_fnspec = true;
2952	else if (spec.arg_specified_p (i))
2953	{
2954	/* Properties that apply to pointers only must not be
2955	present, because we don't make pointers further
2956	indirect. */
2957	gcc_checking_assert
2958	(!spec.arg_max_access_size_given_by_arg_p (i, NULL));
2959	gcc_checking_assert (!spec.arg_copied_to_arg_p (i, NULL));
2960
2961	/* Any claim of direct access only is invalidated by
2962	adding an indirection level. */
2963	if (spec.arg_direct_p (i))
2964	drop_fnspec = true;
2965
2966	/* If there's a claim the argument is not read from, the
2967	added indirection invalidates it: if the argument is
2968	used at all, then the pointer will necessarily be
2969	read. */
2970	if (!spec.arg_maybe_read_p (i)
2971	&& spec.arg_used_p (i))
2972	drop_fnspec = true;
2973	}
2974	}
2975	}
2976
2977	/* ??? Maybe we could adjust it instead. Note we don't need
2978	to mess with attribute access: pointer-typed parameters are
2979	not modified, so they can remain unchanged. */
2980	if (drop_fnspec)
2981	remove_named_attribute_unsharing (name: "fn spec",
2982	attrs: &TYPE_ATTRIBUTES (nftype));
2983
2984	TREE_TYPE (nnode->decl) = nftype;
2985	}
2986
2987	#if ATTR_FNSPEC_DECONST_WATERMARK
2988	{
2989	int flags = flags_from_decl_or_type (nnode->decl);
2990	tree fnspec = lookup_attribute ("fn spec", TREE_TYPE (nnode->decl));
2991
2992	if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
2993	{
2994	size_t xargs = 1 + int (is_stdarg) + int (apply_args);
2995	size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
2996	auto_vec<char> nspecv (tgtlen);
2997	char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
2998	bool no_writes_p = true;
2999	if (fnspec)
3000	{
3001	tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
3002	curlen = TREE_STRING_LENGTH (fnspecstr);
3003	memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
3004	if (!(flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS))
3005	&& curlen >= 2
3006	&& nspec[1] != 'c' && nspec[1] != 'C'
3007	&& nspec[1] != 'p' && nspec[1] != 'P')
3008	no_writes_p = false;
3009	}
3010	if (!curlen)
3011	{
3012	nspec[curlen++] = '.';
3013	nspec[curlen++] = ((flags & ECF_CONST)
3014	? 'c'
3015	: (flags & ECF_PURE)
3016	? 'p'
3017	: ' ');
3018	}
3019	while (curlen < tgtlen - 2 * xargs)
3020	{
3021	nspec[curlen++] = '.';
3022	nspec[curlen++] = ' ';
3023	}
3024
3025	/* These extra args are unlikely to be present in const or pure
3026	functions. It's conceivable that a function that takes variable
3027	arguments, or that passes its arguments on to another function,
3028	could be const or pure, but it would not modify the arguments, and,
3029	being pure or const, it couldn't possibly modify or even access
3030	memory referenced by them. But it can read from these internal
3031	data structures created by the wrapper, and from any
3032	argument-passing memory referenced by them, so we denote the
3033	possibility of reading from multiple levels of indirection, but
3034	only of reading because const/pure. */
3035	if (apply_args)
3036	{
3037	nspec[curlen++] = 'r';
3038	nspec[curlen++] = ' ';
3039	}
3040	if (is_stdarg)
3041	{
3042	nspec[curlen++] = (no_writes_p ? 'r' : '.');
3043	nspec[curlen++] = (no_writes_p ? 't' : ' ');
3044	}
3045
3046	nspec[curlen++] = 'W';
3047	nspec[curlen++] = 't';
3048
3049	/* The type has already been copied before adding parameters. */
3050	gcc_checking_assert (TYPE_ARG_TYPES (TREE_TYPE (nnode->decl))
3051	!= TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
3052	TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl))
3053	= tree_cons (get_identifier ("fn spec"),
3054	build_tree_list (NULL_TREE,
3055	build_string (tgtlen, nspec)),
3056	TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)));
3057	}
3058	}
3059	#endif
3060
3061	{
3062	tree decl = onode->decl;
3063	cgraph_node *target = nnode;
3064
3065	{ // copied from create_wrapper
3066
3067	/* Preserve DECL_RESULT so we get right by reference flag. */
3068	tree decl_result = DECL_RESULT (decl);
3069
3070	/* Remove the function's body but keep arguments to be reused
3071	for thunk. */
3072	onode->release_body (keep_arguments: true);
3073	onode->reset (/* unlike create_wrapper: preserve_comdat_group = */preserve_comdat_group: true);
3074
3075	DECL_UNINLINABLE (decl) = false;
3076	DECL_RESULT (decl) = decl_result;
3077	DECL_INITIAL (decl) = NULL;
3078	allocate_struct_function (decl, false);
3079	set_cfun (NULL);
3080
3081	/* Turn alias into thunk and expand it into GIMPLE representation. */
3082	onode->definition = true;
3083
3084	thunk_info::get_create (node: onode);
3085	onode->thunk = true;
3086	onode->create_edge (callee: target, NULL, count: onode->count);
3087	onode->callees->can_throw_external = !TREE_NOTHROW (target->decl);
3088
3089	tree arguments = DECL_ARGUMENTS (decl);
3090
3091	while (arguments)
3092	{
3093	TREE_ADDRESSABLE (arguments) = false;
3094	arguments = TREE_CHAIN (arguments);
3095	}
3096
3097	{
3098	tree alias = onode->callees->callee->decl;
3099	tree thunk_fndecl = decl;
3100	tree a;
3101
3102	int nxargs = 1 + is_stdarg + apply_args;
3103
3104	{ // Simplified from expand_thunk.
3105	tree restype;
3106	basic_block bb, then_bb, else_bb, return_bb;
3107	gimple_stmt_iterator bsi;
3108	int nargs = 0;
3109	tree arg;
3110	int i;
3111	tree resdecl;
3112	tree restmp = NULL;
3113
3114	gcall *call;
3115	greturn *ret;
3116	bool alias_is_noreturn = TREE_THIS_VOLATILE (alias);
3117
3118	a = DECL_ARGUMENTS (thunk_fndecl);
3119
3120	current_function_decl = thunk_fndecl;
3121
3122	/* Ensure thunks are emitted in their correct sections. */
3123	resolve_unique_section (thunk_fndecl, 0,
3124	flag_function_sections);
3125
3126	bitmap_obstack_initialize (NULL);
3127
3128	/* Build the return declaration for the function. */
3129	restype = TREE_TYPE (TREE_TYPE (thunk_fndecl));
3130	if (DECL_RESULT (thunk_fndecl) == NULL_TREE)
3131	{
3132	resdecl = build_decl (input_location, RESULT_DECL, 0, restype);
3133	DECL_ARTIFICIAL (resdecl) = 1;
3134	DECL_IGNORED_P (resdecl) = 1;
3135	DECL_CONTEXT (resdecl) = thunk_fndecl;
3136	DECL_RESULT (thunk_fndecl) = resdecl;
3137	}
3138	else
3139	resdecl = DECL_RESULT (thunk_fndecl);
3140
3141	profile_count cfg_count = onode->count;
3142	if (!cfg_count.initialized_p ())
3143	cfg_count = profile_count::from_gcov_type (BB_FREQ_MAX).guessed_local ();
3144
3145	bb = then_bb = else_bb = return_bb
3146	= init_lowered_empty_function (thunk_fndecl, true, cfg_count);
3147
3148	bsi = gsi_start_bb (bb);
3149
3150	/* Build call to the function being thunked. */
3151	if (!VOID_TYPE_P (restype)
3152	&& (!alias_is_noreturn
3153	|| TREE_ADDRESSABLE (restype)
3154	|| TREE_CODE (TYPE_SIZE_UNIT (restype)) != INTEGER_CST))
3155	{
3156	if (DECL_BY_REFERENCE (resdecl))
3157	{
3158	restmp = gimple_fold_indirect_ref (resdecl);
3159	if (!restmp)
3160	restmp = build2 (MEM_REF,
3161	TREE_TYPE (TREE_TYPE (resdecl)),
3162	resdecl,
3163	build_int_cst (TREE_TYPE (resdecl), 0));
3164	}
3165	else if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl)))
3166	{
3167	restmp = resdecl;
3168
3169	if (VAR_P (restmp))
3170	{
3171	add_local_decl (cfun, d: restmp);
3172	BLOCK_VARS (DECL_INITIAL (current_function_decl))
3173	= restmp;
3174	}
3175	}
3176	else
3177	restmp = create_tmp_reg (restype, "retval");
3178	}
3179
3180	for (arg = a; arg; arg = DECL_CHAIN (arg))
3181	nargs++;
3182	auto_vec<tree> vargs (nargs + nxargs);
3183	i = 0;
3184	arg = a;
3185
3186	if (nargs)
3187	for (tree nparm = DECL_ARGUMENTS (nnode->decl);
3188	i < nargs;
3189	i++, arg = DECL_CHAIN (arg), nparm = DECL_CHAIN (nparm))
3190	{
3191	tree save_arg = arg;
3192
3193	/* Arrange to pass indirectly the parms, if we decided to do
3194	so, and revert its type in the wrapper. */
3195	if (indirect_nparms.contains (k: nparm))
3196	{
3197	tree ref_type = TREE_TYPE (nparm);
3198	TREE_ADDRESSABLE (arg) = true;
3199	arg = build1 (ADDR_EXPR, ref_type, arg);
3200	}
3201	else if (!TREE_THIS_VOLATILE (arg))
3202	DECL_NOT_GIMPLE_REG_P (arg) = 0;
3203
3204	/* Convert the argument back to the type used by the calling
3205	conventions, e.g. a non-prototyped float type is passed as
3206	double, as in 930603-1.c, and needs to be converted back to
3207	double to be passed on unchanged to the wrapped
3208	function. */
3209	if (TREE_TYPE (nparm) != DECL_ARG_TYPE (nparm))
3210	{
3211	tree tmp = arg;
3212	/* If ARG is e.g. volatile, we must copy and
3213	convert in separate statements. */
3214	if (!is_gimple_val (arg))
3215	{
3216	tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3217	(TREE_TYPE (arg)), "arg");
3218	gimple *stmt = gimple_build_assign (tmp, arg);
3219	gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3220	}
3221	arg = fold_convert (DECL_ARG_TYPE (nparm), tmp);
3222	}
3223
3224	if (!is_gimple_val (arg))
3225	{
3226	tree tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3227	(TREE_TYPE (arg)), "arg");
3228	gimple *stmt = gimple_build_assign (tmp, arg);
3229	gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3230	arg = tmp;
3231	}
3232	vargs.quick_push (obj: arg);
3233	arg = save_arg;
3234	}
3235	/* These strub arguments are adjusted later. */
3236	if (apply_args)
3237	vargs.quick_push (null_pointer_node);
3238	if (is_stdarg)
3239	vargs.quick_push (null_pointer_node);
3240	vargs.quick_push (null_pointer_node);
3241	call = gimple_build_call_vec (build_fold_addr_expr_loc (0, alias),
3242	vargs);
3243	onode->callees->call_stmt = call;
3244	// gimple_call_set_from_thunk (call, true);
3245	if (DECL_STATIC_CHAIN (alias))
3246	{
3247	tree p = DECL_STRUCT_FUNCTION (alias)->static_chain_decl;
3248	tree type = TREE_TYPE (p);
3249	tree decl = build_decl (DECL_SOURCE_LOCATION (thunk_fndecl),
3250	PARM_DECL, create_tmp_var_name ("CHAIN"),
3251	type);
3252	DECL_ARTIFICIAL (decl) = 1;
3253	DECL_IGNORED_P (decl) = 1;
3254	TREE_USED (decl) = 1;
3255	DECL_CONTEXT (decl) = thunk_fndecl;
3256	DECL_ARG_TYPE (decl) = type;
3257	TREE_READONLY (decl) = 1;
3258
3259	struct function *sf = DECL_STRUCT_FUNCTION (thunk_fndecl);
3260	sf->static_chain_decl = decl;
3261
3262	gimple_call_set_chain (call_stmt: call, chain: decl);
3263	}
3264
3265	/* Return slot optimization is always possible and in fact required to
3266	return values with DECL_BY_REFERENCE. */
3267	if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl))
3268	&& (!is_gimple_reg_type (TREE_TYPE (resdecl))
3269	|| DECL_BY_REFERENCE (resdecl)))
3270	gimple_call_set_return_slot_opt (s: call, return_slot_opt_p: true);
3271
3272	if (restmp)
3273	{
3274	gimple_call_set_lhs (gs: call, lhs: restmp);
3275	gcc_assert (useless_type_conversion_p (TREE_TYPE (restmp),
3276	TREE_TYPE (TREE_TYPE (alias))));
3277	}
3278	gsi_insert_after (&bsi, call, GSI_NEW_STMT);
3279	if (!alias_is_noreturn)
3280	{
3281	/* Build return value. */
3282	if (!DECL_BY_REFERENCE (resdecl))
3283	ret = gimple_build_return (restmp);
3284	else
3285	ret = gimple_build_return (resdecl);
3286
3287	gsi_insert_after (&bsi, ret, GSI_NEW_STMT);
3288	}
3289	else
3290	{
3291	remove_edge (single_succ_edge (bb));
3292	}
3293
3294	cfun->gimple_df->in_ssa_p = true;
3295	update_max_bb_count ();
3296	profile_status_for_fn (cfun)
3297	= cfg_count.initialized_p () && cfg_count.ipa_p ()
3298	? PROFILE_READ : PROFILE_GUESSED;
3299	/* FIXME: C++ FE should stop setting TREE_ASM_WRITTEN on thunks. */
3300	// TREE_ASM_WRITTEN (thunk_fndecl) = false;
3301	delete_unreachable_blocks ();
3302	update_ssa (TODO_update_ssa);
3303	checking_verify_flow_info ();
3304	free_dominance_info (CDI_DOMINATORS);
3305
3306	/* Since we want to emit the thunk, we explicitly mark its name as
3307	referenced. */
3308	onode->thunk = false;
3309	onode->lowered = true;
3310	bitmap_obstack_release (NULL);
3311	}
3312	current_function_decl = NULL;
3313	set_cfun (NULL);
3314	}
3315
3316	thunk_info::remove (node: onode);
3317
3318	// some more of create_wrapper at the end of the next block.
3319	}
3320	}
3321
3322	{
3323	tree aaval = NULL_TREE;
3324	tree vaptr = NULL_TREE;
3325	tree wmptr = NULL_TREE;
3326	for (tree arg = DECL_ARGUMENTS (nnode->decl); arg; arg = DECL_CHAIN (arg))
3327	{
3328	aaval = vaptr;
3329	vaptr = wmptr;
3330	wmptr = arg;
3331	}
3332
3333	if (!apply_args)
3334	aaval = NULL_TREE;
3335	/* The trailing args are [apply_args], [va_list_ptr], and
3336	watermark. If we don't have a va_list_ptr, the penultimate
3337	argument is apply_args.
3338	*/
3339	else if (!is_stdarg)
3340	aaval = vaptr;
3341
3342	if (!is_stdarg)
3343	vaptr = NULL_TREE;
3344
3345	DECL_NAME (wmptr) = get_watermark_ptr ();
3346	DECL_ARTIFICIAL (wmptr) = 1;
3347	DECL_IGNORED_P (wmptr) = 1;
3348	TREE_USED (wmptr) = 1;
3349
3350	if (is_stdarg)
3351	{
3352	DECL_NAME (vaptr) = get_va_list_ptr ();
3353	DECL_ARTIFICIAL (vaptr) = 1;
3354	DECL_IGNORED_P (vaptr) = 1;
3355	TREE_USED (vaptr) = 1;
3356	}
3357
3358	if (apply_args)
3359	{
3360	DECL_NAME (aaval) = get_apply_args ();
3361	DECL_ARTIFICIAL (aaval) = 1;
3362	DECL_IGNORED_P (aaval) = 1;
3363	TREE_USED (aaval) = 1;
3364	}
3365
3366	push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
3367
3368	{
3369	edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
3370	gimple_seq seq = call_update_watermark (wmptr, node: nnode, count: e->src->count);
3371	gsi_insert_seq_on_edge_immediate (e, seq);
3372	}
3373
3374	bool any_indirect = !indirect_nparms.is_empty ();
3375
3376	if (any_indirect)
3377	{
3378	basic_block bb;
3379	bool needs_commit = false;
3380	FOR_EACH_BB_FN (bb, cfun)
3381	{
3382	for (gphi_iterator gsi = gsi_start_nonvirtual_phis (bb);
3383	!gsi_end_p (i: gsi);
3384	gsi_next_nonvirtual_phi (i: &gsi))
3385	{
3386	gphi *stmt = gsi.phi ();
3387
3388	walk_stmt_info wi = {};
3389	wi.info = &indirect_nparms;
3390	walk_gimple_op (stmt, walk_make_indirect, &wi);
3391	if (wi.changed && !is_gimple_debug (gs: gsi_stmt (i: gsi)))
3392	if (walk_regimplify_phi (stmt))
3393	needs_commit = true;
3394	}
3395
3396	for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
3397	!gsi_end_p (i: gsi); gsi_next (i: &gsi))
3398	{
3399	gimple *stmt = gsi_stmt (i: gsi);
3400
3401	walk_stmt_info wi = {};
3402	wi.info = &indirect_nparms;
3403	walk_gimple_op (stmt, walk_make_indirect, &wi);
3404	if (wi.changed)
3405	{
3406	if (!is_gimple_debug (gs: stmt))
3407	{
3408	wi.info = &gsi;
3409	walk_gimple_op (stmt, walk_regimplify_addr_expr,
3410	&wi);
3411	}
3412	update_stmt (s: stmt);
3413	}
3414	}
3415	}
3416	if (needs_commit)
3417	gsi_commit_edge_inserts ();
3418	}
3419
3420	if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca
3421	|| is_stdarg || apply_args)
3422	for (cgraph_edge *e = nnode->callees, *enext; e; e = enext)
3423	{
3424	if (!e->call_stmt)
3425	continue;
3426
3427	gcall *call = e->call_stmt;
3428	gimple_stmt_iterator gsi = gsi_for_stmt (call);
3429	tree fndecl = e->callee->decl;
3430
3431	enext = e->next_callee;
3432
3433	if (gimple_alloca_call_p (call))
3434	{
3435	gimple_seq seq = call_update_watermark (wmptr, NULL,
3436	count: gsi_bb (i: gsi)->count);
3437	gsi_insert_finally_seq_after_call (gsi, seq);
3438	}
3439	else if (fndecl && is_stdarg
3440	&& fndecl_built_in_p (node: fndecl, name1: BUILT_IN_VA_START))
3441	{
3442	/* Using a non-default stdarg ABI makes the function ineligible
3443	for internal strub. */
3444	gcc_checking_assert (builtin_decl_explicit (BUILT_IN_VA_START)
3445	== fndecl);
3446	tree bvacopy = builtin_decl_explicit (fncode: BUILT_IN_VA_COPY);
3447	gimple_call_set_fndecl (gs: call, decl: bvacopy);
3448	tree arg = vaptr;
3449	/* The va_copy source must be dereferenced, unless it's an array
3450	type, that would have decayed to a pointer. */
3451	if (TREE_CODE (TREE_TYPE (TREE_TYPE (vaptr))) != ARRAY_TYPE)
3452	{
3453	arg = gimple_fold_indirect_ref (vaptr);
3454	if (!arg)
3455	arg = build2 (MEM_REF,
3456	TREE_TYPE (TREE_TYPE (vaptr)),
3457	vaptr,
3458	build_int_cst (TREE_TYPE (vaptr), 0));
3459	if (!is_gimple_val (arg))
3460	arg = force_gimple_operand_gsi (&gsi, arg, true,
3461	NULL_TREE, true, GSI_SAME_STMT);
3462	}
3463	gimple_call_set_arg (gs: call, index: 1, arg);
3464	update_stmt (s: call);
3465	e->redirect_callee (n: cgraph_node::get_create (bvacopy));
3466	}
3467	else if (fndecl && apply_args
3468	&& fndecl_built_in_p (node: fndecl, name1: BUILT_IN_APPLY_ARGS))
3469	{
3470	tree lhs = gimple_call_lhs (gs: call);
3471	gimple *assign = (lhs
3472	? gimple_build_assign (lhs, aaval)
3473	: gimple_build_nop ());
3474	gsi_replace (&gsi, assign, true);
3475	cgraph_edge::remove (edge: e);
3476	}
3477	}
3478
3479	{ // a little more copied from create_wrapper
3480
3481	/* Inline summary set-up. */
3482	nnode->analyze ();
3483	// inline_analyze_function (nnode);
3484	}
3485
3486	pop_cfun ();
3487	}
3488
3489	{
3490	push_cfun (DECL_STRUCT_FUNCTION (onode->decl));
3491	gimple_stmt_iterator gsi
3492	= gsi_after_labels (bb: single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
3493
3494	gcall *wrcall;
3495	while (!(wrcall = dyn_cast <gcall *> (p: gsi_stmt (i: gsi))))
3496	gsi_next (i: &gsi);
3497
3498	tree swm = create_tmp_var (get_wmt (), ".strub.watermark");
3499	TREE_ADDRESSABLE (swm) = true;
3500	tree swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
3501
3502	tree enter = get_enter ();
3503	gcall *stptr = gimple_build_call (enter, 1, unshare_expr (swmp));
3504	gimple_set_location (g: stptr, location: gimple_location (g: wrcall));
3505	gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
3506	onode->create_edge (callee: cgraph_node::get_create (enter),
3507	call_stmt: stptr, count: gsi_bb (i: gsi)->count, cloning_p: false);
3508
3509	int nargs = gimple_call_num_args (gs: wrcall);
3510
3511	gimple_seq seq = NULL;
3512
3513	if (apply_args)
3514	{
3515	tree aalst = create_tmp_var (ptr_type_node, ".strub.apply_args");
3516	tree bappargs = builtin_decl_explicit (fncode: BUILT_IN_APPLY_ARGS);
3517	gcall *appargs = gimple_build_call (bappargs, 0);
3518	gimple_call_set_lhs (gs: appargs, lhs: aalst);
3519	gimple_set_location (g: appargs, location: gimple_location (g: wrcall));
3520	gsi_insert_before (&gsi, appargs, GSI_SAME_STMT);
3521	gimple_call_set_arg (gs: wrcall, index: nargs - 2 - is_stdarg, arg: aalst);
3522	onode->create_edge (callee: cgraph_node::get_create (bappargs),
3523	call_stmt: appargs, count: gsi_bb (i: gsi)->count, cloning_p: false);
3524	}
3525
3526	if (is_stdarg)
3527	{
3528	tree valst = create_tmp_var (va_list_type_node, ".strub.va_list");
3529	TREE_ADDRESSABLE (valst) = true;
3530	tree vaptr = build1 (ADDR_EXPR,
3531	build_pointer_type (va_list_type_node),
3532	valst);
3533	gimple_call_set_arg (gs: wrcall, index: nargs - 2, arg: unshare_expr (vaptr));
3534
3535	tree bvastart = builtin_decl_explicit (fncode: BUILT_IN_VA_START);
3536	gcall *vastart = gimple_build_call (bvastart, 2,
3537	unshare_expr (vaptr),
3538	integer_zero_node);
3539	gimple_set_location (g: vastart, location: gimple_location (g: wrcall));
3540	gsi_insert_before (&gsi, vastart, GSI_SAME_STMT);
3541	onode->create_edge (callee: cgraph_node::get_create (bvastart),
3542	call_stmt: vastart, count: gsi_bb (i: gsi)->count, cloning_p: false);
3543
3544	tree bvaend = builtin_decl_explicit (fncode: BUILT_IN_VA_END);
3545	gcall *vaend = gimple_build_call (bvaend, 1, unshare_expr (vaptr));
3546	gimple_set_location (g: vaend, location: gimple_location (g: wrcall));
3547	gimple_seq_add_stmt (&seq, vaend);
3548	}
3549
3550	gimple_call_set_arg (gs: wrcall, index: nargs - 1, arg: unshare_expr (swmp));
3551	// gimple_call_set_tail (wrcall, false);
3552	update_stmt (s: wrcall);
3553
3554	{
3555	#if !ATTR_FNSPEC_DECONST_WATERMARK
3556	/* If the call will be assumed to not modify or even read the
3557	watermark, make it read and modified ourselves. */
3558	if ((gimple_call_flags (wrcall)
3559	& (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
3560	{
3561	vec<tree, va_gc> *inputs = NULL;
3562	vec<tree, va_gc> *outputs = NULL;
3563	vec_safe_push (v&: outputs,
3564	obj: build_tree_list
3565	(build_tree_list
3566	(NULL_TREE, build_string (2, "=m")),
3567	swm));
3568	vec_safe_push (v&: inputs,
3569	obj: build_tree_list
3570	(build_tree_list
3571	(NULL_TREE, build_string (1, "m")),
3572	swm));
3573	gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
3574	NULL, NULL);
3575	gimple_seq_add_stmt (&seq, forcemod);
3576
3577	/* If the call will be assumed to not even read the watermark,
3578	make sure it is already in memory before the call. */
3579	if ((gimple_call_flags (wrcall) & ECF_CONST))
3580	{
3581	vec<tree, va_gc> *inputs = NULL;
3582	vec_safe_push (v&: inputs,
3583	obj: build_tree_list
3584	(build_tree_list
3585	(NULL_TREE, build_string (1, "m")),
3586	swm));
3587	gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
3588	NULL, NULL);
3589	gimple_set_location (g: force_store, location: gimple_location (g: wrcall));
3590	gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
3591	}
3592	}
3593	#endif
3594
3595	gcall *sleave = gimple_build_call (get_leave (), 1,
3596	unshare_expr (swmp));
3597	gimple_seq_add_stmt (&seq, sleave);
3598
3599	gassign *clobber = gimple_build_assign (swm,
3600	build_clobber
3601	(TREE_TYPE (swm)));
3602	gimple_seq_add_stmt (&seq, clobber);
3603	}
3604
3605	gsi_insert_finally_seq_after_call (gsi, seq);
3606
3607	/* For nnode, we don't rebuild edges because we wish to retain
3608	any redirections copied to it from earlier passes, so we add
3609	call graph edges explicitly there, but for onode, we create a
3610	fresh function, so we may as well just issue the calls and
3611	then rebuild all cgraph edges. */
3612	// cgraph_edge::rebuild_edges ();
3613	onode->analyze ();
3614	// inline_analyze_function (onode);
3615
3616	pop_cfun ();
3617	}
3618	}
3619
3620	return 0;
3621	}
3622
3623	simple_ipa_opt_pass *
3624	make_pass_ipa_strub (gcc::context *ctxt)
3625	{
3626	return new pass_ipa_strub (ctxt);
3627	}
3628
3629	#include "gt-ipa-strub.h"
3630