1/* Schedule GIMPLE vector statements.
2 Copyright (C) 2020-2024 Free Software Foundation, Inc.
3
4This file is part of GCC.
5
6GCC is free software; you can redistribute it and/or modify it
7under the terms of the GNU General Public License as published by the
8Free Software Foundation; either version 3, or (at your option) any
9later version.
10
11GCC is distributed in the hope that it will be useful, but WITHOUT
12ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14for more details.
15
16You should have received a copy of the GNU General Public License
17along with GCC; see the file COPYING3. If not see
18<http://www.gnu.org/licenses/>. */
19
20#include "config.h"
21#include "system.h"
22#include "coretypes.h"
23#include "backend.h"
24#include "rtl.h"
25#include "tree.h"
26#include "gimple.h"
27#include "tree-pass.h"
28#include "ssa.h"
29#include "expmed.h"
30#include "optabs-tree.h"
31#include "tree-eh.h"
32#include "gimple-iterator.h"
33#include "gimplify-me.h"
34#include "gimplify.h"
35#include "tree-cfg.h"
36#include "bitmap.h"
37#include "tree-ssa-dce.h"
38#include "memmodel.h"
39#include "optabs.h"
40#include "gimple-fold.h"
41#include "internal-fn.h"
42
43/* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
44 internal function based on vector type of selected expansion.
45
46 For vec_set:
47
48 VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
49 =>
50 _7 = u;
51 _8 = .VEC_SET (_7, i_4(D), _1);
52 u = _8;
53
54 For vec_extract:
55
56 _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
57 =>
58 _4 = vD.2208;
59 _3 = .VEC_EXTRACT (_4, idx_2(D)); */
60
61static bool
62gimple_expand_vec_set_extract_expr (struct function *fun,
63 gimple_stmt_iterator *gsi)
64{
65 gcall *new_stmt = NULL;
66 gassign *ass_stmt = NULL;
67 bool cfg_changed = false;
68
69 /* Only consider code == GIMPLE_ASSIGN. */
70 gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi));
71 if (!stmt)
72 return false;
73
74 bool is_extract = false;
75
76 tree lhs = gimple_assign_lhs (gs: stmt);
77 tree rhs = gimple_assign_rhs1 (gs: stmt);
78 tree val, ref;
79 if (TREE_CODE (lhs) == ARRAY_REF)
80 {
81 /* Assume it is a vec_set. */
82 val = rhs;
83 ref = lhs;
84 }
85 else if (TREE_CODE (rhs) == ARRAY_REF)
86 {
87 /* vec_extract. */
88 is_extract = true;
89 val = lhs;
90 ref = rhs;
91 }
92 else
93 return false;
94
95 tree op0 = TREE_OPERAND (ref, 0);
96 if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
97 && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
98 && TYPE_MODE (TREE_TYPE (ref))
99 == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
100 {
101 tree pos = TREE_OPERAND (ref, 1);
102
103 tree view_op0 = TREE_OPERAND (op0, 0);
104 machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
105 machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
106
107 if ((auto_var_in_fn_p (view_op0, fun->decl)
108 || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
109 && !TREE_ADDRESSABLE (view_op0)
110 && ((!is_extract && can_vec_set_var_idx_p (outermode))
111 || (is_extract
112 && can_vec_extract_var_idx_p (outermode, extract_mode))))
113 {
114 location_t loc = gimple_location (g: stmt);
115 tree var_src = make_ssa_name (TREE_TYPE (view_op0));
116
117 ass_stmt = gimple_build_assign (var_src, view_op0);
118 gimple_set_vuse (g: ass_stmt, vuse: gimple_vuse (g: stmt));
119 gimple_set_location (g: ass_stmt, location: loc);
120 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
121
122 if (!is_extract)
123 {
124 tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
125
126 new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
127 val, pos);
128
129 gimple_call_set_lhs (gs: new_stmt, lhs: var_dst);
130 gimple_set_location (g: new_stmt, location: loc);
131 gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
132
133 ass_stmt = gimple_build_assign (view_op0, var_dst);
134 gimple_set_location (g: ass_stmt, location: loc);
135 gimple_move_vops (ass_stmt, stmt);
136 gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
137
138 basic_block bb = gimple_bb (g: stmt);
139 if (gsi_remove (gsi, true)
140 && gimple_purge_dead_eh_edges (bb))
141 cfg_changed = true;
142 *gsi = gsi_for_stmt (ass_stmt);
143 }
144 else
145 {
146 new_stmt
147 = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
148 gimple_call_set_lhs (gs: new_stmt, lhs);
149
150 gsi_replace (gsi, new_stmt, true);
151 cfg_changed = true;
152 }
153 }
154 }
155
156 return cfg_changed;
157}
158
159/* Expand all VEC_COND_EXPR gimple assignments into calls to internal
160 function based on type of selected expansion. */
161
162static gimple *
163gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi,
164 hash_map<tree, unsigned int> *vec_cond_ssa_name_uses)
165{
166 tree lhs, op0a = NULL_TREE, op0b = NULL_TREE;
167 enum tree_code code;
168 enum tree_code tcode;
169 machine_mode cmp_op_mode;
170 bool unsignedp;
171 enum insn_code icode;
172 imm_use_iterator imm_iter;
173
174 /* Only consider code == GIMPLE_ASSIGN. */
175 gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi));
176 if (!stmt)
177 return NULL;
178
179 code = gimple_assign_rhs_code (gs: stmt);
180 if (code != VEC_COND_EXPR)
181 return NULL;
182
183 tree op0 = gimple_assign_rhs1 (gs: stmt);
184 tree op1 = gimple_assign_rhs2 (gs: stmt);
185 tree op2 = gimple_assign_rhs3 (gs: stmt);
186 lhs = gimple_assign_lhs (gs: stmt);
187 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
188
189 /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
190 Those can end up generated by folding and at least for integer mode masks
191 we cannot expect vcond expanders to exist. We lower a ? b : c
192 to (b & a) | (c & ~a). */
193 if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
194 && !VECTOR_MODE_P (mode))
195 {
196 gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
197 gimple_seq stmts = NULL;
198 tree type = TREE_TYPE (lhs);
199 location_t loc = gimple_location (g: stmt);
200 tree tem0 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op1, ops: op0);
201 tree tem1 = gimple_build (seq: &stmts, loc, code: BIT_NOT_EXPR, type, ops: op0);
202 tree tem2 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op2, ops: tem1);
203 tree tem3 = gimple_build (seq: &stmts, loc, code: BIT_IOR_EXPR, type, ops: tem0, ops: tem2);
204 gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
205 return gimple_build_assign (lhs, tem3);
206 }
207
208 bool can_compute_op0 = true;
209 gcc_assert (!COMPARISON_CLASS_P (op0));
210 if (TREE_CODE (op0) == SSA_NAME)
211 {
212 unsigned int used_vec_cond_exprs = 0;
213 unsigned int *slot = vec_cond_ssa_name_uses->get (k: op0);
214 if (slot)
215 used_vec_cond_exprs = *slot;
216 else
217 {
218 gimple *use_stmt;
219 FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0)
220 {
221 gassign *assign = dyn_cast<gassign *> (p: use_stmt);
222 if (assign != NULL
223 && gimple_assign_rhs_code (gs: assign) == VEC_COND_EXPR
224 && gimple_assign_rhs1 (gs: assign) == op0)
225 used_vec_cond_exprs++;
226 }
227 vec_cond_ssa_name_uses->put (k: op0, v: used_vec_cond_exprs);
228 }
229
230 gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
231 if (def_stmt)
232 {
233 tcode = gimple_assign_rhs_code (gs: def_stmt);
234 op0a = gimple_assign_rhs1 (gs: def_stmt);
235 op0b = gimple_assign_rhs2 (gs: def_stmt);
236
237 tree op0_type = TREE_TYPE (op0);
238 tree op0a_type = TREE_TYPE (op0a);
239 if (TREE_CODE_CLASS (tcode) == tcc_comparison)
240 can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
241 tcode);
242
243 /* Try to fold x CMP y ? -1 : 0 to x CMP y. */
244 if (can_compute_op0
245 && integer_minus_onep (op1)
246 && integer_zerop (op2)
247 && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
248 {
249 tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0);
250 gassign *new_stmt = gimple_build_assign (lhs, conv_op);
251 gsi_replace (gsi, new_stmt, true);
252 return new_stmt;
253 }
254
255 /* When the compare has EH we do not want to forward it when
256 it has multiple uses and in general because of the complication
257 with EH redirection. */
258 if (stmt_can_throw_internal (fun, def_stmt))
259 tcode = TREE_CODE (op0);
260
261 /* If we can compute op0 and have multiple uses, keep the SSA
262 name and use vcond_mask. */
263 else if (can_compute_op0
264 && used_vec_cond_exprs >= 2
265 && (get_vcond_mask_icode (vmode: mode, TYPE_MODE (op0_type))
266 != CODE_FOR_nothing))
267 tcode = TREE_CODE (op0);
268 }
269 else
270 tcode = TREE_CODE (op0);
271 }
272 else
273 tcode = TREE_CODE (op0);
274
275 if (TREE_CODE_CLASS (tcode) != tcc_comparison)
276 {
277 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
278 if (get_vcond_mask_icode (vmode: mode, TYPE_MODE (TREE_TYPE (op0)))
279 != CODE_FOR_nothing)
280 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
281 /* Fake op0 < 0. */
282 else
283 {
284 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
285 == MODE_VECTOR_INT);
286 op0a = op0;
287 op0b = build_zero_cst (TREE_TYPE (op0));
288 tcode = LT_EXPR;
289 }
290 }
291 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
292 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
293
294 gcc_assert (known_eq (GET_MODE_NUNITS (mode),
295 GET_MODE_NUNITS (cmp_op_mode)));
296
297 icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: unsignedp);
298 /* Some targets do not have vcondeq and only vcond with NE/EQ
299 but not vcondu, so make sure to also try vcond here as
300 vcond_icode_p would canonicalize the optab query to. */
301 if (icode == CODE_FOR_nothing
302 && (tcode == NE_EXPR || tcode == EQ_EXPR)
303 && ((icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: !unsignedp))
304 != CODE_FOR_nothing))
305 unsignedp = !unsignedp;
306 if (icode == CODE_FOR_nothing)
307 {
308 if (tcode == LT_EXPR
309 && op0a == op0)
310 {
311 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
312 into a constant when only get_vcond_eq_icode is supported.
313 Try changing it to NE_EXPR. */
314 tcode = NE_EXPR;
315 }
316 if ((tcode == EQ_EXPR || tcode == NE_EXPR)
317 && direct_internal_fn_supported_p (fn: IFN_VCONDEQ, TREE_TYPE (lhs),
318 TREE_TYPE (op0a),
319 opt_type: OPTIMIZE_FOR_BOTH))
320 {
321 tree tcode_tree = build_int_cst (integer_type_node, tcode);
322 return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1,
323 op2, tcode_tree);
324 }
325
326 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))
327 && can_compute_op0
328 && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
329 != CODE_FOR_nothing));
330 return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
331 }
332
333 tree tcode_tree = build_int_cst (integer_type_node, tcode);
334 return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND,
335 5, op0a, op0b, op1, op2, tcode_tree);
336}
337
338
339
340namespace {
341
342const pass_data pass_data_gimple_isel =
343{
344 .type: GIMPLE_PASS, /* type */
345 .name: "isel", /* name */
346 .optinfo_flags: OPTGROUP_VEC, /* optinfo_flags */
347 .tv_id: TV_NONE, /* tv_id */
348 PROP_cfg, /* properties_required */
349 .properties_provided: 0, /* properties_provided */
350 .properties_destroyed: 0, /* properties_destroyed */
351 .todo_flags_start: 0, /* todo_flags_start */
352 TODO_update_ssa, /* todo_flags_finish */
353};
354
355class pass_gimple_isel : public gimple_opt_pass
356{
357public:
358 pass_gimple_isel (gcc::context *ctxt)
359 : gimple_opt_pass (pass_data_gimple_isel, ctxt)
360 {}
361
362 /* opt_pass methods: */
363 bool gate (function *) final override
364 {
365 return true;
366 }
367
368 unsigned int execute (function *fun) final override;
369}; // class pass_gimple_isel
370
371
372/* Iterate all gimple statements and perform pre RTL expansion
373 GIMPLE massaging to improve instruction selection. */
374
375unsigned int
376pass_gimple_isel::execute (struct function *fun)
377{
378 gimple_stmt_iterator gsi;
379 basic_block bb;
380 hash_map<tree, unsigned int> vec_cond_ssa_name_uses;
381 auto_bitmap dce_ssa_names;
382 bool cfg_changed = false;
383
384 FOR_EACH_BB_FN (bb, fun)
385 {
386 for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi))
387 {
388 /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
389 calls mapping to supported optabs. */
390 gimple *g = gimple_expand_vec_cond_expr (fun, gsi: &gsi,
391 vec_cond_ssa_name_uses: &vec_cond_ssa_name_uses);
392 if (g != NULL)
393 {
394 tree lhs = gimple_assign_lhs (gs: gsi_stmt (i: gsi));
395 gimple_set_lhs (g, lhs);
396 gsi_replace (&gsi, g, false);
397 }
398
399 /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */
400 cfg_changed |= gimple_expand_vec_set_extract_expr (fun, gsi: &gsi);
401 if (gsi_end_p (i: gsi))
402 break;
403
404 gassign *stmt = dyn_cast <gassign *> (p: *gsi);
405 if (!stmt)
406 continue;
407
408 tree_code code = gimple_assign_rhs_code (gs: stmt);
409 tree lhs = gimple_assign_lhs (gs: stmt);
410 if (TREE_CODE_CLASS (code) == tcc_comparison
411 && !has_single_use (var: lhs))
412 {
413 /* Duplicate COND_EXPR condition defs when they are
414 comparisons so RTL expansion with the help of TER
415 can perform better if conversion. */
416 imm_use_iterator imm_iter;
417 use_operand_p use_p;
418 auto_vec<gassign *, 4> cond_exprs;
419 unsigned cnt = 0;
420 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
421 {
422 if (is_gimple_debug (USE_STMT (use_p)))
423 continue;
424 cnt++;
425 if (gimple_bb (USE_STMT (use_p)) == bb
426 && is_gimple_assign (USE_STMT (use_p))
427 && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use
428 && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR)
429 cond_exprs.safe_push (obj: as_a <gassign *> (USE_STMT (use_p)));
430 }
431 for (unsigned i = cond_exprs.length () == cnt ? 1 : 0;
432 i < cond_exprs.length (); ++i)
433 {
434 gassign *copy = as_a <gassign *> (p: gimple_copy (stmt));
435 tree new_def = duplicate_ssa_name (var: lhs, stmt: copy);
436 gimple_assign_set_lhs (gs: copy, lhs: new_def);
437 auto gsi2 = gsi_for_stmt (cond_exprs[i]);
438 gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
439 gimple_assign_set_rhs1 (gs: cond_exprs[i], rhs: new_def);
440 update_stmt (s: cond_exprs[i]);
441 }
442 }
443 }
444 }
445
446 for (auto it = vec_cond_ssa_name_uses.begin ();
447 it != vec_cond_ssa_name_uses.end (); ++it)
448 bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first));
449
450 simple_dce_from_worklist (dce_ssa_names);
451
452 return cfg_changed ? TODO_cleanup_cfg : 0;
453}
454
455} // anon namespace
456
457gimple_opt_pass *
458make_pass_gimple_isel (gcc::context *ctxt)
459{
460 return new pass_gimple_isel (ctxt);
461}
462
463

source code of gcc/gimple-isel.cc