1 | /* Schedule GIMPLE vector statements. |
2 | Copyright (C) 2020-2024 Free Software Foundation, Inc. |
3 | |
4 | This file is part of GCC. |
5 | |
6 | GCC is free software; you can redistribute it and/or modify it |
7 | under the terms of the GNU General Public License as published by the |
8 | Free Software Foundation; either version 3, or (at your option) any |
9 | later version. |
10 | |
11 | GCC is distributed in the hope that it will be useful, but WITHOUT |
12 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
14 | for more details. |
15 | |
16 | You should have received a copy of the GNU General Public License |
17 | along with GCC; see the file COPYING3. If not see |
18 | <http://www.gnu.org/licenses/>. */ |
19 | |
20 | #include "config.h" |
21 | #include "system.h" |
22 | #include "coretypes.h" |
23 | #include "backend.h" |
24 | #include "rtl.h" |
25 | #include "tree.h" |
26 | #include "gimple.h" |
27 | #include "tree-pass.h" |
28 | #include "ssa.h" |
29 | #include "expmed.h" |
30 | #include "optabs-tree.h" |
31 | #include "tree-eh.h" |
32 | #include "gimple-iterator.h" |
33 | #include "gimplify-me.h" |
34 | #include "gimplify.h" |
35 | #include "tree-cfg.h" |
36 | #include "bitmap.h" |
37 | #include "tree-ssa-dce.h" |
38 | #include "memmodel.h" |
39 | #include "optabs.h" |
40 | #include "gimple-fold.h" |
41 | #include "internal-fn.h" |
42 | |
43 | /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to |
44 | internal function based on vector type of selected expansion. |
45 | |
46 | For vec_set: |
47 | |
48 | VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D); |
49 | => |
50 | _7 = u; |
51 | _8 = .VEC_SET (_7, i_4(D), _1); |
52 | u = _8; |
53 | |
54 | For vec_extract: |
55 | |
56 | _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)]; |
57 | => |
58 | _4 = vD.2208; |
59 | _3 = .VEC_EXTRACT (_4, idx_2(D)); */ |
60 | |
61 | static bool |
62 | gimple_expand_vec_set_extract_expr (struct function *fun, |
63 | gimple_stmt_iterator *gsi) |
64 | { |
65 | gcall *new_stmt = NULL; |
66 | gassign *ass_stmt = NULL; |
67 | bool cfg_changed = false; |
68 | |
69 | /* Only consider code == GIMPLE_ASSIGN. */ |
70 | gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi)); |
71 | if (!stmt) |
72 | return false; |
73 | |
74 | bool = false; |
75 | |
76 | tree lhs = gimple_assign_lhs (gs: stmt); |
77 | tree rhs = gimple_assign_rhs1 (gs: stmt); |
78 | tree val, ref; |
79 | if (TREE_CODE (lhs) == ARRAY_REF) |
80 | { |
81 | /* Assume it is a vec_set. */ |
82 | val = rhs; |
83 | ref = lhs; |
84 | } |
85 | else if (TREE_CODE (rhs) == ARRAY_REF) |
86 | { |
87 | /* vec_extract. */ |
88 | is_extract = true; |
89 | val = lhs; |
90 | ref = rhs; |
91 | } |
92 | else |
93 | return false; |
94 | |
95 | tree op0 = TREE_OPERAND (ref, 0); |
96 | if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0)) |
97 | && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0))) |
98 | && TYPE_MODE (TREE_TYPE (ref)) |
99 | == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0))))) |
100 | { |
101 | tree pos = TREE_OPERAND (ref, 1); |
102 | |
103 | tree view_op0 = TREE_OPERAND (op0, 0); |
104 | machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0)); |
105 | machine_mode = TYPE_MODE (TREE_TYPE (ref)); |
106 | |
107 | if ((auto_var_in_fn_p (view_op0, fun->decl) |
108 | || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0))) |
109 | && !TREE_ADDRESSABLE (view_op0) |
110 | && ((!is_extract && can_vec_set_var_idx_p (outermode)) |
111 | || (is_extract |
112 | && can_vec_extract_var_idx_p (outermode, extract_mode)))) |
113 | { |
114 | location_t loc = gimple_location (g: stmt); |
115 | tree var_src = make_ssa_name (TREE_TYPE (view_op0)); |
116 | |
117 | ass_stmt = gimple_build_assign (var_src, view_op0); |
118 | gimple_set_vuse (g: ass_stmt, vuse: gimple_vuse (g: stmt)); |
119 | gimple_set_location (g: ass_stmt, location: loc); |
120 | gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); |
121 | |
122 | if (!is_extract) |
123 | { |
124 | tree var_dst = make_ssa_name (TREE_TYPE (view_op0)); |
125 | |
126 | new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src, |
127 | val, pos); |
128 | |
129 | gimple_call_set_lhs (gs: new_stmt, lhs: var_dst); |
130 | gimple_set_location (g: new_stmt, location: loc); |
131 | gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT); |
132 | |
133 | ass_stmt = gimple_build_assign (view_op0, var_dst); |
134 | gimple_set_location (g: ass_stmt, location: loc); |
135 | gimple_move_vops (ass_stmt, stmt); |
136 | gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT); |
137 | |
138 | basic_block bb = gimple_bb (g: stmt); |
139 | if (gsi_remove (gsi, true) |
140 | && gimple_purge_dead_eh_edges (bb)) |
141 | cfg_changed = true; |
142 | *gsi = gsi_for_stmt (ass_stmt); |
143 | } |
144 | else |
145 | { |
146 | new_stmt |
147 | = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos); |
148 | gimple_call_set_lhs (gs: new_stmt, lhs); |
149 | |
150 | gsi_replace (gsi, new_stmt, true); |
151 | cfg_changed = true; |
152 | } |
153 | } |
154 | } |
155 | |
156 | return cfg_changed; |
157 | } |
158 | |
159 | /* Expand all VEC_COND_EXPR gimple assignments into calls to internal |
160 | function based on type of selected expansion. */ |
161 | |
162 | static gimple * |
163 | gimple_expand_vec_cond_expr (struct function *fun, gimple_stmt_iterator *gsi, |
164 | hash_map<tree, unsigned int> *vec_cond_ssa_name_uses) |
165 | { |
166 | tree lhs, op0a = NULL_TREE, op0b = NULL_TREE; |
167 | enum tree_code code; |
168 | enum tree_code tcode; |
169 | machine_mode cmp_op_mode; |
170 | bool unsignedp; |
171 | enum insn_code icode; |
172 | imm_use_iterator imm_iter; |
173 | |
174 | /* Only consider code == GIMPLE_ASSIGN. */ |
175 | gassign *stmt = dyn_cast<gassign *> (p: gsi_stmt (i: *gsi)); |
176 | if (!stmt) |
177 | return NULL; |
178 | |
179 | code = gimple_assign_rhs_code (gs: stmt); |
180 | if (code != VEC_COND_EXPR) |
181 | return NULL; |
182 | |
183 | tree op0 = gimple_assign_rhs1 (gs: stmt); |
184 | tree op1 = gimple_assign_rhs2 (gs: stmt); |
185 | tree op2 = gimple_assign_rhs3 (gs: stmt); |
186 | lhs = gimple_assign_lhs (gs: stmt); |
187 | machine_mode mode = TYPE_MODE (TREE_TYPE (lhs)); |
188 | |
189 | /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations. |
190 | Those can end up generated by folding and at least for integer mode masks |
191 | we cannot expect vcond expanders to exist. We lower a ? b : c |
192 | to (b & a) | (c & ~a). */ |
193 | if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs)) |
194 | && !VECTOR_MODE_P (mode)) |
195 | { |
196 | gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1))); |
197 | gimple_seq stmts = NULL; |
198 | tree type = TREE_TYPE (lhs); |
199 | location_t loc = gimple_location (g: stmt); |
200 | tree tem0 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op1, ops: op0); |
201 | tree tem1 = gimple_build (seq: &stmts, loc, code: BIT_NOT_EXPR, type, ops: op0); |
202 | tree tem2 = gimple_build (seq: &stmts, loc, code: BIT_AND_EXPR, type, ops: op2, ops: tem1); |
203 | tree tem3 = gimple_build (seq: &stmts, loc, code: BIT_IOR_EXPR, type, ops: tem0, ops: tem2); |
204 | gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT); |
205 | return gimple_build_assign (lhs, tem3); |
206 | } |
207 | |
208 | bool can_compute_op0 = true; |
209 | gcc_assert (!COMPARISON_CLASS_P (op0)); |
210 | if (TREE_CODE (op0) == SSA_NAME) |
211 | { |
212 | unsigned int used_vec_cond_exprs = 0; |
213 | unsigned int *slot = vec_cond_ssa_name_uses->get (k: op0); |
214 | if (slot) |
215 | used_vec_cond_exprs = *slot; |
216 | else |
217 | { |
218 | gimple *use_stmt; |
219 | FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, op0) |
220 | { |
221 | gassign *assign = dyn_cast<gassign *> (p: use_stmt); |
222 | if (assign != NULL |
223 | && gimple_assign_rhs_code (gs: assign) == VEC_COND_EXPR |
224 | && gimple_assign_rhs1 (gs: assign) == op0) |
225 | used_vec_cond_exprs++; |
226 | } |
227 | vec_cond_ssa_name_uses->put (k: op0, v: used_vec_cond_exprs); |
228 | } |
229 | |
230 | gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0)); |
231 | if (def_stmt) |
232 | { |
233 | tcode = gimple_assign_rhs_code (gs: def_stmt); |
234 | op0a = gimple_assign_rhs1 (gs: def_stmt); |
235 | op0b = gimple_assign_rhs2 (gs: def_stmt); |
236 | |
237 | tree op0_type = TREE_TYPE (op0); |
238 | tree op0a_type = TREE_TYPE (op0a); |
239 | if (TREE_CODE_CLASS (tcode) == tcc_comparison) |
240 | can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type, |
241 | tcode); |
242 | |
243 | /* Try to fold x CMP y ? -1 : 0 to x CMP y. */ |
244 | if (can_compute_op0 |
245 | && integer_minus_onep (op1) |
246 | && integer_zerop (op2) |
247 | && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0))) |
248 | { |
249 | tree conv_op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), op0); |
250 | gassign *new_stmt = gimple_build_assign (lhs, conv_op); |
251 | gsi_replace (gsi, new_stmt, true); |
252 | return new_stmt; |
253 | } |
254 | |
255 | /* When the compare has EH we do not want to forward it when |
256 | it has multiple uses and in general because of the complication |
257 | with EH redirection. */ |
258 | if (stmt_can_throw_internal (fun, def_stmt)) |
259 | tcode = TREE_CODE (op0); |
260 | |
261 | /* If we can compute op0 and have multiple uses, keep the SSA |
262 | name and use vcond_mask. */ |
263 | else if (can_compute_op0 |
264 | && used_vec_cond_exprs >= 2 |
265 | && (get_vcond_mask_icode (vmode: mode, TYPE_MODE (op0_type)) |
266 | != CODE_FOR_nothing)) |
267 | tcode = TREE_CODE (op0); |
268 | } |
269 | else |
270 | tcode = TREE_CODE (op0); |
271 | } |
272 | else |
273 | tcode = TREE_CODE (op0); |
274 | |
275 | if (TREE_CODE_CLASS (tcode) != tcc_comparison) |
276 | { |
277 | gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); |
278 | if (get_vcond_mask_icode (vmode: mode, TYPE_MODE (TREE_TYPE (op0))) |
279 | != CODE_FOR_nothing) |
280 | return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2); |
281 | /* Fake op0 < 0. */ |
282 | else |
283 | { |
284 | gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) |
285 | == MODE_VECTOR_INT); |
286 | op0a = op0; |
287 | op0b = build_zero_cst (TREE_TYPE (op0)); |
288 | tcode = LT_EXPR; |
289 | } |
290 | } |
291 | cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); |
292 | unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); |
293 | |
294 | gcc_assert (known_eq (GET_MODE_NUNITS (mode), |
295 | GET_MODE_NUNITS (cmp_op_mode))); |
296 | |
297 | icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: unsignedp); |
298 | /* Some targets do not have vcondeq and only vcond with NE/EQ |
299 | but not vcondu, so make sure to also try vcond here as |
300 | vcond_icode_p would canonicalize the optab query to. */ |
301 | if (icode == CODE_FOR_nothing |
302 | && (tcode == NE_EXPR || tcode == EQ_EXPR) |
303 | && ((icode = get_vcond_icode (vmode: mode, cmode: cmp_op_mode, uns: !unsignedp)) |
304 | != CODE_FOR_nothing)) |
305 | unsignedp = !unsignedp; |
306 | if (icode == CODE_FOR_nothing) |
307 | { |
308 | if (tcode == LT_EXPR |
309 | && op0a == op0) |
310 | { |
311 | /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR |
312 | into a constant when only get_vcond_eq_icode is supported. |
313 | Try changing it to NE_EXPR. */ |
314 | tcode = NE_EXPR; |
315 | } |
316 | if ((tcode == EQ_EXPR || tcode == NE_EXPR) |
317 | && direct_internal_fn_supported_p (fn: IFN_VCONDEQ, TREE_TYPE (lhs), |
318 | TREE_TYPE (op0a), |
319 | opt_type: OPTIMIZE_FOR_BOTH)) |
320 | { |
321 | tree tcode_tree = build_int_cst (integer_type_node, tcode); |
322 | return gimple_build_call_internal (IFN_VCONDEQ, 5, op0a, op0b, op1, |
323 | op2, tcode_tree); |
324 | } |
325 | |
326 | gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)) |
327 | && can_compute_op0 |
328 | && (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) |
329 | != CODE_FOR_nothing)); |
330 | return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2); |
331 | } |
332 | |
333 | tree tcode_tree = build_int_cst (integer_type_node, tcode); |
334 | return gimple_build_call_internal (unsignedp ? IFN_VCONDU : IFN_VCOND, |
335 | 5, op0a, op0b, op1, op2, tcode_tree); |
336 | } |
337 | |
338 | |
339 | |
340 | namespace { |
341 | |
342 | const pass_data pass_data_gimple_isel = |
343 | { |
344 | .type: GIMPLE_PASS, /* type */ |
345 | .name: "isel" , /* name */ |
346 | .optinfo_flags: OPTGROUP_VEC, /* optinfo_flags */ |
347 | .tv_id: TV_NONE, /* tv_id */ |
348 | PROP_cfg, /* properties_required */ |
349 | .properties_provided: 0, /* properties_provided */ |
350 | .properties_destroyed: 0, /* properties_destroyed */ |
351 | .todo_flags_start: 0, /* todo_flags_start */ |
352 | TODO_update_ssa, /* todo_flags_finish */ |
353 | }; |
354 | |
355 | class pass_gimple_isel : public gimple_opt_pass |
356 | { |
357 | public: |
358 | pass_gimple_isel (gcc::context *ctxt) |
359 | : gimple_opt_pass (pass_data_gimple_isel, ctxt) |
360 | {} |
361 | |
362 | /* opt_pass methods: */ |
363 | bool gate (function *) final override |
364 | { |
365 | return true; |
366 | } |
367 | |
368 | unsigned int execute (function *fun) final override; |
369 | }; // class pass_gimple_isel |
370 | |
371 | |
372 | /* Iterate all gimple statements and perform pre RTL expansion |
373 | GIMPLE massaging to improve instruction selection. */ |
374 | |
375 | unsigned int |
376 | pass_gimple_isel::execute (struct function *fun) |
377 | { |
378 | gimple_stmt_iterator gsi; |
379 | basic_block bb; |
380 | hash_map<tree, unsigned int> vec_cond_ssa_name_uses; |
381 | auto_bitmap dce_ssa_names; |
382 | bool cfg_changed = false; |
383 | |
384 | FOR_EACH_BB_FN (bb, fun) |
385 | { |
386 | for (gsi = gsi_start_bb (bb); !gsi_end_p (i: gsi); gsi_next (i: &gsi)) |
387 | { |
388 | /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function |
389 | calls mapping to supported optabs. */ |
390 | gimple *g = gimple_expand_vec_cond_expr (fun, gsi: &gsi, |
391 | vec_cond_ssa_name_uses: &vec_cond_ssa_name_uses); |
392 | if (g != NULL) |
393 | { |
394 | tree lhs = gimple_assign_lhs (gs: gsi_stmt (i: gsi)); |
395 | gimple_set_lhs (g, lhs); |
396 | gsi_replace (&gsi, g, false); |
397 | } |
398 | |
399 | /* Recognize .VEC_SET and .VEC_EXTRACT patterns. */ |
400 | cfg_changed |= gimple_expand_vec_set_extract_expr (fun, gsi: &gsi); |
401 | if (gsi_end_p (i: gsi)) |
402 | break; |
403 | |
404 | gassign *stmt = dyn_cast <gassign *> (p: *gsi); |
405 | if (!stmt) |
406 | continue; |
407 | |
408 | tree_code code = gimple_assign_rhs_code (gs: stmt); |
409 | tree lhs = gimple_assign_lhs (gs: stmt); |
410 | if (TREE_CODE_CLASS (code) == tcc_comparison |
411 | && !has_single_use (var: lhs)) |
412 | { |
413 | /* Duplicate COND_EXPR condition defs when they are |
414 | comparisons so RTL expansion with the help of TER |
415 | can perform better if conversion. */ |
416 | imm_use_iterator imm_iter; |
417 | use_operand_p use_p; |
418 | auto_vec<gassign *, 4> cond_exprs; |
419 | unsigned cnt = 0; |
420 | FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs) |
421 | { |
422 | if (is_gimple_debug (USE_STMT (use_p))) |
423 | continue; |
424 | cnt++; |
425 | if (gimple_bb (USE_STMT (use_p)) == bb |
426 | && is_gimple_assign (USE_STMT (use_p)) |
427 | && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use |
428 | && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR) |
429 | cond_exprs.safe_push (obj: as_a <gassign *> (USE_STMT (use_p))); |
430 | } |
431 | for (unsigned i = cond_exprs.length () == cnt ? 1 : 0; |
432 | i < cond_exprs.length (); ++i) |
433 | { |
434 | gassign *copy = as_a <gassign *> (p: gimple_copy (stmt)); |
435 | tree new_def = duplicate_ssa_name (var: lhs, stmt: copy); |
436 | gimple_assign_set_lhs (gs: copy, lhs: new_def); |
437 | auto gsi2 = gsi_for_stmt (cond_exprs[i]); |
438 | gsi_insert_before (&gsi2, copy, GSI_SAME_STMT); |
439 | gimple_assign_set_rhs1 (gs: cond_exprs[i], rhs: new_def); |
440 | update_stmt (s: cond_exprs[i]); |
441 | } |
442 | } |
443 | } |
444 | } |
445 | |
446 | for (auto it = vec_cond_ssa_name_uses.begin (); |
447 | it != vec_cond_ssa_name_uses.end (); ++it) |
448 | bitmap_set_bit (dce_ssa_names, SSA_NAME_VERSION ((*it).first)); |
449 | |
450 | simple_dce_from_worklist (dce_ssa_names); |
451 | |
452 | return cfg_changed ? TODO_cleanup_cfg : 0; |
453 | } |
454 | |
455 | } // anon namespace |
456 | |
457 | gimple_opt_pass * |
458 | make_pass_gimple_isel (gcc::context *ctxt) |
459 | { |
460 | return new pass_gimple_isel (ctxt); |
461 | } |
462 | |
463 | |