1	//===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	/// \file
9	///
10	/// This file implements the OpenMPIRBuilder class, which is used as a
11	/// convenient way to create LLVM instructions for OpenMP directives.
12	///
13	//===----------------------------------------------------------------------===//
14
15	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
16	#include "llvm/ADT/SmallSet.h"
17	#include "llvm/ADT/StringExtras.h"
18	#include "llvm/ADT/StringRef.h"
19	#include "llvm/Analysis/AssumptionCache.h"
20	#include "llvm/Analysis/CodeMetrics.h"
21	#include "llvm/Analysis/LoopInfo.h"
22	#include "llvm/Analysis/OptimizationRemarkEmitter.h"
23	#include "llvm/Analysis/ScalarEvolution.h"
24	#include "llvm/Analysis/TargetLibraryInfo.h"
25	#include "llvm/Bitcode/BitcodeReader.h"
26	#include "llvm/Frontend/Offloading/Utility.h"
27	#include "llvm/Frontend/OpenMP/OMPGridValues.h"
28	#include "llvm/IR/Attributes.h"
29	#include "llvm/IR/BasicBlock.h"
30	#include "llvm/IR/CFG.h"
31	#include "llvm/IR/CallingConv.h"
32	#include "llvm/IR/Constant.h"
33	#include "llvm/IR/Constants.h"
34	#include "llvm/IR/DebugInfoMetadata.h"
35	#include "llvm/IR/DerivedTypes.h"
36	#include "llvm/IR/Function.h"
37	#include "llvm/IR/GlobalVariable.h"
38	#include "llvm/IR/IRBuilder.h"
39	#include "llvm/IR/LLVMContext.h"
40	#include "llvm/IR/MDBuilder.h"
41	#include "llvm/IR/Metadata.h"
42	#include "llvm/IR/PassManager.h"
43	#include "llvm/IR/Value.h"
44	#include "llvm/MC/TargetRegistry.h"
45	#include "llvm/Support/CommandLine.h"
46	#include "llvm/Support/ErrorHandling.h"
47	#include "llvm/Support/FileSystem.h"
48	#include "llvm/Target/TargetMachine.h"
49	#include "llvm/Target/TargetOptions.h"
50	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
51	#include "llvm/Transforms/Utils/Cloning.h"
52	#include "llvm/Transforms/Utils/CodeExtractor.h"
53	#include "llvm/Transforms/Utils/LoopPeel.h"
54	#include "llvm/Transforms/Utils/UnrollLoop.h"
55
56	#include <cstdint>
57	#include <optional>
58
59	#define DEBUG_TYPE "openmp-ir-builder"
60
61	using namespace llvm;
62	using namespace omp;
63
64	static cl::opt<bool>
65	OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
66	cl::desc("Use optimistic attributes describing "
67	"'as-if' properties of runtime calls."),
68	cl::init(Val: false));
69
70	static cl::opt<double> UnrollThresholdFactor(
71	"openmp-ir-builder-unroll-threshold-factor", cl::Hidden,
72	cl::desc("Factor for the unroll threshold to account for code "
73	"simplifications still taking place"),
74	cl::init(Val: 1.5));
75
76	#ifndef NDEBUG
77	/// Return whether IP1 and IP2 are ambiguous, i.e. that inserting instructions
78	/// at position IP1 may change the meaning of IP2 or vice-versa. This is because
79	/// an InsertPoint stores the instruction before something is inserted. For
80	/// instance, if both point to the same instruction, two IRBuilders alternating
81	/// creating instruction will cause the instructions to be interleaved.
82	static bool isConflictIP(IRBuilder<>::InsertPoint IP1,
83	IRBuilder<>::InsertPoint IP2) {
84	if (!IP1.isSet() || !IP2.isSet())
85	return false;
86	return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
87	}
88
89	static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
90	// Valid ordered/unordered and base algorithm combinations.
91	switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
92	case OMPScheduleType::UnorderedStaticChunked:
93	case OMPScheduleType::UnorderedStatic:
94	case OMPScheduleType::UnorderedDynamicChunked:
95	case OMPScheduleType::UnorderedGuidedChunked:
96	case OMPScheduleType::UnorderedRuntime:
97	case OMPScheduleType::UnorderedAuto:
98	case OMPScheduleType::UnorderedTrapezoidal:
99	case OMPScheduleType::UnorderedGreedy:
100	case OMPScheduleType::UnorderedBalanced:
101	case OMPScheduleType::UnorderedGuidedIterativeChunked:
102	case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
103	case OMPScheduleType::UnorderedSteal:
104	case OMPScheduleType::UnorderedStaticBalancedChunked:
105	case OMPScheduleType::UnorderedGuidedSimd:
106	case OMPScheduleType::UnorderedRuntimeSimd:
107	case OMPScheduleType::OrderedStaticChunked:
108	case OMPScheduleType::OrderedStatic:
109	case OMPScheduleType::OrderedDynamicChunked:
110	case OMPScheduleType::OrderedGuidedChunked:
111	case OMPScheduleType::OrderedRuntime:
112	case OMPScheduleType::OrderedAuto:
113	case OMPScheduleType::OrderdTrapezoidal:
114	case OMPScheduleType::NomergeUnorderedStaticChunked:
115	case OMPScheduleType::NomergeUnorderedStatic:
116	case OMPScheduleType::NomergeUnorderedDynamicChunked:
117	case OMPScheduleType::NomergeUnorderedGuidedChunked:
118	case OMPScheduleType::NomergeUnorderedRuntime:
119	case OMPScheduleType::NomergeUnorderedAuto:
120	case OMPScheduleType::NomergeUnorderedTrapezoidal:
121	case OMPScheduleType::NomergeUnorderedGreedy:
122	case OMPScheduleType::NomergeUnorderedBalanced:
123	case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
124	case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
125	case OMPScheduleType::NomergeUnorderedSteal:
126	case OMPScheduleType::NomergeOrderedStaticChunked:
127	case OMPScheduleType::NomergeOrderedStatic:
128	case OMPScheduleType::NomergeOrderedDynamicChunked:
129	case OMPScheduleType::NomergeOrderedGuidedChunked:
130	case OMPScheduleType::NomergeOrderedRuntime:
131	case OMPScheduleType::NomergeOrderedAuto:
132	case OMPScheduleType::NomergeOrderedTrapezoidal:
133	break;
134	default:
135	return false;
136	}
137
138	// Must not set both monotonicity modifiers at the same time.
139	OMPScheduleType MonotonicityFlags =
140	SchedType & OMPScheduleType::MonotonicityMask;
141	if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
142	return false;
143
144	return true;
145	}
146	#endif
147
148	static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
149	if (T.isAMDGPU()) {
150	StringRef Features =
151	Kernel->getFnAttribute(Kind: "target-features").getValueAsString();
152	if (Features.count(Str: "+wavefrontsize64"))
153	return omp::getAMDGPUGridValues<64>();
154	return omp::getAMDGPUGridValues<32>();
155	}
156	if (T.isNVPTX())
157	return omp::NVPTXGridValues;
158	llvm_unreachable("No grid value available for this architecture!");
159	}
160
161	/// Determine which scheduling algorithm to use, determined from schedule clause
162	/// arguments.
163	static OMPScheduleType
164	getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks,
165	bool HasSimdModifier) {
166	// Currently, the default schedule it static.
167	switch (ClauseKind) {
168	case OMP_SCHEDULE_Default:
169	case OMP_SCHEDULE_Static:
170	return HasChunks ? OMPScheduleType::BaseStaticChunked
171	: OMPScheduleType::BaseStatic;
172	case OMP_SCHEDULE_Dynamic:
173	return OMPScheduleType::BaseDynamicChunked;
174	case OMP_SCHEDULE_Guided:
175	return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
176	: OMPScheduleType::BaseGuidedChunked;
177	case OMP_SCHEDULE_Auto:
178	return llvm::omp::OMPScheduleType::BaseAuto;
179	case OMP_SCHEDULE_Runtime:
180	return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
181	: OMPScheduleType::BaseRuntime;
182	}
183	llvm_unreachable("unhandled schedule clause argument");
184	}
185
186	/// Adds ordering modifier flags to schedule type.
187	static OMPScheduleType
188	getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType,
189	bool HasOrderedClause) {
190	assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
191	OMPScheduleType::None &&
192	"Must not have ordering nor monotonicity flags already set");
193
194	OMPScheduleType OrderingModifier = HasOrderedClause
195	? OMPScheduleType::ModifierOrdered
196	: OMPScheduleType::ModifierUnordered;
197	OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
198
199	// Unsupported combinations
200	if (OrderingScheduleType ==
201	(OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
202	return OMPScheduleType::OrderedGuidedChunked;
203	else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
204	OMPScheduleType::ModifierOrdered))
205	return OMPScheduleType::OrderedRuntime;
206
207	return OrderingScheduleType;
208	}
209
210	/// Adds monotonicity modifier flags to schedule type.
211	static OMPScheduleType
212	getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType,
213	bool HasSimdModifier, bool HasMonotonic,
214	bool HasNonmonotonic, bool HasOrderedClause) {
215	assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
216	OMPScheduleType::None &&
217	"Must not have monotonicity flags already set");
218	assert((!HasMonotonic || !HasNonmonotonic) &&
219	"Monotonic and Nonmonotonic are contradicting each other");
220
221	if (HasMonotonic) {
222	return ScheduleType | OMPScheduleType::ModifierMonotonic;
223	} else if (HasNonmonotonic) {
224	return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
225	} else {
226	// OpenMP 5.1, 2.11.4 Worksharing-Loop Construct, Description.
227	// If the static schedule kind is specified or if the ordered clause is
228	// specified, and if the nonmonotonic modifier is not specified, the
229	// effect is as if the monotonic modifier is specified. Otherwise, unless
230	// the monotonic modifier is specified, the effect is as if the
231	// nonmonotonic modifier is specified.
232	OMPScheduleType BaseScheduleType =
233	ScheduleType & ~OMPScheduleType::ModifierMask;
234	if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
235	(BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
236	HasOrderedClause) {
237	// The monotonic is used by default in openmp runtime library, so no need
238	// to set it.
239	return ScheduleType;
240	} else {
241	return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
242	}
243	}
244	}
245
246	/// Determine the schedule type using schedule and ordering clause arguments.
247	static OMPScheduleType
248	computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks,
249	bool HasSimdModifier, bool HasMonotonicModifier,
250	bool HasNonmonotonicModifier, bool HasOrderedClause) {
251	OMPScheduleType BaseSchedule =
252	getOpenMPBaseScheduleType(ClauseKind, HasChunks, HasSimdModifier);
253	OMPScheduleType OrderedSchedule =
254	getOpenMPOrderingScheduleType(BaseScheduleType: BaseSchedule, HasOrderedClause);
255	OMPScheduleType Result = getOpenMPMonotonicityScheduleType(
256	ScheduleType: OrderedSchedule, HasSimdModifier, HasMonotonic: HasMonotonicModifier,
257	HasNonmonotonic: HasNonmonotonicModifier, HasOrderedClause);
258
259	assert(isValidWorkshareLoopScheduleType(Result));
260	return Result;
261	}
262
263	/// Make \p Source branch to \p Target.
264	///
265	/// Handles two situations:
266	/// * \p Source already has an unconditional branch.
267	/// * \p Source is a degenerate block (no terminator because the BB is
268	/// the current head of the IR construction).
269	static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL) {
270	if (Instruction *Term = Source->getTerminator()) {
271	auto *Br = cast<BranchInst>(Val: Term);
272	assert(!Br->isConditional() &&
273	"BB's terminator must be an unconditional branch (or degenerate)");
274	BasicBlock *Succ = Br->getSuccessor(i: 0);
275	Succ->removePredecessor(Pred: Source, /*KeepOneInputPHIs=*/true);
276	Br->setSuccessor(idx: 0, NewSucc: Target);
277	return;
278	}
279
280	auto *NewBr = BranchInst::Create(IfTrue: Target, InsertAtEnd: Source);
281	NewBr->setDebugLoc(DL);
282	}
283
284	void llvm::spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New,
285	bool CreateBranch) {
286	assert(New->getFirstInsertionPt() == New->begin() &&
287	"Target BB must not have PHI nodes");
288
289	// Move instructions to new block.
290	BasicBlock *Old = IP.getBlock();
291	New->splice(ToIt: New->begin(), FromBB: Old, FromBeginIt: IP.getPoint(), FromEndIt: Old->end());
292
293	if (CreateBranch)
294	BranchInst::Create(IfTrue: New, InsertAtEnd: Old);
295	}
296
297	void llvm::spliceBB(IRBuilder<> &Builder, BasicBlock *New, bool CreateBranch) {
298	DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
299	BasicBlock *Old = Builder.GetInsertBlock();
300
301	spliceBB(IP: Builder.saveIP(), New, CreateBranch);
302	if (CreateBranch)
303	Builder.SetInsertPoint(Old->getTerminator());
304	else
305	Builder.SetInsertPoint(Old);
306
307	// SetInsertPoint also updates the Builder's debug location, but we want to
308	// keep the one the Builder was configured to use.
309	Builder.SetCurrentDebugLocation(DebugLoc);
310	}
311
312	BasicBlock *llvm::splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch,
313	llvm::Twine Name) {
314	BasicBlock *Old = IP.getBlock();
315	BasicBlock *New = BasicBlock::Create(
316	Context&: Old->getContext(), Name: Name.isTriviallyEmpty() ? Old->getName() : Name,
317	Parent: Old->getParent(), InsertBefore: Old->getNextNode());
318	spliceBB(IP, New, CreateBranch);
319	New->replaceSuccessorsPhiUsesWith(Old, New);
320	return New;
321	}
322
323	BasicBlock *llvm::splitBB(IRBuilderBase &Builder, bool CreateBranch,
324	llvm::Twine Name) {
325	DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
326	BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, Name);
327	if (CreateBranch)
328	Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
329	else
330	Builder.SetInsertPoint(Builder.GetInsertBlock());
331	// SetInsertPoint also updates the Builder's debug location, but we want to
332	// keep the one the Builder was configured to use.
333	Builder.SetCurrentDebugLocation(DebugLoc);
334	return New;
335	}
336
337	BasicBlock *llvm::splitBB(IRBuilder<> &Builder, bool CreateBranch,
338	llvm::Twine Name) {
339	DebugLoc DebugLoc = Builder.getCurrentDebugLocation();
340	BasicBlock *New = splitBB(IP: Builder.saveIP(), CreateBranch, Name);
341	if (CreateBranch)
342	Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
343	else
344	Builder.SetInsertPoint(Builder.GetInsertBlock());
345	// SetInsertPoint also updates the Builder's debug location, but we want to
346	// keep the one the Builder was configured to use.
347	Builder.SetCurrentDebugLocation(DebugLoc);
348	return New;
349	}
350
351	BasicBlock *llvm::splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch,
352	llvm::Twine Suffix) {
353	BasicBlock *Old = Builder.GetInsertBlock();
354	return splitBB(Builder, CreateBranch, Name: Old->getName() + Suffix);
355	}
356
357	// This function creates a fake integer value and a fake use for the integer
358	// value. It returns the fake value created. This is useful in modeling the
359	// extra arguments to the outlined functions.
360	Value *createFakeIntVal(IRBuilder<> &Builder,
361	OpenMPIRBuilder::InsertPointTy OuterAllocaIP,
362	std::stack<Instruction *> &ToBeDeleted,
363	OpenMPIRBuilder::InsertPointTy InnerAllocaIP,
364	const Twine &Name = "", bool AsPtr = true) {
365	Builder.restoreIP(IP: OuterAllocaIP);
366	Instruction *FakeVal;
367	AllocaInst *FakeValAddr =
368	Builder.CreateAlloca(Ty: Builder.getInt32Ty(), ArraySize: nullptr, Name: Name + ".addr");
369	ToBeDeleted.push(x: FakeValAddr);
370
371	if (AsPtr) {
372	FakeVal = FakeValAddr;
373	} else {
374	FakeVal =
375	Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: FakeValAddr, Name: Name + ".val");
376	ToBeDeleted.push(x: FakeVal);
377	}
378
379	// Generate a fake use of this value
380	Builder.restoreIP(IP: InnerAllocaIP);
381	Instruction *UseFakeVal;
382	if (AsPtr) {
383	UseFakeVal =
384	Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: FakeVal, Name: Name + ".use");
385	} else {
386	UseFakeVal =
387	cast<BinaryOperator>(Val: Builder.CreateAdd(LHS: FakeVal, RHS: Builder.getInt32(C: 10)));
388	}
389	ToBeDeleted.push(x: UseFakeVal);
390	return FakeVal;
391	}
392
393	//===----------------------------------------------------------------------===//
394	// OpenMPIRBuilderConfig
395	//===----------------------------------------------------------------------===//
396
397	namespace {
398	LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
399	/// Values for bit flags for marking which requires clauses have been used.
400	enum OpenMPOffloadingRequiresDirFlags {
401	/// flag undefined.
402	OMP_REQ_UNDEFINED = 0x000,
403	/// no requires directive present.
404	OMP_REQ_NONE = 0x001,
405	/// reverse_offload clause.
406	OMP_REQ_REVERSE_OFFLOAD = 0x002,
407	/// unified_address clause.
408	OMP_REQ_UNIFIED_ADDRESS = 0x004,
409	/// unified_shared_memory clause.
410	OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
411	/// dynamic_allocators clause.
412	OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
413	LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS)
414	};
415
416	} // anonymous namespace
417
418	OpenMPIRBuilderConfig::OpenMPIRBuilderConfig()
419	: RequiresFlags(OMP_REQ_UNDEFINED) {}
420
421	OpenMPIRBuilderConfig::OpenMPIRBuilderConfig(
422	bool IsTargetDevice, bool IsGPU, bool OpenMPOffloadMandatory,
423	bool HasRequiresReverseOffload, bool HasRequiresUnifiedAddress,
424	bool HasRequiresUnifiedSharedMemory, bool HasRequiresDynamicAllocators)
425	: IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
426	OpenMPOffloadMandatory(OpenMPOffloadMandatory),
427	RequiresFlags(OMP_REQ_UNDEFINED) {
428	if (HasRequiresReverseOffload)
429	RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
430	if (HasRequiresUnifiedAddress)
431	RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
432	if (HasRequiresUnifiedSharedMemory)
433	RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
434	if (HasRequiresDynamicAllocators)
435	RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
436	}
437
438	bool OpenMPIRBuilderConfig::hasRequiresReverseOffload() const {
439	return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
440	}
441
442	bool OpenMPIRBuilderConfig::hasRequiresUnifiedAddress() const {
443	return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
444	}
445
446	bool OpenMPIRBuilderConfig::hasRequiresUnifiedSharedMemory() const {
447	return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
448	}
449
450	bool OpenMPIRBuilderConfig::hasRequiresDynamicAllocators() const {
451	return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
452	}
453
454	int64_t OpenMPIRBuilderConfig::getRequiresFlags() const {
455	return hasRequiresFlags() ? RequiresFlags
456	: static_cast<int64_t>(OMP_REQ_NONE);
457	}
458
459	void OpenMPIRBuilderConfig::setHasRequiresReverseOffload(bool Value) {
460	if (Value)
461	RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
462	else
463	RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
464	}
465
466	void OpenMPIRBuilderConfig::setHasRequiresUnifiedAddress(bool Value) {
467	if (Value)
468	RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
469	else
470	RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
471	}
472
473	void OpenMPIRBuilderConfig::setHasRequiresUnifiedSharedMemory(bool Value) {
474	if (Value)
475	RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
476	else
477	RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
478	}
479
480	void OpenMPIRBuilderConfig::setHasRequiresDynamicAllocators(bool Value) {
481	if (Value)
482	RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
483	else
484	RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
485	}
486
487	//===----------------------------------------------------------------------===//
488	// OpenMPIRBuilder
489	//===----------------------------------------------------------------------===//
490
491	void OpenMPIRBuilder::getKernelArgsVector(TargetKernelArgs &KernelArgs,
492	IRBuilderBase &Builder,
493	SmallVector<Value *> &ArgsVector) {
494	Value *Version = Builder.getInt32(OMP_KERNEL_ARG_VERSION);
495	Value *PointerNum = Builder.getInt32(C: KernelArgs.NumTargetItems);
496	auto Int32Ty = Type::getInt32Ty(C&: Builder.getContext());
497	Value *ZeroArray = Constant::getNullValue(Ty: ArrayType::get(ElementType: Int32Ty, NumElements: 3));
498	Value *Flags = Builder.getInt64(C: KernelArgs.HasNoWait);
499
500	Value *NumTeams3D =
501	Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumTeams, Idxs: {0});
502	Value *NumThreads3D =
503	Builder.CreateInsertValue(Agg: ZeroArray, Val: KernelArgs.NumThreads, Idxs: {0});
504
505	ArgsVector = {Version,
506	PointerNum,
507	KernelArgs.RTArgs.BasePointersArray,
508	KernelArgs.RTArgs.PointersArray,
509	KernelArgs.RTArgs.SizesArray,
510	KernelArgs.RTArgs.MapTypesArray,
511	KernelArgs.RTArgs.MapNamesArray,
512	KernelArgs.RTArgs.MappersArray,
513	KernelArgs.NumIterations,
514	Flags,
515	NumTeams3D,
516	NumThreads3D,
517	KernelArgs.DynCGGroupMem};
518	}
519
520	void OpenMPIRBuilder::addAttributes(omp::RuntimeFunction FnID, Function &Fn) {
521	LLVMContext &Ctx = Fn.getContext();
522
523	// Get the function's current attributes.
524	auto Attrs = Fn.getAttributes();
525	auto FnAttrs = Attrs.getFnAttrs();
526	auto RetAttrs = Attrs.getRetAttrs();
527	SmallVector<AttributeSet, 4> ArgAttrs;
528	for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
529	ArgAttrs.emplace_back(Args: Attrs.getParamAttrs(ArgNo));
530
531	// Add AS to FnAS while taking special care with integer extensions.
532	auto addAttrSet = [&](AttributeSet &FnAS, const AttributeSet &AS,
533	bool Param = true) -> void {
534	bool HasSignExt = AS.hasAttribute(Attribute::SExt);
535	bool HasZeroExt = AS.hasAttribute(Attribute::ZExt);
536	if (HasSignExt || HasZeroExt) {
537	assert(AS.getNumAttributes() == 1 &&
538	"Currently not handling extension attr combined with others.");
539	if (Param) {
540	if (auto AK = TargetLibraryInfo::getExtAttrForI32Param(T, HasSignExt))
541	FnAS = FnAS.addAttribute(Ctx, AK);
542	} else if (auto AK =
543	TargetLibraryInfo::getExtAttrForI32Return(T, HasSignExt))
544	FnAS = FnAS.addAttribute(Ctx, AK);
545	} else {
546	FnAS = FnAS.addAttributes(C&: Ctx, AS);
547	}
548	};
549
550	#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
551	#include "llvm/Frontend/OpenMP/OMPKinds.def"
552
553	// Add attributes to the function declaration.
554	switch (FnID) {
555	#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
556	case Enum: \
557	FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
558	addAttrSet(RetAttrs, RetAttrSet, /*Param*/ false); \
559	for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
560	addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
561	Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
562	break;
563	#include "llvm/Frontend/OpenMP/OMPKinds.def"
564	default:
565	// Attributes are optional.
566	break;
567	}
568	}
569
570	FunctionCallee
571	OpenMPIRBuilder::getOrCreateRuntimeFunction(Module &M, RuntimeFunction FnID) {
572	FunctionType *FnTy = nullptr;
573	Function *Fn = nullptr;
574
575	// Try to find the declation in the module first.
576	switch (FnID) {
577	#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
578	case Enum: \
579	FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
580	IsVarArg); \
581	Fn = M.getFunction(Str); \
582	break;
583	#include "llvm/Frontend/OpenMP/OMPKinds.def"
584	}
585
586	if (!Fn) {
587	// Create a new declaration if we need one.
588	switch (FnID) {
589	#define OMP_RTL(Enum, Str, ...) \
590	case Enum: \
591	Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
592	break;
593	#include "llvm/Frontend/OpenMP/OMPKinds.def"
594	}
595
596	// Add information if the runtime function takes a callback function
597	if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
598	if (!Fn->hasMetadata(KindID: LLVMContext::MD_callback)) {
599	LLVMContext &Ctx = Fn->getContext();
600	MDBuilder MDB(Ctx);
601	// Annotate the callback behavior of the runtime function:
602	// - The callback callee is argument number 2 (microtask).
603	// - The first two arguments of the callback callee are unknown (-1).
604	// - All variadic arguments to the runtime function are passed to the
605	// callback callee.
606	Fn->addMetadata(
607	KindID: LLVMContext::MD_callback,
608	MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
609	CalleeArgNo: 2, Arguments: {-1, -1}, /* VarArgsArePassed */ true)}));
610	}
611	}
612
613	LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
614	<< " with type " << *Fn->getFunctionType() << "\n");
615	addAttributes(FnID, Fn&: *Fn);
616
617	} else {
618	LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
619	<< " with type " << *Fn->getFunctionType() << "\n");
620	}
621
622	assert(Fn && "Failed to create OpenMP runtime function");
623
624	return {FnTy, Fn};
625	}
626
627	Function *OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr(RuntimeFunction FnID) {
628	FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
629	auto *Fn = dyn_cast<llvm::Function>(Val: RTLFn.getCallee());
630	assert(Fn && "Failed to create OpenMP runtime function pointer");
631	return Fn;
632	}
633
634	void OpenMPIRBuilder::initialize() { initializeTypes(M); }
635
636	void OpenMPIRBuilder::finalize(Function *Fn) {
637	SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
638	SmallVector<BasicBlock *, 32> Blocks;
639	SmallVector<OutlineInfo, 16> DeferredOutlines;
640	for (OutlineInfo &OI : OutlineInfos) {
641	// Skip functions that have not finalized yet; may happen with nested
642	// function generation.
643	if (Fn && OI.getFunction() != Fn) {
644	DeferredOutlines.push_back(Elt: OI);
645	continue;
646	}
647
648	ParallelRegionBlockSet.clear();
649	Blocks.clear();
650	OI.collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
651
652	Function *OuterFn = OI.getFunction();
653	CodeExtractorAnalysisCache CEAC(*OuterFn);
654	// If we generate code for the target device, we need to allocate
655	// struct for aggregate params in the device default alloca address space.
656	// OpenMP runtime requires that the params of the extracted functions are
657	// passed as zero address space pointers. This flag ensures that
658	// CodeExtractor generates correct code for extracted functions
659	// which are used by OpenMP runtime.
660	bool ArgsInZeroAddressSpace = Config.isTargetDevice();
661	CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
662	/* AggregateArgs */ true,
663	/* BlockFrequencyInfo */ nullptr,
664	/* BranchProbabilityInfo */ nullptr,
665	/* AssumptionCache */ nullptr,
666	/* AllowVarArgs */ true,
667	/* AllowAlloca */ true,
668	/* AllocaBlock*/ OI.OuterAllocaBB,
669	/* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
670
671	LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
672	LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()
673	<< " Exit: " << OI.ExitBB->getName() << "\n");
674	assert(Extractor.isEligible() &&
675	"Expected OpenMP outlining to be possible!");
676
677	for (auto *V : OI.ExcludeArgsFromAggregate)
678	Extractor.excludeArgFromAggregate(Arg: V);
679
680	Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
681
682	// Forward target-cpu, target-features attributes to the outlined function.
683	auto TargetCpuAttr = OuterFn->getFnAttribute(Kind: "target-cpu");
684	if (TargetCpuAttr.isStringAttribute())
685	OutlinedFn->addFnAttr(Attr: TargetCpuAttr);
686
687	auto TargetFeaturesAttr = OuterFn->getFnAttribute(Kind: "target-features");
688	if (TargetFeaturesAttr.isStringAttribute())
689	OutlinedFn->addFnAttr(Attr: TargetFeaturesAttr);
690
691	LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
692	LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
693	assert(OutlinedFn->getReturnType()->isVoidTy() &&
694	"OpenMP outlined functions should not return a value!");
695
696	// For compability with the clang CG we move the outlined function after the
697	// one with the parallel region.
698	OutlinedFn->removeFromParent();
699	M.getFunctionList().insertAfter(where: OuterFn->getIterator(), New: OutlinedFn);
700
701	// Remove the artificial entry introduced by the extractor right away, we
702	// made our own entry block after all.
703	{
704	BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
705	assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB);
706	assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
707	// Move instructions from the to-be-deleted ArtificialEntry to the entry
708	// basic block of the parallel region. CodeExtractor generates
709	// instructions to unwrap the aggregate argument and may sink
710	// allocas/bitcasts for values that are solely used in the outlined region
711	// and do not escape.
712	assert(!ArtificialEntry.empty() &&
713	"Expected instructions to add in the outlined region entry");
714	for (BasicBlock::reverse_iterator It = ArtificialEntry.rbegin(),
715	End = ArtificialEntry.rend();
716	It != End;) {
717	Instruction &I = *It;
718	It++;
719
720	if (I.isTerminator())
721	continue;
722
723	I.moveBeforePreserving(BB&: *OI.EntryBB, I: OI.EntryBB->getFirstInsertionPt());
724	}
725
726	OI.EntryBB->moveBefore(MovePos: &ArtificialEntry);
727	ArtificialEntry.eraseFromParent();
728	}
729	assert(&OutlinedFn->getEntryBlock() == OI.EntryBB);
730	assert(OutlinedFn && OutlinedFn->getNumUses() == 1);
731
732	// Run a user callback, e.g. to add attributes.
733	if (OI.PostOutlineCB)
734	OI.PostOutlineCB(*OutlinedFn);
735	}
736
737	// Remove work items that have been completed.
738	OutlineInfos = std::move(DeferredOutlines);
739
740	EmitMetadataErrorReportFunctionTy &&ErrorReportFn =
741	[](EmitMetadataErrorKind Kind,
742	const TargetRegionEntryInfo &EntryInfo) -> void {
743	errs() << "Error of kind: " << Kind
744	<< " when emitting offload entries and metadata during "
745	"OMPIRBuilder finalization \n";
746	};
747
748	if (!OffloadInfoManager.empty())
749	createOffloadEntriesAndInfoMetadata(ErrorReportFunction&: ErrorReportFn);
750	}
751
752	OpenMPIRBuilder::~OpenMPIRBuilder() {
753	assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
754	}
755
756	GlobalValue *OpenMPIRBuilder::createGlobalFlag(unsigned Value, StringRef Name) {
757	IntegerType *I32Ty = Type::getInt32Ty(C&: M.getContext());
758	auto *GV =
759	new GlobalVariable(M, I32Ty,
760	/* isConstant = */ true, GlobalValue::WeakODRLinkage,
761	ConstantInt::get(Ty: I32Ty, V: Value), Name);
762	GV->setVisibility(GlobalValue::HiddenVisibility);
763
764	return GV;
765	}
766
767	Constant *OpenMPIRBuilder::getOrCreateIdent(Constant *SrcLocStr,
768	uint32_t SrcLocStrSize,
769	IdentFlag LocFlags,
770	unsigned Reserve2Flags) {
771	// Enable "C-mode".
772	LocFlags |= OMP_IDENT_FLAG_KMPC;
773
774	Constant *&Ident =
775	IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
776	if (!Ident) {
777	Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
778	Constant *IdentData[] = {I32Null,
779	ConstantInt::get(Ty: Int32, V: uint32_t(LocFlags)),
780	ConstantInt::get(Ty: Int32, V: Reserve2Flags),
781	ConstantInt::get(Ty: Int32, V: SrcLocStrSize), SrcLocStr};
782	Constant *Initializer =
783	ConstantStruct::get(T: OpenMPIRBuilder::Ident, V: IdentData);
784
785	// Look for existing encoding of the location + flags, not needed but
786	// minimizes the difference to the existing solution while we transition.
787	for (GlobalVariable &GV : M.globals())
788	if (GV.getValueType() == OpenMPIRBuilder::Ident && GV.hasInitializer())
789	if (GV.getInitializer() == Initializer)
790	Ident = &GV;
791
792	if (!Ident) {
793	auto *GV = new GlobalVariable(
794	M, OpenMPIRBuilder::Ident,
795	/* isConstant = */ true, GlobalValue::PrivateLinkage, Initializer, "",
796	nullptr, GlobalValue::NotThreadLocal,
797	M.getDataLayout().getDefaultGlobalsAddressSpace());
798	GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
799	GV->setAlignment(Align(8));
800	Ident = GV;
801	}
802	}
803
804	return ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: Ident, Ty: IdentPtr);
805	}
806
807	Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef LocStr,
808	uint32_t &SrcLocStrSize) {
809	SrcLocStrSize = LocStr.size();
810	Constant *&SrcLocStr = SrcLocStrMap[LocStr];
811	if (!SrcLocStr) {
812	Constant *Initializer =
813	ConstantDataArray::getString(Context&: M.getContext(), Initializer: LocStr);
814
815	// Look for existing encoding of the location, not needed but minimizes the
816	// difference to the existing solution while we transition.
817	for (GlobalVariable &GV : M.globals())
818	if (GV.isConstant() && GV.hasInitializer() &&
819	GV.getInitializer() == Initializer)
820	return SrcLocStr = ConstantExpr::getPointerCast(C: &GV, Ty: Int8Ptr);
821
822	SrcLocStr = Builder.CreateGlobalStringPtr(Str: LocStr, /* Name */ "",
823	/* AddressSpace */ 0, M: &M);
824	}
825	return SrcLocStr;
826	}
827
828	Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(StringRef FunctionName,
829	StringRef FileName,
830	unsigned Line, unsigned Column,
831	uint32_t &SrcLocStrSize) {
832	SmallString<128> Buffer;
833	Buffer.push_back(Elt: ';');
834	Buffer.append(RHS: FileName);
835	Buffer.push_back(Elt: ';');
836	Buffer.append(RHS: FunctionName);
837	Buffer.push_back(Elt: ';');
838	Buffer.append(RHS: std::to_string(val: Line));
839	Buffer.push_back(Elt: ';');
840	Buffer.append(RHS: std::to_string(val: Column));
841	Buffer.push_back(Elt: ';');
842	Buffer.push_back(Elt: ';');
843	return getOrCreateSrcLocStr(LocStr: Buffer.str(), SrcLocStrSize);
844	}
845
846	Constant *
847	OpenMPIRBuilder::getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize) {
848	StringRef UnknownLoc = ";unknown;unknown;0;0;;";
849	return getOrCreateSrcLocStr(LocStr: UnknownLoc, SrcLocStrSize);
850	}
851
852	Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(DebugLoc DL,
853	uint32_t &SrcLocStrSize,
854	Function *F) {
855	DILocation *DIL = DL.get();
856	if (!DIL)
857	return getOrCreateDefaultSrcLocStr(SrcLocStrSize);
858	StringRef FileName = M.getName();
859	if (DIFile *DIF = DIL->getFile())
860	if (std::optional<StringRef> Source = DIF->getSource())
861	FileName = *Source;
862	StringRef Function = DIL->getScope()->getSubprogram()->getName();
863	if (Function.empty() && F)
864	Function = F->getName();
865	return getOrCreateSrcLocStr(FunctionName: Function, FileName, Line: DIL->getLine(),
866	Column: DIL->getColumn(), SrcLocStrSize);
867	}
868
869	Constant *OpenMPIRBuilder::getOrCreateSrcLocStr(const LocationDescription &Loc,
870	uint32_t &SrcLocStrSize) {
871	return getOrCreateSrcLocStr(DL: Loc.DL, SrcLocStrSize,
872	F: Loc.IP.getBlock()->getParent());
873	}
874
875	Value *OpenMPIRBuilder::getOrCreateThreadID(Value *Ident) {
876	return Builder.CreateCall(
877	Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num), Args: Ident,
878	Name: "omp_global_thread_num");
879	}
880
881	OpenMPIRBuilder::InsertPointTy
882	OpenMPIRBuilder::createBarrier(const LocationDescription &Loc, Directive DK,
883	bool ForceSimpleCall, bool CheckCancelFlag) {
884	if (!updateToLocation(Loc))
885	return Loc.IP;
886	return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
887	}
888
889	OpenMPIRBuilder::InsertPointTy
890	OpenMPIRBuilder::emitBarrierImpl(const LocationDescription &Loc, Directive Kind,
891	bool ForceSimpleCall, bool CheckCancelFlag) {
892	// Build call __kmpc_cancel_barrier(loc, thread_id) or
893	// __kmpc_barrier(loc, thread_id);
894
895	IdentFlag BarrierLocFlags;
896	switch (Kind) {
897	case OMPD_for:
898	BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
899	break;
900	case OMPD_sections:
901	BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
902	break;
903	case OMPD_single:
904	BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
905	break;
906	case OMPD_barrier:
907	BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
908	break;
909	default:
910	BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
911	break;
912	}
913
914	uint32_t SrcLocStrSize;
915	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
916	Value *Args[] = {
917	getOrCreateIdent(SrcLocStr, SrcLocStrSize, LocFlags: BarrierLocFlags),
918	getOrCreateThreadID(Ident: getOrCreateIdent(SrcLocStr, SrcLocStrSize))};
919
920	// If we are in a cancellable parallel region, barriers are cancellation
921	// points.
922	// TODO: Check why we would force simple calls or to ignore the cancel flag.
923	bool UseCancelBarrier =
924	!ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
925
926	Value *Result =
927	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(
928	FnID: UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
929	: OMPRTL___kmpc_barrier),
930	Args);
931
932	if (UseCancelBarrier && CheckCancelFlag)
933	emitCancelationCheckImpl(Result, OMPD_parallel);
934
935	return Builder.saveIP();
936	}
937
938	OpenMPIRBuilder::InsertPointTy
939	OpenMPIRBuilder::createCancel(const LocationDescription &Loc,
940	Value *IfCondition,
941	omp::Directive CanceledDirective) {
942	if (!updateToLocation(Loc))
943	return Loc.IP;
944
945	// LLVM utilities like blocks with terminators.
946	auto *UI = Builder.CreateUnreachable();
947
948	Instruction *ThenTI = UI, *ElseTI = nullptr;
949	if (IfCondition)
950	SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: UI, ThenTerm: &ThenTI, ElseTerm: &ElseTI);
951	Builder.SetInsertPoint(ThenTI);
952
953	Value *CancelKind = nullptr;
954	switch (CanceledDirective) {
955	#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
956	case DirectiveEnum: \
957	CancelKind = Builder.getInt32(Value); \
958	break;
959	#include "llvm/Frontend/OpenMP/OMPKinds.def"
960	default:
961	llvm_unreachable("Unknown cancel kind!");
962	}
963
964	uint32_t SrcLocStrSize;
965	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
966	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
967	Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
968	Value *Result = Builder.CreateCall(
969	Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_cancel), Args);
970	auto ExitCB = [this, CanceledDirective, Loc](InsertPointTy IP) {
971	if (CanceledDirective == OMPD_parallel) {
972	IRBuilder<>::InsertPointGuard IPG(Builder);
973	Builder.restoreIP(IP);
974	createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
975	omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
976	/* CheckCancelFlag */ false);
977	}
978	};
979
980	// The actual cancel logic is shared with others, e.g., cancel_barriers.
981	emitCancelationCheckImpl(Result, CanceledDirective, ExitCB);
982
983	// Update the insertion point and remove the terminator we introduced.
984	Builder.SetInsertPoint(UI->getParent());
985	UI->eraseFromParent();
986
987	return Builder.saveIP();
988	}
989
990	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
991	const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
992	Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
993	Value *HostPtr, ArrayRef<Value *> KernelArgs) {
994	if (!updateToLocation(Loc))
995	return Loc.IP;
996
997	Builder.restoreIP(IP: AllocaIP);
998	auto *KernelArgsPtr =
999	Builder.CreateAlloca(Ty: OpenMPIRBuilder::KernelArgs, ArraySize: nullptr, Name: "kernel_args");
1000	Builder.restoreIP(IP: Loc.IP);
1001
1002	for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) {
1003	llvm::Value *Arg =
1004	Builder.CreateStructGEP(Ty: OpenMPIRBuilder::KernelArgs, Ptr: KernelArgsPtr, Idx: I);
1005	Builder.CreateAlignedStore(
1006	Val: KernelArgs[I], Ptr: Arg,
1007	Align: M.getDataLayout().getPrefTypeAlign(Ty: KernelArgs[I]->getType()));
1008	}
1009
1010	SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
1011	NumThreads, HostPtr, KernelArgsPtr};
1012
1013	Return = Builder.CreateCall(
1014	Callee: getOrCreateRuntimeFunction(M, FnID: OMPRTL___tgt_target_kernel),
1015	Args: OffloadingArgs);
1016
1017	return Builder.saveIP();
1018	}
1019
1020	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitKernelLaunch(
1021	const LocationDescription &Loc, Function *OutlinedFn, Value *OutlinedFnID,
1022	EmitFallbackCallbackTy emitTargetCallFallbackCB, TargetKernelArgs &Args,
1023	Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP) {
1024
1025	if (!updateToLocation(Loc))
1026	return Loc.IP;
1027
1028	Builder.restoreIP(IP: Loc.IP);
1029	// On top of the arrays that were filled up, the target offloading call
1030	// takes as arguments the device id as well as the host pointer. The host
1031	// pointer is used by the runtime library to identify the current target
1032	// region, so it only has to be unique and not necessarily point to
1033	// anything. It could be the pointer to the outlined function that
1034	// implements the target region, but we aren't using that so that the
1035	// compiler doesn't need to keep that, and could therefore inline the host
1036	// function if proven worthwhile during optimization.
1037
1038	// From this point on, we need to have an ID of the target region defined.
1039	assert(OutlinedFnID && "Invalid outlined function ID!");
1040	(void)OutlinedFnID;
1041
1042	// Return value of the runtime offloading call.
1043	Value *Return = nullptr;
1044
1045	// Arguments for the target kernel.
1046	SmallVector<Value *> ArgsVector;
1047	getKernelArgsVector(KernelArgs&: Args, Builder, ArgsVector);
1048
1049	// The target region is an outlined function launched by the runtime
1050	// via calls to __tgt_target_kernel().
1051	//
1052	// Note that on the host and CPU targets, the runtime implementation of
1053	// these calls simply call the outlined function without forking threads.
1054	// The outlined functions themselves have runtime calls to
1055	// __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by
1056	// the compiler in emitTeamsCall() and emitParallelCall().
1057	//
1058	// In contrast, on the NVPTX target, the implementation of
1059	// __tgt_target_teams() launches a GPU kernel with the requested number
1060	// of teams and threads so no additional calls to the runtime are required.
1061	// Check the error code and execute the host version if required.
1062	Builder.restoreIP(IP: emitTargetKernel(Loc: Builder, AllocaIP, Return, Ident: RTLoc, DeviceID,
1063	NumTeams: Args.NumTeams, NumThreads: Args.NumThreads,
1064	HostPtr: OutlinedFnID, KernelArgs: ArgsVector));
1065
1066	BasicBlock *OffloadFailedBlock =
1067	BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.failed");
1068	BasicBlock *OffloadContBlock =
1069	BasicBlock::Create(Context&: Builder.getContext(), Name: "omp_offload.cont");
1070	Value *Failed = Builder.CreateIsNotNull(Arg: Return);
1071	Builder.CreateCondBr(Cond: Failed, True: OffloadFailedBlock, False: OffloadContBlock);
1072
1073	auto CurFn = Builder.GetInsertBlock()->getParent();
1074	emitBlock(BB: OffloadFailedBlock, CurFn);
1075	Builder.restoreIP(IP: emitTargetCallFallbackCB(Builder.saveIP()));
1076	emitBranch(Target: OffloadContBlock);
1077	emitBlock(BB: OffloadContBlock, CurFn, /*IsFinished=*/true);
1078	return Builder.saveIP();
1079	}
1080
1081	void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag,
1082	omp::Directive CanceledDirective,
1083	FinalizeCallbackTy ExitCB) {
1084	assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
1085	"Unexpected cancellation!");
1086
1087	// For a cancel barrier we create two new blocks.
1088	BasicBlock *BB = Builder.GetInsertBlock();
1089	BasicBlock *NonCancellationBlock;
1090	if (Builder.GetInsertPoint() == BB->end()) {
1091	// TODO: This branch will not be needed once we moved to the
1092	// OpenMPIRBuilder codegen completely.
1093	NonCancellationBlock = BasicBlock::Create(
1094	Context&: BB->getContext(), Name: BB->getName() + ".cont", Parent: BB->getParent());
1095	} else {
1096	NonCancellationBlock = SplitBlock(Old: BB, SplitPt: &*Builder.GetInsertPoint());
1097	BB->getTerminator()->eraseFromParent();
1098	Builder.SetInsertPoint(BB);
1099	}
1100	BasicBlock *CancellationBlock = BasicBlock::Create(
1101	Context&: BB->getContext(), Name: BB->getName() + ".cncl", Parent: BB->getParent());
1102
1103	// Jump to them based on the return value.
1104	Value *Cmp = Builder.CreateIsNull(Arg: CancelFlag);
1105	Builder.CreateCondBr(Cond: Cmp, True: NonCancellationBlock, False: CancellationBlock,
1106	/* TODO weight */ BranchWeights: nullptr, Unpredictable: nullptr);
1107
1108	// From the cancellation block we finalize all variables and go to the
1109	// post finalization block that is known to the FiniCB callback.
1110	Builder.SetInsertPoint(CancellationBlock);
1111	if (ExitCB)
1112	ExitCB(Builder.saveIP());
1113	auto &FI = FinalizationStack.back();
1114	FI.FiniCB(Builder.saveIP());
1115
1116	// The continuation block is where code generation continues.
1117	Builder.SetInsertPoint(TheBB: NonCancellationBlock, IP: NonCancellationBlock->begin());
1118	}
1119
1120	// Callback used to create OpenMP runtime calls to support
1121	// omp parallel clause for the device.
1122	// We need to use this callback to replace call to the OutlinedFn in OuterFn
1123	// by the call to the OpenMP DeviceRTL runtime function (kmpc_parallel_51)
1124	static void targetParallelCallback(
1125	OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn,
1126	BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition,
1127	Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1128	Value *ThreadID, const SmallVector<Instruction *, 4> &ToBeDeleted) {
1129	// Add some known attributes.
1130	IRBuilder<> &Builder = OMPIRBuilder->Builder;
1131	OutlinedFn.addParamAttr(0, Attribute::NoAlias);
1132	OutlinedFn.addParamAttr(1, Attribute::NoAlias);
1133	OutlinedFn.addParamAttr(0, Attribute::NoUndef);
1134	OutlinedFn.addParamAttr(1, Attribute::NoUndef);
1135	OutlinedFn.addFnAttr(Attribute::NoUnwind);
1136
1137	assert(OutlinedFn.arg_size() >= 2 &&
1138	"Expected at least tid and bounded tid as arguments");
1139	unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1140
1141	CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1142	assert(CI && "Expected call instruction to outlined function");
1143	CI->getParent()->setName("omp_parallel");
1144
1145	Builder.SetInsertPoint(CI);
1146	Type *PtrTy = OMPIRBuilder->VoidPtr;
1147	Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1148
1149	// Add alloca for kernel args
1150	OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.saveIP();
1151	Builder.SetInsertPoint(TheBB: OuterAllocaBB, IP: OuterAllocaBB->getFirstInsertionPt());
1152	AllocaInst *ArgsAlloca =
1153	Builder.CreateAlloca(Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars));
1154	Value *Args = ArgsAlloca;
1155	// Add address space cast if array for storing arguments is not allocated
1156	// in address space 0
1157	if (ArgsAlloca->getAddressSpace())
1158	Args = Builder.CreatePointerCast(V: ArgsAlloca, DestTy: PtrTy);
1159	Builder.restoreIP(IP: CurrentIP);
1160
1161	// Store captured vars which are used by kmpc_parallel_51
1162	for (unsigned Idx = 0; Idx < NumCapturedVars; Idx++) {
1163	Value *V = *(CI->arg_begin() + 2 + Idx);
1164	Value *StoreAddress = Builder.CreateConstInBoundsGEP2_64(
1165	Ty: ArrayType::get(ElementType: PtrTy, NumElements: NumCapturedVars), Ptr: Args, Idx0: 0, Idx1: Idx);
1166	Builder.CreateStore(Val: V, Ptr: StoreAddress);
1167	}
1168
1169	Value *Cond =
1170	IfCondition ? Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32)
1171	: Builder.getInt32(C: 1);
1172
1173	// Build kmpc_parallel_51 call
1174	Value *Parallel51CallArgs[] = {
1175	/* identifier*/ Ident,
1176	/* global thread num*/ ThreadID,
1177	/* if expression */ Cond,
1178	/* number of threads */ NumThreads ? NumThreads : Builder.getInt32(C: -1),
1179	/* Proc bind */ Builder.getInt32(C: -1),
1180	/* outlined function */
1181	Builder.CreateBitCast(V: &OutlinedFn, DestTy: OMPIRBuilder->ParallelTaskPtr),
1182	/* wrapper function */ NullPtrValue,
1183	/* arguments of the outlined funciton*/ Args,
1184	/* number of arguments */ Builder.getInt64(C: NumCapturedVars)};
1185
1186	FunctionCallee RTLFn =
1187	OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_parallel_51);
1188
1189	Builder.CreateCall(Callee: RTLFn, Args: Parallel51CallArgs);
1190
1191	LLVM_DEBUG(dbgs() << "With kmpc_parallel_51 placed: "
1192	<< *Builder.GetInsertBlock()->getParent() << "\n");
1193
1194	// Initialize the local TID stack location with the argument value.
1195	Builder.SetInsertPoint(PrivTID);
1196	Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1197	Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1198	Ptr: PrivTIDAddr);
1199
1200	// Remove redundant call to the outlined function.
1201	CI->eraseFromParent();
1202
1203	for (Instruction *I : ToBeDeleted) {
1204	I->eraseFromParent();
1205	}
1206	}
1207
1208	// Callback used to create OpenMP runtime calls to support
1209	// omp parallel clause for the host.
1210	// We need to use this callback to replace call to the OutlinedFn in OuterFn
1211	// by the call to the OpenMP host runtime function ( __kmpc_fork_call[_if])
1212	static void
1213	hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn,
1214	Function *OuterFn, Value *Ident, Value *IfCondition,
1215	Instruction *PrivTID, AllocaInst *PrivTIDAddr,
1216	const SmallVector<Instruction *, 4> &ToBeDeleted) {
1217	IRBuilder<> &Builder = OMPIRBuilder->Builder;
1218	FunctionCallee RTLFn;
1219	if (IfCondition) {
1220	RTLFn =
1221	OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call_if);
1222	} else {
1223	RTLFn =
1224	OMPIRBuilder->getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_fork_call);
1225	}
1226	if (auto *F = dyn_cast<Function>(Val: RTLFn.getCallee())) {
1227	if (!F->hasMetadata(KindID: LLVMContext::MD_callback)) {
1228	LLVMContext &Ctx = F->getContext();
1229	MDBuilder MDB(Ctx);
1230	// Annotate the callback behavior of the __kmpc_fork_call:
1231	// - The callback callee is argument number 2 (microtask).
1232	// - The first two arguments of the callback callee are unknown (-1).
1233	// - All variadic arguments to the __kmpc_fork_call are passed to the
1234	// callback callee.
1235	F->addMetadata(KindID: LLVMContext::MD_callback,
1236	MD&: *MDNode::get(Context&: Ctx, MDs: {MDB.createCallbackEncoding(
1237	CalleeArgNo: 2, Arguments: {-1, -1},
1238	/* VarArgsArePassed */ true)}));
1239	}
1240	}
1241	// Add some known attributes.
1242	OutlinedFn.addParamAttr(0, Attribute::NoAlias);
1243	OutlinedFn.addParamAttr(1, Attribute::NoAlias);
1244	OutlinedFn.addFnAttr(Attribute::NoUnwind);
1245
1246	assert(OutlinedFn.arg_size() >= 2 &&
1247	"Expected at least tid and bounded tid as arguments");
1248	unsigned NumCapturedVars = OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
1249
1250	CallInst *CI = cast<CallInst>(Val: OutlinedFn.user_back());
1251	CI->getParent()->setName("omp_parallel");
1252	Builder.SetInsertPoint(CI);
1253
1254	// Build call __kmpc_fork_call[_if](Ident, n, microtask, var1, .., varn);
1255	Value *ForkCallArgs[] = {
1256	Ident, Builder.getInt32(C: NumCapturedVars),
1257	Builder.CreateBitCast(V: &OutlinedFn, DestTy: OMPIRBuilder->ParallelTaskPtr)};
1258
1259	SmallVector<Value *, 16> RealArgs;
1260	RealArgs.append(in_start: std::begin(arr&: ForkCallArgs), in_end: std::end(arr&: ForkCallArgs));
1261	if (IfCondition) {
1262	Value *Cond = Builder.CreateSExtOrTrunc(V: IfCondition, DestTy: OMPIRBuilder->Int32);
1263	RealArgs.push_back(Elt: Cond);
1264	}
1265	RealArgs.append(in_start: CI->arg_begin() + /* tid & bound tid */ 2, in_end: CI->arg_end());
1266
1267	// __kmpc_fork_call_if always expects a void ptr as the last argument
1268	// If there are no arguments, pass a null pointer.
1269	auto PtrTy = OMPIRBuilder->VoidPtr;
1270	if (IfCondition && NumCapturedVars == 0) {
1271	Value *NullPtrValue = Constant::getNullValue(Ty: PtrTy);
1272	RealArgs.push_back(Elt: NullPtrValue);
1273	}
1274	if (IfCondition && RealArgs.back()->getType() != PtrTy)
1275	RealArgs.back() = Builder.CreateBitCast(V: RealArgs.back(), DestTy: PtrTy);
1276
1277	Builder.CreateCall(Callee: RTLFn, Args: RealArgs);
1278
1279	LLVM_DEBUG(dbgs() << "With fork_call placed: "
1280	<< *Builder.GetInsertBlock()->getParent() << "\n");
1281
1282	// Initialize the local TID stack location with the argument value.
1283	Builder.SetInsertPoint(PrivTID);
1284	Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
1285	Builder.CreateStore(Val: Builder.CreateLoad(Ty: OMPIRBuilder->Int32, Ptr: OutlinedAI),
1286	Ptr: PrivTIDAddr);
1287
1288	// Remove redundant call to the outlined function.
1289	CI->eraseFromParent();
1290
1291	for (Instruction *I : ToBeDeleted) {
1292	I->eraseFromParent();
1293	}
1294	}
1295
1296	IRBuilder<>::InsertPoint OpenMPIRBuilder::createParallel(
1297	const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
1298	BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
1299	FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
1300	omp::ProcBindKind ProcBind, bool IsCancellable) {
1301	assert(!isConflictIP(Loc.IP, OuterAllocaIP) && "IPs must not be ambiguous");
1302
1303	if (!updateToLocation(Loc))
1304	return Loc.IP;
1305
1306	uint32_t SrcLocStrSize;
1307	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1308	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1309	Value *ThreadID = getOrCreateThreadID(Ident);
1310	// If we generate code for the target device, we need to allocate
1311	// struct for aggregate params in the device default alloca address space.
1312	// OpenMP runtime requires that the params of the extracted functions are
1313	// passed as zero address space pointers. This flag ensures that extracted
1314	// function arguments are declared in zero address space
1315	bool ArgsInZeroAddressSpace = Config.isTargetDevice();
1316
1317	// Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
1318	// only if we compile for host side.
1319	if (NumThreads && !Config.isTargetDevice()) {
1320	Value *Args[] = {
1321	Ident, ThreadID,
1322	Builder.CreateIntCast(V: NumThreads, DestTy: Int32, /*isSigned*/ false)};
1323	Builder.CreateCall(
1324	Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_threads), Args);
1325	}
1326
1327	if (ProcBind != OMP_PROC_BIND_default) {
1328	// Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
1329	Value *Args[] = {
1330	Ident, ThreadID,
1331	ConstantInt::get(Ty: Int32, V: unsigned(ProcBind), /*isSigned=*/IsSigned: true)};
1332	Builder.CreateCall(
1333	Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_proc_bind), Args);
1334	}
1335
1336	BasicBlock *InsertBB = Builder.GetInsertBlock();
1337	Function *OuterFn = InsertBB->getParent();
1338
1339	// Save the outer alloca block because the insertion iterator may get
1340	// invalidated and we still need this later.
1341	BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
1342
1343	// Vector to remember instructions we used only during the modeling but which
1344	// we want to delete at the end.
1345	SmallVector<Instruction *, 4> ToBeDeleted;
1346
1347	// Change the location to the outer alloca insertion point to create and
1348	// initialize the allocas we pass into the parallel region.
1349	Builder.restoreIP(IP: OuterAllocaIP);
1350	AllocaInst *TIDAddrAlloca = Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr");
1351	AllocaInst *ZeroAddrAlloca =
1352	Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "zero.addr");
1353	Instruction *TIDAddr = TIDAddrAlloca;
1354	Instruction *ZeroAddr = ZeroAddrAlloca;
1355	if (ArgsInZeroAddressSpace && M.getDataLayout().getAllocaAddrSpace() != 0) {
1356	// Add additional casts to enforce pointers in zero address space
1357	TIDAddr = new AddrSpaceCastInst(
1358	TIDAddrAlloca, PointerType ::get(C&: M.getContext(), AddressSpace: 0), "tid.addr.ascast");
1359	TIDAddr->insertAfter(InsertPos: TIDAddrAlloca);
1360	ToBeDeleted.push_back(Elt: TIDAddr);
1361	ZeroAddr = new AddrSpaceCastInst(ZeroAddrAlloca,
1362	PointerType ::get(C&: M.getContext(), AddressSpace: 0),
1363	"zero.addr.ascast");
1364	ZeroAddr->insertAfter(InsertPos: ZeroAddrAlloca);
1365	ToBeDeleted.push_back(Elt: ZeroAddr);
1366	}
1367
1368	// We only need TIDAddr and ZeroAddr for modeling purposes to get the
1369	// associated arguments in the outlined function, so we delete them later.
1370	ToBeDeleted.push_back(Elt: TIDAddrAlloca);
1371	ToBeDeleted.push_back(Elt: ZeroAddrAlloca);
1372
1373	// Create an artificial insertion point that will also ensure the blocks we
1374	// are about to split are not degenerated.
1375	auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
1376
1377	BasicBlock *EntryBB = UI->getParent();
1378	BasicBlock *PRegEntryBB = EntryBB->splitBasicBlock(I: UI, BBName: "omp.par.entry");
1379	BasicBlock *PRegBodyBB = PRegEntryBB->splitBasicBlock(I: UI, BBName: "omp.par.region");
1380	BasicBlock *PRegPreFiniBB =
1381	PRegBodyBB->splitBasicBlock(I: UI, BBName: "omp.par.pre_finalize");
1382	BasicBlock *PRegExitBB = PRegPreFiniBB->splitBasicBlock(I: UI, BBName: "omp.par.exit");
1383
1384	auto FiniCBWrapper = [&](InsertPointTy IP) {
1385	// Hide "open-ended" blocks from the given FiniCB by setting the right jump
1386	// target to the region exit block.
1387	if (IP.getBlock()->end() == IP.getPoint()) {
1388	IRBuilder<>::InsertPointGuard IPG(Builder);
1389	Builder.restoreIP(IP);
1390	Instruction *I = Builder.CreateBr(Dest: PRegExitBB);
1391	IP = InsertPointTy(I->getParent(), I->getIterator());
1392	}
1393	assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
1394	IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
1395	"Unexpected insertion point for finalization call!");
1396	return FiniCB(IP);
1397	};
1398
1399	FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
1400
1401	// Generate the privatization allocas in the block that will become the entry
1402	// of the outlined function.
1403	Builder.SetInsertPoint(PRegEntryBB->getTerminator());
1404	InsertPointTy InnerAllocaIP = Builder.saveIP();
1405
1406	AllocaInst *PrivTIDAddr =
1407	Builder.CreateAlloca(Ty: Int32, ArraySize: nullptr, Name: "tid.addr.local");
1408	Instruction *PrivTID = Builder.CreateLoad(Ty: Int32, Ptr: PrivTIDAddr, Name: "tid");
1409
1410	// Add some fake uses for OpenMP provided arguments.
1411	ToBeDeleted.push_back(Elt: Builder.CreateLoad(Ty: Int32, Ptr: TIDAddr, Name: "tid.addr.use"));
1412	Instruction *ZeroAddrUse =
1413	Builder.CreateLoad(Ty: Int32, Ptr: ZeroAddr, Name: "zero.addr.use");
1414	ToBeDeleted.push_back(Elt: ZeroAddrUse);
1415
1416	// EntryBB
1417	// |
1418	// V
1419	// PRegionEntryBB <- Privatization allocas are placed here.
1420	// |
1421	// V
1422	// PRegionBodyBB <- BodeGen is invoked here.
1423	// |
1424	// V
1425	// PRegPreFiniBB <- The block we will start finalization from.
1426	// |
1427	// V
1428	// PRegionExitBB <- A common exit to simplify block collection.
1429	//
1430
1431	LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
1432
1433	// Let the caller create the body.
1434	assert(BodyGenCB && "Expected body generation callback!");
1435	InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
1436	BodyGenCB(InnerAllocaIP, CodeGenIP);
1437
1438	LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
1439
1440	OutlineInfo OI;
1441	if (Config.isTargetDevice()) {
1442	// Generate OpenMP target specific runtime call
1443	OI.PostOutlineCB = [=, ToBeDeletedVec =
1444	std::move(ToBeDeleted)](Function &OutlinedFn) {
1445	targetParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, OuterAllocaBB: OuterAllocaBlock, Ident,
1446	IfCondition, NumThreads, PrivTID, PrivTIDAddr,
1447	ThreadID, ToBeDeleted: ToBeDeletedVec);
1448	};
1449	} else {
1450	// Generate OpenMP host runtime call
1451	OI.PostOutlineCB = [=, ToBeDeletedVec =
1452	std::move(ToBeDeleted)](Function &OutlinedFn) {
1453	hostParallelCallback(OMPIRBuilder: this, OutlinedFn, OuterFn, Ident, IfCondition,
1454	PrivTID, PrivTIDAddr, ToBeDeleted: ToBeDeletedVec);
1455	};
1456	}
1457
1458	// Adjust the finalization stack, verify the adjustment, and call the
1459	// finalize function a last time to finalize values between the pre-fini
1460	// block and the exit block if we left the parallel "the normal way".
1461	auto FiniInfo = FinalizationStack.pop_back_val();
1462	(void)FiniInfo;
1463	assert(FiniInfo.DK == OMPD_parallel &&
1464	"Unexpected finalization stack state!");
1465
1466	Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
1467
1468	InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
1469	FiniCB(PreFiniIP);
1470
1471	OI.OuterAllocaBB = OuterAllocaBlock;
1472	OI.EntryBB = PRegEntryBB;
1473	OI.ExitBB = PRegExitBB;
1474
1475	SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
1476	SmallVector<BasicBlock *, 32> Blocks;
1477	OI.collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
1478
1479	// Ensure a single exit node for the outlined region by creating one.
1480	// We might have multiple incoming edges to the exit now due to finalizations,
1481	// e.g., cancel calls that cause the control flow to leave the region.
1482	BasicBlock *PRegOutlinedExitBB = PRegExitBB;
1483	PRegExitBB = SplitBlock(Old: PRegExitBB, SplitPt: &*PRegExitBB->getFirstInsertionPt());
1484	PRegOutlinedExitBB->setName("omp.par.outlined.exit");
1485	Blocks.push_back(Elt: PRegOutlinedExitBB);
1486
1487	CodeExtractorAnalysisCache CEAC(*OuterFn);
1488	CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
1489	/* AggregateArgs */ false,
1490	/* BlockFrequencyInfo */ nullptr,
1491	/* BranchProbabilityInfo */ nullptr,
1492	/* AssumptionCache */ nullptr,
1493	/* AllowVarArgs */ true,
1494	/* AllowAlloca */ true,
1495	/* AllocationBlock */ OuterAllocaBlock,
1496	/* Suffix */ ".omp_par", ArgsInZeroAddressSpace);
1497
1498	// Find inputs to, outputs from the code region.
1499	BasicBlock *CommonExit = nullptr;
1500	SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
1501	Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
1502	Extractor.findInputsOutputs(Inputs, Outputs, Allocas: SinkingCands);
1503
1504	LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
1505
1506	FunctionCallee TIDRTLFn =
1507	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_global_thread_num);
1508
1509	auto PrivHelper = [&](Value &V) {
1510	if (&V == TIDAddr || &V == ZeroAddr) {
1511	OI.ExcludeArgsFromAggregate.push_back(Elt: &V);
1512	return;
1513	}
1514
1515	SetVector<Use *> Uses;
1516	for (Use &U : V.uses())
1517	if (auto *UserI = dyn_cast<Instruction>(Val: U.getUser()))
1518	if (ParallelRegionBlockSet.count(Ptr: UserI->getParent()))
1519	Uses.insert(X: &U);
1520
1521	// __kmpc_fork_call expects extra arguments as pointers. If the input
1522	// already has a pointer type, everything is fine. Otherwise, store the
1523	// value onto stack and load it back inside the to-be-outlined region. This
1524	// will ensure only the pointer will be passed to the function.
1525	// FIXME: if there are more than 15 trailing arguments, they must be
1526	// additionally packed in a struct.
1527	Value *Inner = &V;
1528	if (!V.getType()->isPointerTy()) {
1529	IRBuilder<>::InsertPointGuard Guard(Builder);
1530	LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
1531
1532	Builder.restoreIP(IP: OuterAllocaIP);
1533	Value *Ptr =
1534	Builder.CreateAlloca(Ty: V.getType(), ArraySize: nullptr, Name: V.getName() + ".reloaded");
1535
1536	// Store to stack at end of the block that currently branches to the entry
1537	// block of the to-be-outlined region.
1538	Builder.SetInsertPoint(TheBB: InsertBB,
1539	IP: InsertBB->getTerminator()->getIterator());
1540	Builder.CreateStore(Val: &V, Ptr);
1541
1542	// Load back next to allocations in the to-be-outlined region.
1543	Builder.restoreIP(IP: InnerAllocaIP);
1544	Inner = Builder.CreateLoad(Ty: V.getType(), Ptr);
1545	}
1546
1547	Value *ReplacementValue = nullptr;
1548	CallInst *CI = dyn_cast<CallInst>(Val: &V);
1549	if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
1550	ReplacementValue = PrivTID;
1551	} else {
1552	Builder.restoreIP(
1553	IP: PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
1554	assert(ReplacementValue &&
1555	"Expected copy/create callback to set replacement value!");
1556	if (ReplacementValue == &V)
1557	return;
1558	}
1559
1560	for (Use *UPtr : Uses)
1561	UPtr->set(ReplacementValue);
1562	};
1563
1564	// Reset the inner alloca insertion as it will be used for loading the values
1565	// wrapped into pointers before passing them into the to-be-outlined region.
1566	// Configure it to insert immediately after the fake use of zero address so
1567	// that they are available in the generated body and so that the
1568	// OpenMP-related values (thread ID and zero address pointers) remain leading
1569	// in the argument list.
1570	InnerAllocaIP = IRBuilder<>::InsertPoint(
1571	ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
1572
1573	// Reset the outer alloca insertion point to the entry of the relevant block
1574	// in case it was invalidated.
1575	OuterAllocaIP = IRBuilder<>::InsertPoint(
1576	OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
1577
1578	for (Value *Input : Inputs) {
1579	LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
1580	PrivHelper(*Input);
1581	}
1582	LLVM_DEBUG({
1583	for (Value *Output : Outputs)
1584	LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
1585	});
1586	assert(Outputs.empty() &&
1587	"OpenMP outlining should not produce live-out values!");
1588
1589	LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
1590	LLVM_DEBUG({
1591	for (auto *BB : Blocks)
1592	dbgs() << " PBR: " << BB->getName() << "\n";
1593	});
1594
1595	// Register the outlined info.
1596	addOutlineInfo(OI: std::move(OI));
1597
1598	InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
1599	UI->eraseFromParent();
1600
1601	return AfterIP;
1602	}
1603
1604	void OpenMPIRBuilder::emitFlush(const LocationDescription &Loc) {
1605	// Build call void __kmpc_flush(ident_t *loc)
1606	uint32_t SrcLocStrSize;
1607	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1608	Value *Args[] = {getOrCreateIdent(SrcLocStr, SrcLocStrSize)};
1609
1610	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_flush), Args);
1611	}
1612
1613	void OpenMPIRBuilder::createFlush(const LocationDescription &Loc) {
1614	if (!updateToLocation(Loc))
1615	return;
1616	emitFlush(Loc);
1617	}
1618
1619	void OpenMPIRBuilder::emitTaskwaitImpl(const LocationDescription &Loc) {
1620	// Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
1621	// global_tid);
1622	uint32_t SrcLocStrSize;
1623	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1624	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1625	Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
1626
1627	// Ignore return result until untied tasks are supported.
1628	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskwait),
1629	Args);
1630	}
1631
1632	void OpenMPIRBuilder::createTaskwait(const LocationDescription &Loc) {
1633	if (!updateToLocation(Loc))
1634	return;
1635	emitTaskwaitImpl(Loc);
1636	}
1637
1638	void OpenMPIRBuilder::emitTaskyieldImpl(const LocationDescription &Loc) {
1639	// Build call __kmpc_omp_taskyield(loc, thread_id, 0);
1640	uint32_t SrcLocStrSize;
1641	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1642	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1643	Constant *I32Null = ConstantInt::getNullValue(Ty: Int32);
1644	Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
1645
1646	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_taskyield),
1647	Args);
1648	}
1649
1650	void OpenMPIRBuilder::createTaskyield(const LocationDescription &Loc) {
1651	if (!updateToLocation(Loc))
1652	return;
1653	emitTaskyieldImpl(Loc);
1654	}
1655
1656	OpenMPIRBuilder::InsertPointTy
1657	OpenMPIRBuilder::createTask(const LocationDescription &Loc,
1658	InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB,
1659	bool Tied, Value *Final, Value *IfCondition,
1660	SmallVector<DependData> Dependencies) {
1661
1662	if (!updateToLocation(Loc))
1663	return InsertPointTy();
1664
1665	uint32_t SrcLocStrSize;
1666	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1667	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1668	// The current basic block is split into four basic blocks. After outlining,
1669	// they will be mapped as follows:
1670	// ```
1671	// def current_fn() {
1672	// current_basic_block:
1673	// br label %task.exit
1674	// task.exit:
1675	// ; instructions after task
1676	// }
1677	// def outlined_fn() {
1678	// task.alloca:
1679	// br label %task.body
1680	// task.body:
1681	// ret void
1682	// }
1683	// ```
1684	BasicBlock *TaskExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.exit");
1685	BasicBlock *TaskBodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "task.body");
1686	BasicBlock *TaskAllocaBB =
1687	splitBB(Builder, /*CreateBranch=*/true, Name: "task.alloca");
1688
1689	InsertPointTy TaskAllocaIP =
1690	InsertPointTy(TaskAllocaBB, TaskAllocaBB->begin());
1691	InsertPointTy TaskBodyIP = InsertPointTy(TaskBodyBB, TaskBodyBB->begin());
1692	BodyGenCB(TaskAllocaIP, TaskBodyIP);
1693
1694	OutlineInfo OI;
1695	OI.EntryBB = TaskAllocaBB;
1696	OI.OuterAllocaBB = AllocaIP.getBlock();
1697	OI.ExitBB = TaskExitBB;
1698
1699	// Add the thread ID argument.
1700	std::stack<Instruction *> ToBeDeleted;
1701	OI.ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
1702	Builder, OuterAllocaIP: AllocaIP, ToBeDeleted, InnerAllocaIP: TaskAllocaIP, Name: "global.tid", AsPtr: false));
1703
1704	OI.PostOutlineCB = [this, Ident, Tied, Final, IfCondition, Dependencies,
1705	TaskAllocaBB, ToBeDeleted](Function &OutlinedFn) mutable {
1706	// Replace the Stale CI by appropriate RTL function call.
1707	assert(OutlinedFn.getNumUses() == 1 &&
1708	"there must be a single user for the outlined function");
1709	CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
1710
1711	// HasShareds is true if any variables are captured in the outlined region,
1712	// false otherwise.
1713	bool HasShareds = StaleCI->arg_size() > 1;
1714	Builder.SetInsertPoint(StaleCI);
1715
1716	// Gather the arguments for emitting the runtime call for
1717	// @__kmpc_omp_task_alloc
1718	Function *TaskAllocFn =
1719	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_alloc);
1720
1721	// Arguments - `loc_ref` (Ident) and `gtid` (ThreadID)
1722	// call.
1723	Value *ThreadID = getOrCreateThreadID(Ident);
1724
1725	// Argument - `flags`
1726	// Task is tied iff (Flags & 1) == 1.
1727	// Task is untied iff (Flags & 1) == 0.
1728	// Task is final iff (Flags & 2) == 2.
1729	// Task is not final iff (Flags & 2) == 0.
1730	// TODO: Handle the other flags.
1731	Value *Flags = Builder.getInt32(C: Tied);
1732	if (Final) {
1733	Value *FinalFlag =
1734	Builder.CreateSelect(C: Final, True: Builder.getInt32(C: 2), False: Builder.getInt32(C: 0));
1735	Flags = Builder.CreateOr(LHS: FinalFlag, RHS: Flags);
1736	}
1737
1738	// Argument - `sizeof_kmp_task_t` (TaskSize)
1739	// Tasksize refers to the size in bytes of kmp_task_t data structure
1740	// including private vars accessed in task.
1741	// TODO: add kmp_task_t_with_privates (privates)
1742	Value *TaskSize = Builder.getInt64(
1743	C: divideCeil(Numerator: M.getDataLayout().getTypeSizeInBits(Ty: Task), Denominator: 8));
1744
1745	// Argument - `sizeof_shareds` (SharedsSize)
1746	// SharedsSize refers to the shareds array size in the kmp_task_t data
1747	// structure.
1748	Value *SharedsSize = Builder.getInt64(C: 0);
1749	if (HasShareds) {
1750	AllocaInst *ArgStructAlloca =
1751	dyn_cast<AllocaInst>(Val: StaleCI->getArgOperand(i: 1));
1752	assert(ArgStructAlloca &&
1753	"Unable to find the alloca instruction corresponding to arguments "
1754	"for extracted function");
1755	StructType *ArgStructType =
1756	dyn_cast<StructType>(Val: ArgStructAlloca->getAllocatedType());
1757	assert(ArgStructType && "Unable to find struct type corresponding to "
1758	"arguments for extracted function");
1759	SharedsSize =
1760	Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(Ty: ArgStructType));
1761	}
1762	// Emit the @__kmpc_omp_task_alloc runtime call
1763	// The runtime call returns a pointer to an area where the task captured
1764	// variables must be copied before the task is run (TaskData)
1765	CallInst *TaskData = Builder.CreateCall(
1766	Callee: TaskAllocFn, Args: {/*loc_ref=*/Ident, /*gtid=*/ThreadID, /*flags=*/Flags,
1767	/*sizeof_task=*/TaskSize, /*sizeof_shared=*/SharedsSize,
1768	/*task_func=*/&OutlinedFn});
1769
1770	// Copy the arguments for outlined function
1771	if (HasShareds) {
1772	Value *Shareds = StaleCI->getArgOperand(i: 1);
1773	Align Alignment = TaskData->getPointerAlignment(DL: M.getDataLayout());
1774	Value *TaskShareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: TaskData);
1775	Builder.CreateMemCpy(Dst: TaskShareds, DstAlign: Alignment, Src: Shareds, SrcAlign: Alignment,
1776	Size: SharedsSize);
1777	}
1778
1779	Value *DepArray = nullptr;
1780	if (Dependencies.size()) {
1781	InsertPointTy OldIP = Builder.saveIP();
1782	Builder.SetInsertPoint(
1783	&OldIP.getBlock()->getParent()->getEntryBlock().back());
1784
1785	Type *DepArrayTy = ArrayType::get(ElementType: DependInfo, NumElements: Dependencies.size());
1786	DepArray = Builder.CreateAlloca(Ty: DepArrayTy, ArraySize: nullptr, Name: ".dep.arr.addr");
1787
1788	unsigned P = 0;
1789	for (const DependData &Dep : Dependencies) {
1790	Value *Base =
1791	Builder.CreateConstInBoundsGEP2_64(Ty: DepArrayTy, Ptr: DepArray, Idx0: 0, Idx1: P);
1792	// Store the pointer to the variable
1793	Value *Addr = Builder.CreateStructGEP(
1794	Ty: DependInfo, Ptr: Base,
1795	Idx: static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
1796	Value *DepValPtr =
1797	Builder.CreatePtrToInt(V: Dep.DepVal, DestTy: Builder.getInt64Ty());
1798	Builder.CreateStore(Val: DepValPtr, Ptr: Addr);
1799	// Store the size of the variable
1800	Value *Size = Builder.CreateStructGEP(
1801	Ty: DependInfo, Ptr: Base,
1802	Idx: static_cast<unsigned int>(RTLDependInfoFields::Len));
1803	Builder.CreateStore(Val: Builder.getInt64(C: M.getDataLayout().getTypeStoreSize(
1804	Ty: Dep.DepValueType)),
1805	Ptr: Size);
1806	// Store the dependency kind
1807	Value *Flags = Builder.CreateStructGEP(
1808	Ty: DependInfo, Ptr: Base,
1809	Idx: static_cast<unsigned int>(RTLDependInfoFields::Flags));
1810	Builder.CreateStore(
1811	Val: ConstantInt::get(Ty: Builder.getInt8Ty(),
1812	V: static_cast<unsigned int>(Dep.DepKind)),
1813	Ptr: Flags);
1814	++P;
1815	}
1816
1817	Builder.restoreIP(IP: OldIP);
1818	}
1819
1820	// In the presence of the `if` clause, the following IR is generated:
1821	// ...
1822	// %data = call @__kmpc_omp_task_alloc(...)
1823	// br i1 %if_condition, label %then, label %else
1824	// then:
1825	// call @__kmpc_omp_task(...)
1826	// br label %exit
1827	// else:
1828	// call @__kmpc_omp_task_begin_if0(...)
1829	// call @outlined_fn(...)
1830	// call @__kmpc_omp_task_complete_if0(...)
1831	// br label %exit
1832	// exit:
1833	// ...
1834	if (IfCondition) {
1835	// `SplitBlockAndInsertIfThenElse` requires the block to have a
1836	// terminator.
1837	splitBB(Builder, /*CreateBranch=*/true, Name: "if.end");
1838	Instruction *IfTerminator =
1839	Builder.GetInsertPoint()->getParent()->getTerminator();
1840	Instruction *ThenTI = IfTerminator, *ElseTI = nullptr;
1841	Builder.SetInsertPoint(IfTerminator);
1842	SplitBlockAndInsertIfThenElse(Cond: IfCondition, SplitBefore: IfTerminator, ThenTerm: &ThenTI,
1843	ElseTerm: &ElseTI);
1844	Builder.SetInsertPoint(ElseTI);
1845	Function *TaskBeginFn =
1846	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_begin_if0);
1847	Function *TaskCompleteFn =
1848	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_complete_if0);
1849	Builder.CreateCall(Callee: TaskBeginFn, Args: {Ident, ThreadID, TaskData});
1850	CallInst *CI = nullptr;
1851	if (HasShareds)
1852	CI = Builder.CreateCall(Callee: &OutlinedFn, Args: {ThreadID, TaskData});
1853	else
1854	CI = Builder.CreateCall(Callee: &OutlinedFn, Args: {ThreadID});
1855	CI->setDebugLoc(StaleCI->getDebugLoc());
1856	Builder.CreateCall(Callee: TaskCompleteFn, Args: {Ident, ThreadID, TaskData});
1857	Builder.SetInsertPoint(ThenTI);
1858	}
1859
1860	if (Dependencies.size()) {
1861	Function *TaskFn =
1862	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task_with_deps);
1863	Builder.CreateCall(
1864	Callee: TaskFn,
1865	Args: {Ident, ThreadID, TaskData, Builder.getInt32(C: Dependencies.size()),
1866	DepArray, ConstantInt::get(Ty: Builder.getInt32Ty(), V: 0),
1867	ConstantPointerNull::get(T: PointerType::getUnqual(C&: M.getContext()))});
1868
1869	} else {
1870	// Emit the @__kmpc_omp_task runtime call to spawn the task
1871	Function *TaskFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_omp_task);
1872	Builder.CreateCall(Callee: TaskFn, Args: {Ident, ThreadID, TaskData});
1873	}
1874
1875	StaleCI->eraseFromParent();
1876
1877	Builder.SetInsertPoint(TheBB: TaskAllocaBB, IP: TaskAllocaBB->begin());
1878	if (HasShareds) {
1879	LoadInst *Shareds = Builder.CreateLoad(Ty: VoidPtr, Ptr: OutlinedFn.getArg(i: 1));
1880	OutlinedFn.getArg(i: 1)->replaceUsesWithIf(
1881	New: Shareds, ShouldReplace: [Shareds](Use &U) { return U.getUser() != Shareds; });
1882	}
1883
1884	while (!ToBeDeleted.empty()) {
1885	ToBeDeleted.top()->eraseFromParent();
1886	ToBeDeleted.pop();
1887	}
1888	};
1889
1890	addOutlineInfo(OI: std::move(OI));
1891	Builder.SetInsertPoint(TheBB: TaskExitBB, IP: TaskExitBB->begin());
1892
1893	return Builder.saveIP();
1894	}
1895
1896	OpenMPIRBuilder::InsertPointTy
1897	OpenMPIRBuilder::createTaskgroup(const LocationDescription &Loc,
1898	InsertPointTy AllocaIP,
1899	BodyGenCallbackTy BodyGenCB) {
1900	if (!updateToLocation(Loc))
1901	return InsertPointTy();
1902
1903	uint32_t SrcLocStrSize;
1904	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
1905	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
1906	Value *ThreadID = getOrCreateThreadID(Ident);
1907
1908	// Emit the @__kmpc_taskgroup runtime call to start the taskgroup
1909	Function *TaskgroupFn =
1910	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_taskgroup);
1911	Builder.CreateCall(Callee: TaskgroupFn, Args: {Ident, ThreadID});
1912
1913	BasicBlock *TaskgroupExitBB = splitBB(Builder, CreateBranch: true, Name: "taskgroup.exit");
1914	BodyGenCB(AllocaIP, Builder.saveIP());
1915
1916	Builder.SetInsertPoint(TaskgroupExitBB);
1917	// Emit the @__kmpc_end_taskgroup runtime call to end the taskgroup
1918	Function *EndTaskgroupFn =
1919	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_taskgroup);
1920	Builder.CreateCall(Callee: EndTaskgroupFn, Args: {Ident, ThreadID});
1921
1922	return Builder.saveIP();
1923	}
1924
1925	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSections(
1926	const LocationDescription &Loc, InsertPointTy AllocaIP,
1927	ArrayRef<StorableBodyGenCallbackTy> SectionCBs, PrivatizeCallbackTy PrivCB,
1928	FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait) {
1929	assert(!isConflictIP(AllocaIP, Loc.IP) && "Dedicated IP allocas required");
1930
1931	if (!updateToLocation(Loc))
1932	return Loc.IP;
1933
1934	auto FiniCBWrapper = [&](InsertPointTy IP) {
1935	if (IP.getBlock()->end() != IP.getPoint())
1936	return FiniCB(IP);
1937	// This must be done otherwise any nested constructs using FinalizeOMPRegion
1938	// will fail because that function requires the Finalization Basic Block to
1939	// have a terminator, which is already removed by EmitOMPRegionBody.
1940	// IP is currently at cancelation block.
1941	// We need to backtrack to the condition block to fetch
1942	// the exit block and create a branch from cancelation
1943	// to exit block.
1944	IRBuilder<>::InsertPointGuard IPG(Builder);
1945	Builder.restoreIP(IP);
1946	auto *CaseBB = IP.getBlock()->getSinglePredecessor();
1947	auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
1948	auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
1949	Instruction *I = Builder.CreateBr(Dest: ExitBB);
1950	IP = InsertPointTy(I->getParent(), I->getIterator());
1951	return FiniCB(IP);
1952	};
1953
1954	FinalizationStack.push_back({FiniCBWrapper, OMPD_sections, IsCancellable});
1955
1956	// Each section is emitted as a switch case
1957	// Each finalization callback is handled from clang.EmitOMPSectionDirective()
1958	// -> OMP.createSection() which generates the IR for each section
1959	// Iterate through all sections and emit a switch construct:
1960	// switch (IV) {
1961	// case 0:
1962	// <SectionStmt[0]>;
1963	// break;
1964	// ...
1965	// case <NumSection> - 1:
1966	// <SectionStmt[<NumSection> - 1]>;
1967	// break;
1968	// }
1969	// ...
1970	// section_loop.after:
1971	// <FiniCB>;
1972	auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, Value *IndVar) {
1973	Builder.restoreIP(IP: CodeGenIP);
1974	BasicBlock *Continue =
1975	splitBBWithSuffix(Builder, /*CreateBranch=*/false, Suffix: ".sections.after");
1976	Function *CurFn = Continue->getParent();
1977	SwitchInst *SwitchStmt = Builder.CreateSwitch(V: IndVar, Dest: Continue);
1978
1979	unsigned CaseNumber = 0;
1980	for (auto SectionCB : SectionCBs) {
1981	BasicBlock *CaseBB = BasicBlock::Create(
1982	Context&: M.getContext(), Name: "omp_section_loop.body.case", Parent: CurFn, InsertBefore: Continue);
1983	SwitchStmt->addCase(OnVal: Builder.getInt32(C: CaseNumber), Dest: CaseBB);
1984	Builder.SetInsertPoint(CaseBB);
1985	BranchInst *CaseEndBr = Builder.CreateBr(Dest: Continue);
1986	SectionCB(InsertPointTy(),
1987	{CaseEndBr->getParent(), CaseEndBr->getIterator()});
1988	CaseNumber++;
1989	}
1990	// remove the existing terminator from body BB since there can be no
1991	// terminators after switch/case
1992	};
1993	// Loop body ends here
1994	// LowerBound, UpperBound, and STride for createCanonicalLoop
1995	Type *I32Ty = Type::getInt32Ty(C&: M.getContext());
1996	Value *LB = ConstantInt::get(Ty: I32Ty, V: 0);
1997	Value *UB = ConstantInt::get(Ty: I32Ty, V: SectionCBs.size());
1998	Value *ST = ConstantInt::get(Ty: I32Ty, V: 1);
1999	llvm::CanonicalLoopInfo *LoopInfo = createCanonicalLoop(
2000	Loc, BodyGenCB: LoopBodyGenCB, Start: LB, Stop: UB, Step: ST, IsSigned: true, InclusiveStop: false, ComputeIP: AllocaIP, Name: "section_loop");
2001	InsertPointTy AfterIP =
2002	applyStaticWorkshareLoop(DL: Loc.DL, CLI: LoopInfo, AllocaIP, NeedsBarrier: !IsNowait);
2003
2004	// Apply the finalization callback in LoopAfterBB
2005	auto FiniInfo = FinalizationStack.pop_back_val();
2006	assert(FiniInfo.DK == OMPD_sections &&
2007	"Unexpected finalization stack state!");
2008	if (FinalizeCallbackTy &CB = FiniInfo.FiniCB) {
2009	Builder.restoreIP(IP: AfterIP);
2010	BasicBlock *FiniBB =
2011	splitBBWithSuffix(Builder, /*CreateBranch=*/true, Suffix: "sections.fini");
2012	CB(Builder.saveIP());
2013	AfterIP = {FiniBB, FiniBB->begin()};
2014	}
2015
2016	return AfterIP;
2017	}
2018
2019	OpenMPIRBuilder::InsertPointTy
2020	OpenMPIRBuilder::createSection(const LocationDescription &Loc,
2021	BodyGenCallbackTy BodyGenCB,
2022	FinalizeCallbackTy FiniCB) {
2023	if (!updateToLocation(Loc))
2024	return Loc.IP;
2025
2026	auto FiniCBWrapper = [&](InsertPointTy IP) {
2027	if (IP.getBlock()->end() != IP.getPoint())
2028	return FiniCB(IP);
2029	// This must be done otherwise any nested constructs using FinalizeOMPRegion
2030	// will fail because that function requires the Finalization Basic Block to
2031	// have a terminator, which is already removed by EmitOMPRegionBody.
2032	// IP is currently at cancelation block.
2033	// We need to backtrack to the condition block to fetch
2034	// the exit block and create a branch from cancelation
2035	// to exit block.
2036	IRBuilder<>::InsertPointGuard IPG(Builder);
2037	Builder.restoreIP(IP);
2038	auto *CaseBB = Loc.IP.getBlock();
2039	auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
2040	auto *ExitBB = CondBB->getTerminator()->getSuccessor(Idx: 1);
2041	Instruction *I = Builder.CreateBr(Dest: ExitBB);
2042	IP = InsertPointTy(I->getParent(), I->getIterator());
2043	return FiniCB(IP);
2044	};
2045
2046	Directive OMPD = Directive::OMPD_sections;
2047	// Since we are using Finalization Callback here, HasFinalize
2048	// and IsCancellable have to be true
2049	return EmitOMPInlinedRegion(OMPD, nullptr, nullptr, BodyGenCB, FiniCBWrapper,
2050	/*Conditional*/ false, /*hasFinalize*/ true,
2051	/*IsCancellable*/ true);
2052	}
2053
2054	/// Create a function with a unique name and a "void (i8*, i8*)" signature in
2055	/// the given module and return it.
2056	Function *getFreshReductionFunc(Module &M) {
2057	Type *VoidTy = Type::getVoidTy(C&: M.getContext());
2058	Type *Int8PtrTy = PointerType::getUnqual(C&: M.getContext());
2059	auto *FuncTy =
2060	FunctionType::get(Result: VoidTy, Params: {Int8PtrTy, Int8PtrTy}, /* IsVarArg */ isVarArg: false);
2061	return Function::Create(Ty: FuncTy, Linkage: GlobalVariable::InternalLinkage,
2062	AddrSpace: M.getDataLayout().getDefaultGlobalsAddressSpace(),
2063	N: ".omp.reduction.func", M: &M);
2064	}
2065
2066	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createReductions(
2067	const LocationDescription &Loc, InsertPointTy AllocaIP,
2068	ArrayRef<ReductionInfo> ReductionInfos, bool IsNoWait) {
2069	for (const ReductionInfo &RI : ReductionInfos) {
2070	(void)RI;
2071	assert(RI.Variable && "expected non-null variable");
2072	assert(RI.PrivateVariable && "expected non-null private variable");
2073	assert(RI.ReductionGen && "expected non-null reduction generator callback");
2074	assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
2075	"expected variables and their private equivalents to have the same "
2076	"type");
2077	assert(RI.Variable->getType()->isPointerTy() &&
2078	"expected variables to be pointers");
2079	}
2080
2081	if (!updateToLocation(Loc))
2082	return InsertPointTy();
2083
2084	BasicBlock *InsertBlock = Loc.IP.getBlock();
2085	BasicBlock *ContinuationBlock =
2086	InsertBlock->splitBasicBlock(I: Loc.IP.getPoint(), BBName: "reduce.finalize");
2087	InsertBlock->getTerminator()->eraseFromParent();
2088
2089	// Create and populate array of type-erased pointers to private reduction
2090	// values.
2091	unsigned NumReductions = ReductionInfos.size();
2092	Type *RedArrayTy = ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: NumReductions);
2093	Builder.restoreIP(IP: AllocaIP);
2094	Value *RedArray = Builder.CreateAlloca(Ty: RedArrayTy, ArraySize: nullptr, Name: "red.array");
2095
2096	Builder.SetInsertPoint(TheBB: InsertBlock, IP: InsertBlock->end());
2097
2098	for (auto En : enumerate(First&: ReductionInfos)) {
2099	unsigned Index = En.index();
2100	const ReductionInfo &RI = En.value();
2101	Value *RedArrayElemPtr = Builder.CreateConstInBoundsGEP2_64(
2102	Ty: RedArrayTy, Ptr: RedArray, Idx0: 0, Idx1: Index, Name: "red.array.elem." + Twine(Index));
2103	Builder.CreateStore(Val: RI.PrivateVariable, Ptr: RedArrayElemPtr);
2104	}
2105
2106	// Emit a call to the runtime function that orchestrates the reduction.
2107	// Declare the reduction function in the process.
2108	Function *Func = Builder.GetInsertBlock()->getParent();
2109	Module *Module = Func->getParent();
2110	uint32_t SrcLocStrSize;
2111	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2112	bool CanGenerateAtomic =
2113	llvm::all_of(Range&: ReductionInfos, P: [](const ReductionInfo &RI) {
2114	return RI.AtomicReductionGen;
2115	});
2116	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize,
2117	LocFlags: CanGenerateAtomic
2118	? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
2119	: IdentFlag(0));
2120	Value *ThreadId = getOrCreateThreadID(Ident);
2121	Constant *NumVariables = Builder.getInt32(C: NumReductions);
2122	const DataLayout &DL = Module->getDataLayout();
2123	unsigned RedArrayByteSize = DL.getTypeStoreSize(Ty: RedArrayTy);
2124	Constant *RedArraySize = Builder.getInt64(C: RedArrayByteSize);
2125	Function *ReductionFunc = getFreshReductionFunc(M&: *Module);
2126	Value *Lock = getOMPCriticalRegionLock(CriticalName: ".reduction");
2127	Function *ReduceFunc = getOrCreateRuntimeFunctionPtr(
2128	FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
2129	: RuntimeFunction::OMPRTL___kmpc_reduce);
2130	CallInst *ReduceCall =
2131	Builder.CreateCall(Callee: ReduceFunc,
2132	Args: {Ident, ThreadId, NumVariables, RedArraySize, RedArray,
2133	ReductionFunc, Lock},
2134	Name: "reduce");
2135
2136	// Create final reduction entry blocks for the atomic and non-atomic case.
2137	// Emit IR that dispatches control flow to one of the blocks based on the
2138	// reduction supporting the atomic mode.
2139	BasicBlock *NonAtomicRedBlock =
2140	BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.nonatomic", Parent: Func);
2141	BasicBlock *AtomicRedBlock =
2142	BasicBlock::Create(Context&: Module->getContext(), Name: "reduce.switch.atomic", Parent: Func);
2143	SwitchInst *Switch =
2144	Builder.CreateSwitch(V: ReduceCall, Dest: ContinuationBlock, /* NumCases */ 2);
2145	Switch->addCase(OnVal: Builder.getInt32(C: 1), Dest: NonAtomicRedBlock);
2146	Switch->addCase(OnVal: Builder.getInt32(C: 2), Dest: AtomicRedBlock);
2147
2148	// Populate the non-atomic reduction using the elementwise reduction function.
2149	// This loads the elements from the global and private variables and reduces
2150	// them before storing back the result to the global variable.
2151	Builder.SetInsertPoint(NonAtomicRedBlock);
2152	for (auto En : enumerate(First&: ReductionInfos)) {
2153	const ReductionInfo &RI = En.value();
2154	Type *ValueType = RI.ElementType;
2155	Value *RedValue = Builder.CreateLoad(Ty: ValueType, Ptr: RI.Variable,
2156	Name: "red.value." + Twine(En.index()));
2157	Value *PrivateRedValue =
2158	Builder.CreateLoad(Ty: ValueType, Ptr: RI.PrivateVariable,
2159	Name: "red.private.value." + Twine(En.index()));
2160	Value *Reduced;
2161	Builder.restoreIP(
2162	IP: RI.ReductionGen(Builder.saveIP(), RedValue, PrivateRedValue, Reduced));
2163	if (!Builder.GetInsertBlock())
2164	return InsertPointTy();
2165	Builder.CreateStore(Val: Reduced, Ptr: RI.Variable);
2166	}
2167	Function *EndReduceFunc = getOrCreateRuntimeFunctionPtr(
2168	FnID: IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
2169	: RuntimeFunction::OMPRTL___kmpc_end_reduce);
2170	Builder.CreateCall(Callee: EndReduceFunc, Args: {Ident, ThreadId, Lock});
2171	Builder.CreateBr(Dest: ContinuationBlock);
2172
2173	// Populate the atomic reduction using the atomic elementwise reduction
2174	// function. There are no loads/stores here because they will be happening
2175	// inside the atomic elementwise reduction.
2176	Builder.SetInsertPoint(AtomicRedBlock);
2177	if (CanGenerateAtomic) {
2178	for (const ReductionInfo &RI : ReductionInfos) {
2179	Builder.restoreIP(IP: RI.AtomicReductionGen(Builder.saveIP(), RI.ElementType,
2180	RI.Variable, RI.PrivateVariable));
2181	if (!Builder.GetInsertBlock())
2182	return InsertPointTy();
2183	}
2184	Builder.CreateBr(Dest: ContinuationBlock);
2185	} else {
2186	Builder.CreateUnreachable();
2187	}
2188
2189	// Populate the outlined reduction function using the elementwise reduction
2190	// function. Partial values are extracted from the type-erased array of
2191	// pointers to private variables.
2192	BasicBlock *ReductionFuncBlock =
2193	BasicBlock::Create(Context&: Module->getContext(), Name: "", Parent: ReductionFunc);
2194	Builder.SetInsertPoint(ReductionFuncBlock);
2195	Value *LHSArrayPtr = ReductionFunc->getArg(i: 0);
2196	Value *RHSArrayPtr = ReductionFunc->getArg(i: 1);
2197
2198	for (auto En : enumerate(First&: ReductionInfos)) {
2199	const ReductionInfo &RI = En.value();
2200	Value *LHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
2201	Ty: RedArrayTy, Ptr: LHSArrayPtr, Idx0: 0, Idx1: En.index());
2202	Value *LHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: LHSI8PtrPtr);
2203	Value *LHSPtr = Builder.CreateBitCast(V: LHSI8Ptr, DestTy: RI.Variable->getType());
2204	Value *LHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: LHSPtr);
2205	Value *RHSI8PtrPtr = Builder.CreateConstInBoundsGEP2_64(
2206	Ty: RedArrayTy, Ptr: RHSArrayPtr, Idx0: 0, Idx1: En.index());
2207	Value *RHSI8Ptr = Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: RHSI8PtrPtr);
2208	Value *RHSPtr =
2209	Builder.CreateBitCast(V: RHSI8Ptr, DestTy: RI.PrivateVariable->getType());
2210	Value *RHS = Builder.CreateLoad(Ty: RI.ElementType, Ptr: RHSPtr);
2211	Value *Reduced;
2212	Builder.restoreIP(IP: RI.ReductionGen(Builder.saveIP(), LHS, RHS, Reduced));
2213	if (!Builder.GetInsertBlock())
2214	return InsertPointTy();
2215	Builder.CreateStore(Val: Reduced, Ptr: LHSPtr);
2216	}
2217	Builder.CreateRetVoid();
2218
2219	Builder.SetInsertPoint(ContinuationBlock);
2220	return Builder.saveIP();
2221	}
2222
2223	OpenMPIRBuilder::InsertPointTy
2224	OpenMPIRBuilder::createMaster(const LocationDescription &Loc,
2225	BodyGenCallbackTy BodyGenCB,
2226	FinalizeCallbackTy FiniCB) {
2227
2228	if (!updateToLocation(Loc))
2229	return Loc.IP;
2230
2231	Directive OMPD = Directive::OMPD_master;
2232	uint32_t SrcLocStrSize;
2233	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2234	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2235	Value *ThreadId = getOrCreateThreadID(Ident);
2236	Value *Args[] = {Ident, ThreadId};
2237
2238	Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_master);
2239	Instruction *EntryCall = Builder.CreateCall(Callee: EntryRTLFn, Args);
2240
2241	Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_master);
2242	Instruction *ExitCall = Builder.CreateCall(Callee: ExitRTLFn, Args);
2243
2244	return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
2245	/*Conditional*/ true, /*hasFinalize*/ true);
2246	}
2247
2248	OpenMPIRBuilder::InsertPointTy
2249	OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
2250	BodyGenCallbackTy BodyGenCB,
2251	FinalizeCallbackTy FiniCB, Value *Filter) {
2252	if (!updateToLocation(Loc))
2253	return Loc.IP;
2254
2255	Directive OMPD = Directive::OMPD_masked;
2256	uint32_t SrcLocStrSize;
2257	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
2258	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2259	Value *ThreadId = getOrCreateThreadID(Ident);
2260	Value *Args[] = {Ident, ThreadId, Filter};
2261	Value *ArgsEnd[] = {Ident, ThreadId};
2262
2263	Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_masked);
2264	Instruction *EntryCall = Builder.CreateCall(Callee: EntryRTLFn, Args);
2265
2266	Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_masked);
2267	Instruction *ExitCall = Builder.CreateCall(Callee: ExitRTLFn, Args: ArgsEnd);
2268
2269	return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
2270	/*Conditional*/ true, /*hasFinalize*/ true);
2271	}
2272
2273	CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
2274	DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
2275	BasicBlock *PostInsertBefore, const Twine &Name) {
2276	Module *M = F->getParent();
2277	LLVMContext &Ctx = M->getContext();
2278	Type *IndVarTy = TripCount->getType();
2279
2280	// Create the basic block structure.
2281	BasicBlock *Preheader =
2282	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".preheader", Parent: F, InsertBefore: PreInsertBefore);
2283	BasicBlock *Header =
2284	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".header", Parent: F, InsertBefore: PreInsertBefore);
2285	BasicBlock *Cond =
2286	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".cond", Parent: F, InsertBefore: PreInsertBefore);
2287	BasicBlock *Body =
2288	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".body", Parent: F, InsertBefore: PreInsertBefore);
2289	BasicBlock *Latch =
2290	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".inc", Parent: F, InsertBefore: PostInsertBefore);
2291	BasicBlock *Exit =
2292	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".exit", Parent: F, InsertBefore: PostInsertBefore);
2293	BasicBlock *After =
2294	BasicBlock::Create(Context&: Ctx, Name: "omp_" + Name + ".after", Parent: F, InsertBefore: PostInsertBefore);
2295
2296	// Use specified DebugLoc for new instructions.
2297	Builder.SetCurrentDebugLocation(DL);
2298
2299	Builder.SetInsertPoint(Preheader);
2300	Builder.CreateBr(Dest: Header);
2301
2302	Builder.SetInsertPoint(Header);
2303	PHINode *IndVarPHI = Builder.CreatePHI(Ty: IndVarTy, NumReservedValues: 2, Name: "omp_" + Name + ".iv");
2304	IndVarPHI->addIncoming(V: ConstantInt::get(Ty: IndVarTy, V: 0), BB: Preheader);
2305	Builder.CreateBr(Dest: Cond);
2306
2307	Builder.SetInsertPoint(Cond);
2308	Value *Cmp =
2309	Builder.CreateICmpULT(LHS: IndVarPHI, RHS: TripCount, Name: "omp_" + Name + ".cmp");
2310	Builder.CreateCondBr(Cond: Cmp, True: Body, False: Exit);
2311
2312	Builder.SetInsertPoint(Body);
2313	Builder.CreateBr(Dest: Latch);
2314
2315	Builder.SetInsertPoint(Latch);
2316	Value *Next = Builder.CreateAdd(LHS: IndVarPHI, RHS: ConstantInt::get(Ty: IndVarTy, V: 1),
2317	Name: "omp_" + Name + ".next", /*HasNUW=*/true);
2318	Builder.CreateBr(Dest: Header);
2319	IndVarPHI->addIncoming(V: Next, BB: Latch);
2320
2321	Builder.SetInsertPoint(Exit);
2322	Builder.CreateBr(Dest: After);
2323
2324	// Remember and return the canonical control flow.
2325	LoopInfos.emplace_front();
2326	CanonicalLoopInfo *CL = &LoopInfos.front();
2327
2328	CL->Header = Header;
2329	CL->Cond = Cond;
2330	CL->Latch = Latch;
2331	CL->Exit = Exit;
2332
2333	#ifndef NDEBUG
2334	CL->assertOK();
2335	#endif
2336	return CL;
2337	}
2338
2339	CanonicalLoopInfo *
2340	OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
2341	LoopBodyGenCallbackTy BodyGenCB,
2342	Value *TripCount, const Twine &Name) {
2343	BasicBlock *BB = Loc.IP.getBlock();
2344	BasicBlock *NextBB = BB->getNextNode();
2345
2346	CanonicalLoopInfo *CL = createLoopSkeleton(DL: Loc.DL, TripCount, F: BB->getParent(),
2347	PreInsertBefore: NextBB, PostInsertBefore: NextBB, Name);
2348	BasicBlock *After = CL->getAfter();
2349
2350	// If location is not set, don't connect the loop.
2351	if (updateToLocation(Loc)) {
2352	// Split the loop at the insertion point: Branch to the preheader and move
2353	// every following instruction to after the loop (the After BB). Also, the
2354	// new successor is the loop's after block.
2355	spliceBB(Builder, New: After, /*CreateBranch=*/false);
2356	Builder.CreateBr(Dest: CL->getPreheader());
2357	}
2358
2359	// Emit the body content. We do it after connecting the loop to the CFG to
2360	// avoid that the callback encounters degenerate BBs.
2361	BodyGenCB(CL->getBodyIP(), CL->getIndVar());
2362
2363	#ifndef NDEBUG
2364	CL->assertOK();
2365	#endif
2366	return CL;
2367	}
2368
2369	CanonicalLoopInfo *OpenMPIRBuilder::createCanonicalLoop(
2370	const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
2371	Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
2372	InsertPointTy ComputeIP, const Twine &Name) {
2373
2374	// Consider the following difficulties (assuming 8-bit signed integers):
2375	// * Adding \p Step to the loop counter which passes \p Stop may overflow:
2376	// DO I = 1, 100, 50
2377	/// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
2378	// DO I = 100, 0, -128
2379
2380	// Start, Stop and Step must be of the same integer type.
2381	auto *IndVarTy = cast<IntegerType>(Val: Start->getType());
2382	assert(IndVarTy == Stop->getType() && "Stop type mismatch");
2383	assert(IndVarTy == Step->getType() && "Step type mismatch");
2384
2385	LocationDescription ComputeLoc =
2386	ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
2387	updateToLocation(Loc: ComputeLoc);
2388
2389	ConstantInt *Zero = ConstantInt::get(Ty: IndVarTy, V: 0);
2390	ConstantInt *One = ConstantInt::get(Ty: IndVarTy, V: 1);
2391
2392	// Like Step, but always positive.
2393	Value *Incr = Step;
2394
2395	// Distance between Start and Stop; always positive.
2396	Value *Span;
2397
2398	// Condition whether there are no iterations are executed at all, e.g. because
2399	// UB < LB.
2400	Value *ZeroCmp;
2401
2402	if (IsSigned) {
2403	// Ensure that increment is positive. If not, negate and invert LB and UB.
2404	Value *IsNeg = Builder.CreateICmpSLT(LHS: Step, RHS: Zero);
2405	Incr = Builder.CreateSelect(C: IsNeg, True: Builder.CreateNeg(V: Step), False: Step);
2406	Value *LB = Builder.CreateSelect(C: IsNeg, True: Stop, False: Start);
2407	Value *UB = Builder.CreateSelect(C: IsNeg, True: Start, False: Stop);
2408	Span = Builder.CreateSub(LHS: UB, RHS: LB, Name: "", HasNUW: false, HasNSW: true);
2409	ZeroCmp = Builder.CreateICmp(
2410	P: InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, LHS: UB, RHS: LB);
2411	} else {
2412	Span = Builder.CreateSub(LHS: Stop, RHS: Start, Name: "", HasNUW: true);
2413	ZeroCmp = Builder.CreateICmp(
2414	P: InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, LHS: Stop, RHS: Start);
2415	}
2416
2417	Value *CountIfLooping;
2418	if (InclusiveStop) {
2419	CountIfLooping = Builder.CreateAdd(LHS: Builder.CreateUDiv(LHS: Span, RHS: Incr), RHS: One);
2420	} else {
2421	// Avoid incrementing past stop since it could overflow.
2422	Value *CountIfTwo = Builder.CreateAdd(
2423	LHS: Builder.CreateUDiv(LHS: Builder.CreateSub(LHS: Span, RHS: One), RHS: Incr), RHS: One);
2424	Value *OneCmp = Builder.CreateICmp(P: CmpInst::ICMP_ULE, LHS: Span, RHS: Incr);
2425	CountIfLooping = Builder.CreateSelect(C: OneCmp, True: One, False: CountIfTwo);
2426	}
2427	Value *TripCount = Builder.CreateSelect(C: ZeroCmp, True: Zero, False: CountIfLooping,
2428	Name: "omp_" + Name + ".tripcount");
2429
2430	auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
2431	Builder.restoreIP(IP: CodeGenIP);
2432	Value *Span = Builder.CreateMul(LHS: IV, RHS: Step);
2433	Value *IndVar = Builder.CreateAdd(LHS: Span, RHS: Start);
2434	BodyGenCB(Builder.saveIP(), IndVar);
2435	};
2436	LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
2437	return createCanonicalLoop(Loc: LoopLoc, BodyGenCB: BodyGen, TripCount, Name);
2438	}
2439
2440	// Returns an LLVM function to call for initializing loop bounds using OpenMP
2441	// static scheduling depending on `type`. Only i32 and i64 are supported by the
2442	// runtime. Always interpret integers as unsigned similarly to
2443	// CanonicalLoopInfo.
2444	static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M,
2445	OpenMPIRBuilder &OMPBuilder) {
2446	unsigned Bitwidth = Ty->getIntegerBitWidth();
2447	if (Bitwidth == 32)
2448	return OMPBuilder.getOrCreateRuntimeFunction(
2449	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
2450	if (Bitwidth == 64)
2451	return OMPBuilder.getOrCreateRuntimeFunction(
2452	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
2453	llvm_unreachable("unknown OpenMP loop iterator bitwidth");
2454	}
2455
2456	OpenMPIRBuilder::InsertPointTy
2457	OpenMPIRBuilder::applyStaticWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI,
2458	InsertPointTy AllocaIP,
2459	bool NeedsBarrier) {
2460	assert(CLI->isValid() && "Requires a valid canonical loop");
2461	assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
2462	"Require dedicated allocate IP");
2463
2464	// Set up the source location value for OpenMP runtime.
2465	Builder.restoreIP(IP: CLI->getPreheaderIP());
2466	Builder.SetCurrentDebugLocation(DL);
2467
2468	uint32_t SrcLocStrSize;
2469	Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2470	Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2471
2472	// Declare useful OpenMP runtime functions.
2473	Value *IV = CLI->getIndVar();
2474	Type *IVTy = IV->getType();
2475	FunctionCallee StaticInit = getKmpcForStaticInitForType(Ty: IVTy, M, OMPBuilder&: *this);
2476	FunctionCallee StaticFini =
2477	getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
2478
2479	// Allocate space for computed loop bounds as expected by the "init" function.
2480	Builder.restoreIP(IP: AllocaIP);
2481	Type *I32Type = Type::getInt32Ty(C&: M.getContext());
2482	Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
2483	Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
2484	Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
2485	Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
2486
2487	// At the end of the preheader, prepare for calling the "init" function by
2488	// storing the current loop bounds into the allocated space. A canonical loop
2489	// always iterates from 0 to trip-count with step 1. Note that "init" expects
2490	// and produces an inclusive upper bound.
2491	Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2492	Constant *Zero = ConstantInt::get(Ty: IVTy, V: 0);
2493	Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
2494	Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
2495	Value *UpperBound = Builder.CreateSub(LHS: CLI->getTripCount(), RHS: One);
2496	Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
2497	Builder.CreateStore(Val: One, Ptr: PStride);
2498
2499	Value *ThreadNum = getOrCreateThreadID(Ident: SrcLoc);
2500
2501	Constant *SchedulingType = ConstantInt::get(
2502	Ty: I32Type, V: static_cast<int>(OMPScheduleType::UnorderedStatic));
2503
2504	// Call the "init" function and update the trip count of the loop with the
2505	// value it produced.
2506	Builder.CreateCall(Callee: StaticInit,
2507	Args: {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
2508	PUpperBound, PStride, One, Zero});
2509	Value *LowerBound = Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound);
2510	Value *InclusiveUpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound);
2511	Value *TripCountMinusOne = Builder.CreateSub(LHS: InclusiveUpperBound, RHS: LowerBound);
2512	Value *TripCount = Builder.CreateAdd(LHS: TripCountMinusOne, RHS: One);
2513	CLI->setTripCount(TripCount);
2514
2515	// Update all uses of the induction variable except the one in the condition
2516	// block that compares it with the actual upper bound, and the increment in
2517	// the latch block.
2518
2519	CLI->mapIndVar(Updater: [&](Instruction *OldIV) -> Value * {
2520	Builder.SetInsertPoint(TheBB: CLI->getBody(),
2521	IP: CLI->getBody()->getFirstInsertionPt());
2522	Builder.SetCurrentDebugLocation(DL);
2523	return Builder.CreateAdd(LHS: OldIV, RHS: LowerBound);
2524	});
2525
2526	// In the "exit" block, call the "fini" function.
2527	Builder.SetInsertPoint(TheBB: CLI->getExit(),
2528	IP: CLI->getExit()->getTerminator()->getIterator());
2529	Builder.CreateCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
2530
2531	// Add the barrier if requested.
2532	if (NeedsBarrier)
2533	createBarrier(LocationDescription(Builder.saveIP(), DL),
2534	omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
2535	/* CheckCancelFlag */ false);
2536
2537	InsertPointTy AfterIP = CLI->getAfterIP();
2538	CLI->invalidate();
2539
2540	return AfterIP;
2541	}
2542
2543	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(
2544	DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2545	bool NeedsBarrier, Value *ChunkSize) {
2546	assert(CLI->isValid() && "Requires a valid canonical loop");
2547	assert(ChunkSize && "Chunk size is required");
2548
2549	LLVMContext &Ctx = CLI->getFunction()->getContext();
2550	Value *IV = CLI->getIndVar();
2551	Value *OrigTripCount = CLI->getTripCount();
2552	Type *IVTy = IV->getType();
2553	assert(IVTy->getIntegerBitWidth() <= 64 &&
2554	"Max supported tripcount bitwidth is 64 bits");
2555	Type *InternalIVTy = IVTy->getIntegerBitWidth() <= 32 ? Type::getInt32Ty(C&: Ctx)
2556	: Type::getInt64Ty(C&: Ctx);
2557	Type *I32Type = Type::getInt32Ty(C&: M.getContext());
2558	Constant *Zero = ConstantInt::get(Ty: InternalIVTy, V: 0);
2559	Constant *One = ConstantInt::get(Ty: InternalIVTy, V: 1);
2560
2561	// Declare useful OpenMP runtime functions.
2562	FunctionCallee StaticInit =
2563	getKmpcForStaticInitForType(Ty: InternalIVTy, M, OMPBuilder&: *this);
2564	FunctionCallee StaticFini =
2565	getOrCreateRuntimeFunction(M, FnID: omp::OMPRTL___kmpc_for_static_fini);
2566
2567	// Allocate space for computed loop bounds as expected by the "init" function.
2568	Builder.restoreIP(IP: AllocaIP);
2569	Builder.SetCurrentDebugLocation(DL);
2570	Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
2571	Value *PLowerBound =
2572	Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.lowerbound");
2573	Value *PUpperBound =
2574	Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.upperbound");
2575	Value *PStride = Builder.CreateAlloca(Ty: InternalIVTy, ArraySize: nullptr, Name: "p.stride");
2576
2577	// Set up the source location value for the OpenMP runtime.
2578	Builder.restoreIP(IP: CLI->getPreheaderIP());
2579	Builder.SetCurrentDebugLocation(DL);
2580
2581	// TODO: Detect overflow in ubsan or max-out with current tripcount.
2582	Value *CastedChunkSize =
2583	Builder.CreateZExtOrTrunc(V: ChunkSize, DestTy: InternalIVTy, Name: "chunksize");
2584	Value *CastedTripCount =
2585	Builder.CreateZExt(V: OrigTripCount, DestTy: InternalIVTy, Name: "tripcount");
2586
2587	Constant *SchedulingType = ConstantInt::get(
2588	Ty: I32Type, V: static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
2589	Builder.CreateStore(Val: Zero, Ptr: PLowerBound);
2590	Value *OrigUpperBound = Builder.CreateSub(LHS: CastedTripCount, RHS: One);
2591	Builder.CreateStore(Val: OrigUpperBound, Ptr: PUpperBound);
2592	Builder.CreateStore(Val: One, Ptr: PStride);
2593
2594	// Call the "init" function and update the trip count of the loop with the
2595	// value it produced.
2596	uint32_t SrcLocStrSize;
2597	Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2598	Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2599	Value *ThreadNum = getOrCreateThreadID(Ident: SrcLoc);
2600	Builder.CreateCall(Callee: StaticInit,
2601	Args: {/*loc=*/SrcLoc, /*global_tid=*/ThreadNum,
2602	/*schedtype=*/SchedulingType, /*plastiter=*/PLastIter,
2603	/*plower=*/PLowerBound, /*pupper=*/PUpperBound,
2604	/*pstride=*/PStride, /*incr=*/One,
2605	/*chunk=*/CastedChunkSize});
2606
2607	// Load values written by the "init" function.
2608	Value *FirstChunkStart =
2609	Builder.CreateLoad(Ty: InternalIVTy, Ptr: PLowerBound, Name: "omp_firstchunk.lb");
2610	Value *FirstChunkStop =
2611	Builder.CreateLoad(Ty: InternalIVTy, Ptr: PUpperBound, Name: "omp_firstchunk.ub");
2612	Value *FirstChunkEnd = Builder.CreateAdd(LHS: FirstChunkStop, RHS: One);
2613	Value *ChunkRange =
2614	Builder.CreateSub(LHS: FirstChunkEnd, RHS: FirstChunkStart, Name: "omp_chunk.range");
2615	Value *NextChunkStride =
2616	Builder.CreateLoad(Ty: InternalIVTy, Ptr: PStride, Name: "omp_dispatch.stride");
2617
2618	// Create outer "dispatch" loop for enumerating the chunks.
2619	BasicBlock *DispatchEnter = splitBB(Builder, CreateBranch: true);
2620	Value *DispatchCounter;
2621	CanonicalLoopInfo *DispatchCLI = createCanonicalLoop(
2622	Loc: {Builder.saveIP(), DL},
2623	BodyGenCB: [&](InsertPointTy BodyIP, Value *Counter) { DispatchCounter = Counter; },
2624	Start: FirstChunkStart, Stop: CastedTripCount, Step: NextChunkStride,
2625	/*IsSigned=*/false, /*InclusiveStop=*/false, /*ComputeIP=*/{},
2626	Name: "dispatch");
2627
2628	// Remember the BasicBlocks of the dispatch loop we need, then invalidate to
2629	// not have to preserve the canonical invariant.
2630	BasicBlock *DispatchBody = DispatchCLI->getBody();
2631	BasicBlock *DispatchLatch = DispatchCLI->getLatch();
2632	BasicBlock *DispatchExit = DispatchCLI->getExit();
2633	BasicBlock *DispatchAfter = DispatchCLI->getAfter();
2634	DispatchCLI->invalidate();
2635
2636	// Rewire the original loop to become the chunk loop inside the dispatch loop.
2637	redirectTo(Source: DispatchAfter, Target: CLI->getAfter(), DL);
2638	redirectTo(Source: CLI->getExit(), Target: DispatchLatch, DL);
2639	redirectTo(Source: DispatchBody, Target: DispatchEnter, DL);
2640
2641	// Prepare the prolog of the chunk loop.
2642	Builder.restoreIP(IP: CLI->getPreheaderIP());
2643	Builder.SetCurrentDebugLocation(DL);
2644
2645	// Compute the number of iterations of the chunk loop.
2646	Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
2647	Value *ChunkEnd = Builder.CreateAdd(LHS: DispatchCounter, RHS: ChunkRange);
2648	Value *IsLastChunk =
2649	Builder.CreateICmpUGE(LHS: ChunkEnd, RHS: CastedTripCount, Name: "omp_chunk.is_last");
2650	Value *CountUntilOrigTripCount =
2651	Builder.CreateSub(LHS: CastedTripCount, RHS: DispatchCounter);
2652	Value *ChunkTripCount = Builder.CreateSelect(
2653	C: IsLastChunk, True: CountUntilOrigTripCount, False: ChunkRange, Name: "omp_chunk.tripcount");
2654	Value *BackcastedChunkTC =
2655	Builder.CreateTrunc(V: ChunkTripCount, DestTy: IVTy, Name: "omp_chunk.tripcount.trunc");
2656	CLI->setTripCount(BackcastedChunkTC);
2657
2658	// Update all uses of the induction variable except the one in the condition
2659	// block that compares it with the actual upper bound, and the increment in
2660	// the latch block.
2661	Value *BackcastedDispatchCounter =
2662	Builder.CreateTrunc(V: DispatchCounter, DestTy: IVTy, Name: "omp_dispatch.iv.trunc");
2663	CLI->mapIndVar(Updater: [&](Instruction *) -> Value * {
2664	Builder.restoreIP(IP: CLI->getBodyIP());
2665	return Builder.CreateAdd(LHS: IV, RHS: BackcastedDispatchCounter);
2666	});
2667
2668	// In the "exit" block, call the "fini" function.
2669	Builder.SetInsertPoint(TheBB: DispatchExit, IP: DispatchExit->getFirstInsertionPt());
2670	Builder.CreateCall(Callee: StaticFini, Args: {SrcLoc, ThreadNum});
2671
2672	// Add the barrier if requested.
2673	if (NeedsBarrier)
2674	createBarrier(LocationDescription(Builder.saveIP(), DL), OMPD_for,
2675	/*ForceSimpleCall=*/false, /*CheckCancelFlag=*/false);
2676
2677	#ifndef NDEBUG
2678	// Even though we currently do not support applying additional methods to it,
2679	// the chunk loop should remain a canonical loop.
2680	CLI->assertOK();
2681	#endif
2682
2683	return {DispatchAfter, DispatchAfter->getFirstInsertionPt()};
2684	}
2685
2686	// Returns an LLVM function to call for executing an OpenMP static worksharing
2687	// for loop depending on `type`. Only i32 and i64 are supported by the runtime.
2688	// Always interpret integers as unsigned similarly to CanonicalLoopInfo.
2689	static FunctionCallee
2690	getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder,
2691	WorksharingLoopType LoopType) {
2692	unsigned Bitwidth = Ty->getIntegerBitWidth();
2693	Module &M = OMPBuilder->M;
2694	switch (LoopType) {
2695	case WorksharingLoopType::ForStaticLoop:
2696	if (Bitwidth == 32)
2697	return OMPBuilder->getOrCreateRuntimeFunction(
2698	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
2699	if (Bitwidth == 64)
2700	return OMPBuilder->getOrCreateRuntimeFunction(
2701	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
2702	break;
2703	case WorksharingLoopType::DistributeStaticLoop:
2704	if (Bitwidth == 32)
2705	return OMPBuilder->getOrCreateRuntimeFunction(
2706	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
2707	if (Bitwidth == 64)
2708	return OMPBuilder->getOrCreateRuntimeFunction(
2709	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
2710	break;
2711	case WorksharingLoopType::DistributeForStaticLoop:
2712	if (Bitwidth == 32)
2713	return OMPBuilder->getOrCreateRuntimeFunction(
2714	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
2715	if (Bitwidth == 64)
2716	return OMPBuilder->getOrCreateRuntimeFunction(
2717	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
2718	break;
2719	}
2720	if (Bitwidth != 32 && Bitwidth != 64) {
2721	llvm_unreachable("Unknown OpenMP loop iterator bitwidth");
2722	}
2723	llvm_unreachable("Unknown type of OpenMP worksharing loop");
2724	}
2725
2726	// Inserts a call to proper OpenMP Device RTL function which handles
2727	// loop worksharing.
2728	static void createTargetLoopWorkshareCall(
2729	OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType,
2730	BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg,
2731	Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn) {
2732	Type *TripCountTy = TripCount->getType();
2733	Module &M = OMPBuilder->M;
2734	IRBuilder<> &Builder = OMPBuilder->Builder;
2735	FunctionCallee RTLFn =
2736	getKmpcForStaticLoopForType(Ty: TripCountTy, OMPBuilder, LoopType);
2737	SmallVector<Value *, 8> RealArgs;
2738	RealArgs.push_back(Elt: Ident);
2739	RealArgs.push_back(Elt: Builder.CreateBitCast(V: &LoopBodyFn, DestTy: ParallelTaskPtr));
2740	RealArgs.push_back(Elt: LoopBodyArg);
2741	RealArgs.push_back(Elt: TripCount);
2742	if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
2743	RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
2744	Builder.CreateCall(Callee: RTLFn, Args: RealArgs);
2745	return;
2746	}
2747	FunctionCallee RTLNumThreads = OMPBuilder->getOrCreateRuntimeFunction(
2748	M, FnID: omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
2749	Builder.restoreIP(IP: {InsertBlock, std::prev(x: InsertBlock->end())});
2750	Value *NumThreads = Builder.CreateCall(Callee: RTLNumThreads, Args: {});
2751
2752	RealArgs.push_back(
2753	Elt: Builder.CreateZExtOrTrunc(V: NumThreads, DestTy: TripCountTy, Name: "num.threads.cast"));
2754	RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
2755	if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
2756	RealArgs.push_back(Elt: ConstantInt::get(Ty: TripCountTy, V: 0));
2757	}
2758
2759	Builder.CreateCall(Callee: RTLFn, Args: RealArgs);
2760	}
2761
2762	static void
2763	workshareLoopTargetCallback(OpenMPIRBuilder *OMPIRBuilder,
2764	CanonicalLoopInfo *CLI, Value *Ident,
2765	Function &OutlinedFn, Type *ParallelTaskPtr,
2766	const SmallVector<Instruction *, 4> &ToBeDeleted,
2767	WorksharingLoopType LoopType) {
2768	IRBuilder<> &Builder = OMPIRBuilder->Builder;
2769	BasicBlock *Preheader = CLI->getPreheader();
2770	Value *TripCount = CLI->getTripCount();
2771
2772	// After loop body outling, the loop body contains only set up
2773	// of loop body argument structure and the call to the outlined
2774	// loop body function. Firstly, we need to move setup of loop body args
2775	// into loop preheader.
2776	Preheader->splice(ToIt: std::prev(x: Preheader->end()), FromBB: CLI->getBody(),
2777	FromBeginIt: CLI->getBody()->begin(), FromEndIt: std::prev(x: CLI->getBody()->end()));
2778
2779	// The next step is to remove the whole loop. We do not it need anymore.
2780	// That's why make an unconditional branch from loop preheader to loop
2781	// exit block
2782	Builder.restoreIP(IP: {Preheader, Preheader->end()});
2783	Preheader->getTerminator()->eraseFromParent();
2784	Builder.CreateBr(Dest: CLI->getExit());
2785
2786	// Delete dead loop blocks
2787	OpenMPIRBuilder::OutlineInfo CleanUpInfo;
2788	SmallPtrSet<BasicBlock *, 32> RegionBlockSet;
2789	SmallVector<BasicBlock *, 32> BlocksToBeRemoved;
2790	CleanUpInfo.EntryBB = CLI->getHeader();
2791	CleanUpInfo.ExitBB = CLI->getExit();
2792	CleanUpInfo.collectBlocks(BlockSet&: RegionBlockSet, BlockVector&: BlocksToBeRemoved);
2793	DeleteDeadBlocks(BBs: BlocksToBeRemoved);
2794
2795	// Find the instruction which corresponds to loop body argument structure
2796	// and remove the call to loop body function instruction.
2797	Value *LoopBodyArg;
2798	User *OutlinedFnUser = OutlinedFn.getUniqueUndroppableUser();
2799	assert(OutlinedFnUser &&
2800	"Expected unique undroppable user of outlined function");
2801	CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(Val: OutlinedFnUser);
2802	assert(OutlinedFnCallInstruction && "Expected outlined function call");
2803	assert((OutlinedFnCallInstruction->getParent() == Preheader) &&
2804	"Expected outlined function call to be located in loop preheader");
2805	// Check in case no argument structure has been passed.
2806	if (OutlinedFnCallInstruction->arg_size() > 1)
2807	LoopBodyArg = OutlinedFnCallInstruction->getArgOperand(i: 1);
2808	else
2809	LoopBodyArg = Constant::getNullValue(Ty: Builder.getPtrTy());
2810	OutlinedFnCallInstruction->eraseFromParent();
2811
2812	createTargetLoopWorkshareCall(OMPBuilder: OMPIRBuilder, LoopType, InsertBlock: Preheader, Ident,
2813	LoopBodyArg, ParallelTaskPtr, TripCount,
2814	LoopBodyFn&: OutlinedFn);
2815
2816	for (auto &ToBeDeletedItem : ToBeDeleted)
2817	ToBeDeletedItem->eraseFromParent();
2818	CLI->invalidate();
2819	}
2820
2821	OpenMPIRBuilder::InsertPointTy
2822	OpenMPIRBuilder::applyWorkshareLoopTarget(DebugLoc DL, CanonicalLoopInfo *CLI,
2823	InsertPointTy AllocaIP,
2824	WorksharingLoopType LoopType) {
2825	uint32_t SrcLocStrSize;
2826	Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
2827	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
2828
2829	OutlineInfo OI;
2830	OI.OuterAllocaBB = CLI->getPreheader();
2831	Function *OuterFn = CLI->getPreheader()->getParent();
2832
2833	// Instructions which need to be deleted at the end of code generation
2834	SmallVector<Instruction *, 4> ToBeDeleted;
2835
2836	OI.OuterAllocaBB = AllocaIP.getBlock();
2837
2838	// Mark the body loop as region which needs to be extracted
2839	OI.EntryBB = CLI->getBody();
2840	OI.ExitBB = CLI->getLatch()->splitBasicBlock(I: CLI->getLatch()->begin(),
2841	BBName: "omp.prelatch", Before: true);
2842
2843	// Prepare loop body for extraction
2844	Builder.restoreIP(IP: {CLI->getPreheader(), CLI->getPreheader()->begin()});
2845
2846	// Insert new loop counter variable which will be used only in loop
2847	// body.
2848	AllocaInst *NewLoopCnt = Builder.CreateAlloca(Ty: CLI->getIndVarType(), ArraySize: 0, Name: "");
2849	Instruction *NewLoopCntLoad =
2850	Builder.CreateLoad(Ty: CLI->getIndVarType(), Ptr: NewLoopCnt);
2851	// New loop counter instructions are redundant in the loop preheader when
2852	// code generation for workshare loop is finshed. That's why mark them as
2853	// ready for deletion.
2854	ToBeDeleted.push_back(Elt: NewLoopCntLoad);
2855	ToBeDeleted.push_back(Elt: NewLoopCnt);
2856
2857	// Analyse loop body region. Find all input variables which are used inside
2858	// loop body region.
2859	SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
2860	SmallVector<BasicBlock *, 32> Blocks;
2861	OI.collectBlocks(BlockSet&: ParallelRegionBlockSet, BlockVector&: Blocks);
2862	SmallVector<BasicBlock *, 32> BlocksT(ParallelRegionBlockSet.begin(),
2863	ParallelRegionBlockSet.end());
2864
2865	CodeExtractorAnalysisCache CEAC(*OuterFn);
2866	CodeExtractor Extractor(Blocks,
2867	/* DominatorTree */ nullptr,
2868	/* AggregateArgs */ true,
2869	/* BlockFrequencyInfo */ nullptr,
2870	/* BranchProbabilityInfo */ nullptr,
2871	/* AssumptionCache */ nullptr,
2872	/* AllowVarArgs */ true,
2873	/* AllowAlloca */ true,
2874	/* AllocationBlock */ CLI->getPreheader(),
2875	/* Suffix */ ".omp_wsloop",
2876	/* AggrArgsIn0AddrSpace */ true);
2877
2878	BasicBlock *CommonExit = nullptr;
2879	SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
2880
2881	// Find allocas outside the loop body region which are used inside loop
2882	// body
2883	Extractor.findAllocas(CEAC, SinkCands&: SinkingCands, HoistCands&: HoistingCands, ExitBlock&: CommonExit);
2884
2885	// We need to model loop body region as the function f(cnt, loop_arg).
2886	// That's why we replace loop induction variable by the new counter
2887	// which will be one of loop body function argument
2888	SmallVector<User *> Users(CLI->getIndVar()->user_begin(),
2889	CLI->getIndVar()->user_end());
2890	for (auto Use : Users) {
2891	if (Instruction *Inst = dyn_cast<Instruction>(Val: Use)) {
2892	if (ParallelRegionBlockSet.count(Ptr: Inst->getParent())) {
2893	Inst->replaceUsesOfWith(From: CLI->getIndVar(), To: NewLoopCntLoad);
2894	}
2895	}
2896	}
2897	// Make sure that loop counter variable is not merged into loop body
2898	// function argument structure and it is passed as separate variable
2899	OI.ExcludeArgsFromAggregate.push_back(Elt: NewLoopCntLoad);
2900
2901	// PostOutline CB is invoked when loop body function is outlined and
2902	// loop body is replaced by call to outlined function. We need to add
2903	// call to OpenMP device rtl inside loop preheader. OpenMP device rtl
2904	// function will handle loop control logic.
2905	//
2906	OI.PostOutlineCB = [=, ToBeDeletedVec =
2907	std::move(ToBeDeleted)](Function &OutlinedFn) {
2908	workshareLoopTargetCallback(OMPIRBuilder: this, CLI, Ident, OutlinedFn, ParallelTaskPtr,
2909	ToBeDeleted: ToBeDeletedVec, LoopType);
2910	};
2911	addOutlineInfo(OI: std::move(OI));
2912	return CLI->getAfterIP();
2913	}
2914
2915	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyWorkshareLoop(
2916	DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
2917	bool NeedsBarrier, omp::ScheduleKind SchedKind, Value *ChunkSize,
2918	bool HasSimdModifier, bool HasMonotonicModifier,
2919	bool HasNonmonotonicModifier, bool HasOrderedClause,
2920	WorksharingLoopType LoopType) {
2921	if (Config.isTargetDevice())
2922	return applyWorkshareLoopTarget(DL, CLI, AllocaIP, LoopType);
2923	OMPScheduleType EffectiveScheduleType = computeOpenMPScheduleType(
2924	SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
2925	HasNonmonotonicModifier, HasOrderedClause);
2926
2927	bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
2928	OMPScheduleType::ModifierOrdered;
2929	switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
2930	case OMPScheduleType::BaseStatic:
2931	assert(!ChunkSize && "No chunk size with static-chunked schedule");
2932	if (IsOrdered)
2933	return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
2934	NeedsBarrier, Chunk: ChunkSize);
2935	// FIXME: Monotonicity ignored?
2936	return applyStaticWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier);
2937
2938	case OMPScheduleType::BaseStaticChunked:
2939	if (IsOrdered)
2940	return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
2941	NeedsBarrier, Chunk: ChunkSize);
2942	// FIXME: Monotonicity ignored?
2943	return applyStaticChunkedWorkshareLoop(DL, CLI, AllocaIP, NeedsBarrier,
2944	ChunkSize);
2945
2946	case OMPScheduleType::BaseRuntime:
2947	case OMPScheduleType::BaseAuto:
2948	case OMPScheduleType::BaseGreedy:
2949	case OMPScheduleType::BaseBalanced:
2950	case OMPScheduleType::BaseSteal:
2951	case OMPScheduleType::BaseGuidedSimd:
2952	case OMPScheduleType::BaseRuntimeSimd:
2953	assert(!ChunkSize &&
2954	"schedule type does not support user-defined chunk sizes");
2955	[[fallthrough]];
2956	case OMPScheduleType::BaseDynamicChunked:
2957	case OMPScheduleType::BaseGuidedChunked:
2958	case OMPScheduleType::BaseGuidedIterativeChunked:
2959	case OMPScheduleType::BaseGuidedAnalyticalChunked:
2960	case OMPScheduleType::BaseStaticBalancedChunked:
2961	return applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType: EffectiveScheduleType,
2962	NeedsBarrier, Chunk: ChunkSize);
2963
2964	default:
2965	llvm_unreachable("Unknown/unimplemented schedule kind");
2966	}
2967	}
2968
2969	/// Returns an LLVM function to call for initializing loop bounds using OpenMP
2970	/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2971	/// the runtime. Always interpret integers as unsigned similarly to
2972	/// CanonicalLoopInfo.
2973	static FunctionCallee
2974	getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2975	unsigned Bitwidth = Ty->getIntegerBitWidth();
2976	if (Bitwidth == 32)
2977	return OMPBuilder.getOrCreateRuntimeFunction(
2978	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
2979	if (Bitwidth == 64)
2980	return OMPBuilder.getOrCreateRuntimeFunction(
2981	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
2982	llvm_unreachable("unknown OpenMP loop iterator bitwidth");
2983	}
2984
2985	/// Returns an LLVM function to call for updating the next loop using OpenMP
2986	/// dynamic scheduling depending on `type`. Only i32 and i64 are supported by
2987	/// the runtime. Always interpret integers as unsigned similarly to
2988	/// CanonicalLoopInfo.
2989	static FunctionCallee
2990	getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
2991	unsigned Bitwidth = Ty->getIntegerBitWidth();
2992	if (Bitwidth == 32)
2993	return OMPBuilder.getOrCreateRuntimeFunction(
2994	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
2995	if (Bitwidth == 64)
2996	return OMPBuilder.getOrCreateRuntimeFunction(
2997	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
2998	llvm_unreachable("unknown OpenMP loop iterator bitwidth");
2999	}
3000
3001	/// Returns an LLVM function to call for finalizing the dynamic loop using
3002	/// depending on `type`. Only i32 and i64 are supported by the runtime. Always
3003	/// interpret integers as unsigned similarly to CanonicalLoopInfo.
3004	static FunctionCallee
3005	getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder) {
3006	unsigned Bitwidth = Ty->getIntegerBitWidth();
3007	if (Bitwidth == 32)
3008	return OMPBuilder.getOrCreateRuntimeFunction(
3009	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
3010	if (Bitwidth == 64)
3011	return OMPBuilder.getOrCreateRuntimeFunction(
3012	M, FnID: omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
3013	llvm_unreachable("unknown OpenMP loop iterator bitwidth");
3014	}
3015
3016	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::applyDynamicWorkshareLoop(
3017	DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP,
3018	OMPScheduleType SchedType, bool NeedsBarrier, Value *Chunk) {
3019	assert(CLI->isValid() && "Requires a valid canonical loop");
3020	assert(!isConflictIP(AllocaIP, CLI->getPreheaderIP()) &&
3021	"Require dedicated allocate IP");
3022	assert(isValidWorkshareLoopScheduleType(SchedType) &&
3023	"Require valid schedule type");
3024
3025	bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
3026	OMPScheduleType::ModifierOrdered;
3027
3028	// Set up the source location value for OpenMP runtime.
3029	Builder.SetCurrentDebugLocation(DL);
3030
3031	uint32_t SrcLocStrSize;
3032	Constant *SrcLocStr = getOrCreateSrcLocStr(DL, SrcLocStrSize);
3033	Value *SrcLoc = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3034
3035	// Declare useful OpenMP runtime functions.
3036	Value *IV = CLI->getIndVar();
3037	Type *IVTy = IV->getType();
3038	FunctionCallee DynamicInit = getKmpcForDynamicInitForType(Ty: IVTy, M, OMPBuilder&: *this);
3039	FunctionCallee DynamicNext = getKmpcForDynamicNextForType(Ty: IVTy, M, OMPBuilder&: *this);
3040
3041	// Allocate space for computed loop bounds as expected by the "init" function.
3042	Builder.restoreIP(IP: AllocaIP);
3043	Type *I32Type = Type::getInt32Ty(C&: M.getContext());
3044	Value *PLastIter = Builder.CreateAlloca(Ty: I32Type, ArraySize: nullptr, Name: "p.lastiter");
3045	Value *PLowerBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.lowerbound");
3046	Value *PUpperBound = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.upperbound");
3047	Value *PStride = Builder.CreateAlloca(Ty: IVTy, ArraySize: nullptr, Name: "p.stride");
3048
3049	// At the end of the preheader, prepare for calling the "init" function by
3050	// storing the current loop bounds into the allocated space. A canonical loop
3051	// always iterates from 0 to trip-count with step 1. Note that "init" expects
3052	// and produces an inclusive upper bound.
3053	BasicBlock *PreHeader = CLI->getPreheader();
3054	Builder.SetInsertPoint(PreHeader->getTerminator());
3055	Constant *One = ConstantInt::get(Ty: IVTy, V: 1);
3056	Builder.CreateStore(Val: One, Ptr: PLowerBound);
3057	Value *UpperBound = CLI->getTripCount();
3058	Builder.CreateStore(Val: UpperBound, Ptr: PUpperBound);
3059	Builder.CreateStore(Val: One, Ptr: PStride);
3060
3061	BasicBlock *Header = CLI->getHeader();
3062	BasicBlock *Exit = CLI->getExit();
3063	BasicBlock *Cond = CLI->getCond();
3064	BasicBlock *Latch = CLI->getLatch();
3065	InsertPointTy AfterIP = CLI->getAfterIP();
3066
3067	// The CLI will be "broken" in the code below, as the loop is no longer
3068	// a valid canonical loop.
3069
3070	if (!Chunk)
3071	Chunk = One;
3072
3073	Value *ThreadNum = getOrCreateThreadID(Ident: SrcLoc);
3074
3075	Constant *SchedulingType =
3076	ConstantInt::get(Ty: I32Type, V: static_cast<int>(SchedType));
3077
3078	// Call the "init" function.
3079	Builder.CreateCall(Callee: DynamicInit,
3080	Args: {SrcLoc, ThreadNum, SchedulingType, /* LowerBound */ One,
3081	UpperBound, /* step */ One, Chunk});
3082
3083	// An outer loop around the existing one.
3084	BasicBlock *OuterCond = BasicBlock::Create(
3085	Context&: PreHeader->getContext(), Name: Twine(PreHeader->getName()) + ".outer.cond",
3086	Parent: PreHeader->getParent());
3087	// This needs to be 32-bit always, so can't use the IVTy Zero above.
3088	Builder.SetInsertPoint(TheBB: OuterCond, IP: OuterCond->getFirstInsertionPt());
3089	Value *Res =
3090	Builder.CreateCall(Callee: DynamicNext, Args: {SrcLoc, ThreadNum, PLastIter,
3091	PLowerBound, PUpperBound, PStride});
3092	Constant *Zero32 = ConstantInt::get(Ty: I32Type, V: 0);
3093	Value *MoreWork = Builder.CreateCmp(Pred: CmpInst::ICMP_NE, LHS: Res, RHS: Zero32);
3094	Value *LowerBound =
3095	Builder.CreateSub(LHS: Builder.CreateLoad(Ty: IVTy, Ptr: PLowerBound), RHS: One, Name: "lb");
3096	Builder.CreateCondBr(Cond: MoreWork, True: Header, False: Exit);
3097
3098	// Change PHI-node in loop header to use outer cond rather than preheader,
3099	// and set IV to the LowerBound.
3100	Instruction *Phi = &Header->front();
3101	auto *PI = cast<PHINode>(Val: Phi);
3102	PI->setIncomingBlock(i: 0, BB: OuterCond);
3103	PI->setIncomingValue(i: 0, V: LowerBound);
3104
3105	// Then set the pre-header to jump to the OuterCond
3106	Instruction *Term = PreHeader->getTerminator();
3107	auto *Br = cast<BranchInst>(Val: Term);
3108	Br->setSuccessor(idx: 0, NewSucc: OuterCond);
3109
3110	// Modify the inner condition:
3111	// * Use the UpperBound returned from the DynamicNext call.
3112	// * jump to the loop outer loop when done with one of the inner loops.
3113	Builder.SetInsertPoint(TheBB: Cond, IP: Cond->getFirstInsertionPt());
3114	UpperBound = Builder.CreateLoad(Ty: IVTy, Ptr: PUpperBound, Name: "ub");
3115	Instruction *Comp = &*Builder.GetInsertPoint();
3116	auto *CI = cast<CmpInst>(Val: Comp);
3117	CI->setOperand(i_nocapture: 1, Val_nocapture: UpperBound);
3118	// Redirect the inner exit to branch to outer condition.
3119	Instruction *Branch = &Cond->back();
3120	auto *BI = cast<BranchInst>(Val: Branch);
3121	assert(BI->getSuccessor(1) == Exit);
3122	BI->setSuccessor(idx: 1, NewSucc: OuterCond);
3123
3124	// Call the "fini" function if "ordered" is present in wsloop directive.
3125	if (Ordered) {
3126	Builder.SetInsertPoint(&Latch->back());
3127	FunctionCallee DynamicFini = getKmpcForDynamicFiniForType(Ty: IVTy, M, OMPBuilder&: *this);
3128	Builder.CreateCall(Callee: DynamicFini, Args: {SrcLoc, ThreadNum});
3129	}
3130
3131	// Add the barrier if requested.
3132	if (NeedsBarrier) {
3133	Builder.SetInsertPoint(&Exit->back());
3134	createBarrier(LocationDescription(Builder.saveIP(), DL),
3135	omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
3136	/* CheckCancelFlag */ false);
3137	}
3138
3139	CLI->invalidate();
3140	return AfterIP;
3141	}
3142
3143	/// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
3144	/// after this \p OldTarget will be orphaned.
3145	static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
3146	BasicBlock *NewTarget, DebugLoc DL) {
3147	for (BasicBlock *Pred : make_early_inc_range(Range: predecessors(BB: OldTarget)))
3148	redirectTo(Source: Pred, Target: NewTarget, DL);
3149	}
3150
3151	/// Determine which blocks in \p BBs are reachable from outside and remove the
3152	/// ones that are not reachable from the function.
3153	static void removeUnusedBlocksFromParent(ArrayRef<BasicBlock *> BBs) {
3154	SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
3155	auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
3156	for (Use &U : BB->uses()) {
3157	auto *UseInst = dyn_cast<Instruction>(Val: U.getUser());
3158	if (!UseInst)
3159	continue;
3160	if (BBsToErase.count(Ptr: UseInst->getParent()))
3161	continue;
3162	return true;
3163	}
3164	return false;
3165	};
3166
3167	while (true) {
3168	bool Changed = false;
3169	for (BasicBlock *BB : make_early_inc_range(Range&: BBsToErase)) {
3170	if (HasRemainingUses(BB)) {
3171	BBsToErase.erase(Ptr: BB);
3172	Changed = true;
3173	}
3174	}
3175	if (!Changed)
3176	break;
3177	}
3178
3179	SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
3180	DeleteDeadBlocks(BBs: BBVec);
3181	}
3182
3183	CanonicalLoopInfo *
3184	OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
3185	InsertPointTy ComputeIP) {
3186	assert(Loops.size() >= 1 && "At least one loop required");
3187	size_t NumLoops = Loops.size();
3188
3189	// Nothing to do if there is already just one loop.
3190	if (NumLoops == 1)
3191	return Loops.front();
3192
3193	CanonicalLoopInfo *Outermost = Loops.front();
3194	CanonicalLoopInfo *Innermost = Loops.back();
3195	BasicBlock *OrigPreheader = Outermost->getPreheader();
3196	BasicBlock *OrigAfter = Outermost->getAfter();
3197	Function *F = OrigPreheader->getParent();
3198
3199	// Loop control blocks that may become orphaned later.
3200	SmallVector<BasicBlock *, 12> OldControlBBs;
3201	OldControlBBs.reserve(N: 6 * Loops.size());
3202	for (CanonicalLoopInfo *Loop : Loops)
3203	Loop->collectControlBlocks(BBs&: OldControlBBs);
3204
3205	// Setup the IRBuilder for inserting the trip count computation.
3206	Builder.SetCurrentDebugLocation(DL);
3207	if (ComputeIP.isSet())
3208	Builder.restoreIP(IP: ComputeIP);
3209	else
3210	Builder.restoreIP(IP: Outermost->getPreheaderIP());
3211
3212	// Derive the collapsed' loop trip count.
3213	// TODO: Find common/largest indvar type.
3214	Value *CollapsedTripCount = nullptr;
3215	for (CanonicalLoopInfo *L : Loops) {
3216	assert(L->isValid() &&
3217	"All loops to collapse must be valid canonical loops");
3218	Value *OrigTripCount = L->getTripCount();
3219	if (!CollapsedTripCount) {
3220	CollapsedTripCount = OrigTripCount;
3221	continue;
3222	}
3223
3224	// TODO: Enable UndefinedSanitizer to diagnose an overflow here.
3225	CollapsedTripCount = Builder.CreateMul(LHS: CollapsedTripCount, RHS: OrigTripCount,
3226	Name: {}, /*HasNUW=*/true);
3227	}
3228
3229	// Create the collapsed loop control flow.
3230	CanonicalLoopInfo *Result =
3231	createLoopSkeleton(DL, TripCount: CollapsedTripCount, F,
3232	PreInsertBefore: OrigPreheader->getNextNode(), PostInsertBefore: OrigAfter, Name: "collapsed");
3233
3234	// Build the collapsed loop body code.
3235	// Start with deriving the input loop induction variables from the collapsed
3236	// one, using a divmod scheme. To preserve the original loops' order, the
3237	// innermost loop use the least significant bits.
3238	Builder.restoreIP(IP: Result->getBodyIP());
3239
3240	Value *Leftover = Result->getIndVar();
3241	SmallVector<Value *> NewIndVars;
3242	NewIndVars.resize(N: NumLoops);
3243	for (int i = NumLoops - 1; i >= 1; --i) {
3244	Value *OrigTripCount = Loops[i]->getTripCount();
3245
3246	Value *NewIndVar = Builder.CreateURem(LHS: Leftover, RHS: OrigTripCount);
3247	NewIndVars[i] = NewIndVar;
3248
3249	Leftover = Builder.CreateUDiv(LHS: Leftover, RHS: OrigTripCount);
3250	}
3251	// Outermost loop gets all the remaining bits.
3252	NewIndVars[0] = Leftover;
3253
3254	// Construct the loop body control flow.
3255	// We progressively construct the branch structure following in direction of
3256	// the control flow, from the leading in-between code, the loop nest body, the
3257	// trailing in-between code, and rejoining the collapsed loop's latch.
3258	// ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
3259	// the ContinueBlock is set, continue with that block. If ContinuePred, use
3260	// its predecessors as sources.
3261	BasicBlock *ContinueBlock = Result->getBody();
3262	BasicBlock *ContinuePred = nullptr;
3263	auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
3264	BasicBlock *NextSrc) {
3265	if (ContinueBlock)
3266	redirectTo(Source: ContinueBlock, Target: Dest, DL);
3267	else
3268	redirectAllPredecessorsTo(OldTarget: ContinuePred, NewTarget: Dest, DL);
3269
3270	ContinueBlock = nullptr;
3271	ContinuePred = NextSrc;
3272	};
3273
3274	// The code before the nested loop of each level.
3275	// Because we are sinking it into the nest, it will be executed more often
3276	// that the original loop. More sophisticated schemes could keep track of what
3277	// the in-between code is and instantiate it only once per thread.
3278	for (size_t i = 0; i < NumLoops - 1; ++i)
3279	ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
3280
3281	// Connect the loop nest body.
3282	ContinueWith(Innermost->getBody(), Innermost->getLatch());
3283
3284	// The code after the nested loop at each level.
3285	for (size_t i = NumLoops - 1; i > 0; --i)
3286	ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
3287
3288	// Connect the finished loop to the collapsed loop latch.
3289	ContinueWith(Result->getLatch(), nullptr);
3290
3291	// Replace the input loops with the new collapsed loop.
3292	redirectTo(Source: Outermost->getPreheader(), Target: Result->getPreheader(), DL);
3293	redirectTo(Source: Result->getAfter(), Target: Outermost->getAfter(), DL);
3294
3295	// Replace the input loop indvars with the derived ones.
3296	for (size_t i = 0; i < NumLoops; ++i)
3297	Loops[i]->getIndVar()->replaceAllUsesWith(V: NewIndVars[i]);
3298
3299	// Remove unused parts of the input loops.
3300	removeUnusedBlocksFromParent(BBs: OldControlBBs);
3301
3302	for (CanonicalLoopInfo *L : Loops)
3303	L->invalidate();
3304
3305	#ifndef NDEBUG
3306	Result->assertOK();
3307	#endif
3308	return Result;
3309	}
3310
3311	std::vector<CanonicalLoopInfo *>
3312	OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
3313	ArrayRef<Value *> TileSizes) {
3314	assert(TileSizes.size() == Loops.size() &&
3315	"Must pass as many tile sizes as there are loops");
3316	int NumLoops = Loops.size();
3317	assert(NumLoops >= 1 && "At least one loop to tile required");
3318
3319	CanonicalLoopInfo *OutermostLoop = Loops.front();
3320	CanonicalLoopInfo *InnermostLoop = Loops.back();
3321	Function *F = OutermostLoop->getBody()->getParent();
3322	BasicBlock *InnerEnter = InnermostLoop->getBody();
3323	BasicBlock *InnerLatch = InnermostLoop->getLatch();
3324
3325	// Loop control blocks that may become orphaned later.
3326	SmallVector<BasicBlock *, 12> OldControlBBs;
3327	OldControlBBs.reserve(N: 6 * Loops.size());
3328	for (CanonicalLoopInfo *Loop : Loops)
3329	Loop->collectControlBlocks(BBs&: OldControlBBs);
3330
3331	// Collect original trip counts and induction variable to be accessible by
3332	// index. Also, the structure of the original loops is not preserved during
3333	// the construction of the tiled loops, so do it before we scavenge the BBs of
3334	// any original CanonicalLoopInfo.
3335	SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
3336	for (CanonicalLoopInfo *L : Loops) {
3337	assert(L->isValid() && "All input loops must be valid canonical loops");
3338	OrigTripCounts.push_back(Elt: L->getTripCount());
3339	OrigIndVars.push_back(Elt: L->getIndVar());
3340	}
3341
3342	// Collect the code between loop headers. These may contain SSA definitions
3343	// that are used in the loop nest body. To be usable with in the innermost
3344	// body, these BasicBlocks will be sunk into the loop nest body. That is,
3345	// these instructions may be executed more often than before the tiling.
3346	// TODO: It would be sufficient to only sink them into body of the
3347	// corresponding tile loop.
3348	SmallVector<std::pair<BasicBlock *, BasicBlock *>, 4> InbetweenCode;
3349	for (int i = 0; i < NumLoops - 1; ++i) {
3350	CanonicalLoopInfo *Surrounding = Loops[i];
3351	CanonicalLoopInfo *Nested = Loops[i + 1];
3352
3353	BasicBlock *EnterBB = Surrounding->getBody();
3354	BasicBlock *ExitBB = Nested->getHeader();
3355	InbetweenCode.emplace_back(Args&: EnterBB, Args&: ExitBB);
3356	}
3357
3358	// Compute the trip counts of the floor loops.
3359	Builder.SetCurrentDebugLocation(DL);
3360	Builder.restoreIP(IP: OutermostLoop->getPreheaderIP());
3361	SmallVector<Value *, 4> FloorCount, FloorRems;
3362	for (int i = 0; i < NumLoops; ++i) {
3363	Value *TileSize = TileSizes[i];
3364	Value *OrigTripCount = OrigTripCounts[i];
3365	Type *IVType = OrigTripCount->getType();
3366
3367	Value *FloorTripCount = Builder.CreateUDiv(LHS: OrigTripCount, RHS: TileSize);
3368	Value *FloorTripRem = Builder.CreateURem(LHS: OrigTripCount, RHS: TileSize);
3369
3370	// 0 if tripcount divides the tilesize, 1 otherwise.
3371	// 1 means we need an additional iteration for a partial tile.
3372	//
3373	// Unfortunately we cannot just use the roundup-formula
3374	// (tripcount + tilesize - 1)/tilesize
3375	// because the summation might overflow. We do not want introduce undefined
3376	// behavior when the untiled loop nest did not.
3377	Value *FloorTripOverflow =
3378	Builder.CreateICmpNE(LHS: FloorTripRem, RHS: ConstantInt::get(Ty: IVType, V: 0));
3379
3380	FloorTripOverflow = Builder.CreateZExt(V: FloorTripOverflow, DestTy: IVType);
3381	FloorTripCount =
3382	Builder.CreateAdd(LHS: FloorTripCount, RHS: FloorTripOverflow,
3383	Name: "omp_floor" + Twine(i) + ".tripcount", HasNUW: true);
3384
3385	// Remember some values for later use.
3386	FloorCount.push_back(Elt: FloorTripCount);
3387	FloorRems.push_back(Elt: FloorTripRem);
3388	}
3389
3390	// Generate the new loop nest, from the outermost to the innermost.
3391	std::vector<CanonicalLoopInfo *> Result;
3392	Result.reserve(n: NumLoops * 2);
3393
3394	// The basic block of the surrounding loop that enters the nest generated
3395	// loop.
3396	BasicBlock *Enter = OutermostLoop->getPreheader();
3397
3398	// The basic block of the surrounding loop where the inner code should
3399	// continue.
3400	BasicBlock *Continue = OutermostLoop->getAfter();
3401
3402	// Where the next loop basic block should be inserted.
3403	BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
3404
3405	auto EmbeddNewLoop =
3406	[this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
3407	Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
3408	CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
3409	DL, TripCount, F, PreInsertBefore: InnerEnter, PostInsertBefore: OutroInsertBefore, Name);
3410	redirectTo(Source: Enter, Target: EmbeddedLoop->getPreheader(), DL);
3411	redirectTo(Source: EmbeddedLoop->getAfter(), Target: Continue, DL);
3412
3413	// Setup the position where the next embedded loop connects to this loop.
3414	Enter = EmbeddedLoop->getBody();
3415	Continue = EmbeddedLoop->getLatch();
3416	OutroInsertBefore = EmbeddedLoop->getLatch();
3417	return EmbeddedLoop;
3418	};
3419
3420	auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
3421	const Twine &NameBase) {
3422	for (auto P : enumerate(First&: TripCounts)) {
3423	CanonicalLoopInfo *EmbeddedLoop =
3424	EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
3425	Result.push_back(x: EmbeddedLoop);
3426	}
3427	};
3428
3429	EmbeddNewLoops(FloorCount, "floor");
3430
3431	// Within the innermost floor loop, emit the code that computes the tile
3432	// sizes.
3433	Builder.SetInsertPoint(Enter->getTerminator());
3434	SmallVector<Value *, 4> TileCounts;
3435	for (int i = 0; i < NumLoops; ++i) {
3436	CanonicalLoopInfo *FloorLoop = Result[i];
3437	Value *TileSize = TileSizes[i];
3438
3439	Value *FloorIsEpilogue =
3440	Builder.CreateICmpEQ(LHS: FloorLoop->getIndVar(), RHS: FloorCount[i]);
3441	Value *TileTripCount =
3442	Builder.CreateSelect(C: FloorIsEpilogue, True: FloorRems[i], False: TileSize);
3443
3444	TileCounts.push_back(Elt: TileTripCount);
3445	}
3446
3447	// Create the tile loops.
3448	EmbeddNewLoops(TileCounts, "tile");
3449
3450	// Insert the inbetween code into the body.
3451	BasicBlock *BodyEnter = Enter;
3452	BasicBlock *BodyEntered = nullptr;
3453	for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
3454	BasicBlock *EnterBB = P.first;
3455	BasicBlock *ExitBB = P.second;
3456
3457	if (BodyEnter)
3458	redirectTo(Source: BodyEnter, Target: EnterBB, DL);
3459	else
3460	redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: EnterBB, DL);
3461
3462	BodyEnter = nullptr;
3463	BodyEntered = ExitBB;
3464	}
3465
3466	// Append the original loop nest body into the generated loop nest body.
3467	if (BodyEnter)
3468	redirectTo(Source: BodyEnter, Target: InnerEnter, DL);
3469	else
3470	redirectAllPredecessorsTo(OldTarget: BodyEntered, NewTarget: InnerEnter, DL);
3471	redirectAllPredecessorsTo(OldTarget: InnerLatch, NewTarget: Continue, DL);
3472
3473	// Replace the original induction variable with an induction variable computed
3474	// from the tile and floor induction variables.
3475	Builder.restoreIP(IP: Result.back()->getBodyIP());
3476	for (int i = 0; i < NumLoops; ++i) {
3477	CanonicalLoopInfo *FloorLoop = Result[i];
3478	CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
3479	Value *OrigIndVar = OrigIndVars[i];
3480	Value *Size = TileSizes[i];
3481
3482	Value *Scale =
3483	Builder.CreateMul(LHS: Size, RHS: FloorLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
3484	Value *Shift =
3485	Builder.CreateAdd(LHS: Scale, RHS: TileLoop->getIndVar(), Name: {}, /*HasNUW=*/true);
3486	OrigIndVar->replaceAllUsesWith(V: Shift);
3487	}
3488
3489	// Remove unused parts of the original loops.
3490	removeUnusedBlocksFromParent(BBs: OldControlBBs);
3491
3492	for (CanonicalLoopInfo *L : Loops)
3493	L->invalidate();
3494
3495	#ifndef NDEBUG
3496	for (CanonicalLoopInfo *GenL : Result)
3497	GenL->assertOK();
3498	#endif
3499	return Result;
3500	}
3501
3502	/// Attach metadata \p Properties to the basic block described by \p BB. If the
3503	/// basic block already has metadata, the basic block properties are appended.
3504	static void addBasicBlockMetadata(BasicBlock *BB,
3505	ArrayRef<Metadata *> Properties) {
3506	// Nothing to do if no property to attach.
3507	if (Properties.empty())
3508	return;
3509
3510	LLVMContext &Ctx = BB->getContext();
3511	SmallVector<Metadata *> NewProperties;
3512	NewProperties.push_back(Elt: nullptr);
3513
3514	// If the basic block already has metadata, prepend it to the new metadata.
3515	MDNode *Existing = BB->getTerminator()->getMetadata(KindID: LLVMContext::MD_loop);
3516	if (Existing)
3517	append_range(C&: NewProperties, R: drop_begin(RangeOrContainer: Existing->operands(), N: 1));
3518
3519	append_range(C&: NewProperties, R&: Properties);
3520	MDNode *BasicBlockID = MDNode::getDistinct(Context&: Ctx, MDs: NewProperties);
3521	BasicBlockID->replaceOperandWith(I: 0, New: BasicBlockID);
3522
3523	BB->getTerminator()->setMetadata(KindID: LLVMContext::MD_loop, Node: BasicBlockID);
3524	}
3525
3526	/// Attach loop metadata \p Properties to the loop described by \p Loop. If the
3527	/// loop already has metadata, the loop properties are appended.
3528	static void addLoopMetadata(CanonicalLoopInfo *Loop,
3529	ArrayRef<Metadata *> Properties) {
3530	assert(Loop->isValid() && "Expecting a valid CanonicalLoopInfo");
3531
3532	// Attach metadata to the loop's latch
3533	BasicBlock *Latch = Loop->getLatch();
3534	assert(Latch && "A valid CanonicalLoopInfo must have a unique latch");
3535	addBasicBlockMetadata(BB: Latch, Properties);
3536	}
3537
3538	/// Attach llvm.access.group metadata to the memref instructions of \p Block
3539	static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup,
3540	LoopInfo &LI) {
3541	for (Instruction &I : *Block) {
3542	if (I.mayReadOrWriteMemory()) {
3543	// TODO: This instruction may already have access group from
3544	// other pragmas e.g. #pragma clang loop vectorize. Append
3545	// so that the existing metadata is not overwritten.
3546	I.setMetadata(KindID: LLVMContext::MD_access_group, Node: AccessGroup);
3547	}
3548	}
3549	}
3550
3551	void OpenMPIRBuilder::unrollLoopFull(DebugLoc, CanonicalLoopInfo *Loop) {
3552	LLVMContext &Ctx = Builder.getContext();
3553	addLoopMetadata(
3554	Loop, Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
3555	MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.full"))});
3556	}
3557
3558	void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) {
3559	LLVMContext &Ctx = Builder.getContext();
3560	addLoopMetadata(
3561	Loop, Properties: {
3562	MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
3563	});
3564	}
3565
3566	void OpenMPIRBuilder::createIfVersion(CanonicalLoopInfo *CanonicalLoop,
3567	Value *IfCond, ValueToValueMapTy &VMap,
3568	const Twine &NamePrefix) {
3569	Function *F = CanonicalLoop->getFunction();
3570
3571	// Define where if branch should be inserted
3572	Instruction *SplitBefore;
3573	if (Instruction::classof(V: IfCond)) {
3574	SplitBefore = dyn_cast<Instruction>(Val: IfCond);
3575	} else {
3576	SplitBefore = CanonicalLoop->getPreheader()->getTerminator();
3577	}
3578
3579	// TODO: We should not rely on pass manager. Currently we use pass manager
3580	// only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
3581	// object. We should have a method which returns all blocks between
3582	// CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
3583	FunctionAnalysisManager FAM;
3584	FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
3585	FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
3586	FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
3587
3588	// Get the loop which needs to be cloned
3589	LoopAnalysis LIA;
3590	LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
3591	Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
3592
3593	// Create additional blocks for the if statement
3594	BasicBlock *Head = SplitBefore->getParent();
3595	Instruction *HeadOldTerm = Head->getTerminator();
3596	llvm::LLVMContext &C = Head->getContext();
3597	llvm::BasicBlock *ThenBlock = llvm::BasicBlock::Create(
3598	Context&: C, Name: NamePrefix + ".if.then", Parent: Head->getParent(), InsertBefore: Head->getNextNode());
3599	llvm::BasicBlock *ElseBlock = llvm::BasicBlock::Create(
3600	Context&: C, Name: NamePrefix + ".if.else", Parent: Head->getParent(), InsertBefore: CanonicalLoop->getExit());
3601
3602	// Create if condition branch.
3603	Builder.SetInsertPoint(HeadOldTerm);
3604	Instruction *BrInstr =
3605	Builder.CreateCondBr(Cond: IfCond, True: ThenBlock, /*ifFalse*/ False: ElseBlock);
3606	InsertPointTy IP{BrInstr->getParent(), ++BrInstr->getIterator()};
3607	// Then block contains branch to omp loop which needs to be vectorized
3608	spliceBB(IP, New: ThenBlock, CreateBranch: false);
3609	ThenBlock->replaceSuccessorsPhiUsesWith(Old: Head, New: ThenBlock);
3610
3611	Builder.SetInsertPoint(ElseBlock);
3612
3613	// Clone loop for the else branch
3614	SmallVector<BasicBlock *, 8> NewBlocks;
3615
3616	VMap[CanonicalLoop->getPreheader()] = ElseBlock;
3617	for (BasicBlock *Block : L->getBlocks()) {
3618	BasicBlock *NewBB = CloneBasicBlock(BB: Block, VMap, NameSuffix: "", F);
3619	NewBB->moveBefore(MovePos: CanonicalLoop->getExit());
3620	VMap[Block] = NewBB;
3621	NewBlocks.push_back(Elt: NewBB);
3622	}
3623	remapInstructionsInBlocks(Blocks: NewBlocks, VMap);
3624	Builder.CreateBr(Dest: NewBlocks.front());
3625	}
3626
3627	unsigned
3628	OpenMPIRBuilder::getOpenMPDefaultSimdAlign(const Triple &TargetTriple,
3629	const StringMap<bool> &Features) {
3630	if (TargetTriple.isX86()) {
3631	if (Features.lookup(Key: "avx512f"))
3632	return 512;
3633	else if (Features.lookup(Key: "avx"))
3634	return 256;
3635	return 128;
3636	}
3637	if (TargetTriple.isPPC())
3638	return 128;
3639	if (TargetTriple.isWasm())
3640	return 128;
3641	return 0;
3642	}
3643
3644	void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
3645	MapVector<Value *, Value *> AlignedVars,
3646	Value *IfCond, OrderKind Order,
3647	ConstantInt *Simdlen, ConstantInt *Safelen) {
3648	LLVMContext &Ctx = Builder.getContext();
3649
3650	Function *F = CanonicalLoop->getFunction();
3651
3652	// TODO: We should not rely on pass manager. Currently we use pass manager
3653	// only for getting llvm::Loop which corresponds to given CanonicalLoopInfo
3654	// object. We should have a method which returns all blocks between
3655	// CanonicalLoopInfo::getHeader() and CanonicalLoopInfo::getAfter()
3656	FunctionAnalysisManager FAM;
3657	FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
3658	FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
3659	FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
3660
3661	LoopAnalysis LIA;
3662	LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
3663
3664	Loop *L = LI.getLoopFor(BB: CanonicalLoop->getHeader());
3665	if (AlignedVars.size()) {
3666	InsertPointTy IP = Builder.saveIP();
3667	Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator());
3668	for (auto &AlignedItem : AlignedVars) {
3669	Value *AlignedPtr = AlignedItem.first;
3670	Value *Alignment = AlignedItem.second;
3671	Builder.CreateAlignmentAssumption(DL: F->getParent()->getDataLayout(),
3672	PtrValue: AlignedPtr, Alignment);
3673	}
3674	Builder.restoreIP(IP);
3675	}
3676
3677	if (IfCond) {
3678	ValueToValueMapTy VMap;
3679	createIfVersion(CanonicalLoop, IfCond, VMap, NamePrefix: "simd");
3680	// Add metadata to the cloned loop which disables vectorization
3681	Value *MappedLatch = VMap.lookup(Val: CanonicalLoop->getLatch());
3682	assert(MappedLatch &&
3683	"Cannot find value which corresponds to original loop latch");
3684	assert(isa<BasicBlock>(MappedLatch) &&
3685	"Cannot cast mapped latch block value to BasicBlock");
3686	BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(Val: MappedLatch);
3687	ConstantAsMetadata *BoolConst =
3688	ConstantAsMetadata::get(C: ConstantInt::getFalse(Ty: Type::getInt1Ty(C&: Ctx)));
3689	addBasicBlockMetadata(
3690	BB: NewLatchBlock,
3691	Properties: {MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"),
3692	BoolConst})});
3693	}
3694
3695	SmallSet<BasicBlock *, 8> Reachable;
3696
3697	// Get the basic blocks from the loop in which memref instructions
3698	// can be found.
3699	// TODO: Generalize getting all blocks inside a CanonicalizeLoopInfo,
3700	// preferably without running any passes.
3701	for (BasicBlock *Block : L->getBlocks()) {
3702	if (Block == CanonicalLoop->getCond() ||
3703	Block == CanonicalLoop->getHeader())
3704	continue;
3705	Reachable.insert(Ptr: Block);
3706	}
3707
3708	SmallVector<Metadata *> LoopMDList;
3709
3710	// In presence of finite 'safelen', it may be unsafe to mark all
3711	// the memory instructions parallel, because loop-carried
3712	// dependences of 'safelen' iterations are possible.
3713	// If clause order(concurrent) is specified then the memory instructions
3714	// are marked parallel even if 'safelen' is finite.
3715	if ((Safelen == nullptr) || (Order == OrderKind::OMP_ORDER_concurrent)) {
3716	// Add access group metadata to memory-access instructions.
3717	MDNode *AccessGroup = MDNode::getDistinct(Context&: Ctx, MDs: {});
3718	for (BasicBlock *BB : Reachable)
3719	addSimdMetadata(Block: BB, AccessGroup, LI);
3720	// TODO: If the loop has existing parallel access metadata, have
3721	// to combine two lists.
3722	LoopMDList.push_back(Elt: MDNode::get(
3723	Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.parallel_accesses"), AccessGroup}));
3724	}
3725
3726	// Use the above access group metadata to create loop level
3727	// metadata, which should be distinct for each loop.
3728	ConstantAsMetadata *BoolConst =
3729	ConstantAsMetadata::get(C: ConstantInt::getTrue(Ty: Type::getInt1Ty(C&: Ctx)));
3730	LoopMDList.push_back(Elt: MDNode::get(
3731	Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.enable"), BoolConst}));
3732
3733	if (Simdlen || Safelen) {
3734	// If both simdlen and safelen clauses are specified, the value of the
3735	// simdlen parameter must be less than or equal to the value of the safelen
3736	// parameter. Therefore, use safelen only in the absence of simdlen.
3737	ConstantInt *VectorizeWidth = Simdlen == nullptr ? Safelen : Simdlen;
3738	LoopMDList.push_back(
3739	Elt: MDNode::get(Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.vectorize.width"),
3740	ConstantAsMetadata::get(C: VectorizeWidth)}));
3741	}
3742
3743	addLoopMetadata(Loop: CanonicalLoop, Properties: LoopMDList);
3744	}
3745
3746	/// Create the TargetMachine object to query the backend for optimization
3747	/// preferences.
3748	///
3749	/// Ideally, this would be passed from the front-end to the OpenMPBuilder, but
3750	/// e.g. Clang does not pass it to its CodeGen layer and creates it only when
3751	/// needed for the LLVM pass pipline. We use some default options to avoid
3752	/// having to pass too many settings from the frontend that probably do not
3753	/// matter.
3754	///
3755	/// Currently, TargetMachine is only used sometimes by the unrollLoopPartial
3756	/// method. If we are going to use TargetMachine for more purposes, especially
3757	/// those that are sensitive to TargetOptions, RelocModel and CodeModel, it
3758	/// might become be worth requiring front-ends to pass on their TargetMachine,
3759	/// or at least cache it between methods. Note that while fontends such as Clang
3760	/// have just a single main TargetMachine per translation unit, "target-cpu" and
3761	/// "target-features" that determine the TargetMachine are per-function and can
3762	/// be overrided using __attribute__((target("OPTIONS"))).
3763	static std::unique_ptr<TargetMachine>
3764	createTargetMachine(Function *F, CodeGenOptLevel OptLevel) {
3765	Module *M = F->getParent();
3766
3767	StringRef CPU = F->getFnAttribute(Kind: "target-cpu").getValueAsString();
3768	StringRef Features = F->getFnAttribute(Kind: "target-features").getValueAsString();
3769	const std::string &Triple = M->getTargetTriple();
3770
3771	std::string Error;
3772	const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
3773	if (!TheTarget)
3774	return {};
3775
3776	llvm::TargetOptions Options;
3777	return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
3778	TT: Triple, CPU, Features, Options, /*RelocModel=*/RM: std::nullopt,
3779	/*CodeModel=*/CM: std::nullopt, OL: OptLevel));
3780	}
3781
3782	/// Heuristically determine the best-performant unroll factor for \p CLI. This
3783	/// depends on the target processor. We are re-using the same heuristics as the
3784	/// LoopUnrollPass.
3785	static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI) {
3786	Function *F = CLI->getFunction();
3787
3788	// Assume the user requests the most aggressive unrolling, even if the rest of
3789	// the code is optimized using a lower setting.
3790	CodeGenOptLevel OptLevel = CodeGenOptLevel::Aggressive;
3791	std::unique_ptr<TargetMachine> TM = createTargetMachine(F, OptLevel);
3792
3793	FunctionAnalysisManager FAM;
3794	FAM.registerPass(PassBuilder: []() { return TargetLibraryAnalysis(); });
3795	FAM.registerPass(PassBuilder: []() { return AssumptionAnalysis(); });
3796	FAM.registerPass(PassBuilder: []() { return DominatorTreeAnalysis(); });
3797	FAM.registerPass(PassBuilder: []() { return LoopAnalysis(); });
3798	FAM.registerPass(PassBuilder: []() { return ScalarEvolutionAnalysis(); });
3799	FAM.registerPass(PassBuilder: []() { return PassInstrumentationAnalysis(); });
3800	TargetIRAnalysis TIRA;
3801	if (TM)
3802	TIRA = TargetIRAnalysis(
3803	[&](const Function &F) { return TM->getTargetTransformInfo(F); });
3804	FAM.registerPass(PassBuilder: [&]() { return TIRA; });
3805
3806	TargetIRAnalysis::Result &&TTI = TIRA.run(F: *F, FAM);
3807	ScalarEvolutionAnalysis SEA;
3808	ScalarEvolution &&SE = SEA.run(F&: *F, AM&: FAM);
3809	DominatorTreeAnalysis DTA;
3810	DominatorTree &&DT = DTA.run(F&: *F, FAM);
3811	LoopAnalysis LIA;
3812	LoopInfo &&LI = LIA.run(F&: *F, AM&: FAM);
3813	AssumptionAnalysis ACT;
3814	AssumptionCache &&AC = ACT.run(F&: *F, FAM);
3815	OptimizationRemarkEmitter ORE{F};
3816
3817	Loop *L = LI.getLoopFor(BB: CLI->getHeader());
3818	assert(L && "Expecting CanonicalLoopInfo to be recognized as a loop");
3819
3820	TargetTransformInfo::UnrollingPreferences UP =
3821	gatherUnrollingPreferences(L, SE, TTI,
3822	/*BlockFrequencyInfo=*/BFI: nullptr,
3823	/*ProfileSummaryInfo=*/PSI: nullptr, ORE, OptLevel: static_cast<int>(OptLevel),
3824	/*UserThreshold=*/std::nullopt,
3825	/*UserCount=*/std::nullopt,
3826	/*UserAllowPartial=*/true,
3827	/*UserAllowRuntime=*/UserRuntime: true,
3828	/*UserUpperBound=*/std::nullopt,
3829	/*UserFullUnrollMaxCount=*/std::nullopt);
3830
3831	UP.Force = true;
3832
3833	// Account for additional optimizations taking place before the LoopUnrollPass
3834	// would unroll the loop.
3835	UP.Threshold *= UnrollThresholdFactor;
3836	UP.PartialThreshold *= UnrollThresholdFactor;
3837
3838	// Use normal unroll factors even if the rest of the code is optimized for
3839	// size.
3840	UP.OptSizeThreshold = UP.Threshold;
3841	UP.PartialOptSizeThreshold = UP.PartialThreshold;
3842
3843	LLVM_DEBUG(dbgs() << "Unroll heuristic thresholds:\n"
3844	<< " Threshold=" << UP.Threshold << "\n"
3845	<< " PartialThreshold=" << UP.PartialThreshold << "\n"
3846	<< " OptSizeThreshold=" << UP.OptSizeThreshold << "\n"
3847	<< " PartialOptSizeThreshold="
3848	<< UP.PartialOptSizeThreshold << "\n");
3849
3850	// Disable peeling.
3851	TargetTransformInfo::PeelingPreferences PP =
3852	gatherPeelingPreferences(L, SE, TTI,
3853	/*UserAllowPeeling=*/false,
3854	/*UserAllowProfileBasedPeeling=*/false,
3855	/*UnrollingSpecficValues=*/false);
3856
3857	SmallPtrSet<const Value *, 32> EphValues;
3858	CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
3859
3860	// Assume that reads and writes to stack variables can be eliminated by
3861	// Mem2Reg, SROA or LICM. That is, don't count them towards the loop body's
3862	// size.
3863	for (BasicBlock *BB : L->blocks()) {
3864	for (Instruction &I : *BB) {
3865	Value *Ptr;
3866	if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
3867	Ptr = Load->getPointerOperand();
3868	} else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
3869	Ptr = Store->getPointerOperand();
3870	} else
3871	continue;
3872
3873	Ptr = Ptr->stripPointerCasts();
3874
3875	if (auto *Alloca = dyn_cast<AllocaInst>(Val: Ptr)) {
3876	if (Alloca->getParent() == &F->getEntryBlock())
3877	EphValues.insert(Ptr: &I);
3878	}
3879	}
3880	}
3881
3882	UnrollCostEstimator UCE(L, TTI, EphValues, UP.BEInsns);
3883
3884	// Loop is not unrollable if the loop contains certain instructions.
3885	if (!UCE.canUnroll() || UCE.Convergent) {
3886	LLVM_DEBUG(dbgs() << "Loop not considered unrollable\n");
3887	return 1;
3888	}
3889
3890	LLVM_DEBUG(dbgs() << "Estimated loop size is " << UCE.getRolledLoopSize()
3891	<< "\n");
3892
3893	// TODO: Determine trip count of \p CLI if constant, computeUnrollCount might
3894	// be able to use it.
3895	int TripCount = 0;
3896	int MaxTripCount = 0;
3897	bool MaxOrZero = false;
3898	unsigned TripMultiple = 0;
3899
3900	bool UseUpperBound = false;
3901	computeUnrollCount(L, TTI, DT, LI: &LI, AC: &AC, SE, EphValues, ORE: &ORE, TripCount,
3902	MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,
3903	UseUpperBound);
3904	unsigned Factor = UP.Count;
3905	LLVM_DEBUG(dbgs() << "Suggesting unroll factor of " << Factor << "\n");
3906
3907	// This function returns 1 to signal to not unroll a loop.
3908	if (Factor == 0)
3909	return 1;
3910	return Factor;
3911	}
3912
3913	void OpenMPIRBuilder::unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop,
3914	int32_t Factor,
3915	CanonicalLoopInfo **UnrolledCLI) {
3916	assert(Factor >= 0 && "Unroll factor must not be negative");
3917
3918	Function *F = Loop->getFunction();
3919	LLVMContext &Ctx = F->getContext();
3920
3921	// If the unrolled loop is not used for another loop-associated directive, it
3922	// is sufficient to add metadata for the LoopUnrollPass.
3923	if (!UnrolledCLI) {
3924	SmallVector<Metadata *, 2> LoopMetadata;
3925	LoopMetadata.push_back(
3926	Elt: MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")));
3927
3928	if (Factor >= 1) {
3929	ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3930	C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
3931	LoopMetadata.push_back(Elt: MDNode::get(
3932	Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst}));
3933	}
3934
3935	addLoopMetadata(Loop, Properties: LoopMetadata);
3936	return;
3937	}
3938
3939	// Heuristically determine the unroll factor.
3940	if (Factor == 0)
3941	Factor = computeHeuristicUnrollFactor(CLI: Loop);
3942
3943	// No change required with unroll factor 1.
3944	if (Factor == 1) {
3945	*UnrolledCLI = Loop;
3946	return;
3947	}
3948
3949	assert(Factor >= 2 &&
3950	"unrolling only makes sense with a factor of 2 or larger");
3951
3952	Type *IndVarTy = Loop->getIndVarType();
3953
3954	// Apply partial unrolling by tiling the loop by the unroll-factor, then fully
3955	// unroll the inner loop.
3956	Value *FactorVal =
3957	ConstantInt::get(Ty: IndVarTy, V: APInt(IndVarTy->getIntegerBitWidth(), Factor,
3958	/*isSigned=*/false));
3959	std::vector<CanonicalLoopInfo *> LoopNest =
3960	tileLoops(DL, Loops: {Loop}, TileSizes: {FactorVal});
3961	assert(LoopNest.size() == 2 && "Expect 2 loops after tiling");
3962	*UnrolledCLI = LoopNest[0];
3963	CanonicalLoopInfo *InnerLoop = LoopNest[1];
3964
3965	// LoopUnrollPass can only fully unroll loops with constant trip count.
3966	// Unroll by the unroll factor with a fallback epilog for the remainder
3967	// iterations if necessary.
3968	ConstantAsMetadata *FactorConst = ConstantAsMetadata::get(
3969	C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: APInt(32, Factor)));
3970	addLoopMetadata(
3971	Loop: InnerLoop,
3972	Properties: {MDNode::get(Context&: Ctx, MDs: MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.enable")),
3973	MDNode::get(
3974	Context&: Ctx, MDs: {MDString::get(Context&: Ctx, Str: "llvm.loop.unroll.count"), FactorConst})});
3975
3976	#ifndef NDEBUG
3977	(*UnrolledCLI)->assertOK();
3978	#endif
3979	}
3980
3981	OpenMPIRBuilder::InsertPointTy
3982	OpenMPIRBuilder::createCopyPrivate(const LocationDescription &Loc,
3983	llvm::Value *BufSize, llvm::Value *CpyBuf,
3984	llvm::Value *CpyFn, llvm::Value *DidIt) {
3985	if (!updateToLocation(Loc))
3986	return Loc.IP;
3987
3988	uint32_t SrcLocStrSize;
3989	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
3990	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
3991	Value *ThreadId = getOrCreateThreadID(Ident);
3992
3993	llvm::Value *DidItLD = Builder.CreateLoad(Ty: Builder.getInt32Ty(), Ptr: DidIt);
3994
3995	Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
3996
3997	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_copyprivate);
3998	Builder.CreateCall(Callee: Fn, Args);
3999
4000	return Builder.saveIP();
4001	}
4002
4003	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createSingle(
4004	const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
4005	FinalizeCallbackTy FiniCB, bool IsNowait, llvm::Value *DidIt) {
4006
4007	if (!updateToLocation(Loc))
4008	return Loc.IP;
4009
4010	// If needed (i.e. not null), initialize `DidIt` with 0
4011	if (DidIt) {
4012	Builder.CreateStore(Val: Builder.getInt32(C: 0), Ptr: DidIt);
4013	}
4014
4015	Directive OMPD = Directive::OMPD_single;
4016	uint32_t SrcLocStrSize;
4017	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4018	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4019	Value *ThreadId = getOrCreateThreadID(Ident);
4020	Value *Args[] = {Ident, ThreadId};
4021
4022	Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_single);
4023	Instruction *EntryCall = Builder.CreateCall(Callee: EntryRTLFn, Args);
4024
4025	Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_single);
4026	Instruction *ExitCall = Builder.CreateCall(Callee: ExitRTLFn, Args);
4027
4028	// generates the following:
4029	// if (__kmpc_single()) {
4030	// .... single region ...
4031	// __kmpc_end_single
4032	// }
4033	// __kmpc_barrier
4034
4035	EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
4036	/*Conditional*/ true,
4037	/*hasFinalize*/ true);
4038	if (!IsNowait)
4039	createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
4040	omp::Directive::OMPD_unknown, /* ForceSimpleCall */ false,
4041	/* CheckCancelFlag */ false);
4042	return Builder.saveIP();
4043	}
4044
4045	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCritical(
4046	const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
4047	FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
4048
4049	if (!updateToLocation(Loc))
4050	return Loc.IP;
4051
4052	Directive OMPD = Directive::OMPD_critical;
4053	uint32_t SrcLocStrSize;
4054	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4055	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4056	Value *ThreadId = getOrCreateThreadID(Ident);
4057	Value *LockVar = getOMPCriticalRegionLock(CriticalName);
4058	Value *Args[] = {Ident, ThreadId, LockVar};
4059
4060	SmallVector<llvm::Value *, 4> EnterArgs(std::begin(arr&: Args), std::end(arr&: Args));
4061	Function *RTFn = nullptr;
4062	if (HintInst) {
4063	// Add Hint to entry Args and create call
4064	EnterArgs.push_back(Elt: HintInst);
4065	RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical_with_hint);
4066	} else {
4067	RTFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_critical);
4068	}
4069	Instruction *EntryCall = Builder.CreateCall(Callee: RTFn, Args: EnterArgs);
4070
4071	Function *ExitRTLFn =
4072	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_critical);
4073	Instruction *ExitCall = Builder.CreateCall(Callee: ExitRTLFn, Args);
4074
4075	return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
4076	/*Conditional*/ false, /*hasFinalize*/ true);
4077	}
4078
4079	OpenMPIRBuilder::InsertPointTy
4080	OpenMPIRBuilder::createOrderedDepend(const LocationDescription &Loc,
4081	InsertPointTy AllocaIP, unsigned NumLoops,
4082	ArrayRef<llvm::Value *> StoreValues,
4083	const Twine &Name, bool IsDependSource) {
4084	assert(
4085	llvm::all_of(StoreValues,
4086	[](Value *SV) { return SV->getType()->isIntegerTy(64); }) &&
4087	"OpenMP runtime requires depend vec with i64 type");
4088
4089	if (!updateToLocation(Loc))
4090	return Loc.IP;
4091
4092	// Allocate space for vector and generate alloc instruction.
4093	auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumLoops);
4094	Builder.restoreIP(IP: AllocaIP);
4095	AllocaInst *ArgsBase = Builder.CreateAlloca(Ty: ArrI64Ty, ArraySize: nullptr, Name);
4096	ArgsBase->setAlignment(Align(8));
4097	Builder.restoreIP(IP: Loc.IP);
4098
4099	// Store the index value with offset in depend vector.
4100	for (unsigned I = 0; I < NumLoops; ++I) {
4101	Value *DependAddrGEPIter = Builder.CreateInBoundsGEP(
4102	Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: I)});
4103	StoreInst *STInst = Builder.CreateStore(Val: StoreValues[I], Ptr: DependAddrGEPIter);
4104	STInst->setAlignment(Align(8));
4105	}
4106
4107	Value *DependBaseAddrGEP = Builder.CreateInBoundsGEP(
4108	Ty: ArrI64Ty, Ptr: ArgsBase, IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: 0)});
4109
4110	uint32_t SrcLocStrSize;
4111	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4112	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4113	Value *ThreadId = getOrCreateThreadID(Ident);
4114	Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
4115
4116	Function *RTLFn = nullptr;
4117	if (IsDependSource)
4118	RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_post);
4119	else
4120	RTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_doacross_wait);
4121	Builder.CreateCall(Callee: RTLFn, Args);
4122
4123	return Builder.saveIP();
4124	}
4125
4126	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createOrderedThreadsSimd(
4127	const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
4128	FinalizeCallbackTy FiniCB, bool IsThreads) {
4129	if (!updateToLocation(Loc))
4130	return Loc.IP;
4131
4132	Directive OMPD = Directive::OMPD_ordered;
4133	Instruction *EntryCall = nullptr;
4134	Instruction *ExitCall = nullptr;
4135
4136	if (IsThreads) {
4137	uint32_t SrcLocStrSize;
4138	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4139	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4140	Value *ThreadId = getOrCreateThreadID(Ident);
4141	Value *Args[] = {Ident, ThreadId};
4142
4143	Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_ordered);
4144	EntryCall = Builder.CreateCall(Callee: EntryRTLFn, Args);
4145
4146	Function *ExitRTLFn =
4147	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_end_ordered);
4148	ExitCall = Builder.CreateCall(Callee: ExitRTLFn, Args);
4149	}
4150
4151	return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
4152	/*Conditional*/ false, /*hasFinalize*/ true);
4153	}
4154
4155	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
4156	Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
4157	BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
4158	bool HasFinalize, bool IsCancellable) {
4159
4160	if (HasFinalize)
4161	FinalizationStack.push_back({FiniCB, OMPD, IsCancellable});
4162
4163	// Create inlined region's entry and body blocks, in preparation
4164	// for conditional creation
4165	BasicBlock *EntryBB = Builder.GetInsertBlock();
4166	Instruction *SplitPos = EntryBB->getTerminator();
4167	if (!isa_and_nonnull<BranchInst>(Val: SplitPos))
4168	SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
4169	BasicBlock *ExitBB = EntryBB->splitBasicBlock(I: SplitPos, BBName: "omp_region.end");
4170	BasicBlock *FiniBB =
4171	EntryBB->splitBasicBlock(I: EntryBB->getTerminator(), BBName: "omp_region.finalize");
4172
4173	Builder.SetInsertPoint(EntryBB->getTerminator());
4174	emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
4175
4176	// generate body
4177	BodyGenCB(/* AllocaIP */ InsertPointTy(),
4178	/* CodeGenIP */ Builder.saveIP());
4179
4180	// emit exit call and do any needed finalization.
4181	auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
4182	assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
4183	FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
4184	"Unexpected control flow graph state!!");
4185	emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
4186	assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&
4187	"Unexpected Control Flow State!");
4188	MergeBlockIntoPredecessor(BB: FiniBB);
4189
4190	// If we are skipping the region of a non conditional, remove the exit
4191	// block, and clear the builder's insertion point.
4192	assert(SplitPos->getParent() == ExitBB &&
4193	"Unexpected Insertion point location!");
4194	auto merged = MergeBlockIntoPredecessor(BB: ExitBB);
4195	BasicBlock *ExitPredBB = SplitPos->getParent();
4196	auto InsertBB = merged ? ExitPredBB : ExitBB;
4197	if (!isa_and_nonnull<BranchInst>(Val: SplitPos))
4198	SplitPos->eraseFromParent();
4199	Builder.SetInsertPoint(InsertBB);
4200
4201	return Builder.saveIP();
4202	}
4203
4204	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
4205	Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
4206	// if nothing to do, Return current insertion point.
4207	if (!Conditional || !EntryCall)
4208	return Builder.saveIP();
4209
4210	BasicBlock *EntryBB = Builder.GetInsertBlock();
4211	Value *CallBool = Builder.CreateIsNotNull(Arg: EntryCall);
4212	auto *ThenBB = BasicBlock::Create(Context&: M.getContext(), Name: "omp_region.body");
4213	auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
4214
4215	// Emit thenBB and set the Builder's insertion point there for
4216	// body generation next. Place the block after the current block.
4217	Function *CurFn = EntryBB->getParent();
4218	CurFn->insert(Position: std::next(x: EntryBB->getIterator()), BB: ThenBB);
4219
4220	// Move Entry branch to end of ThenBB, and replace with conditional
4221	// branch (If-stmt)
4222	Instruction *EntryBBTI = EntryBB->getTerminator();
4223	Builder.CreateCondBr(Cond: CallBool, True: ThenBB, False: ExitBB);
4224	EntryBBTI->removeFromParent();
4225	Builder.SetInsertPoint(UI);
4226	Builder.Insert(I: EntryBBTI);
4227	UI->eraseFromParent();
4228	Builder.SetInsertPoint(ThenBB->getTerminator());
4229
4230	// return an insertion point to ExitBB.
4231	return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
4232	}
4233
4234	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
4235	omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
4236	bool HasFinalize) {
4237
4238	Builder.restoreIP(IP: FinIP);
4239
4240	// If there is finalization to do, emit it before the exit call
4241	if (HasFinalize) {
4242	assert(!FinalizationStack.empty() &&
4243	"Unexpected finalization stack state!");
4244
4245	FinalizationInfo Fi = FinalizationStack.pop_back_val();
4246	assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
4247
4248	Fi.FiniCB(FinIP);
4249
4250	BasicBlock *FiniBB = FinIP.getBlock();
4251	Instruction *FiniBBTI = FiniBB->getTerminator();
4252
4253	// set Builder IP for call creation
4254	Builder.SetInsertPoint(FiniBBTI);
4255	}
4256
4257	if (!ExitCall)
4258	return Builder.saveIP();
4259
4260	// place the Exitcall as last instruction before Finalization block terminator
4261	ExitCall->removeFromParent();
4262	Builder.Insert(I: ExitCall);
4263
4264	return IRBuilder<>::InsertPoint(ExitCall->getParent(),
4265	ExitCall->getIterator());
4266	}
4267
4268	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createCopyinClauseBlocks(
4269	InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
4270	llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
4271	if (!IP.isSet())
4272	return IP;
4273
4274	IRBuilder<>::InsertPointGuard IPG(Builder);
4275
4276	// creates the following CFG structure
4277	// OMP_Entry : (MasterAddr != PrivateAddr)?
4278	// F T
4279	// | \
4280	// | copin.not.master
4281	// | /
4282	// v /
4283	// copyin.not.master.end
4284	// |
4285	// v
4286	// OMP.Entry.Next
4287
4288	BasicBlock *OMP_Entry = IP.getBlock();
4289	Function *CurFn = OMP_Entry->getParent();
4290	BasicBlock *CopyBegin =
4291	BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master", Parent: CurFn);
4292	BasicBlock *CopyEnd = nullptr;
4293
4294	// If entry block is terminated, split to preserve the branch to following
4295	// basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
4296	if (isa_and_nonnull<BranchInst>(Val: OMP_Entry->getTerminator())) {
4297	CopyEnd = OMP_Entry->splitBasicBlock(I: OMP_Entry->getTerminator(),
4298	BBName: "copyin.not.master.end");
4299	OMP_Entry->getTerminator()->eraseFromParent();
4300	} else {
4301	CopyEnd =
4302	BasicBlock::Create(Context&: M.getContext(), Name: "copyin.not.master.end", Parent: CurFn);
4303	}
4304
4305	Builder.SetInsertPoint(OMP_Entry);
4306	Value *MasterPtr = Builder.CreatePtrToInt(V: MasterAddr, DestTy: IntPtrTy);
4307	Value *PrivatePtr = Builder.CreatePtrToInt(V: PrivateAddr, DestTy: IntPtrTy);
4308	Value *cmp = Builder.CreateICmpNE(LHS: MasterPtr, RHS: PrivatePtr);
4309	Builder.CreateCondBr(Cond: cmp, True: CopyBegin, False: CopyEnd);
4310
4311	Builder.SetInsertPoint(CopyBegin);
4312	if (BranchtoEnd)
4313	Builder.SetInsertPoint(Builder.CreateBr(Dest: CopyEnd));
4314
4315	return Builder.saveIP();
4316	}
4317
4318	CallInst *OpenMPIRBuilder::createOMPAlloc(const LocationDescription &Loc,
4319	Value *Size, Value *Allocator,
4320	std::string Name) {
4321	IRBuilder<>::InsertPointGuard IPG(Builder);
4322	Builder.restoreIP(IP: Loc.IP);
4323
4324	uint32_t SrcLocStrSize;
4325	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4326	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4327	Value *ThreadId = getOrCreateThreadID(Ident);
4328	Value *Args[] = {ThreadId, Size, Allocator};
4329
4330	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_alloc);
4331
4332	return Builder.CreateCall(Callee: Fn, Args, Name);
4333	}
4334
4335	CallInst *OpenMPIRBuilder::createOMPFree(const LocationDescription &Loc,
4336	Value *Addr, Value *Allocator,
4337	std::string Name) {
4338	IRBuilder<>::InsertPointGuard IPG(Builder);
4339	Builder.restoreIP(IP: Loc.IP);
4340
4341	uint32_t SrcLocStrSize;
4342	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4343	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4344	Value *ThreadId = getOrCreateThreadID(Ident);
4345	Value *Args[] = {ThreadId, Addr, Allocator};
4346	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_free);
4347	return Builder.CreateCall(Callee: Fn, Args, Name);
4348	}
4349
4350	CallInst *OpenMPIRBuilder::createOMPInteropInit(
4351	const LocationDescription &Loc, Value *InteropVar,
4352	omp::OMPInteropType InteropType, Value *Device, Value *NumDependences,
4353	Value *DependenceAddress, bool HaveNowaitClause) {
4354	IRBuilder<>::InsertPointGuard IPG(Builder);
4355	Builder.restoreIP(IP: Loc.IP);
4356
4357	uint32_t SrcLocStrSize;
4358	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4359	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4360	Value *ThreadId = getOrCreateThreadID(Ident);
4361	if (Device == nullptr)
4362	Device = ConstantInt::get(Ty: Int32, V: -1);
4363	Constant *InteropTypeVal = ConstantInt::get(Ty: Int32, V: (int)InteropType);
4364	if (NumDependences == nullptr) {
4365	NumDependences = ConstantInt::get(Ty: Int32, V: 0);
4366	PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
4367	DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
4368	}
4369	Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
4370	Value *Args[] = {
4371	Ident, ThreadId, InteropVar, InteropTypeVal,
4372	Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
4373
4374	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_init);
4375
4376	return Builder.CreateCall(Callee: Fn, Args);
4377	}
4378
4379	CallInst *OpenMPIRBuilder::createOMPInteropDestroy(
4380	const LocationDescription &Loc, Value *InteropVar, Value *Device,
4381	Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause) {
4382	IRBuilder<>::InsertPointGuard IPG(Builder);
4383	Builder.restoreIP(IP: Loc.IP);
4384
4385	uint32_t SrcLocStrSize;
4386	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4387	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4388	Value *ThreadId = getOrCreateThreadID(Ident);
4389	if (Device == nullptr)
4390	Device = ConstantInt::get(Ty: Int32, V: -1);
4391	if (NumDependences == nullptr) {
4392	NumDependences = ConstantInt::get(Ty: Int32, V: 0);
4393	PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
4394	DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
4395	}
4396	Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
4397	Value *Args[] = {
4398	Ident, ThreadId, InteropVar, Device,
4399	NumDependences, DependenceAddress, HaveNowaitClauseVal};
4400
4401	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_destroy);
4402
4403	return Builder.CreateCall(Callee: Fn, Args);
4404	}
4405
4406	CallInst *OpenMPIRBuilder::createOMPInteropUse(const LocationDescription &Loc,
4407	Value *InteropVar, Value *Device,
4408	Value *NumDependences,
4409	Value *DependenceAddress,
4410	bool HaveNowaitClause) {
4411	IRBuilder<>::InsertPointGuard IPG(Builder);
4412	Builder.restoreIP(IP: Loc.IP);
4413	uint32_t SrcLocStrSize;
4414	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4415	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4416	Value *ThreadId = getOrCreateThreadID(Ident);
4417	if (Device == nullptr)
4418	Device = ConstantInt::get(Ty: Int32, V: -1);
4419	if (NumDependences == nullptr) {
4420	NumDependences = ConstantInt::get(Ty: Int32, V: 0);
4421	PointerType *PointerTypeVar = PointerType::getUnqual(C&: M.getContext());
4422	DependenceAddress = ConstantPointerNull::get(T: PointerTypeVar);
4423	}
4424	Value *HaveNowaitClauseVal = ConstantInt::get(Ty: Int32, V: HaveNowaitClause);
4425	Value *Args[] = {
4426	Ident, ThreadId, InteropVar, Device,
4427	NumDependences, DependenceAddress, HaveNowaitClauseVal};
4428
4429	Function *Fn = getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___tgt_interop_use);
4430
4431	return Builder.CreateCall(Callee: Fn, Args);
4432	}
4433
4434	CallInst *OpenMPIRBuilder::createCachedThreadPrivate(
4435	const LocationDescription &Loc, llvm::Value *Pointer,
4436	llvm::ConstantInt *Size, const llvm::Twine &Name) {
4437	IRBuilder<>::InsertPointGuard IPG(Builder);
4438	Builder.restoreIP(IP: Loc.IP);
4439
4440	uint32_t SrcLocStrSize;
4441	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4442	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4443	Value *ThreadId = getOrCreateThreadID(Ident);
4444	Constant *ThreadPrivateCache =
4445	getOrCreateInternalVariable(Ty: Int8PtrPtr, Name: Name.str());
4446	llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
4447
4448	Function *Fn =
4449	getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_threadprivate_cached);
4450
4451	return Builder.CreateCall(Callee: Fn, Args);
4452	}
4453
4454	OpenMPIRBuilder::InsertPointTy
4455	OpenMPIRBuilder::createTargetInit(const LocationDescription &Loc, bool IsSPMD,
4456	int32_t MinThreadsVal, int32_t MaxThreadsVal,
4457	int32_t MinTeamsVal, int32_t MaxTeamsVal) {
4458	if (!updateToLocation(Loc))
4459	return Loc.IP;
4460
4461	uint32_t SrcLocStrSize;
4462	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4463	Constant *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4464	Constant *IsSPMDVal = ConstantInt::getSigned(
4465	Ty: Int8, V: IsSPMD ? OMP_TGT_EXEC_MODE_SPMD : OMP_TGT_EXEC_MODE_GENERIC);
4466	Constant *UseGenericStateMachineVal = ConstantInt::getSigned(Ty: Int8, V: !IsSPMD);
4467	Constant *MayUseNestedParallelismVal = ConstantInt::getSigned(Ty: Int8, V: true);
4468	Constant *DebugIndentionLevelVal = ConstantInt::getSigned(Ty: Int16, V: 0);
4469
4470	Function *Kernel = Builder.GetInsertBlock()->getParent();
4471
4472	// Manifest the launch configuration in the metadata matching the kernel
4473	// environment.
4474	if (MinTeamsVal > 1 || MaxTeamsVal > 0)
4475	writeTeamsForKernel(T, Kernel&: *Kernel, LB: MinTeamsVal, UB: MaxTeamsVal);
4476
4477	// For max values, < 0 means unset, == 0 means set but unknown.
4478	if (MaxThreadsVal < 0)
4479	MaxThreadsVal = std::max(
4480	a: int32_t(getGridValue(T, Kernel).GV_Default_WG_Size), b: MinThreadsVal);
4481
4482	if (MaxThreadsVal > 0)
4483	writeThreadBoundsForKernel(T, Kernel&: *Kernel, LB: MinThreadsVal, UB: MaxThreadsVal);
4484
4485	Constant *MinThreads = ConstantInt::getSigned(Ty: Int32, V: MinThreadsVal);
4486	Constant *MaxThreads = ConstantInt::getSigned(Ty: Int32, V: MaxThreadsVal);
4487	Constant *MinTeams = ConstantInt::getSigned(Ty: Int32, V: MinTeamsVal);
4488	Constant *MaxTeams = ConstantInt::getSigned(Ty: Int32, V: MaxTeamsVal);
4489	Constant *ReductionDataSize = ConstantInt::getSigned(Ty: Int32, V: 0);
4490	Constant *ReductionBufferLength = ConstantInt::getSigned(Ty: Int32, V: 0);
4491
4492	// We need to strip the debug prefix to get the correct kernel name.
4493	StringRef KernelName = Kernel->getName();
4494	const std::string DebugPrefix = "_debug__";
4495	if (KernelName.ends_with(Suffix: DebugPrefix))
4496	KernelName = KernelName.drop_back(N: DebugPrefix.length());
4497
4498	Function *Fn = getOrCreateRuntimeFunctionPtr(
4499	FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_init);
4500	const DataLayout &DL = Fn->getParent()->getDataLayout();
4501
4502	Twine DynamicEnvironmentName = KernelName + "_dynamic_environment";
4503	Constant *DynamicEnvironmentInitializer =
4504	ConstantStruct::get(T: DynamicEnvironment, V: {DebugIndentionLevelVal});
4505	GlobalVariable *DynamicEnvironmentGV = new GlobalVariable(
4506	M, DynamicEnvironment, /*IsConstant=*/false, GlobalValue::WeakODRLinkage,
4507	DynamicEnvironmentInitializer, DynamicEnvironmentName,
4508	/*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
4509	DL.getDefaultGlobalsAddressSpace());
4510	DynamicEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
4511
4512	Constant *DynamicEnvironment =
4513	DynamicEnvironmentGV->getType() == DynamicEnvironmentPtr
4514	? DynamicEnvironmentGV
4515	: ConstantExpr::getAddrSpaceCast(C: DynamicEnvironmentGV,
4516	Ty: DynamicEnvironmentPtr);
4517
4518	Constant *ConfigurationEnvironmentInitializer = ConstantStruct::get(
4519	T: ConfigurationEnvironment, V: {
4520	UseGenericStateMachineVal,
4521	MayUseNestedParallelismVal,
4522	IsSPMDVal,
4523	MinThreads,
4524	MaxThreads,
4525	MinTeams,
4526	MaxTeams,
4527	ReductionDataSize,
4528	ReductionBufferLength,
4529	});
4530	Constant *KernelEnvironmentInitializer = ConstantStruct::get(
4531	T: KernelEnvironment, V: {
4532	ConfigurationEnvironmentInitializer,
4533	Ident,
4534	DynamicEnvironment,
4535	});
4536	Twine KernelEnvironmentName = KernelName + "_kernel_environment";
4537	GlobalVariable *KernelEnvironmentGV = new GlobalVariable(
4538	M, KernelEnvironment, /*IsConstant=*/true, GlobalValue::WeakODRLinkage,
4539	KernelEnvironmentInitializer, KernelEnvironmentName,
4540	/*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
4541	DL.getDefaultGlobalsAddressSpace());
4542	KernelEnvironmentGV->setVisibility(GlobalValue::ProtectedVisibility);
4543
4544	Constant *KernelEnvironment =
4545	KernelEnvironmentGV->getType() == KernelEnvironmentPtr
4546	? KernelEnvironmentGV
4547	: ConstantExpr::getAddrSpaceCast(C: KernelEnvironmentGV,
4548	Ty: KernelEnvironmentPtr);
4549	Value *KernelLaunchEnvironment = Kernel->getArg(i: 0);
4550	CallInst *ThreadKind =
4551	Builder.CreateCall(Callee: Fn, Args: {KernelEnvironment, KernelLaunchEnvironment});
4552
4553	Value *ExecUserCode = Builder.CreateICmpEQ(
4554	LHS: ThreadKind, RHS: ConstantInt::get(Ty: ThreadKind->getType(), V: -1),
4555	Name: "exec_user_code");
4556
4557	// ThreadKind = __kmpc_target_init(...)
4558	// if (ThreadKind == -1)
4559	// user_code
4560	// else
4561	// return;
4562
4563	auto *UI = Builder.CreateUnreachable();
4564	BasicBlock *CheckBB = UI->getParent();
4565	BasicBlock *UserCodeEntryBB = CheckBB->splitBasicBlock(I: UI, BBName: "user_code.entry");
4566
4567	BasicBlock *WorkerExitBB = BasicBlock::Create(
4568	Context&: CheckBB->getContext(), Name: "worker.exit", Parent: CheckBB->getParent());
4569	Builder.SetInsertPoint(WorkerExitBB);
4570	Builder.CreateRetVoid();
4571
4572	auto *CheckBBTI = CheckBB->getTerminator();
4573	Builder.SetInsertPoint(CheckBBTI);
4574	Builder.CreateCondBr(Cond: ExecUserCode, True: UI->getParent(), False: WorkerExitBB);
4575
4576	CheckBBTI->eraseFromParent();
4577	UI->eraseFromParent();
4578
4579	// Continue in the "user_code" block, see diagram above and in
4580	// openmp/libomptarget/deviceRTLs/common/include/target.h .
4581	return InsertPointTy(UserCodeEntryBB, UserCodeEntryBB->getFirstInsertionPt());
4582	}
4583
4584	void OpenMPIRBuilder::createTargetDeinit(const LocationDescription &Loc,
4585	int32_t TeamsReductionDataSize,
4586	int32_t TeamsReductionBufferLength) {
4587	if (!updateToLocation(Loc))
4588	return;
4589
4590	Function *Fn = getOrCreateRuntimeFunctionPtr(
4591	FnID: omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
4592
4593	Builder.CreateCall(Callee: Fn, Args: {});
4594
4595	if (!TeamsReductionBufferLength || !TeamsReductionDataSize)
4596	return;
4597
4598	Function *Kernel = Builder.GetInsertBlock()->getParent();
4599	// We need to strip the debug prefix to get the correct kernel name.
4600	StringRef KernelName = Kernel->getName();
4601	const std::string DebugPrefix = "_debug__";
4602	if (KernelName.ends_with(Suffix: DebugPrefix))
4603	KernelName = KernelName.drop_back(N: DebugPrefix.length());
4604	auto *KernelEnvironmentGV =
4605	M.getNamedGlobal(Name: (KernelName + "_kernel_environment").str());
4606	assert(KernelEnvironmentGV && "Expected kernel environment global\n");
4607	auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
4608	auto *NewInitializer = ConstantFoldInsertValueInstruction(
4609	Agg: KernelEnvironmentInitializer,
4610	Val: ConstantInt::get(Ty: Int32, V: TeamsReductionDataSize), Idxs: {0, 7});
4611	NewInitializer = ConstantFoldInsertValueInstruction(
4612	Agg: NewInitializer, Val: ConstantInt::get(Ty: Int32, V: TeamsReductionBufferLength),
4613	Idxs: {0, 8});
4614	KernelEnvironmentGV->setInitializer(NewInitializer);
4615	}
4616
4617	static MDNode *getNVPTXMDNode(Function &Kernel, StringRef Name) {
4618	Module &M = *Kernel.getParent();
4619	NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "nvvm.annotations");
4620	for (auto *Op : MD->operands()) {
4621	if (Op->getNumOperands() != 3)
4622	continue;
4623	auto *KernelOp = dyn_cast<ConstantAsMetadata>(Val: Op->getOperand(I: 0));
4624	if (!KernelOp || KernelOp->getValue() != &Kernel)
4625	continue;
4626	auto *Prop = dyn_cast<MDString>(Val: Op->getOperand(I: 1));
4627	if (!Prop || Prop->getString() != Name)
4628	continue;
4629	return Op;
4630	}
4631	return nullptr;
4632	}
4633
4634	static void updateNVPTXMetadata(Function &Kernel, StringRef Name, int32_t Value,
4635	bool Min) {
4636	// Update the "maxntidx" metadata for NVIDIA, or add it.
4637	MDNode *ExistingOp = getNVPTXMDNode(Kernel, Name);
4638	if (ExistingOp) {
4639	auto *OldVal = cast<ConstantAsMetadata>(Val: ExistingOp->getOperand(I: 2));
4640	int32_t OldLimit = cast<ConstantInt>(Val: OldVal->getValue())->getZExtValue();
4641	ExistingOp->replaceOperandWith(
4642	I: 2, New: ConstantAsMetadata::get(C: ConstantInt::get(
4643	Ty: OldVal->getValue()->getType(),
4644	V: Min ? std::min(a: OldLimit, b: Value) : std::max(a: OldLimit, b: Value))));
4645	} else {
4646	LLVMContext &Ctx = Kernel.getContext();
4647	Metadata *MDVals[] = {ConstantAsMetadata::get(C: &Kernel),
4648	MDString::get(Context&: Ctx, Str: Name),
4649	ConstantAsMetadata::get(
4650	C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: Value))};
4651	// Append metadata to nvvm.annotations
4652	Module &M = *Kernel.getParent();
4653	NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "nvvm.annotations");
4654	MD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDVals));
4655	}
4656	}
4657
4658	std::pair<int32_t, int32_t>
4659	OpenMPIRBuilder::readThreadBoundsForKernel(const Triple &T, Function &Kernel) {
4660	int32_t ThreadLimit =
4661	Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_thread_limit");
4662
4663	if (T.isAMDGPU()) {
4664	const auto &Attr = Kernel.getFnAttribute(Kind: "amdgpu-flat-work-group-size");
4665	if (!Attr.isValid() || !Attr.isStringAttribute())
4666	return {0, ThreadLimit};
4667	auto [LBStr, UBStr] = Attr.getValueAsString().split(Separator: ',');
4668	int32_t LB, UB;
4669	if (!llvm::to_integer(S: UBStr, Num&: UB, Base: 10))
4670	return {0, ThreadLimit};
4671	UB = ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB;
4672	if (!llvm::to_integer(S: LBStr, Num&: LB, Base: 10))
4673	return {0, UB};
4674	return {LB, UB};
4675	}
4676
4677	if (MDNode *ExistingOp = getNVPTXMDNode(Kernel, Name: "maxntidx")) {
4678	auto *OldVal = cast<ConstantAsMetadata>(Val: ExistingOp->getOperand(I: 2));
4679	int32_t UB = cast<ConstantInt>(Val: OldVal->getValue())->getZExtValue();
4680	return {0, ThreadLimit ? std::min(a: ThreadLimit, b: UB) : UB};
4681	}
4682	return {0, ThreadLimit};
4683	}
4684
4685	void OpenMPIRBuilder::writeThreadBoundsForKernel(const Triple &T,
4686	Function &Kernel, int32_t LB,
4687	int32_t UB) {
4688	Kernel.addFnAttr(Kind: "omp_target_thread_limit", Val: std::to_string(val: UB));
4689
4690	if (T.isAMDGPU()) {
4691	Kernel.addFnAttr(Kind: "amdgpu-flat-work-group-size",
4692	Val: llvm::utostr(X: LB) + "," + llvm::utostr(X: UB));
4693	return;
4694	}
4695
4696	updateNVPTXMetadata(Kernel, Name: "maxntidx", Value: UB, Min: true);
4697	}
4698
4699	std::pair<int32_t, int32_t>
4700	OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
4701	// TODO: Read from backend annotations if available.
4702	return {0, Kernel.getFnAttributeAsParsedInteger(Kind: "omp_target_num_teams")};
4703	}
4704
4705	void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
4706	int32_t LB, int32_t UB) {
4707	if (T.isNVPTX()) {
4708	if (UB > 0)
4709	updateNVPTXMetadata(Kernel, Name: "maxclusterrank", Value: UB, Min: true);
4710	updateNVPTXMetadata(Kernel, Name: "minctasm", Value: LB, Min: false);
4711	}
4712	Kernel.addFnAttr(Kind: "omp_target_num_teams", Val: std::to_string(val: LB));
4713	}
4714
4715	void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
4716	Function *OutlinedFn) {
4717	if (Config.isTargetDevice()) {
4718	OutlinedFn->setLinkage(GlobalValue::WeakODRLinkage);
4719	// TODO: Determine if DSO local can be set to true.
4720	OutlinedFn->setDSOLocal(false);
4721	OutlinedFn->setVisibility(GlobalValue::ProtectedVisibility);
4722	if (T.isAMDGCN())
4723	OutlinedFn->setCallingConv(CallingConv::AMDGPU_KERNEL);
4724	}
4725	}
4726
4727	Constant *OpenMPIRBuilder::createOutlinedFunctionID(Function *OutlinedFn,
4728	StringRef EntryFnIDName) {
4729	if (Config.isTargetDevice()) {
4730	assert(OutlinedFn && "The outlined function must exist if embedded");
4731	return OutlinedFn;
4732	}
4733
4734	return new GlobalVariable(
4735	M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::WeakAnyLinkage,
4736	Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnIDName);
4737	}
4738
4739	Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(Function *OutlinedFn,
4740	StringRef EntryFnName) {
4741	if (OutlinedFn)
4742	return OutlinedFn;
4743
4744	assert(!M.getGlobalVariable(EntryFnName, true) &&
4745	"Named kernel already exists?");
4746	return new GlobalVariable(
4747	M, Builder.getInt8Ty(), /*isConstant=*/true, GlobalValue::InternalLinkage,
4748	Constant::getNullValue(Ty: Builder.getInt8Ty()), EntryFnName);
4749	}
4750
4751	void OpenMPIRBuilder::emitTargetRegionFunction(
4752	TargetRegionEntryInfo &EntryInfo,
4753	FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry,
4754	Function *&OutlinedFn, Constant *&OutlinedFnID) {
4755
4756	SmallString<64> EntryFnName;
4757	OffloadInfoManager.getTargetRegionEntryFnName(Name&: EntryFnName, EntryInfo);
4758
4759	OutlinedFn = Config.isTargetDevice() || !Config.openMPOffloadMandatory()
4760	? GenerateFunctionCallback(EntryFnName)
4761	: nullptr;
4762
4763	// If this target outline function is not an offload entry, we don't need to
4764	// register it. This may be in the case of a false if clause, or if there are
4765	// no OpenMP targets.
4766	if (!IsOffloadEntry)
4767	return;
4768
4769	std::string EntryFnIDName =
4770	Config.isTargetDevice()
4771	? std::string(EntryFnName)
4772	: createPlatformSpecificName(Parts: {EntryFnName, "region_id"});
4773
4774	OutlinedFnID = registerTargetRegionFunction(EntryInfo, OutlinedFunction: OutlinedFn,
4775	EntryFnName, EntryFnIDName);
4776	}
4777
4778	Constant *OpenMPIRBuilder::registerTargetRegionFunction(
4779	TargetRegionEntryInfo &EntryInfo, Function *OutlinedFn,
4780	StringRef EntryFnName, StringRef EntryFnIDName) {
4781	if (OutlinedFn)
4782	setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
4783	auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
4784	auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
4785	OffloadInfoManager.registerTargetRegionEntryInfo(
4786	EntryInfo, Addr: EntryAddr, ID: OutlinedFnID,
4787	Flags: OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion);
4788	return OutlinedFnID;
4789	}
4790
4791	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
4792	const LocationDescription &Loc, InsertPointTy AllocaIP,
4793	InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
4794	TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB,
4795	omp::RuntimeFunction *MapperFunc,
4796	function_ref<InsertPointTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)>
4797	BodyGenCB,
4798	function_ref<void(unsigned int, Value *)> DeviceAddrCB,
4799	function_ref<Value *(unsigned int)> CustomMapperCB, Value *SrcLocInfo) {
4800	if (!updateToLocation(Loc))
4801	return InsertPointTy();
4802
4803	// Disable TargetData CodeGen on Device pass.
4804	if (Config.IsTargetDevice.value_or(u: false))
4805	return Builder.saveIP();
4806
4807	Builder.restoreIP(IP: CodeGenIP);
4808	bool IsStandAlone = !BodyGenCB;
4809	MapInfosTy *MapInfo;
4810	// Generate the code for the opening of the data environment. Capture all the
4811	// arguments of the runtime call by reference because they are used in the
4812	// closing of the region.
4813	auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4814	MapInfo = &GenMapInfoCB(Builder.saveIP());
4815	emitOffloadingArrays(AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: *MapInfo, Info,
4816	/*IsNonContiguous=*/true, DeviceAddrCB,
4817	CustomMapperCB);
4818
4819	TargetDataRTArgs RTArgs;
4820	emitOffloadingArraysArgument(Builder, RTArgs, Info,
4821	EmitDebug: !MapInfo->Names.empty());
4822
4823	// Emit the number of elements in the offloading arrays.
4824	Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
4825
4826	// Source location for the ident struct
4827	if (!SrcLocInfo) {
4828	uint32_t SrcLocStrSize;
4829	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4830	SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4831	}
4832
4833	Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
4834	PointerNum, RTArgs.BasePointersArray,
4835	RTArgs.PointersArray, RTArgs.SizesArray,
4836	RTArgs.MapTypesArray, RTArgs.MapNamesArray,
4837	RTArgs.MappersArray};
4838
4839	if (IsStandAlone) {
4840	assert(MapperFunc && "MapperFunc missing for standalone target data");
4841	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(FnID: *MapperFunc),
4842	Args: OffloadingArgs);
4843	} else {
4844	Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
4845	FnID: omp::OMPRTL___tgt_target_data_begin_mapper);
4846
4847	Builder.CreateCall(Callee: BeginMapperFunc, Args: OffloadingArgs);
4848
4849	for (auto DeviceMap : Info.DevicePtrInfoMap) {
4850	if (isa<AllocaInst>(Val: DeviceMap.second.second)) {
4851	auto *LI =
4852	Builder.CreateLoad(Ty: Builder.getPtrTy(), Ptr: DeviceMap.second.first);
4853	Builder.CreateStore(Val: LI, Ptr: DeviceMap.second.second);
4854	}
4855	}
4856
4857	// If device pointer privatization is required, emit the body of the
4858	// region here. It will have to be duplicated: with and without
4859	// privatization.
4860	Builder.restoreIP(IP: BodyGenCB(Builder.saveIP(), BodyGenTy::Priv));
4861	}
4862	};
4863
4864	// If we need device pointer privatization, we need to emit the body of the
4865	// region with no privatization in the 'else' branch of the conditional.
4866	// Otherwise, we don't have to do anything.
4867	auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4868	Builder.restoreIP(IP: BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv));
4869	};
4870
4871	// Generate code for the closing of the data region.
4872	auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4873	TargetDataRTArgs RTArgs;
4874	emitOffloadingArraysArgument(Builder, RTArgs, Info, EmitDebug: !MapInfo->Names.empty(),
4875	/*ForEndCall=*/true);
4876
4877	// Emit the number of elements in the offloading arrays.
4878	Value *PointerNum = Builder.getInt32(C: Info.NumberOfPtrs);
4879
4880	// Source location for the ident struct
4881	if (!SrcLocInfo) {
4882	uint32_t SrcLocStrSize;
4883	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
4884	SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
4885	}
4886
4887	Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
4888	PointerNum, RTArgs.BasePointersArray,
4889	RTArgs.PointersArray, RTArgs.SizesArray,
4890	RTArgs.MapTypesArray, RTArgs.MapNamesArray,
4891	RTArgs.MappersArray};
4892	Function *EndMapperFunc =
4893	getOrCreateRuntimeFunctionPtr(FnID: omp::OMPRTL___tgt_target_data_end_mapper);
4894
4895	Builder.CreateCall(Callee: EndMapperFunc, Args: OffloadingArgs);
4896	};
4897
4898	// We don't have to do anything to close the region if the if clause evaluates
4899	// to false.
4900	auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4901
4902	if (BodyGenCB) {
4903	if (IfCond) {
4904	emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: BeginElseGen, AllocaIP);
4905	} else {
4906	BeginThenGen(AllocaIP, Builder.saveIP());
4907	}
4908
4909	// If we don't require privatization of device pointers, we emit the body in
4910	// between the runtime calls. This avoids duplicating the body code.
4911	Builder.restoreIP(IP: BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
4912
4913	if (IfCond) {
4914	emitIfClause(Cond: IfCond, ThenGen: EndThenGen, ElseGen: EndElseGen, AllocaIP);
4915	} else {
4916	EndThenGen(AllocaIP, Builder.saveIP());
4917	}
4918	} else {
4919	if (IfCond) {
4920	emitIfClause(Cond: IfCond, ThenGen: BeginThenGen, ElseGen: EndElseGen, AllocaIP);
4921	} else {
4922	BeginThenGen(AllocaIP, Builder.saveIP());
4923	}
4924	}
4925
4926	return Builder.saveIP();
4927	}
4928
4929	FunctionCallee
4930	OpenMPIRBuilder::createForStaticInitFunction(unsigned IVSize, bool IVSigned,
4931	bool IsGPUDistribute) {
4932	assert((IVSize == 32 || IVSize == 64) &&
4933	"IV size is not compatible with the omp runtime");
4934	RuntimeFunction Name;
4935	if (IsGPUDistribute)
4936	Name = IVSize == 32
4937	? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
4938	: omp::OMPRTL___kmpc_distribute_static_init_4u)
4939	: (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
4940	: omp::OMPRTL___kmpc_distribute_static_init_8u);
4941	else
4942	Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
4943	: omp::OMPRTL___kmpc_for_static_init_4u)
4944	: (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
4945	: omp::OMPRTL___kmpc_for_static_init_8u);
4946
4947	return getOrCreateRuntimeFunction(M, FnID: Name);
4948	}
4949
4950	FunctionCallee OpenMPIRBuilder::createDispatchInitFunction(unsigned IVSize,
4951	bool IVSigned) {
4952	assert((IVSize == 32 || IVSize == 64) &&
4953	"IV size is not compatible with the omp runtime");
4954	RuntimeFunction Name = IVSize == 32
4955	? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
4956	: omp::OMPRTL___kmpc_dispatch_init_4u)
4957	: (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
4958	: omp::OMPRTL___kmpc_dispatch_init_8u);
4959
4960	return getOrCreateRuntimeFunction(M, FnID: Name);
4961	}
4962
4963	FunctionCallee OpenMPIRBuilder::createDispatchNextFunction(unsigned IVSize,
4964	bool IVSigned) {
4965	assert((IVSize == 32 || IVSize == 64) &&
4966	"IV size is not compatible with the omp runtime");
4967	RuntimeFunction Name = IVSize == 32
4968	? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
4969	: omp::OMPRTL___kmpc_dispatch_next_4u)
4970	: (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
4971	: omp::OMPRTL___kmpc_dispatch_next_8u);
4972
4973	return getOrCreateRuntimeFunction(M, FnID: Name);
4974	}
4975
4976	FunctionCallee OpenMPIRBuilder::createDispatchFiniFunction(unsigned IVSize,
4977	bool IVSigned) {
4978	assert((IVSize == 32 || IVSize == 64) &&
4979	"IV size is not compatible with the omp runtime");
4980	RuntimeFunction Name = IVSize == 32
4981	? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
4982	: omp::OMPRTL___kmpc_dispatch_fini_4u)
4983	: (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
4984	: omp::OMPRTL___kmpc_dispatch_fini_8u);
4985
4986	return getOrCreateRuntimeFunction(M, FnID: Name);
4987	}
4988
4989	static void replaceConstatExprUsesInFuncWithInstr(ConstantExpr *ConstExpr,
4990	Function *Func) {
4991	for (User *User : make_early_inc_range(Range: ConstExpr->users()))
4992	if (auto *Instr = dyn_cast<Instruction>(Val: User))
4993	if (Instr->getFunction() == Func)
4994	Instr->replaceUsesOfWith(From: ConstExpr, To: ConstExpr->getAsInstruction(InsertBefore: Instr));
4995	}
4996
4997	static void replaceConstantValueUsesInFuncWithInstr(llvm::Value *Input,
4998	Function *Func) {
4999	for (User *User : make_early_inc_range(Range: Input->users()))
5000	if (auto *Const = dyn_cast<Constant>(Val: User))
5001	if (auto *ConstExpr = dyn_cast<ConstantExpr>(Val: Const))
5002	replaceConstatExprUsesInFuncWithInstr(ConstExpr, Func);
5003	}
5004
5005	static Function *createOutlinedFunction(
5006	OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName,
5007	SmallVectorImpl<Value *> &Inputs,
5008	OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
5009	OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
5010	SmallVector<Type *> ParameterTypes;
5011	if (OMPBuilder.Config.isTargetDevice()) {
5012	// Add the "implicit" runtime argument we use to provide launch specific
5013	// information for target devices.
5014	auto *Int8PtrTy = PointerType::getUnqual(C&: Builder.getContext());
5015	ParameterTypes.push_back(Elt: Int8PtrTy);
5016
5017	// All parameters to target devices are passed as pointers
5018	// or i64. This assumes 64-bit address spaces/pointers.
5019	for (auto &Arg : Inputs)
5020	ParameterTypes.push_back(Elt: Arg->getType()->isPointerTy()
5021	? Arg->getType()
5022	: Type::getInt64Ty(C&: Builder.getContext()));
5023	} else {
5024	for (auto &Arg : Inputs)
5025	ParameterTypes.push_back(Elt: Arg->getType());
5026	}
5027
5028	auto FuncType = FunctionType::get(Result: Builder.getVoidTy(), Params: ParameterTypes,
5029	/*isVarArg*/ false);
5030	auto Func = Function::Create(Ty: FuncType, Linkage: GlobalValue::InternalLinkage, N: FuncName,
5031	M: Builder.GetInsertBlock()->getModule());
5032
5033	// Save insert point.
5034	auto OldInsertPoint = Builder.saveIP();
5035
5036	// Generate the region into the function.
5037	BasicBlock *EntryBB = BasicBlock::Create(Context&: Builder.getContext(), Name: "entry", Parent: Func);
5038	Builder.SetInsertPoint(EntryBB);
5039
5040	// Insert target init call in the device compilation pass.
5041	if (OMPBuilder.Config.isTargetDevice())
5042	Builder.restoreIP(IP: OMPBuilder.createTargetInit(Loc: Builder, /*IsSPMD*/ false));
5043
5044	BasicBlock *UserCodeEntryBB = Builder.GetInsertBlock();
5045
5046	// Insert target deinit call in the device compilation pass.
5047	Builder.restoreIP(IP: CBFunc(Builder.saveIP(), Builder.saveIP()));
5048	if (OMPBuilder.Config.isTargetDevice())
5049	OMPBuilder.createTargetDeinit(Loc: Builder);
5050
5051	// Insert return instruction.
5052	Builder.CreateRetVoid();
5053
5054	// New Alloca IP at entry point of created device function.
5055	Builder.SetInsertPoint(EntryBB->getFirstNonPHI());
5056	auto AllocaIP = Builder.saveIP();
5057
5058	Builder.SetInsertPoint(UserCodeEntryBB->getFirstNonPHIOrDbg());
5059
5060	// Skip the artificial dyn_ptr on the device.
5061	const auto &ArgRange =
5062	OMPBuilder.Config.isTargetDevice()
5063	? make_range(x: Func->arg_begin() + 1, y: Func->arg_end())
5064	: Func->args();
5065
5066	// Rewrite uses of input valus to parameters.
5067	for (auto InArg : zip(t&: Inputs, u: ArgRange)) {
5068	Value *Input = std::get<0>(t&: InArg);
5069	Argument &Arg = std::get<1>(t&: InArg);
5070	Value *InputCopy = nullptr;
5071
5072	Builder.restoreIP(
5073	IP: ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.saveIP()));
5074
5075	// Things like GEP's can come in the form of Constants. Constants and
5076	// ConstantExpr's do not have access to the knowledge of what they're
5077	// contained in, so we must dig a little to find an instruction so we can
5078	// tell if they're used inside of the function we're outlining. We also
5079	// replace the original constant expression with a new instruction
5080	// equivalent; an instruction as it allows easy modification in the
5081	// following loop, as we can now know the constant (instruction) is owned by
5082	// our target function and replaceUsesOfWith can now be invoked on it
5083	// (cannot do this with constants it seems). A brand new one also allows us
5084	// to be cautious as it is perhaps possible the old expression was used
5085	// inside of the function but exists and is used externally (unlikely by the
5086	// nature of a Constant, but still).
5087	replaceConstantValueUsesInFuncWithInstr(Input, Func);
5088
5089	// Collect all the instructions
5090	for (User *User : make_early_inc_range(Range: Input->users()))
5091	if (auto *Instr = dyn_cast<Instruction>(Val: User))
5092	if (Instr->getFunction() == Func)
5093	Instr->replaceUsesOfWith(From: Input, To: InputCopy);
5094	}
5095
5096	// Restore insert point.
5097	Builder.restoreIP(IP: OldInsertPoint);
5098
5099	return Func;
5100	}
5101
5102	static void emitTargetOutlinedFunction(
5103	OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
5104	TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn,
5105	Constant *&OutlinedFnID, SmallVectorImpl<Value *> &Inputs,
5106	OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc,
5107	OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB) {
5108
5109	OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =
5110	[&OMPBuilder, &Builder, &Inputs, &CBFunc,
5111	&ArgAccessorFuncCB](StringRef EntryFnName) {
5112	return createOutlinedFunction(OMPBuilder, Builder, FuncName: EntryFnName, Inputs,
5113	CBFunc, ArgAccessorFuncCB);
5114	};
5115
5116	OMPBuilder.emitTargetRegionFunction(EntryInfo, GenerateFunctionCallback&: GenerateOutlinedFunction, IsOffloadEntry: true,
5117	OutlinedFn, OutlinedFnID);
5118	}
5119
5120	static void emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder,
5121	OpenMPIRBuilder::InsertPointTy AllocaIP,
5122	Function *OutlinedFn, Constant *OutlinedFnID,
5123	int32_t NumTeams, int32_t NumThreads,
5124	SmallVectorImpl<Value *> &Args,
5125	OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB) {
5126
5127	OpenMPIRBuilder::TargetDataInfo Info(
5128	/*RequiresDevicePointerInfo=*/false,
5129	/*SeparateBeginEndCalls=*/true);
5130
5131	OpenMPIRBuilder::MapInfosTy &MapInfo = GenMapInfoCB(Builder.saveIP());
5132	OMPBuilder.emitOffloadingArrays(AllocaIP, CodeGenIP: Builder.saveIP(), CombinedInfo&: MapInfo, Info,
5133	/*IsNonContiguous=*/true);
5134
5135	OpenMPIRBuilder::TargetDataRTArgs RTArgs;
5136	OMPBuilder.emitOffloadingArraysArgument(Builder, RTArgs, Info,
5137	EmitDebug: !MapInfo.Names.empty());
5138
5139	// emitKernelLaunch
5140	auto &&EmitTargetCallFallbackCB =
5141	[&](OpenMPIRBuilder::InsertPointTy IP) -> OpenMPIRBuilder::InsertPointTy {
5142	Builder.restoreIP(IP);
5143	Builder.CreateCall(Callee: OutlinedFn, Args);
5144	return Builder.saveIP();
5145	};
5146
5147	unsigned NumTargetItems = MapInfo.BasePointers.size();
5148	// TODO: Use correct device ID
5149	Value *DeviceID = Builder.getInt64(C: OMP_DEVICEID_UNDEF);
5150	Value *NumTeamsVal = Builder.getInt32(C: NumTeams);
5151	Value *NumThreadsVal = Builder.getInt32(C: NumThreads);
5152	uint32_t SrcLocStrSize;
5153	Constant *SrcLocStr = OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);
5154	Value *RTLoc = OMPBuilder.getOrCreateIdent(SrcLocStr, SrcLocStrSize,
5155	LocFlags: llvm::omp::IdentFlag(0), Reserve2Flags: 0);
5156	// TODO: Use correct NumIterations
5157	Value *NumIterations = Builder.getInt64(C: 0);
5158	// TODO: Use correct DynCGGroupMem
5159	Value *DynCGGroupMem = Builder.getInt32(C: 0);
5160
5161	bool HasNoWait = false;
5162
5163	OpenMPIRBuilder::TargetKernelArgs KArgs(NumTargetItems, RTArgs, NumIterations,
5164	NumTeamsVal, NumThreadsVal,
5165	DynCGGroupMem, HasNoWait);
5166
5167	Builder.restoreIP(IP: OMPBuilder.emitKernelLaunch(
5168	Loc: Builder, OutlinedFn, OutlinedFnID, emitTargetCallFallbackCB: EmitTargetCallFallbackCB, Args&: KArgs,
5169	DeviceID, RTLoc, AllocaIP));
5170	}
5171
5172	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTarget(
5173	const LocationDescription &Loc, InsertPointTy AllocaIP,
5174	InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo, int32_t NumTeams,
5175	int32_t NumThreads, SmallVectorImpl<Value *> &Args,
5176	GenMapInfoCallbackTy GenMapInfoCB,
5177	OpenMPIRBuilder::TargetBodyGenCallbackTy CBFunc,
5178	OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB) {
5179	if (!updateToLocation(Loc))
5180	return InsertPointTy();
5181
5182	Builder.restoreIP(IP: CodeGenIP);
5183
5184	Function *OutlinedFn;
5185	Constant *OutlinedFnID;
5186	emitTargetOutlinedFunction(OMPBuilder&: *this, Builder, EntryInfo, OutlinedFn,
5187	OutlinedFnID, Inputs&: Args, CBFunc, ArgAccessorFuncCB);
5188	if (!Config.isTargetDevice())
5189	emitTargetCall(OMPBuilder&: *this, Builder, AllocaIP, OutlinedFn, OutlinedFnID, NumTeams,
5190	NumThreads, Args, GenMapInfoCB);
5191
5192	return Builder.saveIP();
5193	}
5194
5195	std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
5196	StringRef FirstSeparator,
5197	StringRef Separator) {
5198	SmallString<128> Buffer;
5199	llvm::raw_svector_ostream OS(Buffer);
5200	StringRef Sep = FirstSeparator;
5201	for (StringRef Part : Parts) {
5202	OS << Sep << Part;
5203	Sep = Separator;
5204	}
5205	return OS.str().str();
5206	}
5207
5208	std::string
5209	OpenMPIRBuilder::createPlatformSpecificName(ArrayRef<StringRef> Parts) const {
5210	return OpenMPIRBuilder::getNameWithSeparators(Parts, FirstSeparator: Config.firstSeparator(),
5211	Separator: Config.separator());
5212	}
5213
5214	GlobalVariable *
5215	OpenMPIRBuilder::getOrCreateInternalVariable(Type *Ty, const StringRef &Name,
5216	unsigned AddressSpace) {
5217	auto &Elem = *InternalVars.try_emplace(Key: Name, Args: nullptr).first;
5218	if (Elem.second) {
5219	assert(Elem.second->getValueType() == Ty &&
5220	"OMP internal variable has different type than requested");
5221	} else {
5222	// TODO: investigate the appropriate linkage type used for the global
5223	// variable for possibly changing that to internal or private, or maybe
5224	// create different versions of the function for different OMP internal
5225	// variables.
5226	auto Linkage = this->M.getTargetTriple().rfind(s: "wasm32") == 0
5227	? GlobalValue::ExternalLinkage
5228	: GlobalValue::CommonLinkage;
5229	auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
5230	Constant::getNullValue(Ty), Elem.first(),
5231	/*InsertBefore=*/nullptr,
5232	GlobalValue::NotThreadLocal, AddressSpace);
5233	const DataLayout &DL = M.getDataLayout();
5234	const llvm::Align TypeAlign = DL.getABITypeAlign(Ty);
5235	const llvm::Align PtrAlign = DL.getPointerABIAlignment(AS: AddressSpace);
5236	GV->setAlignment(std::max(a: TypeAlign, b: PtrAlign));
5237	Elem.second = GV;
5238	}
5239
5240	return Elem.second;
5241	}
5242
5243	Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
5244	std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
5245	std::string Name = getNameWithSeparators(Parts: {Prefix, "var"}, FirstSeparator: ".", Separator: ".");
5246	return getOrCreateInternalVariable(Ty: KmpCriticalNameTy, Name);
5247	}
5248
5249	Value *OpenMPIRBuilder::getSizeInBytes(Value *BasePtr) {
5250	LLVMContext &Ctx = Builder.getContext();
5251	Value *Null =
5252	Constant::getNullValue(Ty: PointerType::getUnqual(C&: BasePtr->getContext()));
5253	Value *SizeGep =
5254	Builder.CreateGEP(Ty: BasePtr->getType(), Ptr: Null, IdxList: Builder.getInt32(C: 1));
5255	Value *SizePtrToInt = Builder.CreatePtrToInt(V: SizeGep, DestTy: Type::getInt64Ty(C&: Ctx));
5256	return SizePtrToInt;
5257	}
5258
5259	GlobalVariable *
5260	OpenMPIRBuilder::createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
5261	std::string VarName) {
5262	llvm::Constant *MaptypesArrayInit =
5263	llvm::ConstantDataArray::get(Context&: M.getContext(), Elts&: Mappings);
5264	auto *MaptypesArrayGlobal = new llvm::GlobalVariable(
5265	M, MaptypesArrayInit->getType(),
5266	/*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MaptypesArrayInit,
5267	VarName);
5268	MaptypesArrayGlobal->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
5269	return MaptypesArrayGlobal;
5270	}
5271
5272	void OpenMPIRBuilder::createMapperAllocas(const LocationDescription &Loc,
5273	InsertPointTy AllocaIP,
5274	unsigned NumOperands,
5275	struct MapperAllocas &MapperAllocas) {
5276	if (!updateToLocation(Loc))
5277	return;
5278
5279	auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
5280	auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
5281	Builder.restoreIP(IP: AllocaIP);
5282	AllocaInst *ArgsBase = Builder.CreateAlloca(
5283	Ty: ArrI8PtrTy, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
5284	AllocaInst *Args = Builder.CreateAlloca(Ty: ArrI8PtrTy, /* ArraySize = */ nullptr,
5285	Name: ".offload_ptrs");
5286	AllocaInst *ArgSizes = Builder.CreateAlloca(
5287	Ty: ArrI64Ty, /* ArraySize = */ nullptr, Name: ".offload_sizes");
5288	Builder.restoreIP(IP: Loc.IP);
5289	MapperAllocas.ArgsBase = ArgsBase;
5290	MapperAllocas.Args = Args;
5291	MapperAllocas.ArgSizes = ArgSizes;
5292	}
5293
5294	void OpenMPIRBuilder::emitMapperCall(const LocationDescription &Loc,
5295	Function *MapperFunc, Value *SrcLocInfo,
5296	Value *MaptypesArg, Value *MapnamesArg,
5297	struct MapperAllocas &MapperAllocas,
5298	int64_t DeviceID, unsigned NumOperands) {
5299	if (!updateToLocation(Loc))
5300	return;
5301
5302	auto *ArrI8PtrTy = ArrayType::get(ElementType: Int8Ptr, NumElements: NumOperands);
5303	auto *ArrI64Ty = ArrayType::get(ElementType: Int64, NumElements: NumOperands);
5304	Value *ArgsBaseGEP =
5305	Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.ArgsBase,
5306	IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
5307	Value *ArgsGEP =
5308	Builder.CreateInBoundsGEP(Ty: ArrI8PtrTy, Ptr: MapperAllocas.Args,
5309	IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
5310	Value *ArgSizesGEP =
5311	Builder.CreateInBoundsGEP(Ty: ArrI64Ty, Ptr: MapperAllocas.ArgSizes,
5312	IdxList: {Builder.getInt32(C: 0), Builder.getInt32(C: 0)});
5313	Value *NullPtr =
5314	Constant::getNullValue(Ty: PointerType::getUnqual(C&: Int8Ptr->getContext()));
5315	Builder.CreateCall(Callee: MapperFunc,
5316	Args: {SrcLocInfo, Builder.getInt64(C: DeviceID),
5317	Builder.getInt32(C: NumOperands), ArgsBaseGEP, ArgsGEP,
5318	ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
5319	}
5320
5321	void OpenMPIRBuilder::emitOffloadingArraysArgument(IRBuilderBase &Builder,
5322	TargetDataRTArgs &RTArgs,
5323	TargetDataInfo &Info,
5324	bool EmitDebug,
5325	bool ForEndCall) {
5326	assert((!ForEndCall || Info.separateBeginEndCalls()) &&
5327	"expected region end call to runtime only when end call is separate");
5328	auto UnqualPtrTy = PointerType::getUnqual(C&: M.getContext());
5329	auto VoidPtrTy = UnqualPtrTy;
5330	auto VoidPtrPtrTy = UnqualPtrTy;
5331	auto Int64Ty = Type::getInt64Ty(C&: M.getContext());
5332	auto Int64PtrTy = UnqualPtrTy;
5333
5334	if (!Info.NumberOfPtrs) {
5335	RTArgs.BasePointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5336	RTArgs.PointersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5337	RTArgs.SizesArray = ConstantPointerNull::get(T: Int64PtrTy);
5338	RTArgs.MapTypesArray = ConstantPointerNull::get(T: Int64PtrTy);
5339	RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5340	RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5341	return;
5342	}
5343
5344	RTArgs.BasePointersArray = Builder.CreateConstInBoundsGEP2_32(
5345	Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs),
5346	Ptr: Info.RTArgs.BasePointersArray,
5347	/*Idx0=*/0, /*Idx1=*/0);
5348	RTArgs.PointersArray = Builder.CreateConstInBoundsGEP2_32(
5349	Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray,
5350	/*Idx0=*/0,
5351	/*Idx1=*/0);
5352	RTArgs.SizesArray = Builder.CreateConstInBoundsGEP2_32(
5353	Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
5354	/*Idx0=*/0, /*Idx1=*/0);
5355	RTArgs.MapTypesArray = Builder.CreateConstInBoundsGEP2_32(
5356	Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs),
5357	Ptr: ForEndCall && Info.RTArgs.MapTypesArrayEnd ? Info.RTArgs.MapTypesArrayEnd
5358	: Info.RTArgs.MapTypesArray,
5359	/*Idx0=*/0,
5360	/*Idx1=*/0);
5361
5362	// Only emit the mapper information arrays if debug information is
5363	// requested.
5364	if (!EmitDebug)
5365	RTArgs.MapNamesArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5366	else
5367	RTArgs.MapNamesArray = Builder.CreateConstInBoundsGEP2_32(
5368	Ty: ArrayType::get(ElementType: VoidPtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.MapNamesArray,
5369	/*Idx0=*/0,
5370	/*Idx1=*/0);
5371	// If there is no user-defined mapper, set the mapper array to nullptr to
5372	// avoid an unnecessary data privatization
5373	if (!Info.HasMapper)
5374	RTArgs.MappersArray = ConstantPointerNull::get(T: VoidPtrPtrTy);
5375	else
5376	RTArgs.MappersArray =
5377	Builder.CreatePointerCast(V: Info.RTArgs.MappersArray, DestTy: VoidPtrPtrTy);
5378	}
5379
5380	void OpenMPIRBuilder::emitNonContiguousDescriptor(InsertPointTy AllocaIP,
5381	InsertPointTy CodeGenIP,
5382	MapInfosTy &CombinedInfo,
5383	TargetDataInfo &Info) {
5384	MapInfosTy::StructNonContiguousInfo &NonContigInfo =
5385	CombinedInfo.NonContigInfo;
5386
5387	// Build an array of struct descriptor_dim and then assign it to
5388	// offload_args.
5389	//
5390	// struct descriptor_dim {
5391	// uint64_t offset;
5392	// uint64_t count;
5393	// uint64_t stride
5394	// };
5395	Type *Int64Ty = Builder.getInt64Ty();
5396	StructType *DimTy = StructType::create(
5397	Context&: M.getContext(), Elements: ArrayRef<Type *>({Int64Ty, Int64Ty, Int64Ty}),
5398	Name: "struct.descriptor_dim");
5399
5400	enum { OffsetFD = 0, CountFD, StrideFD };
5401	// We need two index variable here since the size of "Dims" is the same as
5402	// the size of Components, however, the size of offset, count, and stride is
5403	// equal to the size of base declaration that is non-contiguous.
5404	for (unsigned I = 0, L = 0, E = NonContigInfo.Dims.size(); I < E; ++I) {
5405	// Skip emitting ir if dimension size is 1 since it cannot be
5406	// non-contiguous.
5407	if (NonContigInfo.Dims[I] == 1)
5408	continue;
5409	Builder.restoreIP(IP: AllocaIP);
5410	ArrayType *ArrayTy = ArrayType::get(ElementType: DimTy, NumElements: NonContigInfo.Dims[I]);
5411	AllocaInst *DimsAddr =
5412	Builder.CreateAlloca(Ty: ArrayTy, /* ArraySize = */ nullptr, Name: "dims");
5413	Builder.restoreIP(IP: CodeGenIP);
5414	for (unsigned II = 0, EE = NonContigInfo.Dims[I]; II < EE; ++II) {
5415	unsigned RevIdx = EE - II - 1;
5416	Value *DimsLVal = Builder.CreateInBoundsGEP(
5417	Ty: DimsAddr->getAllocatedType(), Ptr: DimsAddr,
5418	IdxList: {Builder.getInt64(C: 0), Builder.getInt64(C: II)});
5419	// Offset
5420	Value *OffsetLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: OffsetFD);
5421	Builder.CreateAlignedStore(
5422	Val: NonContigInfo.Offsets[L][RevIdx], Ptr: OffsetLVal,
5423	Align: M.getDataLayout().getPrefTypeAlign(Ty: OffsetLVal->getType()));
5424	// Count
5425	Value *CountLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: CountFD);
5426	Builder.CreateAlignedStore(
5427	Val: NonContigInfo.Counts[L][RevIdx], Ptr: CountLVal,
5428	Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
5429	// Stride
5430	Value *StrideLVal = Builder.CreateStructGEP(Ty: DimTy, Ptr: DimsLVal, Idx: StrideFD);
5431	Builder.CreateAlignedStore(
5432	Val: NonContigInfo.Strides[L][RevIdx], Ptr: StrideLVal,
5433	Align: M.getDataLayout().getPrefTypeAlign(Ty: CountLVal->getType()));
5434	}
5435	// args[I] = &dims
5436	Builder.restoreIP(IP: CodeGenIP);
5437	Value *DAddr = Builder.CreatePointerBitCastOrAddrSpaceCast(
5438	V: DimsAddr, DestTy: Builder.getPtrTy());
5439	Value *P = Builder.CreateConstInBoundsGEP2_32(
5440	Ty: ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs),
5441	Ptr: Info.RTArgs.PointersArray, Idx0: 0, Idx1: I);
5442	Builder.CreateAlignedStore(
5443	Val: DAddr, Ptr: P, Align: M.getDataLayout().getPrefTypeAlign(Ty: Builder.getPtrTy()));
5444	++L;
5445	}
5446	}
5447
5448	void OpenMPIRBuilder::emitOffloadingArrays(
5449	InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo,
5450	TargetDataInfo &Info, bool IsNonContiguous,
5451	function_ref<void(unsigned int, Value *)> DeviceAddrCB,
5452	function_ref<Value *(unsigned int)> CustomMapperCB) {
5453
5454	// Reset the array information.
5455	Info.clearArrayInfo();
5456	Info.NumberOfPtrs = CombinedInfo.BasePointers.size();
5457
5458	if (Info.NumberOfPtrs == 0)
5459	return;
5460
5461	Builder.restoreIP(IP: AllocaIP);
5462	// Detect if we have any capture size requiring runtime evaluation of the
5463	// size so that a constant array could be eventually used.
5464	ArrayType *PointerArrayType =
5465	ArrayType::get(ElementType: Builder.getPtrTy(), NumElements: Info.NumberOfPtrs);
5466
5467	Info.RTArgs.BasePointersArray = Builder.CreateAlloca(
5468	Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_baseptrs");
5469
5470	Info.RTArgs.PointersArray = Builder.CreateAlloca(
5471	Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_ptrs");
5472	AllocaInst *MappersArray = Builder.CreateAlloca(
5473	Ty: PointerArrayType, /* ArraySize = */ nullptr, Name: ".offload_mappers");
5474	Info.RTArgs.MappersArray = MappersArray;
5475
5476	// If we don't have any VLA types or other types that require runtime
5477	// evaluation, we can use a constant array for the map sizes, otherwise we
5478	// need to fill up the arrays as we do for the pointers.
5479	Type *Int64Ty = Builder.getInt64Ty();
5480	SmallVector<Constant *> ConstSizes(CombinedInfo.Sizes.size(),
5481	ConstantInt::get(Ty: Int64Ty, V: 0));
5482	SmallBitVector RuntimeSizes(CombinedInfo.Sizes.size());
5483	for (unsigned I = 0, E = CombinedInfo.Sizes.size(); I < E; ++I) {
5484	if (auto *CI = dyn_cast<Constant>(Val: CombinedInfo.Sizes[I])) {
5485	if (!isa<ConstantExpr>(Val: CI) && !isa<GlobalValue>(Val: CI)) {
5486	if (IsNonContiguous &&
5487	static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
5488	CombinedInfo.Types[I] &
5489	OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
5490	ConstSizes[I] =
5491	ConstantInt::get(Ty: Int64Ty, V: CombinedInfo.NonContigInfo.Dims[I]);
5492	else
5493	ConstSizes[I] = CI;
5494	continue;
5495	}
5496	}
5497	RuntimeSizes.set(I);
5498	}
5499
5500	if (RuntimeSizes.all()) {
5501	ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
5502	Info.RTArgs.SizesArray = Builder.CreateAlloca(
5503	Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
5504	Builder.restoreIP(IP: CodeGenIP);
5505	} else {
5506	auto *SizesArrayInit = ConstantArray::get(
5507	T: ArrayType::get(ElementType: Int64Ty, NumElements: ConstSizes.size()), V: ConstSizes);
5508	std::string Name = createPlatformSpecificName(Parts: {"offload_sizes"});
5509	auto *SizesArrayGbl =
5510	new GlobalVariable(M, SizesArrayInit->getType(), /*isConstant=*/true,
5511	GlobalValue::PrivateLinkage, SizesArrayInit, Name);
5512	SizesArrayGbl->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
5513
5514	if (!RuntimeSizes.any()) {
5515	Info.RTArgs.SizesArray = SizesArrayGbl;
5516	} else {
5517	unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
5518	Align OffloadSizeAlign = M.getDataLayout().getABIIntegerTypeAlignment(BitWidth: 64);
5519	ArrayType *SizeArrayType = ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs);
5520	AllocaInst *Buffer = Builder.CreateAlloca(
5521	Ty: SizeArrayType, /* ArraySize = */ nullptr, Name: ".offload_sizes");
5522	Buffer->setAlignment(OffloadSizeAlign);
5523	Builder.restoreIP(IP: CodeGenIP);
5524	Builder.CreateMemCpy(
5525	Dst: Buffer, DstAlign: M.getDataLayout().getPrefTypeAlign(Ty: Buffer->getType()),
5526	Src: SizesArrayGbl, SrcAlign: OffloadSizeAlign,
5527	Size: Builder.getIntN(
5528	N: IndexSize,
5529	C: Buffer->getAllocationSize(DL: M.getDataLayout())->getFixedValue()));
5530
5531	Info.RTArgs.SizesArray = Buffer;
5532	}
5533	Builder.restoreIP(IP: CodeGenIP);
5534	}
5535
5536	// The map types are always constant so we don't need to generate code to
5537	// fill arrays. Instead, we create an array constant.
5538	SmallVector<uint64_t, 4> Mapping;
5539	for (auto mapFlag : CombinedInfo.Types)
5540	Mapping.push_back(
5541	Elt: static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
5542	mapFlag));
5543	std::string MaptypesName = createPlatformSpecificName(Parts: {"offload_maptypes"});
5544	auto *MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
5545	Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
5546
5547	// The information types are only built if provided.
5548	if (!CombinedInfo.Names.empty()) {
5549	std::string MapnamesName = createPlatformSpecificName(Parts: {"offload_mapnames"});
5550	auto *MapNamesArrayGbl =
5551	createOffloadMapnames(Names&: CombinedInfo.Names, VarName: MapnamesName);
5552	Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
5553	} else {
5554	Info.RTArgs.MapNamesArray =
5555	Constant::getNullValue(Ty: PointerType::getUnqual(C&: Builder.getContext()));
5556	}
5557
5558	// If there's a present map type modifier, it must not be applied to the end
5559	// of a region, so generate a separate map type array in that case.
5560	if (Info.separateBeginEndCalls()) {
5561	bool EndMapTypesDiffer = false;
5562	for (uint64_t &Type : Mapping) {
5563	if (Type & static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
5564	OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
5565	Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
5566	OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
5567	EndMapTypesDiffer = true;
5568	}
5569	}
5570	if (EndMapTypesDiffer) {
5571	MapTypesArrayGbl = createOffloadMaptypes(Mappings&: Mapping, VarName: MaptypesName);
5572	Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
5573	}
5574	}
5575
5576	PointerType *PtrTy = Builder.getPtrTy();
5577	for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) {
5578	Value *BPVal = CombinedInfo.BasePointers[I];
5579	Value *BP = Builder.CreateConstInBoundsGEP2_32(
5580	Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.BasePointersArray,
5581	Idx0: 0, Idx1: I);
5582	Builder.CreateAlignedStore(Val: BPVal, Ptr: BP,
5583	Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
5584
5585	if (Info.requiresDevicePointerInfo()) {
5586	if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Pointer) {
5587	CodeGenIP = Builder.saveIP();
5588	Builder.restoreIP(IP: AllocaIP);
5589	Info.DevicePtrInfoMap[BPVal] = {BP, Builder.CreateAlloca(Ty: PtrTy)};
5590	Builder.restoreIP(IP: CodeGenIP);
5591	if (DeviceAddrCB)
5592	DeviceAddrCB(I, Info.DevicePtrInfoMap[BPVal].second);
5593	} else if (CombinedInfo.DevicePointers[I] == DeviceInfoTy::Address) {
5594	Info.DevicePtrInfoMap[BPVal] = {BP, BP};
5595	if (DeviceAddrCB)
5596	DeviceAddrCB(I, BP);
5597	}
5598	}
5599
5600	Value *PVal = CombinedInfo.Pointers[I];
5601	Value *P = Builder.CreateConstInBoundsGEP2_32(
5602	Ty: ArrayType::get(ElementType: PtrTy, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.PointersArray, Idx0: 0,
5603	Idx1: I);
5604	// TODO: Check alignment correct.
5605	Builder.CreateAlignedStore(Val: PVal, Ptr: P,
5606	Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
5607
5608	if (RuntimeSizes.test(Idx: I)) {
5609	Value *S = Builder.CreateConstInBoundsGEP2_32(
5610	Ty: ArrayType::get(ElementType: Int64Ty, NumElements: Info.NumberOfPtrs), Ptr: Info.RTArgs.SizesArray,
5611	/*Idx0=*/0,
5612	/*Idx1=*/I);
5613	Builder.CreateAlignedStore(Val: Builder.CreateIntCast(V: CombinedInfo.Sizes[I],
5614	DestTy: Int64Ty,
5615	/*isSigned=*/true),
5616	Ptr: S, Align: M.getDataLayout().getPrefTypeAlign(Ty: PtrTy));
5617	}
5618	// Fill up the mapper array.
5619	unsigned IndexSize = M.getDataLayout().getIndexSizeInBits(AS: 0);
5620	Value *MFunc = ConstantPointerNull::get(T: PtrTy);
5621	if (CustomMapperCB)
5622	if (Value *CustomMFunc = CustomMapperCB(I))
5623	MFunc = Builder.CreatePointerCast(V: CustomMFunc, DestTy: PtrTy);
5624	Value *MAddr = Builder.CreateInBoundsGEP(
5625	Ty: MappersArray->getAllocatedType(), Ptr: MappersArray,
5626	IdxList: {Builder.getIntN(N: IndexSize, C: 0), Builder.getIntN(N: IndexSize, C: I)});
5627	Builder.CreateAlignedStore(
5628	Val: MFunc, Ptr: MAddr, Align: M.getDataLayout().getPrefTypeAlign(Ty: MAddr->getType()));
5629	}
5630
5631	if (!IsNonContiguous || CombinedInfo.NonContigInfo.Offsets.empty() ||
5632	Info.NumberOfPtrs == 0)
5633	return;
5634	emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
5635	}
5636
5637	void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
5638	BasicBlock *CurBB = Builder.GetInsertBlock();
5639
5640	if (!CurBB || CurBB->getTerminator()) {
5641	// If there is no insert point or the previous block is already
5642	// terminated, don't touch it.
5643	} else {
5644	// Otherwise, create a fall-through branch.
5645	Builder.CreateBr(Dest: Target);
5646	}
5647
5648	Builder.ClearInsertionPoint();
5649	}
5650
5651	void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
5652	bool IsFinished) {
5653	BasicBlock *CurBB = Builder.GetInsertBlock();
5654
5655	// Fall out of the current block (if necessary).
5656	emitBranch(Target: BB);
5657
5658	if (IsFinished && BB->use_empty()) {
5659	BB->eraseFromParent();
5660	return;
5661	}
5662
5663	// Place the block after the current block, if possible, or else at
5664	// the end of the function.
5665	if (CurBB && CurBB->getParent())
5666	CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
5667	else
5668	CurFn->insert(Position: CurFn->end(), BB);
5669	Builder.SetInsertPoint(BB);
5670	}
5671
5672	void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
5673	BodyGenCallbackTy ElseGen,
5674	InsertPointTy AllocaIP) {
5675	// If the condition constant folds and can be elided, try to avoid emitting
5676	// the condition and the dead arm of the if/else.
5677	if (auto *CI = dyn_cast<ConstantInt>(Val: Cond)) {
5678	auto CondConstant = CI->getSExtValue();
5679	if (CondConstant)
5680	ThenGen(AllocaIP, Builder.saveIP());
5681	else
5682	ElseGen(AllocaIP, Builder.saveIP());
5683	return;
5684	}
5685
5686	Function *CurFn = Builder.GetInsertBlock()->getParent();
5687
5688	// Otherwise, the condition did not fold, or we couldn't elide it. Just
5689	// emit the conditional branch.
5690	BasicBlock *ThenBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.then");
5691	BasicBlock *ElseBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.else");
5692	BasicBlock *ContBlock = BasicBlock::Create(Context&: M.getContext(), Name: "omp_if.end");
5693	Builder.CreateCondBr(Cond, True: ThenBlock, False: ElseBlock);
5694	// Emit the 'then' code.
5695	emitBlock(BB: ThenBlock, CurFn);
5696	ThenGen(AllocaIP, Builder.saveIP());
5697	emitBranch(Target: ContBlock);
5698	// Emit the 'else' code if present.
5699	// There is no need to emit line number for unconditional branch.
5700	emitBlock(BB: ElseBlock, CurFn);
5701	ElseGen(AllocaIP, Builder.saveIP());
5702	// There is no need to emit line number for unconditional branch.
5703	emitBranch(Target: ContBlock);
5704	// Emit the continuation block for code after the if.
5705	emitBlock(BB: ContBlock, CurFn, /*IsFinished=*/true);
5706	}
5707
5708	bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
5709	const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
5710	assert(!(AO == AtomicOrdering::NotAtomic ||
5711	AO == llvm::AtomicOrdering::Unordered) &&
5712	"Unexpected Atomic Ordering.");
5713
5714	bool Flush = false;
5715	llvm::AtomicOrdering FlushAO = AtomicOrdering::Monotonic;
5716
5717	switch (AK) {
5718	case Read:
5719	if (AO == AtomicOrdering::Acquire || AO == AtomicOrdering::AcquireRelease ||
5720	AO == AtomicOrdering::SequentiallyConsistent) {
5721	FlushAO = AtomicOrdering::Acquire;
5722	Flush = true;
5723	}
5724	break;
5725	case Write:
5726	case Compare:
5727	case Update:
5728	if (AO == AtomicOrdering::Release || AO == AtomicOrdering::AcquireRelease ||
5729	AO == AtomicOrdering::SequentiallyConsistent) {
5730	FlushAO = AtomicOrdering::Release;
5731	Flush = true;
5732	}
5733	break;
5734	case Capture:
5735	switch (AO) {
5736	case AtomicOrdering::Acquire:
5737	FlushAO = AtomicOrdering::Acquire;
5738	Flush = true;
5739	break;
5740	case AtomicOrdering::Release:
5741	FlushAO = AtomicOrdering::Release;
5742	Flush = true;
5743	break;
5744	case AtomicOrdering::AcquireRelease:
5745	case AtomicOrdering::SequentiallyConsistent:
5746	FlushAO = AtomicOrdering::AcquireRelease;
5747	Flush = true;
5748	break;
5749	default:
5750	// do nothing - leave silently.
5751	break;
5752	}
5753	}
5754
5755	if (Flush) {
5756	// Currently Flush RT call still doesn't take memory_ordering, so for when
5757	// that happens, this tries to do the resolution of which atomic ordering
5758	// to use with but issue the flush call
5759	// TODO: pass `FlushAO` after memory ordering support is added
5760	(void)FlushAO;
5761	emitFlush(Loc);
5762	}
5763
5764	// for AO == AtomicOrdering::Monotonic and all other case combinations
5765	// do nothing
5766	return Flush;
5767	}
5768
5769	OpenMPIRBuilder::InsertPointTy
5770	OpenMPIRBuilder::createAtomicRead(const LocationDescription &Loc,
5771	AtomicOpValue &X, AtomicOpValue &V,
5772	AtomicOrdering AO) {
5773	if (!updateToLocation(Loc))
5774	return Loc.IP;
5775
5776	assert(X.Var->getType()->isPointerTy() &&
5777	"OMP Atomic expects a pointer to target memory");
5778	Type *XElemTy = X.ElemTy;
5779	assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
5780	XElemTy->isPointerTy()) &&
5781	"OMP atomic read expected a scalar type");
5782
5783	Value *XRead = nullptr;
5784
5785	if (XElemTy->isIntegerTy()) {
5786	LoadInst *XLD =
5787	Builder.CreateLoad(Ty: XElemTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.read");
5788	XLD->setAtomic(Ordering: AO);
5789	XRead = cast<Value>(Val: XLD);
5790	} else {
5791	// We need to perform atomic op as integer
5792	IntegerType *IntCastTy =
5793	IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
5794	LoadInst *XLoad =
5795	Builder.CreateLoad(Ty: IntCastTy, Ptr: X.Var, isVolatile: X.IsVolatile, Name: "omp.atomic.load");
5796	XLoad->setAtomic(Ordering: AO);
5797	if (XElemTy->isFloatingPointTy()) {
5798	XRead = Builder.CreateBitCast(V: XLoad, DestTy: XElemTy, Name: "atomic.flt.cast");
5799	} else {
5800	XRead = Builder.CreateIntToPtr(V: XLoad, DestTy: XElemTy, Name: "atomic.ptr.cast");
5801	}
5802	}
5803	checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Read);
5804	Builder.CreateStore(Val: XRead, Ptr: V.Var, isVolatile: V.IsVolatile);
5805	return Builder.saveIP();
5806	}
5807
5808	OpenMPIRBuilder::InsertPointTy
5809	OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
5810	AtomicOpValue &X, Value *Expr,
5811	AtomicOrdering AO) {
5812	if (!updateToLocation(Loc))
5813	return Loc.IP;
5814
5815	assert(X.Var->getType()->isPointerTy() &&
5816	"OMP Atomic expects a pointer to target memory");
5817	Type *XElemTy = X.ElemTy;
5818	assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
5819	XElemTy->isPointerTy()) &&
5820	"OMP atomic write expected a scalar type");
5821
5822	if (XElemTy->isIntegerTy()) {
5823	StoreInst *XSt = Builder.CreateStore(Val: Expr, Ptr: X.Var, isVolatile: X.IsVolatile);
5824	XSt->setAtomic(Ordering: AO);
5825	} else {
5826	// We need to bitcast and perform atomic op as integers
5827	IntegerType *IntCastTy =
5828	IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
5829	Value *ExprCast =
5830	Builder.CreateBitCast(V: Expr, DestTy: IntCastTy, Name: "atomic.src.int.cast");
5831	StoreInst *XSt = Builder.CreateStore(Val: ExprCast, Ptr: X.Var, isVolatile: X.IsVolatile);
5832	XSt->setAtomic(Ordering: AO);
5833	}
5834
5835	checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Write);
5836	return Builder.saveIP();
5837	}
5838
5839	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicUpdate(
5840	const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
5841	Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
5842	AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
5843	assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
5844	if (!updateToLocation(Loc))
5845	return Loc.IP;
5846
5847	LLVM_DEBUG({
5848	Type *XTy = X.Var->getType();
5849	assert(XTy->isPointerTy() &&
5850	"OMP Atomic expects a pointer to target memory");
5851	Type *XElemTy = X.ElemTy;
5852	assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
5853	XElemTy->isPointerTy()) &&
5854	"OMP atomic update expected a scalar type");
5855	assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
5856	(RMWOp != AtomicRMWInst::UMax) && (RMWOp != AtomicRMWInst::UMin) &&
5857	"OpenMP atomic does not support LT or GT operations");
5858	});
5859
5860	emitAtomicUpdate(AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp, UpdateOp,
5861	VolatileX: X.IsVolatile, IsXBinopExpr);
5862	checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Update);
5863	return Builder.saveIP();
5864	}
5865
5866	// FIXME: Duplicating AtomicExpand
5867	Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
5868	AtomicRMWInst::BinOp RMWOp) {
5869	switch (RMWOp) {
5870	case AtomicRMWInst::Add:
5871	return Builder.CreateAdd(LHS: Src1, RHS: Src2);
5872	case AtomicRMWInst::Sub:
5873	return Builder.CreateSub(LHS: Src1, RHS: Src2);
5874	case AtomicRMWInst::And:
5875	return Builder.CreateAnd(LHS: Src1, RHS: Src2);
5876	case AtomicRMWInst::Nand:
5877	return Builder.CreateNeg(V: Builder.CreateAnd(LHS: Src1, RHS: Src2));
5878	case AtomicRMWInst::Or:
5879	return Builder.CreateOr(LHS: Src1, RHS: Src2);
5880	case AtomicRMWInst::Xor:
5881	return Builder.CreateXor(LHS: Src1, RHS: Src2);
5882	case AtomicRMWInst::Xchg:
5883	case AtomicRMWInst::FAdd:
5884	case AtomicRMWInst::FSub:
5885	case AtomicRMWInst::BAD_BINOP:
5886	case AtomicRMWInst::Max:
5887	case AtomicRMWInst::Min:
5888	case AtomicRMWInst::UMax:
5889	case AtomicRMWInst::UMin:
5890	case AtomicRMWInst::FMax:
5891	case AtomicRMWInst::FMin:
5892	case AtomicRMWInst::UIncWrap:
5893	case AtomicRMWInst::UDecWrap:
5894	llvm_unreachable("Unsupported atomic update operation");
5895	}
5896	llvm_unreachable("Unsupported atomic update operation");
5897	}
5898
5899	std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
5900	InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
5901	AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
5902	AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
5903	// TODO: handle the case where XElemTy is not byte-sized or not a power of 2
5904	// or a complex datatype.
5905	bool emitRMWOp = false;
5906	switch (RMWOp) {
5907	case AtomicRMWInst::Add:
5908	case AtomicRMWInst::And:
5909	case AtomicRMWInst::Nand:
5910	case AtomicRMWInst::Or:
5911	case AtomicRMWInst::Xor:
5912	case AtomicRMWInst::Xchg:
5913	emitRMWOp = XElemTy;
5914	break;
5915	case AtomicRMWInst::Sub:
5916	emitRMWOp = (IsXBinopExpr && XElemTy);
5917	break;
5918	default:
5919	emitRMWOp = false;
5920	}
5921	emitRMWOp &= XElemTy->isIntegerTy();
5922
5923	std::pair<Value *, Value *> Res;
5924	if (emitRMWOp) {
5925	Res.first = Builder.CreateAtomicRMW(Op: RMWOp, Ptr: X, Val: Expr, Align: llvm::MaybeAlign(), Ordering: AO);
5926	// not needed except in case of postfix captures. Generate anyway for
5927	// consistency with the else part. Will be removed with any DCE pass.
5928	// AtomicRMWInst::Xchg does not have a coressponding instruction.
5929	if (RMWOp == AtomicRMWInst::Xchg)
5930	Res.second = Res.first;
5931	else
5932	Res.second = emitRMWOpAsInstruction(Src1: Res.first, Src2: Expr, RMWOp);
5933	} else {
5934	IntegerType *IntCastTy =
5935	IntegerType::get(C&: M.getContext(), NumBits: XElemTy->getScalarSizeInBits());
5936	LoadInst *OldVal =
5937	Builder.CreateLoad(Ty: IntCastTy, Ptr: X, Name: X->getName() + ".atomic.load");
5938	OldVal->setAtomic(Ordering: AO);
5939	// CurBB
5940	// | /---\
5941	// ContBB |
5942	// | \---/
5943	// ExitBB
5944	BasicBlock *CurBB = Builder.GetInsertBlock();
5945	Instruction *CurBBTI = CurBB->getTerminator();
5946	CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
5947	BasicBlock *ExitBB =
5948	CurBB->splitBasicBlock(I: CurBBTI, BBName: X->getName() + ".atomic.exit");
5949	BasicBlock *ContBB = CurBB->splitBasicBlock(I: CurBB->getTerminator(),
5950	BBName: X->getName() + ".atomic.cont");
5951	ContBB->getTerminator()->eraseFromParent();
5952	Builder.restoreIP(IP: AllocaIP);
5953	AllocaInst *NewAtomicAddr = Builder.CreateAlloca(Ty: XElemTy);
5954	NewAtomicAddr->setName(X->getName() + "x.new.val");
5955	Builder.SetInsertPoint(ContBB);
5956	llvm::PHINode *PHI = Builder.CreatePHI(Ty: OldVal->getType(), NumReservedValues: 2);
5957	PHI->addIncoming(V: OldVal, BB: CurBB);
5958	bool IsIntTy = XElemTy->isIntegerTy();
5959	Value *OldExprVal = PHI;
5960	if (!IsIntTy) {
5961	if (XElemTy->isFloatingPointTy()) {
5962	OldExprVal = Builder.CreateBitCast(V: PHI, DestTy: XElemTy,
5963	Name: X->getName() + ".atomic.fltCast");
5964	} else {
5965	OldExprVal = Builder.CreateIntToPtr(V: PHI, DestTy: XElemTy,
5966	Name: X->getName() + ".atomic.ptrCast");
5967	}
5968	}
5969
5970	Value *Upd = UpdateOp(OldExprVal, Builder);
5971	Builder.CreateStore(Val: Upd, Ptr: NewAtomicAddr);
5972	LoadInst *DesiredVal = Builder.CreateLoad(Ty: IntCastTy, Ptr: NewAtomicAddr);
5973	AtomicOrdering Failure =
5974	llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
5975	AtomicCmpXchgInst *Result = Builder.CreateAtomicCmpXchg(
5976	Ptr: X, Cmp: PHI, New: DesiredVal, Align: llvm::MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
5977	Result->setVolatile(VolatileX);
5978	Value *PreviousVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
5979	Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
5980	PHI->addIncoming(V: PreviousVal, BB: Builder.GetInsertBlock());
5981	Builder.CreateCondBr(Cond: SuccessFailureVal, True: ExitBB, False: ContBB);
5982
5983	Res.first = OldExprVal;
5984	Res.second = Upd;
5985
5986	// set Insertion point in exit block
5987	if (UnreachableInst *ExitTI =
5988	dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
5989	CurBBTI->eraseFromParent();
5990	Builder.SetInsertPoint(ExitBB);
5991	} else {
5992	Builder.SetInsertPoint(ExitTI);
5993	}
5994	}
5995
5996	return Res;
5997	}
5998
5999	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCapture(
6000	const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
6001	AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
6002	AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
6003	bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
6004	if (!updateToLocation(Loc))
6005	return Loc.IP;
6006
6007	LLVM_DEBUG({
6008	Type *XTy = X.Var->getType();
6009	assert(XTy->isPointerTy() &&
6010	"OMP Atomic expects a pointer to target memory");
6011	Type *XElemTy = X.ElemTy;
6012	assert((XElemTy->isFloatingPointTy() || XElemTy->isIntegerTy() ||
6013	XElemTy->isPointerTy()) &&
6014	"OMP atomic capture expected a scalar type");
6015	assert((RMWOp != AtomicRMWInst::Max) && (RMWOp != AtomicRMWInst::Min) &&
6016	"OpenMP atomic does not support LT or GT operations");
6017	});
6018
6019	// If UpdateExpr is 'x' updated with some `expr` not based on 'x',
6020	// 'x' is simply atomically rewritten with 'expr'.
6021	AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
6022	std::pair<Value *, Value *> Result =
6023	emitAtomicUpdate(AllocaIP, X: X.Var, XElemTy: X.ElemTy, Expr, AO, RMWOp: AtomicOp, UpdateOp,
6024	VolatileX: X.IsVolatile, IsXBinopExpr);
6025
6026	Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
6027	Builder.CreateStore(Val: CapturedVal, Ptr: V.Var, isVolatile: V.IsVolatile);
6028
6029	checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Capture);
6030	return Builder.saveIP();
6031	}
6032
6033	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
6034	const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
6035	AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
6036	omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
6037	bool IsFailOnly) {
6038
6039	AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
6040	return createAtomicCompare(Loc, X, V, R, E, D, AO, Op, IsXBinopExpr,
6041	IsPostfixUpdate, IsFailOnly, Failure);
6042	}
6043
6044	OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare(
6045	const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V,
6046	AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO,
6047	omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate,
6048	bool IsFailOnly, AtomicOrdering Failure) {
6049
6050	if (!updateToLocation(Loc))
6051	return Loc.IP;
6052
6053	assert(X.Var->getType()->isPointerTy() &&
6054	"OMP atomic expects a pointer to target memory");
6055	// compare capture
6056	if (V.Var) {
6057	assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type");
6058	assert(V.ElemTy == X.ElemTy && "x and v must be of same type");
6059	}
6060
6061	bool IsInteger = E->getType()->isIntegerTy();
6062
6063	if (Op == OMPAtomicCompareOp::EQ) {
6064	AtomicCmpXchgInst *Result = nullptr;
6065	if (!IsInteger) {
6066	IntegerType *IntCastTy =
6067	IntegerType::get(C&: M.getContext(), NumBits: X.ElemTy->getScalarSizeInBits());
6068	Value *EBCast = Builder.CreateBitCast(V: E, DestTy: IntCastTy);
6069	Value *DBCast = Builder.CreateBitCast(V: D, DestTy: IntCastTy);
6070	Result = Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: EBCast, New: DBCast, Align: MaybeAlign(),
6071	SuccessOrdering: AO, FailureOrdering: Failure);
6072	} else {
6073	Result =
6074	Builder.CreateAtomicCmpXchg(Ptr: X.Var, Cmp: E, New: D, Align: MaybeAlign(), SuccessOrdering: AO, FailureOrdering: Failure);
6075	}
6076
6077	if (V.Var) {
6078	Value *OldValue = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/0);
6079	if (!IsInteger)
6080	OldValue = Builder.CreateBitCast(V: OldValue, DestTy: X.ElemTy);
6081	assert(OldValue->getType() == V.ElemTy &&
6082	"OldValue and V must be of same type");
6083	if (IsPostfixUpdate) {
6084	Builder.CreateStore(Val: OldValue, Ptr: V.Var, isVolatile: V.IsVolatile);
6085	} else {
6086	Value *SuccessOrFail = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
6087	if (IsFailOnly) {
6088	// CurBB----
6089	// | |
6090	// v |
6091	// ContBB |
6092	// | |
6093	// v |
6094	// ExitBB <-
6095	//
6096	// where ContBB only contains the store of old value to 'v'.
6097	BasicBlock *CurBB = Builder.GetInsertBlock();
6098	Instruction *CurBBTI = CurBB->getTerminator();
6099	CurBBTI = CurBBTI ? CurBBTI : Builder.CreateUnreachable();
6100	BasicBlock *ExitBB = CurBB->splitBasicBlock(
6101	I: CurBBTI, BBName: X.Var->getName() + ".atomic.exit");
6102	BasicBlock *ContBB = CurBB->splitBasicBlock(
6103	I: CurBB->getTerminator(), BBName: X.Var->getName() + ".atomic.cont");
6104	ContBB->getTerminator()->eraseFromParent();
6105	CurBB->getTerminator()->eraseFromParent();
6106
6107	Builder.CreateCondBr(Cond: SuccessOrFail, True: ExitBB, False: ContBB);
6108
6109	Builder.SetInsertPoint(ContBB);
6110	Builder.CreateStore(Val: OldValue, Ptr: V.Var);
6111	Builder.CreateBr(Dest: ExitBB);
6112
6113	if (UnreachableInst *ExitTI =
6114	dyn_cast<UnreachableInst>(Val: ExitBB->getTerminator())) {
6115	CurBBTI->eraseFromParent();
6116	Builder.SetInsertPoint(ExitBB);
6117	} else {
6118	Builder.SetInsertPoint(ExitTI);
6119	}
6120	} else {
6121	Value *CapturedValue =
6122	Builder.CreateSelect(C: SuccessOrFail, True: E, False: OldValue);
6123	Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
6124	}
6125	}
6126	}
6127	// The comparison result has to be stored.
6128	if (R.Var) {
6129	assert(R.Var->getType()->isPointerTy() &&
6130	"r.var must be of pointer type");
6131	assert(R.ElemTy->isIntegerTy() && "r must be of integral type");
6132
6133	Value *SuccessFailureVal = Builder.CreateExtractValue(Agg: Result, /*Idxs=*/1);
6134	Value *ResultCast = R.IsSigned
6135	? Builder.CreateSExt(V: SuccessFailureVal, DestTy: R.ElemTy)
6136	: Builder.CreateZExt(V: SuccessFailureVal, DestTy: R.ElemTy);
6137	Builder.CreateStore(Val: ResultCast, Ptr: R.Var, isVolatile: R.IsVolatile);
6138	}
6139	} else {
6140	assert((Op == OMPAtomicCompareOp::MAX || Op == OMPAtomicCompareOp::MIN) &&
6141	"Op should be either max or min at this point");
6142	assert(!IsFailOnly && "IsFailOnly is only valid when the comparison is ==");
6143
6144	// Reverse the ordop as the OpenMP forms are different from LLVM forms.
6145	// Let's take max as example.
6146	// OpenMP form:
6147	// x = x > expr ? expr : x;
6148	// LLVM form:
6149	// *ptr = *ptr > val ? *ptr : val;
6150	// We need to transform to LLVM form.
6151	// x = x <= expr ? x : expr;
6152	AtomicRMWInst::BinOp NewOp;
6153	if (IsXBinopExpr) {
6154	if (IsInteger) {
6155	if (X.IsSigned)
6156	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min
6157	: AtomicRMWInst::Max;
6158	else
6159	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin
6160	: AtomicRMWInst::UMax;
6161	} else {
6162	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin
6163	: AtomicRMWInst::FMax;
6164	}
6165	} else {
6166	if (IsInteger) {
6167	if (X.IsSigned)
6168	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max
6169	: AtomicRMWInst::Min;
6170	else
6171	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax
6172	: AtomicRMWInst::UMin;
6173	} else {
6174	NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax
6175	: AtomicRMWInst::FMin;
6176	}
6177	}
6178
6179	AtomicRMWInst *OldValue =
6180	Builder.CreateAtomicRMW(Op: NewOp, Ptr: X.Var, Val: E, Align: MaybeAlign(), Ordering: AO);
6181	if (V.Var) {
6182	Value *CapturedValue = nullptr;
6183	if (IsPostfixUpdate) {
6184	CapturedValue = OldValue;
6185	} else {
6186	CmpInst::Predicate Pred;
6187	switch (NewOp) {
6188	case AtomicRMWInst::Max:
6189	Pred = CmpInst::ICMP_SGT;
6190	break;
6191	case AtomicRMWInst::UMax:
6192	Pred = CmpInst::ICMP_UGT;
6193	break;
6194	case AtomicRMWInst::FMax:
6195	Pred = CmpInst::FCMP_OGT;
6196	break;
6197	case AtomicRMWInst::Min:
6198	Pred = CmpInst::ICMP_SLT;
6199	break;
6200	case AtomicRMWInst::UMin:
6201	Pred = CmpInst::ICMP_ULT;
6202	break;
6203	case AtomicRMWInst::FMin:
6204	Pred = CmpInst::FCMP_OLT;
6205	break;
6206	default:
6207	llvm_unreachable("unexpected comparison op");
6208	}
6209	Value *NonAtomicCmp = Builder.CreateCmp(Pred, LHS: OldValue, RHS: E);
6210	CapturedValue = Builder.CreateSelect(C: NonAtomicCmp, True: E, False: OldValue);
6211	}
6212	Builder.CreateStore(Val: CapturedValue, Ptr: V.Var, isVolatile: V.IsVolatile);
6213	}
6214	}
6215
6216	checkAndEmitFlushAfterAtomic(Loc, AO, AK: AtomicKind::Compare);
6217
6218	return Builder.saveIP();
6219	}
6220
6221	OpenMPIRBuilder::InsertPointTy
6222	OpenMPIRBuilder::createTeams(const LocationDescription &Loc,
6223	BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower,
6224	Value *NumTeamsUpper, Value *ThreadLimit,
6225	Value *IfExpr) {
6226	if (!updateToLocation(Loc))
6227	return InsertPointTy();
6228
6229	uint32_t SrcLocStrSize;
6230	Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
6231	Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
6232	Function *CurrentFunction = Builder.GetInsertBlock()->getParent();
6233
6234	// Outer allocation basicblock is the entry block of the current function.
6235	BasicBlock &OuterAllocaBB = CurrentFunction->getEntryBlock();
6236	if (&OuterAllocaBB == Builder.GetInsertBlock()) {
6237	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.entry");
6238	Builder.SetInsertPoint(TheBB: BodyBB, IP: BodyBB->begin());
6239	}
6240
6241	// The current basic block is split into four basic blocks. After outlining,
6242	// they will be mapped as follows:
6243	// ```
6244	// def current_fn() {
6245	// current_basic_block:
6246	// br label %teams.exit
6247	// teams.exit:
6248	// ; instructions after teams
6249	// }
6250	//
6251	// def outlined_fn() {
6252	// teams.alloca:
6253	// br label %teams.body
6254	// teams.body:
6255	// ; instructions within teams body
6256	// }
6257	// ```
6258	BasicBlock *ExitBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.exit");
6259	BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, Name: "teams.body");
6260	BasicBlock *AllocaBB =
6261	splitBB(Builder, /*CreateBranch=*/true, Name: "teams.alloca");
6262
6263	bool SubClausesPresent =
6264	(NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
6265	// Push num_teams
6266	if (!Config.isTargetDevice() && SubClausesPresent) {
6267	assert((NumTeamsLower == nullptr || NumTeamsUpper != nullptr) &&
6268	"if lowerbound is non-null, then upperbound must also be non-null "
6269	"for bounds on num_teams");
6270
6271	if (NumTeamsUpper == nullptr)
6272	NumTeamsUpper = Builder.getInt32(C: 0);
6273
6274	if (NumTeamsLower == nullptr)
6275	NumTeamsLower = NumTeamsUpper;
6276
6277	if (IfExpr) {
6278	assert(IfExpr->getType()->isIntegerTy() &&
6279	"argument to if clause must be an integer value");
6280
6281	// upper = ifexpr ? upper : 1
6282	if (IfExpr->getType() != Int1)
6283	IfExpr = Builder.CreateICmpNE(LHS: IfExpr,
6284	RHS: ConstantInt::get(Ty: IfExpr->getType(), V: 0));
6285	NumTeamsUpper = Builder.CreateSelect(
6286	C: IfExpr, True: NumTeamsUpper, False: Builder.getInt32(C: 1), Name: "numTeamsUpper");
6287
6288	// lower = ifexpr ? lower : 1
6289	NumTeamsLower = Builder.CreateSelect(
6290	C: IfExpr, True: NumTeamsLower, False: Builder.getInt32(C: 1), Name: "numTeamsLower");
6291	}
6292
6293	if (ThreadLimit == nullptr)
6294	ThreadLimit = Builder.getInt32(C: 0);
6295
6296	Value *ThreadNum = getOrCreateThreadID(Ident);
6297	Builder.CreateCall(
6298	Callee: getOrCreateRuntimeFunctionPtr(FnID: OMPRTL___kmpc_push_num_teams_51),
6299	Args: {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});
6300	}
6301	// Generate the body of teams.
6302	InsertPointTy AllocaIP(AllocaBB, AllocaBB->begin());
6303	InsertPointTy CodeGenIP(BodyBB, BodyBB->begin());
6304	BodyGenCB(AllocaIP, CodeGenIP);
6305
6306	OutlineInfo OI;
6307	OI.EntryBB = AllocaBB;
6308	OI.ExitBB = ExitBB;
6309	OI.OuterAllocaBB = &OuterAllocaBB;
6310
6311	// Insert fake values for global tid and bound tid.
6312	std::stack<Instruction *> ToBeDeleted;
6313	InsertPointTy OuterAllocaIP(&OuterAllocaBB, OuterAllocaBB.begin());
6314	OI.ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
6315	Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "gid", AsPtr: true));
6316	OI.ExcludeArgsFromAggregate.push_back(Elt: createFakeIntVal(
6317	Builder, OuterAllocaIP, ToBeDeleted, InnerAllocaIP: AllocaIP, Name: "tid", AsPtr: true));
6318
6319	auto HostPostOutlineCB = [this, Ident,
6320	ToBeDeleted](Function &OutlinedFn) mutable {
6321	// The stale call instruction will be replaced with a new call instruction
6322	// for runtime call with the outlined function.
6323
6324	assert(OutlinedFn.getNumUses() == 1 &&
6325	"there must be a single user for the outlined function");
6326	CallInst *StaleCI = cast<CallInst>(Val: OutlinedFn.user_back());
6327	ToBeDeleted.push(x: StaleCI);
6328
6329	assert((OutlinedFn.arg_size() == 2 || OutlinedFn.arg_size() == 3) &&
6330	"Outlined function must have two or three arguments only");
6331
6332	bool HasShared = OutlinedFn.arg_size() == 3;
6333
6334	OutlinedFn.getArg(i: 0)->setName("global.tid.ptr");
6335	OutlinedFn.getArg(i: 1)->setName("bound.tid.ptr");
6336	if (HasShared)
6337	OutlinedFn.getArg(i: 2)->setName("data");
6338
6339	// Call to the runtime function for teams in the current function.
6340	assert(StaleCI && "Error while outlining - no CallInst user found for the "
6341	"outlined function.");
6342	Builder.SetInsertPoint(StaleCI);
6343	SmallVector<Value *> Args = {
6344	Ident, Builder.getInt32(C: StaleCI->arg_size() - 2), &OutlinedFn};
6345	if (HasShared)
6346	Args.push_back(Elt: StaleCI->getArgOperand(i: 2));
6347	Builder.CreateCall(Callee: getOrCreateRuntimeFunctionPtr(
6348	FnID: omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
6349	Args);
6350
6351	while (!ToBeDeleted.empty()) {
6352	ToBeDeleted.top()->eraseFromParent();
6353	ToBeDeleted.pop();
6354	}
6355	};
6356
6357	if (!Config.isTargetDevice())
6358	OI.PostOutlineCB = HostPostOutlineCB;
6359
6360	addOutlineInfo(OI: std::move(OI));
6361
6362	Builder.SetInsertPoint(TheBB: ExitBB, IP: ExitBB->begin());
6363
6364	return Builder.saveIP();
6365	}
6366
6367	GlobalVariable *
6368	OpenMPIRBuilder::createOffloadMapnames(SmallVectorImpl<llvm::Constant *> &Names,
6369	std::string VarName) {
6370	llvm::Constant *MapNamesArrayInit = llvm::ConstantArray::get(
6371	T: llvm::ArrayType::get(ElementType: llvm::PointerType::getUnqual(C&: M.getContext()),
6372	NumElements: Names.size()),
6373	V: Names);
6374	auto *MapNamesArrayGlobal = new llvm::GlobalVariable(
6375	M, MapNamesArrayInit->getType(),
6376	/*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, MapNamesArrayInit,
6377	VarName);
6378	return MapNamesArrayGlobal;
6379	}
6380
6381	// Create all simple and struct types exposed by the runtime and remember
6382	// the llvm::PointerTypes of them for easy access later.
6383	void OpenMPIRBuilder::initializeTypes(Module &M) {
6384	LLVMContext &Ctx = M.getContext();
6385	StructType *T;
6386	#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
6387	#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
6388	VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
6389	VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
6390	#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
6391	VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
6392	VarName##Ptr = PointerType::getUnqual(VarName);
6393	#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
6394	T = StructType::getTypeByName(Ctx, StructName); \
6395	if (!T) \
6396	T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
6397	VarName = T; \
6398	VarName##Ptr = PointerType::getUnqual(T);
6399	#include "llvm/Frontend/OpenMP/OMPKinds.def"
6400	}
6401
6402	void OpenMPIRBuilder::OutlineInfo::collectBlocks(
6403	SmallPtrSetImpl<BasicBlock *> &BlockSet,
6404	SmallVectorImpl<BasicBlock *> &BlockVector) {
6405	SmallVector<BasicBlock *, 32> Worklist;
6406	BlockSet.insert(Ptr: EntryBB);
6407	BlockSet.insert(Ptr: ExitBB);
6408
6409	Worklist.push_back(Elt: EntryBB);
6410	while (!Worklist.empty()) {
6411	BasicBlock *BB = Worklist.pop_back_val();
6412	BlockVector.push_back(Elt: BB);
6413	for (BasicBlock *SuccBB : successors(BB))
6414	if (BlockSet.insert(Ptr: SuccBB).second)
6415	Worklist.push_back(Elt: SuccBB);
6416	}
6417	}
6418
6419	void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr,
6420	uint64_t Size, int32_t Flags,
6421	GlobalValue::LinkageTypes,
6422	StringRef Name) {
6423	if (!Config.isGPU()) {
6424	llvm::offloading::emitOffloadingEntry(
6425	M, Addr: ID, Name: Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0,
6426	SectionName: "omp_offloading_entries");
6427	return;
6428	}
6429	// TODO: Add support for global variables on the device after declare target
6430	// support.
6431	Function *Fn = dyn_cast<Function>(Val: Addr);
6432	if (!Fn)
6433	return;
6434
6435	Module &M = *(Fn->getParent());
6436	LLVMContext &Ctx = M.getContext();
6437
6438	// Get "nvvm.annotations" metadata node.
6439	NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "nvvm.annotations");
6440
6441	Metadata *MDVals[] = {
6442	ConstantAsMetadata::get(C: Fn), MDString::get(Context&: Ctx, Str: "kernel"),
6443	ConstantAsMetadata::get(C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: 1))};
6444	// Append metadata to nvvm.annotations.
6445	MD->addOperand(M: MDNode::get(Context&: Ctx, MDs: MDVals));
6446
6447	// Add a function attribute for the kernel.
6448	Fn->addFnAttr(Attr: Attribute::get(Context&: Ctx, Kind: "kernel"));
6449	if (T.isAMDGCN())
6450	Fn->addFnAttr(Kind: "uniform-work-group-size", Val: "true");
6451	Fn->addFnAttr(Attribute::MustProgress);
6452	}
6453
6454	// We only generate metadata for function that contain target regions.
6455	void OpenMPIRBuilder::createOffloadEntriesAndInfoMetadata(
6456	EmitMetadataErrorReportFunctionTy &ErrorFn) {
6457
6458	// If there are no entries, we don't need to do anything.
6459	if (OffloadInfoManager.empty())
6460	return;
6461
6462	LLVMContext &C = M.getContext();
6463	SmallVector<std::pair<const OffloadEntriesInfoManager::OffloadEntryInfo *,
6464	TargetRegionEntryInfo>,
6465	16>
6466	OrderedEntries(OffloadInfoManager.size());
6467
6468	// Auxiliary methods to create metadata values and strings.
6469	auto &&GetMDInt = [this](unsigned V) {
6470	return ConstantAsMetadata::get(C: ConstantInt::get(Ty: Builder.getInt32Ty(), V));
6471	};
6472
6473	auto &&GetMDString = [&C](StringRef V) { return MDString::get(Context&: C, Str: V); };
6474
6475	// Create the offloading info metadata node.
6476	NamedMDNode *MD = M.getOrInsertNamedMetadata(Name: "omp_offload.info");
6477	auto &&TargetRegionMetadataEmitter =
6478	[&C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
6479	const TargetRegionEntryInfo &EntryInfo,
6480	const OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion &E) {
6481	// Generate metadata for target regions. Each entry of this metadata
6482	// contains:
6483	// - Entry 0 -> Kind of this type of metadata (0).
6484	// - Entry 1 -> Device ID of the file where the entry was identified.
6485	// - Entry 2 -> File ID of the file where the entry was identified.
6486	// - Entry 3 -> Mangled name of the function where the entry was
6487	// identified.
6488	// - Entry 4 -> Line in the file where the entry was identified.
6489	// - Entry 5 -> Count of regions at this DeviceID/FilesID/Line.
6490	// - Entry 6 -> Order the entry was created.
6491	// The first element of the metadata node is the kind.
6492	Metadata *Ops[] = {
6493	GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
6494	GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
6495	GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
6496	GetMDInt(E.getOrder())};
6497
6498	// Save this entry in the right position of the ordered entries array.
6499	OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y: EntryInfo);
6500
6501	// Add metadata to the named metadata node.
6502	MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
6503	};
6504
6505	OffloadInfoManager.actOnTargetRegionEntriesInfo(Action: TargetRegionMetadataEmitter);
6506
6507	// Create function that emits metadata for each device global variable entry;
6508	auto &&DeviceGlobalVarMetadataEmitter =
6509	[&C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
6510	StringRef MangledName,
6511	const OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar &E) {
6512	// Generate metadata for global variables. Each entry of this metadata
6513	// contains:
6514	// - Entry 0 -> Kind of this type of metadata (1).
6515	// - Entry 1 -> Mangled name of the variable.
6516	// - Entry 2 -> Declare target kind.
6517	// - Entry 3 -> Order the entry was created.
6518	// The first element of the metadata node is the kind.
6519	Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
6520	GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
6521
6522	// Save this entry in the right position of the ordered entries array.
6523	TargetRegionEntryInfo varInfo(MangledName, 0, 0, 0);
6524	OrderedEntries[E.getOrder()] = std::make_pair(x: &E, y&: varInfo);
6525
6526	// Add metadata to the named metadata node.
6527	MD->addOperand(M: MDNode::get(Context&: C, MDs: Ops));
6528	};
6529
6530	OffloadInfoManager.actOnDeviceGlobalVarEntriesInfo(
6531	Action: DeviceGlobalVarMetadataEmitter);
6532
6533	for (const auto &E : OrderedEntries) {
6534	assert(E.first && "All ordered entries must exist!");
6535	if (const auto *CE =
6536	dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
6537	Val: E.first)) {
6538	if (!CE->getID() || !CE->getAddress()) {
6539	// Do not blame the entry if the parent funtion is not emitted.
6540	TargetRegionEntryInfo EntryInfo = E.second;
6541	StringRef FnName = EntryInfo.ParentName;
6542	if (!M.getNamedValue(Name: FnName))
6543	continue;
6544	ErrorFn(EMIT_MD_TARGET_REGION_ERROR, EntryInfo);
6545	continue;
6546	}
6547	createOffloadEntry(ID: CE->getID(), Addr: CE->getAddress(),
6548	/*Size=*/0, Flags: CE->getFlags(),
6549	GlobalValue::WeakAnyLinkage);
6550	} else if (const auto *CE = dyn_cast<
6551	OffloadEntriesInfoManager::OffloadEntryInfoDeviceGlobalVar>(
6552	Val: E.first)) {
6553	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags =
6554	static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
6555	CE->getFlags());
6556	switch (Flags) {
6557	case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter:
6558	case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo:
6559	if (Config.isTargetDevice() && Config.hasRequiresUnifiedSharedMemory())
6560	continue;
6561	if (!CE->getAddress()) {
6562	ErrorFn(EMIT_MD_DECLARE_TARGET_ERROR, E.second);
6563	continue;
6564	}
6565	// The vaiable has no definition - no need to add the entry.
6566	if (CE->getVarSize() == 0)
6567	continue;
6568	break;
6569	case OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink:
6570	assert(((Config.isTargetDevice() && !CE->getAddress()) ||
6571	(!Config.isTargetDevice() && CE->getAddress())) &&
6572	"Declaret target link address is set.");
6573	if (Config.isTargetDevice())
6574	continue;
6575	if (!CE->getAddress()) {
6576	ErrorFn(EMIT_MD_GLOBAL_VAR_LINK_ERROR, TargetRegionEntryInfo());
6577	continue;
6578	}
6579	break;
6580	default:
6581	break;
6582	}
6583
6584	// Hidden or internal symbols on the device are not externally visible.
6585	// We should not attempt to register them by creating an offloading
6586	// entry. Indirect variables are handled separately on the device.
6587	if (auto *GV = dyn_cast<GlobalValue>(Val: CE->getAddress()))
6588	if ((GV->hasLocalLinkage() || GV->hasHiddenVisibility()) &&
6589	Flags != OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
6590	continue;
6591
6592	// Indirect globals need to use a special name that doesn't match the name
6593	// of the associated host global.
6594	if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
6595	createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
6596	Flags, CE->getLinkage(), Name: CE->getVarName());
6597	else
6598	createOffloadEntry(ID: CE->getAddress(), Addr: CE->getAddress(), Size: CE->getVarSize(),
6599	Flags, CE->getLinkage());
6600
6601	} else {
6602	llvm_unreachable("Unsupported entry kind.");
6603	}
6604	}
6605	}
6606
6607	void TargetRegionEntryInfo::getTargetRegionEntryFnName(
6608	SmallVectorImpl<char> &Name, StringRef ParentName, unsigned DeviceID,
6609	unsigned FileID, unsigned Line, unsigned Count) {
6610	raw_svector_ostream OS(Name);
6611	OS << "__omp_offloading" << llvm::format(Fmt: "_%x", Vals: DeviceID)
6612	<< llvm::format(Fmt: "_%x_", Vals: FileID) << ParentName << "_l" << Line;
6613	if (Count)
6614	OS << "_" << Count;
6615	}
6616
6617	void OffloadEntriesInfoManager::getTargetRegionEntryFnName(
6618	SmallVectorImpl<char> &Name, const TargetRegionEntryInfo &EntryInfo) {
6619	unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
6620	TargetRegionEntryInfo::getTargetRegionEntryFnName(
6621	Name, ParentName: EntryInfo.ParentName, DeviceID: EntryInfo.DeviceID, FileID: EntryInfo.FileID,
6622	Line: EntryInfo.Line, Count: NewCount);
6623	}
6624
6625	TargetRegionEntryInfo
6626	OpenMPIRBuilder::getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack,
6627	StringRef ParentName) {
6628	sys::fs::UniqueID ID;
6629	auto FileIDInfo = CallBack();
6630	if (auto EC = sys::fs::getUniqueID(Path: std::get<0>(t&: FileIDInfo), Result&: ID)) {
6631	report_fatal_error(reason: ("Unable to get unique ID for file, during "
6632	"getTargetEntryUniqueInfo, error message: " +
6633	EC.message())
6634	.c_str());
6635	}
6636
6637	return TargetRegionEntryInfo(ParentName, ID.getDevice(), ID.getFile(),
6638	std::get<1>(t&: FileIDInfo));
6639	}
6640
6641	unsigned OpenMPIRBuilder::getFlagMemberOffset() {
6642	unsigned Offset = 0;
6643	for (uint64_t Remain =
6644	static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
6645	omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);
6646	!(Remain & 1); Remain = Remain >> 1)
6647	Offset++;
6648	return Offset;
6649	}
6650
6651	omp::OpenMPOffloadMappingFlags
6652	OpenMPIRBuilder::getMemberOfFlag(unsigned Position) {
6653	// Rotate by getFlagMemberOffset() bits.
6654	return static_cast<omp::OpenMPOffloadMappingFlags>(((uint64_t)Position + 1)
6655	<< getFlagMemberOffset());
6656	}
6657
6658	void OpenMPIRBuilder::setCorrectMemberOfFlag(
6659	omp::OpenMPOffloadMappingFlags &Flags,
6660	omp::OpenMPOffloadMappingFlags MemberOfFlag) {
6661	// If the entry is PTR_AND_OBJ but has not been marked with the special
6662	// placeholder value 0xFFFF in the MEMBER_OF field, then it should not be
6663	// marked as MEMBER_OF.
6664	if (static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
6665	Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ) &&
6666	static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>>(
6667	(Flags & omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=
6668	omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF))
6669	return;
6670
6671	// Reset the placeholder value to prepare the flag for the assignment of the
6672	// proper MEMBER_OF value.
6673	Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
6674	Flags |= MemberOfFlag;
6675	}
6676
6677	Constant *OpenMPIRBuilder::getAddrOfDeclareTargetVar(
6678	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
6679	OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
6680	bool IsDeclaration, bool IsExternallyVisible,
6681	TargetRegionEntryInfo EntryInfo, StringRef MangledName,
6682	std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
6683	std::vector<Triple> TargetTriple, Type *LlvmPtrTy,
6684	std::function<Constant *()> GlobalInitializer,
6685	std::function<GlobalValue::LinkageTypes()> VariableLinkage) {
6686	// TODO: convert this to utilise the IRBuilder Config rather than
6687	// a passed down argument.
6688	if (OpenMPSIMD)
6689	return nullptr;
6690
6691	if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink ||
6692	((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
6693	CaptureClause ==
6694	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
6695	Config.hasRequiresUnifiedSharedMemory())) {
6696	SmallString<64> PtrName;
6697	{
6698	raw_svector_ostream OS(PtrName);
6699	OS << MangledName;
6700	if (!IsExternallyVisible)
6701	OS << format(Fmt: "_%x", Vals: EntryInfo.FileID);
6702	OS << "_decl_tgt_ref_ptr";
6703	}
6704
6705	Value *Ptr = M.getNamedValue(Name: PtrName);
6706
6707	if (!Ptr) {
6708	GlobalValue *GlobalValue = M.getNamedValue(Name: MangledName);
6709	Ptr = getOrCreateInternalVariable(Ty: LlvmPtrTy, Name: PtrName);
6710
6711	auto *GV = cast<GlobalVariable>(Val: Ptr);
6712	GV->setLinkage(GlobalValue::WeakAnyLinkage);
6713
6714	if (!Config.isTargetDevice()) {
6715	if (GlobalInitializer)
6716	GV->setInitializer(GlobalInitializer());
6717	else
6718	GV->setInitializer(GlobalValue);
6719	}
6720
6721	registerTargetGlobalVariable(
6722	CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
6723	EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
6724	GlobalInitializer, VariableLinkage, LlvmPtrTy, Addr: cast<Constant>(Val: Ptr));
6725	}
6726
6727	return cast<Constant>(Val: Ptr);
6728	}
6729
6730	return nullptr;
6731	}
6732
6733	void OpenMPIRBuilder::registerTargetGlobalVariable(
6734	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause,
6735	OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause,
6736	bool IsDeclaration, bool IsExternallyVisible,
6737	TargetRegionEntryInfo EntryInfo, StringRef MangledName,
6738	std::vector<GlobalVariable *> &GeneratedRefs, bool OpenMPSIMD,
6739	std::vector<Triple> TargetTriple,
6740	std::function<Constant *()> GlobalInitializer,
6741	std::function<GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy,
6742	Constant *Addr) {
6743	if (DeviceClause != OffloadEntriesInfoManager::OMPTargetDeviceClauseAny ||
6744	(TargetTriple.empty() && !Config.isTargetDevice()))
6745	return;
6746
6747	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind Flags;
6748	StringRef VarName;
6749	int64_t VarSize;
6750	GlobalValue::LinkageTypes Linkage;
6751
6752	if ((CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo ||
6753	CaptureClause ==
6754	OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter) &&
6755	!Config.hasRequiresUnifiedSharedMemory()) {
6756	Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
6757	VarName = MangledName;
6758	GlobalValue *LlvmVal = M.getNamedValue(Name: VarName);
6759
6760	if (!IsDeclaration)
6761	VarSize = divideCeil(
6762	Numerator: M.getDataLayout().getTypeSizeInBits(Ty: LlvmVal->getValueType()), Denominator: 8);
6763	else
6764	VarSize = 0;
6765	Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->getLinkage();
6766
6767	// This is a workaround carried over from Clang which prevents undesired
6768	// optimisation of internal variables.
6769	if (Config.isTargetDevice() &&
6770	(!IsExternallyVisible || Linkage == GlobalValue::LinkOnceODRLinkage)) {
6771	// Do not create a "ref-variable" if the original is not also available
6772	// on the host.
6773	if (!OffloadInfoManager.hasDeviceGlobalVarEntryInfo(VarName))
6774	return;
6775
6776	std::string RefName = createPlatformSpecificName(Parts: {VarName, "ref"});
6777
6778	if (!M.getNamedValue(Name: RefName)) {
6779	Constant *AddrRef =
6780	getOrCreateInternalVariable(Ty: Addr->getType(), Name: RefName);
6781	auto *GvAddrRef = cast<GlobalVariable>(Val: AddrRef);
6782	GvAddrRef->setConstant(true);
6783	GvAddrRef->setLinkage(GlobalValue::InternalLinkage);
6784	GvAddrRef->setInitializer(Addr);
6785	GeneratedRefs.push_back(x: GvAddrRef);
6786	}
6787	}
6788	} else {
6789	if (CaptureClause == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink)
6790	Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;
6791	else
6792	Flags = OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;
6793
6794	if (Config.isTargetDevice()) {
6795	VarName = (Addr) ? Addr->getName() : "";
6796	Addr = nullptr;
6797	} else {
6798	Addr = getAddrOfDeclareTargetVar(
6799	CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
6800	EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
6801	LlvmPtrTy, GlobalInitializer, VariableLinkage);
6802	VarName = (Addr) ? Addr->getName() : "";
6803	}
6804	VarSize = M.getDataLayout().getPointerSize();
6805	Linkage = GlobalValue::WeakAnyLinkage;
6806	}
6807
6808	OffloadInfoManager.registerDeviceGlobalVarEntryInfo(VarName, Addr, VarSize,
6809	Flags, Linkage);
6810	}
6811
6812	/// Loads all the offload entries information from the host IR
6813	/// metadata.
6814	void OpenMPIRBuilder::loadOffloadInfoMetadata(Module &M) {
6815	// If we are in target mode, load the metadata from the host IR. This code has
6816	// to match the metadata creation in createOffloadEntriesAndInfoMetadata().
6817
6818	NamedMDNode *MD = M.getNamedMetadata(Name: ompOffloadInfoName);
6819	if (!MD)
6820	return;
6821
6822	for (MDNode *MN : MD->operands()) {
6823	auto &&GetMDInt = [MN](unsigned Idx) {
6824	auto *V = cast<ConstantAsMetadata>(Val: MN->getOperand(I: Idx));
6825	return cast<ConstantInt>(Val: V->getValue())->getZExtValue();
6826	};
6827
6828	auto &&GetMDString = [MN](unsigned Idx) {
6829	auto *V = cast<MDString>(Val: MN->getOperand(I: Idx));
6830	return V->getString();
6831	};
6832
6833	switch (GetMDInt(0)) {
6834	default:
6835	llvm_unreachable("Unexpected metadata!");
6836	break;
6837	case OffloadEntriesInfoManager::OffloadEntryInfo::
6838	OffloadingEntryInfoTargetRegion: {
6839	TargetRegionEntryInfo EntryInfo(/*ParentName=*/GetMDString(3),
6840	/*DeviceID=*/GetMDInt(1),
6841	/*FileID=*/GetMDInt(2),
6842	/*Line=*/GetMDInt(4),
6843	/*Count=*/GetMDInt(5));
6844	OffloadInfoManager.initializeTargetRegionEntryInfo(EntryInfo,
6845	/*Order=*/GetMDInt(6));
6846	break;
6847	}
6848	case OffloadEntriesInfoManager::OffloadEntryInfo::
6849	OffloadingEntryInfoDeviceGlobalVar:
6850	OffloadInfoManager.initializeDeviceGlobalVarEntryInfo(
6851	/*MangledName=*/Name: GetMDString(1),
6852	Flags: static_cast<OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind>(
6853	/*Flags=*/GetMDInt(2)),
6854	/*Order=*/GetMDInt(3));
6855	break;
6856	}
6857	}
6858	}
6859
6860	void OpenMPIRBuilder::loadOffloadInfoMetadata(StringRef HostFilePath) {
6861	if (HostFilePath.empty())
6862	return;
6863
6864	auto Buf = MemoryBuffer::getFile(Filename: HostFilePath);
6865	if (std::error_code Err = Buf.getError()) {
6866	report_fatal_error(reason: ("error opening host file from host file path inside of "
6867	"OpenMPIRBuilder: " +
6868	Err.message())
6869	.c_str());
6870	}
6871
6872	LLVMContext Ctx;
6873	auto M = expectedToErrorOrAndEmitErrors(
6874	Ctx, Val: parseBitcodeFile(Buffer: Buf.get()->getMemBufferRef(), Context&: Ctx));
6875	if (std::error_code Err = M.getError()) {
6876	report_fatal_error(
6877	reason: ("error parsing host file inside of OpenMPIRBuilder: " + Err.message())
6878	.c_str());
6879	}
6880
6881	loadOffloadInfoMetadata(M&: *M.get());
6882	}
6883
6884	Function *OpenMPIRBuilder::createRegisterRequires(StringRef Name) {
6885	// Skip the creation of the registration function if this is device codegen
6886	if (Config.isTargetDevice())
6887	return nullptr;
6888
6889	Builder.ClearInsertionPoint();
6890
6891	// Create registration function prototype
6892	auto *RegFnTy = FunctionType::get(Result: Builder.getVoidTy(), isVarArg: {});
6893	auto *RegFn = Function::Create(
6894	Ty: RegFnTy, Linkage: GlobalVariable::LinkageTypes::InternalLinkage, N: Name, M);
6895	RegFn->setSection(".text.startup");
6896	RegFn->addFnAttr(Attribute::NoInline);
6897	RegFn->addFnAttr(Attribute::NoUnwind);
6898
6899	// Create registration function body
6900	auto *BB = BasicBlock::Create(Context&: M.getContext(), Name: "entry", Parent: RegFn);
6901	ConstantInt *FlagsVal =
6902	ConstantInt::getSigned(Ty: Builder.getInt64Ty(), V: Config.getRequiresFlags());
6903	Function *RTLRegFn = getOrCreateRuntimeFunctionPtr(
6904	FnID: omp::RuntimeFunction::OMPRTL___tgt_register_requires);
6905
6906	Builder.SetInsertPoint(BB);
6907	Builder.CreateCall(Callee: RTLRegFn, Args: {FlagsVal});
6908	Builder.CreateRetVoid();
6909
6910	return RegFn;
6911	}
6912
6913	//===----------------------------------------------------------------------===//
6914	// OffloadEntriesInfoManager
6915	//===----------------------------------------------------------------------===//
6916
6917	bool OffloadEntriesInfoManager::empty() const {
6918	return OffloadEntriesTargetRegion.empty() &&
6919	OffloadEntriesDeviceGlobalVar.empty();
6920	}
6921
6922	unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
6923	const TargetRegionEntryInfo &EntryInfo) const {
6924	auto It = OffloadEntriesTargetRegionCount.find(
6925	x: getTargetRegionEntryCountKey(EntryInfo));
6926	if (It == OffloadEntriesTargetRegionCount.end())
6927	return 0;
6928	return It->second;
6929	}
6930
6931	void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
6932	const TargetRegionEntryInfo &EntryInfo) {
6933	OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
6934	EntryInfo.Count + 1;
6935	}
6936
6937	/// Initialize target region entry.
6938	void OffloadEntriesInfoManager::initializeTargetRegionEntryInfo(
6939	const TargetRegionEntryInfo &EntryInfo, unsigned Order) {
6940	OffloadEntriesTargetRegion[EntryInfo] =
6941	OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
6942	OMPTargetRegionEntryTargetRegion);
6943	++OffloadingEntriesNum;
6944	}
6945
6946	void OffloadEntriesInfoManager::registerTargetRegionEntryInfo(
6947	TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID,
6948	OMPTargetRegionEntryKind Flags) {
6949	assert(EntryInfo.Count == 0 && "expected default EntryInfo");
6950
6951	// Update the EntryInfo with the next available count for this location.
6952	EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
6953
6954	// If we are emitting code for a target, the entry is already initialized,
6955	// only has to be registered.
6956	if (OMPBuilder->Config.isTargetDevice()) {
6957	// This could happen if the device compilation is invoked standalone.
6958	if (!hasTargetRegionEntryInfo(EntryInfo)) {
6959	return;
6960	}
6961	auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
6962	Entry.setAddress(Addr);
6963	Entry.setID(ID);
6964	Entry.setFlags(Flags);
6965	} else {
6966	if (Flags == OffloadEntriesInfoManager::OMPTargetRegionEntryTargetRegion &&
6967	hasTargetRegionEntryInfo(EntryInfo, /*IgnoreAddressId*/ true))
6968	return;
6969	assert(!hasTargetRegionEntryInfo(EntryInfo) &&
6970	"Target region entry already registered!");
6971	OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags);
6972	OffloadEntriesTargetRegion[EntryInfo] = Entry;
6973	++OffloadingEntriesNum;
6974	}
6975	incrementTargetRegionEntryInfoCount(EntryInfo);
6976	}
6977
6978	bool OffloadEntriesInfoManager::hasTargetRegionEntryInfo(
6979	TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId) const {
6980
6981	// Update the EntryInfo with the next available count for this location.
6982	EntryInfo.Count = getTargetRegionEntryInfoCount(EntryInfo);
6983
6984	auto It = OffloadEntriesTargetRegion.find(x: EntryInfo);
6985	if (It == OffloadEntriesTargetRegion.end()) {
6986	return false;
6987	}
6988	// Fail if this entry is already registered.
6989	if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
6990	return false;
6991	return true;
6992	}
6993
6994	void OffloadEntriesInfoManager::actOnTargetRegionEntriesInfo(
6995	const OffloadTargetRegionEntryInfoActTy &Action) {
6996	// Scan all target region entries and perform the provided action.
6997	for (const auto &It : OffloadEntriesTargetRegion) {
6998	Action(It.first, It.second);
6999	}
7000	}
7001
7002	void OffloadEntriesInfoManager::initializeDeviceGlobalVarEntryInfo(
7003	StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order) {
7004	OffloadEntriesDeviceGlobalVar.try_emplace(Key: Name, Args&: Order, Args&: Flags);
7005	++OffloadingEntriesNum;
7006	}
7007
7008	void OffloadEntriesInfoManager::registerDeviceGlobalVarEntryInfo(
7009	StringRef VarName, Constant *Addr, int64_t VarSize,
7010	OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage) {
7011	if (OMPBuilder->Config.isTargetDevice()) {
7012	// This could happen if the device compilation is invoked standalone.
7013	if (!hasDeviceGlobalVarEntryInfo(VarName))
7014	return;
7015	auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
7016	if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
7017	if (Entry.getVarSize() == 0) {
7018	Entry.setVarSize(VarSize);
7019	Entry.setLinkage(Linkage);
7020	}
7021	return;
7022	}
7023	Entry.setVarSize(VarSize);
7024	Entry.setLinkage(Linkage);
7025	Entry.setAddress(Addr);
7026	} else {
7027	if (hasDeviceGlobalVarEntryInfo(VarName)) {
7028	auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
7029	assert(Entry.isValid() && Entry.getFlags() == Flags &&
7030	"Entry not initialized!");
7031	if (Entry.getVarSize() == 0) {
7032	Entry.setVarSize(VarSize);
7033	Entry.setLinkage(Linkage);
7034	}
7035	return;
7036	}
7037	if (Flags == OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect)
7038	OffloadEntriesDeviceGlobalVar.try_emplace(Key: VarName, Args&: OffloadingEntriesNum,
7039	Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage,
7040	Args: VarName.str());
7041	else
7042	OffloadEntriesDeviceGlobalVar.try_emplace(
7043	Key: VarName, Args&: OffloadingEntriesNum, Args&: Addr, Args&: VarSize, Args&: Flags, Args&: Linkage, Args: "");
7044	++OffloadingEntriesNum;
7045	}
7046	}
7047
7048	void OffloadEntriesInfoManager::actOnDeviceGlobalVarEntriesInfo(
7049	const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
7050	// Scan all target region entries and perform the provided action.
7051	for (const auto &E : OffloadEntriesDeviceGlobalVar)
7052	Action(E.getKey(), E.getValue());
7053	}
7054
7055	//===----------------------------------------------------------------------===//
7056	// CanonicalLoopInfo
7057	//===----------------------------------------------------------------------===//
7058
7059	void CanonicalLoopInfo::collectControlBlocks(
7060	SmallVectorImpl<BasicBlock *> &BBs) {
7061	// We only count those BBs as control block for which we do not need to
7062	// reverse the CFG, i.e. not the loop body which can contain arbitrary control
7063	// flow. For consistency, this also means we do not add the Body block, which
7064	// is just the entry to the body code.
7065	BBs.reserve(N: BBs.size() + 6);
7066	BBs.append(IL: {getPreheader(), Header, Cond, Latch, Exit, getAfter()});
7067	}
7068
7069	BasicBlock *CanonicalLoopInfo::getPreheader() const {
7070	assert(isValid() && "Requires a valid canonical loop");
7071	for (BasicBlock *Pred : predecessors(BB: Header)) {
7072	if (Pred != Latch)
7073	return Pred;
7074	}
7075	llvm_unreachable("Missing preheader");
7076	}
7077
7078	void CanonicalLoopInfo::setTripCount(Value *TripCount) {
7079	assert(isValid() && "Requires a valid canonical loop");
7080
7081	Instruction *CmpI = &getCond()->front();
7082	assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
7083	CmpI->setOperand(i: 1, Val: TripCount);
7084
7085	#ifndef NDEBUG
7086	assertOK();
7087	#endif
7088	}
7089
7090	void CanonicalLoopInfo::mapIndVar(
7091	llvm::function_ref<Value *(Instruction *)> Updater) {
7092	assert(isValid() && "Requires a valid canonical loop");
7093
7094	Instruction *OldIV = getIndVar();
7095
7096	// Record all uses excluding those introduced by the updater. Uses by the
7097	// CanonicalLoopInfo itself to keep track of the number of iterations are
7098	// excluded.
7099	SmallVector<Use *> ReplacableUses;
7100	for (Use &U : OldIV->uses()) {
7101	auto *User = dyn_cast<Instruction>(Val: U.getUser());
7102	if (!User)
7103	continue;
7104	if (User->getParent() == getCond())
7105	continue;
7106	if (User->getParent() == getLatch())
7107	continue;
7108	ReplacableUses.push_back(Elt: &U);
7109	}
7110
7111	// Run the updater that may introduce new uses
7112	Value *NewIV = Updater(OldIV);
7113
7114	// Replace the old uses with the value returned by the updater.
7115	for (Use *U : ReplacableUses)
7116	U->set(NewIV);
7117
7118	#ifndef NDEBUG
7119	assertOK();
7120	#endif
7121	}
7122
7123	void CanonicalLoopInfo::assertOK() const {
7124	#ifndef NDEBUG
7125	// No constraints if this object currently does not describe a loop.
7126	if (!isValid())
7127	return;
7128
7129	BasicBlock *Preheader = getPreheader();
7130	BasicBlock *Body = getBody();
7131	BasicBlock *After = getAfter();
7132
7133	// Verify standard control-flow we use for OpenMP loops.
7134	assert(Preheader);
7135	assert(isa<BranchInst>(Preheader->getTerminator()) &&
7136	"Preheader must terminate with unconditional branch");
7137	assert(Preheader->getSingleSuccessor() == Header &&
7138	"Preheader must jump to header");
7139
7140	assert(Header);
7141	assert(isa<BranchInst>(Header->getTerminator()) &&
7142	"Header must terminate with unconditional branch");
7143	assert(Header->getSingleSuccessor() == Cond &&
7144	"Header must jump to exiting block");
7145
7146	assert(Cond);
7147	assert(Cond->getSinglePredecessor() == Header &&
7148	"Exiting block only reachable from header");
7149
7150	assert(isa<BranchInst>(Cond->getTerminator()) &&
7151	"Exiting block must terminate with conditional branch");
7152	assert(size(successors(Cond)) == 2 &&
7153	"Exiting block must have two successors");
7154	assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
7155	"Exiting block's first successor jump to the body");
7156	assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
7157	"Exiting block's second successor must exit the loop");
7158
7159	assert(Body);
7160	assert(Body->getSinglePredecessor() == Cond &&
7161	"Body only reachable from exiting block");
7162	assert(!isa<PHINode>(Body->front()));
7163
7164	assert(Latch);
7165	assert(isa<BranchInst>(Latch->getTerminator()) &&
7166	"Latch must terminate with unconditional branch");
7167	assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
7168	// TODO: To support simple redirecting of the end of the body code that has
7169	// multiple; introduce another auxiliary basic block like preheader and after.
7170	assert(Latch->getSinglePredecessor() != nullptr);
7171	assert(!isa<PHINode>(Latch->front()));
7172
7173	assert(Exit);
7174	assert(isa<BranchInst>(Exit->getTerminator()) &&
7175	"Exit block must terminate with unconditional branch");
7176	assert(Exit->getSingleSuccessor() == After &&
7177	"Exit block must jump to after block");
7178
7179	assert(After);
7180	assert(After->getSinglePredecessor() == Exit &&
7181	"After block only reachable from exit block");
7182	assert(After->empty() || !isa<PHINode>(After->front()));
7183
7184	Instruction *IndVar = getIndVar();
7185	assert(IndVar && "Canonical induction variable not found?");
7186	assert(isa<IntegerType>(IndVar->getType()) &&
7187	"Induction variable must be an integer");
7188	assert(cast<PHINode>(IndVar)->getParent() == Header &&
7189	"Induction variable must be a PHI in the loop header");
7190	assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
7191	assert(
7192	cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
7193	assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
7194
7195	auto *NextIndVar = cast<PHINode>(Val: IndVar)->getIncomingValue(i: 1);
7196	assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
7197	assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
7198	assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
7199	assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
7200	->isOne());
7201
7202	Value *TripCount = getTripCount();
7203	assert(TripCount && "Loop trip count not found?");
7204	assert(IndVar->getType() == TripCount->getType() &&
7205	"Trip count and induction variable must have the same type");
7206
7207	auto *CmpI = cast<CmpInst>(Val: &Cond->front());
7208	assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
7209	"Exit condition must be a signed less-than comparison");
7210	assert(CmpI->getOperand(0) == IndVar &&
7211	"Exit condition must compare the induction variable");
7212	assert(CmpI->getOperand(1) == TripCount &&
7213	"Exit condition must compare with the trip count");
7214	#endif
7215	}
7216
7217	void CanonicalLoopInfo::invalidate() {
7218	Header = nullptr;
7219	Cond = nullptr;
7220	Latch = nullptr;
7221	Exit = nullptr;
7222	}
7223