1	//===----- CGOpenMPRuntime.h - Interface to OpenMP Runtimes -----*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This provides a class for OpenMP runtime code generation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
14	#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
15
16	#include "CGValue.h"
17	#include "clang/AST/DeclOpenMP.h"
18	#include "clang/AST/GlobalDecl.h"
19	#include "clang/AST/Type.h"
20	#include "clang/Basic/OpenMPKinds.h"
21	#include "clang/Basic/SourceLocation.h"
22	#include "llvm/ADT/DenseMap.h"
23	#include "llvm/ADT/PointerIntPair.h"
24	#include "llvm/ADT/SmallPtrSet.h"
25	#include "llvm/ADT/StringMap.h"
26	#include "llvm/ADT/StringSet.h"
27	#include "llvm/Frontend/OpenMP/OMPConstants.h"
28	#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
29	#include "llvm/IR/Function.h"
30	#include "llvm/IR/ValueHandle.h"
31	#include "llvm/Support/AtomicOrdering.h"
32
33	namespace llvm {
34	class ArrayType;
35	class Constant;
36	class FunctionType;
37	class GlobalVariable;
38	class Type;
39	class Value;
40	class OpenMPIRBuilder;
41	} // namespace llvm
42
43	namespace clang {
44	class Expr;
45	class OMPDependClause;
46	class OMPExecutableDirective;
47	class OMPLoopDirective;
48	class VarDecl;
49	class OMPDeclareReductionDecl;
50
51	namespace CodeGen {
52	class Address;
53	class CodeGenFunction;
54	class CodeGenModule;
55
56	/// A basic class for pre|post-action for advanced codegen sequence for OpenMP
57	/// region.
58	class PrePostActionTy {
59	public:
60	explicit PrePostActionTy() {}
61	virtual void Enter(CodeGenFunction &CGF) {}
62	virtual void Exit(CodeGenFunction &CGF) {}
63	virtual ~PrePostActionTy() {}
64	};
65
66	/// Class provides a way to call simple version of codegen for OpenMP region, or
67	/// an advanced with possible pre|post-actions in codegen.
68	class RegionCodeGenTy final {
69	intptr_t CodeGen;
70	typedef void (*CodeGenTy)(intptr_t, CodeGenFunction &, PrePostActionTy &);
71	CodeGenTy Callback;
72	mutable PrePostActionTy *PrePostAction;
73	RegionCodeGenTy() = delete;
74	template <typename Callable>
75	static void CallbackFn(intptr_t CodeGen, CodeGenFunction &CGF,
76	PrePostActionTy &Action) {
77	return (*reinterpret_cast<Callable *>(CodeGen))(CGF, Action);
78	}
79
80	public:
81	template <typename Callable>
82	RegionCodeGenTy(
83	Callable &&CodeGen,
84	std::enable_if_t<!std::is_same<std::remove_reference_t<Callable>,
85	RegionCodeGenTy>::value> * = nullptr)
86	: CodeGen(reinterpret_cast<intptr_t>(&CodeGen)),
87	Callback(CallbackFn<std::remove_reference_t<Callable>>),
88	PrePostAction(nullptr) {}
89	void setAction(PrePostActionTy &Action) const { PrePostAction = &Action; }
90	void operator()(CodeGenFunction &CGF) const;
91	};
92
93	struct OMPTaskDataTy final {
94	SmallVector<const Expr *, 4> PrivateVars;
95	SmallVector<const Expr *, 4> PrivateCopies;
96	SmallVector<const Expr *, 4> FirstprivateVars;
97	SmallVector<const Expr *, 4> FirstprivateCopies;
98	SmallVector<const Expr *, 4> FirstprivateInits;
99	SmallVector<const Expr *, 4> LastprivateVars;
100	SmallVector<const Expr *, 4> LastprivateCopies;
101	SmallVector<const Expr *, 4> ReductionVars;
102	SmallVector<const Expr *, 4> ReductionOrigs;
103	SmallVector<const Expr *, 4> ReductionCopies;
104	SmallVector<const Expr *, 4> ReductionOps;
105	SmallVector<CanonicalDeclPtr<const VarDecl>, 4> PrivateLocals;
106	struct DependData {
107	OpenMPDependClauseKind DepKind = OMPC_DEPEND_unknown;
108	const Expr *IteratorExpr = nullptr;
109	SmallVector<const Expr *, 4> DepExprs;
110	explicit DependData() = default;
111	DependData(OpenMPDependClauseKind DepKind, const Expr *IteratorExpr)
112	: DepKind(DepKind), IteratorExpr(IteratorExpr) {}
113	};
114	SmallVector<DependData, 4> Dependences;
115	llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
116	llvm::PointerIntPair<llvm::Value *, 1, bool> Schedule;
117	llvm::PointerIntPair<llvm::Value *, 1, bool> Priority;
118	llvm::Value *Reductions = nullptr;
119	unsigned NumberOfParts = 0;
120	bool Tied = true;
121	bool Nogroup = false;
122	bool IsReductionWithTaskMod = false;
123	bool IsWorksharingReduction = false;
124	bool HasNowaitClause = false;
125	};
126
127	/// Class intended to support codegen of all kind of the reduction clauses.
128	class ReductionCodeGen {
129	private:
130	/// Data required for codegen of reduction clauses.
131	struct ReductionData {
132	/// Reference to the item shared between tasks to reduce into.
133	const Expr *Shared = nullptr;
134	/// Reference to the original item.
135	const Expr *Ref = nullptr;
136	/// Helper expression for generation of private copy.
137	const Expr *Private = nullptr;
138	/// Helper expression for generation reduction operation.
139	const Expr *ReductionOp = nullptr;
140	ReductionData(const Expr *Shared, const Expr *Ref, const Expr *Private,
141	const Expr *ReductionOp)
142	: Shared(Shared), Ref(Ref), Private(Private), ReductionOp(ReductionOp) {
143	}
144	};
145	/// List of reduction-based clauses.
146	SmallVector<ReductionData, 4> ClausesData;
147
148	/// List of addresses of shared variables/expressions.
149	SmallVector<std::pair<LValue, LValue>, 4> SharedAddresses;
150	/// List of addresses of original variables/expressions.
151	SmallVector<std::pair<LValue, LValue>, 4> OrigAddresses;
152	/// Sizes of the reduction items in chars.
153	SmallVector<std::pair<llvm::Value *, llvm::Value *>, 4> Sizes;
154	/// Base declarations for the reduction items.
155	SmallVector<const VarDecl *, 4> BaseDecls;
156
157	/// Emits lvalue for shared expression.
158	LValue emitSharedLValue(CodeGenFunction &CGF, const Expr *E);
159	/// Emits upper bound for shared expression (if array section).
160	LValue emitSharedLValueUB(CodeGenFunction &CGF, const Expr *E);
161	/// Performs aggregate initialization.
162	/// \param N Number of reduction item in the common list.
163	/// \param PrivateAddr Address of the corresponding private item.
164	/// \param SharedAddr Address of the original shared variable.
165	/// \param DRD Declare reduction construct used for reduction item.
166	void emitAggregateInitialization(CodeGenFunction &CGF, unsigned N,
167	Address PrivateAddr, Address SharedAddr,
168	const OMPDeclareReductionDecl *DRD);
169
170	public:
171	ReductionCodeGen(ArrayRef<const Expr *> Shareds, ArrayRef<const Expr *> Origs,
172	ArrayRef<const Expr *> Privates,
173	ArrayRef<const Expr *> ReductionOps);
174	/// Emits lvalue for the shared and original reduction item.
175	/// \param N Number of the reduction item.
176	void emitSharedOrigLValue(CodeGenFunction &CGF, unsigned N);
177	/// Emits the code for the variable-modified type, if required.
178	/// \param N Number of the reduction item.
179	void emitAggregateType(CodeGenFunction &CGF, unsigned N);
180	/// Emits the code for the variable-modified type, if required.
181	/// \param N Number of the reduction item.
182	/// \param Size Size of the type in chars.
183	void emitAggregateType(CodeGenFunction &CGF, unsigned N, llvm::Value *Size);
184	/// Performs initialization of the private copy for the reduction item.
185	/// \param N Number of the reduction item.
186	/// \param PrivateAddr Address of the corresponding private item.
187	/// \param DefaultInit Default initialization sequence that should be
188	/// performed if no reduction specific initialization is found.
189	/// \param SharedAddr Address of the original shared variable.
190	void
191	emitInitialization(CodeGenFunction &CGF, unsigned N, Address PrivateAddr,
192	Address SharedAddr,
193	llvm::function_ref<bool(CodeGenFunction &)> DefaultInit);
194	/// Returns true if the private copy requires cleanups.
195	bool needCleanups(unsigned N);
196	/// Emits cleanup code for the reduction item.
197	/// \param N Number of the reduction item.
198	/// \param PrivateAddr Address of the corresponding private item.
199	void emitCleanups(CodeGenFunction &CGF, unsigned N, Address PrivateAddr);
200	/// Adjusts \p PrivatedAddr for using instead of the original variable
201	/// address in normal operations.
202	/// \param N Number of the reduction item.
203	/// \param PrivateAddr Address of the corresponding private item.
204	Address adjustPrivateAddress(CodeGenFunction &CGF, unsigned N,
205	Address PrivateAddr);
206	/// Returns LValue for the reduction item.
207	LValue getSharedLValue(unsigned N) const { return SharedAddresses[N].first; }
208	/// Returns LValue for the original reduction item.
209	LValue getOrigLValue(unsigned N) const { return OrigAddresses[N].first; }
210	/// Returns the size of the reduction item (in chars and total number of
211	/// elements in the item), or nullptr, if the size is a constant.
212	std::pair<llvm::Value *, llvm::Value *> getSizes(unsigned N) const {
213	return Sizes[N];
214	}
215	/// Returns the base declaration of the reduction item.
216	const VarDecl *getBaseDecl(unsigned N) const { return BaseDecls[N]; }
217	/// Returns the base declaration of the reduction item.
218	const Expr *getRefExpr(unsigned N) const { return ClausesData[N].Ref; }
219	/// Returns true if the initialization of the reduction item uses initializer
220	/// from declare reduction construct.
221	bool usesReductionInitializer(unsigned N) const;
222	/// Return the type of the private item.
223	QualType getPrivateType(unsigned N) const {
224	return cast<VarDecl>(Val: cast<DeclRefExpr>(Val: ClausesData[N].Private)->getDecl())
225	->getType();
226	}
227	};
228
229	class CGOpenMPRuntime {
230	public:
231	/// Allows to disable automatic handling of functions used in target regions
232	/// as those marked as `omp declare target`.
233	class DisableAutoDeclareTargetRAII {
234	CodeGenModule &CGM;
235	bool SavedShouldMarkAsGlobal = false;
236
237	public:
238	DisableAutoDeclareTargetRAII(CodeGenModule &CGM);
239	~DisableAutoDeclareTargetRAII();
240	};
241
242	/// Manages list of nontemporal decls for the specified directive.
243	class NontemporalDeclsRAII {
244	CodeGenModule &CGM;
245	const bool NeedToPush;
246
247	public:
248	NontemporalDeclsRAII(CodeGenModule &CGM, const OMPLoopDirective &S);
249	~NontemporalDeclsRAII();
250	};
251
252	/// Manages list of nontemporal decls for the specified directive.
253	class UntiedTaskLocalDeclsRAII {
254	CodeGenModule &CGM;
255	const bool NeedToPush;
256
257	public:
258	UntiedTaskLocalDeclsRAII(
259	CodeGenFunction &CGF,
260	const llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
261	std::pair<Address, Address>> &LocalVars);
262	~UntiedTaskLocalDeclsRAII();
263	};
264
265	/// Maps the expression for the lastprivate variable to the global copy used
266	/// to store new value because original variables are not mapped in inner
267	/// parallel regions. Only private copies are captured but we need also to
268	/// store private copy in shared address.
269	/// Also, stores the expression for the private loop counter and it
270	/// threaprivate name.
271	struct LastprivateConditionalData {
272	llvm::MapVector<CanonicalDeclPtr<const Decl>, SmallString<16>>
273	DeclToUniqueName;
274	LValue IVLVal;
275	llvm::Function *Fn = nullptr;
276	bool Disabled = false;
277	};
278	/// Manages list of lastprivate conditional decls for the specified directive.
279	class LastprivateConditionalRAII {
280	enum class ActionToDo {
281	DoNotPush,
282	PushAsLastprivateConditional,
283	DisableLastprivateConditional,
284	};
285	CodeGenModule &CGM;
286	ActionToDo Action = ActionToDo::DoNotPush;
287
288	/// Check and try to disable analysis of inner regions for changes in
289	/// lastprivate conditional.
290	void tryToDisableInnerAnalysis(const OMPExecutableDirective &S,
291	llvm::DenseSet<CanonicalDeclPtr<const Decl>>
292	&NeedToAddForLPCsAsDisabled) const;
293
294	LastprivateConditionalRAII(CodeGenFunction &CGF,
295	const OMPExecutableDirective &S);
296
297	public:
298	explicit LastprivateConditionalRAII(CodeGenFunction &CGF,
299	const OMPExecutableDirective &S,
300	LValue IVLVal);
301	static LastprivateConditionalRAII disable(CodeGenFunction &CGF,
302	const OMPExecutableDirective &S);
303	~LastprivateConditionalRAII();
304	};
305
306	llvm::OpenMPIRBuilder &getOMPBuilder() { return OMPBuilder; }
307
308	protected:
309	CodeGenModule &CGM;
310
311	/// An OpenMP-IR-Builder instance.
312	llvm::OpenMPIRBuilder OMPBuilder;
313
314	/// Helper to determine the min/max number of threads/teams for \p D.
315	void computeMinAndMaxThreadsAndTeams(const OMPExecutableDirective &D,
316	CodeGenFunction &CGF,
317	int32_t &MinThreadsVal,
318	int32_t &MaxThreadsVal,
319	int32_t &MinTeamsVal,
320	int32_t &MaxTeamsVal);
321
322	/// Helper to emit outlined function for 'target' directive.
323	/// \param D Directive to emit.
324	/// \param ParentName Name of the function that encloses the target region.
325	/// \param OutlinedFn Outlined function value to be defined by this call.
326	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
327	/// \param IsOffloadEntry True if the outlined function is an offload entry.
328	/// \param CodeGen Lambda codegen specific to an accelerator device.
329	/// An outlined function may not be an entry if, e.g. the if clause always
330	/// evaluates to false.
331	virtual void emitTargetOutlinedFunctionHelper(const OMPExecutableDirective &D,
332	StringRef ParentName,
333	llvm::Function *&OutlinedFn,
334	llvm::Constant *&OutlinedFnID,
335	bool IsOffloadEntry,
336	const RegionCodeGenTy &CodeGen);
337
338	/// Returns pointer to ident_t type.
339	llvm::Type *getIdentTyPointerTy();
340
341	/// Gets thread id value for the current thread.
342	///
343	llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
344
345	/// Get the function name of an outlined region.
346	std::string getOutlinedHelperName(StringRef Name) const;
347	std::string getOutlinedHelperName(CodeGenFunction &CGF) const;
348
349	/// Get the function name of a reduction function.
350	std::string getReductionFuncName(StringRef Name) const;
351
352	/// Emits \p Callee function call with arguments \p Args with location \p Loc.
353	void emitCall(CodeGenFunction &CGF, SourceLocation Loc,
354	llvm::FunctionCallee Callee,
355	ArrayRef<llvm::Value *> Args = std::nullopt) const;
356
357	/// Emits address of the word in a memory where current thread id is
358	/// stored.
359	virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc);
360
361	void setLocThreadIdInsertPt(CodeGenFunction &CGF,
362	bool AtCurrentPoint = false);
363	void clearLocThreadIdInsertPt(CodeGenFunction &CGF);
364
365	/// Check if the default location must be constant.
366	/// Default is false to support OMPT/OMPD.
367	virtual bool isDefaultLocationConstant() const { return false; }
368
369	/// Returns additional flags that can be stored in reserved_2 field of the
370	/// default location.
371	virtual unsigned getDefaultLocationReserved2Flags() const { return 0; }
372
373	/// Returns default flags for the barriers depending on the directive, for
374	/// which this barier is going to be emitted.
375	static unsigned getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind);
376
377	/// Get the LLVM type for the critical name.
378	llvm::ArrayType *getKmpCriticalNameTy() const {return KmpCriticalNameTy;}
379
380	/// Returns corresponding lock object for the specified critical region
381	/// name. If the lock object does not exist it is created, otherwise the
382	/// reference to the existing copy is returned.
383	/// \param CriticalName Name of the critical region.
384	///
385	llvm::Value *getCriticalRegionLock(StringRef CriticalName);
386
387	protected:
388	/// Map for SourceLocation and OpenMP runtime library debug locations.
389	typedef llvm::DenseMap<SourceLocation, llvm::Value *> OpenMPDebugLocMapTy;
390	OpenMPDebugLocMapTy OpenMPDebugLocMap;
391	/// The type for a microtask which gets passed to __kmpc_fork_call().
392	/// Original representation is:
393	/// typedef void (kmpc_micro)(kmp_int32 global_tid, kmp_int32 bound_tid,...);
394	llvm::FunctionType *Kmpc_MicroTy = nullptr;
395	/// Stores debug location and ThreadID for the function.
396	struct DebugLocThreadIdTy {
397	llvm::Value *DebugLoc;
398	llvm::Value *ThreadID;
399	/// Insert point for the service instructions.
400	llvm::AssertingVH<llvm::Instruction> ServiceInsertPt = nullptr;
401	};
402	/// Map of local debug location, ThreadId and functions.
403	typedef llvm::DenseMap<llvm::Function *, DebugLocThreadIdTy>
404	OpenMPLocThreadIDMapTy;
405	OpenMPLocThreadIDMapTy OpenMPLocThreadIDMap;
406	/// Map of UDRs and corresponding combiner/initializer.
407	typedef llvm::DenseMap<const OMPDeclareReductionDecl *,
408	std::pair<llvm::Function *, llvm::Function *>>
409	UDRMapTy;
410	UDRMapTy UDRMap;
411	/// Map of functions and locally defined UDRs.
412	typedef llvm::DenseMap<llvm::Function *,
413	SmallVector<const OMPDeclareReductionDecl *, 4>>
414	FunctionUDRMapTy;
415	FunctionUDRMapTy FunctionUDRMap;
416	/// Map from the user-defined mapper declaration to its corresponding
417	/// functions.
418	llvm::DenseMap<const OMPDeclareMapperDecl *, llvm::Function *> UDMMap;
419	/// Map of functions and their local user-defined mappers.
420	using FunctionUDMMapTy =
421	llvm::DenseMap<llvm::Function *,
422	SmallVector<const OMPDeclareMapperDecl *, 4>>;
423	FunctionUDMMapTy FunctionUDMMap;
424	/// Maps local variables marked as lastprivate conditional to their internal
425	/// types.
426	llvm::DenseMap<llvm::Function *,
427	llvm::DenseMap<CanonicalDeclPtr<const Decl>,
428	std::tuple<QualType, const FieldDecl *,
429	const FieldDecl *, LValue>>>
430	LastprivateConditionalToTypes;
431	/// Maps function to the position of the untied task locals stack.
432	llvm::DenseMap<llvm::Function *, unsigned> FunctionToUntiedTaskStackMap;
433	/// Type kmp_critical_name, originally defined as typedef kmp_int32
434	/// kmp_critical_name[8];
435	llvm::ArrayType *KmpCriticalNameTy;
436	/// An ordered map of auto-generated variables to their unique names.
437	/// It stores variables with the following names: 1) ".gomp_critical_user_" +
438	/// <critical_section_name> + ".var" for "omp critical" directives; 2)
439	/// <mangled_name_for_global_var> + ".cache." for cache for threadprivate
440	/// variables.
441	llvm::StringMap<llvm::AssertingVH<llvm::GlobalVariable>,
442	llvm::BumpPtrAllocator> InternalVars;
443	/// Type typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *);
444	llvm::Type *KmpRoutineEntryPtrTy = nullptr;
445	QualType KmpRoutineEntryPtrQTy;
446	/// Type typedef struct kmp_task {
447	/// void * shareds; /**< pointer to block of pointers to
448	/// shared vars */
449	/// kmp_routine_entry_t routine; /**< pointer to routine to call for
450	/// executing task */
451	/// kmp_int32 part_id; /**< part id for the task */
452	/// kmp_routine_entry_t destructors; /* pointer to function to invoke
453	/// deconstructors of firstprivate C++ objects */
454	/// } kmp_task_t;
455	QualType KmpTaskTQTy;
456	/// Saved kmp_task_t for task directive.
457	QualType SavedKmpTaskTQTy;
458	/// Saved kmp_task_t for taskloop-based directive.
459	QualType SavedKmpTaskloopTQTy;
460	/// Type typedef struct kmp_depend_info {
461	/// kmp_intptr_t base_addr;
462	/// size_t len;
463	/// struct {
464	/// bool in:1;
465	/// bool out:1;
466	/// } flags;
467	/// } kmp_depend_info_t;
468	QualType KmpDependInfoTy;
469	/// Type typedef struct kmp_task_affinity_info {
470	/// kmp_intptr_t base_addr;
471	/// size_t len;
472	/// struct {
473	/// bool flag1 : 1;
474	/// bool flag2 : 1;
475	/// kmp_int32 reserved : 30;
476	/// } flags;
477	/// } kmp_task_affinity_info_t;
478	QualType KmpTaskAffinityInfoTy;
479	/// struct kmp_dim { // loop bounds info casted to kmp_int64
480	/// kmp_int64 lo; // lower
481	/// kmp_int64 up; // upper
482	/// kmp_int64 st; // stride
483	/// };
484	QualType KmpDimTy;
485
486	bool ShouldMarkAsGlobal = true;
487	/// List of the emitted declarations.
488	llvm::DenseSet<CanonicalDeclPtr<const Decl>> AlreadyEmittedTargetDecls;
489	/// List of the global variables with their addresses that should not be
490	/// emitted for the target.
491	llvm::StringMap<llvm::WeakTrackingVH> EmittedNonTargetVariables;
492
493	/// List of variables that can become declare target implicitly and, thus,
494	/// must be emitted.
495	llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
496
497	using NontemporalDeclsSet = llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>>;
498	/// Stack for list of declarations in current context marked as nontemporal.
499	/// The set is the union of all current stack elements.
500	llvm::SmallVector<NontemporalDeclsSet, 4> NontemporalDeclsStack;
501
502	using UntiedLocalVarsAddressesMap =
503	llvm::MapVector<CanonicalDeclPtr<const VarDecl>,
504	std::pair<Address, Address>>;
505	llvm::SmallVector<UntiedLocalVarsAddressesMap, 4> UntiedLocalVarsStack;
506
507	/// Stack for list of addresses of declarations in current context marked as
508	/// lastprivate conditional. The set is the union of all current stack
509	/// elements.
510	llvm::SmallVector<LastprivateConditionalData, 4> LastprivateConditionalStack;
511
512	/// Flag for keeping track of weather a requires unified_shared_memory
513	/// directive is present.
514	bool HasRequiresUnifiedSharedMemory = false;
515
516	/// Atomic ordering from the omp requires directive.
517	llvm::AtomicOrdering RequiresAtomicOrdering = llvm::AtomicOrdering::Monotonic;
518
519	/// Flag for keeping track of weather a target region has been emitted.
520	bool HasEmittedTargetRegion = false;
521
522	/// Flag for keeping track of weather a device routine has been emitted.
523	/// Device routines are specific to the
524	bool HasEmittedDeclareTargetRegion = false;
525
526	/// Start scanning from statement \a S and emit all target regions
527	/// found along the way.
528	/// \param S Starting statement.
529	/// \param ParentName Name of the function declaration that is being scanned.
530	void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName);
531
532	/// Build type kmp_routine_entry_t (if not built yet).
533	void emitKmpRoutineEntryT(QualType KmpInt32Ty);
534
535	/// Returns pointer to kmpc_micro type.
536	llvm::Type *getKmpc_MicroPointerTy();
537
538	/// If the specified mangled name is not in the module, create and
539	/// return threadprivate cache object. This object is a pointer's worth of
540	/// storage that's reserved for use by the OpenMP runtime.
541	/// \param VD Threadprivate variable.
542	/// \return Cache variable for the specified threadprivate.
543	llvm::Constant *getOrCreateThreadPrivateCache(const VarDecl *VD);
544
545	/// Set of threadprivate variables with the generated initializer.
546	llvm::StringSet<> ThreadPrivateWithDefinition;
547
548	/// Set of declare target variables with the generated initializer.
549	llvm::StringSet<> DeclareTargetWithDefinition;
550
551	/// Emits initialization code for the threadprivate variables.
552	/// \param VDAddr Address of the global variable \a VD.
553	/// \param Ctor Pointer to a global init function for \a VD.
554	/// \param CopyCtor Pointer to a global copy function for \a VD.
555	/// \param Dtor Pointer to a global destructor function for \a VD.
556	/// \param Loc Location of threadprivate declaration.
557	void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr,
558	llvm::Value *Ctor, llvm::Value *CopyCtor,
559	llvm::Value *Dtor, SourceLocation Loc);
560
561	/// Emit the array initialization or deletion portion for user-defined mapper
562	/// code generation.
563	void emitUDMapperArrayInitOrDel(CodeGenFunction &MapperCGF,
564	llvm::Value *Handle, llvm::Value *BasePtr,
565	llvm::Value *Ptr, llvm::Value *Size,
566	llvm::Value *MapType, llvm::Value *MapName,
567	CharUnits ElementSize,
568	llvm::BasicBlock *ExitBB, bool IsInit);
569
570	struct TaskResultTy {
571	llvm::Value *NewTask = nullptr;
572	llvm::Function *TaskEntry = nullptr;
573	llvm::Value *NewTaskNewTaskTTy = nullptr;
574	LValue TDBase;
575	const RecordDecl *KmpTaskTQTyRD = nullptr;
576	llvm::Value *TaskDupFn = nullptr;
577	};
578	/// Emit task region for the task directive. The task region is emitted in
579	/// several steps:
580	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
581	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
582	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
583	/// function:
584	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
585	/// TaskFunction(gtid, tt->part_id, tt->shareds);
586	/// return 0;
587	/// }
588	/// 2. Copy a list of shared variables to field shareds of the resulting
589	/// structure kmp_task_t returned by the previous call (if any).
590	/// 3. Copy a pointer to destructions function to field destructions of the
591	/// resulting structure kmp_task_t.
592	/// \param D Current task directive.
593	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
594	/// /*part_id*/, captured_struct */*__context*/);
595	/// \param SharedsTy A type which contains references the shared variables.
596	/// \param Shareds Context with the list of shared variables from the \p
597	/// TaskFunction.
598	/// \param Data Additional data for task generation like tiednsee, final
599	/// state, list of privates etc.
600	TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
601	const OMPExecutableDirective &D,
602	llvm::Function *TaskFunction, QualType SharedsTy,
603	Address Shareds, const OMPTaskDataTy &Data);
604
605	/// Emit update for lastprivate conditional data.
606	void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
607	StringRef UniqueDeclName, LValue LVal,
608	SourceLocation Loc);
609
610	/// Returns the number of the elements and the address of the depobj
611	/// dependency array.
612	/// \return Number of elements in depobj array and the pointer to the array of
613	/// dependencies.
614	std::pair<llvm::Value *, LValue> getDepobjElements(CodeGenFunction &CGF,
615	LValue DepobjLVal,
616	SourceLocation Loc);
617
618	SmallVector<llvm::Value *, 4>
619	emitDepobjElementsSizes(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
620	const OMPTaskDataTy::DependData &Data);
621
622	void emitDepobjElements(CodeGenFunction &CGF, QualType &KmpDependInfoTy,
623	LValue PosLVal, const OMPTaskDataTy::DependData &Data,
624	Address DependenciesArray);
625
626	public:
627	explicit CGOpenMPRuntime(CodeGenModule &CGM);
628	virtual ~CGOpenMPRuntime() {}
629	virtual void clear();
630
631	/// Emits object of ident_t type with info for source location.
632	/// \param Flags Flags for OpenMP location.
633	/// \param EmitLoc emit source location with debug-info is off.
634	///
635	llvm::Value *emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
636	unsigned Flags = 0, bool EmitLoc = false);
637
638	/// Emit the number of teams for a target directive. Inspect the num_teams
639	/// clause associated with a teams construct combined or closely nested
640	/// with the target directive.
641	///
642	/// Emit a team of size one for directives such as 'target parallel' that
643	/// have no associated teams construct.
644	///
645	/// Otherwise, return nullptr.
646	const Expr *getNumTeamsExprForTargetDirective(CodeGenFunction &CGF,
647	const OMPExecutableDirective &D,
648	int32_t &MinTeamsVal,
649	int32_t &MaxTeamsVal);
650	llvm::Value *emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
651	const OMPExecutableDirective &D);
652
653	/// Check for a number of threads upper bound constant value (stored in \p
654	/// UpperBound), or expression (returned). If the value is conditional (via an
655	/// if-clause), store the condition in \p CondExpr. Similarly, a potential
656	/// thread limit expression is stored in \p ThreadLimitExpr. If \p
657	/// UpperBoundOnly is true, no expression evaluation is perfomed.
658	const Expr *getNumThreadsExprForTargetDirective(
659	CodeGenFunction &CGF, const OMPExecutableDirective &D,
660	int32_t &UpperBound, bool UpperBoundOnly,
661	llvm::Value **CondExpr = nullptr, const Expr **ThreadLimitExpr = nullptr);
662
663	/// Emit an expression that denotes the number of threads a target region
664	/// shall use. Will generate "i32 0" to allow the runtime to choose.
665	llvm::Value *
666	emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
667	const OMPExecutableDirective &D);
668
669	/// Return the trip count of loops associated with constructs / 'target teams
670	/// distribute' and 'teams distribute parallel for'. \param SizeEmitter Emits
671	/// the int64 value for the number of iterations of the associated loop.
672	llvm::Value *emitTargetNumIterationsCall(
673	CodeGenFunction &CGF, const OMPExecutableDirective &D,
674	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
675	const OMPLoopDirective &D)>
676	SizeEmitter);
677
678	/// Returns true if the current target is a GPU.
679	virtual bool isGPU() const { return false; }
680
681	/// Check if the variable length declaration is delayed:
682	virtual bool isDelayedVariableLengthDecl(CodeGenFunction &CGF,
683	const VarDecl *VD) const {
684	return false;
685	};
686
687	/// Get call to __kmpc_alloc_shared
688	virtual std::pair<llvm::Value *, llvm::Value *>
689	getKmpcAllocShared(CodeGenFunction &CGF, const VarDecl *VD) {
690	llvm_unreachable("not implemented");
691	}
692
693	/// Get call to __kmpc_free_shared
694	virtual void getKmpcFreeShared(
695	CodeGenFunction &CGF,
696	const std::pair<llvm::Value *, llvm::Value *> &AddrSizePair) {
697	llvm_unreachable("not implemented");
698	}
699
700	/// Emits code for OpenMP 'if' clause using specified \a CodeGen
701	/// function. Here is the logic:
702	/// if (Cond) {
703	/// ThenGen();
704	/// } else {
705	/// ElseGen();
706	/// }
707	void emitIfClause(CodeGenFunction &CGF, const Expr *Cond,
708	const RegionCodeGenTy &ThenGen,
709	const RegionCodeGenTy &ElseGen);
710
711	/// Checks if the \p Body is the \a CompoundStmt and returns its child
712	/// statement iff there is only one that is not evaluatable at the compile
713	/// time.
714	static const Stmt *getSingleCompoundChild(ASTContext &Ctx, const Stmt *Body);
715
716	/// Get the platform-specific name separator.
717	std::string getName(ArrayRef<StringRef> Parts) const;
718
719	/// Emit code for the specified user defined reduction construct.
720	virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
721	const OMPDeclareReductionDecl *D);
722	/// Get combiner/initializer for the specified user-defined reduction, if any.
723	virtual std::pair<llvm::Function *, llvm::Function *>
724	getUserDefinedReduction(const OMPDeclareReductionDecl *D);
725
726	/// Emit the function for the user defined mapper construct.
727	void emitUserDefinedMapper(const OMPDeclareMapperDecl *D,
728	CodeGenFunction *CGF = nullptr);
729	/// Get the function for the specified user-defined mapper. If it does not
730	/// exist, create one.
731	llvm::Function *
732	getOrCreateUserDefinedMapperFunc(const OMPDeclareMapperDecl *D);
733
734	/// Emits outlined function for the specified OpenMP parallel directive
735	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
736	/// kmp_int32 BoundID, struct context_vars*).
737	/// \param CGF Reference to current CodeGenFunction.
738	/// \param D OpenMP directive.
739	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
740	/// \param InnermostKind Kind of innermost directive (for simple directives it
741	/// is a directive itself, for combined - its innermost directive).
742	/// \param CodeGen Code generation sequence for the \a D directive.
743	virtual llvm::Function *emitParallelOutlinedFunction(
744	CodeGenFunction &CGF, const OMPExecutableDirective &D,
745	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
746	const RegionCodeGenTy &CodeGen);
747
748	/// Emits outlined function for the specified OpenMP teams directive
749	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
750	/// kmp_int32 BoundID, struct context_vars*).
751	/// \param CGF Reference to current CodeGenFunction.
752	/// \param D OpenMP directive.
753	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
754	/// \param InnermostKind Kind of innermost directive (for simple directives it
755	/// is a directive itself, for combined - its innermost directive).
756	/// \param CodeGen Code generation sequence for the \a D directive.
757	virtual llvm::Function *emitTeamsOutlinedFunction(
758	CodeGenFunction &CGF, const OMPExecutableDirective &D,
759	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
760	const RegionCodeGenTy &CodeGen);
761
762	/// Emits outlined function for the OpenMP task directive \a D. This
763	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
764	/// TaskT).
765	/// \param D OpenMP directive.
766	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
767	/// \param PartIDVar Variable for partition id in the current OpenMP untied
768	/// task region.
769	/// \param TaskTVar Variable for task_t argument.
770	/// \param InnermostKind Kind of innermost directive (for simple directives it
771	/// is a directive itself, for combined - its innermost directive).
772	/// \param CodeGen Code generation sequence for the \a D directive.
773	/// \param Tied true if task is generated for tied task, false otherwise.
774	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
775	/// tasks.
776	///
777	virtual llvm::Function *emitTaskOutlinedFunction(
778	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
779	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
780	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
781	bool Tied, unsigned &NumberOfParts);
782
783	/// Cleans up references to the objects in finished function.
784	///
785	virtual void functionFinished(CodeGenFunction &CGF);
786
787	/// Emits code for parallel or serial call of the \a OutlinedFn with
788	/// variables captured in a record which address is stored in \a
789	/// CapturedStruct.
790	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
791	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
792	/// \param CapturedVars A pointer to the record with the references to
793	/// variables used in \a OutlinedFn function.
794	/// \param IfCond Condition in the associated 'if' clause, if it was
795	/// specified, nullptr otherwise.
796	/// \param NumThreads The value corresponding to the num_threads clause, if
797	/// any, or nullptr.
798	///
799	virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
800	llvm::Function *OutlinedFn,
801	ArrayRef<llvm::Value *> CapturedVars,
802	const Expr *IfCond, llvm::Value *NumThreads);
803
804	/// Emits a critical region.
805	/// \param CriticalName Name of the critical region.
806	/// \param CriticalOpGen Generator for the statement associated with the given
807	/// critical region.
808	/// \param Hint Value of the 'hint' clause (optional).
809	virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
810	const RegionCodeGenTy &CriticalOpGen,
811	SourceLocation Loc,
812	const Expr *Hint = nullptr);
813
814	/// Emits a master region.
815	/// \param MasterOpGen Generator for the statement associated with the given
816	/// master region.
817	virtual void emitMasterRegion(CodeGenFunction &CGF,
818	const RegionCodeGenTy &MasterOpGen,
819	SourceLocation Loc);
820
821	/// Emits a masked region.
822	/// \param MaskedOpGen Generator for the statement associated with the given
823	/// masked region.
824	virtual void emitMaskedRegion(CodeGenFunction &CGF,
825	const RegionCodeGenTy &MaskedOpGen,
826	SourceLocation Loc,
827	const Expr *Filter = nullptr);
828
829	/// Emits code for a taskyield directive.
830	virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc);
831
832	/// Emit __kmpc_error call for error directive
833	/// extern void __kmpc_error(ident_t *loc, int severity, const char *message);
834	virtual void emitErrorCall(CodeGenFunction &CGF, SourceLocation Loc, Expr *ME,
835	bool IsFatal);
836
837	/// Emit a taskgroup region.
838	/// \param TaskgroupOpGen Generator for the statement associated with the
839	/// given taskgroup region.
840	virtual void emitTaskgroupRegion(CodeGenFunction &CGF,
841	const RegionCodeGenTy &TaskgroupOpGen,
842	SourceLocation Loc);
843
844	/// Emits a single region.
845	/// \param SingleOpGen Generator for the statement associated with the given
846	/// single region.
847	virtual void emitSingleRegion(CodeGenFunction &CGF,
848	const RegionCodeGenTy &SingleOpGen,
849	SourceLocation Loc,
850	ArrayRef<const Expr *> CopyprivateVars,
851	ArrayRef<const Expr *> DestExprs,
852	ArrayRef<const Expr *> SrcExprs,
853	ArrayRef<const Expr *> AssignmentOps);
854
855	/// Emit an ordered region.
856	/// \param OrderedOpGen Generator for the statement associated with the given
857	/// ordered region.
858	virtual void emitOrderedRegion(CodeGenFunction &CGF,
859	const RegionCodeGenTy &OrderedOpGen,
860	SourceLocation Loc, bool IsThreads);
861
862	/// Emit an implicit/explicit barrier for OpenMP threads.
863	/// \param Kind Directive for which this implicit barrier call must be
864	/// generated. Must be OMPD_barrier for explicit barrier generation.
865	/// \param EmitChecks true if need to emit checks for cancellation barriers.
866	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
867	/// runtime class decides which one to emit (simple or with cancellation
868	/// checks).
869	///
870	virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
871	OpenMPDirectiveKind Kind,
872	bool EmitChecks = true,
873	bool ForceSimpleCall = false);
874
875	/// Check if the specified \a ScheduleKind is static non-chunked.
876	/// This kind of worksharing directive is emitted without outer loop.
877	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
878	/// \param Chunked True if chunk is specified in the clause.
879	///
880	virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind,
881	bool Chunked) const;
882
883	/// Check if the specified \a ScheduleKind is static non-chunked.
884	/// This kind of distribute directive is emitted without outer loop.
885	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
886	/// \param Chunked True if chunk is specified in the clause.
887	///
888	virtual bool isStaticNonchunked(OpenMPDistScheduleClauseKind ScheduleKind,
889	bool Chunked) const;
890
891	/// Check if the specified \a ScheduleKind is static chunked.
892	/// \param ScheduleKind Schedule kind specified in the 'schedule' clause.
893	/// \param Chunked True if chunk is specified in the clause.
894	///
895	virtual bool isStaticChunked(OpenMPScheduleClauseKind ScheduleKind,
896	bool Chunked) const;
897
898	/// Check if the specified \a ScheduleKind is static non-chunked.
899	/// \param ScheduleKind Schedule kind specified in the 'dist_schedule' clause.
900	/// \param Chunked True if chunk is specified in the clause.
901	///
902	virtual bool isStaticChunked(OpenMPDistScheduleClauseKind ScheduleKind,
903	bool Chunked) const;
904
905	/// Check if the specified \a ScheduleKind is dynamic.
906	/// This kind of worksharing directive is emitted without outer loop.
907	/// \param ScheduleKind Schedule Kind specified in the 'schedule' clause.
908	///
909	virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const;
910
911	/// struct with the values to be passed to the dispatch runtime function
912	struct DispatchRTInput {
913	/// Loop lower bound
914	llvm::Value *LB = nullptr;
915	/// Loop upper bound
916	llvm::Value *UB = nullptr;
917	/// Chunk size specified using 'schedule' clause (nullptr if chunk
918	/// was not specified)
919	llvm::Value *Chunk = nullptr;
920	DispatchRTInput() = default;
921	DispatchRTInput(llvm::Value *LB, llvm::Value *UB, llvm::Value *Chunk)
922	: LB(LB), UB(UB), Chunk(Chunk) {}
923	};
924
925	/// Call the appropriate runtime routine to initialize it before start
926	/// of loop.
927
928	/// This is used for non static scheduled types and when the ordered
929	/// clause is present on the loop construct.
930	/// Depending on the loop schedule, it is necessary to call some runtime
931	/// routine before start of the OpenMP loop to get the loop upper / lower
932	/// bounds \a LB and \a UB and stride \a ST.
933	///
934	/// \param CGF Reference to current CodeGenFunction.
935	/// \param Loc Clang source location.
936	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
937	/// \param IVSize Size of the iteration variable in bits.
938	/// \param IVSigned Sign of the iteration variable.
939	/// \param Ordered true if loop is ordered, false otherwise.
940	/// \param DispatchValues struct containing llvm values for lower bound, upper
941	/// bound, and chunk expression.
942	/// For the default (nullptr) value, the chunk 1 will be used.
943	///
944	virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
945	const OpenMPScheduleTy &ScheduleKind,
946	unsigned IVSize, bool IVSigned, bool Ordered,
947	const DispatchRTInput &DispatchValues);
948
949	/// Struct with the values to be passed to the static runtime function
950	struct StaticRTInput {
951	/// Size of the iteration variable in bits.
952	unsigned IVSize = 0;
953	/// Sign of the iteration variable.
954	bool IVSigned = false;
955	/// true if loop is ordered, false otherwise.
956	bool Ordered = false;
957	/// Address of the output variable in which the flag of the last iteration
958	/// is returned.
959	Address IL = Address::invalid();
960	/// Address of the output variable in which the lower iteration number is
961	/// returned.
962	Address LB = Address::invalid();
963	/// Address of the output variable in which the upper iteration number is
964	/// returned.
965	Address UB = Address::invalid();
966	/// Address of the output variable in which the stride value is returned
967	/// necessary to generated the static_chunked scheduled loop.
968	Address ST = Address::invalid();
969	/// Value of the chunk for the static_chunked scheduled loop. For the
970	/// default (nullptr) value, the chunk 1 will be used.
971	llvm::Value *Chunk = nullptr;
972	StaticRTInput(unsigned IVSize, bool IVSigned, bool Ordered, Address IL,
973	Address LB, Address UB, Address ST,
974	llvm::Value *Chunk = nullptr)
975	: IVSize(IVSize), IVSigned(IVSigned), Ordered(Ordered), IL(IL), LB(LB),
976	UB(UB), ST(ST), Chunk(Chunk) {}
977	};
978	/// Call the appropriate runtime routine to initialize it before start
979	/// of loop.
980	///
981	/// This is used only in case of static schedule, when the user did not
982	/// specify a ordered clause on the loop construct.
983	/// Depending on the loop schedule, it is necessary to call some runtime
984	/// routine before start of the OpenMP loop to get the loop upper / lower
985	/// bounds LB and UB and stride ST.
986	///
987	/// \param CGF Reference to current CodeGenFunction.
988	/// \param Loc Clang source location.
989	/// \param DKind Kind of the directive.
990	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
991	/// \param Values Input arguments for the construct.
992	///
993	virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
994	OpenMPDirectiveKind DKind,
995	const OpenMPScheduleTy &ScheduleKind,
996	const StaticRTInput &Values);
997
998	///
999	/// \param CGF Reference to current CodeGenFunction.
1000	/// \param Loc Clang source location.
1001	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1002	/// \param Values Input arguments for the construct.
1003	///
1004	virtual void emitDistributeStaticInit(CodeGenFunction &CGF,
1005	SourceLocation Loc,
1006	OpenMPDistScheduleClauseKind SchedKind,
1007	const StaticRTInput &Values);
1008
1009	/// Call the appropriate runtime routine to notify that we finished
1010	/// iteration of the ordered loop with the dynamic scheduling.
1011	///
1012	/// \param CGF Reference to current CodeGenFunction.
1013	/// \param Loc Clang source location.
1014	/// \param IVSize Size of the iteration variable in bits.
1015	/// \param IVSigned Sign of the iteration variable.
1016	///
1017	virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF,
1018	SourceLocation Loc, unsigned IVSize,
1019	bool IVSigned);
1020
1021	/// Call the appropriate runtime routine to notify that we finished
1022	/// all the work with current loop.
1023	///
1024	/// \param CGF Reference to current CodeGenFunction.
1025	/// \param Loc Clang source location.
1026	/// \param DKind Kind of the directive for which the static finish is emitted.
1027	///
1028	virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1029	OpenMPDirectiveKind DKind);
1030
1031	/// Call __kmpc_dispatch_next(
1032	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1033	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1034	/// kmp_int[32|64] *p_stride);
1035	/// \param IVSize Size of the iteration variable in bits.
1036	/// \param IVSigned Sign of the iteration variable.
1037	/// \param IL Address of the output variable in which the flag of the
1038	/// last iteration is returned.
1039	/// \param LB Address of the output variable in which the lower iteration
1040	/// number is returned.
1041	/// \param UB Address of the output variable in which the upper iteration
1042	/// number is returned.
1043	/// \param ST Address of the output variable in which the stride value is
1044	/// returned.
1045	virtual llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1046	unsigned IVSize, bool IVSigned,
1047	Address IL, Address LB,
1048	Address UB, Address ST);
1049
1050	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1051	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1052	/// clause.
1053	/// \param NumThreads An integer value of threads.
1054	virtual void emitNumThreadsClause(CodeGenFunction &CGF,
1055	llvm::Value *NumThreads,
1056	SourceLocation Loc);
1057
1058	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1059	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1060	virtual void emitProcBindClause(CodeGenFunction &CGF,
1061	llvm::omp::ProcBindKind ProcBind,
1062	SourceLocation Loc);
1063
1064	/// Returns address of the threadprivate variable for the current
1065	/// thread.
1066	/// \param VD Threadprivate variable.
1067	/// \param VDAddr Address of the global variable \a VD.
1068	/// \param Loc Location of the reference to threadprivate var.
1069	/// \return Address of the threadprivate variable for the current thread.
1070	virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF,
1071	const VarDecl *VD,
1072	Address VDAddr,
1073	SourceLocation Loc);
1074
1075	/// Returns the address of the variable marked as declare target with link
1076	/// clause OR as declare target with to clause and unified memory.
1077	virtual Address getAddrOfDeclareTargetVar(const VarDecl *VD);
1078
1079	/// Emit a code for initialization of threadprivate variable. It emits
1080	/// a call to runtime library which adds initial value to the newly created
1081	/// threadprivate variable (if it is not constant) and registers destructor
1082	/// for the variable (if any).
1083	/// \param VD Threadprivate variable.
1084	/// \param VDAddr Address of the global variable \a VD.
1085	/// \param Loc Location of threadprivate declaration.
1086	/// \param PerformInit true if initialization expression is not constant.
1087	virtual llvm::Function *
1088	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1089	SourceLocation Loc, bool PerformInit,
1090	CodeGenFunction *CGF = nullptr);
1091
1092	/// Emit code for handling declare target functions in the runtime.
1093	/// \param FD Declare target function.
1094	/// \param Addr Address of the global \a FD.
1095	/// \param PerformInit true if initialization expression is not constant.
1096	virtual void emitDeclareTargetFunction(const FunctionDecl *FD,
1097	llvm::GlobalValue *GV);
1098
1099	/// Creates artificial threadprivate variable with name \p Name and type \p
1100	/// VarType.
1101	/// \param VarType Type of the artificial threadprivate variable.
1102	/// \param Name Name of the artificial threadprivate variable.
1103	virtual Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1104	QualType VarType,
1105	StringRef Name);
1106
1107	/// Emit flush of the variables specified in 'omp flush' directive.
1108	/// \param Vars List of variables to flush.
1109	virtual void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1110	SourceLocation Loc, llvm::AtomicOrdering AO);
1111
1112	/// Emit task region for the task directive. The task region is
1113	/// emitted in several steps:
1114	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1115	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1116	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1117	/// function:
1118	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1119	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1120	/// return 0;
1121	/// }
1122	/// 2. Copy a list of shared variables to field shareds of the resulting
1123	/// structure kmp_task_t returned by the previous call (if any).
1124	/// 3. Copy a pointer to destructions function to field destructions of the
1125	/// resulting structure kmp_task_t.
1126	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1127	/// kmp_task_t *new_task), where new_task is a resulting structure from
1128	/// previous items.
1129	/// \param D Current task directive.
1130	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1131	/// /*part_id*/, captured_struct */*__context*/);
1132	/// \param SharedsTy A type which contains references the shared variables.
1133	/// \param Shareds Context with the list of shared variables from the \p
1134	/// TaskFunction.
1135	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1136	/// otherwise.
1137	/// \param Data Additional data for task generation like tiednsee, final
1138	/// state, list of privates etc.
1139	virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1140	const OMPExecutableDirective &D,
1141	llvm::Function *TaskFunction, QualType SharedsTy,
1142	Address Shareds, const Expr *IfCond,
1143	const OMPTaskDataTy &Data);
1144
1145	/// Emit task region for the taskloop directive. The taskloop region is
1146	/// emitted in several steps:
1147	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1148	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1149	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1150	/// function:
1151	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1152	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1153	/// return 0;
1154	/// }
1155	/// 2. Copy a list of shared variables to field shareds of the resulting
1156	/// structure kmp_task_t returned by the previous call (if any).
1157	/// 3. Copy a pointer to destructions function to field destructions of the
1158	/// resulting structure kmp_task_t.
1159	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
1160	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
1161	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
1162	/// is a resulting structure from
1163	/// previous items.
1164	/// \param D Current task directive.
1165	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1166	/// /*part_id*/, captured_struct */*__context*/);
1167	/// \param SharedsTy A type which contains references the shared variables.
1168	/// \param Shareds Context with the list of shared variables from the \p
1169	/// TaskFunction.
1170	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1171	/// otherwise.
1172	/// \param Data Additional data for task generation like tiednsee, final
1173	/// state, list of privates etc.
1174	virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
1175	const OMPLoopDirective &D,
1176	llvm::Function *TaskFunction,
1177	QualType SharedsTy, Address Shareds,
1178	const Expr *IfCond, const OMPTaskDataTy &Data);
1179
1180	/// Emit code for the directive that does not require outlining.
1181	///
1182	/// \param InnermostKind Kind of innermost directive (for simple directives it
1183	/// is a directive itself, for combined - its innermost directive).
1184	/// \param CodeGen Code generation sequence for the \a D directive.
1185	/// \param HasCancel true if region has inner cancel directive, false
1186	/// otherwise.
1187	virtual void emitInlinedDirective(CodeGenFunction &CGF,
1188	OpenMPDirectiveKind InnermostKind,
1189	const RegionCodeGenTy &CodeGen,
1190	bool HasCancel = false);
1191
1192	/// Emits reduction function.
1193	/// \param ReducerName Name of the function calling the reduction.
1194	/// \param ArgsElemType Array type containing pointers to reduction variables.
1195	/// \param Privates List of private copies for original reduction arguments.
1196	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1197	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1198	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1199	/// or 'operator binop(LHS, RHS)'.
1200	llvm::Function *emitReductionFunction(
1201	StringRef ReducerName, SourceLocation Loc, llvm::Type *ArgsElemType,
1202	ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs,
1203	ArrayRef<const Expr *> RHSExprs, ArrayRef<const Expr *> ReductionOps);
1204
1205	/// Emits single reduction combiner
1206	void emitSingleReductionCombiner(CodeGenFunction &CGF,
1207	const Expr *ReductionOp,
1208	const Expr *PrivateRef,
1209	const DeclRefExpr *LHS,
1210	const DeclRefExpr *RHS);
1211
1212	struct ReductionOptionsTy {
1213	bool WithNowait;
1214	bool SimpleReduction;
1215	OpenMPDirectiveKind ReductionKind;
1216	};
1217	/// Emit a code for reduction clause. Next code should be emitted for
1218	/// reduction:
1219	/// \code
1220	///
1221	/// static kmp_critical_name lock = { 0 };
1222	///
1223	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
1224	/// ...
1225	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
1226	/// ...
1227	/// }
1228	///
1229	/// ...
1230	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
1231	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
1232	/// RedList, reduce_func, &<lock>)) {
1233	/// case 1:
1234	/// ...
1235	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
1236	/// ...
1237	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
1238	/// break;
1239	/// case 2:
1240	/// ...
1241	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
1242	/// ...
1243	/// break;
1244	/// default:;
1245	/// }
1246	/// \endcode
1247	///
1248	/// \param Privates List of private copies for original reduction arguments.
1249	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
1250	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
1251	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
1252	/// or 'operator binop(LHS, RHS)'.
1253	/// \param Options List of options for reduction codegen:
1254	/// WithNowait true if parent directive has also nowait clause, false
1255	/// otherwise.
1256	/// SimpleReduction Emit reduction operation only. Used for omp simd
1257	/// directive on the host.
1258	/// ReductionKind The kind of reduction to perform.
1259	virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
1260	ArrayRef<const Expr *> Privates,
1261	ArrayRef<const Expr *> LHSExprs,
1262	ArrayRef<const Expr *> RHSExprs,
1263	ArrayRef<const Expr *> ReductionOps,
1264	ReductionOptionsTy Options);
1265
1266	/// Emit a code for initialization of task reduction clause. Next code
1267	/// should be emitted for reduction:
1268	/// \code
1269	///
1270	/// _taskred_item_t red_data[n];
1271	/// ...
1272	/// red_data[i].shar = &shareds[i];
1273	/// red_data[i].orig = &origs[i];
1274	/// red_data[i].size = sizeof(origs[i]);
1275	/// red_data[i].f_init = (void*)RedInit<i>;
1276	/// red_data[i].f_fini = (void*)RedDest<i>;
1277	/// red_data[i].f_comb = (void*)RedOp<i>;
1278	/// red_data[i].flags = <Flag_i>;
1279	/// ...
1280	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
1281	/// \endcode
1282	/// For reduction clause with task modifier it emits the next call:
1283	/// \code
1284	///
1285	/// _taskred_item_t red_data[n];
1286	/// ...
1287	/// red_data[i].shar = &shareds[i];
1288	/// red_data[i].orig = &origs[i];
1289	/// red_data[i].size = sizeof(origs[i]);
1290	/// red_data[i].f_init = (void*)RedInit<i>;
1291	/// red_data[i].f_fini = (void*)RedDest<i>;
1292	/// red_data[i].f_comb = (void*)RedOp<i>;
1293	/// red_data[i].flags = <Flag_i>;
1294	/// ...
1295	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
1296	/// red_data);
1297	/// \endcode
1298	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
1299	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
1300	/// \param Data Additional data for task generation like tiedness, final
1301	/// state, list of privates, reductions etc.
1302	virtual llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF,
1303	SourceLocation Loc,
1304	ArrayRef<const Expr *> LHSExprs,
1305	ArrayRef<const Expr *> RHSExprs,
1306	const OMPTaskDataTy &Data);
1307
1308	/// Emits the following code for reduction clause with task modifier:
1309	/// \code
1310	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
1311	/// \endcode
1312	virtual void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
1313	bool IsWorksharingReduction);
1314
1315	/// Required to resolve existing problems in the runtime. Emits threadprivate
1316	/// variables to store the size of the VLAs/array sections for
1317	/// initializer/combiner/finalizer functions.
1318	/// \param RCG Allows to reuse an existing data for the reductions.
1319	/// \param N Reduction item for which fixups must be emitted.
1320	virtual void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
1321	ReductionCodeGen &RCG, unsigned N);
1322
1323	/// Get the address of `void *` type of the privatue copy of the reduction
1324	/// item specified by the \p SharedLVal.
1325	/// \param ReductionsPtr Pointer to the reduction data returned by the
1326	/// emitTaskReductionInit function.
1327	/// \param SharedLVal Address of the original reduction item.
1328	virtual Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
1329	llvm::Value *ReductionsPtr,
1330	LValue SharedLVal);
1331
1332	/// Emit code for 'taskwait' directive.
1333	virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
1334	const OMPTaskDataTy &Data);
1335
1336	/// Emit code for 'cancellation point' construct.
1337	/// \param CancelRegion Region kind for which the cancellation point must be
1338	/// emitted.
1339	///
1340	virtual void emitCancellationPointCall(CodeGenFunction &CGF,
1341	SourceLocation Loc,
1342	OpenMPDirectiveKind CancelRegion);
1343
1344	/// Emit code for 'cancel' construct.
1345	/// \param IfCond Condition in the associated 'if' clause, if it was
1346	/// specified, nullptr otherwise.
1347	/// \param CancelRegion Region kind for which the cancel must be emitted.
1348	///
1349	virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
1350	const Expr *IfCond,
1351	OpenMPDirectiveKind CancelRegion);
1352
1353	/// Emit outilined function for 'target' directive.
1354	/// \param D Directive to emit.
1355	/// \param ParentName Name of the function that encloses the target region.
1356	/// \param OutlinedFn Outlined function value to be defined by this call.
1357	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
1358	/// \param IsOffloadEntry True if the outlined function is an offload entry.
1359	/// \param CodeGen Code generation sequence for the \a D directive.
1360	/// An outlined function may not be an entry if, e.g. the if clause always
1361	/// evaluates to false.
1362	virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
1363	StringRef ParentName,
1364	llvm::Function *&OutlinedFn,
1365	llvm::Constant *&OutlinedFnID,
1366	bool IsOffloadEntry,
1367	const RegionCodeGenTy &CodeGen);
1368
1369	/// Emit the target offloading code associated with \a D. The emitted
1370	/// code attempts offloading the execution to the device, an the event of
1371	/// a failure it executes the host version outlined in \a OutlinedFn.
1372	/// \param D Directive to emit.
1373	/// \param OutlinedFn Host version of the code to be offloaded.
1374	/// \param OutlinedFnID ID of host version of the code to be offloaded.
1375	/// \param IfCond Expression evaluated in if clause associated with the target
1376	/// directive, or null if no if clause is used.
1377	/// \param Device Expression evaluated in device clause associated with the
1378	/// target directive, or null if no device clause is used and device modifier.
1379	/// \param SizeEmitter Callback to emit number of iterations for loop-based
1380	/// directives.
1381	virtual void emitTargetCall(
1382	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1383	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
1384	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
1385	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
1386	const OMPLoopDirective &D)>
1387	SizeEmitter);
1388
1389	/// Emit the target regions enclosed in \a GD function definition or
1390	/// the function itself in case it is a valid device function. Returns true if
1391	/// \a GD was dealt with successfully.
1392	/// \param GD Function to scan.
1393	virtual bool emitTargetFunctions(GlobalDecl GD);
1394
1395	/// Emit the global variable if it is a valid device global variable.
1396	/// Returns true if \a GD was dealt with successfully.
1397	/// \param GD Variable declaration to emit.
1398	virtual bool emitTargetGlobalVariable(GlobalDecl GD);
1399
1400	/// Checks if the provided global decl \a GD is a declare target variable and
1401	/// registers it when emitting code for the host.
1402	virtual void registerTargetGlobalVariable(const VarDecl *VD,
1403	llvm::Constant *Addr);
1404
1405	/// Emit the global \a GD if it is meaningful for the target. Returns
1406	/// if it was emitted successfully.
1407	/// \param GD Global to scan.
1408	virtual bool emitTargetGlobal(GlobalDecl GD);
1409
1410	/// Creates and returns a registration function for when at least one
1411	/// requires directives was used in the current module.
1412	llvm::Function *emitRequiresDirectiveRegFun();
1413
1414	/// Creates all the offload entries in the current compilation unit
1415	/// along with the associated metadata.
1416	void createOffloadEntriesAndInfoMetadata();
1417
1418	/// Emits code for teams call of the \a OutlinedFn with
1419	/// variables captured in a record which address is stored in \a
1420	/// CapturedStruct.
1421	/// \param OutlinedFn Outlined function to be run by team masters. Type of
1422	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1423	/// \param CapturedVars A pointer to the record with the references to
1424	/// variables used in \a OutlinedFn function.
1425	///
1426	virtual void emitTeamsCall(CodeGenFunction &CGF,
1427	const OMPExecutableDirective &D,
1428	SourceLocation Loc, llvm::Function *OutlinedFn,
1429	ArrayRef<llvm::Value *> CapturedVars);
1430
1431	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
1432	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
1433	/// for num_teams clause.
1434	/// \param NumTeams An integer expression of teams.
1435	/// \param ThreadLimit An integer expression of threads.
1436	virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
1437	const Expr *ThreadLimit, SourceLocation Loc);
1438
1439	/// Emits call to void __kmpc_set_thread_limit(ident_t *loc, kmp_int32
1440	/// global_tid, kmp_int32 thread_limit) to generate code for
1441	/// thread_limit clause on target directive
1442	/// \param ThreadLimit An integer expression of threads.
1443	virtual void emitThreadLimitClause(CodeGenFunction &CGF,
1444	const Expr *ThreadLimit,
1445	SourceLocation Loc);
1446
1447	/// Struct that keeps all the relevant information that should be kept
1448	/// throughout a 'target data' region.
1449	class TargetDataInfo : public llvm::OpenMPIRBuilder::TargetDataInfo {
1450	public:
1451	explicit TargetDataInfo() : llvm::OpenMPIRBuilder::TargetDataInfo() {}
1452	explicit TargetDataInfo(bool RequiresDevicePointerInfo,
1453	bool SeparateBeginEndCalls)
1454	: llvm::OpenMPIRBuilder::TargetDataInfo(RequiresDevicePointerInfo,
1455	SeparateBeginEndCalls) {}
1456	/// Map between the a declaration of a capture and the corresponding new
1457	/// llvm address where the runtime returns the device pointers.
1458	llvm::DenseMap<const ValueDecl *, llvm::Value *> CaptureDeviceAddrMap;
1459	};
1460
1461	/// Emit the target data mapping code associated with \a D.
1462	/// \param D Directive to emit.
1463	/// \param IfCond Expression evaluated in if clause associated with the
1464	/// target directive, or null if no device clause is used.
1465	/// \param Device Expression evaluated in device clause associated with the
1466	/// target directive, or null if no device clause is used.
1467	/// \param Info A record used to store information that needs to be preserved
1468	/// until the region is closed.
1469	virtual void emitTargetDataCalls(CodeGenFunction &CGF,
1470	const OMPExecutableDirective &D,
1471	const Expr *IfCond, const Expr *Device,
1472	const RegionCodeGenTy &CodeGen,
1473	CGOpenMPRuntime::TargetDataInfo &Info);
1474
1475	/// Emit the data mapping/movement code associated with the directive
1476	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
1477	/// \param D Directive to emit.
1478	/// \param IfCond Expression evaluated in if clause associated with the target
1479	/// directive, or null if no if clause is used.
1480	/// \param Device Expression evaluated in device clause associated with the
1481	/// target directive, or null if no device clause is used.
1482	virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
1483	const OMPExecutableDirective &D,
1484	const Expr *IfCond,
1485	const Expr *Device);
1486
1487	/// Marks function \a Fn with properly mangled versions of vector functions.
1488	/// \param FD Function marked as 'declare simd'.
1489	/// \param Fn LLVM function that must be marked with 'declare simd'
1490	/// attributes.
1491	virtual void emitDeclareSimdFunction(const FunctionDecl *FD,
1492	llvm::Function *Fn);
1493
1494	/// Emit initialization for doacross loop nesting support.
1495	/// \param D Loop-based construct used in doacross nesting construct.
1496	virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
1497	ArrayRef<Expr *> NumIterations);
1498
1499	/// Emit code for doacross ordered directive with 'depend' clause.
1500	/// \param C 'depend' clause with 'sink|source' dependency kind.
1501	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1502	const OMPDependClause *C);
1503
1504	/// Emit code for doacross ordered directive with 'doacross' clause.
1505	/// \param C 'doacross' clause with 'sink|source' dependence type.
1506	virtual void emitDoacrossOrdered(CodeGenFunction &CGF,
1507	const OMPDoacrossClause *C);
1508
1509	/// Translates the native parameter of outlined function if this is required
1510	/// for target.
1511	/// \param FD Field decl from captured record for the parameter.
1512	/// \param NativeParam Parameter itself.
1513	virtual const VarDecl *translateParameter(const FieldDecl *FD,
1514	const VarDecl *NativeParam) const {
1515	return NativeParam;
1516	}
1517
1518	/// Gets the address of the native argument basing on the address of the
1519	/// target-specific parameter.
1520	/// \param NativeParam Parameter itself.
1521	/// \param TargetParam Corresponding target-specific parameter.
1522	virtual Address getParameterAddress(CodeGenFunction &CGF,
1523	const VarDecl *NativeParam,
1524	const VarDecl *TargetParam) const;
1525
1526	/// Choose default schedule type and chunk value for the
1527	/// dist_schedule clause.
1528	virtual void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF,
1529	const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind,
1530	llvm::Value *&Chunk) const {}
1531
1532	/// Choose default schedule type and chunk value for the
1533	/// schedule clause.
1534	virtual void getDefaultScheduleAndChunk(CodeGenFunction &CGF,
1535	const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind,
1536	const Expr *&ChunkExpr) const;
1537
1538	/// Emits call of the outlined function with the provided arguments,
1539	/// translating these arguments to correct target-specific arguments.
1540	virtual void
1541	emitOutlinedFunctionCall(CodeGenFunction &CGF, SourceLocation Loc,
1542	llvm::FunctionCallee OutlinedFn,
1543	ArrayRef<llvm::Value *> Args = std::nullopt) const;
1544
1545	/// Emits OpenMP-specific function prolog.
1546	/// Required for device constructs.
1547	virtual void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D);
1548
1549	/// Gets the OpenMP-specific address of the local variable.
1550	virtual Address getAddressOfLocalVariable(CodeGenFunction &CGF,
1551	const VarDecl *VD);
1552
1553	/// Marks the declaration as already emitted for the device code and returns
1554	/// true, if it was marked already, and false, otherwise.
1555	bool markAsGlobalTarget(GlobalDecl GD);
1556
1557	/// Emit deferred declare target variables marked for deferred emission.
1558	void emitDeferredTargetDecls() const;
1559
1560	/// Adjust some parameters for the target-based directives, like addresses of
1561	/// the variables captured by reference in lambdas.
1562	virtual void
1563	adjustTargetSpecificDataForLambdas(CodeGenFunction &CGF,
1564	const OMPExecutableDirective &D) const;
1565
1566	/// Perform check on requires decl to ensure that target architecture
1567	/// supports unified addressing
1568	virtual void processRequiresDirective(const OMPRequiresDecl *D);
1569
1570	/// Gets default memory ordering as specified in requires directive.
1571	llvm::AtomicOrdering getDefaultMemoryOrdering() const;
1572
1573	/// Checks if the variable has associated OMPAllocateDeclAttr attribute with
1574	/// the predefined allocator and translates it into the corresponding address
1575	/// space.
1576	virtual bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS);
1577
1578	/// Return whether the unified_shared_memory has been specified.
1579	bool hasRequiresUnifiedSharedMemory() const;
1580
1581	/// Checks if the \p VD variable is marked as nontemporal declaration in
1582	/// current context.
1583	bool isNontemporalDecl(const ValueDecl *VD) const;
1584
1585	/// Create specialized alloca to handle lastprivate conditionals.
1586	Address emitLastprivateConditionalInit(CodeGenFunction &CGF,
1587	const VarDecl *VD);
1588
1589	/// Checks if the provided \p LVal is lastprivate conditional and emits the
1590	/// code to update the value of the original variable.
1591	/// \code
1592	/// lastprivate(conditional: a)
1593	/// ...
1594	/// <type> a;
1595	/// lp_a = ...;
1596	/// #pragma omp critical(a)
1597	/// if (last_iv_a <= iv) {
1598	/// last_iv_a = iv;
1599	/// global_a = lp_a;
1600	/// }
1601	/// \endcode
1602	virtual void checkAndEmitLastprivateConditional(CodeGenFunction &CGF,
1603	const Expr *LHS);
1604
1605	/// Checks if the lastprivate conditional was updated in inner region and
1606	/// writes the value.
1607	/// \code
1608	/// lastprivate(conditional: a)
1609	/// ...
1610	/// <type> a;bool Fired = false;
1611	/// #pragma omp ... shared(a)
1612	/// {
1613	/// lp_a = ...;
1614	/// Fired = true;
1615	/// }
1616	/// if (Fired) {
1617	/// #pragma omp critical(a)
1618	/// if (last_iv_a <= iv) {
1619	/// last_iv_a = iv;
1620	/// global_a = lp_a;
1621	/// }
1622	/// Fired = false;
1623	/// }
1624	/// \endcode
1625	virtual void checkAndEmitSharedLastprivateConditional(
1626	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1627	const llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> &IgnoredDecls);
1628
1629	/// Gets the address of the global copy used for lastprivate conditional
1630	/// update, if any.
1631	/// \param PrivLVal LValue for the private copy.
1632	/// \param VD Original lastprivate declaration.
1633	virtual void emitLastprivateConditionalFinalUpdate(CodeGenFunction &CGF,
1634	LValue PrivLVal,
1635	const VarDecl *VD,
1636	SourceLocation Loc);
1637
1638	/// Emits list of dependecies based on the provided data (array of
1639	/// dependence/expression pairs).
1640	/// \returns Pointer to the first element of the array casted to VoidPtr type.
1641	std::pair<llvm::Value *, Address>
1642	emitDependClause(CodeGenFunction &CGF,
1643	ArrayRef<OMPTaskDataTy::DependData> Dependencies,
1644	SourceLocation Loc);
1645
1646	/// Emits list of dependecies based on the provided data (array of
1647	/// dependence/expression pairs) for depobj construct. In this case, the
1648	/// variable is allocated in dynamically. \returns Pointer to the first
1649	/// element of the array casted to VoidPtr type.
1650	Address emitDepobjDependClause(CodeGenFunction &CGF,
1651	const OMPTaskDataTy::DependData &Dependencies,
1652	SourceLocation Loc);
1653
1654	/// Emits the code to destroy the dependency object provided in depobj
1655	/// directive.
1656	void emitDestroyClause(CodeGenFunction &CGF, LValue DepobjLVal,
1657	SourceLocation Loc);
1658
1659	/// Updates the dependency kind in the specified depobj object.
1660	/// \param DepobjLVal LValue for the main depobj object.
1661	/// \param NewDepKind New dependency kind.
1662	void emitUpdateClause(CodeGenFunction &CGF, LValue DepobjLVal,
1663	OpenMPDependClauseKind NewDepKind, SourceLocation Loc);
1664
1665	/// Initializes user defined allocators specified in the uses_allocators
1666	/// clauses.
1667	void emitUsesAllocatorsInit(CodeGenFunction &CGF, const Expr *Allocator,
1668	const Expr *AllocatorTraits);
1669
1670	/// Destroys user defined allocators specified in the uses_allocators clause.
1671	void emitUsesAllocatorsFini(CodeGenFunction &CGF, const Expr *Allocator);
1672
1673	/// Returns true if the variable is a local variable in untied task.
1674	bool isLocalVarInUntiedTask(CodeGenFunction &CGF, const VarDecl *VD) const;
1675	};
1676
1677	/// Class supports emissionof SIMD-only code.
1678	class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
1679	public:
1680	explicit CGOpenMPSIMDRuntime(CodeGenModule &CGM) : CGOpenMPRuntime(CGM) {}
1681	~CGOpenMPSIMDRuntime() override {}
1682
1683	/// Emits outlined function for the specified OpenMP parallel directive
1684	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1685	/// kmp_int32 BoundID, struct context_vars*).
1686	/// \param CGF Reference to current CodeGenFunction.
1687	/// \param D OpenMP directive.
1688	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1689	/// \param InnermostKind Kind of innermost directive (for simple directives it
1690	/// is a directive itself, for combined - its innermost directive).
1691	/// \param CodeGen Code generation sequence for the \a D directive.
1692	llvm::Function *emitParallelOutlinedFunction(
1693	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1694	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1695	const RegionCodeGenTy &CodeGen) override;
1696
1697	/// Emits outlined function for the specified OpenMP teams directive
1698	/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
1699	/// kmp_int32 BoundID, struct context_vars*).
1700	/// \param CGF Reference to current CodeGenFunction.
1701	/// \param D OpenMP directive.
1702	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1703	/// \param InnermostKind Kind of innermost directive (for simple directives it
1704	/// is a directive itself, for combined - its innermost directive).
1705	/// \param CodeGen Code generation sequence for the \a D directive.
1706	llvm::Function *emitTeamsOutlinedFunction(
1707	CodeGenFunction &CGF, const OMPExecutableDirective &D,
1708	const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind,
1709	const RegionCodeGenTy &CodeGen) override;
1710
1711	/// Emits outlined function for the OpenMP task directive \a D. This
1712	/// outlined function has type void(*)(kmp_int32 ThreadID, struct task_t*
1713	/// TaskT).
1714	/// \param D OpenMP directive.
1715	/// \param ThreadIDVar Variable for thread id in the current OpenMP region.
1716	/// \param PartIDVar Variable for partition id in the current OpenMP untied
1717	/// task region.
1718	/// \param TaskTVar Variable for task_t argument.
1719	/// \param InnermostKind Kind of innermost directive (for simple directives it
1720	/// is a directive itself, for combined - its innermost directive).
1721	/// \param CodeGen Code generation sequence for the \a D directive.
1722	/// \param Tied true if task is generated for tied task, false otherwise.
1723	/// \param NumberOfParts Number of parts in untied task. Ignored for tied
1724	/// tasks.
1725	///
1726	llvm::Function *emitTaskOutlinedFunction(
1727	const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1728	const VarDecl *PartIDVar, const VarDecl *TaskTVar,
1729	OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
1730	bool Tied, unsigned &NumberOfParts) override;
1731
1732	/// Emits code for parallel or serial call of the \a OutlinedFn with
1733	/// variables captured in a record which address is stored in \a
1734	/// CapturedStruct.
1735	/// \param OutlinedFn Outlined function to be run in parallel threads. Type of
1736	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
1737	/// \param CapturedVars A pointer to the record with the references to
1738	/// variables used in \a OutlinedFn function.
1739	/// \param IfCond Condition in the associated 'if' clause, if it was
1740	/// specified, nullptr otherwise.
1741	/// \param NumThreads The value corresponding to the num_threads clause, if
1742	/// any, or nullptr.
1743	///
1744	void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
1745	llvm::Function *OutlinedFn,
1746	ArrayRef<llvm::Value *> CapturedVars,
1747	const Expr *IfCond, llvm::Value *NumThreads) override;
1748
1749	/// Emits a critical region.
1750	/// \param CriticalName Name of the critical region.
1751	/// \param CriticalOpGen Generator for the statement associated with the given
1752	/// critical region.
1753	/// \param Hint Value of the 'hint' clause (optional).
1754	void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName,
1755	const RegionCodeGenTy &CriticalOpGen,
1756	SourceLocation Loc,
1757	const Expr *Hint = nullptr) override;
1758
1759	/// Emits a master region.
1760	/// \param MasterOpGen Generator for the statement associated with the given
1761	/// master region.
1762	void emitMasterRegion(CodeGenFunction &CGF,
1763	const RegionCodeGenTy &MasterOpGen,
1764	SourceLocation Loc) override;
1765
1766	/// Emits a masked region.
1767	/// \param MaskedOpGen Generator for the statement associated with the given
1768	/// masked region.
1769	void emitMaskedRegion(CodeGenFunction &CGF,
1770	const RegionCodeGenTy &MaskedOpGen, SourceLocation Loc,
1771	const Expr *Filter = nullptr) override;
1772
1773	/// Emits a masked region.
1774	/// \param MaskedOpGen Generator for the statement associated with the given
1775	/// masked region.
1776
1777	/// Emits code for a taskyield directive.
1778	void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc) override;
1779
1780	/// Emit a taskgroup region.
1781	/// \param TaskgroupOpGen Generator for the statement associated with the
1782	/// given taskgroup region.
1783	void emitTaskgroupRegion(CodeGenFunction &CGF,
1784	const RegionCodeGenTy &TaskgroupOpGen,
1785	SourceLocation Loc) override;
1786
1787	/// Emits a single region.
1788	/// \param SingleOpGen Generator for the statement associated with the given
1789	/// single region.
1790	void emitSingleRegion(CodeGenFunction &CGF,
1791	const RegionCodeGenTy &SingleOpGen, SourceLocation Loc,
1792	ArrayRef<const Expr *> CopyprivateVars,
1793	ArrayRef<const Expr *> DestExprs,
1794	ArrayRef<const Expr *> SrcExprs,
1795	ArrayRef<const Expr *> AssignmentOps) override;
1796
1797	/// Emit an ordered region.
1798	/// \param OrderedOpGen Generator for the statement associated with the given
1799	/// ordered region.
1800	void emitOrderedRegion(CodeGenFunction &CGF,
1801	const RegionCodeGenTy &OrderedOpGen,
1802	SourceLocation Loc, bool IsThreads) override;
1803
1804	/// Emit an implicit/explicit barrier for OpenMP threads.
1805	/// \param Kind Directive for which this implicit barrier call must be
1806	/// generated. Must be OMPD_barrier for explicit barrier generation.
1807	/// \param EmitChecks true if need to emit checks for cancellation barriers.
1808	/// \param ForceSimpleCall true simple barrier call must be emitted, false if
1809	/// runtime class decides which one to emit (simple or with cancellation
1810	/// checks).
1811	///
1812	void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
1813	OpenMPDirectiveKind Kind, bool EmitChecks = true,
1814	bool ForceSimpleCall = false) override;
1815
1816	/// This is used for non static scheduled types and when the ordered
1817	/// clause is present on the loop construct.
1818	/// Depending on the loop schedule, it is necessary to call some runtime
1819	/// routine before start of the OpenMP loop to get the loop upper / lower
1820	/// bounds \a LB and \a UB and stride \a ST.
1821	///
1822	/// \param CGF Reference to current CodeGenFunction.
1823	/// \param Loc Clang source location.
1824	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1825	/// \param IVSize Size of the iteration variable in bits.
1826	/// \param IVSigned Sign of the iteration variable.
1827	/// \param Ordered true if loop is ordered, false otherwise.
1828	/// \param DispatchValues struct containing llvm values for lower bound, upper
1829	/// bound, and chunk expression.
1830	/// For the default (nullptr) value, the chunk 1 will be used.
1831	///
1832	void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc,
1833	const OpenMPScheduleTy &ScheduleKind,
1834	unsigned IVSize, bool IVSigned, bool Ordered,
1835	const DispatchRTInput &DispatchValues) override;
1836
1837	/// Call the appropriate runtime routine to initialize it before start
1838	/// of loop.
1839	///
1840	/// This is used only in case of static schedule, when the user did not
1841	/// specify a ordered clause on the loop construct.
1842	/// Depending on the loop schedule, it is necessary to call some runtime
1843	/// routine before start of the OpenMP loop to get the loop upper / lower
1844	/// bounds LB and UB and stride ST.
1845	///
1846	/// \param CGF Reference to current CodeGenFunction.
1847	/// \param Loc Clang source location.
1848	/// \param DKind Kind of the directive.
1849	/// \param ScheduleKind Schedule kind, specified by the 'schedule' clause.
1850	/// \param Values Input arguments for the construct.
1851	///
1852	void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1853	OpenMPDirectiveKind DKind,
1854	const OpenMPScheduleTy &ScheduleKind,
1855	const StaticRTInput &Values) override;
1856
1857	///
1858	/// \param CGF Reference to current CodeGenFunction.
1859	/// \param Loc Clang source location.
1860	/// \param SchedKind Schedule kind, specified by the 'dist_schedule' clause.
1861	/// \param Values Input arguments for the construct.
1862	///
1863	void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc,
1864	OpenMPDistScheduleClauseKind SchedKind,
1865	const StaticRTInput &Values) override;
1866
1867	/// Call the appropriate runtime routine to notify that we finished
1868	/// iteration of the ordered loop with the dynamic scheduling.
1869	///
1870	/// \param CGF Reference to current CodeGenFunction.
1871	/// \param Loc Clang source location.
1872	/// \param IVSize Size of the iteration variable in bits.
1873	/// \param IVSigned Sign of the iteration variable.
1874	///
1875	void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc,
1876	unsigned IVSize, bool IVSigned) override;
1877
1878	/// Call the appropriate runtime routine to notify that we finished
1879	/// all the work with current loop.
1880	///
1881	/// \param CGF Reference to current CodeGenFunction.
1882	/// \param Loc Clang source location.
1883	/// \param DKind Kind of the directive for which the static finish is emitted.
1884	///
1885	void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc,
1886	OpenMPDirectiveKind DKind) override;
1887
1888	/// Call __kmpc_dispatch_next(
1889	/// ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter,
1890	/// kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper,
1891	/// kmp_int[32|64] *p_stride);
1892	/// \param IVSize Size of the iteration variable in bits.
1893	/// \param IVSigned Sign of the iteration variable.
1894	/// \param IL Address of the output variable in which the flag of the
1895	/// last iteration is returned.
1896	/// \param LB Address of the output variable in which the lower iteration
1897	/// number is returned.
1898	/// \param UB Address of the output variable in which the upper iteration
1899	/// number is returned.
1900	/// \param ST Address of the output variable in which the stride value is
1901	/// returned.
1902	llvm::Value *emitForNext(CodeGenFunction &CGF, SourceLocation Loc,
1903	unsigned IVSize, bool IVSigned, Address IL,
1904	Address LB, Address UB, Address ST) override;
1905
1906	/// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32
1907	/// global_tid, kmp_int32 num_threads) to generate code for 'num_threads'
1908	/// clause.
1909	/// \param NumThreads An integer value of threads.
1910	void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads,
1911	SourceLocation Loc) override;
1912
1913	/// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32
1914	/// global_tid, int proc_bind) to generate code for 'proc_bind' clause.
1915	void emitProcBindClause(CodeGenFunction &CGF,
1916	llvm::omp::ProcBindKind ProcBind,
1917	SourceLocation Loc) override;
1918
1919	/// Returns address of the threadprivate variable for the current
1920	/// thread.
1921	/// \param VD Threadprivate variable.
1922	/// \param VDAddr Address of the global variable \a VD.
1923	/// \param Loc Location of the reference to threadprivate var.
1924	/// \return Address of the threadprivate variable for the current thread.
1925	Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD,
1926	Address VDAddr, SourceLocation Loc) override;
1927
1928	/// Emit a code for initialization of threadprivate variable. It emits
1929	/// a call to runtime library which adds initial value to the newly created
1930	/// threadprivate variable (if it is not constant) and registers destructor
1931	/// for the variable (if any).
1932	/// \param VD Threadprivate variable.
1933	/// \param VDAddr Address of the global variable \a VD.
1934	/// \param Loc Location of threadprivate declaration.
1935	/// \param PerformInit true if initialization expression is not constant.
1936	llvm::Function *
1937	emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr,
1938	SourceLocation Loc, bool PerformInit,
1939	CodeGenFunction *CGF = nullptr) override;
1940
1941	/// Creates artificial threadprivate variable with name \p Name and type \p
1942	/// VarType.
1943	/// \param VarType Type of the artificial threadprivate variable.
1944	/// \param Name Name of the artificial threadprivate variable.
1945	Address getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF,
1946	QualType VarType,
1947	StringRef Name) override;
1948
1949	/// Emit flush of the variables specified in 'omp flush' directive.
1950	/// \param Vars List of variables to flush.
1951	void emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *> Vars,
1952	SourceLocation Loc, llvm::AtomicOrdering AO) override;
1953
1954	/// Emit task region for the task directive. The task region is
1955	/// emitted in several steps:
1956	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1957	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1958	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1959	/// function:
1960	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1961	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1962	/// return 0;
1963	/// }
1964	/// 2. Copy a list of shared variables to field shareds of the resulting
1965	/// structure kmp_task_t returned by the previous call (if any).
1966	/// 3. Copy a pointer to destructions function to field destructions of the
1967	/// resulting structure kmp_task_t.
1968	/// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
1969	/// kmp_task_t *new_task), where new_task is a resulting structure from
1970	/// previous items.
1971	/// \param D Current task directive.
1972	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
1973	/// /*part_id*/, captured_struct */*__context*/);
1974	/// \param SharedsTy A type which contains references the shared variables.
1975	/// \param Shareds Context with the list of shared variables from the \p
1976	/// TaskFunction.
1977	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
1978	/// otherwise.
1979	/// \param Data Additional data for task generation like tiednsee, final
1980	/// state, list of privates etc.
1981	void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
1982	const OMPExecutableDirective &D,
1983	llvm::Function *TaskFunction, QualType SharedsTy,
1984	Address Shareds, const Expr *IfCond,
1985	const OMPTaskDataTy &Data) override;
1986
1987	/// Emit task region for the taskloop directive. The taskloop region is
1988	/// emitted in several steps:
1989	/// 1. Emit a call to kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32
1990	/// gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1991	/// kmp_routine_entry_t *task_entry). Here task_entry is a pointer to the
1992	/// function:
1993	/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
1994	/// TaskFunction(gtid, tt->part_id, tt->shareds);
1995	/// return 0;
1996	/// }
1997	/// 2. Copy a list of shared variables to field shareds of the resulting
1998	/// structure kmp_task_t returned by the previous call (if any).
1999	/// 3. Copy a pointer to destructions function to field destructions of the
2000	/// resulting structure kmp_task_t.
2001	/// 4. Emit a call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t
2002	/// *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int
2003	/// nogroup, int sched, kmp_uint64 grainsize, void *task_dup ), where new_task
2004	/// is a resulting structure from
2005	/// previous items.
2006	/// \param D Current task directive.
2007	/// \param TaskFunction An LLVM function with type void (*)(i32 /*gtid*/, i32
2008	/// /*part_id*/, captured_struct */*__context*/);
2009	/// \param SharedsTy A type which contains references the shared variables.
2010	/// \param Shareds Context with the list of shared variables from the \p
2011	/// TaskFunction.
2012	/// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
2013	/// otherwise.
2014	/// \param Data Additional data for task generation like tiednsee, final
2015	/// state, list of privates etc.
2016	void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
2017	const OMPLoopDirective &D, llvm::Function *TaskFunction,
2018	QualType SharedsTy, Address Shareds, const Expr *IfCond,
2019	const OMPTaskDataTy &Data) override;
2020
2021	/// Emit a code for reduction clause. Next code should be emitted for
2022	/// reduction:
2023	/// \code
2024	///
2025	/// static kmp_critical_name lock = { 0 };
2026	///
2027	/// void reduce_func(void *lhs[<n>], void *rhs[<n>]) {
2028	/// ...
2029	/// *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]);
2030	/// ...
2031	/// }
2032	///
2033	/// ...
2034	/// void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]};
2035	/// switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList),
2036	/// RedList, reduce_func, &<lock>)) {
2037	/// case 1:
2038	/// ...
2039	/// <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]);
2040	/// ...
2041	/// __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>);
2042	/// break;
2043	/// case 2:
2044	/// ...
2045	/// Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]));
2046	/// ...
2047	/// break;
2048	/// default:;
2049	/// }
2050	/// \endcode
2051	///
2052	/// \param Privates List of private copies for original reduction arguments.
2053	/// \param LHSExprs List of LHS in \a ReductionOps reduction operations.
2054	/// \param RHSExprs List of RHS in \a ReductionOps reduction operations.
2055	/// \param ReductionOps List of reduction operations in form 'LHS binop RHS'
2056	/// or 'operator binop(LHS, RHS)'.
2057	/// \param Options List of options for reduction codegen:
2058	/// WithNowait true if parent directive has also nowait clause, false
2059	/// otherwise.
2060	/// SimpleReduction Emit reduction operation only. Used for omp simd
2061	/// directive on the host.
2062	/// ReductionKind The kind of reduction to perform.
2063	void emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
2064	ArrayRef<const Expr *> Privates,
2065	ArrayRef<const Expr *> LHSExprs,
2066	ArrayRef<const Expr *> RHSExprs,
2067	ArrayRef<const Expr *> ReductionOps,
2068	ReductionOptionsTy Options) override;
2069
2070	/// Emit a code for initialization of task reduction clause. Next code
2071	/// should be emitted for reduction:
2072	/// \code
2073	///
2074	/// _taskred_item_t red_data[n];
2075	/// ...
2076	/// red_data[i].shar = &shareds[i];
2077	/// red_data[i].orig = &origs[i];
2078	/// red_data[i].size = sizeof(origs[i]);
2079	/// red_data[i].f_init = (void*)RedInit<i>;
2080	/// red_data[i].f_fini = (void*)RedDest<i>;
2081	/// red_data[i].f_comb = (void*)RedOp<i>;
2082	/// red_data[i].flags = <Flag_i>;
2083	/// ...
2084	/// void* tg1 = __kmpc_taskred_init(gtid, n, red_data);
2085	/// \endcode
2086	/// For reduction clause with task modifier it emits the next call:
2087	/// \code
2088	///
2089	/// _taskred_item_t red_data[n];
2090	/// ...
2091	/// red_data[i].shar = &shareds[i];
2092	/// red_data[i].orig = &origs[i];
2093	/// red_data[i].size = sizeof(origs[i]);
2094	/// red_data[i].f_init = (void*)RedInit<i>;
2095	/// red_data[i].f_fini = (void*)RedDest<i>;
2096	/// red_data[i].f_comb = (void*)RedOp<i>;
2097	/// red_data[i].flags = <Flag_i>;
2098	/// ...
2099	/// void* tg1 = __kmpc_taskred_modifier_init(loc, gtid, is_worksharing, n,
2100	/// red_data);
2101	/// \endcode
2102	/// \param LHSExprs List of LHS in \a Data.ReductionOps reduction operations.
2103	/// \param RHSExprs List of RHS in \a Data.ReductionOps reduction operations.
2104	/// \param Data Additional data for task generation like tiedness, final
2105	/// state, list of privates, reductions etc.
2106	llvm::Value *emitTaskReductionInit(CodeGenFunction &CGF, SourceLocation Loc,
2107	ArrayRef<const Expr *> LHSExprs,
2108	ArrayRef<const Expr *> RHSExprs,
2109	const OMPTaskDataTy &Data) override;
2110
2111	/// Emits the following code for reduction clause with task modifier:
2112	/// \code
2113	/// __kmpc_task_reduction_modifier_fini(loc, gtid, is_worksharing);
2114	/// \endcode
2115	void emitTaskReductionFini(CodeGenFunction &CGF, SourceLocation Loc,
2116	bool IsWorksharingReduction) override;
2117
2118	/// Required to resolve existing problems in the runtime. Emits threadprivate
2119	/// variables to store the size of the VLAs/array sections for
2120	/// initializer/combiner/finalizer functions + emits threadprivate variable to
2121	/// store the pointer to the original reduction item for the custom
2122	/// initializer defined by declare reduction construct.
2123	/// \param RCG Allows to reuse an existing data for the reductions.
2124	/// \param N Reduction item for which fixups must be emitted.
2125	void emitTaskReductionFixups(CodeGenFunction &CGF, SourceLocation Loc,
2126	ReductionCodeGen &RCG, unsigned N) override;
2127
2128	/// Get the address of `void *` type of the privatue copy of the reduction
2129	/// item specified by the \p SharedLVal.
2130	/// \param ReductionsPtr Pointer to the reduction data returned by the
2131	/// emitTaskReductionInit function.
2132	/// \param SharedLVal Address of the original reduction item.
2133	Address getTaskReductionItem(CodeGenFunction &CGF, SourceLocation Loc,
2134	llvm::Value *ReductionsPtr,
2135	LValue SharedLVal) override;
2136
2137	/// Emit code for 'taskwait' directive.
2138	void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
2139	const OMPTaskDataTy &Data) override;
2140
2141	/// Emit code for 'cancellation point' construct.
2142	/// \param CancelRegion Region kind for which the cancellation point must be
2143	/// emitted.
2144	///
2145	void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc,
2146	OpenMPDirectiveKind CancelRegion) override;
2147
2148	/// Emit code for 'cancel' construct.
2149	/// \param IfCond Condition in the associated 'if' clause, if it was
2150	/// specified, nullptr otherwise.
2151	/// \param CancelRegion Region kind for which the cancel must be emitted.
2152	///
2153	void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc,
2154	const Expr *IfCond,
2155	OpenMPDirectiveKind CancelRegion) override;
2156
2157	/// Emit outilined function for 'target' directive.
2158	/// \param D Directive to emit.
2159	/// \param ParentName Name of the function that encloses the target region.
2160	/// \param OutlinedFn Outlined function value to be defined by this call.
2161	/// \param OutlinedFnID Outlined function ID value to be defined by this call.
2162	/// \param IsOffloadEntry True if the outlined function is an offload entry.
2163	/// \param CodeGen Code generation sequence for the \a D directive.
2164	/// An outlined function may not be an entry if, e.g. the if clause always
2165	/// evaluates to false.
2166	void emitTargetOutlinedFunction(const OMPExecutableDirective &D,
2167	StringRef ParentName,
2168	llvm::Function *&OutlinedFn,
2169	llvm::Constant *&OutlinedFnID,
2170	bool IsOffloadEntry,
2171	const RegionCodeGenTy &CodeGen) override;
2172
2173	/// Emit the target offloading code associated with \a D. The emitted
2174	/// code attempts offloading the execution to the device, an the event of
2175	/// a failure it executes the host version outlined in \a OutlinedFn.
2176	/// \param D Directive to emit.
2177	/// \param OutlinedFn Host version of the code to be offloaded.
2178	/// \param OutlinedFnID ID of host version of the code to be offloaded.
2179	/// \param IfCond Expression evaluated in if clause associated with the target
2180	/// directive, or null if no if clause is used.
2181	/// \param Device Expression evaluated in device clause associated with the
2182	/// target directive, or null if no device clause is used and device modifier.
2183	void emitTargetCall(
2184	CodeGenFunction &CGF, const OMPExecutableDirective &D,
2185	llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
2186	llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
2187	llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
2188	const OMPLoopDirective &D)>
2189	SizeEmitter) override;
2190
2191	/// Emit the target regions enclosed in \a GD function definition or
2192	/// the function itself in case it is a valid device function. Returns true if
2193	/// \a GD was dealt with successfully.
2194	/// \param GD Function to scan.
2195	bool emitTargetFunctions(GlobalDecl GD) override;
2196
2197	/// Emit the global variable if it is a valid device global variable.
2198	/// Returns true if \a GD was dealt with successfully.
2199	/// \param GD Variable declaration to emit.
2200	bool emitTargetGlobalVariable(GlobalDecl GD) override;
2201
2202	/// Emit the global \a GD if it is meaningful for the target. Returns
2203	/// if it was emitted successfully.
2204	/// \param GD Global to scan.
2205	bool emitTargetGlobal(GlobalDecl GD) override;
2206
2207	/// Emits code for teams call of the \a OutlinedFn with
2208	/// variables captured in a record which address is stored in \a
2209	/// CapturedStruct.
2210	/// \param OutlinedFn Outlined function to be run by team masters. Type of
2211	/// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*).
2212	/// \param CapturedVars A pointer to the record with the references to
2213	/// variables used in \a OutlinedFn function.
2214	///
2215	void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
2216	SourceLocation Loc, llvm::Function *OutlinedFn,
2217	ArrayRef<llvm::Value *> CapturedVars) override;
2218
2219	/// Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32
2220	/// global_tid, kmp_int32 num_teams, kmp_int32 thread_limit) to generate code
2221	/// for num_teams clause.
2222	/// \param NumTeams An integer expression of teams.
2223	/// \param ThreadLimit An integer expression of threads.
2224	void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams,
2225	const Expr *ThreadLimit, SourceLocation Loc) override;
2226
2227	/// Emit the target data mapping code associated with \a D.
2228	/// \param D Directive to emit.
2229	/// \param IfCond Expression evaluated in if clause associated with the
2230	/// target directive, or null if no device clause is used.
2231	/// \param Device Expression evaluated in device clause associated with the
2232	/// target directive, or null if no device clause is used.
2233	/// \param Info A record used to store information that needs to be preserved
2234	/// until the region is closed.
2235	void emitTargetDataCalls(CodeGenFunction &CGF,
2236	const OMPExecutableDirective &D, const Expr *IfCond,
2237	const Expr *Device, const RegionCodeGenTy &CodeGen,
2238	CGOpenMPRuntime::TargetDataInfo &Info) override;
2239
2240	/// Emit the data mapping/movement code associated with the directive
2241	/// \a D that should be of the form 'target [{enter|exit} data | update]'.
2242	/// \param D Directive to emit.
2243	/// \param IfCond Expression evaluated in if clause associated with the target
2244	/// directive, or null if no if clause is used.
2245	/// \param Device Expression evaluated in device clause associated with the
2246	/// target directive, or null if no device clause is used.
2247	void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
2248	const OMPExecutableDirective &D,
2249	const Expr *IfCond,
2250	const Expr *Device) override;
2251
2252	/// Emit initialization for doacross loop nesting support.
2253	/// \param D Loop-based construct used in doacross nesting construct.
2254	void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D,
2255	ArrayRef<Expr *> NumIterations) override;
2256
2257	/// Emit code for doacross ordered directive with 'depend' clause.
2258	/// \param C 'depend' clause with 'sink|source' dependency kind.
2259	void emitDoacrossOrdered(CodeGenFunction &CGF,
2260	const OMPDependClause *C) override;
2261
2262	/// Emit code for doacross ordered directive with 'doacross' clause.
2263	/// \param C 'doacross' clause with 'sink|source' dependence type.
2264	void emitDoacrossOrdered(CodeGenFunction &CGF,
2265	const OMPDoacrossClause *C) override;
2266
2267	/// Translates the native parameter of outlined function if this is required
2268	/// for target.
2269	/// \param FD Field decl from captured record for the parameter.
2270	/// \param NativeParam Parameter itself.
2271	const VarDecl *translateParameter(const FieldDecl *FD,
2272	const VarDecl *NativeParam) const override;
2273
2274	/// Gets the address of the native argument basing on the address of the
2275	/// target-specific parameter.
2276	/// \param NativeParam Parameter itself.
2277	/// \param TargetParam Corresponding target-specific parameter.
2278	Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam,
2279	const VarDecl *TargetParam) const override;
2280
2281	/// Gets the OpenMP-specific address of the local variable.
2282	Address getAddressOfLocalVariable(CodeGenFunction &CGF,
2283	const VarDecl *VD) override {
2284	return Address::invalid();
2285	}
2286	};
2287
2288	} // namespace CodeGen
2289	// Utility for openmp doacross clause kind
2290	namespace {
2291	template <typename T> class OMPDoacrossKind {
2292	public:
2293	bool isSink(const T *) { return false; }
2294	bool isSource(const T *) { return false; }
2295	};
2296	template <> class OMPDoacrossKind<OMPDependClause> {
2297	public:
2298	bool isSink(const OMPDependClause *C) {
2299	return C->getDependencyKind() == OMPC_DEPEND_sink;
2300	}
2301	bool isSource(const OMPDependClause *C) {
2302	return C->getDependencyKind() == OMPC_DEPEND_source;
2303	}
2304	};
2305	template <> class OMPDoacrossKind<OMPDoacrossClause> {
2306	public:
2307	bool isSource(const OMPDoacrossClause *C) {
2308	return C->getDependenceType() == OMPC_DOACROSS_source ||
2309	C->getDependenceType() == OMPC_DOACROSS_source_omp_cur_iteration;
2310	}
2311	bool isSink(const OMPDoacrossClause *C) {
2312	return C->getDependenceType() == OMPC_DOACROSS_sink ||
2313	C->getDependenceType() == OMPC_DOACROSS_sink_omp_cur_iteration;
2314	}
2315	};
2316	} // namespace
2317	} // namespace clang
2318
2319	#endif
2320