1//===---- CGOpenMPRuntimeGPU.cpp - Interface to OpenMP GPU Runtimes ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This provides a generalized class for OpenMP runtime code generation
10// specialized by GPU targets NVPTX and AMDGCN.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CGOpenMPRuntimeGPU.h"
15#include "CGOpenMPRuntimeNVPTX.h"
16#include "CodeGenFunction.h"
17#include "clang/AST/Attr.h"
18#include "clang/AST/DeclOpenMP.h"
19#include "clang/AST/StmtOpenMP.h"
20#include "clang/AST/StmtVisitor.h"
21#include "clang/Basic/Cuda.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/Frontend/OpenMP/OMPGridValues.h"
24#include "llvm/IR/IntrinsicsNVPTX.h"
25
26using namespace clang;
27using namespace CodeGen;
28using namespace llvm::omp;
29
30namespace {
31/// Pre(post)-action for different OpenMP constructs specialized for NVPTX.
32class NVPTXActionTy final : public PrePostActionTy {
33 llvm::FunctionCallee EnterCallee = nullptr;
34 ArrayRef<llvm::Value *> EnterArgs;
35 llvm::FunctionCallee ExitCallee = nullptr;
36 ArrayRef<llvm::Value *> ExitArgs;
37 bool Conditional = false;
38 llvm::BasicBlock *ContBlock = nullptr;
39
40public:
41 NVPTXActionTy(llvm::FunctionCallee EnterCallee,
42 ArrayRef<llvm::Value *> EnterArgs,
43 llvm::FunctionCallee ExitCallee,
44 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false)
45 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
46 ExitArgs(ExitArgs), Conditional(Conditional) {}
47 void Enter(CodeGenFunction &CGF) override {
48 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
49 if (Conditional) {
50 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
51 auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
52 ContBlock = CGF.createBasicBlock("omp_if.end");
53 // Generate the branch (If-stmt)
54 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
55 CGF.EmitBlock(ThenBlock);
56 }
57 }
58 void Done(CodeGenFunction &CGF) {
59 // Emit the rest of blocks/branches
60 CGF.EmitBranch(ContBlock);
61 CGF.EmitBlock(ContBlock, true);
62 }
63 void Exit(CodeGenFunction &CGF) override {
64 CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
65 }
66};
67
68/// A class to track the execution mode when codegening directives within
69/// a target region. The appropriate mode (SPMD|NON-SPMD) is set on entry
70/// to the target region and used by containing directives such as 'parallel'
71/// to emit optimized code.
72class ExecutionRuntimeModesRAII {
73private:
74 CGOpenMPRuntimeGPU::ExecutionMode SavedExecMode =
75 CGOpenMPRuntimeGPU::EM_Unknown;
76 CGOpenMPRuntimeGPU::ExecutionMode &ExecMode;
77 bool SavedRuntimeMode = false;
78 bool *RuntimeMode = nullptr;
79
80public:
81 /// Constructor for Non-SPMD mode.
82 ExecutionRuntimeModesRAII(CGOpenMPRuntimeGPU::ExecutionMode &ExecMode)
83 : ExecMode(ExecMode) {
84 SavedExecMode = ExecMode;
85 ExecMode = CGOpenMPRuntimeGPU::EM_NonSPMD;
86 }
87 /// Constructor for SPMD mode.
88 ExecutionRuntimeModesRAII(CGOpenMPRuntimeGPU::ExecutionMode &ExecMode,
89 bool &RuntimeMode, bool FullRuntimeMode)
90 : ExecMode(ExecMode), RuntimeMode(&RuntimeMode) {
91 SavedExecMode = ExecMode;
92 SavedRuntimeMode = RuntimeMode;
93 ExecMode = CGOpenMPRuntimeGPU::EM_SPMD;
94 RuntimeMode = FullRuntimeMode;
95 }
96 ~ExecutionRuntimeModesRAII() {
97 ExecMode = SavedExecMode;
98 if (RuntimeMode)
99 *RuntimeMode = SavedRuntimeMode;
100 }
101};
102
103/// GPU Configuration: This information can be derived from cuda registers,
104/// however, providing compile time constants helps generate more efficient
105/// code. For all practical purposes this is fine because the configuration
106/// is the same for all known NVPTX architectures.
107enum MachineConfiguration : unsigned {
108 /// See "llvm/Frontend/OpenMP/OMPGridValues.h" for various related target
109 /// specific Grid Values like GV_Warp_Size, GV_Warp_Size_Log2,
110 /// and GV_Warp_Size_Log2_Mask.
111
112 /// Global memory alignment for performance.
113 GlobalMemoryAlignment = 128,
114
115 /// Maximal size of the shared memory buffer.
116 SharedMemorySize = 128,
117};
118
119static const ValueDecl *getPrivateItem(const Expr *RefExpr) {
120 RefExpr = RefExpr->IgnoreParens();
121 if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(RefExpr)) {
122 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts();
123 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
124 Base = TempASE->getBase()->IgnoreParenImpCasts();
125 RefExpr = Base;
126 } else if (auto *OASE = dyn_cast<OMPArraySectionExpr>(RefExpr)) {
127 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts();
128 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
129 Base = TempOASE->getBase()->IgnoreParenImpCasts();
130 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base))
131 Base = TempASE->getBase()->IgnoreParenImpCasts();
132 RefExpr = Base;
133 }
134 RefExpr = RefExpr->IgnoreParenImpCasts();
135 if (const auto *DE = dyn_cast<DeclRefExpr>(RefExpr))
136 return cast<ValueDecl>(DE->getDecl()->getCanonicalDecl());
137 const auto *ME = cast<MemberExpr>(RefExpr);
138 return cast<ValueDecl>(ME->getMemberDecl()->getCanonicalDecl());
139}
140
141
142static RecordDecl *buildRecordForGlobalizedVars(
143 ASTContext &C, ArrayRef<const ValueDecl *> EscapedDecls,
144 ArrayRef<const ValueDecl *> EscapedDeclsForTeams,
145 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
146 &MappedDeclsFields, int BufSize) {
147 using VarsDataTy = std::pair<CharUnits /*Align*/, const ValueDecl *>;
148 if (EscapedDecls.empty() && EscapedDeclsForTeams.empty())
149 return nullptr;
150 SmallVector<VarsDataTy, 4> GlobalizedVars;
151 for (const ValueDecl *D : EscapedDecls)
152 GlobalizedVars.emplace_back(
153 CharUnits::fromQuantity(std::max(
154 C.getDeclAlign(D).getQuantity(),
155 static_cast<CharUnits::QuantityType>(GlobalMemoryAlignment))),
156 D);
157 for (const ValueDecl *D : EscapedDeclsForTeams)
158 GlobalizedVars.emplace_back(C.getDeclAlign(D), D);
159 llvm::stable_sort(GlobalizedVars, [](VarsDataTy L, VarsDataTy R) {
160 return L.first > R.first;
161 });
162
163 // Build struct _globalized_locals_ty {
164 // /* globalized vars */[WarSize] align (max(decl_align,
165 // GlobalMemoryAlignment))
166 // /* globalized vars */ for EscapedDeclsForTeams
167 // };
168 RecordDecl *GlobalizedRD = C.buildImplicitRecord("_globalized_locals_ty");
169 GlobalizedRD->startDefinition();
170 llvm::SmallPtrSet<const ValueDecl *, 16> SingleEscaped(
171 EscapedDeclsForTeams.begin(), EscapedDeclsForTeams.end());
172 for (const auto &Pair : GlobalizedVars) {
173 const ValueDecl *VD = Pair.second;
174 QualType Type = VD->getType();
175 if (Type->isLValueReferenceType())
176 Type = C.getPointerType(Type.getNonReferenceType());
177 else
178 Type = Type.getNonReferenceType();
179 SourceLocation Loc = VD->getLocation();
180 FieldDecl *Field;
181 if (SingleEscaped.count(VD)) {
182 Field = FieldDecl::Create(
183 C, GlobalizedRD, Loc, Loc, VD->getIdentifier(), Type,
184 C.getTrivialTypeSourceInfo(Type, SourceLocation()),
185 /*BW=*/nullptr, /*Mutable=*/false,
186 /*InitStyle=*/ICIS_NoInit);
187 Field->setAccess(AS_public);
188 if (VD->hasAttrs()) {
189 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()),
190 E(VD->getAttrs().end());
191 I != E; ++I)
192 Field->addAttr(*I);
193 }
194 } else {
195 llvm::APInt ArraySize(32, BufSize);
196 Type = C.getConstantArrayType(Type, ArraySize, nullptr, ArrayType::Normal,
197 0);
198 Field = FieldDecl::Create(
199 C, GlobalizedRD, Loc, Loc, VD->getIdentifier(), Type,
200 C.getTrivialTypeSourceInfo(Type, SourceLocation()),
201 /*BW=*/nullptr, /*Mutable=*/false,
202 /*InitStyle=*/ICIS_NoInit);
203 Field->setAccess(AS_public);
204 llvm::APInt Align(32, std::max(C.getDeclAlign(VD).getQuantity(),
205 static_cast<CharUnits::QuantityType>(
206 GlobalMemoryAlignment)));
207 Field->addAttr(AlignedAttr::CreateImplicit(
208 C, /*IsAlignmentExpr=*/true,
209 IntegerLiteral::Create(C, Align,
210 C.getIntTypeForBitwidth(32, /*Signed=*/0),
211 SourceLocation()),
212 {}, AttributeCommonInfo::AS_GNU, AlignedAttr::GNU_aligned));
213 }
214 GlobalizedRD->addDecl(Field);
215 MappedDeclsFields.try_emplace(VD, Field);
216 }
217 GlobalizedRD->completeDefinition();
218 return GlobalizedRD;
219}
220
221/// Get the list of variables that can escape their declaration context.
222class CheckVarsEscapingDeclContext final
223 : public ConstStmtVisitor<CheckVarsEscapingDeclContext> {
224 CodeGenFunction &CGF;
225 llvm::SetVector<const ValueDecl *> EscapedDecls;
226 llvm::SetVector<const ValueDecl *> EscapedVariableLengthDecls;
227 llvm::SmallPtrSet<const Decl *, 4> EscapedParameters;
228 RecordDecl *GlobalizedRD = nullptr;
229 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
230 bool AllEscaped = false;
231 bool IsForCombinedParallelRegion = false;
232
233 void markAsEscaped(const ValueDecl *VD) {
234 // Do not globalize declare target variables.
235 if (!isa<VarDecl>(VD) ||
236 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD))
237 return;
238 VD = cast<ValueDecl>(VD->getCanonicalDecl());
239 // Use user-specified allocation.
240 if (VD->hasAttrs() && VD->hasAttr<OMPAllocateDeclAttr>())
241 return;
242 // Variables captured by value must be globalized.
243 if (auto *CSI = CGF.CapturedStmtInfo) {
244 if (const FieldDecl *FD = CSI->lookup(cast<VarDecl>(VD))) {
245 // Check if need to capture the variable that was already captured by
246 // value in the outer region.
247 if (!IsForCombinedParallelRegion) {
248 if (!FD->hasAttrs())
249 return;
250 const auto *Attr = FD->getAttr<OMPCaptureKindAttr>();
251 if (!Attr)
252 return;
253 if (((Attr->getCaptureKind() != OMPC_map) &&
254 !isOpenMPPrivate(Attr->getCaptureKind())) ||
255 ((Attr->getCaptureKind() == OMPC_map) &&
256 !FD->getType()->isAnyPointerType()))
257 return;
258 }
259 if (!FD->getType()->isReferenceType()) {
260 assert(!VD->getType()->isVariablyModifiedType() &&
261 "Parameter captured by value with variably modified type");
262 EscapedParameters.insert(VD);
263 } else if (!IsForCombinedParallelRegion) {
264 return;
265 }
266 }
267 }
268 if ((!CGF.CapturedStmtInfo ||
269 (IsForCombinedParallelRegion && CGF.CapturedStmtInfo)) &&
270 VD->getType()->isReferenceType())
271 // Do not globalize variables with reference type.
272 return;
273 if (VD->getType()->isVariablyModifiedType())
274 EscapedVariableLengthDecls.insert(VD);
275 else
276 EscapedDecls.insert(VD);
277 }
278
279 void VisitValueDecl(const ValueDecl *VD) {
280 if (VD->getType()->isLValueReferenceType())
281 markAsEscaped(VD);
282 if (const auto *VarD = dyn_cast<VarDecl>(VD)) {
283 if (!isa<ParmVarDecl>(VarD) && VarD->hasInit()) {
284 const bool SavedAllEscaped = AllEscaped;
285 AllEscaped = VD->getType()->isLValueReferenceType();
286 Visit(VarD->getInit());
287 AllEscaped = SavedAllEscaped;
288 }
289 }
290 }
291 void VisitOpenMPCapturedStmt(const CapturedStmt *S,
292 ArrayRef<OMPClause *> Clauses,
293 bool IsCombinedParallelRegion) {
294 if (!S)
295 return;
296 for (const CapturedStmt::Capture &C : S->captures()) {
297 if (C.capturesVariable() && !C.capturesVariableByCopy()) {
298 const ValueDecl *VD = C.getCapturedVar();
299 bool SavedIsForCombinedParallelRegion = IsForCombinedParallelRegion;
300 if (IsCombinedParallelRegion) {
301 // Check if the variable is privatized in the combined construct and
302 // those private copies must be shared in the inner parallel
303 // directive.
304 IsForCombinedParallelRegion = false;
305 for (const OMPClause *C : Clauses) {
306 if (!isOpenMPPrivate(C->getClauseKind()) ||
307 C->getClauseKind() == OMPC_reduction ||
308 C->getClauseKind() == OMPC_linear ||
309 C->getClauseKind() == OMPC_private)
310 continue;
311 ArrayRef<const Expr *> Vars;
312 if (const auto *PC = dyn_cast<OMPFirstprivateClause>(C))
313 Vars = PC->getVarRefs();
314 else if (const auto *PC = dyn_cast<OMPLastprivateClause>(C))
315 Vars = PC->getVarRefs();
316 else
317 llvm_unreachable("Unexpected clause.");
318 for (const auto *E : Vars) {
319 const Decl *D =
320 cast<DeclRefExpr>(E)->getDecl()->getCanonicalDecl();
321 if (D == VD->getCanonicalDecl()) {
322 IsForCombinedParallelRegion = true;
323 break;
324 }
325 }
326 if (IsForCombinedParallelRegion)
327 break;
328 }
329 }
330 markAsEscaped(VD);
331 if (isa<OMPCapturedExprDecl>(VD))
332 VisitValueDecl(VD);
333 IsForCombinedParallelRegion = SavedIsForCombinedParallelRegion;
334 }
335 }
336 }
337
338 void buildRecordForGlobalizedVars(bool IsInTTDRegion) {
339 assert(!GlobalizedRD &&
340 "Record for globalized variables is built already.");
341 ArrayRef<const ValueDecl *> EscapedDeclsForParallel, EscapedDeclsForTeams;
342 unsigned WarpSize = CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
343 if (IsInTTDRegion)
344 EscapedDeclsForTeams = EscapedDecls.getArrayRef();
345 else
346 EscapedDeclsForParallel = EscapedDecls.getArrayRef();
347 GlobalizedRD = ::buildRecordForGlobalizedVars(
348 CGF.getContext(), EscapedDeclsForParallel, EscapedDeclsForTeams,
349 MappedDeclsFields, WarpSize);
350 }
351
352public:
353 CheckVarsEscapingDeclContext(CodeGenFunction &CGF,
354 ArrayRef<const ValueDecl *> TeamsReductions)
355 : CGF(CGF), EscapedDecls(TeamsReductions.begin(), TeamsReductions.end()) {
356 }
357 virtual ~CheckVarsEscapingDeclContext() = default;
358 void VisitDeclStmt(const DeclStmt *S) {
359 if (!S)
360 return;
361 for (const Decl *D : S->decls())
362 if (const auto *VD = dyn_cast_or_null<ValueDecl>(D))
363 VisitValueDecl(VD);
364 }
365 void VisitOMPExecutableDirective(const OMPExecutableDirective *D) {
366 if (!D)
367 return;
368 if (!D->hasAssociatedStmt())
369 return;
370 if (const auto *S =
371 dyn_cast_or_null<CapturedStmt>(D->getAssociatedStmt())) {
372 // Do not analyze directives that do not actually require capturing,
373 // like `omp for` or `omp simd` directives.
374 llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
375 getOpenMPCaptureRegions(CaptureRegions, D->getDirectiveKind());
376 if (CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown) {
377 VisitStmt(S->getCapturedStmt());
378 return;
379 }
380 VisitOpenMPCapturedStmt(
381 S, D->clauses(),
382 CaptureRegions.back() == OMPD_parallel &&
383 isOpenMPDistributeDirective(D->getDirectiveKind()));
384 }
385 }
386 void VisitCapturedStmt(const CapturedStmt *S) {
387 if (!S)
388 return;
389 for (const CapturedStmt::Capture &C : S->captures()) {
390 if (C.capturesVariable() && !C.capturesVariableByCopy()) {
391 const ValueDecl *VD = C.getCapturedVar();
392 markAsEscaped(VD);
393 if (isa<OMPCapturedExprDecl>(VD))
394 VisitValueDecl(VD);
395 }
396 }
397 }
398 void VisitLambdaExpr(const LambdaExpr *E) {
399 if (!E)
400 return;
401 for (const LambdaCapture &C : E->captures()) {
402 if (C.capturesVariable()) {
403 if (C.getCaptureKind() == LCK_ByRef) {
404 const ValueDecl *VD = C.getCapturedVar();
405 markAsEscaped(VD);
406 if (E->isInitCapture(&C) || isa<OMPCapturedExprDecl>(VD))
407 VisitValueDecl(VD);
408 }
409 }
410 }
411 }
412 void VisitBlockExpr(const BlockExpr *E) {
413 if (!E)
414 return;
415 for (const BlockDecl::Capture &C : E->getBlockDecl()->captures()) {
416 if (C.isByRef()) {
417 const VarDecl *VD = C.getVariable();
418 markAsEscaped(VD);
419 if (isa<OMPCapturedExprDecl>(VD) || VD->isInitCapture())
420 VisitValueDecl(VD);
421 }
422 }
423 }
424 void VisitCallExpr(const CallExpr *E) {
425 if (!E)
426 return;
427 for (const Expr *Arg : E->arguments()) {
428 if (!Arg)
429 continue;
430 if (Arg->isLValue()) {
431 const bool SavedAllEscaped = AllEscaped;
432 AllEscaped = true;
433 Visit(Arg);
434 AllEscaped = SavedAllEscaped;
435 } else {
436 Visit(Arg);
437 }
438 }
439 Visit(E->getCallee());
440 }
441 void VisitDeclRefExpr(const DeclRefExpr *E) {
442 if (!E)
443 return;
444 const ValueDecl *VD = E->getDecl();
445 if (AllEscaped)
446 markAsEscaped(VD);
447 if (isa<OMPCapturedExprDecl>(VD))
448 VisitValueDecl(VD);
449 else if (const auto *VarD = dyn_cast<VarDecl>(VD))
450 if (VarD->isInitCapture())
451 VisitValueDecl(VD);
452 }
453 void VisitUnaryOperator(const UnaryOperator *E) {
454 if (!E)
455 return;
456 if (E->getOpcode() == UO_AddrOf) {
457 const bool SavedAllEscaped = AllEscaped;
458 AllEscaped = true;
459 Visit(E->getSubExpr());
460 AllEscaped = SavedAllEscaped;
461 } else {
462 Visit(E->getSubExpr());
463 }
464 }
465 void VisitImplicitCastExpr(const ImplicitCastExpr *E) {
466 if (!E)
467 return;
468 if (E->getCastKind() == CK_ArrayToPointerDecay) {
469 const bool SavedAllEscaped = AllEscaped;
470 AllEscaped = true;
471 Visit(E->getSubExpr());
472 AllEscaped = SavedAllEscaped;
473 } else {
474 Visit(E->getSubExpr());
475 }
476 }
477 void VisitExpr(const Expr *E) {
478 if (!E)
479 return;
480 bool SavedAllEscaped = AllEscaped;
481 if (!E->isLValue())
482 AllEscaped = false;
483 for (const Stmt *Child : E->children())
484 if (Child)
485 Visit(Child);
486 AllEscaped = SavedAllEscaped;
487 }
488 void VisitStmt(const Stmt *S) {
489 if (!S)
490 return;
491 for (const Stmt *Child : S->children())
492 if (Child)
493 Visit(Child);
494 }
495
496 /// Returns the record that handles all the escaped local variables and used
497 /// instead of their original storage.
498 const RecordDecl *getGlobalizedRecord(bool IsInTTDRegion) {
499 if (!GlobalizedRD)
500 buildRecordForGlobalizedVars(IsInTTDRegion);
501 return GlobalizedRD;
502 }
503
504 /// Returns the field in the globalized record for the escaped variable.
505 const FieldDecl *getFieldForGlobalizedVar(const ValueDecl *VD) const {
506 assert(GlobalizedRD &&
507 "Record for globalized variables must be generated already.");
508 auto I = MappedDeclsFields.find(VD);
509 if (I == MappedDeclsFields.end())
510 return nullptr;
511 return I->getSecond();
512 }
513
514 /// Returns the list of the escaped local variables/parameters.
515 ArrayRef<const ValueDecl *> getEscapedDecls() const {
516 return EscapedDecls.getArrayRef();
517 }
518
519 /// Checks if the escaped local variable is actually a parameter passed by
520 /// value.
521 const llvm::SmallPtrSetImpl<const Decl *> &getEscapedParameters() const {
522 return EscapedParameters;
523 }
524
525 /// Returns the list of the escaped variables with the variably modified
526 /// types.
527 ArrayRef<const ValueDecl *> getEscapedVariableLengthDecls() const {
528 return EscapedVariableLengthDecls.getArrayRef();
529 }
530};
531} // anonymous namespace
532
533/// Get the id of the warp in the block.
534/// We assume that the warp size is 32, which is always the case
535/// on the NVPTX device, to generate more efficient code.
536static llvm::Value *getNVPTXWarpID(CodeGenFunction &CGF) {
537 CGBuilderTy &Bld = CGF.Builder;
538 unsigned LaneIDBits =
539 CGF.getTarget().getGridValue(llvm::omp::GV_Warp_Size_Log2);
540 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
541 return Bld.CreateAShr(RT.getGPUThreadID(CGF), LaneIDBits, "nvptx_warp_id");
542}
543
544/// Get the id of the current lane in the Warp.
545/// We assume that the warp size is 32, which is always the case
546/// on the NVPTX device, to generate more efficient code.
547static llvm::Value *getNVPTXLaneID(CodeGenFunction &CGF) {
548 CGBuilderTy &Bld = CGF.Builder;
549 unsigned LaneIDMask = CGF.getContext().getTargetInfo().getGridValue(
550 llvm::omp::GV_Warp_Size_Log2_Mask);
551 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
552 return Bld.CreateAnd(RT.getGPUThreadID(CGF), Bld.getInt32(LaneIDMask),
553 "nvptx_lane_id");
554}
555
556/// Get the value of the thread_limit clause in the teams directive.
557/// For the 'generic' execution mode, the runtime encodes thread_limit in
558/// the launch parameters, always starting thread_limit+warpSize threads per
559/// CTA. The threads in the last warp are reserved for master execution.
560/// For the 'spmd' execution mode, all threads in a CTA are part of the team.
561static llvm::Value *getThreadLimit(CodeGenFunction &CGF,
562 bool IsInSPMDExecutionMode = false) {
563 CGBuilderTy &Bld = CGF.Builder;
564 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
565 llvm::Value *ThreadLimit = nullptr;
566 if (IsInSPMDExecutionMode)
567 ThreadLimit = RT.getGPUNumThreads(CGF);
568 else {
569 llvm::Value *GPUNumThreads = RT.getGPUNumThreads(CGF);
570 llvm::Value *GPUWarpSize = RT.getGPUWarpSize(CGF);
571 ThreadLimit = Bld.CreateNUWSub(GPUNumThreads, GPUWarpSize, "thread_limit");
572 }
573 assert(ThreadLimit != nullptr && "Expected non-null ThreadLimit");
574 return ThreadLimit;
575}
576
577/// Get the thread id of the OMP master thread.
578/// The master thread id is the first thread (lane) of the last warp in the
579/// GPU block. Warp size is assumed to be some power of 2.
580/// Thread id is 0 indexed.
581/// E.g: If NumThreads is 33, master id is 32.
582/// If NumThreads is 64, master id is 32.
583/// If NumThreads is 1024, master id is 992.
584static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) {
585 CGBuilderTy &Bld = CGF.Builder;
586 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
587 llvm::Value *NumThreads = RT.getGPUNumThreads(CGF);
588 // We assume that the warp size is a power of 2.
589 llvm::Value *Mask = Bld.CreateNUWSub(RT.getGPUWarpSize(CGF), Bld.getInt32(1));
590
591 llvm::Value *NumThreadsSubOne = Bld.CreateNUWSub(NumThreads, Bld.getInt32(1));
592 return Bld.CreateAnd(NumThreadsSubOne, Bld.CreateNot(Mask), "master_tid");
593}
594
595CGOpenMPRuntimeGPU::WorkerFunctionState::WorkerFunctionState(
596 CodeGenModule &CGM, SourceLocation Loc)
597 : WorkerFn(nullptr), CGFI(CGM.getTypes().arrangeNullaryFunction()),
598 Loc(Loc) {
599 createWorkerFunction(CGM);
600}
601
602void CGOpenMPRuntimeGPU::WorkerFunctionState::createWorkerFunction(
603 CodeGenModule &CGM) {
604 // Create an worker function with no arguments.
605
606 WorkerFn = llvm::Function::Create(
607 CGM.getTypes().GetFunctionType(CGFI), llvm::GlobalValue::InternalLinkage,
608 /*placeholder=*/"_worker", &CGM.getModule());
609 CGM.SetInternalFunctionAttributes(GlobalDecl(), WorkerFn, CGFI);
610 WorkerFn->setDoesNotRecurse();
611}
612
613CGOpenMPRuntimeGPU::ExecutionMode
614CGOpenMPRuntimeGPU::getExecutionMode() const {
615 return CurrentExecutionMode;
616}
617
618static CGOpenMPRuntimeGPU::DataSharingMode
619getDataSharingMode(CodeGenModule &CGM) {
620 return CGM.getLangOpts().OpenMPCUDAMode ? CGOpenMPRuntimeGPU::CUDA
621 : CGOpenMPRuntimeGPU::Generic;
622}
623
624/// Check for inner (nested) SPMD construct, if any
625static bool hasNestedSPMDDirective(ASTContext &Ctx,
626 const OMPExecutableDirective &D) {
627 const auto *CS = D.getInnermostCapturedStmt();
628 const auto *Body =
629 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
630 const Stmt *ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
631
632 if (const auto *NestedDir =
633 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
634 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
635 switch (D.getDirectiveKind()) {
636 case OMPD_target:
637 if (isOpenMPParallelDirective(DKind))
638 return true;
639 if (DKind == OMPD_teams) {
640 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
641 /*IgnoreCaptured=*/true);
642 if (!Body)
643 return false;
644 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
645 if (const auto *NND =
646 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
647 DKind = NND->getDirectiveKind();
648 if (isOpenMPParallelDirective(DKind))
649 return true;
650 }
651 }
652 return false;
653 case OMPD_target_teams:
654 return isOpenMPParallelDirective(DKind);
655 case OMPD_target_simd:
656 case OMPD_target_parallel:
657 case OMPD_target_parallel_for:
658 case OMPD_target_parallel_for_simd:
659 case OMPD_target_teams_distribute:
660 case OMPD_target_teams_distribute_simd:
661 case OMPD_target_teams_distribute_parallel_for:
662 case OMPD_target_teams_distribute_parallel_for_simd:
663 case OMPD_parallel:
664 case OMPD_for:
665 case OMPD_parallel_for:
666 case OMPD_parallel_master:
667 case OMPD_parallel_sections:
668 case OMPD_for_simd:
669 case OMPD_parallel_for_simd:
670 case OMPD_cancel:
671 case OMPD_cancellation_point:
672 case OMPD_ordered:
673 case OMPD_threadprivate:
674 case OMPD_allocate:
675 case OMPD_task:
676 case OMPD_simd:
677 case OMPD_sections:
678 case OMPD_section:
679 case OMPD_single:
680 case OMPD_master:
681 case OMPD_critical:
682 case OMPD_taskyield:
683 case OMPD_barrier:
684 case OMPD_taskwait:
685 case OMPD_taskgroup:
686 case OMPD_atomic:
687 case OMPD_flush:
688 case OMPD_depobj:
689 case OMPD_scan:
690 case OMPD_teams:
691 case OMPD_target_data:
692 case OMPD_target_exit_data:
693 case OMPD_target_enter_data:
694 case OMPD_distribute:
695 case OMPD_distribute_simd:
696 case OMPD_distribute_parallel_for:
697 case OMPD_distribute_parallel_for_simd:
698 case OMPD_teams_distribute:
699 case OMPD_teams_distribute_simd:
700 case OMPD_teams_distribute_parallel_for:
701 case OMPD_teams_distribute_parallel_for_simd:
702 case OMPD_target_update:
703 case OMPD_declare_simd:
704 case OMPD_declare_variant:
705 case OMPD_begin_declare_variant:
706 case OMPD_end_declare_variant:
707 case OMPD_declare_target:
708 case OMPD_end_declare_target:
709 case OMPD_declare_reduction:
710 case OMPD_declare_mapper:
711 case OMPD_taskloop:
712 case OMPD_taskloop_simd:
713 case OMPD_master_taskloop:
714 case OMPD_master_taskloop_simd:
715 case OMPD_parallel_master_taskloop:
716 case OMPD_parallel_master_taskloop_simd:
717 case OMPD_requires:
718 case OMPD_unknown:
719 default:
720 llvm_unreachable("Unexpected directive.");
721 }
722 }
723
724 return false;
725}
726
727static bool supportsSPMDExecutionMode(ASTContext &Ctx,
728 const OMPExecutableDirective &D) {
729 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
730 switch (DirectiveKind) {
731 case OMPD_target:
732 case OMPD_target_teams:
733 return hasNestedSPMDDirective(Ctx, D);
734 case OMPD_target_parallel:
735 case OMPD_target_parallel_for:
736 case OMPD_target_parallel_for_simd:
737 case OMPD_target_teams_distribute_parallel_for:
738 case OMPD_target_teams_distribute_parallel_for_simd:
739 case OMPD_target_simd:
740 case OMPD_target_teams_distribute_simd:
741 return true;
742 case OMPD_target_teams_distribute:
743 return false;
744 case OMPD_parallel:
745 case OMPD_for:
746 case OMPD_parallel_for:
747 case OMPD_parallel_master:
748 case OMPD_parallel_sections:
749 case OMPD_for_simd:
750 case OMPD_parallel_for_simd:
751 case OMPD_cancel:
752 case OMPD_cancellation_point:
753 case OMPD_ordered:
754 case OMPD_threadprivate:
755 case OMPD_allocate:
756 case OMPD_task:
757 case OMPD_simd:
758 case OMPD_sections:
759 case OMPD_section:
760 case OMPD_single:
761 case OMPD_master:
762 case OMPD_critical:
763 case OMPD_taskyield:
764 case OMPD_barrier:
765 case OMPD_taskwait:
766 case OMPD_taskgroup:
767 case OMPD_atomic:
768 case OMPD_flush:
769 case OMPD_depobj:
770 case OMPD_scan:
771 case OMPD_teams:
772 case OMPD_target_data:
773 case OMPD_target_exit_data:
774 case OMPD_target_enter_data:
775 case OMPD_distribute:
776 case OMPD_distribute_simd:
777 case OMPD_distribute_parallel_for:
778 case OMPD_distribute_parallel_for_simd:
779 case OMPD_teams_distribute:
780 case OMPD_teams_distribute_simd:
781 case OMPD_teams_distribute_parallel_for:
782 case OMPD_teams_distribute_parallel_for_simd:
783 case OMPD_target_update:
784 case OMPD_declare_simd:
785 case OMPD_declare_variant:
786 case OMPD_begin_declare_variant:
787 case OMPD_end_declare_variant:
788 case OMPD_declare_target:
789 case OMPD_end_declare_target:
790 case OMPD_declare_reduction:
791 case OMPD_declare_mapper:
792 case OMPD_taskloop:
793 case OMPD_taskloop_simd:
794 case OMPD_master_taskloop:
795 case OMPD_master_taskloop_simd:
796 case OMPD_parallel_master_taskloop:
797 case OMPD_parallel_master_taskloop_simd:
798 case OMPD_requires:
799 case OMPD_unknown:
800 default:
801 break;
802 }
803 llvm_unreachable(
804 "Unknown programming model for OpenMP directive on NVPTX target.");
805}
806
807/// Check if the directive is loops based and has schedule clause at all or has
808/// static scheduling.
809static bool hasStaticScheduling(const OMPExecutableDirective &D) {
810 assert(isOpenMPWorksharingDirective(D.getDirectiveKind()) &&
811 isOpenMPLoopDirective(D.getDirectiveKind()) &&
812 "Expected loop-based directive.");
813 return !D.hasClausesOfKind<OMPOrderedClause>() &&
814 (!D.hasClausesOfKind<OMPScheduleClause>() ||
815 llvm::any_of(D.getClausesOfKind<OMPScheduleClause>(),
816 [](const OMPScheduleClause *C) {
817 return C->getScheduleKind() == OMPC_SCHEDULE_static;
818 }));
819}
820
821/// Check for inner (nested) lightweight runtime construct, if any
822static bool hasNestedLightweightDirective(ASTContext &Ctx,
823 const OMPExecutableDirective &D) {
824 assert(supportsSPMDExecutionMode(Ctx, D) && "Expected SPMD mode directive.");
825 const auto *CS = D.getInnermostCapturedStmt();
826 const auto *Body =
827 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true);
828 const Stmt *ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
829
830 if (const auto *NestedDir =
831 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
832 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind();
833 switch (D.getDirectiveKind()) {
834 case OMPD_target:
835 if (isOpenMPParallelDirective(DKind) &&
836 isOpenMPWorksharingDirective(DKind) && isOpenMPLoopDirective(DKind) &&
837 hasStaticScheduling(*NestedDir))
838 return true;
839 if (DKind == OMPD_teams_distribute_simd || DKind == OMPD_simd)
840 return true;
841 if (DKind == OMPD_parallel) {
842 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
843 /*IgnoreCaptured=*/true);
844 if (!Body)
845 return false;
846 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
847 if (const auto *NND =
848 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
849 DKind = NND->getDirectiveKind();
850 if (isOpenMPWorksharingDirective(DKind) &&
851 isOpenMPLoopDirective(DKind) && hasStaticScheduling(*NND))
852 return true;
853 }
854 } else if (DKind == OMPD_teams) {
855 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
856 /*IgnoreCaptured=*/true);
857 if (!Body)
858 return false;
859 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
860 if (const auto *NND =
861 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
862 DKind = NND->getDirectiveKind();
863 if (isOpenMPParallelDirective(DKind) &&
864 isOpenMPWorksharingDirective(DKind) &&
865 isOpenMPLoopDirective(DKind) && hasStaticScheduling(*NND))
866 return true;
867 if (DKind == OMPD_parallel) {
868 Body = NND->getInnermostCapturedStmt()->IgnoreContainers(
869 /*IgnoreCaptured=*/true);
870 if (!Body)
871 return false;
872 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
873 if (const auto *NND =
874 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
875 DKind = NND->getDirectiveKind();
876 if (isOpenMPWorksharingDirective(DKind) &&
877 isOpenMPLoopDirective(DKind) && hasStaticScheduling(*NND))
878 return true;
879 }
880 }
881 }
882 }
883 return false;
884 case OMPD_target_teams:
885 if (isOpenMPParallelDirective(DKind) &&
886 isOpenMPWorksharingDirective(DKind) && isOpenMPLoopDirective(DKind) &&
887 hasStaticScheduling(*NestedDir))
888 return true;
889 if (DKind == OMPD_distribute_simd || DKind == OMPD_simd)
890 return true;
891 if (DKind == OMPD_parallel) {
892 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(
893 /*IgnoreCaptured=*/true);
894 if (!Body)
895 return false;
896 ChildStmt = CGOpenMPRuntime::getSingleCompoundChild(Ctx, Body);
897 if (const auto *NND =
898 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {
899 DKind = NND->getDirectiveKind();
900 if (isOpenMPWorksharingDirective(DKind) &&
901 isOpenMPLoopDirective(DKind) && hasStaticScheduling(*NND))
902 return true;
903 }
904 }
905 return false;
906 case OMPD_target_parallel:
907 if (DKind == OMPD_simd)
908 return true;
909 return isOpenMPWorksharingDirective(DKind) &&
910 isOpenMPLoopDirective(DKind) && hasStaticScheduling(*NestedDir);
911 case OMPD_target_teams_distribute:
912 case OMPD_target_simd:
913 case OMPD_target_parallel_for:
914 case OMPD_target_parallel_for_simd:
915 case OMPD_target_teams_distribute_simd:
916 case OMPD_target_teams_distribute_parallel_for:
917 case OMPD_target_teams_distribute_parallel_for_simd:
918 case OMPD_parallel:
919 case OMPD_for:
920 case OMPD_parallel_for:
921 case OMPD_parallel_master:
922 case OMPD_parallel_sections:
923 case OMPD_for_simd:
924 case OMPD_parallel_for_simd:
925 case OMPD_cancel:
926 case OMPD_cancellation_point:
927 case OMPD_ordered:
928 case OMPD_threadprivate:
929 case OMPD_allocate:
930 case OMPD_task:
931 case OMPD_simd:
932 case OMPD_sections:
933 case OMPD_section:
934 case OMPD_single:
935 case OMPD_master:
936 case OMPD_critical:
937 case OMPD_taskyield:
938 case OMPD_barrier:
939 case OMPD_taskwait:
940 case OMPD_taskgroup:
941 case OMPD_atomic:
942 case OMPD_flush:
943 case OMPD_depobj:
944 case OMPD_scan:
945 case OMPD_teams:
946 case OMPD_target_data:
947 case OMPD_target_exit_data:
948 case OMPD_target_enter_data:
949 case OMPD_distribute:
950 case OMPD_distribute_simd:
951 case OMPD_distribute_parallel_for:
952 case OMPD_distribute_parallel_for_simd:
953 case OMPD_teams_distribute:
954 case OMPD_teams_distribute_simd:
955 case OMPD_teams_distribute_parallel_for:
956 case OMPD_teams_distribute_parallel_for_simd:
957 case OMPD_target_update:
958 case OMPD_declare_simd:
959 case OMPD_declare_variant:
960 case OMPD_begin_declare_variant:
961 case OMPD_end_declare_variant:
962 case OMPD_declare_target:
963 case OMPD_end_declare_target:
964 case OMPD_declare_reduction:
965 case OMPD_declare_mapper:
966 case OMPD_taskloop:
967 case OMPD_taskloop_simd:
968 case OMPD_master_taskloop:
969 case OMPD_master_taskloop_simd:
970 case OMPD_parallel_master_taskloop:
971 case OMPD_parallel_master_taskloop_simd:
972 case OMPD_requires:
973 case OMPD_unknown:
974 default:
975 llvm_unreachable("Unexpected directive.");
976 }
977 }
978
979 return false;
980}
981
982/// Checks if the construct supports lightweight runtime. It must be SPMD
983/// construct + inner loop-based construct with static scheduling.
984static bool supportsLightweightRuntime(ASTContext &Ctx,
985 const OMPExecutableDirective &D) {
986 if (!supportsSPMDExecutionMode(Ctx, D))
987 return false;
988 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind();
989 switch (DirectiveKind) {
990 case OMPD_target:
991 case OMPD_target_teams:
992 case OMPD_target_parallel:
993 return hasNestedLightweightDirective(Ctx, D);
994 case OMPD_target_parallel_for:
995 case OMPD_target_parallel_for_simd:
996 case OMPD_target_teams_distribute_parallel_for:
997 case OMPD_target_teams_distribute_parallel_for_simd:
998 // (Last|First)-privates must be shared in parallel region.
999 return hasStaticScheduling(D);
1000 case OMPD_target_simd:
1001 case OMPD_target_teams_distribute_simd:
1002 return true;
1003 case OMPD_target_teams_distribute:
1004 return false;
1005 case OMPD_parallel:
1006 case OMPD_for:
1007 case OMPD_parallel_for:
1008 case OMPD_parallel_master:
1009 case OMPD_parallel_sections:
1010 case OMPD_for_simd:
1011 case OMPD_parallel_for_simd:
1012 case OMPD_cancel:
1013 case OMPD_cancellation_point:
1014 case OMPD_ordered:
1015 case OMPD_threadprivate:
1016 case OMPD_allocate:
1017 case OMPD_task:
1018 case OMPD_simd:
1019 case OMPD_sections:
1020 case OMPD_section:
1021 case OMPD_single:
1022 case OMPD_master:
1023 case OMPD_critical:
1024 case OMPD_taskyield:
1025 case OMPD_barrier:
1026 case OMPD_taskwait:
1027 case OMPD_taskgroup:
1028 case OMPD_atomic:
1029 case OMPD_flush:
1030 case OMPD_depobj:
1031 case OMPD_scan:
1032 case OMPD_teams:
1033 case OMPD_target_data:
1034 case OMPD_target_exit_data:
1035 case OMPD_target_enter_data:
1036 case OMPD_distribute:
1037 case OMPD_distribute_simd:
1038 case OMPD_distribute_parallel_for:
1039 case OMPD_distribute_parallel_for_simd:
1040 case OMPD_teams_distribute:
1041 case OMPD_teams_distribute_simd:
1042 case OMPD_teams_distribute_parallel_for:
1043 case OMPD_teams_distribute_parallel_for_simd:
1044 case OMPD_target_update:
1045 case OMPD_declare_simd:
1046 case OMPD_declare_variant:
1047 case OMPD_begin_declare_variant:
1048 case OMPD_end_declare_variant:
1049 case OMPD_declare_target:
1050 case OMPD_end_declare_target:
1051 case OMPD_declare_reduction:
1052 case OMPD_declare_mapper:
1053 case OMPD_taskloop:
1054 case OMPD_taskloop_simd:
1055 case OMPD_master_taskloop:
1056 case OMPD_master_taskloop_simd:
1057 case OMPD_parallel_master_taskloop:
1058 case OMPD_parallel_master_taskloop_simd:
1059 case OMPD_requires:
1060 case OMPD_unknown:
1061 default:
1062 break;
1063 }
1064 llvm_unreachable(
1065 "Unknown programming model for OpenMP directive on NVPTX target.");
1066}
1067
1068void CGOpenMPRuntimeGPU::emitNonSPMDKernel(const OMPExecutableDirective &D,
1069 StringRef ParentName,
1070 llvm::Function *&OutlinedFn,
1071 llvm::Constant *&OutlinedFnID,
1072 bool IsOffloadEntry,
1073 const RegionCodeGenTy &CodeGen) {
1074 ExecutionRuntimeModesRAII ModeRAII(CurrentExecutionMode);
1075 EntryFunctionState EST;
1076 WorkerFunctionState WST(CGM, D.getBeginLoc());
1077 Work.clear();
1078 WrapperFunctionsMap.clear();
1079
1080 // Emit target region as a standalone region.
1081 class NVPTXPrePostActionTy : public PrePostActionTy {
1082 CGOpenMPRuntimeGPU::EntryFunctionState &EST;
1083 CGOpenMPRuntimeGPU::WorkerFunctionState &WST;
1084
1085 public:
1086 NVPTXPrePostActionTy(CGOpenMPRuntimeGPU::EntryFunctionState &EST,
1087 CGOpenMPRuntimeGPU::WorkerFunctionState &WST)
1088 : EST(EST), WST(WST) {}
1089 void Enter(CodeGenFunction &CGF) override {
1090 auto &RT =
1091 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1092 RT.emitNonSPMDEntryHeader(CGF, EST, WST);
1093 // Skip target region initialization.
1094 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1095 }
1096 void Exit(CodeGenFunction &CGF) override {
1097 auto &RT =
1098 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1099 RT.clearLocThreadIdInsertPt(CGF);
1100 RT.emitNonSPMDEntryFooter(CGF, EST);
1101 }
1102 } Action(EST, WST);
1103 CodeGen.setAction(Action);
1104 IsInTTDRegion = true;
1105 // Reserve place for the globalized memory.
1106 GlobalizedRecords.emplace_back();
1107 if (!KernelStaticGlobalized) {
1108 KernelStaticGlobalized = new llvm::GlobalVariable(
1109 CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
1110 llvm::GlobalValue::InternalLinkage,
1111 llvm::UndefValue::get(CGM.VoidPtrTy),
1112 "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
1113 llvm::GlobalValue::NotThreadLocal,
1114 CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
1115 }
1116 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
1117 IsOffloadEntry, CodeGen);
1118 IsInTTDRegion = false;
1119
1120 // Now change the name of the worker function to correspond to this target
1121 // region's entry function.
1122 WST.WorkerFn->setName(Twine(OutlinedFn->getName(), "_worker"));
1123
1124 // Create the worker function
1125 emitWorkerFunction(WST);
1126}
1127
1128// Setup NVPTX threads for master-worker OpenMP scheme.
1129void CGOpenMPRuntimeGPU::emitNonSPMDEntryHeader(CodeGenFunction &CGF,
1130 EntryFunctionState &EST,
1131 WorkerFunctionState &WST) {
1132 CGBuilderTy &Bld = CGF.Builder;
1133
1134 llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker");
1135 llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck");
1136 llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master");
1137 EST.ExitBB = CGF.createBasicBlock(".exit");
1138
1139 auto &RT = static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1140 llvm::Value *GPUThreadID = RT.getGPUThreadID(CGF);
1141 llvm::Value *ThreadLimit = getThreadLimit(CGF);
1142 llvm::Value *IsWorker = Bld.CreateICmpULT(GPUThreadID, ThreadLimit);
1143 Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
1144
1145 CGF.EmitBlock(WorkerBB);
1146 emitCall(CGF, WST.Loc, WST.WorkerFn);
1147 CGF.EmitBranch(EST.ExitBB);
1148
1149 CGF.EmitBlock(MasterCheckBB);
1150 GPUThreadID = RT.getGPUThreadID(CGF);
1151 llvm::Value *MasterThreadID = getMasterThreadID(CGF);
1152 llvm::Value *IsMaster = Bld.CreateICmpEQ(GPUThreadID, MasterThreadID);
1153 Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
1154
1155 CGF.EmitBlock(MasterBB);
1156 IsInTargetMasterThreadRegion = true;
1157 // SEQUENTIAL (MASTER) REGION START
1158 // First action in sequential region:
1159 // Initialize the state of the OpenMP runtime library on the GPU.
1160 // TODO: Optimize runtime initialization and pass in correct value.
1161 llvm::Value *Args[] = {getThreadLimit(CGF),
1162 Bld.getInt16(/*RequiresOMPRuntime=*/1)};
1163 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1164 CGM.getModule(), OMPRTL___kmpc_kernel_init),
1165 Args);
1166
1167 // For data sharing, we need to initialize the stack.
1168 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1169 CGM.getModule(), OMPRTL___kmpc_data_sharing_init_stack));
1170
1171 emitGenericVarsProlog(CGF, WST.Loc);
1172}
1173
1174void CGOpenMPRuntimeGPU::emitNonSPMDEntryFooter(CodeGenFunction &CGF,
1175 EntryFunctionState &EST) {
1176 IsInTargetMasterThreadRegion = false;
1177 if (!CGF.HaveInsertPoint())
1178 return;
1179
1180 emitGenericVarsEpilog(CGF);
1181
1182 if (!EST.ExitBB)
1183 EST.ExitBB = CGF.createBasicBlock(".exit");
1184
1185 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier");
1186 CGF.EmitBranch(TerminateBB);
1187
1188 CGF.EmitBlock(TerminateBB);
1189 // Signal termination condition.
1190 // TODO: Optimize runtime initialization and pass in correct value.
1191 llvm::Value *Args[] = {CGF.Builder.getInt16(/*IsOMPRuntimeInitialized=*/1)};
1192 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1193 CGM.getModule(), OMPRTL___kmpc_kernel_deinit),
1194 Args);
1195 // Barrier to terminate worker threads.
1196 syncCTAThreads(CGF);
1197 // Master thread jumps to exit point.
1198 CGF.EmitBranch(EST.ExitBB);
1199
1200 CGF.EmitBlock(EST.ExitBB);
1201 EST.ExitBB = nullptr;
1202}
1203
1204void CGOpenMPRuntimeGPU::emitSPMDKernel(const OMPExecutableDirective &D,
1205 StringRef ParentName,
1206 llvm::Function *&OutlinedFn,
1207 llvm::Constant *&OutlinedFnID,
1208 bool IsOffloadEntry,
1209 const RegionCodeGenTy &CodeGen) {
1210 ExecutionRuntimeModesRAII ModeRAII(
1211 CurrentExecutionMode, RequiresFullRuntime,
1212 CGM.getLangOpts().OpenMPCUDAForceFullRuntime ||
1213 !supportsLightweightRuntime(CGM.getContext(), D));
1214 EntryFunctionState EST;
1215
1216 // Emit target region as a standalone region.
1217 class NVPTXPrePostActionTy : public PrePostActionTy {
1218 CGOpenMPRuntimeGPU &RT;
1219 CGOpenMPRuntimeGPU::EntryFunctionState &EST;
1220 const OMPExecutableDirective &D;
1221
1222 public:
1223 NVPTXPrePostActionTy(CGOpenMPRuntimeGPU &RT,
1224 CGOpenMPRuntimeGPU::EntryFunctionState &EST,
1225 const OMPExecutableDirective &D)
1226 : RT(RT), EST(EST), D(D) {}
1227 void Enter(CodeGenFunction &CGF) override {
1228 RT.emitSPMDEntryHeader(CGF, EST, D);
1229 // Skip target region initialization.
1230 RT.setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1231 }
1232 void Exit(CodeGenFunction &CGF) override {
1233 RT.clearLocThreadIdInsertPt(CGF);
1234 RT.emitSPMDEntryFooter(CGF, EST);
1235 }
1236 } Action(*this, EST, D);
1237 CodeGen.setAction(Action);
1238 IsInTTDRegion = true;
1239 // Reserve place for the globalized memory.
1240 GlobalizedRecords.emplace_back();
1241 if (!KernelStaticGlobalized) {
1242 KernelStaticGlobalized = new llvm::GlobalVariable(
1243 CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
1244 llvm::GlobalValue::InternalLinkage,
1245 llvm::UndefValue::get(CGM.VoidPtrTy),
1246 "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
1247 llvm::GlobalValue::NotThreadLocal,
1248 CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
1249 }
1250 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
1251 IsOffloadEntry, CodeGen);
1252 IsInTTDRegion = false;
1253}
1254
1255void CGOpenMPRuntimeGPU::emitSPMDEntryHeader(
1256 CodeGenFunction &CGF, EntryFunctionState &EST,
1257 const OMPExecutableDirective &D) {
1258 CGBuilderTy &Bld = CGF.Builder;
1259
1260 // Setup BBs in entry function.
1261 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute");
1262 EST.ExitBB = CGF.createBasicBlock(".exit");
1263
1264 llvm::Value *Args[] = {getThreadLimit(CGF, /*IsInSPMDExecutionMode=*/true),
1265 /*RequiresOMPRuntime=*/
1266 Bld.getInt16(RequiresFullRuntime ? 1 : 0)};
1267 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1268 CGM.getModule(), OMPRTL___kmpc_spmd_kernel_init),
1269 Args);
1270
1271 if (RequiresFullRuntime) {
1272 // For data sharing, we need to initialize the stack.
1273 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1274 CGM.getModule(), OMPRTL___kmpc_data_sharing_init_stack_spmd));
1275 }
1276
1277 CGF.EmitBranch(ExecuteBB);
1278
1279 CGF.EmitBlock(ExecuteBB);
1280
1281 IsInTargetMasterThreadRegion = true;
1282}
1283
1284void CGOpenMPRuntimeGPU::emitSPMDEntryFooter(CodeGenFunction &CGF,
1285 EntryFunctionState &EST) {
1286 IsInTargetMasterThreadRegion = false;
1287 if (!CGF.HaveInsertPoint())
1288 return;
1289
1290 if (!EST.ExitBB)
1291 EST.ExitBB = CGF.createBasicBlock(".exit");
1292
1293 llvm::BasicBlock *OMPDeInitBB = CGF.createBasicBlock(".omp.deinit");
1294 CGF.EmitBranch(OMPDeInitBB);
1295
1296 CGF.EmitBlock(OMPDeInitBB);
1297 // DeInitialize the OMP state in the runtime; called by all active threads.
1298 llvm::Value *Args[] = {/*RequiresOMPRuntime=*/
1299 CGF.Builder.getInt16(RequiresFullRuntime ? 1 : 0)};
1300 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1301 CGM.getModule(), OMPRTL___kmpc_spmd_kernel_deinit_v2),
1302 Args);
1303 CGF.EmitBranch(EST.ExitBB);
1304
1305 CGF.EmitBlock(EST.ExitBB);
1306 EST.ExitBB = nullptr;
1307}
1308
1309// Create a unique global variable to indicate the execution mode of this target
1310// region. The execution mode is either 'generic', or 'spmd' depending on the
1311// target directive. This variable is picked up by the offload library to setup
1312// the device appropriately before kernel launch. If the execution mode is
1313// 'generic', the runtime reserves one warp for the master, otherwise, all
1314// warps participate in parallel work.
1315static void setPropertyExecutionMode(CodeGenModule &CGM, StringRef Name,
1316 bool Mode) {
1317 auto *GVMode =
1318 new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
1319 llvm::GlobalValue::WeakAnyLinkage,
1320 llvm::ConstantInt::get(CGM.Int8Ty, Mode ? 0 : 1),
1321 Twine(Name, "_exec_mode"));
1322 CGM.addCompilerUsedGlobal(GVMode);
1323}
1324
1325void CGOpenMPRuntimeGPU::emitWorkerFunction(WorkerFunctionState &WST) {
1326 ASTContext &Ctx = CGM.getContext();
1327
1328 CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
1329 CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, WST.CGFI, {},
1330 WST.Loc, WST.Loc);
1331 emitWorkerLoop(CGF, WST);
1332 CGF.FinishFunction();
1333}
1334
1335void CGOpenMPRuntimeGPU::emitWorkerLoop(CodeGenFunction &CGF,
1336 WorkerFunctionState &WST) {
1337 //
1338 // The workers enter this loop and wait for parallel work from the master.
1339 // When the master encounters a parallel region it sets up the work + variable
1340 // arguments, and wakes up the workers. The workers first check to see if
1341 // they are required for the parallel region, i.e., within the # of requested
1342 // parallel threads. The activated workers load the variable arguments and
1343 // execute the parallel work.
1344 //
1345
1346 CGBuilderTy &Bld = CGF.Builder;
1347
1348 llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work");
1349 llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers");
1350 llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel");
1351 llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel");
1352 llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel");
1353 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
1354
1355 CGF.EmitBranch(AwaitBB);
1356
1357 // Workers wait for work from master.
1358 CGF.EmitBlock(AwaitBB);
1359 // Wait for parallel work
1360 syncCTAThreads(CGF);
1361
1362 Address WorkFn =
1363 CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn");
1364 Address ExecStatus =
1365 CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status");
1366 CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
1367 CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy));
1368
1369 // TODO: Optimize runtime initialization and pass in correct value.
1370 llvm::Value *Args[] = {WorkFn.getPointer()};
1371 llvm::Value *Ret =
1372 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1373 CGM.getModule(), OMPRTL___kmpc_kernel_parallel),
1374 Args);
1375 Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
1376
1377 // On termination condition (workid == 0), exit loop.
1378 llvm::Value *WorkID = Bld.CreateLoad(WorkFn);
1379 llvm::Value *ShouldTerminate = Bld.CreateIsNull(WorkID, "should_terminate");
1380 Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB);
1381
1382 // Activate requested workers.
1383 CGF.EmitBlock(SelectWorkersBB);
1384 llvm::Value *IsActive =
1385 Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active");
1386 Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB);
1387
1388 // Signal start of parallel region.
1389 CGF.EmitBlock(ExecuteBB);
1390 // Skip initialization.
1391 setLocThreadIdInsertPt(CGF, /*AtCurrentPoint=*/true);
1392
1393 // Process work items: outlined parallel functions.
1394 for (llvm::Function *W : Work) {
1395 // Try to match this outlined function.
1396 llvm::Value *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy);
1397
1398 llvm::Value *WorkFnMatch =
1399 Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match");
1400
1401 llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn");
1402 llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next");
1403 Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB);
1404
1405 // Execute this outlined function.
1406 CGF.EmitBlock(ExecuteFNBB);
1407
1408 // Insert call to work function via shared wrapper. The shared
1409 // wrapper takes two arguments:
1410 // - the parallelism level;
1411 // - the thread ID;
1412 emitCall(CGF, WST.Loc, W,
1413 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1414
1415 // Go to end of parallel region.
1416 CGF.EmitBranch(TerminateBB);
1417
1418 CGF.EmitBlock(CheckNextBB);
1419 }
1420 // Default case: call to outlined function through pointer if the target
1421 // region makes a declare target call that may contain an orphaned parallel
1422 // directive.
1423 auto *ParallelFnTy =
1424 llvm::FunctionType::get(CGM.VoidTy, {CGM.Int16Ty, CGM.Int32Ty},
1425 /*isVarArg=*/false);
1426 llvm::Value *WorkFnCast =
1427 Bld.CreateBitCast(WorkID, ParallelFnTy->getPointerTo());
1428 // Insert call to work function via shared wrapper. The shared
1429 // wrapper takes two arguments:
1430 // - the parallelism level;
1431 // - the thread ID;
1432 emitCall(CGF, WST.Loc, {ParallelFnTy, WorkFnCast},
1433 {Bld.getInt16(/*ParallelLevel=*/0), getThreadID(CGF, WST.Loc)});
1434 // Go to end of parallel region.
1435 CGF.EmitBranch(TerminateBB);
1436
1437 // Signal end of parallel region.
1438 CGF.EmitBlock(TerminateBB);
1439 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1440 CGM.getModule(), OMPRTL___kmpc_kernel_end_parallel),
1441 llvm::None);
1442 CGF.EmitBranch(BarrierBB);
1443
1444 // All active and inactive workers wait at a barrier after parallel region.
1445 CGF.EmitBlock(BarrierBB);
1446 // Barrier after parallel region.
1447 syncCTAThreads(CGF);
1448 CGF.EmitBranch(AwaitBB);
1449
1450 // Exit target region.
1451 CGF.EmitBlock(ExitBB);
1452 // Skip initialization.
1453 clearLocThreadIdInsertPt(CGF);
1454}
1455
1456void CGOpenMPRuntimeGPU::createOffloadEntry(llvm::Constant *ID,
1457 llvm::Constant *Addr,
1458 uint64_t Size, int32_t,
1459 llvm::GlobalValue::LinkageTypes) {
1460 // TODO: Add support for global variables on the device after declare target
1461 // support.
1462 if (!isa<llvm::Function>(Addr))
1463 return;
1464 llvm::Module &M = CGM.getModule();
1465 llvm::LLVMContext &Ctx = CGM.getLLVMContext();
1466
1467 // Get "nvvm.annotations" metadata node
1468 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
1469
1470 llvm::Metadata *MDVals[] = {
1471 llvm::ConstantAsMetadata::get(Addr), llvm::MDString::get(Ctx, "kernel"),
1472 llvm::ConstantAsMetadata::get(
1473 llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
1474 // Append metadata to nvvm.annotations
1475 MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
1476}
1477
1478void CGOpenMPRuntimeGPU::emitTargetOutlinedFunction(
1479 const OMPExecutableDirective &D, StringRef ParentName,
1480 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
1481 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
1482 if (!IsOffloadEntry) // Nothing to do.
1483 return;
1484
1485 assert(!ParentName.empty() && "Invalid target region parent name!");
1486
1487 bool Mode = supportsSPMDExecutionMode(CGM.getContext(), D);
1488 if (Mode)
1489 emitSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
1490 CodeGen);
1491 else
1492 emitNonSPMDKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
1493 CodeGen);
1494
1495 setPropertyExecutionMode(CGM, OutlinedFn->getName(), Mode);
1496}
1497
1498namespace {
1499LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
1500/// Enum for accesseing the reserved_2 field of the ident_t struct.
1501enum ModeFlagsTy : unsigned {
1502 /// Bit set to 1 when in SPMD mode.
1503 KMP_IDENT_SPMD_MODE = 0x01,
1504 /// Bit set to 1 when a simplified runtime is used.
1505 KMP_IDENT_SIMPLE_RT_MODE = 0x02,
1506 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/KMP_IDENT_SIMPLE_RT_MODE)
1507};
1508
1509/// Special mode Undefined. Is the combination of Non-SPMD mode + SimpleRuntime.
1510static const ModeFlagsTy UndefinedMode =
1511 (~KMP_IDENT_SPMD_MODE) & KMP_IDENT_SIMPLE_RT_MODE;
1512} // anonymous namespace
1513
1514unsigned CGOpenMPRuntimeGPU::getDefaultLocationReserved2Flags() const {
1515 switch (getExecutionMode()) {
1516 case EM_SPMD:
1517 if (requiresFullRuntime())
1518 return KMP_IDENT_SPMD_MODE & (~KMP_IDENT_SIMPLE_RT_MODE);
1519 return KMP_IDENT_SPMD_MODE | KMP_IDENT_SIMPLE_RT_MODE;
1520 case EM_NonSPMD:
1521 assert(requiresFullRuntime() && "Expected full runtime.");
1522 return (~KMP_IDENT_SPMD_MODE) & (~KMP_IDENT_SIMPLE_RT_MODE);
1523 case EM_Unknown:
1524 return UndefinedMode;
1525 }
1526 llvm_unreachable("Unknown flags are requested.");
1527}
1528
1529CGOpenMPRuntimeGPU::CGOpenMPRuntimeGPU(CodeGenModule &CGM)
1530 : CGOpenMPRuntime(CGM, "_", "$") {
1531 if (!CGM.getLangOpts().OpenMPIsDevice)
1532 llvm_unreachable("OpenMP NVPTX can only handle device code.");
1533}
1534
1535void CGOpenMPRuntimeGPU::emitProcBindClause(CodeGenFunction &CGF,
1536 ProcBindKind ProcBind,
1537 SourceLocation Loc) {
1538 // Do nothing in case of SPMD mode and L0 parallel.
1539 if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD)
1540 return;
1541
1542 CGOpenMPRuntime::emitProcBindClause(CGF, ProcBind, Loc);
1543}
1544
1545void CGOpenMPRuntimeGPU::emitNumThreadsClause(CodeGenFunction &CGF,
1546 llvm::Value *NumThreads,
1547 SourceLocation Loc) {
1548 // Do nothing in case of SPMD mode and L0 parallel.
1549 if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD)
1550 return;
1551
1552 CGOpenMPRuntime::emitNumThreadsClause(CGF, NumThreads, Loc);
1553}
1554
1555void CGOpenMPRuntimeGPU::emitNumTeamsClause(CodeGenFunction &CGF,
1556 const Expr *NumTeams,
1557 const Expr *ThreadLimit,
1558 SourceLocation Loc) {}
1559
1560llvm::Function *CGOpenMPRuntimeGPU::emitParallelOutlinedFunction(
1561 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1562 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1563 // Emit target region as a standalone region.
1564 class NVPTXPrePostActionTy : public PrePostActionTy {
1565 bool &IsInParallelRegion;
1566 bool PrevIsInParallelRegion;
1567
1568 public:
1569 NVPTXPrePostActionTy(bool &IsInParallelRegion)
1570 : IsInParallelRegion(IsInParallelRegion) {}
1571 void Enter(CodeGenFunction &CGF) override {
1572 PrevIsInParallelRegion = IsInParallelRegion;
1573 IsInParallelRegion = true;
1574 }
1575 void Exit(CodeGenFunction &CGF) override {
1576 IsInParallelRegion = PrevIsInParallelRegion;
1577 }
1578 } Action(IsInParallelRegion);
1579 CodeGen.setAction(Action);
1580 bool PrevIsInTTDRegion = IsInTTDRegion;
1581 IsInTTDRegion = false;
1582 bool PrevIsInTargetMasterThreadRegion = IsInTargetMasterThreadRegion;
1583 IsInTargetMasterThreadRegion = false;
1584 auto *OutlinedFun =
1585 cast<llvm::Function>(CGOpenMPRuntime::emitParallelOutlinedFunction(
1586 D, ThreadIDVar, InnermostKind, CodeGen));
1587 IsInTargetMasterThreadRegion = PrevIsInTargetMasterThreadRegion;
1588 IsInTTDRegion = PrevIsInTTDRegion;
1589 if (getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD &&
1590 !IsInParallelRegion) {
1591 llvm::Function *WrapperFun =
1592 createParallelDataSharingWrapper(OutlinedFun, D);
1593 WrapperFunctionsMap[OutlinedFun] = WrapperFun;
1594 }
1595
1596 return OutlinedFun;
1597}
1598
1599/// Get list of lastprivate variables from the teams distribute ... or
1600/// teams {distribute ...} directives.
1601static void
1602getDistributeLastprivateVars(ASTContext &Ctx, const OMPExecutableDirective &D,
1603 llvm::SmallVectorImpl<const ValueDecl *> &Vars) {
1604 assert(isOpenMPTeamsDirective(D.getDirectiveKind()) &&
1605 "expected teams directive.");
1606 const OMPExecutableDirective *Dir = &D;
1607 if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
1608 if (const Stmt *S = CGOpenMPRuntime::getSingleCompoundChild(
1609 Ctx,
1610 D.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers(
1611 /*IgnoreCaptured=*/true))) {
1612 Dir = dyn_cast_or_null<OMPExecutableDirective>(S);
1613 if (Dir && !isOpenMPDistributeDirective(Dir->getDirectiveKind()))
1614 Dir = nullptr;
1615 }
1616 }
1617 if (!Dir)
1618 return;
1619 for (const auto *C : Dir->getClausesOfKind<OMPLastprivateClause>()) {
1620 for (const Expr *E : C->getVarRefs())
1621 Vars.push_back(getPrivateItem(E));
1622 }
1623}
1624
1625/// Get list of reduction variables from the teams ... directives.
1626static void
1627getTeamsReductionVars(ASTContext &Ctx, const OMPExecutableDirective &D,
1628 llvm::SmallVectorImpl<const ValueDecl *> &Vars) {
1629 assert(isOpenMPTeamsDirective(D.getDirectiveKind()) &&
1630 "expected teams directive.");
1631 for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
1632 for (const Expr *E : C->privates())
1633 Vars.push_back(getPrivateItem(E));
1634 }
1635}
1636
1637llvm::Function *CGOpenMPRuntimeGPU::emitTeamsOutlinedFunction(
1638 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
1639 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
1640 SourceLocation Loc = D.getBeginLoc();
1641
1642 const RecordDecl *GlobalizedRD = nullptr;
1643 llvm::SmallVector<const ValueDecl *, 4> LastPrivatesReductions;
1644 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *> MappedDeclsFields;
1645 unsigned WarpSize = CGM.getTarget().getGridValue(llvm::omp::GV_Warp_Size);
1646 // Globalize team reductions variable unconditionally in all modes.
1647 if (getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD)
1648 getTeamsReductionVars(CGM.getContext(), D, LastPrivatesReductions);
1649 if (getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD) {
1650 getDistributeLastprivateVars(CGM.getContext(), D, LastPrivatesReductions);
1651 if (!LastPrivatesReductions.empty()) {
1652 GlobalizedRD = ::buildRecordForGlobalizedVars(
1653 CGM.getContext(), llvm::None, LastPrivatesReductions,
1654 MappedDeclsFields, WarpSize);
1655 }
1656 } else if (!LastPrivatesReductions.empty()) {
1657 assert(!TeamAndReductions.first &&
1658 "Previous team declaration is not expected.");
1659 TeamAndReductions.first = D.getCapturedStmt(OMPD_teams)->getCapturedDecl();
1660 std::swap(TeamAndReductions.second, LastPrivatesReductions);
1661 }
1662
1663 // Emit target region as a standalone region.
1664 class NVPTXPrePostActionTy : public PrePostActionTy {
1665 SourceLocation &Loc;
1666 const RecordDecl *GlobalizedRD;
1667 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
1668 &MappedDeclsFields;
1669
1670 public:
1671 NVPTXPrePostActionTy(
1672 SourceLocation &Loc, const RecordDecl *GlobalizedRD,
1673 llvm::SmallDenseMap<const ValueDecl *, const FieldDecl *>
1674 &MappedDeclsFields)
1675 : Loc(Loc), GlobalizedRD(GlobalizedRD),
1676 MappedDeclsFields(MappedDeclsFields) {}
1677 void Enter(CodeGenFunction &CGF) override {
1678 auto &Rt =
1679 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
1680 if (GlobalizedRD) {
1681 auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first;
1682 I->getSecond().GlobalRecord = GlobalizedRD;
1683 I->getSecond().MappedParams =
1684 std::make_unique<CodeGenFunction::OMPMapVars>();
1685 DeclToAddrMapTy &Data = I->getSecond().LocalVarData;
1686 for (const auto &Pair : MappedDeclsFields) {
1687 assert(Pair.getFirst()->isCanonicalDecl() &&
1688 "Expected canonical declaration");
1689 Data.insert(std::make_pair(Pair.getFirst(),
1690 MappedVarData(Pair.getSecond(),
1691 /*IsOnePerTeam=*/true)));
1692 }
1693 }
1694 Rt.emitGenericVarsProlog(CGF, Loc);
1695 }
1696 void Exit(CodeGenFunction &CGF) override {
1697 static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime())
1698 .emitGenericVarsEpilog(CGF);
1699 }
1700 } Action(Loc, GlobalizedRD, MappedDeclsFields);
1701 CodeGen.setAction(Action);
1702 llvm::Function *OutlinedFun = CGOpenMPRuntime::emitTeamsOutlinedFunction(
1703 D, ThreadIDVar, InnermostKind, CodeGen);
1704
1705 return OutlinedFun;
1706}
1707
1708void CGOpenMPRuntimeGPU::emitGenericVarsProlog(CodeGenFunction &CGF,
1709 SourceLocation Loc,
1710 bool WithSPMDCheck) {
1711 if (getDataSharingMode(CGM) != CGOpenMPRuntimeGPU::Generic &&
1712 getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD)
1713 return;
1714
1715 CGBuilderTy &Bld = CGF.Builder;
1716
1717 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
1718 if (I == FunctionGlobalizedDecls.end())
1719 return;
1720 if (const RecordDecl *GlobalizedVarsRecord = I->getSecond().GlobalRecord) {
1721 QualType GlobalRecTy = CGM.getContext().getRecordType(GlobalizedVarsRecord);
1722 QualType SecGlobalRecTy;
1723
1724 // Recover pointer to this function's global record. The runtime will
1725 // handle the specifics of the allocation of the memory.
1726 // Use actual memory size of the record including the padding
1727 // for alignment purposes.
1728 unsigned Alignment =
1729 CGM.getContext().getTypeAlignInChars(GlobalRecTy).getQuantity();
1730 unsigned GlobalRecordSize =
1731 CGM.getContext().getTypeSizeInChars(GlobalRecTy).getQuantity();
1732 GlobalRecordSize = llvm::alignTo(GlobalRecordSize, Alignment);
1733
1734 llvm::PointerType *GlobalRecPtrTy =
1735 CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo();
1736 llvm::Value *GlobalRecCastAddr;
1737 llvm::Value *IsTTD = nullptr;
1738 if (!IsInTTDRegion &&
1739 (WithSPMDCheck ||
1740 getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
1741 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
1742 llvm::BasicBlock *SPMDBB = CGF.createBasicBlock(".spmd");
1743 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
1744 if (I->getSecond().SecondaryGlobalRecord.hasValue()) {
1745 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
1746 llvm::Value *ThreadID = getThreadID(CGF, Loc);
1747 llvm::Value *PL = CGF.EmitRuntimeCall(
1748 OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
1749 OMPRTL___kmpc_parallel_level),
1750 {RTLoc, ThreadID});
1751 IsTTD = Bld.CreateIsNull(PL);
1752 }
1753 llvm::Value *IsSPMD = Bld.CreateIsNotNull(
1754 CGF.EmitNounwindRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
1755 CGM.getModule(), OMPRTL___kmpc_is_spmd_exec_mode)));
1756 Bld.CreateCondBr(IsSPMD, SPMDBB, NonSPMDBB);
1757 // There is no need to emit line number for unconditional branch.
1758 (void)ApplyDebugLocation::CreateEmpty(CGF);
1759 CGF.EmitBlock(SPMDBB);
1760 Address RecPtr = Address(llvm::ConstantPointerNull::get(GlobalRecPtrTy),
1761 CharUnits::fromQuantity(Alignment));
1762 CGF.EmitBranch(ExitBB);
1763 // There is no need to emit line number for unconditional branch.
1764 (void)ApplyDebugLocation::CreateEmpty(CGF);
1765 CGF.EmitBlock(NonSPMDBB);
1766 llvm::Value *Size = llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize);
1767 if (const RecordDecl *SecGlobalizedVarsRecord =
1768 I->getSecond().SecondaryGlobalRecord.getValueOr(nullptr)) {
1769 SecGlobalRecTy =
1770 CGM.getContext().getRecordType(SecGlobalizedVarsRecord);
1771
1772 // Recover pointer to this function's global record. The runtime will
1773 // handle the specifics of the allocation of the memory.
1774 // Use actual memory size of the record including the padding
1775 // for alignment purposes.
1776 unsigned Alignment =
1777 CGM.getContext().getTypeAlignInChars(SecGlobalRecTy).getQuantity();
1778 unsigned GlobalRecordSize =
1779 CGM.getContext().getTypeSizeInChars(SecGlobalRecTy).getQuantity();
1780 GlobalRecordSize = llvm::alignTo(GlobalRecordSize, Alignment);
1781 Size = Bld.CreateSelect(
1782 IsTTD, llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize), Size);
1783 }
1784 // TODO: allow the usage of shared memory to be controlled by
1785 // the user, for now, default to global.
1786 llvm::Value *GlobalRecordSizeArg[] = {
1787 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
1788 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1789 OMPBuilder.getOrCreateRuntimeFunction(
1790 CGM.getModule(), OMPRTL___kmpc_data_sharing_coalesced_push_stack),
1791 GlobalRecordSizeArg);
1792 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1793 GlobalRecValue, GlobalRecPtrTy);
1794 CGF.EmitBlock(ExitBB);
1795 auto *Phi = Bld.CreatePHI(GlobalRecPtrTy,
1796 /*NumReservedValues=*/2, "_select_stack");
1797 Phi->addIncoming(RecPtr.getPointer(), SPMDBB);
1798 Phi->addIncoming(GlobalRecCastAddr, NonSPMDBB);
1799 GlobalRecCastAddr = Phi;
1800 I->getSecond().GlobalRecordAddr = Phi;
1801 I->getSecond().IsInSPMDModeFlag = IsSPMD;
1802 } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
1803 assert(GlobalizedRecords.back().Records.size() < 2 &&
1804 "Expected less than 2 globalized records: one for target and one "
1805 "for teams.");
1806 unsigned Offset = 0;
1807 for (const RecordDecl *RD : GlobalizedRecords.back().Records) {
1808 QualType RDTy = CGM.getContext().getRecordType(RD);
1809 unsigned Alignment =
1810 CGM.getContext().getTypeAlignInChars(RDTy).getQuantity();
1811 unsigned Size = CGM.getContext().getTypeSizeInChars(RDTy).getQuantity();
1812 Offset =
1813 llvm::alignTo(llvm::alignTo(Offset, Alignment) + Size, Alignment);
1814 }
1815 unsigned Alignment =
1816 CGM.getContext().getTypeAlignInChars(GlobalRecTy).getQuantity();
1817 Offset = llvm::alignTo(Offset, Alignment);
1818 GlobalizedRecords.back().Records.push_back(GlobalizedVarsRecord);
1819 ++GlobalizedRecords.back().RegionCounter;
1820 if (GlobalizedRecords.back().Records.size() == 1) {
1821 assert(KernelStaticGlobalized &&
1822 "Kernel static pointer must be initialized already.");
1823 auto *UseSharedMemory = new llvm::GlobalVariable(
1824 CGM.getModule(), CGM.Int16Ty, /*isConstant=*/true,
1825 llvm::GlobalValue::InternalLinkage, nullptr,
1826 "_openmp_static_kernel$is_shared");
1827 UseSharedMemory->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1828 QualType Int16Ty = CGM.getContext().getIntTypeForBitwidth(
1829 /*DestWidth=*/16, /*Signed=*/0);
1830 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
1831 Address(UseSharedMemory,
1832 CGM.getContext().getTypeAlignInChars(Int16Ty)),
1833 /*Volatile=*/false, Int16Ty, Loc);
1834 auto *StaticGlobalized = new llvm::GlobalVariable(
1835 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
1836 llvm::GlobalValue::CommonLinkage, nullptr);
1837 auto *RecSize = new llvm::GlobalVariable(
1838 CGM.getModule(), CGM.SizeTy, /*isConstant=*/true,
1839 llvm::GlobalValue::InternalLinkage, nullptr,
1840 "_openmp_static_kernel$size");
1841 RecSize->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1842 llvm::Value *Ld = CGF.EmitLoadOfScalar(
1843 Address(RecSize, CGM.getSizeAlign()), /*Volatile=*/false,
1844 CGM.getContext().getSizeType(), Loc);
1845 llvm::Value *ResAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1846 KernelStaticGlobalized, CGM.VoidPtrPtrTy);
1847 llvm::Value *GlobalRecordSizeArg[] = {
1848 llvm::ConstantInt::get(
1849 CGM.Int16Ty,
1850 getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD ? 1 : 0),
1851 StaticGlobalized, Ld, IsInSharedMemory, ResAddr};
1852 CGF.EmitRuntimeCall(
1853 OMPBuilder.getOrCreateRuntimeFunction(
1854 CGM.getModule(), OMPRTL___kmpc_get_team_static_memory),
1855 GlobalRecordSizeArg);
1856 GlobalizedRecords.back().Buffer = StaticGlobalized;
1857 GlobalizedRecords.back().RecSize = RecSize;
1858 GlobalizedRecords.back().UseSharedMemory = UseSharedMemory;
1859 GlobalizedRecords.back().Loc = Loc;
1860 }
1861 assert(KernelStaticGlobalized && "Global address must be set already.");
1862 Address FrameAddr = CGF.EmitLoadOfPointer(
1863 Address(KernelStaticGlobalized, CGM.getPointerAlign()),
1864 CGM.getContext()
1865 .getPointerType(CGM.getContext().VoidPtrTy)
1866 .castAs<PointerType>());
1867 llvm::Value *GlobalRecValue =
1868 Bld.CreateConstInBoundsGEP(FrameAddr, Offset).getPointer();
1869 I->getSecond().GlobalRecordAddr = GlobalRecValue;
1870 I->getSecond().IsInSPMDModeFlag = nullptr;
1871 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1872 GlobalRecValue, CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo());
1873 } else {
1874 // TODO: allow the usage of shared memory to be controlled by
1875 // the user, for now, default to global.
1876 bool UseSharedMemory =
1877 IsInTTDRegion && GlobalRecordSize <= SharedMemorySize;
1878 llvm::Value *GlobalRecordSizeArg[] = {
1879 llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize),
1880 CGF.Builder.getInt16(UseSharedMemory ? 1 : 0)};
1881 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1882 OMPBuilder.getOrCreateRuntimeFunction(
1883 CGM.getModule(),
1884 IsInTTDRegion ? OMPRTL___kmpc_data_sharing_push_stack
1885 : OMPRTL___kmpc_data_sharing_coalesced_push_stack),
1886 GlobalRecordSizeArg);
1887 GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1888 GlobalRecValue, GlobalRecPtrTy);
1889 I->getSecond().GlobalRecordAddr = GlobalRecValue;
1890 I->getSecond().IsInSPMDModeFlag = nullptr;
1891 }
1892 LValue Base =
1893 CGF.MakeNaturalAlignPointeeAddrLValue(GlobalRecCastAddr, GlobalRecTy);
1894
1895 // Emit the "global alloca" which is a GEP from the global declaration
1896 // record using the pointer returned by the runtime.
1897 LValue SecBase;
1898 decltype(I->getSecond().LocalVarData)::const_iterator SecIt;
1899 if (IsTTD) {
1900 SecIt = I->getSecond().SecondaryLocalVarData->begin();
1901 llvm::PointerType *SecGlobalRecPtrTy =
1902 CGF.ConvertTypeForMem(SecGlobalRecTy)->getPointerTo();
1903 SecBase = CGF.MakeNaturalAlignPointeeAddrLValue(
1904 Bld.CreatePointerBitCastOrAddrSpaceCast(
1905 I->getSecond().GlobalRecordAddr, SecGlobalRecPtrTy),
1906 SecGlobalRecTy);
1907 }
1908 for (auto &Rec : I->getSecond().LocalVarData) {
1909 bool EscapedParam = I->getSecond().EscapedParameters.count(Rec.first);
1910 llvm::Value *ParValue;
1911 if (EscapedParam) {
1912 const auto *VD = cast<VarDecl>(Rec.first);
1913 LValue ParLVal =
1914 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
1915 ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc);
1916 }
1917 LValue VarAddr = CGF.EmitLValueForField(Base, Rec.second.FD);
1918 // Emit VarAddr basing on lane-id if required.
1919 QualType VarTy;
1920 if (Rec.second.IsOnePerTeam) {
1921 VarTy = Rec.second.FD->getType();
1922 } else {
1923 Address Addr = VarAddr.getAddress(CGF);
1924 llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
1925 Addr.getElementType(), Addr.getPointer(),
1926 {Bld.getInt32(0), getNVPTXLaneID(CGF)});
1927 VarTy =
1928 Rec.second.FD->getType()->castAsArrayTypeUnsafe()->getElementType();
1929 VarAddr = CGF.MakeAddrLValue(
1930 Address(Ptr, CGM.getContext().getDeclAlign(Rec.first)), VarTy,
1931 AlignmentSource::Decl);
1932 }
1933 Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
1934 if (!IsInTTDRegion &&
1935 (WithSPMDCheck ||
1936 getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
1937 assert(I->getSecond().IsInSPMDModeFlag &&
1938 "Expected unknown execution mode or required SPMD check.");
1939 if (IsTTD) {
1940 assert(SecIt->second.IsOnePerTeam &&
1941 "Secondary glob data must be one per team.");
1942 LValue SecVarAddr = CGF.EmitLValueForField(SecBase, SecIt->second.FD);
1943 VarAddr.setAddress(
1944 Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF),
1945 VarAddr.getPointer(CGF)),
1946 VarAddr.getAlignment()));
1947 Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
1948 }
1949 Address GlobalPtr = Rec.second.PrivateAddr;
1950 Address LocalAddr = CGF.CreateMemTemp(VarTy, Rec.second.FD->getName());
1951 Rec.second.PrivateAddr = Address(
1952 Bld.CreateSelect(I->getSecond().IsInSPMDModeFlag,
1953 LocalAddr.getPointer(), GlobalPtr.getPointer()),
1954 LocalAddr.getAlignment());
1955 }
1956 if (EscapedParam) {
1957 const auto *VD = cast<VarDecl>(Rec.first);
1958 CGF.EmitStoreOfScalar(ParValue, VarAddr);
1959 I->getSecond().MappedParams->setVarAddr(CGF, VD,
1960 VarAddr.getAddress(CGF));
1961 }
1962 if (IsTTD)
1963 ++SecIt;
1964 }
1965 }
1966 for (const ValueDecl *VD : I->getSecond().EscapedVariableLengthDecls) {
1967 // Recover pointer to this function's global record. The runtime will
1968 // handle the specifics of the allocation of the memory.
1969 // Use actual memory size of the record including the padding
1970 // for alignment purposes.
1971 CGBuilderTy &Bld = CGF.Builder;
1972 llvm::Value *Size = CGF.getTypeSize(VD->getType());
1973 CharUnits Align = CGM.getContext().getDeclAlign(VD);
1974 Size = Bld.CreateNUWAdd(
1975 Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1));
1976 llvm::Value *AlignVal =
1977 llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity());
1978 Size = Bld.CreateUDiv(Size, AlignVal);
1979 Size = Bld.CreateNUWMul(Size, AlignVal);
1980 // TODO: allow the usage of shared memory to be controlled by
1981 // the user, for now, default to global.
1982 llvm::Value *GlobalRecordSizeArg[] = {
1983 Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
1984 llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
1985 OMPBuilder.getOrCreateRuntimeFunction(
1986 CGM.getModule(), OMPRTL___kmpc_data_sharing_coalesced_push_stack),
1987 GlobalRecordSizeArg);
1988 llvm::Value *GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
1989 GlobalRecValue, CGF.ConvertTypeForMem(VD->getType())->getPointerTo());
1990 LValue Base = CGF.MakeAddrLValue(GlobalRecCastAddr, VD->getType(),
1991 CGM.getContext().getDeclAlign(VD),
1992 AlignmentSource::Decl);
1993 I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD),
1994 Base.getAddress(CGF));
1995 I->getSecond().EscapedVariableLengthDeclsAddrs.emplace_back(GlobalRecValue);
1996 }
1997 I->getSecond().MappedParams->apply(CGF);
1998}
1999
2000void CGOpenMPRuntimeGPU::emitGenericVarsEpilog(CodeGenFunction &CGF,
2001 bool WithSPMDCheck) {
2002 if (getDataSharingMode(CGM) != CGOpenMPRuntimeGPU::Generic &&
2003 getExecutionMode() != CGOpenMPRuntimeGPU::EM_SPMD)
2004 return;
2005
2006 const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
2007 if (I != FunctionGlobalizedDecls.end()) {
2008 I->getSecond().MappedParams->restore(CGF);
2009 if (!CGF.HaveInsertPoint())
2010 return;
2011 for (llvm::Value *Addr :
2012 llvm::reverse(I->getSecond().EscapedVariableLengthDeclsAddrs)) {
2013 CGF.EmitRuntimeCall(
2014 OMPBuilder.getOrCreateRuntimeFunction(
2015 CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
2016 Addr);
2017 }
2018 if (I->getSecond().GlobalRecordAddr) {
2019 if (!IsInTTDRegion &&
2020 (WithSPMDCheck ||
2021 getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
2022 CGBuilderTy &Bld = CGF.Builder;
2023 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
2024 llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
2025 Bld.CreateCondBr(I->getSecond().IsInSPMDModeFlag, ExitBB, NonSPMDBB);
2026 // There is no need to emit line number for unconditional branch.
2027 (void)ApplyDebugLocation::CreateEmpty(CGF);
2028 CGF.EmitBlock(NonSPMDBB);
2029 CGF.EmitRuntimeCall(
2030 OMPBuilder.getOrCreateRuntimeFunction(
2031 CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
2032 CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr));
2033 CGF.EmitBlock(ExitBB);
2034 } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
2035 assert(GlobalizedRecords.back().RegionCounter > 0 &&
2036 "region counter must be > 0.");
2037 --GlobalizedRecords.back().RegionCounter;
2038 // Emit the restore function only in the target region.
2039 if (GlobalizedRecords.back().RegionCounter == 0) {
2040 QualType Int16Ty = CGM.getContext().getIntTypeForBitwidth(
2041 /*DestWidth=*/16, /*Signed=*/0);
2042 llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
2043 Address(GlobalizedRecords.back().UseSharedMemory,
2044 CGM.getContext().getTypeAlignInChars(Int16Ty)),
2045 /*Volatile=*/false, Int16Ty, GlobalizedRecords.back().Loc);
2046 llvm::Value *Args[] = {
2047 llvm::ConstantInt::get(
2048 CGM.Int16Ty,
2049 getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD ? 1 : 0),
2050 IsInSharedMemory};
2051 CGF.EmitRuntimeCall(
2052 OMPBuilder.getOrCreateRuntimeFunction(
2053 CGM.getModule(), OMPRTL___kmpc_restore_team_static_memory),
2054 Args);
2055 }
2056 } else {
2057 CGF.EmitRuntimeCall(
2058 OMPBuilder.getOrCreateRuntimeFunction(
2059 CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
2060 I->getSecond().GlobalRecordAddr);
2061 }
2062 }
2063 }
2064}
2065
2066void CGOpenMPRuntimeGPU::emitTeamsCall(CodeGenFunction &CGF,
2067 const OMPExecutableDirective &D,
2068 SourceLocation Loc,
2069 llvm::Function *OutlinedFn,
2070 ArrayRef<llvm::Value *> CapturedVars) {
2071 if (!CGF.HaveInsertPoint())
2072 return;
2073
2074 Address ZeroAddr = CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty,
2075 /*Name=*/".zero.addr");
2076 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
2077 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
2078 OutlinedFnArgs.push_back(emitThreadIDAddress(CGF, Loc).getPointer());
2079 OutlinedFnArgs.push_back(ZeroAddr.getPointer());
2080 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
2081 emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs);
2082}
2083
2084void CGOpenMPRuntimeGPU::emitParallelCall(CodeGenFunction &CGF,
2085 SourceLocation Loc,
2086 llvm::Function *OutlinedFn,
2087 ArrayRef<llvm::Value *> CapturedVars,
2088 const Expr *IfCond) {
2089 if (!CGF.HaveInsertPoint())
2090 return;
2091
2092 auto &&ParallelGen = [this, Loc, OutlinedFn, CapturedVars,
2093 IfCond](CodeGenFunction &CGF, PrePostActionTy &Action) {
2094 CGBuilderTy &Bld = CGF.Builder;
2095 llvm::Function *WFn = WrapperFunctionsMap[OutlinedFn];
2096 llvm::Value *ID = llvm::ConstantPointerNull::get(CGM.Int8PtrTy);
2097 if (WFn) {
2098 ID = Bld.CreateBitOrPointerCast(WFn, CGM.Int8PtrTy);
2099 // Remember for post-processing in worker loop.
2100 Work.emplace_back(WFn);
2101 }
2102 llvm::Value *FnPtr = Bld.CreateBitOrPointerCast(OutlinedFn, CGM.Int8PtrTy);
2103
2104 // Create a private scope that will globalize the arguments
2105 // passed from the outside of the target region.
2106 // TODO: Is that needed?
2107 CodeGenFunction::OMPPrivateScope PrivateArgScope(CGF);
2108
2109 Address CapturedVarsAddrs = CGF.CreateDefaultAlignTempAlloca(
2110 llvm::ArrayType::get(CGM.VoidPtrTy, CapturedVars.size()),
2111 "captured_vars_addrs");
2112 // There's something to share.
2113 if (!CapturedVars.empty()) {
2114 // Prepare for parallel region. Indicate the outlined function.
2115 ASTContext &Ctx = CGF.getContext();
2116 unsigned Idx = 0;
2117 for (llvm::Value *V : CapturedVars) {
2118 Address Dst = Bld.CreateConstArrayGEP(CapturedVarsAddrs, Idx);
2119 llvm::Value *PtrV;
2120 if (V->getType()->isIntegerTy())
2121 PtrV = Bld.CreateIntToPtr(V, CGF.VoidPtrTy);
2122 else
2123 PtrV = Bld.CreatePointerBitCastOrAddrSpaceCast(V, CGF.VoidPtrTy);
2124 CGF.EmitStoreOfScalar(PtrV, Dst, /*Volatile=*/false,
2125 Ctx.getPointerType(Ctx.VoidPtrTy));
2126 ++Idx;
2127 }
2128 }
2129
2130 llvm::Value *IfCondVal = nullptr;
2131 if (IfCond)
2132 IfCondVal = Bld.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.Int32Ty,
2133 /* isSigned */ false);
2134 else
2135 IfCondVal = llvm::ConstantInt::get(CGF.Int32Ty, 1);
2136
2137 assert(IfCondVal && "Expected a value");
2138 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
2139 llvm::Value *Args[] = {
2140 RTLoc,
2141 getThreadID(CGF, Loc),
2142 IfCondVal,
2143 llvm::ConstantInt::get(CGF.Int32Ty, -1),
2144 llvm::ConstantInt::get(CGF.Int32Ty, -1),
2145 FnPtr,
2146 ID,
2147 Bld.CreateBitOrPointerCast(CapturedVarsAddrs.getPointer(),
2148 CGF.VoidPtrPtrTy),
2149 llvm::ConstantInt::get(CGM.SizeTy, CapturedVars.size())};
2150 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2151 CGM.getModule(), OMPRTL___kmpc_parallel_51),
2152 Args);
2153 };
2154
2155 RegionCodeGenTy RCG(ParallelGen);
2156 RCG(CGF);
2157}
2158
2159void CGOpenMPRuntimeGPU::syncCTAThreads(CodeGenFunction &CGF) {
2160 // Always emit simple barriers!
2161 if (!CGF.HaveInsertPoint())
2162 return;
2163 // Build call __kmpc_barrier_simple_spmd(nullptr, 0);
2164 // This function does not use parameters, so we can emit just default values.
2165 llvm::Value *Args[] = {
2166 llvm::ConstantPointerNull::get(
2167 cast<llvm::PointerType>(getIdentTyPointerTy())),
2168 llvm::ConstantInt::get(CGF.Int32Ty, /*V=*/0, /*isSigned=*/true)};
2169 CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
2170 CGM.getModule(), OMPRTL___kmpc_barrier_simple_spmd),
2171 Args);
2172}
2173
2174void CGOpenMPRuntimeGPU::emitBarrierCall(CodeGenFunction &CGF,
2175 SourceLocation Loc,
2176 OpenMPDirectiveKind Kind, bool,
2177 bool) {
2178 // Always emit simple barriers!
2179 if (!CGF.HaveInsertPoint())
2180 return;
2181 // Build call __kmpc_cancel_barrier(loc, thread_id);
2182 unsigned Flags = getDefaultFlagsForBarriers(Kind);
2183 llvm::Value *Args[] = {