1 | //===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This contains code dealing with C++ code generation of coroutines. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "CGCleanup.h" |
14 | #include "CodeGenFunction.h" |
15 | #include "llvm/ADT/ScopeExit.h" |
16 | #include "clang/AST/StmtCXX.h" |
17 | #include "clang/AST/StmtVisitor.h" |
18 | |
19 | using namespace clang; |
20 | using namespace CodeGen; |
21 | |
22 | using llvm::Value; |
23 | using llvm::BasicBlock; |
24 | |
25 | namespace { |
26 | enum class AwaitKind { Init, Normal, Yield, Final }; |
27 | static constexpr llvm::StringLiteral AwaitKindStr[] = {"init" , "await" , "yield" , |
28 | "final" }; |
29 | } |
30 | |
31 | struct clang::CodeGen::CGCoroData { |
32 | // What is the current await expression kind and how many |
33 | // await/yield expressions were encountered so far. |
34 | // These are used to generate pretty labels for await expressions in LLVM IR. |
35 | AwaitKind CurrentAwaitKind = AwaitKind::Init; |
36 | unsigned AwaitNum = 0; |
37 | unsigned YieldNum = 0; |
38 | |
39 | // How many co_return statements are in the coroutine. Used to decide whether |
40 | // we need to add co_return; equivalent at the end of the user authored body. |
41 | unsigned CoreturnCount = 0; |
42 | |
43 | // A branch to this block is emitted when coroutine needs to suspend. |
44 | llvm::BasicBlock *SuspendBB = nullptr; |
45 | |
46 | // The promise type's 'unhandled_exception' handler, if it defines one. |
47 | Stmt *ExceptionHandler = nullptr; |
48 | |
49 | // A temporary i1 alloca that stores whether 'await_resume' threw an |
50 | // exception. If it did, 'true' is stored in this variable, and the coroutine |
51 | // body must be skipped. If the promise type does not define an exception |
52 | // handler, this is null. |
53 | llvm::Value *ResumeEHVar = nullptr; |
54 | |
55 | // Stores the jump destination just before the coroutine memory is freed. |
56 | // This is the destination that every suspend point jumps to for the cleanup |
57 | // branch. |
58 | CodeGenFunction::JumpDest CleanupJD; |
59 | |
60 | // Stores the jump destination just before the final suspend. The co_return |
61 | // statements jumps to this point after calling return_xxx promise member. |
62 | CodeGenFunction::JumpDest FinalJD; |
63 | |
64 | // Stores the llvm.coro.id emitted in the function so that we can supply it |
65 | // as the first argument to coro.begin, coro.alloc and coro.free intrinsics. |
66 | // Note: llvm.coro.id returns a token that cannot be directly expressed in a |
67 | // builtin. |
68 | llvm::CallInst *CoroId = nullptr; |
69 | |
70 | // Stores the llvm.coro.begin emitted in the function so that we can replace |
71 | // all coro.frame intrinsics with direct SSA value of coro.begin that returns |
72 | // the address of the coroutine frame of the current coroutine. |
73 | llvm::CallInst *CoroBegin = nullptr; |
74 | |
75 | // Stores the last emitted coro.free for the deallocate expressions, we use it |
76 | // to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). |
77 | llvm::CallInst *LastCoroFree = nullptr; |
78 | |
79 | // If coro.id came from the builtin, remember the expression to give better |
80 | // diagnostic. If CoroIdExpr is nullptr, the coro.id was created by |
81 | // EmitCoroutineBody. |
82 | CallExpr const *CoroIdExpr = nullptr; |
83 | }; |
84 | |
85 | // Defining these here allows to keep CGCoroData private to this file. |
86 | clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} |
87 | CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} |
88 | |
89 | static void createCoroData(CodeGenFunction &CGF, |
90 | CodeGenFunction::CGCoroInfo &CurCoro, |
91 | llvm::CallInst *CoroId, |
92 | CallExpr const *CoroIdExpr = nullptr) { |
93 | if (CurCoro.Data) { |
94 | if (CurCoro.Data->CoroIdExpr) |
95 | CGF.CGM.Error(loc: CoroIdExpr->getBeginLoc(), |
96 | error: "only one __builtin_coro_id can be used in a function" ); |
97 | else if (CoroIdExpr) |
98 | CGF.CGM.Error(loc: CoroIdExpr->getBeginLoc(), |
99 | error: "__builtin_coro_id shall not be used in a C++ coroutine" ); |
100 | else |
101 | llvm_unreachable("EmitCoroutineBodyStatement called twice?" ); |
102 | |
103 | return; |
104 | } |
105 | |
106 | CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); |
107 | CurCoro.Data->CoroId = CoroId; |
108 | CurCoro.Data->CoroIdExpr = CoroIdExpr; |
109 | } |
110 | |
111 | // Synthesize a pretty name for a suspend point. |
112 | static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) { |
113 | unsigned No = 0; |
114 | switch (Kind) { |
115 | case AwaitKind::Init: |
116 | case AwaitKind::Final: |
117 | break; |
118 | case AwaitKind::Normal: |
119 | No = ++Coro.AwaitNum; |
120 | break; |
121 | case AwaitKind::Yield: |
122 | No = ++Coro.YieldNum; |
123 | break; |
124 | } |
125 | SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]); |
126 | if (No > 1) { |
127 | Twine(No).toVector(Out&: Prefix); |
128 | } |
129 | return Prefix; |
130 | } |
131 | |
132 | // Check if function can throw based on prototype noexcept, also works for |
133 | // destructors which are implicitly noexcept but can be marked noexcept(false). |
134 | static bool FunctionCanThrow(const FunctionDecl *D) { |
135 | const auto *Proto = D->getType()->getAs<FunctionProtoType>(); |
136 | if (!Proto) { |
137 | // Function proto is not found, we conservatively assume throwing. |
138 | return true; |
139 | } |
140 | return !isNoexceptExceptionSpec(Proto->getExceptionSpecType()) || |
141 | Proto->canThrow() != CT_Cannot; |
142 | } |
143 | |
144 | static bool ResumeStmtCanThrow(const Stmt *S) { |
145 | if (const auto *CE = dyn_cast<CallExpr>(Val: S)) { |
146 | const auto *Callee = CE->getDirectCallee(); |
147 | if (!Callee) |
148 | // We don't have direct callee. Conservatively assume throwing. |
149 | return true; |
150 | |
151 | if (FunctionCanThrow(D: Callee)) |
152 | return true; |
153 | |
154 | // Fall through to visit the children. |
155 | } |
156 | |
157 | if (const auto *TE = dyn_cast<CXXBindTemporaryExpr>(Val: S)) { |
158 | // Special handling of CXXBindTemporaryExpr here as calling of Dtor of the |
159 | // temporary is not part of `children()` as covered in the fall through. |
160 | // We need to mark entire statement as throwing if the destructor of the |
161 | // temporary throws. |
162 | const auto *Dtor = TE->getTemporary()->getDestructor(); |
163 | if (FunctionCanThrow(Dtor)) |
164 | return true; |
165 | |
166 | // Fall through to visit the children. |
167 | } |
168 | |
169 | for (const auto *child : S->children()) |
170 | if (ResumeStmtCanThrow(S: child)) |
171 | return true; |
172 | |
173 | return false; |
174 | } |
175 | |
176 | // Emit suspend expression which roughly looks like: |
177 | // |
178 | // auto && x = CommonExpr(); |
179 | // if (!x.await_ready()) { |
180 | // llvm_coro_save(); |
181 | // x.await_suspend(...); (*) |
182 | // llvm_coro_suspend(); (**) |
183 | // } |
184 | // x.await_resume(); |
185 | // |
186 | // where the result of the entire expression is the result of x.await_resume() |
187 | // |
188 | // (*) If x.await_suspend return type is bool, it allows to veto a suspend: |
189 | // if (x.await_suspend(...)) |
190 | // llvm_coro_suspend(); |
191 | // |
192 | // (**) llvm_coro_suspend() encodes three possible continuations as |
193 | // a switch instruction: |
194 | // |
195 | // %where-to = call i8 @llvm.coro.suspend(...) |
196 | // switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend |
197 | // i8 0, label %yield.ready ; go here when resumed |
198 | // i8 1, label %yield.cleanup ; go here when destroyed |
199 | // ] |
200 | // |
201 | // See llvm's docs/Coroutines.rst for more details. |
202 | // |
203 | namespace { |
204 | struct LValueOrRValue { |
205 | LValue LV; |
206 | RValue RV; |
207 | }; |
208 | } |
209 | static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, |
210 | CoroutineSuspendExpr const &S, |
211 | AwaitKind Kind, AggValueSlot aggSlot, |
212 | bool ignoreResult, bool forLValue) { |
213 | auto *E = S.getCommonExpr(); |
214 | |
215 | auto Binder = |
216 | CodeGenFunction::OpaqueValueMappingData::bind(CGF, ov: S.getOpaqueValue(), e: E); |
217 | auto UnbindOnExit = llvm::make_scope_exit(F: [&] { Binder.unbind(CGF); }); |
218 | |
219 | auto Prefix = buildSuspendPrefixStr(Coro, Kind); |
220 | BasicBlock *ReadyBlock = CGF.createBasicBlock(name: Prefix + Twine(".ready" )); |
221 | BasicBlock *SuspendBlock = CGF.createBasicBlock(name: Prefix + Twine(".suspend" )); |
222 | BasicBlock *CleanupBlock = CGF.createBasicBlock(name: Prefix + Twine(".cleanup" )); |
223 | |
224 | // If expression is ready, no need to suspend. |
225 | CGF.EmitBranchOnBoolExpr(Cond: S.getReadyExpr(), TrueBlock: ReadyBlock, FalseBlock: SuspendBlock, TrueCount: 0); |
226 | |
227 | // Otherwise, emit suspend logic. |
228 | CGF.EmitBlock(BB: SuspendBlock); |
229 | |
230 | auto &Builder = CGF.Builder; |
231 | llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); |
232 | auto *NullPtr = llvm::ConstantPointerNull::get(T: CGF.CGM.Int8PtrTy); |
233 | auto *SaveCall = Builder.CreateCall(Callee: CoroSave, Args: {NullPtr}); |
234 | |
235 | CGF.CurCoro.InSuspendBlock = true; |
236 | auto *SuspendRet = CGF.EmitScalarExpr(E: S.getSuspendExpr()); |
237 | CGF.CurCoro.InSuspendBlock = false; |
238 | |
239 | if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(Bitwidth: 1)) { |
240 | // Veto suspension if requested by bool returning await_suspend. |
241 | BasicBlock *RealSuspendBlock = |
242 | CGF.createBasicBlock(name: Prefix + Twine(".suspend.bool" )); |
243 | CGF.Builder.CreateCondBr(Cond: SuspendRet, True: RealSuspendBlock, False: ReadyBlock); |
244 | CGF.EmitBlock(BB: RealSuspendBlock); |
245 | } |
246 | |
247 | // Emit the suspend point. |
248 | const bool IsFinalSuspend = (Kind == AwaitKind::Final); |
249 | llvm::Function *CoroSuspend = |
250 | CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); |
251 | auto *SuspendResult = Builder.CreateCall( |
252 | Callee: CoroSuspend, Args: {SaveCall, Builder.getInt1(V: IsFinalSuspend)}); |
253 | |
254 | // Create a switch capturing three possible continuations. |
255 | auto *Switch = Builder.CreateSwitch(V: SuspendResult, Dest: Coro.SuspendBB, NumCases: 2); |
256 | Switch->addCase(OnVal: Builder.getInt8(C: 0), Dest: ReadyBlock); |
257 | Switch->addCase(OnVal: Builder.getInt8(C: 1), Dest: CleanupBlock); |
258 | |
259 | // Emit cleanup for this suspend point. |
260 | CGF.EmitBlock(BB: CleanupBlock); |
261 | CGF.EmitBranchThroughCleanup(Dest: Coro.CleanupJD); |
262 | |
263 | // Emit await_resume expression. |
264 | CGF.EmitBlock(BB: ReadyBlock); |
265 | |
266 | // Exception handling requires additional IR. If the 'await_resume' function |
267 | // is marked as 'noexcept', we avoid generating this additional IR. |
268 | CXXTryStmt *TryStmt = nullptr; |
269 | if (Coro.ExceptionHandler && Kind == AwaitKind::Init && |
270 | ResumeStmtCanThrow(S.getResumeExpr())) { |
271 | Coro.ResumeEHVar = |
272 | CGF.CreateTempAlloca(Ty: Builder.getInt1Ty(), Name: Prefix + Twine("resume.eh" )); |
273 | Builder.CreateFlagStore(Value: true, Addr: Coro.ResumeEHVar); |
274 | |
275 | auto Loc = S.getResumeExpr()->getExprLoc(); |
276 | auto *Catch = new (CGF.getContext()) |
277 | CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler); |
278 | auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(), |
279 | FPOptionsOverride(), Loc, Loc); |
280 | TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch); |
281 | CGF.EnterCXXTryStmt(S: *TryStmt); |
282 | CGF.EmitStmt(S: TryBody); |
283 | // We don't use EmitCXXTryStmt here. We need to store to ResumeEHVar that |
284 | // doesn't exist in the body. |
285 | Builder.CreateFlagStore(Value: false, Addr: Coro.ResumeEHVar); |
286 | CGF.ExitCXXTryStmt(S: *TryStmt); |
287 | LValueOrRValue Res; |
288 | // We are not supposed to obtain the value from init suspend await_resume(). |
289 | Res.RV = RValue::getIgnored(); |
290 | return Res; |
291 | } |
292 | |
293 | LValueOrRValue Res; |
294 | if (forLValue) |
295 | Res.LV = CGF.EmitLValue(E: S.getResumeExpr()); |
296 | else |
297 | Res.RV = CGF.EmitAnyExpr(E: S.getResumeExpr(), aggSlot, ignoreResult); |
298 | |
299 | return Res; |
300 | } |
301 | |
302 | RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, |
303 | AggValueSlot aggSlot, |
304 | bool ignoreResult) { |
305 | return emitSuspendExpression(*this, *CurCoro.Data, E, |
306 | CurCoro.Data->CurrentAwaitKind, aggSlot, |
307 | ignoreResult, /*forLValue*/false).RV; |
308 | } |
309 | RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, |
310 | AggValueSlot aggSlot, |
311 | bool ignoreResult) { |
312 | return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, |
313 | aggSlot, ignoreResult, /*forLValue*/false).RV; |
314 | } |
315 | |
316 | void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { |
317 | ++CurCoro.Data->CoreturnCount; |
318 | const Expr *RV = S.getOperand(); |
319 | if (RV && RV->getType()->isVoidType() && !isa<InitListExpr>(Val: RV)) { |
320 | // Make sure to evaluate the non initlist expression of a co_return |
321 | // with a void expression for side effects. |
322 | RunCleanupsScope cleanupScope(*this); |
323 | EmitIgnoredExpr(E: RV); |
324 | } |
325 | EmitStmt(S.getPromiseCall()); |
326 | EmitBranchThroughCleanup(Dest: CurCoro.Data->FinalJD); |
327 | } |
328 | |
329 | |
330 | #ifndef NDEBUG |
331 | static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, |
332 | const CoroutineSuspendExpr *E) { |
333 | const auto *RE = E->getResumeExpr(); |
334 | // Is it possible for RE to be a CXXBindTemporaryExpr wrapping |
335 | // a MemberCallExpr? |
336 | assert(isa<CallExpr>(RE) && "unexpected suspend expression type" ); |
337 | return cast<CallExpr>(Val: RE)->getCallReturnType(Ctx); |
338 | } |
339 | #endif |
340 | |
341 | LValue |
342 | CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { |
343 | assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
344 | "Can't have a scalar return unless the return type is a " |
345 | "reference type!" ); |
346 | return emitSuspendExpression(*this, *CurCoro.Data, *E, |
347 | CurCoro.Data->CurrentAwaitKind, AggValueSlot::ignored(), |
348 | /*ignoreResult*/false, /*forLValue*/true).LV; |
349 | } |
350 | |
351 | LValue |
352 | CodeGenFunction::EmitCoyieldLValue(const CoyieldExpr *E) { |
353 | assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
354 | "Can't have a scalar return unless the return type is a " |
355 | "reference type!" ); |
356 | return emitSuspendExpression(*this, *CurCoro.Data, *E, |
357 | AwaitKind::Yield, AggValueSlot::ignored(), |
358 | /*ignoreResult*/false, /*forLValue*/true).LV; |
359 | } |
360 | |
361 | // Hunts for the parameter reference in the parameter copy/move declaration. |
362 | namespace { |
363 | struct GetParamRef : public StmtVisitor<GetParamRef> { |
364 | public: |
365 | DeclRefExpr *Expr = nullptr; |
366 | GetParamRef() {} |
367 | void VisitDeclRefExpr(DeclRefExpr *E) { |
368 | assert(Expr == nullptr && "multilple declref in param move" ); |
369 | Expr = E; |
370 | } |
371 | void VisitStmt(Stmt *S) { |
372 | for (auto *C : S->children()) { |
373 | if (C) |
374 | Visit(C); |
375 | } |
376 | } |
377 | }; |
378 | } |
379 | |
380 | // This class replaces references to parameters to their copies by changing |
381 | // the addresses in CGF.LocalDeclMap and restoring back the original values in |
382 | // its destructor. |
383 | |
384 | namespace { |
385 | struct ParamReferenceReplacerRAII { |
386 | CodeGenFunction::DeclMapTy SavedLocals; |
387 | CodeGenFunction::DeclMapTy& LocalDeclMap; |
388 | |
389 | ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap) |
390 | : LocalDeclMap(LocalDeclMap) {} |
391 | |
392 | void addCopy(DeclStmt const *PM) { |
393 | // Figure out what param it refers to. |
394 | |
395 | assert(PM->isSingleDecl()); |
396 | VarDecl const*VD = static_cast<VarDecl const*>(PM->getSingleDecl()); |
397 | Expr const *InitExpr = VD->getInit(); |
398 | GetParamRef Visitor; |
399 | Visitor.Visit(const_cast<Expr*>(InitExpr)); |
400 | assert(Visitor.Expr); |
401 | DeclRefExpr *DREOrig = Visitor.Expr; |
402 | auto *PD = DREOrig->getDecl(); |
403 | |
404 | auto it = LocalDeclMap.find(PD); |
405 | assert(it != LocalDeclMap.end() && "parameter is not found" ); |
406 | SavedLocals.insert({ PD, it->second }); |
407 | |
408 | auto copyIt = LocalDeclMap.find(VD); |
409 | assert(copyIt != LocalDeclMap.end() && "parameter copy is not found" ); |
410 | it->second = copyIt->getSecond(); |
411 | } |
412 | |
413 | ~ParamReferenceReplacerRAII() { |
414 | for (auto&& SavedLocal : SavedLocals) { |
415 | LocalDeclMap.insert(KV: {SavedLocal.first, SavedLocal.second}); |
416 | } |
417 | } |
418 | }; |
419 | } |
420 | |
421 | // For WinEH exception representation backend needs to know what funclet coro.end |
422 | // belongs to. That information is passed in a funclet bundle. |
423 | static SmallVector<llvm::OperandBundleDef, 1> |
424 | getBundlesForCoroEnd(CodeGenFunction &CGF) { |
425 | SmallVector<llvm::OperandBundleDef, 1> BundleList; |
426 | |
427 | if (llvm::Instruction *EHPad = CGF.CurrentFuncletPad) |
428 | BundleList.emplace_back(Args: "funclet" , Args&: EHPad); |
429 | |
430 | return BundleList; |
431 | } |
432 | |
433 | namespace { |
434 | // We will insert coro.end to cut any of the destructors for objects that |
435 | // do not need to be destroyed once the coroutine is resumed. |
436 | // See llvm/docs/Coroutines.rst for more details about coro.end. |
437 | struct CallCoroEnd final : public EHScopeStack::Cleanup { |
438 | void Emit(CodeGenFunction &CGF, Flags flags) override { |
439 | auto &CGM = CGF.CGM; |
440 | auto *NullPtr = llvm::ConstantPointerNull::get(T: CGF.Int8PtrTy); |
441 | llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
442 | // See if we have a funclet bundle to associate coro.end with. (WinEH) |
443 | auto Bundles = getBundlesForCoroEnd(CGF); |
444 | auto *CoroEnd = |
445 | CGF.Builder.CreateCall(Callee: CoroEndFn, |
446 | Args: {NullPtr, CGF.Builder.getTrue(), |
447 | llvm::ConstantTokenNone::get(Context&: CoroEndFn->getContext())}, |
448 | OpBundles: Bundles); |
449 | if (Bundles.empty()) { |
450 | // Otherwise, (landingpad model), create a conditional branch that leads |
451 | // either to a cleanup block or a block with EH resume instruction. |
452 | auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true); |
453 | auto *CleanupContBB = CGF.createBasicBlock(name: "cleanup.cont" ); |
454 | CGF.Builder.CreateCondBr(Cond: CoroEnd, True: ResumeBB, False: CleanupContBB); |
455 | CGF.EmitBlock(BB: CleanupContBB); |
456 | } |
457 | } |
458 | }; |
459 | } |
460 | |
461 | namespace { |
462 | // Make sure to call coro.delete on scope exit. |
463 | struct CallCoroDelete final : public EHScopeStack::Cleanup { |
464 | Stmt *Deallocate; |
465 | |
466 | // Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;" |
467 | |
468 | // Note: That deallocation will be emitted twice: once for a normal exit and |
469 | // once for exceptional exit. This usage is safe because Deallocate does not |
470 | // contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr() |
471 | // builds a single call to a deallocation function which is safe to emit |
472 | // multiple times. |
473 | void Emit(CodeGenFunction &CGF, Flags) override { |
474 | // Remember the current point, as we are going to emit deallocation code |
475 | // first to get to coro.free instruction that is an argument to a delete |
476 | // call. |
477 | BasicBlock *SaveInsertBlock = CGF.Builder.GetInsertBlock(); |
478 | |
479 | auto *FreeBB = CGF.createBasicBlock(name: "coro.free" ); |
480 | CGF.EmitBlock(BB: FreeBB); |
481 | CGF.EmitStmt(S: Deallocate); |
482 | |
483 | auto *AfterFreeBB = CGF.createBasicBlock(name: "after.coro.free" ); |
484 | CGF.EmitBlock(BB: AfterFreeBB); |
485 | |
486 | // We should have captured coro.free from the emission of deallocate. |
487 | auto *CoroFree = CGF.CurCoro.Data->LastCoroFree; |
488 | if (!CoroFree) { |
489 | CGF.CGM.Error(loc: Deallocate->getBeginLoc(), |
490 | error: "Deallocation expressoin does not refer to coro.free" ); |
491 | return; |
492 | } |
493 | |
494 | // Get back to the block we were originally and move coro.free there. |
495 | auto *InsertPt = SaveInsertBlock->getTerminator(); |
496 | CoroFree->moveBefore(MovePos: InsertPt); |
497 | CGF.Builder.SetInsertPoint(InsertPt); |
498 | |
499 | // Add if (auto *mem = coro.free) Deallocate; |
500 | auto *NullPtr = llvm::ConstantPointerNull::get(T: CGF.Int8PtrTy); |
501 | auto *Cond = CGF.Builder.CreateICmpNE(LHS: CoroFree, RHS: NullPtr); |
502 | CGF.Builder.CreateCondBr(Cond, True: FreeBB, False: AfterFreeBB); |
503 | |
504 | // No longer need old terminator. |
505 | InsertPt->eraseFromParent(); |
506 | CGF.Builder.SetInsertPoint(AfterFreeBB); |
507 | } |
508 | explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {} |
509 | }; |
510 | } |
511 | |
512 | namespace { |
513 | struct GetReturnObjectManager { |
514 | CodeGenFunction &CGF; |
515 | CGBuilderTy &Builder; |
516 | const CoroutineBodyStmt &S; |
517 | // When true, performs RVO for the return object. |
518 | bool DirectEmit = false; |
519 | |
520 | Address GroActiveFlag; |
521 | CodeGenFunction::AutoVarEmission GroEmission; |
522 | |
523 | GetReturnObjectManager(CodeGenFunction &CGF, const CoroutineBodyStmt &S) |
524 | : CGF(CGF), Builder(CGF.Builder), S(S), GroActiveFlag(Address::invalid()), |
525 | GroEmission(CodeGenFunction::AutoVarEmission::invalid()) { |
526 | // The call to get_Âreturn_Âobject is sequenced before the call to |
527 | // initial_Âsuspend and is invoked at most once, but there are caveats |
528 | // regarding on whether the prvalue result object may be initialized |
529 | // directly/eager or delayed, depending on the types involved. |
530 | // |
531 | // More info at https://github.com/cplusplus/papers/issues/1414 |
532 | // |
533 | // The general cases: |
534 | // 1. Same type of get_return_object and coroutine return type (direct |
535 | // emission): |
536 | // - Constructed in the return slot. |
537 | // 2. Different types (delayed emission): |
538 | // - Constructed temporary object prior to initial suspend initialized with |
539 | // a call to get_return_object() |
540 | // - When coroutine needs to to return to the caller and needs to construct |
541 | // return value for the coroutine it is initialized with expiring value of |
542 | // the temporary obtained above. |
543 | // |
544 | // Direct emission for void returning coroutines or GROs. |
545 | DirectEmit = [&]() { |
546 | auto *RVI = S.getReturnValueInit(); |
547 | assert(RVI && "expected RVI" ); |
548 | auto GroType = RVI->getType(); |
549 | return CGF.getContext().hasSameType(GroType, CGF.FnRetTy); |
550 | }(); |
551 | } |
552 | |
553 | // The gro variable has to outlive coroutine frame and coroutine promise, but, |
554 | // it can only be initialized after coroutine promise was created, thus, we |
555 | // split its emission in two parts. EmitGroAlloca emits an alloca and sets up |
556 | // cleanups. Later when coroutine promise is available we initialize the gro |
557 | // and sets the flag that the cleanup is now active. |
558 | void EmitGroAlloca() { |
559 | if (DirectEmit) |
560 | return; |
561 | |
562 | auto *GroDeclStmt = dyn_cast_or_null<DeclStmt>(Val: S.getResultDecl()); |
563 | if (!GroDeclStmt) { |
564 | // If get_return_object returns void, no need to do an alloca. |
565 | return; |
566 | } |
567 | |
568 | auto *GroVarDecl = cast<VarDecl>(Val: GroDeclStmt->getSingleDecl()); |
569 | |
570 | // Set GRO flag that it is not initialized yet |
571 | GroActiveFlag = CGF.CreateTempAlloca(Ty: Builder.getInt1Ty(), align: CharUnits::One(), |
572 | Name: "gro.active" ); |
573 | Builder.CreateStore(Val: Builder.getFalse(), Addr: GroActiveFlag); |
574 | |
575 | GroEmission = CGF.EmitAutoVarAlloca(var: *GroVarDecl); |
576 | auto *GroAlloca = dyn_cast_or_null<llvm::AllocaInst>( |
577 | Val: GroEmission.getOriginalAllocatedAddress().getPointer()); |
578 | assert(GroAlloca && "expected alloca to be emitted" ); |
579 | GroAlloca->setMetadata(KindID: llvm::LLVMContext::MD_coro_outside_frame, |
580 | Node: llvm::MDNode::get(Context&: CGF.CGM.getLLVMContext(), MDs: {})); |
581 | |
582 | // Remember the top of EHStack before emitting the cleanup. |
583 | auto old_top = CGF.EHStack.stable_begin(); |
584 | CGF.EmitAutoVarCleanups(emission: GroEmission); |
585 | auto top = CGF.EHStack.stable_begin(); |
586 | |
587 | // Make the cleanup conditional on gro.active |
588 | for (auto b = CGF.EHStack.find(sp: top), e = CGF.EHStack.find(sp: old_top); b != e; |
589 | b++) { |
590 | if (auto *Cleanup = dyn_cast<EHCleanupScope>(Val: &*b)) { |
591 | assert(!Cleanup->hasActiveFlag() && "cleanup already has active flag?" ); |
592 | Cleanup->setActiveFlag(GroActiveFlag); |
593 | Cleanup->setTestFlagInEHCleanup(); |
594 | Cleanup->setTestFlagInNormalCleanup(); |
595 | } |
596 | } |
597 | } |
598 | |
599 | void EmitGroInit() { |
600 | if (DirectEmit) { |
601 | // ReturnValue should be valid as long as the coroutine's return type |
602 | // is not void. The assertion could help us to reduce the check later. |
603 | assert(CGF.ReturnValue.isValid() == (bool)S.getReturnStmt()); |
604 | // Now we have the promise, initialize the GRO. |
605 | // We need to emit `get_return_object` first. According to: |
606 | // [dcl.fct.def.coroutine]p7 |
607 | // The call to get_return_Âobject is sequenced before the call to |
608 | // initial_suspend and is invoked at most once. |
609 | // |
610 | // So we couldn't emit return value when we emit return statment, |
611 | // otherwise the call to get_return_object wouldn't be in front |
612 | // of initial_suspend. |
613 | if (CGF.ReturnValue.isValid()) { |
614 | CGF.EmitAnyExprToMem(E: S.getReturnValue(), Location: CGF.ReturnValue, |
615 | Quals: S.getReturnValue()->getType().getQualifiers(), |
616 | /*IsInit*/ IsInitializer: true); |
617 | } |
618 | return; |
619 | } |
620 | |
621 | if (!GroActiveFlag.isValid()) { |
622 | // No Gro variable was allocated. Simply emit the call to |
623 | // get_return_object. |
624 | CGF.EmitStmt(S: S.getResultDecl()); |
625 | return; |
626 | } |
627 | |
628 | CGF.EmitAutoVarInit(emission: GroEmission); |
629 | Builder.CreateStore(Val: Builder.getTrue(), Addr: GroActiveFlag); |
630 | } |
631 | }; |
632 | } // namespace |
633 | |
634 | static void emitBodyAndFallthrough(CodeGenFunction &CGF, |
635 | const CoroutineBodyStmt &S, Stmt *Body) { |
636 | CGF.EmitStmt(S: Body); |
637 | const bool CanFallthrough = CGF.Builder.GetInsertBlock(); |
638 | if (CanFallthrough) |
639 | if (Stmt *OnFallthrough = S.getFallthroughHandler()) |
640 | CGF.EmitStmt(S: OnFallthrough); |
641 | } |
642 | |
643 | void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { |
644 | auto *NullPtr = llvm::ConstantPointerNull::get(T: Builder.getPtrTy()); |
645 | auto &TI = CGM.getContext().getTargetInfo(); |
646 | unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); |
647 | |
648 | auto *EntryBB = Builder.GetInsertBlock(); |
649 | auto *AllocBB = createBasicBlock(name: "coro.alloc" ); |
650 | auto *InitBB = createBasicBlock(name: "coro.init" ); |
651 | auto *FinalBB = createBasicBlock(name: "coro.final" ); |
652 | auto *RetBB = createBasicBlock(name: "coro.ret" ); |
653 | |
654 | auto *CoroId = Builder.CreateCall( |
655 | CGM.getIntrinsic(llvm::Intrinsic::coro_id), |
656 | {Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); |
657 | createCoroData(*this, CurCoro, CoroId); |
658 | CurCoro.Data->SuspendBB = RetBB; |
659 | assert(ShouldEmitLifetimeMarkers && |
660 | "Must emit lifetime intrinsics for coroutines" ); |
661 | |
662 | // Backend is allowed to elide memory allocations, to help it, emit |
663 | // auto mem = coro.alloc() ? 0 : ... allocation code ...; |
664 | auto *CoroAlloc = Builder.CreateCall( |
665 | CGM.getIntrinsic(llvm::Intrinsic::coro_alloc), {CoroId}); |
666 | |
667 | Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB); |
668 | |
669 | EmitBlock(BB: AllocBB); |
670 | auto *AllocateCall = EmitScalarExpr(E: S.getAllocate()); |
671 | auto *AllocOrInvokeContBB = Builder.GetInsertBlock(); |
672 | |
673 | // Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided. |
674 | if (auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) { |
675 | auto *RetOnFailureBB = createBasicBlock(name: "coro.ret.on.failure" ); |
676 | |
677 | // See if allocation was successful. |
678 | auto *NullPtr = llvm::ConstantPointerNull::get(T: Int8PtrTy); |
679 | auto *Cond = Builder.CreateICmpNE(LHS: AllocateCall, RHS: NullPtr); |
680 | // Expect the allocation to be successful. |
681 | emitCondLikelihoodViaExpectIntrinsic(Cond, LH: Stmt::LH_Likely); |
682 | Builder.CreateCondBr(Cond, True: InitBB, False: RetOnFailureBB); |
683 | |
684 | // If not, return OnAllocFailure object. |
685 | EmitBlock(BB: RetOnFailureBB); |
686 | EmitStmt(S: RetOnAllocFailure); |
687 | } |
688 | else { |
689 | Builder.CreateBr(Dest: InitBB); |
690 | } |
691 | |
692 | EmitBlock(BB: InitBB); |
693 | |
694 | // Pass the result of the allocation to coro.begin. |
695 | auto *Phi = Builder.CreatePHI(Ty: VoidPtrTy, NumReservedValues: 2); |
696 | Phi->addIncoming(V: NullPtr, BB: EntryBB); |
697 | Phi->addIncoming(V: AllocateCall, BB: AllocOrInvokeContBB); |
698 | auto *CoroBegin = Builder.CreateCall( |
699 | CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi}); |
700 | CurCoro.Data->CoroBegin = CoroBegin; |
701 | |
702 | GetReturnObjectManager GroManager(*this, S); |
703 | GroManager.EmitGroAlloca(); |
704 | |
705 | CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(Target: RetBB); |
706 | { |
707 | CGDebugInfo *DI = getDebugInfo(); |
708 | ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap); |
709 | CodeGenFunction::RunCleanupsScope ResumeScope(*this); |
710 | EHStack.pushCleanup<CallCoroDelete>(Kind: NormalAndEHCleanup, A: S.getDeallocate()); |
711 | |
712 | // Create mapping between parameters and copy-params for coroutine function. |
713 | llvm::ArrayRef<const Stmt *> ParamMoves = S.getParamMoves(); |
714 | assert( |
715 | (ParamMoves.size() == 0 || (ParamMoves.size() == FnArgs.size())) && |
716 | "ParamMoves and FnArgs should be the same size for coroutine function" ); |
717 | if (ParamMoves.size() == FnArgs.size() && DI) |
718 | for (const auto Pair : llvm::zip(t&: FnArgs, u&: ParamMoves)) |
719 | DI->getCoroutineParameterMappings().insert( |
720 | KV: {std::get<0>(t: Pair), std::get<1>(t: Pair)}); |
721 | |
722 | // Create parameter copies. We do it before creating a promise, since an |
723 | // evolution of coroutine TS may allow promise constructor to observe |
724 | // parameter copies. |
725 | for (auto *PM : S.getParamMoves()) { |
726 | EmitStmt(S: PM); |
727 | ParamReplacer.addCopy(PM: cast<DeclStmt>(Val: PM)); |
728 | // TODO: if(CoroParam(...)) need to surround ctor and dtor |
729 | // for the copy, so that llvm can elide it if the copy is |
730 | // not needed. |
731 | } |
732 | |
733 | EmitStmt(S: S.getPromiseDeclStmt()); |
734 | |
735 | Address PromiseAddr = GetAddrOfLocalVar(VD: S.getPromiseDecl()); |
736 | auto *PromiseAddrVoidPtr = |
737 | new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "" , CoroId); |
738 | // Update CoroId to refer to the promise. We could not do it earlier because |
739 | // promise local variable was not emitted yet. |
740 | CoroId->setArgOperand(1, PromiseAddrVoidPtr); |
741 | |
742 | // Now we have the promise, initialize the GRO |
743 | GroManager.EmitGroInit(); |
744 | |
745 | EHStack.pushCleanup<CallCoroEnd>(Kind: EHCleanup); |
746 | |
747 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Init; |
748 | CurCoro.Data->ExceptionHandler = S.getExceptionHandler(); |
749 | EmitStmt(S: S.getInitSuspendStmt()); |
750 | CurCoro.Data->FinalJD = getJumpDestInCurrentScope(Target: FinalBB); |
751 | |
752 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal; |
753 | |
754 | if (CurCoro.Data->ExceptionHandler) { |
755 | // If we generated IR to record whether an exception was thrown from |
756 | // 'await_resume', then use that IR to determine whether the coroutine |
757 | // body should be skipped. |
758 | // If we didn't generate the IR (perhaps because 'await_resume' was marked |
759 | // as 'noexcept'), then we skip this check. |
760 | BasicBlock *ContBB = nullptr; |
761 | if (CurCoro.Data->ResumeEHVar) { |
762 | BasicBlock *BodyBB = createBasicBlock(name: "coro.resumed.body" ); |
763 | ContBB = createBasicBlock(name: "coro.resumed.cont" ); |
764 | Value *SkipBody = Builder.CreateFlagLoad(Addr: CurCoro.Data->ResumeEHVar, |
765 | Name: "coro.resumed.eh" ); |
766 | Builder.CreateCondBr(Cond: SkipBody, True: ContBB, False: BodyBB); |
767 | EmitBlock(BB: BodyBB); |
768 | } |
769 | |
770 | auto Loc = S.getBeginLoc(); |
771 | CXXCatchStmt Catch(Loc, /*exDecl=*/nullptr, |
772 | CurCoro.Data->ExceptionHandler); |
773 | auto *TryStmt = |
774 | CXXTryStmt::Create(C: getContext(), tryLoc: Loc, tryBlock: S.getBody(), handlers: &Catch); |
775 | |
776 | EnterCXXTryStmt(S: *TryStmt); |
777 | emitBodyAndFallthrough(*this, S, TryStmt->getTryBlock()); |
778 | ExitCXXTryStmt(S: *TryStmt); |
779 | |
780 | if (ContBB) |
781 | EmitBlock(BB: ContBB); |
782 | } |
783 | else { |
784 | emitBodyAndFallthrough(*this, S, S.getBody()); |
785 | } |
786 | |
787 | // See if we need to generate final suspend. |
788 | const bool CanFallthrough = Builder.GetInsertBlock(); |
789 | const bool HasCoreturns = CurCoro.Data->CoreturnCount > 0; |
790 | if (CanFallthrough || HasCoreturns) { |
791 | EmitBlock(BB: FinalBB); |
792 | CurCoro.Data->CurrentAwaitKind = AwaitKind::Final; |
793 | EmitStmt(S: S.getFinalSuspendStmt()); |
794 | } else { |
795 | // We don't need FinalBB. Emit it to make sure the block is deleted. |
796 | EmitBlock(BB: FinalBB, /*IsFinished=*/true); |
797 | } |
798 | } |
799 | |
800 | EmitBlock(BB: RetBB); |
801 | // Emit coro.end before getReturnStmt (and parameter destructors), since |
802 | // resume and destroy parts of the coroutine should not include them. |
803 | llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
804 | Builder.CreateCall(Callee: CoroEnd, |
805 | Args: {NullPtr, Builder.getFalse(), |
806 | llvm::ConstantTokenNone::get(Context&: CoroEnd->getContext())}); |
807 | |
808 | if (Stmt *Ret = S.getReturnStmt()) { |
809 | // Since we already emitted the return value above, so we shouldn't |
810 | // emit it again here. |
811 | if (GroManager.DirectEmit) |
812 | cast<ReturnStmt>(Val: Ret)->setRetValue(nullptr); |
813 | EmitStmt(S: Ret); |
814 | } |
815 | |
816 | // LLVM require the frontend to mark the coroutine. |
817 | CurFn->setPresplitCoroutine(); |
818 | |
819 | if (CXXRecordDecl *RD = FnRetTy->getAsCXXRecordDecl(); |
820 | RD && RD->hasAttr<CoroOnlyDestroyWhenCompleteAttr>()) |
821 | CurFn->setCoroDestroyOnlyWhenComplete(); |
822 | } |
823 | |
824 | // Emit coroutine intrinsic and patch up arguments of the token type. |
825 | RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, |
826 | unsigned int IID) { |
827 | SmallVector<llvm::Value *, 8> Args; |
828 | switch (IID) { |
829 | default: |
830 | break; |
831 | // The coro.frame builtin is replaced with an SSA value of the coro.begin |
832 | // intrinsic. |
833 | case llvm::Intrinsic::coro_frame: { |
834 | if (CurCoro.Data && CurCoro.Data->CoroBegin) { |
835 | return RValue::get(V: CurCoro.Data->CoroBegin); |
836 | } |
837 | CGM.Error(loc: E->getBeginLoc(), error: "this builtin expect that __builtin_coro_begin " |
838 | "has been used earlier in this function" ); |
839 | auto *NullPtr = llvm::ConstantPointerNull::get(T: Builder.getPtrTy()); |
840 | return RValue::get(V: NullPtr); |
841 | } |
842 | case llvm::Intrinsic::coro_size: { |
843 | auto &Context = getContext(); |
844 | CanQualType SizeTy = Context.getSizeType(); |
845 | llvm::IntegerType *T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy)); |
846 | llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_size, T); |
847 | return RValue::get(V: Builder.CreateCall(Callee: F)); |
848 | } |
849 | case llvm::Intrinsic::coro_align: { |
850 | auto &Context = getContext(); |
851 | CanQualType SizeTy = Context.getSizeType(); |
852 | llvm::IntegerType *T = Builder.getIntNTy(N: Context.getTypeSize(T: SizeTy)); |
853 | llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_align, T); |
854 | return RValue::get(V: Builder.CreateCall(Callee: F)); |
855 | } |
856 | // The following three intrinsics take a token parameter referring to a token |
857 | // returned by earlier call to @llvm.coro.id. Since we cannot represent it in |
858 | // builtins, we patch it up here. |
859 | case llvm::Intrinsic::coro_alloc: |
860 | case llvm::Intrinsic::coro_begin: |
861 | case llvm::Intrinsic::coro_free: { |
862 | if (CurCoro.Data && CurCoro.Data->CoroId) { |
863 | Args.push_back(Elt: CurCoro.Data->CoroId); |
864 | break; |
865 | } |
866 | CGM.Error(loc: E->getBeginLoc(), error: "this builtin expect that __builtin_coro_id has" |
867 | " been used earlier in this function" ); |
868 | // Fallthrough to the next case to add TokenNone as the first argument. |
869 | [[fallthrough]]; |
870 | } |
871 | // @llvm.coro.suspend takes a token parameter. Add token 'none' as the first |
872 | // argument. |
873 | case llvm::Intrinsic::coro_suspend: |
874 | Args.push_back(Elt: llvm::ConstantTokenNone::get(Context&: getLLVMContext())); |
875 | break; |
876 | } |
877 | for (const Expr *Arg : E->arguments()) |
878 | Args.push_back(Elt: EmitScalarExpr(E: Arg)); |
879 | // @llvm.coro.end takes a token parameter. Add token 'none' as the last |
880 | // argument. |
881 | if (IID == llvm::Intrinsic::coro_end) |
882 | Args.push_back(Elt: llvm::ConstantTokenNone::get(Context&: getLLVMContext())); |
883 | |
884 | llvm::Function *F = CGM.getIntrinsic(IID); |
885 | llvm::CallInst *Call = Builder.CreateCall(Callee: F, Args); |
886 | |
887 | // Note: The following code is to enable to emit coro.id and coro.begin by |
888 | // hand to experiment with coroutines in C. |
889 | // If we see @llvm.coro.id remember it in the CoroData. We will update |
890 | // coro.alloc, coro.begin and coro.free intrinsics to refer to it. |
891 | if (IID == llvm::Intrinsic::coro_id) { |
892 | createCoroData(CGF&: *this, CurCoro, CoroId: Call, CoroIdExpr: E); |
893 | } |
894 | else if (IID == llvm::Intrinsic::coro_begin) { |
895 | if (CurCoro.Data) |
896 | CurCoro.Data->CoroBegin = Call; |
897 | } |
898 | else if (IID == llvm::Intrinsic::coro_free) { |
899 | // Remember the last coro_free as we need it to build the conditional |
900 | // deletion of the coroutine frame. |
901 | if (CurCoro.Data) |
902 | CurCoro.Data->LastCoroFree = Call; |
903 | } |
904 | return RValue::get(V: Call); |
905 | } |
906 | |