1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/CodeGen/CGFunctionInfo.h"
20#include "clang/Frontend/FrontendDiagnostic.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
25
26using namespace clang;
27using namespace CodeGen;
28
29namespace {
30 class AtomicInfo {
31 CodeGenFunction &CGF;
32 QualType AtomicTy;
33 QualType ValueTy;
34 uint64_t AtomicSizeInBits;
35 uint64_t ValueSizeInBits;
36 CharUnits AtomicAlign;
37 CharUnits ValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
41 CGBitFieldInfo BFI;
42 public:
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .alignTo(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 VoidPtrAddr, OffsetInChars.getQuantity());
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90 VoidPtrAddr,
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92 "atomic_bitfield_base");
93 BFI = OrigBFI;
94 BFI.Offset = Offset;
95 BFI.StorageSize = AtomicSizeInBits;
96 BFI.StorageOffset += OffsetInChars;
97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98 BFI, lvalue.getType(), lvalue.getBaseInfo(),
99 lvalue.getTBAAInfo());
100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101 if (AtomicTy.isNull()) {
102 llvm::APInt Size(
103 /*numBits=*/32,
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy =
106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107 /*IndexTypeQuals=*/0);
108 }
109 AtomicAlign = ValueAlign = lvalue.getAlignment();
110 } else if (lvalue.isVectorElt()) {
111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112 ValueSizeInBits = C.getTypeSize(ValueTy);
113 AtomicTy = lvalue.getType();
114 AtomicSizeInBits = C.getTypeSize(AtomicTy);
115 AtomicAlign = ValueAlign = lvalue.getAlignment();
116 LVal = lvalue;
117 } else {
118 assert(lvalue.isExtVectorElt());
119 ValueTy = lvalue.getType();
120 ValueSizeInBits = C.getTypeSize(ValueTy);
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122 lvalue.getType(), cast<llvm::FixedVectorType>(
123 lvalue.getExtVectorAddress().getElementType())
124 ->getNumElements());
125 AtomicSizeInBits = C.getTypeSize(AtomicTy);
126 AtomicAlign = ValueAlign = lvalue.getAlignment();
127 LVal = lvalue;
128 }
129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131 }
132
133 QualType getAtomicType() const { return AtomicTy; }
134 QualType getValueType() const { return ValueTy; }
135 CharUnits getAtomicAlignment() const { return AtomicAlign; }
136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139 bool shouldUseLibcall() const { return UseLibcall; }
140 const LValue &getAtomicLValue() const { return LVal; }
141 llvm::Value *getAtomicPointer() const {
142 if (LVal.isSimple())
143 return LVal.getPointer(CGF);
144 else if (LVal.isBitField())
145 return LVal.getBitFieldPointer();
146 else if (LVal.isVectorElt())
147 return LVal.getVectorPointer();
148 assert(LVal.isExtVectorElt());
149 return LVal.getExtVectorPointer();
150 }
151 Address getAtomicAddress() const {
152 return Address(getAtomicPointer(), getAtomicAlignment());
153 }
154
155 Address getAtomicAddressAsAtomicIntPointer() const {
156 return emitCastToAtomicIntPointer(getAtomicAddress());
157 }
158
159 /// Is the atomic size larger than the underlying value type?
160 ///
161 /// Note that the absence of padding does not mean that atomic
162 /// objects are completely interchangeable with non-atomic
163 /// objects: we might have promoted the alignment of a type
164 /// without making it bigger.
165 bool hasPadding() const {
166 return (ValueSizeInBits != AtomicSizeInBits);
167 }
168
169 bool emitMemSetZeroIfNecessary() const;
170
171 llvm::Value *getAtomicSizeValue() const {
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173 return CGF.CGM.getSize(size);
174 }
175
176 /// Cast the given pointer to an integer pointer suitable for atomic
177 /// operations if the source.
178 Address emitCastToAtomicIntPointer(Address Addr) const;
179
180 /// If Addr is compatible with the iN that will be used for an atomic
181 /// operation, bitcast it. Otherwise, create a temporary that is suitable
182 /// and copy the value across.
183 Address convertToAtomicIntPointer(Address Addr) const;
184
185 /// Turn an atomic-layout object into an r-value.
186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187 SourceLocation loc, bool AsValue) const;
188
189 /// Converts a rvalue to integer value.
190 llvm::Value *convertRValueToInt(RValue RVal) const;
191
192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193 AggValueSlot ResultSlot,
194 SourceLocation Loc, bool AsValue) const;
195
196 /// Copy an atomic r-value into atomic-layout memory.
197 void emitCopyIntoMemory(RValue rvalue) const;
198
199 /// Project an l-value down to the value field.
200 LValue projectValue() const {
201 assert(LVal.isSimple());
202 Address addr = getAtomicAddress();
203 if (hasPadding())
204 addr = CGF.Builder.CreateStructGEP(addr, 0);
205
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207 LVal.getBaseInfo(), LVal.getTBAAInfo());
208 }
209
210 /// Emits atomic load.
211 /// \returns Loaded value.
212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213 bool AsValue, llvm::AtomicOrdering AO,
214 bool IsVolatile);
215
216 /// Emits atomic compare-and-exchange sequence.
217 /// \param Expected Expected value.
218 /// \param Desired Desired value.
219 /// \param Success Atomic ordering for success operation.
220 /// \param Failure Atomic ordering for failed operation.
221 /// \param IsWeak true if atomic operation is weak, false otherwise.
222 /// \returns Pair of values: previous value from storage (value type) and
223 /// boolean flag (i1 type) with true if success and false otherwise.
224 std::pair<RValue, llvm::Value *>
225 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226 llvm::AtomicOrdering Success =
227 llvm::AtomicOrdering::SequentiallyConsistent,
228 llvm::AtomicOrdering Failure =
229 llvm::AtomicOrdering::SequentiallyConsistent,
230 bool IsWeak = false);
231
232 /// Emits atomic update.
233 /// \param AO Atomic ordering.
234 /// \param UpdateOp Update operation for the current lvalue.
235 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236 const llvm::function_ref<RValue(RValue)> &UpdateOp,
237 bool IsVolatile);
238 /// Emits atomic update.
239 /// \param AO Atomic ordering.
240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241 bool IsVolatile);
242
243 /// Materialize an atomic r-value in atomic-layout memory.
244 Address materializeRValue(RValue rvalue) const;
245
246 /// Creates temp alloca for intermediate operations on atomic value.
247 Address CreateTempAlloca() const;
248 private:
249 bool requiresMemSetZero(llvm::Type *type) const;
250
251
252 /// Emits atomic load as a libcall.
253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254 llvm::AtomicOrdering AO, bool IsVolatile);
255 /// Emits atomic load as LLVM instruction.
256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257 /// Emits atomic compare-and-exchange op as a libcall.
258 llvm::Value *EmitAtomicCompareExchangeLibcall(
259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260 llvm::AtomicOrdering Success =
261 llvm::AtomicOrdering::SequentiallyConsistent,
262 llvm::AtomicOrdering Failure =
263 llvm::AtomicOrdering::SequentiallyConsistent);
264 /// Emits atomic compare-and-exchange op as LLVM instruction.
265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267 llvm::AtomicOrdering Success =
268 llvm::AtomicOrdering::SequentiallyConsistent,
269 llvm::AtomicOrdering Failure =
270 llvm::AtomicOrdering::SequentiallyConsistent,
271 bool IsWeak = false);
272 /// Emit atomic update as libcalls.
273 void
274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275 const llvm::function_ref<RValue(RValue)> &UpdateOp,
276 bool IsVolatile);
277 /// Emit atomic update as LLVM instructions.
278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279 const llvm::function_ref<RValue(RValue)> &UpdateOp,
280 bool IsVolatile);
281 /// Emit atomic update as libcalls.
282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283 bool IsVolatile);
284 /// Emit atomic update as LLVM instructions.
285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286 bool IsVolatile);
287 };
288}
289
290Address AtomicInfo::CreateTempAlloca() const {
291 Address TempAlloca = CGF.CreateMemTemp(
292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293 : AtomicTy,
294 getAtomicAlignment(),
295 "atomic-temp");
296 // Cast to pointer to value type for bitfields.
297 if (LVal.isBitField())
298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299 TempAlloca, getAtomicAddress().getType());
300 return TempAlloca;
301}
302
303static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304 StringRef fnName,
305 QualType resultType,
306 CallArgList &args) {
307 const CGFunctionInfo &fnInfo =
308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310 llvm::AttrBuilder fnAttrB;
311 fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
312 fnAttrB.addAttribute(llvm::Attribute::WillReturn);
313 llvm::AttributeList fnAttrs = llvm::AttributeList::get(
314 CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
315
316 llvm::FunctionCallee fn =
317 CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
318 auto callee = CGCallee::forDirect(fn);
319 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
320}
321
322/// Does a store of the given IR type modify the full expected width?
323static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
324 uint64_t expectedSize) {
325 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
326}
327
328/// Does the atomic type require memsetting to zero before initialization?
329///
330/// The IR type is provided as a way of making certain queries faster.
331bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
332 // If the atomic type has size padding, we definitely need a memset.
333 if (hasPadding()) return true;
334
335 // Otherwise, do some simple heuristics to try to avoid it:
336 switch (getEvaluationKind()) {
337 // For scalars and complexes, check whether the store size of the
338 // type uses the full size.
339 case TEK_Scalar:
340 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
341 case TEK_Complex:
342 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
343 AtomicSizeInBits / 2);
344
345 // Padding in structs has an undefined bit pattern. User beware.
346 case TEK_Aggregate:
347 return false;
348 }
349 llvm_unreachable("bad evaluation kind");
350}
351
352bool AtomicInfo::emitMemSetZeroIfNecessary() const {
353 assert(LVal.isSimple());
354 llvm::Value *addr = LVal.getPointer(CGF);
355 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
356 return false;
357
358 CGF.Builder.CreateMemSet(
359 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
360 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
361 LVal.getAlignment().getAsAlign());
362 return true;
363}
364
365static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
366 Address Dest, Address Ptr,
367 Address Val1, Address Val2,
368 uint64_t Size,
369 llvm::AtomicOrdering SuccessOrder,
370 llvm::AtomicOrdering FailureOrder,
371 llvm::SyncScope::ID Scope) {
372 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
373 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
374 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
375
376 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
377 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
378 Scope);
379 Pair->setVolatile(E->isVolatile());
380 Pair->setWeak(IsWeak);
381
382 // Cmp holds the result of the compare-exchange operation: true on success,
383 // false on failure.
384 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
385 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
386
387 // This basic block is used to hold the store instruction if the operation
388 // failed.
389 llvm::BasicBlock *StoreExpectedBB =
390 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
391
392 // This basic block is the exit point of the operation, we should end up
393 // here regardless of whether or not the operation succeeded.
394 llvm::BasicBlock *ContinueBB =
395 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
396
397 // Update Expected if Expected isn't equal to Old, otherwise branch to the
398 // exit point.
399 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
400
401 CGF.Builder.SetInsertPoint(StoreExpectedBB);
402 // Update the memory at Expected with Old's value.
403 CGF.Builder.CreateStore(Old, Val1);
404 // Finally, branch to the exit point.
405 CGF.Builder.CreateBr(ContinueBB);
406
407 CGF.Builder.SetInsertPoint(ContinueBB);
408 // Update the memory at Dest with Cmp's value.
409 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
410}
411
412/// Given an ordering required on success, emit all possible cmpxchg
413/// instructions to cope with the provided (but possibly only dynamically known)
414/// FailureOrder.
415static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
416 bool IsWeak, Address Dest, Address Ptr,
417 Address Val1, Address Val2,
418 llvm::Value *FailureOrderVal,
419 uint64_t Size,
420 llvm::AtomicOrdering SuccessOrder,
421 llvm::SyncScope::ID Scope) {
422 llvm::AtomicOrdering FailureOrder;
423 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
424 auto FOS = FO->getSExtValue();
425 if (!llvm::isValidAtomicOrderingCABI(FOS))
426 FailureOrder = llvm::AtomicOrdering::Monotonic;
427 else
428 switch ((llvm::AtomicOrderingCABI)FOS) {
429 case llvm::AtomicOrderingCABI::relaxed:
430 // 31.7.2.18: "The failure argument shall not be memory_order_release
431 // nor memory_order_acq_rel". Fallback to monotonic.
432 case llvm::AtomicOrderingCABI::release:
433 case llvm::AtomicOrderingCABI::acq_rel:
434 FailureOrder = llvm::AtomicOrdering::Monotonic;
435 break;
436 case llvm::AtomicOrderingCABI::consume:
437 case llvm::AtomicOrderingCABI::acquire:
438 FailureOrder = llvm::AtomicOrdering::Acquire;
439 break;
440 case llvm::AtomicOrderingCABI::seq_cst:
441 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
442 break;
443 }
444 // Prior to c++17, "the failure argument shall be no stronger than the
445 // success argument". This condition has been lifted and the only
446 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
447 // language version checks.
448 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
449 FailureOrder, Scope);
450 return;
451 }
452
453 // Create all the relevant BB's
454 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
455 *SeqCstBB = nullptr;
456 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
457 if (SuccessOrder != llvm::AtomicOrdering::Monotonic)
458 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
459 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
460 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
461
462 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
463
464 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
465
466 // Emit all the different atomics
467
468 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469 // doesn't matter unless someone is crazy enough to use something that
470 // doesn't fold to a constant for the ordering.
471 CGF.Builder.SetInsertPoint(MonotonicBB);
472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
474 CGF.Builder.CreateBr(ContBB);
475
476 if (AcquireBB) {
477 CGF.Builder.SetInsertPoint(AcquireBB);
478 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
479 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
480 CGF.Builder.CreateBr(ContBB);
481 if (SuccessOrder != llvm::AtomicOrdering::Release)
482 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
483 AcquireBB);
484 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
485 AcquireBB);
486 }
487 if (SeqCstBB) {
488 CGF.Builder.SetInsertPoint(SeqCstBB);
489 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
490 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
491 CGF.Builder.CreateBr(ContBB);
492 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
493 SeqCstBB);
494 }
495
496 CGF.Builder.SetInsertPoint(ContBB);
497}
498
499/// Duplicate the atomic min/max operation in conventional IR for the builtin
500/// variants that return the new rather than the original value.
501static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
502 AtomicExpr::AtomicOp Op,
503 bool IsSigned,
504 llvm::Value *OldVal,
505 llvm::Value *RHS) {
506 llvm::CmpInst::Predicate Pred;
507 switch (Op) {
508 default:
509 llvm_unreachable("Unexpected min/max operation");
510 case AtomicExpr::AO__atomic_max_fetch:
511 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512 break;
513 case AtomicExpr::AO__atomic_min_fetch:
514 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515 break;
516 }
517 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519}
520
521static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
522 Address Ptr, Address Val1, Address Val2,
523 llvm::Value *IsWeak, llvm::Value *FailureOrder,
524 uint64_t Size, llvm::AtomicOrdering Order,
525 llvm::SyncScope::ID Scope) {
526 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527 bool PostOpMinMax = false;
528 unsigned PostOp = 0;
529
530 switch (E->getOp()) {
531 case AtomicExpr::AO__c11_atomic_init:
532 case AtomicExpr::AO__opencl_atomic_init:
533 llvm_unreachable("Already handled!");
534
535 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
537 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
538 FailureOrder, Size, Order, Scope);
539 return;
540 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
541 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
542 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
543 FailureOrder, Size, Order, Scope);
544 return;
545 case AtomicExpr::AO__atomic_compare_exchange:
546 case AtomicExpr::AO__atomic_compare_exchange_n: {
547 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
548 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
549 Val1, Val2, FailureOrder, Size, Order, Scope);
550 } else {
551 // Create all the relevant BB's
552 llvm::BasicBlock *StrongBB =
553 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
554 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
555 llvm::BasicBlock *ContBB =
556 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
557
558 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
559 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
560
561 CGF.Builder.SetInsertPoint(StrongBB);
562 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
563 FailureOrder, Size, Order, Scope);
564 CGF.Builder.CreateBr(ContBB);
565
566 CGF.Builder.SetInsertPoint(WeakBB);
567 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
568 FailureOrder, Size, Order, Scope);
569 CGF.Builder.CreateBr(ContBB);
570
571 CGF.Builder.SetInsertPoint(ContBB);
572 }
573 return;
574 }
575 case AtomicExpr::AO__c11_atomic_load:
576 case AtomicExpr::AO__opencl_atomic_load:
577 case AtomicExpr::AO__atomic_load_n:
578 case AtomicExpr::AO__atomic_load: {
579 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
580 Load->setAtomic(Order, Scope);
581 Load->setVolatile(E->isVolatile());
582 CGF.Builder.CreateStore(Load, Dest);
583 return;
584 }
585
586 case AtomicExpr::AO__c11_atomic_store:
587 case AtomicExpr::AO__opencl_atomic_store:
588 case AtomicExpr::AO__atomic_store:
589 case AtomicExpr::AO__atomic_store_n: {
590 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
591 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
592 Store->setAtomic(Order, Scope);
593 Store->setVolatile(E->isVolatile());
594 return;
595 }
596
597 case AtomicExpr::AO__c11_atomic_exchange:
598 case AtomicExpr::AO__opencl_atomic_exchange:
599 case AtomicExpr::AO__atomic_exchange_n:
600 case AtomicExpr::AO__atomic_exchange:
601 Op = llvm::AtomicRMWInst::Xchg;
602 break;
603
604 case AtomicExpr::AO__atomic_add_fetch:
605 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
606 : llvm::Instruction::Add;
607 LLVM_FALLTHROUGH;
608 case AtomicExpr::AO__c11_atomic_fetch_add:
609 case AtomicExpr::AO__opencl_atomic_fetch_add:
610 case AtomicExpr::AO__atomic_fetch_add:
611 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
612 : llvm::AtomicRMWInst::Add;
613 break;
614
615 case AtomicExpr::AO__atomic_sub_fetch:
616 PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
617 : llvm::Instruction::Sub;
618 LLVM_FALLTHROUGH;
619 case AtomicExpr::AO__c11_atomic_fetch_sub:
620 case AtomicExpr::AO__opencl_atomic_fetch_sub:
621 case AtomicExpr::AO__atomic_fetch_sub:
622 Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
623 : llvm::AtomicRMWInst::Sub;
624 break;
625
626 case AtomicExpr::AO__atomic_min_fetch:
627 PostOpMinMax = true;
628 LLVM_FALLTHROUGH;
629 case AtomicExpr::AO__c11_atomic_fetch_min:
630 case AtomicExpr::AO__opencl_atomic_fetch_min:
631 case AtomicExpr::AO__atomic_fetch_min:
632 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
633 : llvm::AtomicRMWInst::UMin;
634 break;
635
636 case AtomicExpr::AO__atomic_max_fetch:
637 PostOpMinMax = true;
638 LLVM_FALLTHROUGH;
639 case AtomicExpr::AO__c11_atomic_fetch_max:
640 case AtomicExpr::AO__opencl_atomic_fetch_max:
641 case AtomicExpr::AO__atomic_fetch_max:
642 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
643 : llvm::AtomicRMWInst::UMax;
644 break;
645
646 case AtomicExpr::AO__atomic_and_fetch:
647 PostOp = llvm::Instruction::And;
648 LLVM_FALLTHROUGH;
649 case AtomicExpr::AO__c11_atomic_fetch_and:
650 case AtomicExpr::AO__opencl_atomic_fetch_and:
651 case AtomicExpr::AO__atomic_fetch_and:
652 Op = llvm::AtomicRMWInst::And;
653 break;
654
655 case AtomicExpr::AO__atomic_or_fetch:
656 PostOp = llvm::Instruction::Or;
657 LLVM_FALLTHROUGH;
658 case AtomicExpr::AO__c11_atomic_fetch_or:
659 case AtomicExpr::AO__opencl_atomic_fetch_or:
660 case AtomicExpr::AO__atomic_fetch_or:
661 Op = llvm::AtomicRMWInst::Or;
662 break;
663
664 case AtomicExpr::AO__atomic_xor_fetch:
665 PostOp = llvm::Instruction::Xor;
666 LLVM_FALLTHROUGH;
667 case AtomicExpr::AO__c11_atomic_fetch_xor:
668 case AtomicExpr::AO__opencl_atomic_fetch_xor:
669 case AtomicExpr::AO__atomic_fetch_xor:
670 Op = llvm::AtomicRMWInst::Xor;
671 break;
672
673 case AtomicExpr::AO__atomic_nand_fetch:
674 PostOp = llvm::Instruction::And; // the NOT is special cased below
675 LLVM_FALLTHROUGH;
676 case AtomicExpr::AO__atomic_fetch_nand:
677 Op = llvm::AtomicRMWInst::Nand;
678 break;
679 }
680
681 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
682 llvm::AtomicRMWInst *RMWI =
683 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
684 RMWI->setVolatile(E->isVolatile());
685
686 // For __atomic_*_fetch operations, perform the operation again to
687 // determine the value which was written.
688 llvm::Value *Result = RMWI;
689 if (PostOpMinMax)
690 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
691 E->getValueType()->isSignedIntegerType(),
692 RMWI, LoadVal1);
693 else if (PostOp)
694 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
695 LoadVal1);
696 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
697 Result = CGF.Builder.CreateNot(Result);
698 CGF.Builder.CreateStore(Result, Dest);
699}
700
701// This function emits any expression (scalar, complex, or aggregate)
702// into a temporary alloca.
703static Address
704EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
705 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
706 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
707 /*Init*/ true);
708 return DeclPtr;
709}
710
711static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
712 Address Ptr, Address Val1, Address Val2,
713 llvm::Value *IsWeak, llvm::Value *FailureOrder,
714 uint64_t Size, llvm::AtomicOrdering Order,
715 llvm::Value *Scope) {
716 auto ScopeModel = Expr->getScopeModel();
717
718 // LLVM atomic instructions always have synch scope. If clang atomic
719 // expression has no scope operand, use default LLVM synch scope.
720 if (!ScopeModel) {
721 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
722 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
723 return;
724 }
725
726 // Handle constant scope.
727 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
728 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
729 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
730 Order, CGF.CGM.getLLVMContext());
731 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
732 Order, SCID);
733 return;
734 }
735
736 // Handle non-constant scope.
737 auto &Builder = CGF.Builder;
738 auto Scopes = ScopeModel->getRuntimeValues();
739 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
740 for (auto S : Scopes)
741 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
742
743 llvm::BasicBlock *ContBB =
744 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
745
746 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
747 // If unsupported synch scope is encountered at run time, assume a fallback
748 // synch scope value.
749 auto FallBack = ScopeModel->getFallBackValue();
750 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
751 for (auto S : Scopes) {
752 auto *B = BB[S];
753 if (S != FallBack)
754 SI->addCase(Builder.getInt32(S), B);
755
756 Builder.SetInsertPoint(B);
757 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
758 Order,
759 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
760 ScopeModel->map(S),
761 Order,
762 CGF.getLLVMContext()));
763 Builder.CreateBr(ContBB);
764 }
765
766 Builder.SetInsertPoint(ContBB);
767}
768
769static void
770AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
771 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
772 SourceLocation Loc, CharUnits SizeInChars) {
773 if (UseOptimizedLibcall) {
774 // Load value and pass it to the function directly.
775 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
776 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
777 ValTy =
778 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
779 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
780 SizeInBits)->getPointerTo();
781 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
782 Val = CGF.EmitLoadOfScalar(Ptr, false,
783 CGF.getContext().getPointerType(ValTy),
784 Loc);
785 // Coerce the value into an appropriately sized integer type.
786 Args.add(RValue::get(Val), ValTy);
787 } else {
788 // Non-optimized functions always take a reference.
789 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
790 CGF.getContext().VoidPtrTy);
791 }
792}
793
794RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
795 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
796 QualType MemTy = AtomicTy;
797 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
798 MemTy = AT->getValueType();
799 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
800
801 Address Val1 = Address::invalid();
802 Address Val2 = Address::invalid();
803 Address Dest = Address::invalid();
804 Address Ptr = EmitPointerWithAlignment(E->getPtr());
805
806 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
807 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
808 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
809 EmitAtomicInit(E->getVal1(), lvalue);
810 return RValue::get(nullptr);
811 }
812
813 auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
814 uint64_t Size = TInfo.Width.getQuantity();
815 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
816
817 bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
818 bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
819 bool UseLibcall = Misaligned | Oversized;
820 bool ShouldCastToIntPtrTy = true;
821
822 CharUnits MaxInlineWidth =
823 getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
824
825 DiagnosticsEngine &Diags = CGM.getDiags();
826
827 if (Misaligned) {
828 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
829 << (int)TInfo.Width.getQuantity()
830 << (int)Ptr.getAlignment().getQuantity();
831 }
832
833 if (Oversized) {
834 Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
835 << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
836 }
837
838 llvm::Value *Order = EmitScalarExpr(E->getOrder());
839 llvm::Value *Scope =
840 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
841
842 switch (E->getOp()) {
843 case AtomicExpr::AO__c11_atomic_init:
844 case AtomicExpr::AO__opencl_atomic_init:
845 llvm_unreachable("Already handled above with EmitAtomicInit!");
846
847 case AtomicExpr::AO__c11_atomic_load:
848 case AtomicExpr::AO__opencl_atomic_load:
849 case AtomicExpr::AO__atomic_load_n:
850 break;
851
852 case AtomicExpr::AO__atomic_load:
853 Dest = EmitPointerWithAlignment(E->getVal1());
854 break;
855
856 case AtomicExpr::AO__atomic_store:
857 Val1 = EmitPointerWithAlignment(E->getVal1());
858 break;
859
860 case AtomicExpr::AO__atomic_exchange:
861 Val1 = EmitPointerWithAlignment(E->getVal1());
862 Dest = EmitPointerWithAlignment(E->getVal2());
863 break;
864
865 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
866 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
867 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
868 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
869 case AtomicExpr::AO__atomic_compare_exchange_n:
870 case AtomicExpr::AO__atomic_compare_exchange:
871 Val1 = EmitPointerWithAlignment(E->getVal1());
872 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
873 Val2 = EmitPointerWithAlignment(E->getVal2());
874 else
875 Val2 = EmitValToTemp(*this, E->getVal2());
876 OrderFail = EmitScalarExpr(E->getOrderFail());
877 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
878 E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
879 IsWeak = EmitScalarExpr(E->getWeak());
880 break;
881
882 case AtomicExpr::AO__c11_atomic_fetch_add:
883 case AtomicExpr::AO__c11_atomic_fetch_sub:
884 case AtomicExpr::AO__opencl_atomic_fetch_add:
885 case AtomicExpr::AO__opencl_atomic_fetch_sub:
886 if (MemTy->isPointerType()) {
887 // For pointer arithmetic, we're required to do a bit of math:
888 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
889 // ... but only for the C11 builtins. The GNU builtins expect the
890 // user to multiply by sizeof(T).
891 QualType Val1Ty = E->getVal1()->getType();
892 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
893 CharUnits PointeeIncAmt =
894 getContext().getTypeSizeInChars(MemTy->getPointeeType());
895 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
896 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
897 Val1 = Temp;
898 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
899 break;
900 }
901 LLVM_FALLTHROUGH;
902 case AtomicExpr::AO__atomic_fetch_add:
903 case AtomicExpr::AO__atomic_fetch_sub:
904 case AtomicExpr::AO__atomic_add_fetch:
905 case AtomicExpr::AO__atomic_sub_fetch:
906 ShouldCastToIntPtrTy = !MemTy->isFloatingType();
907 LLVM_FALLTHROUGH;
908
909 case AtomicExpr::AO__c11_atomic_store:
910 case AtomicExpr::AO__c11_atomic_exchange:
911 case AtomicExpr::AO__opencl_atomic_store:
912 case AtomicExpr::AO__opencl_atomic_exchange:
913 case AtomicExpr::AO__atomic_store_n:
914 case AtomicExpr::AO__atomic_exchange_n:
915 case AtomicExpr::AO__c11_atomic_fetch_and:
916 case AtomicExpr::AO__c11_atomic_fetch_or:
917 case AtomicExpr::AO__c11_atomic_fetch_xor:
918 case AtomicExpr::AO__c11_atomic_fetch_max:
919 case AtomicExpr::AO__c11_atomic_fetch_min:
920 case AtomicExpr::AO__opencl_atomic_fetch_and:
921 case AtomicExpr::AO__opencl_atomic_fetch_or:
922 case AtomicExpr::AO__opencl_atomic_fetch_xor:
923 case AtomicExpr::AO__opencl_atomic_fetch_min:
924 case AtomicExpr::AO__opencl_atomic_fetch_max:
925 case AtomicExpr::AO__atomic_fetch_and:
926 case AtomicExpr::AO__atomic_fetch_or:
927 case AtomicExpr::AO__atomic_fetch_xor:
928 case AtomicExpr::AO__atomic_fetch_nand:
929 case AtomicExpr::AO__atomic_and_fetch:
930 case AtomicExpr::AO__atomic_or_fetch:
931 case AtomicExpr::AO__atomic_xor_fetch:
932 case AtomicExpr::AO__atomic_nand_fetch:
933 case AtomicExpr::AO__atomic_max_fetch:
934 case AtomicExpr::AO__atomic_min_fetch:
935 case AtomicExpr::AO__atomic_fetch_max:
936 case AtomicExpr::AO__atomic_fetch_min:
937 Val1 = EmitValToTemp(*this, E->getVal1());
938 break;
939 }
940
941 QualType RValTy = E->getType().getUnqualifiedType();
942
943 // The inlined atomics only function on iN types, where N is a power of 2. We
944 // need to make sure (via temporaries if necessary) that all incoming values
945 // are compatible.
946 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
947 AtomicInfo Atomics(*this, AtomicVal);
948
949 if (ShouldCastToIntPtrTy) {
950 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
951 if (Val1.isValid())
952 Val1 = Atomics.convertToAtomicIntPointer(Val1);
953 if (Val2.isValid())
954 Val2 = Atomics.convertToAtomicIntPointer(Val2);
955 }
956 if (Dest.isValid()) {
957 if (ShouldCastToIntPtrTy)
958 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
959 } else if (E->isCmpXChg())
960 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
961 else if (!RValTy->isVoidType()) {
962 Dest = Atomics.CreateTempAlloca();
963 if (ShouldCastToIntPtrTy)
964 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
965 }
966
967 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
968 if (UseLibcall) {
969 bool UseOptimizedLibcall = false;
970 switch (E->getOp()) {
971 case AtomicExpr::AO__c11_atomic_init:
972 case AtomicExpr::AO__opencl_atomic_init:
973 llvm_unreachable("Already handled above with EmitAtomicInit!");
974
975 case AtomicExpr::AO__c11_atomic_fetch_add:
976 case AtomicExpr::AO__opencl_atomic_fetch_add:
977 case AtomicExpr::AO__atomic_fetch_add:
978 case AtomicExpr::AO__c11_atomic_fetch_and:
979 case AtomicExpr::AO__opencl_atomic_fetch_and:
980 case AtomicExpr::AO__atomic_fetch_and:
981 case AtomicExpr::AO__c11_atomic_fetch_or:
982 case AtomicExpr::AO__opencl_atomic_fetch_or:
983 case AtomicExpr::AO__atomic_fetch_or:
984 case AtomicExpr::AO__atomic_fetch_nand:
985 case AtomicExpr::AO__c11_atomic_fetch_sub:
986 case AtomicExpr::AO__opencl_atomic_fetch_sub:
987 case AtomicExpr::AO__atomic_fetch_sub:
988 case AtomicExpr::AO__c11_atomic_fetch_xor:
989 case AtomicExpr::AO__opencl_atomic_fetch_xor:
990 case AtomicExpr::AO__opencl_atomic_fetch_min:
991 case AtomicExpr::AO__opencl_atomic_fetch_max:
992 case AtomicExpr::AO__atomic_fetch_xor:
993 case AtomicExpr::AO__c11_atomic_fetch_max:
994 case AtomicExpr::AO__c11_atomic_fetch_min:
995 case AtomicExpr::AO__atomic_add_fetch:
996 case AtomicExpr::AO__atomic_and_fetch:
997 case AtomicExpr::AO__atomic_nand_fetch:
998 case AtomicExpr::AO__atomic_or_fetch:
999 case AtomicExpr::AO__atomic_sub_fetch:
1000 case AtomicExpr::AO__atomic_xor_fetch:
1001 case AtomicExpr::AO__atomic_fetch_max:
1002 case AtomicExpr::AO__atomic_fetch_min:
1003 case AtomicExpr::AO__atomic_max_fetch:
1004 case AtomicExpr::AO__atomic_min_fetch:
1005 // For these, only library calls for certain sizes exist.
1006 UseOptimizedLibcall = true;
1007 break;
1008
1009 case AtomicExpr::AO__atomic_load:
1010 case AtomicExpr::AO__atomic_store:
1011 case AtomicExpr::AO__atomic_exchange:
1012 case AtomicExpr::AO__atomic_compare_exchange:
1013 // Use the generic version if we don't know that the operand will be
1014 // suitably aligned for the optimized version.
1015 if (Misaligned)
1016 break;
1017 LLVM_FALLTHROUGH;
1018 case AtomicExpr::AO__c11_atomic_load:
1019 case AtomicExpr::AO__c11_atomic_store:
1020 case AtomicExpr::AO__c11_atomic_exchange:
1021 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1022 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1023 case AtomicExpr::AO__opencl_atomic_load:
1024 case AtomicExpr::AO__opencl_atomic_store:
1025 case AtomicExpr::AO__opencl_atomic_exchange:
1026 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1027 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1028 case AtomicExpr::AO__atomic_load_n:
1029 case AtomicExpr::AO__atomic_store_n:
1030 case AtomicExpr::AO__atomic_exchange_n:
1031 case AtomicExpr::AO__atomic_compare_exchange_n:
1032 // Only use optimized library calls for sizes for which they exist.
1033 // FIXME: Size == 16 optimized library functions exist too.
1034 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1035 UseOptimizedLibcall = true;
1036 break;
1037 }
1038
1039 CallArgList Args;
1040 if (!UseOptimizedLibcall) {
1041 // For non-optimized library calls, the size is the first parameter
1042 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1043 getContext().getSizeType());
1044 }
1045 // Atomic address is the first or second parameter
1046 // The OpenCL atomic library functions only accept pointer arguments to
1047 // generic address space.
1048 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1049 if (!E->isOpenCL())
1050 return V;
1051 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1052 if (AS == LangAS::opencl_generic)
1053 return V;
1054 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1055 auto T = V->getType();
1056 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
1057
1058 return getTargetHooks().performAddrSpaceCast(
1059 *this, V, AS, LangAS::opencl_generic, DestType, false);
1060 };
1061
1062 Args.add(RValue::get(CastToGenericAddrSpace(
1063 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1064 getContext().VoidPtrTy);
1065
1066 std::string LibCallName;
1067 QualType LoweredMemTy =
1068 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1069 QualType RetTy;
1070 bool HaveRetTy = false;
1071 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1072 bool PostOpMinMax = false;
1073 switch (E->getOp()) {
1074 case AtomicExpr::AO__c11_atomic_init:
1075 case AtomicExpr::AO__opencl_atomic_init:
1076 llvm_unreachable("Already handled!");
1077
1078 // There is only one libcall for compare an exchange, because there is no
1079 // optimisation benefit possible from a libcall version of a weak compare
1080 // and exchange.
1081 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1082 // void *desired, int success, int failure)
1083 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1084 // int success, int failure)
1085 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1086 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1087 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1088 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1089 case AtomicExpr::AO__atomic_compare_exchange:
1090 case AtomicExpr::AO__atomic_compare_exchange_n:
1091 LibCallName = "__atomic_compare_exchange";
1092 RetTy = getContext().BoolTy;
1093 HaveRetTy = true;
1094 Args.add(
1095 RValue::get(CastToGenericAddrSpace(
1096 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1097 getContext().VoidPtrTy);
1098 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1099 MemTy, E->getExprLoc(), TInfo.Width);
1100 Args.add(RValue::get(Order), getContext().IntTy);
1101 Order = OrderFail;
1102 break;
1103 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1104 // int order)
1105 // T __atomic_exchange_N(T *mem, T val, int order)
1106 case AtomicExpr::AO__c11_atomic_exchange:
1107 case AtomicExpr::AO__opencl_atomic_exchange:
1108 case AtomicExpr::AO__atomic_exchange_n:
1109 case AtomicExpr::AO__atomic_exchange:
1110 LibCallName = "__atomic_exchange";
1111 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1112 MemTy, E->getExprLoc(), TInfo.Width);
1113 break;
1114 // void __atomic_store(size_t size, void *mem, void *val, int order)
1115 // void __atomic_store_N(T *mem, T val, int order)
1116 case AtomicExpr::AO__c11_atomic_store:
1117 case AtomicExpr::AO__opencl_atomic_store:
1118 case AtomicExpr::AO__atomic_store:
1119 case AtomicExpr::AO__atomic_store_n:
1120 LibCallName = "__atomic_store";
1121 RetTy = getContext().VoidTy;
1122 HaveRetTy = true;
1123 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1124 MemTy, E->getExprLoc(), TInfo.Width);
1125 break;
1126 // void __atomic_load(size_t size, void *mem, void *return, int order)
1127 // T __atomic_load_N(T *mem, int order)
1128 case AtomicExpr::AO__c11_atomic_load:
1129 case AtomicExpr::AO__opencl_atomic_load:
1130 case AtomicExpr::AO__atomic_load:
1131 case AtomicExpr::AO__atomic_load_n:
1132 LibCallName = "__atomic_load";
1133 break;
1134 // T __atomic_add_fetch_N(T *mem, T val, int order)
1135 // T __atomic_fetch_add_N(T *mem, T val, int order)
1136 case AtomicExpr::AO__atomic_add_fetch:
1137 PostOp = llvm::Instruction::Add;
1138 LLVM_FALLTHROUGH;
1139 case AtomicExpr::AO__c11_atomic_fetch_add:
1140 case AtomicExpr::AO__opencl_atomic_fetch_add:
1141 case AtomicExpr::AO__atomic_fetch_add:
1142 LibCallName = "__atomic_fetch_add";
1143 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1144 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1145 break;
1146 // T __atomic_and_fetch_N(T *mem, T val, int order)
1147 // T __atomic_fetch_and_N(T *mem, T val, int order)
1148 case AtomicExpr::AO__atomic_and_fetch:
1149 PostOp = llvm::Instruction::And;
1150 LLVM_FALLTHROUGH;
1151 case AtomicExpr::AO__c11_atomic_fetch_and:
1152 case AtomicExpr::AO__opencl_atomic_fetch_and:
1153 case AtomicExpr::AO__atomic_fetch_and:
1154 LibCallName = "__atomic_fetch_and";
1155 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1156 MemTy, E->getExprLoc(), TInfo.Width);
1157 break;
1158 // T __atomic_or_fetch_N(T *mem, T val, int order)
1159 // T __atomic_fetch_or_N(T *mem, T val, int order)
1160 case AtomicExpr::AO__atomic_or_fetch:
1161 PostOp = llvm::Instruction::Or;
1162 LLVM_FALLTHROUGH;
1163 case AtomicExpr::AO__c11_atomic_fetch_or:
1164 case AtomicExpr::AO__opencl_atomic_fetch_or:
1165 case AtomicExpr::AO__atomic_fetch_or:
1166 LibCallName = "__atomic_fetch_or";
1167 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1168 MemTy, E->getExprLoc(), TInfo.Width);
1169 break;
1170 // T __atomic_sub_fetch_N(T *mem, T val, int order)
1171 // T __atomic_fetch_sub_N(T *mem, T val, int order)
1172 case AtomicExpr::AO__atomic_sub_fetch:
1173 PostOp = llvm::Instruction::Sub;
1174 LLVM_FALLTHROUGH;
1175 case AtomicExpr::AO__c11_atomic_fetch_sub:
1176 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1177 case AtomicExpr::AO__atomic_fetch_sub:
1178 LibCallName = "__atomic_fetch_sub";
1179 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1180 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1181 break;
1182 // T __atomic_xor_fetch_N(T *mem, T val, int order)
1183 // T __atomic_fetch_xor_N(T *mem, T val, int order)
1184 case AtomicExpr::AO__atomic_xor_fetch:
1185 PostOp = llvm::Instruction::Xor;
1186 LLVM_FALLTHROUGH;
1187 case AtomicExpr::AO__c11_atomic_fetch_xor:
1188 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1189 case AtomicExpr::AO__atomic_fetch_xor:
1190 LibCallName = "__atomic_fetch_xor";
1191 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1192 MemTy, E->getExprLoc(), TInfo.Width);
1193 break;
1194 case AtomicExpr::AO__atomic_min_fetch:
1195 PostOpMinMax = true;
1196 LLVM_FALLTHROUGH;
1197 case AtomicExpr::AO__c11_atomic_fetch_min:
1198 case AtomicExpr::AO__atomic_fetch_min:
1199 case AtomicExpr::AO__opencl_atomic_fetch_min:
1200 LibCallName = E->getValueType()->isSignedIntegerType()
1201 ? "__atomic_fetch_min"
1202 : "__atomic_fetch_umin";
1203 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1204 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1205 break;
1206 case AtomicExpr::AO__atomic_max_fetch:
1207 PostOpMinMax = true;
1208 LLVM_FALLTHROUGH;
1209 case AtomicExpr::AO__c11_atomic_fetch_max:
1210 case AtomicExpr::AO__atomic_fetch_max:
1211 case AtomicExpr::AO__opencl_atomic_fetch_max:
1212 LibCallName = E->getValueType()->isSignedIntegerType()
1213 ? "__atomic_fetch_max"
1214 : "__atomic_fetch_umax";
1215 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1216 LoweredMemTy, E->getExprLoc(), TInfo.Width);
1217 break;
1218 // T __atomic_nand_fetch_N(T *mem, T val, int order)
1219 // T __atomic_fetch_nand_N(T *mem, T val, int order)
1220 case AtomicExpr::AO__atomic_nand_fetch:
1221 PostOp = llvm::Instruction::And; // the NOT is special cased below
1222 LLVM_FALLTHROUGH;
1223 case AtomicExpr::AO__atomic_fetch_nand:
1224 LibCallName = "__atomic_fetch_nand";
1225 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1226 MemTy, E->getExprLoc(), TInfo.Width);
1227 break;
1228 }
1229
1230 if (E->isOpenCL()) {
1231 LibCallName = std::string("__opencl") +
1232 StringRef(LibCallName).drop_front(1).str();
1233
1234 }
1235 // Optimized functions have the size in their name.
1236 if (UseOptimizedLibcall)
1237 LibCallName += "_" + llvm::utostr(Size);
1238 // By default, assume we return a value of the atomic type.
1239 if (!HaveRetTy) {
1240 if (UseOptimizedLibcall) {
1241 // Value is returned directly.
1242 // The function returns an appropriately sized integer type.
1243 RetTy = getContext().getIntTypeForBitwidth(
1244 getContext().toBits(TInfo.Width), /*Signed=*/false);
1245 } else {
1246 // Value is returned through parameter before the order.
1247 RetTy = getContext().VoidTy;
1248 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1249 getContext().VoidPtrTy);
1250 }
1251 }
1252 // order is always the last parameter
1253 Args.add(RValue::get(Order),
1254 getContext().IntTy);
1255 if (E->isOpenCL())
1256 Args.add(RValue::get(Scope), getContext().IntTy);
1257
1258 // PostOp is only needed for the atomic_*_fetch operations, and
1259 // thus is only needed for and implemented in the
1260 // UseOptimizedLibcall codepath.
1261 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1262
1263 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1264 // The value is returned directly from the libcall.
1265 if (E->isCmpXChg())
1266 return Res;
1267
1268 // The value is returned directly for optimized libcalls but the expr
1269 // provided an out-param.
1270 if (UseOptimizedLibcall && Res.getScalarVal()) {
1271 llvm::Value *ResVal = Res.getScalarVal();
1272 if (PostOpMinMax) {
1273 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1274 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1275 E->getValueType()->isSignedIntegerType(),
1276 ResVal, LoadVal1);
1277 } else if (PostOp) {
1278 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1279 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1280 }
1281 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1282 ResVal = Builder.CreateNot(ResVal);
1283
1284 Builder.CreateStore(
1285 ResVal,
1286 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1287 }
1288
1289 if (RValTy->isVoidType())
1290 return RValue::get(nullptr);
1291
1292 return convertTempToRValue(
1293 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1294 RValTy, E->getExprLoc());
1295 }
1296
1297 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1298 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1299 E->getOp() == AtomicExpr::AO__atomic_store ||
1300 E->getOp() == AtomicExpr::AO__atomic_store_n;
1301 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1302 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1303 E->getOp() == AtomicExpr::AO__atomic_load ||
1304 E->getOp() == AtomicExpr::AO__atomic_load_n;
1305
1306 if (isa<llvm::ConstantInt>(Order)) {
1307 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1308 // We should not ever get to a case where the ordering isn't a valid C ABI
1309 // value, but it's hard to enforce that in general.
1310 if (llvm::isValidAtomicOrderingCABI(ord))
1311 switch ((llvm::AtomicOrderingCABI)ord) {
1312 case llvm::AtomicOrderingCABI::relaxed:
1313 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1314 llvm::AtomicOrdering::Monotonic, Scope);
1315 break;
1316 case llvm::AtomicOrderingCABI::consume:
1317 case llvm::AtomicOrderingCABI::acquire:
1318 if (IsStore)
1319 break; // Avoid crashing on code with undefined behavior
1320 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1321 llvm::AtomicOrdering::Acquire, Scope);
1322 break;
1323 case llvm::AtomicOrderingCABI::release:
1324 if (IsLoad)
1325 break; // Avoid crashing on code with undefined behavior
1326 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1327 llvm::AtomicOrdering::Release, Scope);
1328 break;
1329 case llvm::AtomicOrderingCABI::acq_rel:
1330 if (IsLoad || IsStore)
1331 break; // Avoid crashing on code with undefined behavior
1332 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1333 llvm::AtomicOrdering::AcquireRelease, Scope);
1334 break;
1335 case llvm::AtomicOrderingCABI::seq_cst:
1336 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1337 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1338 break;
1339 }
1340 if (RValTy->isVoidType())
1341 return RValue::get(nullptr);
1342
1343 return convertTempToRValue(
1344 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1345 Dest.getAddressSpace())),
1346 RValTy, E->getExprLoc());
1347 }
1348
1349 // Long case, when Order isn't obviously constant.
1350
1351 // Create all the relevant BB's
1352 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1353 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1354 *SeqCstBB = nullptr;
1355 MonotonicBB = createBasicBlock("monotonic", CurFn);
1356 if (!IsStore)
1357 AcquireBB = createBasicBlock("acquire", CurFn);
1358 if (!IsLoad)
1359 ReleaseBB = createBasicBlock("release", CurFn);
1360 if (!IsLoad && !IsStore)
1361 AcqRelBB = createBasicBlock("acqrel", CurFn);
1362 SeqCstBB = createBasicBlock("seqcst", CurFn);
1363 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1364
1365 // Create the switch for the split
1366 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1367 // doesn't matter unless someone is crazy enough to use something that
1368 // doesn't fold to a constant for the ordering.
1369 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1370 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1371
1372 // Emit all the different atomics
1373 Builder.SetInsertPoint(MonotonicBB);
1374 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1375 llvm::AtomicOrdering::Monotonic, Scope);
1376 Builder.CreateBr(ContBB);
1377 if (!IsStore) {
1378 Builder.SetInsertPoint(AcquireBB);
1379 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1380 llvm::AtomicOrdering::Acquire, Scope);
1381 Builder.CreateBr(ContBB);
1382 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1383 AcquireBB);
1384 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1385 AcquireBB);
1386 }
1387 if (!IsLoad) {
1388 Builder.SetInsertPoint(ReleaseBB);
1389 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1390 llvm::AtomicOrdering::Release, Scope);
1391 Builder.CreateBr(ContBB);
1392 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1393 ReleaseBB);
1394 }
1395 if (!IsLoad && !IsStore) {
1396 Builder.SetInsertPoint(AcqRelBB);
1397 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1398 llvm::AtomicOrdering::AcquireRelease, Scope);
1399 Builder.CreateBr(ContBB);
1400 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1401 AcqRelBB);
1402 }
1403 Builder.SetInsertPoint(SeqCstBB);
1404 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1405 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1406 Builder.CreateBr(ContBB);
1407 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1408 SeqCstBB);
1409
1410 // Cleanup and return
1411 Builder.SetInsertPoint(ContBB);
1412 if (RValTy->isVoidType())
1413 return RValue::get(nullptr);
1414
1415 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1416 return convertTempToRValue(
1417 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1418 Dest.getAddressSpace())),
1419 RValTy, E->getExprLoc());
1420}
1421
1422Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1423 unsigned addrspace =
1424 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1425 llvm::IntegerType *ty =
1426 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1427 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1428}
1429
1430Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1431 llvm::Type *Ty = Addr.getElementType();
1432 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1433 if (SourceSizeInBits != AtomicSizeInBits) {
1434 Address Tmp = CreateTempAlloca();
1435 CGF.Builder.CreateMemCpy(Tmp, Addr,
1436 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1437 Addr = Tmp;
1438 }
1439
1440 return emitCastToAtomicIntPointer(Addr);
1441}
1442
1443RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1444 AggValueSlot resultSlot,
1445 SourceLocation loc,
1446 bool asValue) const {
1447 if (LVal.isSimple()) {
1448 if (EvaluationKind == TEK_Aggregate)
1449 return resultSlot.asRValue();
1450
1451 // Drill into the padding structure if we have one.
1452 if (hasPadding())
1453 addr = CGF.Builder.CreateStructGEP(addr, 0);
1454
1455 // Otherwise, just convert the temporary to an r-value using the
1456 // normal conversion routine.
1457 return CGF.convertTempToRValue(addr, getValueType(), loc);
1458 }
1459 if (!asValue)
1460 // Get RValue from temp memory as atomic for non-simple lvalues
1461 return RValue::get(CGF.Builder.CreateLoad(addr));
1462 if (LVal.isBitField())
1463 return CGF.EmitLoadOfBitfieldLValue(
1464 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1465 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1466 if (LVal.isVectorElt())
1467 return CGF.EmitLoadOfLValue(
1468 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1469 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1470 assert(LVal.isExtVectorElt());
1471 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1472 addr, LVal.getExtVectorElts(), LVal.getType(),
1473 LVal.getBaseInfo(), TBAAAccessInfo()));
1474}
1475
1476RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1477 AggValueSlot ResultSlot,
1478 SourceLocation Loc,
1479 bool AsValue) const {
1480 // Try not to in some easy cases.
1481 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1482 if (getEvaluationKind() == TEK_Scalar &&
1483 (((!LVal.isBitField() ||
1484 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1485 !hasPadding()) ||
1486 !AsValue)) {
1487 auto *ValTy = AsValue
1488 ? CGF.ConvertTypeForMem(ValueTy)
1489 : getAtomicAddress().getType()->getPointerElementType();
1490 if (ValTy->isIntegerTy()) {
1491 assert(IntVal->getType() == ValTy && "Different integer types.");
1492 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1493 } else if (ValTy->isPointerTy())
1494 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1495 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1496 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1497 }
1498
1499 // Create a temporary. This needs to be big enough to hold the
1500 // atomic integer.
1501 Address Temp = Address::invalid();
1502 bool TempIsVolatile = false;
1503 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1504 assert(!ResultSlot.isIgnored());
1505 Temp = ResultSlot.getAddress();
1506 TempIsVolatile = ResultSlot.isVolatile();
1507 } else {
1508 Temp = CreateTempAlloca();
1509 }
1510
1511 // Slam the integer into the temporary.
1512 Address CastTemp = emitCastToAtomicIntPointer(Temp);
1513 CGF.Builder.CreateStore(IntVal, CastTemp)
1514 ->setVolatile(TempIsVolatile);
1515
1516 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1517}
1518
1519void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1520 llvm::AtomicOrdering AO, bool) {
1521 // void __atomic_load(size_t size, void *mem, void *return, int order);
1522 CallArgList Args;
1523 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1524 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1525 CGF.getContext().VoidPtrTy);
1526 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1527 CGF.getContext().VoidPtrTy);
1528 Args.add(
1529 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1530 CGF.getContext().IntTy);
1531 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1532}
1533
1534llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1535 bool IsVolatile) {
1536 // Okay, we're doing this natively.
1537 Address Addr = getAtomicAddressAsAtomicIntPointer();
1538 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1539 Load->setAtomic(AO);
1540
1541 // Other decoration.
1542 if (IsVolatile)
1543 Load->setVolatile(true);
1544 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1545 return Load;
1546}
1547
1548/// An LValue is a candidate for having its loads and stores be made atomic if
1549/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1550/// performing such an operation can be performed without a libcall.
1551bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1552 if (!CGM.getCodeGenOpts().MSVolatile) return false;
1553 AtomicInfo AI(*this, LV);
1554 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1555 // An atomic is inline if we don't need to use a libcall.
1556 bool AtomicIsInline = !AI.shouldUseLibcall();
1557 // MSVC doesn't seem to do this for types wider than a pointer.
1558 if (getContext().getTypeSize(LV.getType()) >
1559 getContext().getTypeSize(getContext().getIntPtrType()))
1560 return false;
1561 return IsVolatile && AtomicIsInline;
1562}
1563
1564RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1565 AggValueSlot Slot) {
1566 llvm::AtomicOrdering AO;
1567 bool IsVolatile = LV.isVolatileQualified();
1568 if (LV.getType()->isAtomicType()) {
1569 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1570 } else {
1571 AO = llvm::AtomicOrdering::Acquire;
1572 IsVolatile = true;
1573 }
1574 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1575}
1576
1577RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1578 bool AsValue, llvm::AtomicOrdering AO,
1579 bool IsVolatile) {
1580 // Check whether we should use a library call.
1581 if (shouldUseLibcall()) {
1582 Address TempAddr = Address::invalid();
1583 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1584 assert(getEvaluationKind() == TEK_Aggregate);
1585 TempAddr = ResultSlot.getAddress();
1586 } else
1587 TempAddr = CreateTempAlloca();
1588
1589 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1590
1591 // Okay, turn that back into the original value or whole atomic (for
1592 // non-simple lvalues) type.
1593 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1594 }
1595
1596 // Okay, we're doing this natively.
1597 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1598
1599 // If we're ignoring an aggregate return, don't do anything.
1600 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1601 return RValue::getAggregate(Address::invalid(), false);
1602
1603 // Okay, turn that back into the original value or atomic (for non-simple
1604 // lvalues) type.
1605 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1606}
1607
1608/// Emit a load from an l-value of atomic type. Note that the r-value
1609/// we produce is an r-value of the atomic *value* type.
1610RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1611 llvm::AtomicOrdering AO, bool IsVolatile,
1612 AggValueSlot resultSlot) {
1613 AtomicInfo Atomics(*this, src);
1614 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1615 IsVolatile);
1616}
1617
1618/// Copy an r-value into memory as part of storing to an atomic type.
1619/// This needs to create a bit-pattern suitable for atomic operations.
1620void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1621 assert(LVal.isSimple());
1622 // If we have an r-value, the rvalue should be of the atomic type,
1623 // which means that the caller is responsible for having zeroed
1624 // any padding. Just do an aggregate copy of that type.
1625 if (rvalue.isAggregate()) {
1626 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1627 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1628 getAtomicType());
1629 bool IsVolatile = rvalue.isVolatileQualified() ||
1630 LVal.isVolatileQualified();
1631 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1632 AggValueSlot::DoesNotOverlap, IsVolatile);
1633 return;
1634 }
1635
1636 // Okay, otherwise we're copying stuff.
1637
1638 // Zero out the buffer if necessary.
1639 emitMemSetZeroIfNecessary();
1640
1641 // Drill past the padding if present.
1642 LValue TempLVal = projectValue();
1643
1644 // Okay, store the rvalue in.
1645 if (rvalue.isScalar()) {
1646 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1647 } else {
1648 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1649 }
1650}
1651
1652
1653/// Materialize an r-value into memory for the purposes of storing it
1654/// to an atomic type.
1655Address AtomicInfo::materializeRValue(RValue rvalue) const {
1656 // Aggregate r-values are already in memory, and EmitAtomicStore
1657 // requires them to be values of the atomic type.
1658 if (rvalue.isAggregate())
1659 return rvalue.getAggregateAddress();
1660
1661 // Otherwise, make a temporary and materialize into it.
1662 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1663 AtomicInfo Atomics(CGF, TempLV);
1664 Atomics.emitCopyIntoMemory(rvalue);
1665 return TempLV.getAddress(CGF);
1666}
1667
1668llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1669 // If we've got a scalar value of the right size, try to avoid going
1670 // through memory.
1671 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1672 llvm::Value *Value = RVal.getScalarVal();
1673 if (isa<llvm::IntegerType>(Value->getType()))
1674 return CGF.EmitToMemory(Value, ValueTy);
1675 else {
1676 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1677 CGF.getLLVMContext(),
1678 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1679 if (isa<llvm::PointerType>(Value->getType()))
1680 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1681 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1682 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1683 }
1684 }
1685 // Otherwise, we need to go through memory.
1686 // Put the r-value in memory.
1687 Address Addr = materializeRValue(RVal);
1688
1689 // Cast the temporary to the atomic int type and pull a value out.
1690 Addr = emitCastToAtomicIntPointer(Addr);
1691 return CGF.Builder.CreateLoad(Addr);
1692}
1693
1694std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1695 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1696 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1697 // Do the atomic store.
1698 Address Addr = getAtomicAddressAsAtomicIntPointer();
1699 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1700 ExpectedVal, DesiredVal,
1701 Success, Failure);
1702 // Other decoration.
1703 Inst->setVolatile(LVal.isVolatileQualified());
1704 Inst->setWeak(IsWeak);
1705
1706 // Okay, turn that back into the original value type.
1707 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1708 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1709 return std::make_pair(PreviousVal, SuccessFailureVal);
1710}
1711
1712llvm::Value *
1713AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1714 llvm::Value *DesiredAddr,
1715 llvm::AtomicOrdering Success,
1716 llvm::AtomicOrdering Failure) {
1717 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1718 // void *desired, int success, int failure);
1719 CallArgList Args;
1720 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1721 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1722 CGF.getContext().VoidPtrTy);
1723 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1724 CGF.getContext().VoidPtrTy);
1725 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1726 CGF.getContext().VoidPtrTy);
1727 Args.add(RValue::get(
1728 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1729 CGF.getContext().IntTy);
1730 Args.add(RValue::get(
1731 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1732 CGF.getContext().IntTy);
1733 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1734 CGF.getContext().BoolTy, Args);
1735
1736 return SuccessFailureRVal.getScalarVal();
1737}
1738
1739std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1740 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1741 llvm::AtomicOrdering Failure, bool IsWeak) {
1742 if (isStrongerThan(Failure, Success))
1743 // Don't assert on undefined behavior "failure argument shall be no stronger
1744 // than the success argument".
1745 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1746
1747 // Check whether we should use a library call.
1748 if (shouldUseLibcall()) {
1749 // Produce a source address.
1750 Address ExpectedAddr = materializeRValue(Expected);
1751 Address DesiredAddr = materializeRValue(Desired);
1752 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1753 DesiredAddr.getPointer(),
1754 Success, Failure);
1755 return std::make_pair(
1756 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1757 SourceLocation(), /*AsValue=*/false),
1758 Res);
1759 }
1760
1761 // If we've got a scalar value of the right size, try to avoid going
1762 // through memory.
1763 auto *ExpectedVal = convertRValueToInt(Expected);
1764 auto *DesiredVal = convertRValueToInt(Desired);
1765 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1766 Failure, IsWeak);
1767 return std::make_pair(
1768 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1769 SourceLocation(), /*AsValue=*/false),
1770 Res.second);
1771}
1772
1773static void
1774EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1775 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1776 Address DesiredAddr) {
1777 RValue UpRVal;
1778 LValue AtomicLVal = Atomics.