1	//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the code for emitting atomic operations.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCall.h"
14	#include "CGRecordLayout.h"
15	#include "CodeGenFunction.h"
16	#include "CodeGenModule.h"
17	#include "TargetInfo.h"
18	#include "clang/AST/ASTContext.h"
19	#include "clang/CodeGen/CGFunctionInfo.h"
20	#include "clang/Frontend/FrontendDiagnostic.h"
21	#include "llvm/ADT/DenseMap.h"
22	#include "llvm/IR/DataLayout.h"
23	#include "llvm/IR/Intrinsics.h"
24	#include "llvm/IR/Operator.h"
25
26	using namespace clang;
27	using namespace CodeGen;
28
29	namespace {
30	class AtomicInfo {
31	CodeGenFunction &CGF;
32	QualType AtomicTy;
33	QualType ValueTy;
34	uint64_t AtomicSizeInBits;
35	uint64_t ValueSizeInBits;
36	CharUnits AtomicAlign;
37	CharUnits ValueAlign;
38	TypeEvaluationKind EvaluationKind;
39	bool UseLibcall;
40	LValue LVal;
41	CGBitFieldInfo BFI;
42	public:
43	AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44	: CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45	EvaluationKind(TEK_Scalar), UseLibcall(true) {
46	assert(!lvalue.isGlobalReg());
47	ASTContext &C = CGF.getContext();
48	if (lvalue.isSimple()) {
49	AtomicTy = lvalue.getType();
50	if (auto *ATy = AtomicTy->getAs<AtomicType>())
51	ValueTy = ATy->getValueType();
52	else
53	ValueTy = AtomicTy;
54	EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56	uint64_t ValueAlignInBits;
57	uint64_t AtomicAlignInBits;
58	TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59	ValueSizeInBits = ValueTI.Width;
60	ValueAlignInBits = ValueTI.Align;
61
62	TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63	AtomicSizeInBits = AtomicTI.Width;
64	AtomicAlignInBits = AtomicTI.Align;
65
66	assert(ValueSizeInBits <= AtomicSizeInBits);
67	assert(ValueAlignInBits <= AtomicAlignInBits);
68
69	AtomicAlign = C.toCharUnitsFromBits(BitSize: AtomicAlignInBits);
70	ValueAlign = C.toCharUnitsFromBits(BitSize: ValueAlignInBits);
71	if (lvalue.getAlignment().isZero())
72	lvalue.setAlignment(AtomicAlign);
73
74	LVal = lvalue;
75	} else if (lvalue.isBitField()) {
76	ValueTy = lvalue.getType();
77	ValueSizeInBits = C.getTypeSize(ValueTy);
78	auto &OrigBFI = lvalue.getBitFieldInfo();
79	auto Offset = OrigBFI.Offset % C.toBits(CharSize: lvalue.getAlignment());
80	AtomicSizeInBits = C.toBits(
81	CharSize: C.toCharUnitsFromBits(BitSize: Offset + OrigBFI.Size + C.getCharWidth() - 1)
82	.alignTo(Align: lvalue.getAlignment()));
83	llvm::Value *BitFieldPtr = lvalue.getBitFieldPointer();
84	auto OffsetInChars =
85	(C.toCharUnitsFromBits(BitSize: OrigBFI.Offset) / lvalue.getAlignment()) *
86	lvalue.getAlignment();
87	llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
88	Ty: CGF.Int8Ty, Ptr: BitFieldPtr, Idx0: OffsetInChars.getQuantity());
89	StoragePtr = CGF.Builder.CreateAddrSpaceCast(
90	V: StoragePtr, DestTy: CGF.UnqualPtrTy, Name: "atomic_bitfield_base");
91	BFI = OrigBFI;
92	BFI.Offset = Offset;
93	BFI.StorageSize = AtomicSizeInBits;
94	BFI.StorageOffset += OffsetInChars;
95	llvm::Type *StorageTy = CGF.Builder.getIntNTy(N: AtomicSizeInBits);
96	LVal = LValue::MakeBitfield(
97	Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
98	lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
99	AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100	if (AtomicTy.isNull()) {
101	llvm::APInt Size(
102	/*numBits=*/32,
103	C.toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity());
104	AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
105	ArraySizeModifier::Normal,
106	/*IndexTypeQuals=*/0);
107	}
108	AtomicAlign = ValueAlign = lvalue.getAlignment();
109	} else if (lvalue.isVectorElt()) {
110	ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111	ValueSizeInBits = C.getTypeSize(ValueTy);
112	AtomicTy = lvalue.getType();
113	AtomicSizeInBits = C.getTypeSize(AtomicTy);
114	AtomicAlign = ValueAlign = lvalue.getAlignment();
115	LVal = lvalue;
116	} else {
117	assert(lvalue.isExtVectorElt());
118	ValueTy = lvalue.getType();
119	ValueSizeInBits = C.getTypeSize(ValueTy);
120	AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121	lvalue.getType(), cast<llvm::FixedVectorType>(
122	lvalue.getExtVectorAddress().getElementType())
123	->getNumElements());
124	AtomicSizeInBits = C.getTypeSize(AtomicTy);
125	AtomicAlign = ValueAlign = lvalue.getAlignment();
126	LVal = lvalue;
127	}
128	UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129	AtomicSizeInBits, AlignmentInBits: C.toBits(CharSize: lvalue.getAlignment()));
130	}
131
132	QualType getAtomicType() const { return AtomicTy; }
133	QualType getValueType() const { return ValueTy; }
134	CharUnits getAtomicAlignment() const { return AtomicAlign; }
135	uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136	uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137	TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138	bool shouldUseLibcall() const { return UseLibcall; }
139	const LValue &getAtomicLValue() const { return LVal; }
140	llvm::Value *getAtomicPointer() const {
141	if (LVal.isSimple())
142	return LVal.getPointer(CGF);
143	else if (LVal.isBitField())
144	return LVal.getBitFieldPointer();
145	else if (LVal.isVectorElt())
146	return LVal.getVectorPointer();
147	assert(LVal.isExtVectorElt());
148	return LVal.getExtVectorPointer();
149	}
150	Address getAtomicAddress() const {
151	llvm::Type *ElTy;
152	if (LVal.isSimple())
153	ElTy = LVal.getAddress(CGF).getElementType();
154	else if (LVal.isBitField())
155	ElTy = LVal.getBitFieldAddress().getElementType();
156	else if (LVal.isVectorElt())
157	ElTy = LVal.getVectorAddress().getElementType();
158	else
159	ElTy = LVal.getExtVectorAddress().getElementType();
160	return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161	}
162
163	Address getAtomicAddressAsAtomicIntPointer() const {
164	return castToAtomicIntPointer(Addr: getAtomicAddress());
165	}
166
167	/// Is the atomic size larger than the underlying value type?
168	///
169	/// Note that the absence of padding does not mean that atomic
170	/// objects are completely interchangeable with non-atomic
171	/// objects: we might have promoted the alignment of a type
172	/// without making it bigger.
173	bool hasPadding() const {
174	return (ValueSizeInBits != AtomicSizeInBits);
175	}
176
177	bool emitMemSetZeroIfNecessary() const;
178
179	llvm::Value *getAtomicSizeValue() const {
180	CharUnits size = CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits);
181	return CGF.CGM.getSize(numChars: size);
182	}
183
184	/// Cast the given pointer to an integer pointer suitable for atomic
185	/// operations if the source.
186	Address castToAtomicIntPointer(Address Addr) const;
187
188	/// If Addr is compatible with the iN that will be used for an atomic
189	/// operation, bitcast it. Otherwise, create a temporary that is suitable
190	/// and copy the value across.
191	Address convertToAtomicIntPointer(Address Addr) const;
192
193	/// Turn an atomic-layout object into an r-value.
194	RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195	SourceLocation loc, bool AsValue) const;
196
197	/// Converts a rvalue to integer value.
198	llvm::Value *convertRValueToInt(RValue RVal) const;
199
200	RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201	AggValueSlot ResultSlot,
202	SourceLocation Loc, bool AsValue) const;
203
204	/// Copy an atomic r-value into atomic-layout memory.
205	void emitCopyIntoMemory(RValue rvalue) const;
206
207	/// Project an l-value down to the value field.
208	LValue projectValue() const {
209	assert(LVal.isSimple());
210	Address addr = getAtomicAddress();
211	if (hasPadding())
212	addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
213
214	return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215	LVal.getBaseInfo(), LVal.getTBAAInfo());
216	}
217
218	/// Emits atomic load.
219	/// \returns Loaded value.
220	RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221	bool AsValue, llvm::AtomicOrdering AO,
222	bool IsVolatile);
223
224	/// Emits atomic compare-and-exchange sequence.
225	/// \param Expected Expected value.
226	/// \param Desired Desired value.
227	/// \param Success Atomic ordering for success operation.
228	/// \param Failure Atomic ordering for failed operation.
229	/// \param IsWeak true if atomic operation is weak, false otherwise.
230	/// \returns Pair of values: previous value from storage (value type) and
231	/// boolean flag (i1 type) with true if success and false otherwise.
232	std::pair<RValue, llvm::Value *>
233	EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234	llvm::AtomicOrdering Success =
235	llvm::AtomicOrdering::SequentiallyConsistent,
236	llvm::AtomicOrdering Failure =
237	llvm::AtomicOrdering::SequentiallyConsistent,
238	bool IsWeak = false);
239
240	/// Emits atomic update.
241	/// \param AO Atomic ordering.
242	/// \param UpdateOp Update operation for the current lvalue.
243	void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244	const llvm::function_ref<RValue(RValue)> &UpdateOp,
245	bool IsVolatile);
246	/// Emits atomic update.
247	/// \param AO Atomic ordering.
248	void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249	bool IsVolatile);
250
251	/// Materialize an atomic r-value in atomic-layout memory.
252	Address materializeRValue(RValue rvalue) const;
253
254	/// Creates temp alloca for intermediate operations on atomic value.
255	Address CreateTempAlloca() const;
256	private:
257	bool requiresMemSetZero(llvm::Type *type) const;
258
259
260	/// Emits atomic load as a libcall.
261	void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262	llvm::AtomicOrdering AO, bool IsVolatile);
263	/// Emits atomic load as LLVM instruction.
264	llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265	/// Emits atomic compare-and-exchange op as a libcall.
266	llvm::Value *EmitAtomicCompareExchangeLibcall(
267	llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268	llvm::AtomicOrdering Success =
269	llvm::AtomicOrdering::SequentiallyConsistent,
270	llvm::AtomicOrdering Failure =
271	llvm::AtomicOrdering::SequentiallyConsistent);
272	/// Emits atomic compare-and-exchange op as LLVM instruction.
273	std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274	llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275	llvm::AtomicOrdering Success =
276	llvm::AtomicOrdering::SequentiallyConsistent,
277	llvm::AtomicOrdering Failure =
278	llvm::AtomicOrdering::SequentiallyConsistent,
279	bool IsWeak = false);
280	/// Emit atomic update as libcalls.
281	void
282	EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283	const llvm::function_ref<RValue(RValue)> &UpdateOp,
284	bool IsVolatile);
285	/// Emit atomic update as LLVM instructions.
286	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287	const llvm::function_ref<RValue(RValue)> &UpdateOp,
288	bool IsVolatile);
289	/// Emit atomic update as libcalls.
290	void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291	bool IsVolatile);
292	/// Emit atomic update as LLVM instructions.
293	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294	bool IsVolatile);
295	};
296	}
297
298	Address AtomicInfo::CreateTempAlloca() const {
299	Address TempAlloca = CGF.CreateMemTemp(
300	(LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301	: AtomicTy,
302	getAtomicAlignment(),
303	"atomic-temp");
304	// Cast to pointer to value type for bitfields.
305	if (LVal.isBitField())
306	return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
307	Addr: TempAlloca, Ty: getAtomicAddress().getType(),
308	ElementTy: getAtomicAddress().getElementType());
309	return TempAlloca;
310	}
311
312	static RValue emitAtomicLibcall(CodeGenFunction &CGF,
313	StringRef fnName,
314	QualType resultType,
315	CallArgList &args) {
316	const CGFunctionInfo &fnInfo =
317	CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318	llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(Info: fnInfo);
319	llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320	fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321	fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322	llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323	C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: fnAttrB);
324
325	llvm::FunctionCallee fn =
326	CGF.CGM.CreateRuntimeFunction(Ty: fnTy, Name: fnName, ExtraAttrs: fnAttrs);
327	auto callee = CGCallee::forDirect(functionPtr: fn);
328	return CGF.EmitCall(CallInfo: fnInfo, Callee: callee, ReturnValue: ReturnValueSlot(), Args: args);
329	}
330
331	/// Does a store of the given IR type modify the full expected width?
332	static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333	uint64_t expectedSize) {
334	return (CGM.getDataLayout().getTypeStoreSize(Ty: type) * 8 == expectedSize);
335	}
336
337	/// Does the atomic type require memsetting to zero before initialization?
338	///
339	/// The IR type is provided as a way of making certain queries faster.
340	bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341	// If the atomic type has size padding, we definitely need a memset.
342	if (hasPadding()) return true;
343
344	// Otherwise, do some simple heuristics to try to avoid it:
345	switch (getEvaluationKind()) {
346	// For scalars and complexes, check whether the store size of the
347	// type uses the full size.
348	case TEK_Scalar:
349	return !isFullSizeType(CGM&: CGF.CGM, type, expectedSize: AtomicSizeInBits);
350	case TEK_Complex:
351	return !isFullSizeType(CGM&: CGF.CGM, type: type->getStructElementType(N: 0),
352	expectedSize: AtomicSizeInBits / 2);
353
354	// Padding in structs has an undefined bit pattern. User beware.
355	case TEK_Aggregate:
356	return false;
357	}
358	llvm_unreachable("bad evaluation kind");
359	}
360
361	bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362	assert(LVal.isSimple());
363	Address addr = LVal.getAddress(CGF);
364	if (!requiresMemSetZero(type: addr.getElementType()))
365	return false;
366
367	CGF.Builder.CreateMemSet(
368	addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369	CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370	LVal.getAlignment().getAsAlign());
371	return true;
372	}
373
374	static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375	Address Dest, Address Ptr,
376	Address Val1, Address Val2,
377	uint64_t Size,
378	llvm::AtomicOrdering SuccessOrder,
379	llvm::AtomicOrdering FailureOrder,
380	llvm::SyncScope::ID Scope) {
381	// Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382	llvm::Value *Expected = CGF.Builder.CreateLoad(Addr: Val1);
383	llvm::Value *Desired = CGF.Builder.CreateLoad(Addr: Val2);
384
385	llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386	Addr: Ptr, Cmp: Expected, New: Desired, SuccessOrdering: SuccessOrder, FailureOrdering: FailureOrder, SSID: Scope);
387	Pair->setVolatile(E->isVolatile());
388	Pair->setWeak(IsWeak);
389
390	// Cmp holds the result of the compare-exchange operation: true on success,
391	// false on failure.
392	llvm::Value *Old = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
393	llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1);
394
395	// This basic block is used to hold the store instruction if the operation
396	// failed.
397	llvm::BasicBlock *StoreExpectedBB =
398	CGF.createBasicBlock(name: "cmpxchg.store_expected", parent: CGF.CurFn);
399
400	// This basic block is the exit point of the operation, we should end up
401	// here regardless of whether or not the operation succeeded.
402	llvm::BasicBlock *ContinueBB =
403	CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
404
405	// Update Expected if Expected isn't equal to Old, otherwise branch to the
406	// exit point.
407	CGF.Builder.CreateCondBr(Cond: Cmp, True: ContinueBB, False: StoreExpectedBB);
408
409	CGF.Builder.SetInsertPoint(StoreExpectedBB);
410	// Update the memory at Expected with Old's value.
411	CGF.Builder.CreateStore(Val: Old, Addr: Val1);
412	// Finally, branch to the exit point.
413	CGF.Builder.CreateBr(Dest: ContinueBB);
414
415	CGF.Builder.SetInsertPoint(ContinueBB);
416	// Update the memory at Dest with Cmp's value.
417	CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
418	}
419
420	/// Given an ordering required on success, emit all possible cmpxchg
421	/// instructions to cope with the provided (but possibly only dynamically known)
422	/// FailureOrder.
423	static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
424	bool IsWeak, Address Dest, Address Ptr,
425	Address Val1, Address Val2,
426	llvm::Value *FailureOrderVal,
427	uint64_t Size,
428	llvm::AtomicOrdering SuccessOrder,
429	llvm::SyncScope::ID Scope) {
430	llvm::AtomicOrdering FailureOrder;
431	if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(Val: FailureOrderVal)) {
432	auto FOS = FO->getSExtValue();
433	if (!llvm::isValidAtomicOrderingCABI(I: FOS))
434	FailureOrder = llvm::AtomicOrdering::Monotonic;
435	else
436	switch ((llvm::AtomicOrderingCABI)FOS) {
437	case llvm::AtomicOrderingCABI::relaxed:
438	// 31.7.2.18: "The failure argument shall not be memory_order_release
439	// nor memory_order_acq_rel". Fallback to monotonic.
440	case llvm::AtomicOrderingCABI::release:
441	case llvm::AtomicOrderingCABI::acq_rel:
442	FailureOrder = llvm::AtomicOrdering::Monotonic;
443	break;
444	case llvm::AtomicOrderingCABI::consume:
445	case llvm::AtomicOrderingCABI::acquire:
446	FailureOrder = llvm::AtomicOrdering::Acquire;
447	break;
448	case llvm::AtomicOrderingCABI::seq_cst:
449	FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
450	break;
451	}
452	// Prior to c++17, "the failure argument shall be no stronger than the
453	// success argument". This condition has been lifted and the only
454	// precondition is 31.7.2.18. Effectively treat this as a DR and skip
455	// language version checks.
456	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
457	FailureOrder, Scope);
458	return;
459	}
460
461	// Create all the relevant BB's
462	auto *MonotonicBB = CGF.createBasicBlock(name: "monotonic_fail", parent: CGF.CurFn);
463	auto *AcquireBB = CGF.createBasicBlock(name: "acquire_fail", parent: CGF.CurFn);
464	auto *SeqCstBB = CGF.createBasicBlock(name: "seqcst_fail", parent: CGF.CurFn);
465	auto *ContBB = CGF.createBasicBlock(name: "atomic.continue", parent: CGF.CurFn);
466
467	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
468	// doesn't matter unless someone is crazy enough to use something that
469	// doesn't fold to a constant for the ordering.
470	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: FailureOrderVal, Dest: MonotonicBB);
471	// Implemented as acquire, since it's the closest in LLVM.
472	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
473	Dest: AcquireBB);
474	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
475	Dest: AcquireBB);
476	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
477	Dest: SeqCstBB);
478
479	// Emit all the different atomics
480	CGF.Builder.SetInsertPoint(MonotonicBB);
481	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
482	Size, SuccessOrder, FailureOrder: llvm::AtomicOrdering::Monotonic, Scope);
483	CGF.Builder.CreateBr(Dest: ContBB);
484
485	CGF.Builder.SetInsertPoint(AcquireBB);
486	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
487	FailureOrder: llvm::AtomicOrdering::Acquire, Scope);
488	CGF.Builder.CreateBr(Dest: ContBB);
489
490	CGF.Builder.SetInsertPoint(SeqCstBB);
491	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
492	FailureOrder: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
493	CGF.Builder.CreateBr(Dest: ContBB);
494
495	CGF.Builder.SetInsertPoint(ContBB);
496	}
497
498	/// Duplicate the atomic min/max operation in conventional IR for the builtin
499	/// variants that return the new rather than the original value.
500	static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
501	AtomicExpr::AtomicOp Op,
502	bool IsSigned,
503	llvm::Value *OldVal,
504	llvm::Value *RHS) {
505	llvm::CmpInst::Predicate Pred;
506	switch (Op) {
507	default:
508	llvm_unreachable("Unexpected min/max operation");
509	case AtomicExpr::AO__atomic_max_fetch:
510	case AtomicExpr::AO__scoped_atomic_max_fetch:
511	Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512	break;
513	case AtomicExpr::AO__atomic_min_fetch:
514	case AtomicExpr::AO__scoped_atomic_min_fetch:
515	Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
516	break;
517	}
518	llvm::Value *Cmp = Builder.CreateICmp(P: Pred, LHS: OldVal, RHS, Name: "tst");
519	return Builder.CreateSelect(C: Cmp, True: OldVal, False: RHS, Name: "newval");
520	}
521
522	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
523	Address Ptr, Address Val1, Address Val2,
524	llvm::Value *IsWeak, llvm::Value *FailureOrder,
525	uint64_t Size, llvm::AtomicOrdering Order,
526	llvm::SyncScope::ID Scope) {
527	llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
528	bool PostOpMinMax = false;
529	unsigned PostOp = 0;
530
531	switch (E->getOp()) {
532	case AtomicExpr::AO__c11_atomic_init:
533	case AtomicExpr::AO__opencl_atomic_init:
534	llvm_unreachable("Already handled!");
535
536	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
537	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
538	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
539	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
540	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
541	return;
542	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
543	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
544	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
545	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
546	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
547	return;
548	case AtomicExpr::AO__atomic_compare_exchange:
549	case AtomicExpr::AO__atomic_compare_exchange_n:
550	case AtomicExpr::AO__scoped_atomic_compare_exchange:
551	case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
552	if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(Val: IsWeak)) {
553	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: IsWeakC->getZExtValue(), Dest, Ptr,
554	Val1, Val2, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
555	} else {
556	// Create all the relevant BB's
557	llvm::BasicBlock *StrongBB =
558	CGF.createBasicBlock(name: "cmpxchg.strong", parent: CGF.CurFn);
559	llvm::BasicBlock *WeakBB = CGF.createBasicBlock(name: "cmxchg.weak", parent: CGF.CurFn);
560	llvm::BasicBlock *ContBB =
561	CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
562
563	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: IsWeak, Dest: WeakBB);
564	SI->addCase(OnVal: CGF.Builder.getInt1(V: false), Dest: StrongBB);
565
566	CGF.Builder.SetInsertPoint(StrongBB);
567	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
568	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
569	CGF.Builder.CreateBr(Dest: ContBB);
570
571	CGF.Builder.SetInsertPoint(WeakBB);
572	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
573	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
574	CGF.Builder.CreateBr(Dest: ContBB);
575
576	CGF.Builder.SetInsertPoint(ContBB);
577	}
578	return;
579	}
580	case AtomicExpr::AO__c11_atomic_load:
581	case AtomicExpr::AO__opencl_atomic_load:
582	case AtomicExpr::AO__hip_atomic_load:
583	case AtomicExpr::AO__atomic_load_n:
584	case AtomicExpr::AO__atomic_load:
585	case AtomicExpr::AO__scoped_atomic_load_n:
586	case AtomicExpr::AO__scoped_atomic_load: {
587	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr: Ptr);
588	Load->setAtomic(Ordering: Order, SSID: Scope);
589	Load->setVolatile(E->isVolatile());
590	CGF.Builder.CreateStore(Val: Load, Addr: Dest);
591	return;
592	}
593
594	case AtomicExpr::AO__c11_atomic_store:
595	case AtomicExpr::AO__opencl_atomic_store:
596	case AtomicExpr::AO__hip_atomic_store:
597	case AtomicExpr::AO__atomic_store:
598	case AtomicExpr::AO__atomic_store_n:
599	case AtomicExpr::AO__scoped_atomic_store:
600	case AtomicExpr::AO__scoped_atomic_store_n: {
601	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
602	llvm::StoreInst *Store = CGF.Builder.CreateStore(Val: LoadVal1, Addr: Ptr);
603	Store->setAtomic(Ordering: Order, SSID: Scope);
604	Store->setVolatile(E->isVolatile());
605	return;
606	}
607
608	case AtomicExpr::AO__c11_atomic_exchange:
609	case AtomicExpr::AO__hip_atomic_exchange:
610	case AtomicExpr::AO__opencl_atomic_exchange:
611	case AtomicExpr::AO__atomic_exchange_n:
612	case AtomicExpr::AO__atomic_exchange:
613	case AtomicExpr::AO__scoped_atomic_exchange_n:
614	case AtomicExpr::AO__scoped_atomic_exchange:
615	Op = llvm::AtomicRMWInst::Xchg;
616	break;
617
618	case AtomicExpr::AO__atomic_add_fetch:
619	case AtomicExpr::AO__scoped_atomic_add_fetch:
620	PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
621	: llvm::Instruction::Add;
622	[[fallthrough]];
623	case AtomicExpr::AO__c11_atomic_fetch_add:
624	case AtomicExpr::AO__hip_atomic_fetch_add:
625	case AtomicExpr::AO__opencl_atomic_fetch_add:
626	case AtomicExpr::AO__atomic_fetch_add:
627	case AtomicExpr::AO__scoped_atomic_fetch_add:
628	Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
629	: llvm::AtomicRMWInst::Add;
630	break;
631
632	case AtomicExpr::AO__atomic_sub_fetch:
633	case AtomicExpr::AO__scoped_atomic_sub_fetch:
634	PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
635	: llvm::Instruction::Sub;
636	[[fallthrough]];
637	case AtomicExpr::AO__c11_atomic_fetch_sub:
638	case AtomicExpr::AO__hip_atomic_fetch_sub:
639	case AtomicExpr::AO__opencl_atomic_fetch_sub:
640	case AtomicExpr::AO__atomic_fetch_sub:
641	case AtomicExpr::AO__scoped_atomic_fetch_sub:
642	Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
643	: llvm::AtomicRMWInst::Sub;
644	break;
645
646	case AtomicExpr::AO__atomic_min_fetch:
647	case AtomicExpr::AO__scoped_atomic_min_fetch:
648	PostOpMinMax = true;
649	[[fallthrough]];
650	case AtomicExpr::AO__c11_atomic_fetch_min:
651	case AtomicExpr::AO__hip_atomic_fetch_min:
652	case AtomicExpr::AO__opencl_atomic_fetch_min:
653	case AtomicExpr::AO__atomic_fetch_min:
654	case AtomicExpr::AO__scoped_atomic_fetch_min:
655	Op = E->getValueType()->isFloatingType()
656	? llvm::AtomicRMWInst::FMin
657	: (E->getValueType()->isSignedIntegerType()
658	? llvm::AtomicRMWInst::Min
659	: llvm::AtomicRMWInst::UMin);
660	break;
661
662	case AtomicExpr::AO__atomic_max_fetch:
663	case AtomicExpr::AO__scoped_atomic_max_fetch:
664	PostOpMinMax = true;
665	[[fallthrough]];
666	case AtomicExpr::AO__c11_atomic_fetch_max:
667	case AtomicExpr::AO__hip_atomic_fetch_max:
668	case AtomicExpr::AO__opencl_atomic_fetch_max:
669	case AtomicExpr::AO__atomic_fetch_max:
670	case AtomicExpr::AO__scoped_atomic_fetch_max:
671	Op = E->getValueType()->isFloatingType()
672	? llvm::AtomicRMWInst::FMax
673	: (E->getValueType()->isSignedIntegerType()
674	? llvm::AtomicRMWInst::Max
675	: llvm::AtomicRMWInst::UMax);
676	break;
677
678	case AtomicExpr::AO__atomic_and_fetch:
679	case AtomicExpr::AO__scoped_atomic_and_fetch:
680	PostOp = llvm::Instruction::And;
681	[[fallthrough]];
682	case AtomicExpr::AO__c11_atomic_fetch_and:
683	case AtomicExpr::AO__hip_atomic_fetch_and:
684	case AtomicExpr::AO__opencl_atomic_fetch_and:
685	case AtomicExpr::AO__atomic_fetch_and:
686	case AtomicExpr::AO__scoped_atomic_fetch_and:
687	Op = llvm::AtomicRMWInst::And;
688	break;
689
690	case AtomicExpr::AO__atomic_or_fetch:
691	case AtomicExpr::AO__scoped_atomic_or_fetch:
692	PostOp = llvm::Instruction::Or;
693	[[fallthrough]];
694	case AtomicExpr::AO__c11_atomic_fetch_or:
695	case AtomicExpr::AO__hip_atomic_fetch_or:
696	case AtomicExpr::AO__opencl_atomic_fetch_or:
697	case AtomicExpr::AO__atomic_fetch_or:
698	case AtomicExpr::AO__scoped_atomic_fetch_or:
699	Op = llvm::AtomicRMWInst::Or;
700	break;
701
702	case AtomicExpr::AO__atomic_xor_fetch:
703	case AtomicExpr::AO__scoped_atomic_xor_fetch:
704	PostOp = llvm::Instruction::Xor;
705	[[fallthrough]];
706	case AtomicExpr::AO__c11_atomic_fetch_xor:
707	case AtomicExpr::AO__hip_atomic_fetch_xor:
708	case AtomicExpr::AO__opencl_atomic_fetch_xor:
709	case AtomicExpr::AO__atomic_fetch_xor:
710	case AtomicExpr::AO__scoped_atomic_fetch_xor:
711	Op = llvm::AtomicRMWInst::Xor;
712	break;
713
714	case AtomicExpr::AO__atomic_nand_fetch:
715	case AtomicExpr::AO__scoped_atomic_nand_fetch:
716	PostOp = llvm::Instruction::And; // the NOT is special cased below
717	[[fallthrough]];
718	case AtomicExpr::AO__c11_atomic_fetch_nand:
719	case AtomicExpr::AO__atomic_fetch_nand:
720	case AtomicExpr::AO__scoped_atomic_fetch_nand:
721	Op = llvm::AtomicRMWInst::Nand;
722	break;
723	}
724
725	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
726	llvm::AtomicRMWInst *RMWI =
727	CGF.Builder.CreateAtomicRMW(Op, Addr: Ptr, Val: LoadVal1, Ordering: Order, SSID: Scope);
728	RMWI->setVolatile(E->isVolatile());
729
730	// For __atomic_*_fetch operations, perform the operation again to
731	// determine the value which was written.
732	llvm::Value *Result = RMWI;
733	if (PostOpMinMax)
734	Result = EmitPostAtomicMinMax(Builder&: CGF.Builder, Op: E->getOp(),
735	IsSigned: E->getValueType()->isSignedIntegerType(),
736	OldVal: RMWI, RHS: LoadVal1);
737	else if (PostOp)
738	Result = CGF.Builder.CreateBinOp(Opc: (llvm::Instruction::BinaryOps)PostOp, LHS: RMWI,
739	RHS: LoadVal1);
740	if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
741	E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
742	Result = CGF.Builder.CreateNot(V: Result);
743	CGF.Builder.CreateStore(Val: Result, Addr: Dest);
744	}
745
746	// This function emits any expression (scalar, complex, or aggregate)
747	// into a temporary alloca.
748	static Address
749	EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
750	Address DeclPtr = CGF.CreateMemTemp(T: E->getType(), Name: ".atomictmp");
751	CGF.EmitAnyExprToMem(E, Location: DeclPtr, Quals: E->getType().getQualifiers(),
752	/*Init*/ IsInitializer: true);
753	return DeclPtr;
754	}
755
756	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
757	Address Ptr, Address Val1, Address Val2,
758	llvm::Value *IsWeak, llvm::Value *FailureOrder,
759	uint64_t Size, llvm::AtomicOrdering Order,
760	llvm::Value *Scope) {
761	auto ScopeModel = Expr->getScopeModel();
762
763	// LLVM atomic instructions always have synch scope. If clang atomic
764	// expression has no scope operand, use default LLVM synch scope.
765	if (!ScopeModel) {
766	EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
767	Order, Scope: CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(SSN: ""));
768	return;
769	}
770
771	// Handle constant scope.
772	if (auto SC = dyn_cast<llvm::ConstantInt>(Val: Scope)) {
773	auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
774	LangOpts: CGF.CGM.getLangOpts(), Scope: ScopeModel->map(SC->getZExtValue()),
775	Ordering: Order, Ctx&: CGF.CGM.getLLVMContext());
776	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
777	Order, SCID);
778	return;
779	}
780
781	// Handle non-constant scope.
782	auto &Builder = CGF.Builder;
783	auto Scopes = ScopeModel->getRuntimeValues();
784	llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
785	for (auto S : Scopes)
786	BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
787
788	llvm::BasicBlock *ContBB =
789	CGF.createBasicBlock(name: "atomic.scope.continue", parent: CGF.CurFn);
790
791	auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
792	// If unsupported synch scope is encountered at run time, assume a fallback
793	// synch scope value.
794	auto FallBack = ScopeModel->getFallBackValue();
795	llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: BB[FallBack]);
796	for (auto S : Scopes) {
797	auto *B = BB[S];
798	if (S != FallBack)
799	SI->addCase(Builder.getInt32(S), B);
800
801	Builder.SetInsertPoint(B);
802	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
803	Order,
804	CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
805	ScopeModel->map(S),
806	Order,
807	CGF.getLLVMContext()));
808	Builder.CreateBr(ContBB);
809	}
810
811	Builder.SetInsertPoint(ContBB);
812	}
813
814	RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
815	QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
816	QualType MemTy = AtomicTy;
817	if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
818	MemTy = AT->getValueType();
819	llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
820
821	Address Val1 = Address::invalid();
822	Address Val2 = Address::invalid();
823	Address Dest = Address::invalid();
824	Address Ptr = EmitPointerWithAlignment(Addr: E->getPtr());
825
826	if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
827	E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
828	LValue lvalue = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
829	EmitAtomicInit(E: E->getVal1(), lvalue);
830	return RValue::get(V: nullptr);
831	}
832
833	auto TInfo = getContext().getTypeInfoInChars(T: AtomicTy);
834	uint64_t Size = TInfo.Width.getQuantity();
835	unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
836
837	CharUnits MaxInlineWidth =
838	getContext().toCharUnitsFromBits(BitSize: MaxInlineWidthInBits);
839	DiagnosticsEngine &Diags = CGM.getDiags();
840	bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
841	bool Oversized = getContext().toBits(CharSize: TInfo.Width) > MaxInlineWidthInBits;
842	if (Misaligned) {
843	Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
844	<< (int)TInfo.Width.getQuantity()
845	<< (int)Ptr.getAlignment().getQuantity();
846	}
847	if (Oversized) {
848	Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
849	<< (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
850	}
851
852	llvm::Value *Order = EmitScalarExpr(E: E->getOrder());
853	llvm::Value *Scope =
854	E->getScopeModel() ? EmitScalarExpr(E: E->getScope()) : nullptr;
855	bool ShouldCastToIntPtrTy = true;
856
857	switch (E->getOp()) {
858	case AtomicExpr::AO__c11_atomic_init:
859	case AtomicExpr::AO__opencl_atomic_init:
860	llvm_unreachable("Already handled above with EmitAtomicInit!");
861
862	case AtomicExpr::AO__atomic_load_n:
863	case AtomicExpr::AO__scoped_atomic_load_n:
864	case AtomicExpr::AO__c11_atomic_load:
865	case AtomicExpr::AO__opencl_atomic_load:
866	case AtomicExpr::AO__hip_atomic_load:
867	break;
868
869	case AtomicExpr::AO__atomic_load:
870	case AtomicExpr::AO__scoped_atomic_load:
871	Dest = EmitPointerWithAlignment(Addr: E->getVal1());
872	break;
873
874	case AtomicExpr::AO__atomic_store:
875	case AtomicExpr::AO__scoped_atomic_store:
876	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
877	break;
878
879	case AtomicExpr::AO__atomic_exchange:
880	case AtomicExpr::AO__scoped_atomic_exchange:
881	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
882	Dest = EmitPointerWithAlignment(Addr: E->getVal2());
883	break;
884
885	case AtomicExpr::AO__atomic_compare_exchange:
886	case AtomicExpr::AO__atomic_compare_exchange_n:
887	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
888	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
889	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
890	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
891	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
892	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
893	case AtomicExpr::AO__scoped_atomic_compare_exchange:
894	case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
895	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
896	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
897	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
898	Val2 = EmitPointerWithAlignment(Addr: E->getVal2());
899	else
900	Val2 = EmitValToTemp(CGF&: *this, E: E->getVal2());
901	OrderFail = EmitScalarExpr(E: E->getOrderFail());
902	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
903	E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
904	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
905	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
906	IsWeak = EmitScalarExpr(E: E->getWeak());
907	break;
908
909	case AtomicExpr::AO__c11_atomic_fetch_add:
910	case AtomicExpr::AO__c11_atomic_fetch_sub:
911	case AtomicExpr::AO__hip_atomic_fetch_add:
912	case AtomicExpr::AO__hip_atomic_fetch_sub:
913	case AtomicExpr::AO__opencl_atomic_fetch_add:
914	case AtomicExpr::AO__opencl_atomic_fetch_sub:
915	if (MemTy->isPointerType()) {
916	// For pointer arithmetic, we're required to do a bit of math:
917	// adding 1 to an int* is not the same as adding 1 to a uintptr_t.
918	// ... but only for the C11 builtins. The GNU builtins expect the
919	// user to multiply by sizeof(T).
920	QualType Val1Ty = E->getVal1()->getType();
921	llvm::Value *Val1Scalar = EmitScalarExpr(E: E->getVal1());
922	CharUnits PointeeIncAmt =
923	getContext().getTypeSizeInChars(T: MemTy->getPointeeType());
924	Val1Scalar = Builder.CreateMul(LHS: Val1Scalar, RHS: CGM.getSize(numChars: PointeeIncAmt));
925	auto Temp = CreateMemTemp(T: Val1Ty, Name: ".atomictmp");
926	Val1 = Temp;
927	EmitStoreOfScalar(value: Val1Scalar, lvalue: MakeAddrLValue(Addr: Temp, T: Val1Ty));
928	break;
929	}
930	[[fallthrough]];
931	case AtomicExpr::AO__atomic_fetch_add:
932	case AtomicExpr::AO__atomic_fetch_max:
933	case AtomicExpr::AO__atomic_fetch_min:
934	case AtomicExpr::AO__atomic_fetch_sub:
935	case AtomicExpr::AO__atomic_add_fetch:
936	case AtomicExpr::AO__atomic_max_fetch:
937	case AtomicExpr::AO__atomic_min_fetch:
938	case AtomicExpr::AO__atomic_sub_fetch:
939	case AtomicExpr::AO__c11_atomic_fetch_max:
940	case AtomicExpr::AO__c11_atomic_fetch_min:
941	case AtomicExpr::AO__opencl_atomic_fetch_max:
942	case AtomicExpr::AO__opencl_atomic_fetch_min:
943	case AtomicExpr::AO__hip_atomic_fetch_max:
944	case AtomicExpr::AO__hip_atomic_fetch_min:
945	case AtomicExpr::AO__scoped_atomic_fetch_add:
946	case AtomicExpr::AO__scoped_atomic_fetch_max:
947	case AtomicExpr::AO__scoped_atomic_fetch_min:
948	case AtomicExpr::AO__scoped_atomic_fetch_sub:
949	case AtomicExpr::AO__scoped_atomic_add_fetch:
950	case AtomicExpr::AO__scoped_atomic_max_fetch:
951	case AtomicExpr::AO__scoped_atomic_min_fetch:
952	case AtomicExpr::AO__scoped_atomic_sub_fetch:
953	ShouldCastToIntPtrTy = !MemTy->isFloatingType();
954	[[fallthrough]];
955
956	case AtomicExpr::AO__atomic_fetch_and:
957	case AtomicExpr::AO__atomic_fetch_nand:
958	case AtomicExpr::AO__atomic_fetch_or:
959	case AtomicExpr::AO__atomic_fetch_xor:
960	case AtomicExpr::AO__atomic_and_fetch:
961	case AtomicExpr::AO__atomic_nand_fetch:
962	case AtomicExpr::AO__atomic_or_fetch:
963	case AtomicExpr::AO__atomic_xor_fetch:
964	case AtomicExpr::AO__atomic_store_n:
965	case AtomicExpr::AO__atomic_exchange_n:
966	case AtomicExpr::AO__c11_atomic_fetch_and:
967	case AtomicExpr::AO__c11_atomic_fetch_nand:
968	case AtomicExpr::AO__c11_atomic_fetch_or:
969	case AtomicExpr::AO__c11_atomic_fetch_xor:
970	case AtomicExpr::AO__c11_atomic_store:
971	case AtomicExpr::AO__c11_atomic_exchange:
972	case AtomicExpr::AO__hip_atomic_fetch_and:
973	case AtomicExpr::AO__hip_atomic_fetch_or:
974	case AtomicExpr::AO__hip_atomic_fetch_xor:
975	case AtomicExpr::AO__hip_atomic_store:
976	case AtomicExpr::AO__hip_atomic_exchange:
977	case AtomicExpr::AO__opencl_atomic_fetch_and:
978	case AtomicExpr::AO__opencl_atomic_fetch_or:
979	case AtomicExpr::AO__opencl_atomic_fetch_xor:
980	case AtomicExpr::AO__opencl_atomic_store:
981	case AtomicExpr::AO__opencl_atomic_exchange:
982	case AtomicExpr::AO__scoped_atomic_fetch_and:
983	case AtomicExpr::AO__scoped_atomic_fetch_nand:
984	case AtomicExpr::AO__scoped_atomic_fetch_or:
985	case AtomicExpr::AO__scoped_atomic_fetch_xor:
986	case AtomicExpr::AO__scoped_atomic_and_fetch:
987	case AtomicExpr::AO__scoped_atomic_nand_fetch:
988	case AtomicExpr::AO__scoped_atomic_or_fetch:
989	case AtomicExpr::AO__scoped_atomic_xor_fetch:
990	case AtomicExpr::AO__scoped_atomic_store_n:
991	case AtomicExpr::AO__scoped_atomic_exchange_n:
992	Val1 = EmitValToTemp(CGF&: *this, E: E->getVal1());
993	break;
994	}
995
996	QualType RValTy = E->getType().getUnqualifiedType();
997
998	// The inlined atomics only function on iN types, where N is a power of 2. We
999	// need to make sure (via temporaries if necessary) that all incoming values
1000	// are compatible.
1001	LValue AtomicVal = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
1002	AtomicInfo Atomics(*this, AtomicVal);
1003
1004	if (ShouldCastToIntPtrTy) {
1005	Ptr = Atomics.castToAtomicIntPointer(Addr: Ptr);
1006	if (Val1.isValid())
1007	Val1 = Atomics.convertToAtomicIntPointer(Addr: Val1);
1008	if (Val2.isValid())
1009	Val2 = Atomics.convertToAtomicIntPointer(Addr: Val2);
1010	}
1011	if (Dest.isValid()) {
1012	if (ShouldCastToIntPtrTy)
1013	Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1014	} else if (E->isCmpXChg())
1015	Dest = CreateMemTemp(T: RValTy, Name: "cmpxchg.bool");
1016	else if (!RValTy->isVoidType()) {
1017	Dest = Atomics.CreateTempAlloca();
1018	if (ShouldCastToIntPtrTy)
1019	Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1020	}
1021
1022	bool PowerOf2Size = (Size & (Size - 1)) == 0;
1023	bool UseLibcall = !PowerOf2Size || (Size > 16);
1024
1025	// For atomics larger than 16 bytes, emit a libcall from the frontend. This
1026	// avoids the overhead of dealing with excessively-large value types in IR.
1027	// Non-power-of-2 values also lower to libcall here, as they are not currently
1028	// permitted in IR instructions (although that constraint could be relaxed in
1029	// the future). For other cases where a libcall is required on a given
1030	// platform, we let the backend handle it (this includes handling for all of
1031	// the size-optimized libcall variants, which are only valid up to 16 bytes.)
1032	//
1033	// See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1034	if (UseLibcall) {
1035	CallArgList Args;
1036	// For non-optimized library calls, the size is the first parameter.
1037	Args.add(rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: SizeTy, V: Size)),
1038	type: getContext().getSizeType());
1039
1040	// The atomic address is the second parameter.
1041	// The OpenCL atomic library functions only accept pointer arguments to
1042	// generic address space.
1043	auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1044	if (!E->isOpenCL())
1045	return V;
1046	auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1047	if (AS == LangAS::opencl_generic)
1048	return V;
1049	auto DestAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
1050	auto *DestType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: DestAS);
1051
1052	return getTargetHooks().performAddrSpaceCast(
1053	CGF&: *this, V, SrcAddr: AS, DestAddr: LangAS::opencl_generic, DestTy: DestType, IsNonNull: false);
1054	};
1055	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Ptr.getPointer(),
1056	E->getPtr()->getType())),
1057	type: getContext().VoidPtrTy);
1058
1059	// The next 1-3 parameters are op-dependent.
1060	std::string LibCallName;
1061	QualType RetTy;
1062	bool HaveRetTy = false;
1063	switch (E->getOp()) {
1064	case AtomicExpr::AO__c11_atomic_init:
1065	case AtomicExpr::AO__opencl_atomic_init:
1066	llvm_unreachable("Already handled!");
1067
1068	// There is only one libcall for compare an exchange, because there is no
1069	// optimisation benefit possible from a libcall version of a weak compare
1070	// and exchange.
1071	// bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1072	// void *desired, int success, int failure)
1073	case AtomicExpr::AO__atomic_compare_exchange:
1074	case AtomicExpr::AO__atomic_compare_exchange_n:
1075	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1076	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1077	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1078	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1079	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1080	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1081	case AtomicExpr::AO__scoped_atomic_compare_exchange:
1082	case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1083	LibCallName = "__atomic_compare_exchange";
1084	RetTy = getContext().BoolTy;
1085	HaveRetTy = true;
1086	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.getPointer(),
1087	E->getVal1()->getType())),
1088	type: getContext().VoidPtrTy);
1089	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val2.getPointer(),
1090	E->getVal2()->getType())),
1091	type: getContext().VoidPtrTy);
1092	Args.add(rvalue: RValue::get(V: Order), type: getContext().IntTy);
1093	Order = OrderFail;
1094	break;
1095	// void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1096	// int order)
1097	case AtomicExpr::AO__atomic_exchange:
1098	case AtomicExpr::AO__atomic_exchange_n:
1099	case AtomicExpr::AO__c11_atomic_exchange:
1100	case AtomicExpr::AO__hip_atomic_exchange:
1101	case AtomicExpr::AO__opencl_atomic_exchange:
1102	case AtomicExpr::AO__scoped_atomic_exchange:
1103	case AtomicExpr::AO__scoped_atomic_exchange_n:
1104	LibCallName = "__atomic_exchange";
1105	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.getPointer(),
1106	E->getVal1()->getType())),
1107	type: getContext().VoidPtrTy);
1108	break;
1109	// void __atomic_store(size_t size, void *mem, void *val, int order)
1110	case AtomicExpr::AO__atomic_store:
1111	case AtomicExpr::AO__atomic_store_n:
1112	case AtomicExpr::AO__c11_atomic_store:
1113	case AtomicExpr::AO__hip_atomic_store:
1114	case AtomicExpr::AO__opencl_atomic_store:
1115	case AtomicExpr::AO__scoped_atomic_store:
1116	case AtomicExpr::AO__scoped_atomic_store_n:
1117	LibCallName = "__atomic_store";
1118	RetTy = getContext().VoidTy;
1119	HaveRetTy = true;
1120	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.getPointer(),
1121	E->getVal1()->getType())),
1122	type: getContext().VoidPtrTy);
1123	break;
1124	// void __atomic_load(size_t size, void *mem, void *return, int order)
1125	case AtomicExpr::AO__atomic_load:
1126	case AtomicExpr::AO__atomic_load_n:
1127	case AtomicExpr::AO__c11_atomic_load:
1128	case AtomicExpr::AO__hip_atomic_load:
1129	case AtomicExpr::AO__opencl_atomic_load:
1130	case AtomicExpr::AO__scoped_atomic_load:
1131	case AtomicExpr::AO__scoped_atomic_load_n:
1132	LibCallName = "__atomic_load";
1133	break;
1134	case AtomicExpr::AO__atomic_add_fetch:
1135	case AtomicExpr::AO__scoped_atomic_add_fetch:
1136	case AtomicExpr::AO__atomic_fetch_add:
1137	case AtomicExpr::AO__c11_atomic_fetch_add:
1138	case AtomicExpr::AO__hip_atomic_fetch_add:
1139	case AtomicExpr::AO__opencl_atomic_fetch_add:
1140	case AtomicExpr::AO__scoped_atomic_fetch_add:
1141	case AtomicExpr::AO__atomic_and_fetch:
1142	case AtomicExpr::AO__scoped_atomic_and_fetch:
1143	case AtomicExpr::AO__atomic_fetch_and:
1144	case AtomicExpr::AO__c11_atomic_fetch_and:
1145	case AtomicExpr::AO__hip_atomic_fetch_and:
1146	case AtomicExpr::AO__opencl_atomic_fetch_and:
1147	case AtomicExpr::AO__scoped_atomic_fetch_and:
1148	case AtomicExpr::AO__atomic_or_fetch:
1149	case AtomicExpr::AO__scoped_atomic_or_fetch:
1150	case AtomicExpr::AO__atomic_fetch_or:
1151	case AtomicExpr::AO__c11_atomic_fetch_or:
1152	case AtomicExpr::AO__hip_atomic_fetch_or:
1153	case AtomicExpr::AO__opencl_atomic_fetch_or:
1154	case AtomicExpr::AO__scoped_atomic_fetch_or:
1155	case AtomicExpr::AO__atomic_sub_fetch:
1156	case AtomicExpr::AO__scoped_atomic_sub_fetch:
1157	case AtomicExpr::AO__atomic_fetch_sub:
1158	case AtomicExpr::AO__c11_atomic_fetch_sub:
1159	case AtomicExpr::AO__hip_atomic_fetch_sub:
1160	case AtomicExpr::AO__opencl_atomic_fetch_sub:
1161	case AtomicExpr::AO__scoped_atomic_fetch_sub:
1162	case AtomicExpr::AO__atomic_xor_fetch:
1163	case AtomicExpr::AO__scoped_atomic_xor_fetch:
1164	case AtomicExpr::AO__atomic_fetch_xor:
1165	case AtomicExpr::AO__c11_atomic_fetch_xor:
1166	case AtomicExpr::AO__hip_atomic_fetch_xor:
1167	case AtomicExpr::AO__opencl_atomic_fetch_xor:
1168	case AtomicExpr::AO__scoped_atomic_fetch_xor:
1169	case AtomicExpr::AO__atomic_nand_fetch:
1170	case AtomicExpr::AO__atomic_fetch_nand:
1171	case AtomicExpr::AO__c11_atomic_fetch_nand:
1172	case AtomicExpr::AO__scoped_atomic_fetch_nand:
1173	case AtomicExpr::AO__scoped_atomic_nand_fetch:
1174	case AtomicExpr::AO__atomic_min_fetch:
1175	case AtomicExpr::AO__atomic_fetch_min:
1176	case AtomicExpr::AO__c11_atomic_fetch_min:
1177	case AtomicExpr::AO__hip_atomic_fetch_min:
1178	case AtomicExpr::AO__opencl_atomic_fetch_min:
1179	case AtomicExpr::AO__scoped_atomic_fetch_min:
1180	case AtomicExpr::AO__scoped_atomic_min_fetch:
1181	case AtomicExpr::AO__atomic_max_fetch:
1182	case AtomicExpr::AO__atomic_fetch_max:
1183	case AtomicExpr::AO__c11_atomic_fetch_max:
1184	case AtomicExpr::AO__hip_atomic_fetch_max:
1185	case AtomicExpr::AO__opencl_atomic_fetch_max:
1186	case AtomicExpr::AO__scoped_atomic_fetch_max:
1187	case AtomicExpr::AO__scoped_atomic_max_fetch:
1188	llvm_unreachable("Integral atomic operations always become atomicrmw!");
1189	}
1190
1191	if (E->isOpenCL()) {
1192	LibCallName =
1193	std::string("__opencl") + StringRef(LibCallName).drop_front(N: 1).str();
1194	}
1195	// By default, assume we return a value of the atomic type.
1196	if (!HaveRetTy) {
1197	// Value is returned through parameter before the order.
1198	RetTy = getContext().VoidTy;
1199	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Dest.getPointer(), RetTy)),
1200	type: getContext().VoidPtrTy);
1201	}
1202	// Order is always the last parameter.
1203	Args.add(rvalue: RValue::get(V: Order),
1204	type: getContext().IntTy);
1205	if (E->isOpenCL())
1206	Args.add(rvalue: RValue::get(V: Scope), type: getContext().IntTy);
1207
1208	RValue Res = emitAtomicLibcall(CGF&: *this, fnName: LibCallName, resultType: RetTy, args&: Args);
1209	// The value is returned directly from the libcall.
1210	if (E->isCmpXChg())
1211	return Res;
1212
1213	if (RValTy->isVoidType())
1214	return RValue::get(V: nullptr);
1215
1216	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1217	type: RValTy, Loc: E->getExprLoc());
1218	}
1219
1220	bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1221	E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1222	E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1223	E->getOp() == AtomicExpr::AO__atomic_store ||
1224	E->getOp() == AtomicExpr::AO__atomic_store_n ||
1225	E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1226	E->getOp() == AtomicExpr::AO__scoped_atomic_store_n;
1227	bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1228	E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1229	E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1230	E->getOp() == AtomicExpr::AO__atomic_load ||
1231	E->getOp() == AtomicExpr::AO__atomic_load_n ||
1232	E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1233	E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1234
1235	if (isa<llvm::ConstantInt>(Val: Order)) {
1236	auto ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
1237	// We should not ever get to a case where the ordering isn't a valid C ABI
1238	// value, but it's hard to enforce that in general.
1239	if (llvm::isValidAtomicOrderingCABI(I: ord))
1240	switch ((llvm::AtomicOrderingCABI)ord) {
1241	case llvm::AtomicOrderingCABI::relaxed:
1242	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1243	Order: llvm::AtomicOrdering::Monotonic, Scope);
1244	break;
1245	case llvm::AtomicOrderingCABI::consume:
1246	case llvm::AtomicOrderingCABI::acquire:
1247	if (IsStore)
1248	break; // Avoid crashing on code with undefined behavior
1249	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1250	Order: llvm::AtomicOrdering::Acquire, Scope);
1251	break;
1252	case llvm::AtomicOrderingCABI::release:
1253	if (IsLoad)
1254	break; // Avoid crashing on code with undefined behavior
1255	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1256	Order: llvm::AtomicOrdering::Release, Scope);
1257	break;
1258	case llvm::AtomicOrderingCABI::acq_rel:
1259	if (IsLoad || IsStore)
1260	break; // Avoid crashing on code with undefined behavior
1261	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1262	Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1263	break;
1264	case llvm::AtomicOrderingCABI::seq_cst:
1265	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1266	Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1267	break;
1268	}
1269	if (RValTy->isVoidType())
1270	return RValue::get(V: nullptr);
1271
1272	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1273	type: RValTy, Loc: E->getExprLoc());
1274	}
1275
1276	// Long case, when Order isn't obviously constant.
1277
1278	// Create all the relevant BB's
1279	llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1280	*ReleaseBB = nullptr, *AcqRelBB = nullptr,
1281	*SeqCstBB = nullptr;
1282	MonotonicBB = createBasicBlock(name: "monotonic", parent: CurFn);
1283	if (!IsStore)
1284	AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
1285	if (!IsLoad)
1286	ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
1287	if (!IsLoad && !IsStore)
1288	AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
1289	SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
1290	llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
1291
1292	// Create the switch for the split
1293	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
1294	// doesn't matter unless someone is crazy enough to use something that
1295	// doesn't fold to a constant for the ordering.
1296	Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
1297	llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: MonotonicBB);
1298
1299	// Emit all the different atomics
1300	Builder.SetInsertPoint(MonotonicBB);
1301	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1302	Order: llvm::AtomicOrdering::Monotonic, Scope);
1303	Builder.CreateBr(Dest: ContBB);
1304	if (!IsStore) {
1305	Builder.SetInsertPoint(AcquireBB);
1306	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1307	Order: llvm::AtomicOrdering::Acquire, Scope);
1308	Builder.CreateBr(Dest: ContBB);
1309	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
1310	Dest: AcquireBB);
1311	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
1312	Dest: AcquireBB);
1313	}
1314	if (!IsLoad) {
1315	Builder.SetInsertPoint(ReleaseBB);
1316	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1317	Order: llvm::AtomicOrdering::Release, Scope);
1318	Builder.CreateBr(Dest: ContBB);
1319	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::release),
1320	Dest: ReleaseBB);
1321	}
1322	if (!IsLoad && !IsStore) {
1323	Builder.SetInsertPoint(AcqRelBB);
1324	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1325	Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1326	Builder.CreateBr(Dest: ContBB);
1327	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acq_rel),
1328	Dest: AcqRelBB);
1329	}
1330	Builder.SetInsertPoint(SeqCstBB);
1331	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1332	Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1333	Builder.CreateBr(Dest: ContBB);
1334	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
1335	Dest: SeqCstBB);
1336
1337	// Cleanup and return
1338	Builder.SetInsertPoint(ContBB);
1339	if (RValTy->isVoidType())
1340	return RValue::get(V: nullptr);
1341
1342	assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1343	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1344	type: RValTy, Loc: E->getExprLoc());
1345	}
1346
1347	Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1348	llvm::IntegerType *ty =
1349	llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: AtomicSizeInBits);
1350	return addr.withElementType(ElemTy: ty);
1351	}
1352
1353	Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1354	llvm::Type *Ty = Addr.getElementType();
1355	uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1356	if (SourceSizeInBits != AtomicSizeInBits) {
1357	Address Tmp = CreateTempAlloca();
1358	CGF.Builder.CreateMemCpy(Dest: Tmp, Src: Addr,
1359	Size: std::min(a: AtomicSizeInBits, b: SourceSizeInBits) / 8);
1360	Addr = Tmp;
1361	}
1362
1363	return castToAtomicIntPointer(addr: Addr);
1364	}
1365
1366	RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1367	AggValueSlot resultSlot,
1368	SourceLocation loc,
1369	bool asValue) const {
1370	if (LVal.isSimple()) {
1371	if (EvaluationKind == TEK_Aggregate)
1372	return resultSlot.asRValue();
1373
1374	// Drill into the padding structure if we have one.
1375	if (hasPadding())
1376	addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
1377
1378	// Otherwise, just convert the temporary to an r-value using the
1379	// normal conversion routine.
1380	return CGF.convertTempToRValue(addr, type: getValueType(), Loc: loc);
1381	}
1382	if (!asValue)
1383	// Get RValue from temp memory as atomic for non-simple lvalues
1384	return RValue::get(V: CGF.Builder.CreateLoad(Addr: addr));
1385	if (LVal.isBitField())
1386	return CGF.EmitLoadOfBitfieldLValue(
1387	LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1388	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1389	if (LVal.isVectorElt())
1390	return CGF.EmitLoadOfLValue(
1391	LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1392	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1393	assert(LVal.isExtVectorElt());
1394	return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1395	addr, LVal.getExtVectorElts(), LVal.getType(),
1396	LVal.getBaseInfo(), TBAAAccessInfo()));
1397	}
1398
1399	RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1400	AggValueSlot ResultSlot,
1401	SourceLocation Loc,
1402	bool AsValue) const {
1403	// Try not to in some easy cases.
1404	assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1405	if (getEvaluationKind() == TEK_Scalar &&
1406	(((!LVal.isBitField() ||
1407	LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1408	!hasPadding()) ||
1409	!AsValue)) {
1410	auto *ValTy = AsValue
1411	? CGF.ConvertTypeForMem(ValueTy)
1412	: getAtomicAddress().getElementType();
1413	if (ValTy->isIntegerTy()) {
1414	assert(IntVal->getType() == ValTy && "Different integer types.");
1415	return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1416	} else if (ValTy->isPointerTy())
1417	return RValue::get(V: CGF.Builder.CreateIntToPtr(V: IntVal, DestTy: ValTy));
1418	else if (llvm::CastInst::isBitCastable(SrcTy: IntVal->getType(), DestTy: ValTy))
1419	return RValue::get(V: CGF.Builder.CreateBitCast(V: IntVal, DestTy: ValTy));
1420	}
1421
1422	// Create a temporary. This needs to be big enough to hold the
1423	// atomic integer.
1424	Address Temp = Address::invalid();
1425	bool TempIsVolatile = false;
1426	if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1427	assert(!ResultSlot.isIgnored());
1428	Temp = ResultSlot.getAddress();
1429	TempIsVolatile = ResultSlot.isVolatile();
1430	} else {
1431	Temp = CreateTempAlloca();
1432	}
1433
1434	// Slam the integer into the temporary.
1435	Address CastTemp = castToAtomicIntPointer(addr: Temp);
1436	CGF.Builder.CreateStore(Val: IntVal, Addr: CastTemp)
1437	->setVolatile(TempIsVolatile);
1438
1439	return convertAtomicTempToRValue(addr: Temp, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1440	}
1441
1442	void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1443	llvm::AtomicOrdering AO, bool) {
1444	// void __atomic_load(size_t size, void *mem, void *return, int order);
1445	CallArgList Args;
1446	Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1447	Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1448	Args.add(rvalue: RValue::get(V: AddForLoaded), type: CGF.getContext().VoidPtrTy);
1449	Args.add(
1450	rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO))),
1451	type: CGF.getContext().IntTy);
1452	emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1453	}
1454
1455	llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1456	bool IsVolatile) {
1457	// Okay, we're doing this natively.
1458	Address Addr = getAtomicAddressAsAtomicIntPointer();
1459	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, Name: "atomic-load");
1460	Load->setAtomic(Ordering: AO);
1461
1462	// Other decoration.
1463	if (IsVolatile)
1464	Load->setVolatile(true);
1465	CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1466	return Load;
1467	}
1468
1469	/// An LValue is a candidate for having its loads and stores be made atomic if
1470	/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1471	/// performing such an operation can be performed without a libcall.
1472	bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1473	if (!CGM.getLangOpts().MSVolatile) return false;
1474	AtomicInfo AI(*this, LV);
1475	bool IsVolatile = LV.isVolatile() || hasVolatileMember(T: LV.getType());
1476	// An atomic is inline if we don't need to use a libcall.
1477	bool AtomicIsInline = !AI.shouldUseLibcall();
1478	// MSVC doesn't seem to do this for types wider than a pointer.
1479	if (getContext().getTypeSize(T: LV.getType()) >
1480	getContext().getTypeSize(T: getContext().getIntPtrType()))
1481	return false;
1482	return IsVolatile && AtomicIsInline;
1483	}
1484
1485	RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1486	AggValueSlot Slot) {
1487	llvm::AtomicOrdering AO;
1488	bool IsVolatile = LV.isVolatileQualified();
1489	if (LV.getType()->isAtomicType()) {
1490	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1491	} else {
1492	AO = llvm::AtomicOrdering::Acquire;
1493	IsVolatile = true;
1494	}
1495	return EmitAtomicLoad(lvalue: LV, loc: SL, AO, IsVolatile, slot: Slot);
1496	}
1497
1498	RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1499	bool AsValue, llvm::AtomicOrdering AO,
1500	bool IsVolatile) {
1501	// Check whether we should use a library call.
1502	if (shouldUseLibcall()) {
1503	Address TempAddr = Address::invalid();
1504	if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1505	assert(getEvaluationKind() == TEK_Aggregate);
1506	TempAddr = ResultSlot.getAddress();
1507	} else
1508	TempAddr = CreateTempAlloca();
1509
1510	EmitAtomicLoadLibcall(AddForLoaded: TempAddr.getPointer(), AO, IsVolatile);
1511
1512	// Okay, turn that back into the original value or whole atomic (for
1513	// non-simple lvalues) type.
1514	return convertAtomicTempToRValue(addr: TempAddr, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1515	}
1516
1517	// Okay, we're doing this natively.
1518	auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1519
1520	// If we're ignoring an aggregate return, don't do anything.
1521	if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1522	return RValue::getAggregate(addr: Address::invalid(), isVolatile: false);
1523
1524	// Okay, turn that back into the original value or atomic (for non-simple
1525	// lvalues) type.
1526	return ConvertIntToValueOrAtomic(IntVal: Load, ResultSlot, Loc, AsValue);
1527	}
1528
1529	/// Emit a load from an l-value of atomic type. Note that the r-value
1530	/// we produce is an r-value of the atomic *value* type.
1531	RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1532	llvm::AtomicOrdering AO, bool IsVolatile,
1533	AggValueSlot resultSlot) {
1534	AtomicInfo Atomics(*this, src);
1535	return Atomics.EmitAtomicLoad(ResultSlot: resultSlot, Loc: loc, /*AsValue=*/true, AO,
1536	IsVolatile);
1537	}
1538
1539	/// Copy an r-value into memory as part of storing to an atomic type.
1540	/// This needs to create a bit-pattern suitable for atomic operations.
1541	void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1542	assert(LVal.isSimple());
1543	// If we have an r-value, the rvalue should be of the atomic type,
1544	// which means that the caller is responsible for having zeroed
1545	// any padding. Just do an aggregate copy of that type.
1546	if (rvalue.isAggregate()) {
1547	LValue Dest = CGF.MakeAddrLValue(Addr: getAtomicAddress(), T: getAtomicType());
1548	LValue Src = CGF.MakeAddrLValue(Addr: rvalue.getAggregateAddress(),
1549	T: getAtomicType());
1550	bool IsVolatile = rvalue.isVolatileQualified() ||
1551	LVal.isVolatileQualified();
1552	CGF.EmitAggregateCopy(Dest, Src, EltTy: getAtomicType(),
1553	MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: IsVolatile);
1554	return;
1555	}
1556
1557	// Okay, otherwise we're copying stuff.
1558
1559	// Zero out the buffer if necessary.
1560	emitMemSetZeroIfNecessary();
1561
1562	// Drill past the padding if present.
1563	LValue TempLVal = projectValue();
1564
1565	// Okay, store the rvalue in.
1566	if (rvalue.isScalar()) {
1567	CGF.EmitStoreOfScalar(value: rvalue.getScalarVal(), lvalue: TempLVal, /*init*/ isInit: true);
1568	} else {
1569	CGF.EmitStoreOfComplex(V: rvalue.getComplexVal(), dest: TempLVal, /*init*/ isInit: true);
1570	}
1571	}
1572
1573
1574	/// Materialize an r-value into memory for the purposes of storing it
1575	/// to an atomic type.
1576	Address AtomicInfo::materializeRValue(RValue rvalue) const {
1577	// Aggregate r-values are already in memory, and EmitAtomicStore
1578	// requires them to be values of the atomic type.
1579	if (rvalue.isAggregate())
1580	return rvalue.getAggregateAddress();
1581
1582	// Otherwise, make a temporary and materialize into it.
1583	LValue TempLV = CGF.MakeAddrLValue(Addr: CreateTempAlloca(), T: getAtomicType());
1584	AtomicInfo Atomics(CGF, TempLV);
1585	Atomics.emitCopyIntoMemory(rvalue);
1586	return TempLV.getAddress(CGF);
1587	}
1588
1589	llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1590	// If we've got a scalar value of the right size, try to avoid going
1591	// through memory.
1592	if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1593	llvm::Value *Value = RVal.getScalarVal();
1594	if (isa<llvm::IntegerType>(Val: Value->getType()))
1595	return CGF.EmitToMemory(Value, ValueTy);
1596	else {
1597	llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1598	CGF.getLLVMContext(),
1599	LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1600	if (isa<llvm::PointerType>(Val: Value->getType()))
1601	return CGF.Builder.CreatePtrToInt(V: Value, DestTy: InputIntTy);
1602	else if (llvm::BitCastInst::isBitCastable(SrcTy: Value->getType(), DestTy: InputIntTy))
1603	return CGF.Builder.CreateBitCast(V: Value, DestTy: InputIntTy);
1604	}
1605	}
1606	// Otherwise, we need to go through memory.
1607	// Put the r-value in memory.
1608	Address Addr = materializeRValue(rvalue: RVal);
1609
1610	// Cast the temporary to the atomic int type and pull a value out.
1611	Addr = castToAtomicIntPointer(addr: Addr);
1612	return CGF.Builder.CreateLoad(Addr);
1613	}
1614
1615	std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1616	llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1617	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1618	// Do the atomic store.
1619	Address Addr = getAtomicAddressAsAtomicIntPointer();
1620	auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, Cmp: ExpectedVal, New: DesiredVal,
1621	SuccessOrdering: Success, FailureOrdering: Failure);
1622	// Other decoration.
1623	Inst->setVolatile(LVal.isVolatileQualified());
1624	Inst->setWeak(IsWeak);
1625
1626	// Okay, turn that back into the original value type.
1627	auto *PreviousVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/0);
1628	auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/1);
1629	return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
1630	}
1631
1632	llvm::Value *
1633	AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1634	llvm::Value *DesiredAddr,
1635	llvm::AtomicOrdering Success,
1636	llvm::AtomicOrdering Failure) {
1637	// bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1638	// void *desired, int success, int failure);
1639	CallArgList Args;
1640	Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1641	Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1642	Args.add(rvalue: RValue::get(V: ExpectedAddr), type: CGF.getContext().VoidPtrTy);
1643	Args.add(rvalue: RValue::get(V: DesiredAddr), type: CGF.getContext().VoidPtrTy);
1644	Args.add(rvalue: RValue::get(
1645	V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Success))),
1646	type: CGF.getContext().IntTy);
1647	Args.add(rvalue: RValue::get(
1648	V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Failure))),
1649	type: CGF.getContext().IntTy);
1650	auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1651	CGF.getContext().BoolTy, Args);
1652
1653	return SuccessFailureRVal.getScalarVal();
1654	}
1655
1656	std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1657	RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1658	llvm::AtomicOrdering Failure, bool IsWeak) {
1659	// Check whether we should use a library call.
1660	if (shouldUseLibcall()) {
1661	// Produce a source address.
1662	Address ExpectedAddr = materializeRValue(rvalue: Expected);
1663	Address DesiredAddr = materializeRValue(rvalue: Desired);
1664	auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedAddr.getPointer(),
1665	DesiredAddr: DesiredAddr.getPointer(),
1666	Success, Failure);
1667	return std::make_pair(
1668	x: convertAtomicTempToRValue(addr: ExpectedAddr, resultSlot: AggValueSlot::ignored(),
1669	loc: SourceLocation(), /*AsValue=*/asValue: false),
1670	y&: Res);
1671	}
1672
1673	// If we've got a scalar value of the right size, try to avoid going
1674	// through memory.
1675	auto *ExpectedVal = convertRValueToInt(RVal: Expected);
1676	auto *DesiredVal = convertRValueToInt(RVal: Desired);
1677	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1678	Failure, IsWeak);
1679	return std::make_pair(
1680	x: ConvertIntToValueOrAtomic(IntVal: Res.first, ResultSlot: AggValueSlot::ignored(),
1681	Loc: SourceLocation(), /*AsValue=*/false),
1682	y&: Res.second);
1683	}
1684
1685	static void
1686	EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1687	const llvm::function_ref<RValue(RValue)> &UpdateOp,
1688	Address DesiredAddr) {
1689	RValue UpRVal;
1690	LValue AtomicLVal = Atomics.getAtomicLValue();
1691	LValue DesiredLVal;
1692	if (AtomicLVal.isSimple()) {
1693	UpRVal = OldRVal;
1694	DesiredLVal = CGF.MakeAddrLValue(Addr: DesiredAddr, T: AtomicLVal.getType());
1695	} else {
1696	// Build new lvalue for temp address.
1697	Address Ptr = Atomics.materializeRValue(rvalue: OldRVal);
1698	LValue UpdateLVal;
1699	if (AtomicLVal.isBitField()) {
1700	UpdateLVal =
1701	LValue::MakeBitfield(Addr: Ptr, Info: AtomicLVal.getBitFieldInfo(),
1702	type: AtomicLVal.getType(),
1703	BaseInfo: AtomicLVal.getBaseInfo(),
1704	TBAAInfo: AtomicLVal.getTBAAInfo());
1705	DesiredLVal =
1706	LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1707	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1708	TBAAInfo: AtomicLVal.getTBAAInfo());
1709	} else if (AtomicLVal.isVectorElt()) {
1710	UpdateLVal = LValue::MakeVectorElt(vecAddress: Ptr, Idx: AtomicLVal.getVectorIdx(),
1711	type: AtomicLVal.getType(),
1712	BaseInfo: AtomicLVal.getBaseInfo(),
1713	TBAAInfo: AtomicLVal.getTBAAInfo());
1714	DesiredLVal = LValue::MakeVectorElt(
1715	vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(), type: AtomicLVal.getType(),
1716	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1717	} else {
1718	assert(AtomicLVal.isExtVectorElt());
1719	UpdateLVal = LValue::MakeExtVectorElt(vecAddress: Ptr, Elts: AtomicLVal.getExtVectorElts(),
1720	type: AtomicLVal.getType(),
1721	BaseInfo: AtomicLVal.getBaseInfo(),
1722	TBAAInfo: AtomicLVal.getTBAAInfo());
1723	DesiredLVal = LValue::MakeExtVectorElt(
1724	vecAddress: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1725	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1726	}
1727	UpRVal = CGF.EmitLoadOfLValue(V: UpdateLVal, Loc: SourceLocation());
1728	}
1729	// Store new value in the corresponding memory area.
1730	RValue NewRVal = UpdateOp(UpRVal);
1731	if (NewRVal.isScalar()) {
1732	CGF.EmitStoreThroughLValue(Src: NewRVal, Dst: DesiredLVal);
1733	} else {
1734	assert(NewRVal.isComplex());
1735	CGF.EmitStoreOfComplex(V: NewRVal.getComplexVal(), dest: DesiredLVal,
1736	/*isInit=*/false);
1737	}
1738	}
1739
1740	void AtomicInfo::EmitAtomicUpdateLibcall(
1741	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1742	bool IsVolatile) {
1743	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1744
1745	Address ExpectedAddr = CreateTempAlloca();
1746
1747	EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.getPointer(), AO, IsVolatile);
1748	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1749	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1750	CGF.EmitBlock(BB: ContBB);
1751	Address DesiredAddr = CreateTempAlloca();
1752	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1753	requiresMemSetZero(getAtomicAddress().getElementType())) {
1754	auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1755	CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1756	}
1757	auto OldRVal = convertAtomicTempToRValue(addr: ExpectedAddr,
1758	resultSlot: AggValueSlot::ignored(),
1759	loc: SourceLocation(), /*AsValue=*/asValue: false);
1760	EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr);
1761	auto *Res =
1762	EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedAddr.getPointer(),
1763	DesiredAddr: DesiredAddr.getPointer(),
1764	Success: AO, Failure);
1765	CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1766	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1767	}
1768
1769	void AtomicInfo::EmitAtomicUpdateOp(
1770	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1771	bool IsVolatile) {
1772	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1773
1774	// Do the atomic load.
1775	auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile);
1776	// For non-simple lvalues perform compare-and-swap procedure.
1777	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1778	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1779	auto *CurBB = CGF.Builder.GetInsertBlock();
1780	CGF.EmitBlock(BB: ContBB);
1781	llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1782	/*NumReservedValues=*/2);
1783	PHI->addIncoming(V: OldVal, BB: CurBB);
1784	Address NewAtomicAddr = CreateTempAlloca();
1785	Address NewAtomicIntAddr = castToAtomicIntPointer(addr: NewAtomicAddr);
1786	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1787	requiresMemSetZero(getAtomicAddress().getElementType())) {
1788	CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1789	}
1790	auto OldRVal = ConvertIntToValueOrAtomic(IntVal: PHI, ResultSlot: AggValueSlot::ignored(),
1791	Loc: SourceLocation(), /*AsValue=*/false);
1792	EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr: NewAtomicAddr);
1793	auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1794	// Try to write new value using cmpxchg operation.
1795	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1796	PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1797	CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1798	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1799	}
1800
1801	static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1802	RValue UpdateRVal, Address DesiredAddr) {
1803	LValue AtomicLVal = Atomics.getAtomicLValue();
1804	LValue DesiredLVal;
1805	// Build new lvalue for temp address.
1806	if (AtomicLVal.isBitField()) {
1807	DesiredLVal =
1808	LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1809	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1810	TBAAInfo: AtomicLVal.getTBAAInfo());
1811	} else if (AtomicLVal.isVectorElt()) {
1812	DesiredLVal =
1813	LValue::MakeVectorElt(vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(),
1814	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1815	TBAAInfo: AtomicLVal.getTBAAInfo());
1816	} else {
1817	assert(AtomicLVal.isExtVectorElt());
1818	DesiredLVal = LValue::MakeExtVectorElt(
1819	vecAddress: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1820	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1821	}
1822	// Store new value in the corresponding memory area.
1823	assert(UpdateRVal.isScalar());
1824	CGF.EmitStoreThroughLValue(Src: UpdateRVal, Dst: DesiredLVal);
1825	}
1826
1827	void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1828	RValue UpdateRVal, bool IsVolatile) {
1829	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1830
1831	Address ExpectedAddr = CreateTempAlloca();
1832
1833	EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.getPointer(), AO, IsVolatile);
1834	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1835	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1836	CGF.EmitBlock(BB: ContBB);
1837	Address DesiredAddr = CreateTempAlloca();
1838	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1839	requiresMemSetZero(getAtomicAddress().getElementType())) {
1840	auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1841	CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1842	}
1843	EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr);
1844	auto *Res =
1845	EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedAddr.getPointer(),
1846	DesiredAddr: DesiredAddr.getPointer(),
1847	Success: AO, Failure);
1848	CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1849	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1850	}
1851
1852	void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1853	bool IsVolatile) {
1854	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1855
1856	// Do the atomic load.
1857	auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile);
1858	// For non-simple lvalues perform compare-and-swap procedure.
1859	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1860	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1861	auto *CurBB = CGF.Builder.GetInsertBlock();
1862	CGF.EmitBlock(BB: ContBB);
1863	llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1864	/*NumReservedValues=*/2);
1865	PHI->addIncoming(V: OldVal, BB: CurBB);
1866	Address NewAtomicAddr = CreateTempAlloca();
1867	Address NewAtomicIntAddr = castToAtomicIntPointer(addr: NewAtomicAddr);
1868	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1869	requiresMemSetZero(getAtomicAddress().getElementType())) {
1870	CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1871	}
1872	EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr: NewAtomicAddr);
1873	auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1874	// Try to write new value using cmpxchg operation.
1875	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1876	PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1877	CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1878	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1879	}
1880
1881	void AtomicInfo::EmitAtomicUpdate(
1882	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1883	bool IsVolatile) {
1884	if (shouldUseLibcall()) {
1885	EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1886	} else {
1887	EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1888	}
1889	}
1890
1891	void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1892	bool IsVolatile) {
1893	if (shouldUseLibcall()) {
1894	EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1895	} else {
1896	EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1897	}
1898	}
1899
1900	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1901	bool isInit) {
1902	bool IsVolatile = lvalue.isVolatileQualified();
1903	llvm::AtomicOrdering AO;
1904	if (lvalue.getType()->isAtomicType()) {
1905	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1906	} else {
1907	AO = llvm::AtomicOrdering::Release;
1908	IsVolatile = true;
1909	}
1910	return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1911	}
1912
1913	/// Emit a store to an l-value of atomic type.
1914	///
1915	/// Note that the r-value is expected to be an r-value *of the atomic
1916	/// type*; this means that for aggregate r-values, it should include
1917	/// storage for any padding that was necessary.
1918	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1919	llvm::AtomicOrdering AO, bool IsVolatile,
1920	bool isInit) {
1921	// If this is an aggregate r-value, it should agree in type except
1922	// maybe for address-space qualification.
1923	assert(!rvalue.isAggregate() ||
1924	rvalue.getAggregateAddress().getElementType() ==
1925	dest.getAddress(*this).getElementType());
1926
1927	AtomicInfo atomics(*this, dest);
1928	LValue LVal = atomics.getAtomicLValue();
1929
1930	// If this is an initialization, just put the value there normally.
1931	if (LVal.isSimple()) {
1932	if (isInit) {
1933	atomics.emitCopyIntoMemory(rvalue);
1934	return;
1935	}
1936
1937	// Check whether we should use a library call.
1938	if (atomics.shouldUseLibcall()) {
1939	// Produce a source address.
1940	Address srcAddr = atomics.materializeRValue(rvalue);
1941
1942	// void __atomic_store(size_t size, void *mem, void *val, int order)
1943	CallArgList args;
1944	args.add(rvalue: RValue::get(V: atomics.getAtomicSizeValue()),
1945	type: getContext().getSizeType());
1946	args.add(rvalue: RValue::get(V: atomics.getAtomicPointer()), type: getContext().VoidPtrTy);
1947	args.add(rvalue: RValue::get(V: srcAddr.getPointer()), type: getContext().VoidPtrTy);
1948	args.add(
1949	rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: IntTy, V: (int)llvm::toCABI(AO))),
1950	type: getContext().IntTy);
1951	emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1952	return;
1953	}
1954
1955	// Okay, we're doing this natively.
1956	llvm::Value *intValue = atomics.convertRValueToInt(RVal: rvalue);
1957
1958	// Do the atomic store.
1959	Address addr = atomics.castToAtomicIntPointer(addr: atomics.getAtomicAddress());
1960	intValue = Builder.CreateIntCast(
1961	V: intValue, DestTy: addr.getElementType(), /*isSigned=*/false);
1962	llvm::StoreInst *store = Builder.CreateStore(Val: intValue, Addr: addr);
1963
1964	if (AO == llvm::AtomicOrdering::Acquire)
1965	AO = llvm::AtomicOrdering::Monotonic;
1966	else if (AO == llvm::AtomicOrdering::AcquireRelease)
1967	AO = llvm::AtomicOrdering::Release;
1968	// Initializations don't need to be atomic.
1969	if (!isInit)
1970	store->setAtomic(Ordering: AO);
1971
1972	// Other decoration.
1973	if (IsVolatile)
1974	store->setVolatile(true);
1975	CGM.DecorateInstructionWithTBAA(Inst: store, TBAAInfo: dest.getTBAAInfo());
1976	return;
1977	}
1978
1979	// Emit simple atomic update operation.
1980	atomics.EmitAtomicUpdate(AO, UpdateRVal: rvalue, IsVolatile);
1981	}
1982
1983	/// Emit a compare-and-exchange op for atomic type.
1984	///
1985	std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1986	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1987	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1988	AggValueSlot Slot) {
1989	// If this is an aggregate r-value, it should agree in type except
1990	// maybe for address-space qualification.
1991	assert(!Expected.isAggregate() ||
1992	Expected.getAggregateAddress().getElementType() ==
1993	Obj.getAddress(*this).getElementType());
1994	assert(!Desired.isAggregate() ||
1995	Desired.getAggregateAddress().getElementType() ==
1996	Obj.getAddress(*this).getElementType());
1997	AtomicInfo Atomics(*this, Obj);
1998
1999	return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2000	IsWeak);
2001	}
2002
2003	void CodeGenFunction::EmitAtomicUpdate(
2004	LValue LVal, llvm::AtomicOrdering AO,
2005	const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2006	AtomicInfo Atomics(*this, LVal);
2007	Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2008	}
2009
2010	void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2011	AtomicInfo atomics(*this, dest);
2012
2013	switch (atomics.getEvaluationKind()) {
2014	case TEK_Scalar: {
2015	llvm::Value *value = EmitScalarExpr(E: init);
2016	atomics.emitCopyIntoMemory(rvalue: RValue::get(V: value));
2017	return;
2018	}
2019
2020	case TEK_Complex: {
2021	ComplexPairTy value = EmitComplexExpr(E: init);
2022	atomics.emitCopyIntoMemory(rvalue: RValue::getComplex(C: value));
2023	return;
2024	}
2025
2026	case TEK_Aggregate: {
2027	// Fix up the destination if the initializer isn't an expression
2028	// of atomic type.
2029	bool Zeroed = false;
2030	if (!init->getType()->isAtomicType()) {
2031	Zeroed = atomics.emitMemSetZeroIfNecessary();
2032	dest = atomics.projectValue();
2033	}
2034
2035	// Evaluate the expression directly into the destination.
2036	AggValueSlot slot = AggValueSlot::forLValue(
2037	LV: dest, CGF&: *this, isDestructed: AggValueSlot::IsNotDestructed,
2038	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2039	mayOverlap: AggValueSlot::DoesNotOverlap,
2040	isZeroed: Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2041
2042	EmitAggExpr(E: init, AS: slot);
2043	return;
2044	}
2045	}
2046	llvm_unreachable("bad evaluation kind");
2047	}
2048