[MS] Fix double evaluation of MSVC builtin arguments
[lldb.git] / clang / lib / CodeGen / CGBuiltin.cpp
1 //===---- CGBuiltin.cpp - Emit LLVM Code for builtins ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Builtin calls as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CGCXXABI.h"
14 #include "CGObjCRuntime.h"
15 #include "CGOpenCLRuntime.h"
16 #include "CGRecordLayout.h"
17 #include "CodeGenFunction.h"
18 #include "CodeGenModule.h"
19 #include "ConstantEmitter.h"
20 #include "PatternInit.h"
21 #include "TargetInfo.h"
22 #include "clang/AST/ASTContext.h"
23 #include "clang/AST/Attr.h"
24 #include "clang/AST/Decl.h"
25 #include "clang/AST/OSLog.h"
26 #include "clang/Basic/TargetBuiltins.h"
27 #include "clang/Basic/TargetInfo.h"
28 #include "clang/CodeGen/CGFunctionInfo.h"
29 #include "llvm/ADT/SmallPtrSet.h"
30 #include "llvm/ADT/StringExtras.h"
31 #include "llvm/Analysis/ValueTracking.h"
32 #include "llvm/IR/DataLayout.h"
33 #include "llvm/IR/InlineAsm.h"
34 #include "llvm/IR/Intrinsics.h"
35 #include "llvm/IR/IntrinsicsAArch64.h"
36 #include "llvm/IR/IntrinsicsAMDGPU.h"
37 #include "llvm/IR/IntrinsicsARM.h"
38 #include "llvm/IR/IntrinsicsBPF.h"
39 #include "llvm/IR/IntrinsicsHexagon.h"
40 #include "llvm/IR/IntrinsicsNVPTX.h"
41 #include "llvm/IR/IntrinsicsPowerPC.h"
42 #include "llvm/IR/IntrinsicsR600.h"
43 #include "llvm/IR/IntrinsicsS390.h"
44 #include "llvm/IR/IntrinsicsWebAssembly.h"
45 #include "llvm/IR/IntrinsicsX86.h"
46 #include "llvm/IR/MDBuilder.h"
47 #include "llvm/IR/MatrixBuilder.h"
48 #include "llvm/Support/ConvertUTF.h"
49 #include "llvm/Support/ScopedPrinter.h"
50 #include "llvm/Support/X86TargetParser.h"
51 #include <sstream>
52
53 using namespace clang;
54 using namespace CodeGen;
55 using namespace llvm;
56
57 static
58 int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
59   return std::min(High, std::max(Low, Value));
60 }
61
62 static void initializeAlloca(CodeGenFunction &CGF, AllocaInst *AI, Value *Size,
63                              Align AlignmentInBytes) {
64   ConstantInt *Byte;
65   switch (CGF.getLangOpts().getTrivialAutoVarInit()) {
66   case LangOptions::TrivialAutoVarInitKind::Uninitialized:
67     // Nothing to initialize.
68     return;
69   case LangOptions::TrivialAutoVarInitKind::Zero:
70     Byte = CGF.Builder.getInt8(0x00);
71     break;
72   case LangOptions::TrivialAutoVarInitKind::Pattern: {
73     llvm::Type *Int8 = llvm::IntegerType::getInt8Ty(CGF.CGM.getLLVMContext());
74     Byte = llvm::dyn_cast<llvm::ConstantInt>(
75         initializationPatternFor(CGF.CGM, Int8));
76     break;
77   }
78   }
79   if (CGF.CGM.stopAutoInit())
80     return;
81   auto *I = CGF.Builder.CreateMemSet(AI, Byte, Size, AlignmentInBytes);
82   I->addAnnotationMetadata("auto-init");
83 }
84
85 /// getBuiltinLibFunction - Given a builtin id for a function like
86 /// "__builtin_fabsf", return a Function* for "fabsf".
87 llvm::Constant *CodeGenModule::getBuiltinLibFunction(const FunctionDecl *FD,
88                                                      unsigned BuiltinID) {
89   assert(Context.BuiltinInfo.isLibFunction(BuiltinID));
90
91   // Get the name, skip over the __builtin_ prefix (if necessary).
92   StringRef Name;
93   GlobalDecl D(FD);
94
95   // If the builtin has been declared explicitly with an assembler label,
96   // use the mangled name. This differs from the plain label on platforms
97   // that prefix labels.
98   if (FD->hasAttr<AsmLabelAttr>())
99     Name = getMangledName(D);
100   else
101     Name = Context.BuiltinInfo.getName(BuiltinID) + 10;
102
103   llvm::FunctionType *Ty =
104     cast<llvm::FunctionType>(getTypes().ConvertType(FD->getType()));
105
106   return GetOrCreateLLVMFunction(Name, Ty, D, /*ForVTable=*/false);
107 }
108
109 /// Emit the conversions required to turn the given value into an
110 /// integer of the given size.
111 static Value *EmitToInt(CodeGenFunction &CGF, llvm::Value *V,
112                         QualType T, llvm::IntegerType *IntType) {
113   V = CGF.EmitToMemory(V, T);
114
115   if (V->getType()->isPointerTy())
116     return CGF.Builder.CreatePtrToInt(V, IntType);
117
118   assert(V->getType() == IntType);
119   return V;
120 }
121
122 static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
123                           QualType T, llvm::Type *ResultType) {
124   V = CGF.EmitFromMemory(V, T);
125
126   if (ResultType->isPointerTy())
127     return CGF.Builder.CreateIntToPtr(V, ResultType);
128
129   assert(V->getType() == ResultType);
130   return V;
131 }
132
133 /// Utility to insert an atomic instruction based on Intrinsic::ID
134 /// and the expression node.
135 static Value *MakeBinaryAtomicValue(
136     CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E,
137     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
138   QualType T = E->getType();
139   assert(E->getArg(0)->getType()->isPointerType());
140   assert(CGF.getContext().hasSameUnqualifiedType(T,
141                                   E->getArg(0)->getType()->getPointeeType()));
142   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
143
144   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
145   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
146
147   llvm::IntegerType *IntType =
148     llvm::IntegerType::get(CGF.getLLVMContext(),
149                            CGF.getContext().getTypeSize(T));
150   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
151
152   llvm::Value *Args[2];
153   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
154   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
155   llvm::Type *ValueType = Args[1]->getType();
156   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
157
158   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
159       Kind, Args[0], Args[1], Ordering);
160   return EmitFromInt(CGF, Result, T, ValueType);
161 }
162
163 static Value *EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E) {
164   Value *Val = CGF.EmitScalarExpr(E->getArg(0));
165   Value *Address = CGF.EmitScalarExpr(E->getArg(1));
166
167   // Convert the type of the pointer to a pointer to the stored type.
168   Val = CGF.EmitToMemory(Val, E->getArg(0)->getType());
169   Value *BC = CGF.Builder.CreateBitCast(
170       Address, llvm::PointerType::getUnqual(Val->getType()), "cast");
171   LValue LV = CGF.MakeNaturalAlignAddrLValue(BC, E->getArg(0)->getType());
172   LV.setNontemporal(true);
173   CGF.EmitStoreOfScalar(Val, LV, false);
174   return nullptr;
175 }
176
177 static Value *EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E) {
178   Value *Address = CGF.EmitScalarExpr(E->getArg(0));
179
180   LValue LV = CGF.MakeNaturalAlignAddrLValue(Address, E->getType());
181   LV.setNontemporal(true);
182   return CGF.EmitLoadOfScalar(LV, E->getExprLoc());
183 }
184
185 static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
186                                llvm::AtomicRMWInst::BinOp Kind,
187                                const CallExpr *E) {
188   return RValue::get(MakeBinaryAtomicValue(CGF, Kind, E));
189 }
190
191 /// Utility to insert an atomic instruction based Intrinsic::ID and
192 /// the expression node, where the return value is the result of the
193 /// operation.
194 static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
195                                    llvm::AtomicRMWInst::BinOp Kind,
196                                    const CallExpr *E,
197                                    Instruction::BinaryOps Op,
198                                    bool Invert = false) {
199   QualType T = E->getType();
200   assert(E->getArg(0)->getType()->isPointerType());
201   assert(CGF.getContext().hasSameUnqualifiedType(T,
202                                   E->getArg(0)->getType()->getPointeeType()));
203   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
204
205   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
206   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
207
208   llvm::IntegerType *IntType =
209     llvm::IntegerType::get(CGF.getLLVMContext(),
210                            CGF.getContext().getTypeSize(T));
211   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
212
213   llvm::Value *Args[2];
214   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
215   llvm::Type *ValueType = Args[1]->getType();
216   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
217   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
218
219   llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
220       Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
221   Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
222   if (Invert)
223     Result =
224         CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
225                                 llvm::ConstantInt::getAllOnesValue(IntType));
226   Result = EmitFromInt(CGF, Result, T, ValueType);
227   return RValue::get(Result);
228 }
229
230 /// Utility to insert an atomic cmpxchg instruction.
231 ///
232 /// @param CGF The current codegen function.
233 /// @param E   Builtin call expression to convert to cmpxchg.
234 ///            arg0 - address to operate on
235 ///            arg1 - value to compare with
236 ///            arg2 - new value
237 /// @param ReturnBool Specifies whether to return success flag of
238 ///                   cmpxchg result or the old value.
239 ///
240 /// @returns result of cmpxchg, according to ReturnBool
241 ///
242 /// Note: In order to lower Microsoft's _InterlockedCompareExchange* intrinsics
243 /// invoke the function EmitAtomicCmpXchgForMSIntrin.
244 static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
245                                      bool ReturnBool) {
246   QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
247   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
248   unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
249
250   llvm::IntegerType *IntType = llvm::IntegerType::get(
251       CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
252   llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
253
254   Value *Args[3];
255   Args[0] = CGF.Builder.CreateBitCast(DestPtr, IntPtrType);
256   Args[1] = CGF.EmitScalarExpr(E->getArg(1));
257   llvm::Type *ValueType = Args[1]->getType();
258   Args[1] = EmitToInt(CGF, Args[1], T, IntType);
259   Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
260
261   Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
262       Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
263       llvm::AtomicOrdering::SequentiallyConsistent);
264   if (ReturnBool)
265     // Extract boolean success flag and zext it to int.
266     return CGF.Builder.CreateZExt(CGF.Builder.CreateExtractValue(Pair, 1),
267                                   CGF.ConvertType(E->getType()));
268   else
269     // Extract old value and emit it using the same type as compare value.
270     return EmitFromInt(CGF, CGF.Builder.CreateExtractValue(Pair, 0), T,
271                        ValueType);
272 }
273
274 /// This function should be invoked to emit atomic cmpxchg for Microsoft's
275 /// _InterlockedCompareExchange* intrinsics which have the following signature:
276 /// T _InterlockedCompareExchange(T volatile *Destination,
277 ///                               T Exchange,
278 ///                               T Comparand);
279 ///
280 /// Whereas the llvm 'cmpxchg' instruction has the following syntax:
281 /// cmpxchg *Destination, Comparand, Exchange.
282 /// So we need to swap Comparand and Exchange when invoking
283 /// CreateAtomicCmpXchg. That is the reason we could not use the above utility
284 /// function MakeAtomicCmpXchgValue since it expects the arguments to be
285 /// already swapped.
286
287 static
288 Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
289     AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
290   assert(E->getArg(0)->getType()->isPointerType());
291   assert(CGF.getContext().hasSameUnqualifiedType(
292       E->getType(), E->getArg(0)->getType()->getPointeeType()));
293   assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
294                                                  E->getArg(1)->getType()));
295   assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
296                                                  E->getArg(2)->getType()));
297
298   auto *Destination = CGF.EmitScalarExpr(E->getArg(0));
299   auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
300   auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
301
302   // For Release ordering, the failure ordering should be Monotonic.
303   auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
304                          AtomicOrdering::Monotonic :
305                          SuccessOrdering;
306
307   auto *Result = CGF.Builder.CreateAtomicCmpXchg(
308                    Destination, Comparand, Exchange,
309                    SuccessOrdering, FailureOrdering);
310   Result->setVolatile(true);
311   return CGF.Builder.CreateExtractValue(Result, 0);
312 }
313
314 static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
315     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
316   assert(E->getArg(0)->getType()->isPointerType());
317
318   auto *IntTy = CGF.ConvertType(E->getType());
319   auto *Result = CGF.Builder.CreateAtomicRMW(
320                    AtomicRMWInst::Add,
321                    CGF.EmitScalarExpr(E->getArg(0)),
322                    ConstantInt::get(IntTy, 1),
323                    Ordering);
324   return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
325 }
326
327 static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E,
328     AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
329   assert(E->getArg(0)->getType()->isPointerType());
330
331   auto *IntTy = CGF.ConvertType(E->getType());
332   auto *Result = CGF.Builder.CreateAtomicRMW(
333                    AtomicRMWInst::Sub,
334                    CGF.EmitScalarExpr(E->getArg(0)),
335                    ConstantInt::get(IntTy, 1),
336                    Ordering);
337   return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
338 }
339
340 // Build a plain volatile load.
341 static Value *EmitISOVolatileLoad(CodeGenFunction &CGF, const CallExpr *E) {
342   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
343   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
344   CharUnits LoadSize = CGF.getContext().getTypeSizeInChars(ElTy);
345   llvm::Type *ITy =
346       llvm::IntegerType::get(CGF.getLLVMContext(), LoadSize.getQuantity() * 8);
347   Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());
348   llvm::LoadInst *Load = CGF.Builder.CreateAlignedLoad(Ptr, LoadSize);
349   Load->setVolatile(true);
350   return Load;
351 }
352
353 // Build a plain volatile store.
354 static Value *EmitISOVolatileStore(CodeGenFunction &CGF, const CallExpr *E) {
355   Value *Ptr = CGF.EmitScalarExpr(E->getArg(0));
356   Value *Value = CGF.EmitScalarExpr(E->getArg(1));
357   QualType ElTy = E->getArg(0)->getType()->getPointeeType();
358   CharUnits StoreSize = CGF.getContext().getTypeSizeInChars(ElTy);
359   llvm::Type *ITy =
360       llvm::IntegerType::get(CGF.getLLVMContext(), StoreSize.getQuantity() * 8);
361   Ptr = CGF.Builder.CreateBitCast(Ptr, ITy->getPointerTo());
362   llvm::StoreInst *Store =
363       CGF.Builder.CreateAlignedStore(Value, Ptr, StoreSize);
364   Store->setVolatile(true);
365   return Store;
366 }
367
368 // Emit a simple mangled intrinsic that has 1 argument and a return type
369 // matching the argument type. Depending on mode, this may be a constrained
370 // floating-point intrinsic.
371 static Value *emitUnaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
372                                 const CallExpr *E, unsigned IntrinsicID,
373                                 unsigned ConstrainedIntrinsicID) {
374   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
375
376   if (CGF.Builder.getIsFPConstrained()) {
377     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
378     return CGF.Builder.CreateConstrainedFPCall(F, { Src0 });
379   } else {
380     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
381     return CGF.Builder.CreateCall(F, Src0);
382   }
383 }
384
385 // Emit an intrinsic that has 2 operands of the same type as its result.
386 // Depending on mode, this may be a constrained floating-point intrinsic.
387 static Value *emitBinaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
388                                 const CallExpr *E, unsigned IntrinsicID,
389                                 unsigned ConstrainedIntrinsicID) {
390   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
391   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
392
393   if (CGF.Builder.getIsFPConstrained()) {
394     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
395     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1 });
396   } else {
397     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
398     return CGF.Builder.CreateCall(F, { Src0, Src1 });
399   }
400 }
401
402 // Emit an intrinsic that has 3 operands of the same type as its result.
403 // Depending on mode, this may be a constrained floating-point intrinsic.
404 static Value *emitTernaryMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
405                                  const CallExpr *E, unsigned IntrinsicID,
406                                  unsigned ConstrainedIntrinsicID) {
407   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
408   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
409   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
410
411   if (CGF.Builder.getIsFPConstrained()) {
412     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Src0->getType());
413     return CGF.Builder.CreateConstrainedFPCall(F, { Src0, Src1, Src2 });
414   } else {
415     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
416     return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
417   }
418 }
419
420 // Emit an intrinsic where all operands are of the same type as the result.
421 // Depending on mode, this may be a constrained floating-point intrinsic.
422 static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
423                                                 unsigned IntrinsicID,
424                                                 unsigned ConstrainedIntrinsicID,
425                                                 llvm::Type *Ty,
426                                                 ArrayRef<Value *> Args) {
427   Function *F;
428   if (CGF.Builder.getIsFPConstrained())
429     F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID, Ty);
430   else
431     F = CGF.CGM.getIntrinsic(IntrinsicID, Ty);
432
433   if (CGF.Builder.getIsFPConstrained())
434     return CGF.Builder.CreateConstrainedFPCall(F, Args);
435   else
436     return CGF.Builder.CreateCall(F, Args);
437 }
438
439 // Emit a simple mangled intrinsic that has 1 argument and a return type
440 // matching the argument type.
441 static Value *emitUnaryBuiltin(CodeGenFunction &CGF,
442                                const CallExpr *E,
443                                unsigned IntrinsicID) {
444   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
445
446   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
447   return CGF.Builder.CreateCall(F, Src0);
448 }
449
450 // Emit an intrinsic that has 2 operands of the same type as its result.
451 static Value *emitBinaryBuiltin(CodeGenFunction &CGF,
452                                 const CallExpr *E,
453                                 unsigned IntrinsicID) {
454   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
455   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
456
457   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
458   return CGF.Builder.CreateCall(F, { Src0, Src1 });
459 }
460
461 // Emit an intrinsic that has 3 operands of the same type as its result.
462 static Value *emitTernaryBuiltin(CodeGenFunction &CGF,
463                                  const CallExpr *E,
464                                  unsigned IntrinsicID) {
465   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
466   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
467   llvm::Value *Src2 = CGF.EmitScalarExpr(E->getArg(2));
468
469   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
470   return CGF.Builder.CreateCall(F, { Src0, Src1, Src2 });
471 }
472
473 // Emit an intrinsic that has 1 float or double operand, and 1 integer.
474 static Value *emitFPIntBuiltin(CodeGenFunction &CGF,
475                                const CallExpr *E,
476                                unsigned IntrinsicID) {
477   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
478   llvm::Value *Src1 = CGF.EmitScalarExpr(E->getArg(1));
479
480   Function *F = CGF.CGM.getIntrinsic(IntrinsicID, Src0->getType());
481   return CGF.Builder.CreateCall(F, {Src0, Src1});
482 }
483
484 // Emit an intrinsic that has overloaded integer result and fp operand.
485 static Value *
486 emitMaybeConstrainedFPToIntRoundBuiltin(CodeGenFunction &CGF, const CallExpr *E,
487                                         unsigned IntrinsicID,
488                                         unsigned ConstrainedIntrinsicID) {
489   llvm::Type *ResultType = CGF.ConvertType(E->getType());
490   llvm::Value *Src0 = CGF.EmitScalarExpr(E->getArg(0));
491
492   if (CGF.Builder.getIsFPConstrained()) {
493     Function *F = CGF.CGM.getIntrinsic(ConstrainedIntrinsicID,
494                                        {ResultType, Src0->getType()});
495     return CGF.Builder.CreateConstrainedFPCall(F, {Src0});
496   } else {
497     Function *F =
498         CGF.CGM.getIntrinsic(IntrinsicID, {ResultType, Src0->getType()});
499     return CGF.Builder.CreateCall(F, Src0);
500   }
501 }
502
503 /// EmitFAbs - Emit a call to @llvm.fabs().
504 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V) {
505   Function *F = CGF.CGM.getIntrinsic(Intrinsic::fabs, V->getType());
506   llvm::CallInst *Call = CGF.Builder.CreateCall(F, V);
507   Call->setDoesNotAccessMemory();
508   return Call;
509 }
510
511 /// Emit the computation of the sign bit for a floating point value. Returns
512 /// the i1 sign bit value.
513 static Value *EmitSignBit(CodeGenFunction &CGF, Value *V) {
514   LLVMContext &C = CGF.CGM.getLLVMContext();
515
516   llvm::Type *Ty = V->getType();
517   int Width = Ty->getPrimitiveSizeInBits();
518   llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
519   V = CGF.Builder.CreateBitCast(V, IntTy);
520   if (Ty->isPPC_FP128Ty()) {
521     // We want the sign bit of the higher-order double. The bitcast we just
522     // did works as if the double-double was stored to memory and then
523     // read as an i128. The "store" will put the higher-order double in the
524     // lower address in both little- and big-Endian modes, but the "load"
525     // will treat those bits as a different part of the i128: the low bits in
526     // little-Endian, the high bits in big-Endian. Therefore, on big-Endian
527     // we need to shift the high bits down to the low before truncating.
528     Width >>= 1;
529     if (CGF.getTarget().isBigEndian()) {
530       Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
531       V = CGF.Builder.CreateLShr(V, ShiftCst);
532     }
533     // We are truncating value in order to extract the higher-order
534     // double, which we will be using to extract the sign from.
535     IntTy = llvm::IntegerType::get(C, Width);
536     V = CGF.Builder.CreateTrunc(V, IntTy);
537   }
538   Value *Zero = llvm::Constant::getNullValue(IntTy);
539   return CGF.Builder.CreateICmpSLT(V, Zero);
540 }
541
542 static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD,
543                               const CallExpr *E, llvm::Constant *calleeValue) {
544   CGCallee callee = CGCallee::forDirect(calleeValue, GlobalDecl(FD));
545   return CGF.EmitCall(E->getCallee()->getType(), callee, E, ReturnValueSlot());
546 }
547
548 /// Emit a call to llvm.{sadd,uadd,ssub,usub,smul,umul}.with.overflow.*
549 /// depending on IntrinsicID.
550 ///
551 /// \arg CGF The current codegen function.
552 /// \arg IntrinsicID The ID for the Intrinsic we wish to generate.
553 /// \arg X The first argument to the llvm.*.with.overflow.*.
554 /// \arg Y The second argument to the llvm.*.with.overflow.*.
555 /// \arg Carry The carry returned by the llvm.*.with.overflow.*.
556 /// \returns The result (i.e. sum/product) returned by the intrinsic.
557 static llvm::Value *EmitOverflowIntrinsic(CodeGenFunction &CGF,
558                                           const llvm::Intrinsic::ID IntrinsicID,
559                                           llvm::Value *X, llvm::Value *Y,
560                                           llvm::Value *&Carry) {
561   // Make sure we have integers of the same width.
562   assert(X->getType() == Y->getType() &&
563          "Arguments must be the same type. (Did you forget to make sure both "
564          "arguments have the same integer width?)");
565
566   Function *Callee = CGF.CGM.getIntrinsic(IntrinsicID, X->getType());
567   llvm::Value *Tmp = CGF.Builder.CreateCall(Callee, {X, Y});
568   Carry = CGF.Builder.CreateExtractValue(Tmp, 1);
569   return CGF.Builder.CreateExtractValue(Tmp, 0);
570 }
571
572 static Value *emitRangedBuiltin(CodeGenFunction &CGF,
573                                 unsigned IntrinsicID,
574                                 int low, int high) {
575     llvm::MDBuilder MDHelper(CGF.getLLVMContext());
576     llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
577     Function *F = CGF.CGM.getIntrinsic(IntrinsicID, {});
578     llvm::Instruction *Call = CGF.Builder.CreateCall(F);
579     Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
580     return Call;
581 }
582
583 namespace {
584   struct WidthAndSignedness {
585     unsigned Width;
586     bool Signed;
587   };
588 }
589
590 static WidthAndSignedness
591 getIntegerWidthAndSignedness(const clang::ASTContext &context,
592                              const clang::QualType Type) {
593   assert(Type->isIntegerType() && "Given type is not an integer.");
594   unsigned Width = Type->isBooleanType()  ? 1
595                    : Type->isExtIntType() ? context.getIntWidth(Type)
596                                           : context.getTypeInfo(Type).Width;
597   bool Signed = Type->isSignedIntegerType();
598   return {Width, Signed};
599 }
600
601 // Given one or more integer types, this function produces an integer type that
602 // encompasses them: any value in one of the given types could be expressed in
603 // the encompassing type.
604 static struct WidthAndSignedness
605 EncompassingIntegerType(ArrayRef<struct WidthAndSignedness> Types) {
606   assert(Types.size() > 0 && "Empty list of types.");
607
608   // If any of the given types is signed, we must return a signed type.
609   bool Signed = false;
610   for (const auto &Type : Types) {
611     Signed |= Type.Signed;
612   }
613
614   // The encompassing type must have a width greater than or equal to the width
615   // of the specified types.  Additionally, if the encompassing type is signed,
616   // its width must be strictly greater than the width of any unsigned types
617   // given.
618   unsigned Width = 0;
619   for (const auto &Type : Types) {
620     unsigned MinWidth = Type.Width + (Signed && !Type.Signed);
621     if (Width < MinWidth) {
622       Width = MinWidth;
623     }
624   }
625
626   return {Width, Signed};
627 }
628
629 Value *CodeGenFunction::EmitVAStartEnd(Value *ArgValue, bool IsStart) {
630   llvm::Type *DestType = Int8PtrTy;
631   if (ArgValue->getType() != DestType)
632     ArgValue =
633         Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
634
635   Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
636   return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
637 }
638
639 /// Checks if using the result of __builtin_object_size(p, @p From) in place of
640 /// __builtin_object_size(p, @p To) is correct
641 static bool areBOSTypesCompatible(int From, int To) {
642   // Note: Our __builtin_object_size implementation currently treats Type=0 and
643   // Type=2 identically. Encoding this implementation detail here may make
644   // improving __builtin_object_size difficult in the future, so it's omitted.
645   return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
646 }
647
648 static llvm::Value *
649 getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType) {
650   return ConstantInt::get(ResType, (Type & 2) ? 0 : -1, /*isSigned=*/true);
651 }
652
653 llvm::Value *
654 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(const Expr *E, unsigned Type,
655                                                  llvm::IntegerType *ResType,
656                                                  llvm::Value *EmittedE,
657                                                  bool IsDynamic) {
658   uint64_t ObjectSize;
659   if (!E->tryEvaluateObjectSize(ObjectSize, getContext(), Type))
660     return emitBuiltinObjectSize(E, Type, ResType, EmittedE, IsDynamic);
661   return ConstantInt::get(ResType, ObjectSize, /*isSigned=*/true);
662 }
663
664 /// Returns a Value corresponding to the size of the given expression.
665 /// This Value may be either of the following:
666 ///   - A llvm::Argument (if E is a param with the pass_object_size attribute on
667 ///     it)
668 ///   - A call to the @llvm.objectsize intrinsic
669 ///
670 /// EmittedE is the result of emitting `E` as a scalar expr. If it's non-null
671 /// and we wouldn't otherwise try to reference a pass_object_size parameter,
672 /// we'll call @llvm.objectsize on EmittedE, rather than emitting E.
673 llvm::Value *
674 CodeGenFunction::emitBuiltinObjectSize(const Expr *E, unsigned Type,
675                                        llvm::IntegerType *ResType,
676                                        llvm::Value *EmittedE, bool IsDynamic) {
677   // We need to reference an argument if the pointer is a parameter with the
678   // pass_object_size attribute.
679   if (auto *D = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts())) {
680     auto *Param = dyn_cast<ParmVarDecl>(D->getDecl());
681     auto *PS = D->getDecl()->getAttr<PassObjectSizeAttr>();
682     if (Param != nullptr && PS != nullptr &&
683         areBOSTypesCompatible(PS->getType(), Type)) {
684       auto Iter = SizeArguments.find(Param);
685       assert(Iter != SizeArguments.end());
686
687       const ImplicitParamDecl *D = Iter->second;
688       auto DIter = LocalDeclMap.find(D);
689       assert(DIter != LocalDeclMap.end());
690
691       return EmitLoadOfScalar(DIter->second, /*Volatile=*/false,
692                               getContext().getSizeType(), E->getBeginLoc());
693     }
694   }
695
696   // LLVM can't handle Type=3 appropriately, and __builtin_object_size shouldn't
697   // evaluate E for side-effects. In either case, we shouldn't lower to
698   // @llvm.objectsize.
699   if (Type == 3 || (!EmittedE && E->HasSideEffects(getContext())))
700     return getDefaultBuiltinObjectSizeResult(Type, ResType);
701
702   Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
703   assert(Ptr->getType()->isPointerTy() &&
704          "Non-pointer passed to __builtin_object_size?");
705
706   Function *F =
707       CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
708
709   // LLVM only supports 0 and 2, make sure that we pass along that as a boolean.
710   Value *Min = Builder.getInt1((Type & 2) != 0);
711   // For GCC compatibility, __builtin_object_size treat NULL as unknown size.
712   Value *NullIsUnknown = Builder.getTrue();
713   Value *Dynamic = Builder.getInt1(IsDynamic);
714   return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown, Dynamic});
715 }
716
717 namespace {
718 /// A struct to generically describe a bit test intrinsic.
719 struct BitTest {
720   enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
721   enum InterlockingKind : uint8_t {
722     Unlocked,
723     Sequential,
724     Acquire,
725     Release,
726     NoFence
727   };
728
729   ActionKind Action;
730   InterlockingKind Interlocking;
731   bool Is64Bit;
732
733   static BitTest decodeBitTestBuiltin(unsigned BuiltinID);
734 };
735 } // namespace
736
737 BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
738   switch (BuiltinID) {
739     // Main portable variants.
740   case Builtin::BI_bittest:
741     return {TestOnly, Unlocked, false};
742   case Builtin::BI_bittestandcomplement:
743     return {Complement, Unlocked, false};
744   case Builtin::BI_bittestandreset:
745     return {Reset, Unlocked, false};
746   case Builtin::BI_bittestandset:
747     return {Set, Unlocked, false};
748   case Builtin::BI_interlockedbittestandreset:
749     return {Reset, Sequential, false};
750   case Builtin::BI_interlockedbittestandset:
751     return {Set, Sequential, false};
752
753     // X86-specific 64-bit variants.
754   case Builtin::BI_bittest64:
755     return {TestOnly, Unlocked, true};
756   case Builtin::BI_bittestandcomplement64:
757     return {Complement, Unlocked, true};
758   case Builtin::BI_bittestandreset64:
759     return {Reset, Unlocked, true};
760   case Builtin::BI_bittestandset64:
761     return {Set, Unlocked, true};
762   case Builtin::BI_interlockedbittestandreset64:
763     return {Reset, Sequential, true};
764   case Builtin::BI_interlockedbittestandset64:
765     return {Set, Sequential, true};
766
767     // ARM/AArch64-specific ordering variants.
768   case Builtin::BI_interlockedbittestandset_acq:
769     return {Set, Acquire, false};
770   case Builtin::BI_interlockedbittestandset_rel:
771     return {Set, Release, false};
772   case Builtin::BI_interlockedbittestandset_nf:
773     return {Set, NoFence, false};
774   case Builtin::BI_interlockedbittestandreset_acq:
775     return {Reset, Acquire, false};
776   case Builtin::BI_interlockedbittestandreset_rel:
777     return {Reset, Release, false};
778   case Builtin::BI_interlockedbittestandreset_nf:
779     return {Reset, NoFence, false};
780   }
781   llvm_unreachable("expected only bittest intrinsics");
782 }
783
784 static char bitActionToX86BTCode(BitTest::ActionKind A) {
785   switch (A) {
786   case BitTest::TestOnly:   return '\0';
787   case BitTest::Complement: return 'c';
788   case BitTest::Reset:      return 'r';
789   case BitTest::Set:        return 's';
790   }
791   llvm_unreachable("invalid action");
792 }
793
794 static llvm::Value *EmitX86BitTestIntrinsic(CodeGenFunction &CGF,
795                                             BitTest BT,
796                                             const CallExpr *E, Value *BitBase,
797                                             Value *BitPos) {
798   char Action = bitActionToX86BTCode(BT.Action);
799   char SizeSuffix = BT.Is64Bit ? 'q' : 'l';
800
801   // Build the assembly.
802   SmallString<64> Asm;
803   raw_svector_ostream AsmOS(Asm);
804   if (BT.Interlocking != BitTest::Unlocked)
805     AsmOS << "lock ";
806   AsmOS << "bt";
807   if (Action)
808     AsmOS << Action;
809   AsmOS << SizeSuffix << " $2, ($1)";
810
811   // Build the constraints. FIXME: We should support immediates when possible.
812   std::string Constraints = "={@ccc},r,r,~{cc},~{memory}";
813   std::string MachineClobbers = CGF.getTarget().getClobbers();
814   if (!MachineClobbers.empty()) {
815     Constraints += ',';
816     Constraints += MachineClobbers;
817   }
818   llvm::IntegerType *IntType = llvm::IntegerType::get(
819       CGF.getLLVMContext(),
820       CGF.getContext().getTypeSize(E->getArg(1)->getType()));
821   llvm::Type *IntPtrType = IntType->getPointerTo();
822   llvm::FunctionType *FTy =
823       llvm::FunctionType::get(CGF.Int8Ty, {IntPtrType, IntType}, false);
824
825   llvm::InlineAsm *IA =
826       llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
827   return CGF.Builder.CreateCall(IA, {BitBase, BitPos});
828 }
829
830 static llvm::AtomicOrdering
831 getBitTestAtomicOrdering(BitTest::InterlockingKind I) {
832   switch (I) {
833   case BitTest::Unlocked:   return llvm::AtomicOrdering::NotAtomic;
834   case BitTest::Sequential: return llvm::AtomicOrdering::SequentiallyConsistent;
835   case BitTest::Acquire:    return llvm::AtomicOrdering::Acquire;
836   case BitTest::Release:    return llvm::AtomicOrdering::Release;
837   case BitTest::NoFence:    return llvm::AtomicOrdering::Monotonic;
838   }
839   llvm_unreachable("invalid interlocking");
840 }
841
842 /// Emit a _bittest* intrinsic. These intrinsics take a pointer to an array of
843 /// bits and a bit position and read and optionally modify the bit at that
844 /// position. The position index can be arbitrarily large, i.e. it can be larger
845 /// than 31 or 63, so we need an indexed load in the general case.
846 static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
847                                          unsigned BuiltinID,
848                                          const CallExpr *E) {
849   Value *BitBase = CGF.EmitScalarExpr(E->getArg(0));
850   Value *BitPos = CGF.EmitScalarExpr(E->getArg(1));
851
852   BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
853
854   // X86 has special BT, BTC, BTR, and BTS instructions that handle the array
855   // indexing operation internally. Use them if possible.
856   if (CGF.getTarget().getTriple().isX86())
857     return EmitX86BitTestIntrinsic(CGF, BT, E, BitBase, BitPos);
858
859   // Otherwise, use generic code to load one byte and test the bit. Use all but
860   // the bottom three bits as the array index, and the bottom three bits to form
861   // a mask.
862   // Bit = BitBaseI8[BitPos >> 3] & (1 << (BitPos & 0x7)) != 0;
863   Value *ByteIndex = CGF.Builder.CreateAShr(
864       BitPos, llvm::ConstantInt::get(BitPos->getType(), 3), "bittest.byteidx");
865   Value *BitBaseI8 = CGF.Builder.CreatePointerCast(BitBase, CGF.Int8PtrTy);
866   Address ByteAddr(CGF.Builder.CreateInBoundsGEP(CGF.Int8Ty, BitBaseI8,
867                                                  ByteIndex, "bittest.byteaddr"),
868                    CharUnits::One());
869   Value *PosLow =
870       CGF.Builder.CreateAnd(CGF.Builder.CreateTrunc(BitPos, CGF.Int8Ty),
871                             llvm::ConstantInt::get(CGF.Int8Ty, 0x7));
872
873   // The updating instructions will need a mask.
874   Value *Mask = nullptr;
875   if (BT.Action != BitTest::TestOnly) {
876     Mask = CGF.Builder.CreateShl(llvm::ConstantInt::get(CGF.Int8Ty, 1), PosLow,
877                                  "bittest.mask");
878   }
879
880   // Check the action and ordering of the interlocked intrinsics.
881   llvm::AtomicOrdering Ordering = getBitTestAtomicOrdering(BT.Interlocking);
882
883   Value *OldByte = nullptr;
884   if (Ordering != llvm::AtomicOrdering::NotAtomic) {
885     // Emit a combined atomicrmw load/store operation for the interlocked
886     // intrinsics.
887     llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
888     if (BT.Action == BitTest::Reset) {
889       Mask = CGF.Builder.CreateNot(Mask);
890       RMWOp = llvm::AtomicRMWInst::And;
891     }
892     OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
893                                           Ordering);
894   } else {
895     // Emit a plain load for the non-interlocked intrinsics.
896     OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
897     Value *NewByte = nullptr;
898     switch (BT.Action) {
899     case BitTest::TestOnly:
900       // Don't store anything.
901       break;
902     case BitTest::Complement:
903       NewByte = CGF.Builder.CreateXor(OldByte, Mask);
904       break;
905     case BitTest::Reset:
906       NewByte = CGF.Builder.CreateAnd(OldByte, CGF.Builder.CreateNot(Mask));
907       break;
908     case BitTest::Set:
909       NewByte = CGF.Builder.CreateOr(OldByte, Mask);
910       break;
911     }
912     if (NewByte)
913       CGF.Builder.CreateStore(NewByte, ByteAddr);
914   }
915
916   // However we loaded the old byte, either by plain load or atomicrmw, shift
917   // the bit into the low position and mask it to 0 or 1.
918   Value *ShiftedByte = CGF.Builder.CreateLShr(OldByte, PosLow, "bittest.shr");
919   return CGF.Builder.CreateAnd(
920       ShiftedByte, llvm::ConstantInt::get(CGF.Int8Ty, 1), "bittest.res");
921 }
922
923 namespace {
924 enum class MSVCSetJmpKind {
925   _setjmpex,
926   _setjmp3,
927   _setjmp
928 };
929 }
930
931 /// MSVC handles setjmp a bit differently on different platforms. On every
932 /// architecture except 32-bit x86, the frame address is passed. On x86, extra
933 /// parameters can be passed as variadic arguments, but we always pass none.
934 static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind,
935                                const CallExpr *E) {
936   llvm::Value *Arg1 = nullptr;
937   llvm::Type *Arg1Ty = nullptr;
938   StringRef Name;
939   bool IsVarArg = false;
940   if (SJKind == MSVCSetJmpKind::_setjmp3) {
941     Name = "_setjmp3";
942     Arg1Ty = CGF.Int32Ty;
943     Arg1 = llvm::ConstantInt::get(CGF.IntTy, 0);
944     IsVarArg = true;
945   } else {
946     Name = SJKind == MSVCSetJmpKind::_setjmp ? "_setjmp" : "_setjmpex";
947     Arg1Ty = CGF.Int8PtrTy;
948     if (CGF.getTarget().getTriple().getArch() == llvm::Triple::aarch64) {
949       Arg1 = CGF.Builder.CreateCall(
950           CGF.CGM.getIntrinsic(Intrinsic::sponentry, CGF.AllocaInt8PtrTy));
951     } else
952       Arg1 = CGF.Builder.CreateCall(
953           CGF.CGM.getIntrinsic(Intrinsic::frameaddress, CGF.AllocaInt8PtrTy),
954           llvm::ConstantInt::get(CGF.Int32Ty, 0));
955   }
956
957   // Mark the call site and declaration with ReturnsTwice.
958   llvm::Type *ArgTypes[2] = {CGF.Int8PtrTy, Arg1Ty};
959   llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
960       CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex,
961       llvm::Attribute::ReturnsTwice);
962   llvm::FunctionCallee SetJmpFn = CGF.CGM.CreateRuntimeFunction(
963       llvm::FunctionType::get(CGF.IntTy, ArgTypes, IsVarArg), Name,
964       ReturnsTwiceAttr, /*Local=*/true);
965
966   llvm::Value *Buf = CGF.Builder.CreateBitOrPointerCast(
967       CGF.EmitScalarExpr(E->getArg(0)), CGF.Int8PtrTy);
968   llvm::Value *Args[] = {Buf, Arg1};
969   llvm::CallBase *CB = CGF.EmitRuntimeCallOrInvoke(SetJmpFn, Args);
970   CB->setAttributes(ReturnsTwiceAttr);
971   return RValue::get(CB);
972 }
973
974 // Many of MSVC builtins are on x64, ARM and AArch64; to avoid repeating code,
975 // we handle them here.
976 enum class CodeGenFunction::MSVCIntrin {
977   _BitScanForward,
978   _BitScanReverse,
979   _InterlockedAnd,
980   _InterlockedDecrement,
981   _InterlockedExchange,
982   _InterlockedExchangeAdd,
983   _InterlockedExchangeSub,
984   _InterlockedIncrement,
985   _InterlockedOr,
986   _InterlockedXor,
987   _InterlockedExchangeAdd_acq,
988   _InterlockedExchangeAdd_rel,
989   _InterlockedExchangeAdd_nf,
990   _InterlockedExchange_acq,
991   _InterlockedExchange_rel,
992   _InterlockedExchange_nf,
993   _InterlockedCompareExchange_acq,
994   _InterlockedCompareExchange_rel,
995   _InterlockedCompareExchange_nf,
996   _InterlockedOr_acq,
997   _InterlockedOr_rel,
998   _InterlockedOr_nf,
999   _InterlockedXor_acq,
1000   _InterlockedXor_rel,
1001   _InterlockedXor_nf,
1002   _InterlockedAnd_acq,
1003   _InterlockedAnd_rel,
1004   _InterlockedAnd_nf,
1005   _InterlockedIncrement_acq,
1006   _InterlockedIncrement_rel,
1007   _InterlockedIncrement_nf,
1008   _InterlockedDecrement_acq,
1009   _InterlockedDecrement_rel,
1010   _InterlockedDecrement_nf,
1011   __fastfail,
1012 };
1013
1014 static Optional<CodeGenFunction::MSVCIntrin>
1015 translateArmToMsvcIntrin(unsigned BuiltinID) {
1016   using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1017   switch (BuiltinID) {
1018   default:
1019     return None;
1020   case ARM::BI_BitScanForward:
1021   case ARM::BI_BitScanForward64:
1022     return MSVCIntrin::_BitScanForward;
1023   case ARM::BI_BitScanReverse:
1024   case ARM::BI_BitScanReverse64:
1025     return MSVCIntrin::_BitScanReverse;
1026   case ARM::BI_InterlockedAnd64:
1027     return MSVCIntrin::_InterlockedAnd;
1028   case ARM::BI_InterlockedExchange64:
1029     return MSVCIntrin::_InterlockedExchange;
1030   case ARM::BI_InterlockedExchangeAdd64:
1031     return MSVCIntrin::_InterlockedExchangeAdd;
1032   case ARM::BI_InterlockedExchangeSub64:
1033     return MSVCIntrin::_InterlockedExchangeSub;
1034   case ARM::BI_InterlockedOr64:
1035     return MSVCIntrin::_InterlockedOr;
1036   case ARM::BI_InterlockedXor64:
1037     return MSVCIntrin::_InterlockedXor;
1038   case ARM::BI_InterlockedDecrement64:
1039     return MSVCIntrin::_InterlockedDecrement;
1040   case ARM::BI_InterlockedIncrement64:
1041     return MSVCIntrin::_InterlockedIncrement;
1042   case ARM::BI_InterlockedExchangeAdd8_acq:
1043   case ARM::BI_InterlockedExchangeAdd16_acq:
1044   case ARM::BI_InterlockedExchangeAdd_acq:
1045   case ARM::BI_InterlockedExchangeAdd64_acq:
1046     return MSVCIntrin::_InterlockedExchangeAdd_acq;
1047   case ARM::BI_InterlockedExchangeAdd8_rel:
1048   case ARM::BI_InterlockedExchangeAdd16_rel:
1049   case ARM::BI_InterlockedExchangeAdd_rel:
1050   case ARM::BI_InterlockedExchangeAdd64_rel:
1051     return MSVCIntrin::_InterlockedExchangeAdd_rel;
1052   case ARM::BI_InterlockedExchangeAdd8_nf:
1053   case ARM::BI_InterlockedExchangeAdd16_nf:
1054   case ARM::BI_InterlockedExchangeAdd_nf:
1055   case ARM::BI_InterlockedExchangeAdd64_nf:
1056     return MSVCIntrin::_InterlockedExchangeAdd_nf;
1057   case ARM::BI_InterlockedExchange8_acq:
1058   case ARM::BI_InterlockedExchange16_acq:
1059   case ARM::BI_InterlockedExchange_acq:
1060   case ARM::BI_InterlockedExchange64_acq:
1061     return MSVCIntrin::_InterlockedExchange_acq;
1062   case ARM::BI_InterlockedExchange8_rel:
1063   case ARM::BI_InterlockedExchange16_rel:
1064   case ARM::BI_InterlockedExchange_rel:
1065   case ARM::BI_InterlockedExchange64_rel:
1066     return MSVCIntrin::_InterlockedExchange_rel;
1067   case ARM::BI_InterlockedExchange8_nf:
1068   case ARM::BI_InterlockedExchange16_nf:
1069   case ARM::BI_InterlockedExchange_nf:
1070   case ARM::BI_InterlockedExchange64_nf:
1071     return MSVCIntrin::_InterlockedExchange_nf;
1072   case ARM::BI_InterlockedCompareExchange8_acq:
1073   case ARM::BI_InterlockedCompareExchange16_acq:
1074   case ARM::BI_InterlockedCompareExchange_acq:
1075   case ARM::BI_InterlockedCompareExchange64_acq:
1076     return MSVCIntrin::_InterlockedCompareExchange_acq;
1077   case ARM::BI_InterlockedCompareExchange8_rel:
1078   case ARM::BI_InterlockedCompareExchange16_rel:
1079   case ARM::BI_InterlockedCompareExchange_rel:
1080   case ARM::BI_InterlockedCompareExchange64_rel:
1081     return MSVCIntrin::_InterlockedCompareExchange_rel;
1082   case ARM::BI_InterlockedCompareExchange8_nf:
1083   case ARM::BI_InterlockedCompareExchange16_nf:
1084   case ARM::BI_InterlockedCompareExchange_nf:
1085   case ARM::BI_InterlockedCompareExchange64_nf:
1086     return MSVCIntrin::_InterlockedCompareExchange_nf;
1087   case ARM::BI_InterlockedOr8_acq:
1088   case ARM::BI_InterlockedOr16_acq:
1089   case ARM::BI_InterlockedOr_acq:
1090   case ARM::BI_InterlockedOr64_acq:
1091     return MSVCIntrin::_InterlockedOr_acq;
1092   case ARM::BI_InterlockedOr8_rel:
1093   case ARM::BI_InterlockedOr16_rel:
1094   case ARM::BI_InterlockedOr_rel:
1095   case ARM::BI_InterlockedOr64_rel:
1096     return MSVCIntrin::_InterlockedOr_rel;
1097   case ARM::BI_InterlockedOr8_nf:
1098   case ARM::BI_InterlockedOr16_nf:
1099   case ARM::BI_InterlockedOr_nf:
1100   case ARM::BI_InterlockedOr64_nf:
1101     return MSVCIntrin::_InterlockedOr_nf;
1102   case ARM::BI_InterlockedXor8_acq:
1103   case ARM::BI_InterlockedXor16_acq:
1104   case ARM::BI_InterlockedXor_acq:
1105   case ARM::BI_InterlockedXor64_acq:
1106     return MSVCIntrin::_InterlockedXor_acq;
1107   case ARM::BI_InterlockedXor8_rel:
1108   case ARM::BI_InterlockedXor16_rel:
1109   case ARM::BI_InterlockedXor_rel:
1110   case ARM::BI_InterlockedXor64_rel:
1111     return MSVCIntrin::_InterlockedXor_rel;
1112   case ARM::BI_InterlockedXor8_nf:
1113   case ARM::BI_InterlockedXor16_nf:
1114   case ARM::BI_InterlockedXor_nf:
1115   case ARM::BI_InterlockedXor64_nf:
1116     return MSVCIntrin::_InterlockedXor_nf;
1117   case ARM::BI_InterlockedAnd8_acq:
1118   case ARM::BI_InterlockedAnd16_acq:
1119   case ARM::BI_InterlockedAnd_acq:
1120   case ARM::BI_InterlockedAnd64_acq:
1121     return MSVCIntrin::_InterlockedAnd_acq;
1122   case ARM::BI_InterlockedAnd8_rel:
1123   case ARM::BI_InterlockedAnd16_rel:
1124   case ARM::BI_InterlockedAnd_rel:
1125   case ARM::BI_InterlockedAnd64_rel:
1126     return MSVCIntrin::_InterlockedAnd_rel;
1127   case ARM::BI_InterlockedAnd8_nf:
1128   case ARM::BI_InterlockedAnd16_nf:
1129   case ARM::BI_InterlockedAnd_nf:
1130   case ARM::BI_InterlockedAnd64_nf:
1131     return MSVCIntrin::_InterlockedAnd_nf;
1132   case ARM::BI_InterlockedIncrement16_acq:
1133   case ARM::BI_InterlockedIncrement_acq:
1134   case ARM::BI_InterlockedIncrement64_acq:
1135     return MSVCIntrin::_InterlockedIncrement_acq;
1136   case ARM::BI_InterlockedIncrement16_rel:
1137   case ARM::BI_InterlockedIncrement_rel:
1138   case ARM::BI_InterlockedIncrement64_rel:
1139     return MSVCIntrin::_InterlockedIncrement_rel;
1140   case ARM::BI_InterlockedIncrement16_nf:
1141   case ARM::BI_InterlockedIncrement_nf:
1142   case ARM::BI_InterlockedIncrement64_nf:
1143     return MSVCIntrin::_InterlockedIncrement_nf;
1144   case ARM::BI_InterlockedDecrement16_acq:
1145   case ARM::BI_InterlockedDecrement_acq:
1146   case ARM::BI_InterlockedDecrement64_acq:
1147     return MSVCIntrin::_InterlockedDecrement_acq;
1148   case ARM::BI_InterlockedDecrement16_rel:
1149   case ARM::BI_InterlockedDecrement_rel:
1150   case ARM::BI_InterlockedDecrement64_rel:
1151     return MSVCIntrin::_InterlockedDecrement_rel;
1152   case ARM::BI_InterlockedDecrement16_nf:
1153   case ARM::BI_InterlockedDecrement_nf:
1154   case ARM::BI_InterlockedDecrement64_nf:
1155     return MSVCIntrin::_InterlockedDecrement_nf;
1156   }
1157   llvm_unreachable("must return from switch");
1158 }
1159
1160 static Optional<CodeGenFunction::MSVCIntrin>
1161 translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
1162   using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1163   switch (BuiltinID) {
1164   default:
1165     return None;
1166   case AArch64::BI_BitScanForward:
1167   case AArch64::BI_BitScanForward64:
1168     return MSVCIntrin::_BitScanForward;
1169   case AArch64::BI_BitScanReverse:
1170   case AArch64::BI_BitScanReverse64:
1171     return MSVCIntrin::_BitScanReverse;
1172   case AArch64::BI_InterlockedAnd64:
1173     return MSVCIntrin::_InterlockedAnd;
1174   case AArch64::BI_InterlockedExchange64:
1175     return MSVCIntrin::_InterlockedExchange;
1176   case AArch64::BI_InterlockedExchangeAdd64:
1177     return MSVCIntrin::_InterlockedExchangeAdd;
1178   case AArch64::BI_InterlockedExchangeSub64:
1179     return MSVCIntrin::_InterlockedExchangeSub;
1180   case AArch64::BI_InterlockedOr64:
1181     return MSVCIntrin::_InterlockedOr;
1182   case AArch64::BI_InterlockedXor64:
1183     return MSVCIntrin::_InterlockedXor;
1184   case AArch64::BI_InterlockedDecrement64:
1185     return MSVCIntrin::_InterlockedDecrement;
1186   case AArch64::BI_InterlockedIncrement64:
1187     return MSVCIntrin::_InterlockedIncrement;
1188   case AArch64::BI_InterlockedExchangeAdd8_acq:
1189   case AArch64::BI_InterlockedExchangeAdd16_acq:
1190   case AArch64::BI_InterlockedExchangeAdd_acq:
1191   case AArch64::BI_InterlockedExchangeAdd64_acq:
1192     return MSVCIntrin::_InterlockedExchangeAdd_acq;
1193   case AArch64::BI_InterlockedExchangeAdd8_rel:
1194   case AArch64::BI_InterlockedExchangeAdd16_rel:
1195   case AArch64::BI_InterlockedExchangeAdd_rel:
1196   case AArch64::BI_InterlockedExchangeAdd64_rel:
1197     return MSVCIntrin::_InterlockedExchangeAdd_rel;
1198   case AArch64::BI_InterlockedExchangeAdd8_nf:
1199   case AArch64::BI_InterlockedExchangeAdd16_nf:
1200   case AArch64::BI_InterlockedExchangeAdd_nf:
1201   case AArch64::BI_InterlockedExchangeAdd64_nf:
1202     return MSVCIntrin::_InterlockedExchangeAdd_nf;
1203   case AArch64::BI_InterlockedExchange8_acq:
1204   case AArch64::BI_InterlockedExchange16_acq:
1205   case AArch64::BI_InterlockedExchange_acq:
1206   case AArch64::BI_InterlockedExchange64_acq:
1207     return MSVCIntrin::_InterlockedExchange_acq;
1208   case AArch64::BI_InterlockedExchange8_rel:
1209   case AArch64::BI_InterlockedExchange16_rel:
1210   case AArch64::BI_InterlockedExchange_rel:
1211   case AArch64::BI_InterlockedExchange64_rel:
1212     return MSVCIntrin::_InterlockedExchange_rel;
1213   case AArch64::BI_InterlockedExchange8_nf:
1214   case AArch64::BI_InterlockedExchange16_nf:
1215   case AArch64::BI_InterlockedExchange_nf:
1216   case AArch64::BI_InterlockedExchange64_nf:
1217     return MSVCIntrin::_InterlockedExchange_nf;
1218   case AArch64::BI_InterlockedCompareExchange8_acq:
1219   case AArch64::BI_InterlockedCompareExchange16_acq:
1220   case AArch64::BI_InterlockedCompareExchange_acq:
1221   case AArch64::BI_InterlockedCompareExchange64_acq:
1222     return MSVCIntrin::_InterlockedCompareExchange_acq;
1223   case AArch64::BI_InterlockedCompareExchange8_rel:
1224   case AArch64::BI_InterlockedCompareExchange16_rel:
1225   case AArch64::BI_InterlockedCompareExchange_rel:
1226   case AArch64::BI_InterlockedCompareExchange64_rel:
1227     return MSVCIntrin::_InterlockedCompareExchange_rel;
1228   case AArch64::BI_InterlockedCompareExchange8_nf:
1229   case AArch64::BI_InterlockedCompareExchange16_nf:
1230   case AArch64::BI_InterlockedCompareExchange_nf:
1231   case AArch64::BI_InterlockedCompareExchange64_nf:
1232     return MSVCIntrin::_InterlockedCompareExchange_nf;
1233   case AArch64::BI_InterlockedOr8_acq:
1234   case AArch64::BI_InterlockedOr16_acq:
1235   case AArch64::BI_InterlockedOr_acq:
1236   case AArch64::BI_InterlockedOr64_acq:
1237     return MSVCIntrin::_InterlockedOr_acq;
1238   case AArch64::BI_InterlockedOr8_rel:
1239   case AArch64::BI_InterlockedOr16_rel:
1240   case AArch64::BI_InterlockedOr_rel:
1241   case AArch64::BI_InterlockedOr64_rel:
1242     return MSVCIntrin::_InterlockedOr_rel;
1243   case AArch64::BI_InterlockedOr8_nf:
1244   case AArch64::BI_InterlockedOr16_nf:
1245   case AArch64::BI_InterlockedOr_nf:
1246   case AArch64::BI_InterlockedOr64_nf:
1247     return MSVCIntrin::_InterlockedOr_nf;
1248   case AArch64::BI_InterlockedXor8_acq:
1249   case AArch64::BI_InterlockedXor16_acq:
1250   case AArch64::BI_InterlockedXor_acq:
1251   case AArch64::BI_InterlockedXor64_acq:
1252     return MSVCIntrin::_InterlockedXor_acq;
1253   case AArch64::BI_InterlockedXor8_rel:
1254   case AArch64::BI_InterlockedXor16_rel:
1255   case AArch64::BI_InterlockedXor_rel:
1256   case AArch64::BI_InterlockedXor64_rel:
1257     return MSVCIntrin::_InterlockedXor_rel;
1258   case AArch64::BI_InterlockedXor8_nf:
1259   case AArch64::BI_InterlockedXor16_nf:
1260   case AArch64::BI_InterlockedXor_nf:
1261   case AArch64::BI_InterlockedXor64_nf:
1262     return MSVCIntrin::_InterlockedXor_nf;
1263   case AArch64::BI_InterlockedAnd8_acq:
1264   case AArch64::BI_InterlockedAnd16_acq:
1265   case AArch64::BI_InterlockedAnd_acq:
1266   case AArch64::BI_InterlockedAnd64_acq:
1267     return MSVCIntrin::_InterlockedAnd_acq;
1268   case AArch64::BI_InterlockedAnd8_rel:
1269   case AArch64::BI_InterlockedAnd16_rel:
1270   case AArch64::BI_InterlockedAnd_rel:
1271   case AArch64::BI_InterlockedAnd64_rel:
1272     return MSVCIntrin::_InterlockedAnd_rel;
1273   case AArch64::BI_InterlockedAnd8_nf:
1274   case AArch64::BI_InterlockedAnd16_nf:
1275   case AArch64::BI_InterlockedAnd_nf:
1276   case AArch64::BI_InterlockedAnd64_nf:
1277     return MSVCIntrin::_InterlockedAnd_nf;
1278   case AArch64::BI_InterlockedIncrement16_acq:
1279   case AArch64::BI_InterlockedIncrement_acq:
1280   case AArch64::BI_InterlockedIncrement64_acq:
1281     return MSVCIntrin::_InterlockedIncrement_acq;
1282   case AArch64::BI_InterlockedIncrement16_rel:
1283   case AArch64::BI_InterlockedIncrement_rel:
1284   case AArch64::BI_InterlockedIncrement64_rel:
1285     return MSVCIntrin::_InterlockedIncrement_rel;
1286   case AArch64::BI_InterlockedIncrement16_nf:
1287   case AArch64::BI_InterlockedIncrement_nf:
1288   case AArch64::BI_InterlockedIncrement64_nf:
1289     return MSVCIntrin::_InterlockedIncrement_nf;
1290   case AArch64::BI_InterlockedDecrement16_acq:
1291   case AArch64::BI_InterlockedDecrement_acq:
1292   case AArch64::BI_InterlockedDecrement64_acq:
1293     return MSVCIntrin::_InterlockedDecrement_acq;
1294   case AArch64::BI_InterlockedDecrement16_rel:
1295   case AArch64::BI_InterlockedDecrement_rel:
1296   case AArch64::BI_InterlockedDecrement64_rel:
1297     return MSVCIntrin::_InterlockedDecrement_rel;
1298   case AArch64::BI_InterlockedDecrement16_nf:
1299   case AArch64::BI_InterlockedDecrement_nf:
1300   case AArch64::BI_InterlockedDecrement64_nf:
1301     return MSVCIntrin::_InterlockedDecrement_nf;
1302   }
1303   llvm_unreachable("must return from switch");
1304 }
1305
1306 static Optional<CodeGenFunction::MSVCIntrin>
1307 translateX86ToMsvcIntrin(unsigned BuiltinID) {
1308   using MSVCIntrin = CodeGenFunction::MSVCIntrin;
1309   switch (BuiltinID) {
1310   default:
1311     return None;
1312   case clang::X86::BI_BitScanForward:
1313   case clang::X86::BI_BitScanForward64:
1314     return MSVCIntrin::_BitScanForward;
1315   case clang::X86::BI_BitScanReverse:
1316   case clang::X86::BI_BitScanReverse64:
1317     return MSVCIntrin::_BitScanReverse;
1318   case clang::X86::BI_InterlockedAnd64:
1319     return MSVCIntrin::_InterlockedAnd;
1320   case clang::X86::BI_InterlockedExchange64:
1321     return MSVCIntrin::_InterlockedExchange;
1322   case clang::X86::BI_InterlockedExchangeAdd64:
1323     return MSVCIntrin::_InterlockedExchangeAdd;
1324   case clang::X86::BI_InterlockedExchangeSub64:
1325     return MSVCIntrin::_InterlockedExchangeSub;
1326   case clang::X86::BI_InterlockedOr64:
1327     return MSVCIntrin::_InterlockedOr;
1328   case clang::X86::BI_InterlockedXor64:
1329     return MSVCIntrin::_InterlockedXor;
1330   case clang::X86::BI_InterlockedDecrement64:
1331     return MSVCIntrin::_InterlockedDecrement;
1332   case clang::X86::BI_InterlockedIncrement64:
1333     return MSVCIntrin::_InterlockedIncrement;
1334   }
1335   llvm_unreachable("must return from switch");
1336 }
1337
1338 // Emit an MSVC intrinsic. Assumes that arguments have *not* been evaluated.
1339 Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
1340                                             const CallExpr *E) {
1341   switch (BuiltinID) {
1342   case MSVCIntrin::_BitScanForward:
1343   case MSVCIntrin::_BitScanReverse: {
1344     Address IndexAddress(EmitPointerWithAlignment(E->getArg(0)));
1345     Value *ArgValue = EmitScalarExpr(E->getArg(1));
1346
1347     llvm::Type *ArgType = ArgValue->getType();
1348     llvm::Type *IndexType =
1349         IndexAddress.getPointer()->getType()->getPointerElementType();
1350     llvm::Type *ResultType = ConvertType(E->getType());
1351
1352     Value *ArgZero = llvm::Constant::getNullValue(ArgType);
1353     Value *ResZero = llvm::Constant::getNullValue(ResultType);
1354     Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
1355
1356     BasicBlock *Begin = Builder.GetInsertBlock();
1357     BasicBlock *End = createBasicBlock("bitscan_end", this->CurFn);
1358     Builder.SetInsertPoint(End);
1359     PHINode *Result = Builder.CreatePHI(ResultType, 2, "bitscan_result");
1360
1361     Builder.SetInsertPoint(Begin);
1362     Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
1363     BasicBlock *NotZero = createBasicBlock("bitscan_not_zero", this->CurFn);
1364     Builder.CreateCondBr(IsZero, End, NotZero);
1365     Result->addIncoming(ResZero, Begin);
1366
1367     Builder.SetInsertPoint(NotZero);
1368
1369     if (BuiltinID == MSVCIntrin::_BitScanForward) {
1370       Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1371       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1372       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1373       Builder.CreateStore(ZeroCount, IndexAddress, false);
1374     } else {
1375       unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
1376       Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
1377
1378       Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1379       Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
1380       ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType, false);
1381       Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
1382       Builder.CreateStore(Index, IndexAddress, false);
1383     }
1384     Builder.CreateBr(End);
1385     Result->addIncoming(ResOne, NotZero);
1386
1387     Builder.SetInsertPoint(End);
1388     return Result;
1389   }
1390   case MSVCIntrin::_InterlockedAnd:
1391     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E);
1392   case MSVCIntrin::_InterlockedExchange:
1393     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E);
1394   case MSVCIntrin::_InterlockedExchangeAdd:
1395     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E);
1396   case MSVCIntrin::_InterlockedExchangeSub:
1397     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Sub, E);
1398   case MSVCIntrin::_InterlockedOr:
1399     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E);
1400   case MSVCIntrin::_InterlockedXor:
1401     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E);
1402   case MSVCIntrin::_InterlockedExchangeAdd_acq:
1403     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1404                                  AtomicOrdering::Acquire);
1405   case MSVCIntrin::_InterlockedExchangeAdd_rel:
1406     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1407                                  AtomicOrdering::Release);
1408   case MSVCIntrin::_InterlockedExchangeAdd_nf:
1409     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Add, E,
1410                                  AtomicOrdering::Monotonic);
1411   case MSVCIntrin::_InterlockedExchange_acq:
1412     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1413                                  AtomicOrdering::Acquire);
1414   case MSVCIntrin::_InterlockedExchange_rel:
1415     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1416                                  AtomicOrdering::Release);
1417   case MSVCIntrin::_InterlockedExchange_nf:
1418     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
1419                                  AtomicOrdering::Monotonic);
1420   case MSVCIntrin::_InterlockedCompareExchange_acq:
1421     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
1422   case MSVCIntrin::_InterlockedCompareExchange_rel:
1423     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Release);
1424   case MSVCIntrin::_InterlockedCompareExchange_nf:
1425     return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Monotonic);
1426   case MSVCIntrin::_InterlockedOr_acq:
1427     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1428                                  AtomicOrdering::Acquire);
1429   case MSVCIntrin::_InterlockedOr_rel:
1430     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1431                                  AtomicOrdering::Release);
1432   case MSVCIntrin::_InterlockedOr_nf:
1433     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Or, E,
1434                                  AtomicOrdering::Monotonic);
1435   case MSVCIntrin::_InterlockedXor_acq:
1436     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1437                                  AtomicOrdering::Acquire);
1438   case MSVCIntrin::_InterlockedXor_rel:
1439     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1440                                  AtomicOrdering::Release);
1441   case MSVCIntrin::_InterlockedXor_nf:
1442     return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xor, E,
1443                                  AtomicOrdering::Monotonic);
1444   case MSVCIntrin::_InterlockedAnd_acq:
1445     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1446                                  AtomicOrdering::Acquire);
1447   case MSVCIntrin::_InterlockedAnd_rel:
1448     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1449                                  AtomicOrdering::Release);
1450   case MSVCIntrin::_InterlockedAnd_nf:
1451     return MakeBinaryAtomicValue(*this, AtomicRMWInst::And, E,
1452                                  AtomicOrdering::Monotonic);
1453   case MSVCIntrin::_InterlockedIncrement_acq:
1454     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Acquire);
1455   case MSVCIntrin::_InterlockedIncrement_rel:
1456     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Release);
1457   case MSVCIntrin::_InterlockedIncrement_nf:
1458     return EmitAtomicIncrementValue(*this, E, AtomicOrdering::Monotonic);
1459   case MSVCIntrin::_InterlockedDecrement_acq:
1460     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Acquire);
1461   case MSVCIntrin::_InterlockedDecrement_rel:
1462     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Release);
1463   case MSVCIntrin::_InterlockedDecrement_nf:
1464     return EmitAtomicDecrementValue(*this, E, AtomicOrdering::Monotonic);
1465
1466   case MSVCIntrin::_InterlockedDecrement:
1467     return EmitAtomicDecrementValue(*this, E);
1468   case MSVCIntrin::_InterlockedIncrement:
1469     return EmitAtomicIncrementValue(*this, E);
1470
1471   case MSVCIntrin::__fastfail: {
1472     // Request immediate process termination from the kernel. The instruction
1473     // sequences to do this are documented on MSDN:
1474     // https://msdn.microsoft.com/en-us/library/dn774154.aspx
1475     llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
1476     StringRef Asm, Constraints;
1477     switch (ISA) {
1478     default:
1479       ErrorUnsupported(E, "__fastfail call for this architecture");
1480       break;
1481     case llvm::Triple::x86:
1482     case llvm::Triple::x86_64:
1483       Asm = "int $$0x29";
1484       Constraints = "{cx}";
1485       break;
1486     case llvm::Triple::thumb:
1487       Asm = "udf #251";
1488       Constraints = "{r0}";
1489       break;
1490     case llvm::Triple::aarch64:
1491       Asm = "brk #0xF003";
1492       Constraints = "{w0}";
1493     }
1494     llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty}, false);
1495     llvm::InlineAsm *IA =
1496         llvm::InlineAsm::get(FTy, Asm, Constraints, /*hasSideEffects=*/true);
1497     llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1498         getLLVMContext(), llvm::AttributeList::FunctionIndex,
1499         llvm::Attribute::NoReturn);
1500     llvm::CallInst *CI = Builder.CreateCall(IA, EmitScalarExpr(E->getArg(0)));
1501     CI->setAttributes(NoReturnAttr);
1502     return CI;
1503   }
1504   }
1505   llvm_unreachable("Incorrect MSVC intrinsic!");
1506 }
1507
1508 namespace {
1509 // ARC cleanup for __builtin_os_log_format
1510 struct CallObjCArcUse final : EHScopeStack::Cleanup {
1511   CallObjCArcUse(llvm::Value *object) : object(object) {}
1512   llvm::Value *object;
1513
1514   void Emit(CodeGenFunction &CGF, Flags flags) override {
1515     CGF.EmitARCIntrinsicUse(object);
1516   }
1517 };
1518 }
1519
1520 Value *CodeGenFunction::EmitCheckedArgForBuiltin(const Expr *E,
1521                                                  BuiltinCheckKind Kind) {
1522   assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
1523           && "Unsupported builtin check kind");
1524
1525   Value *ArgValue = EmitScalarExpr(E);
1526   if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
1527     return ArgValue;
1528
1529   SanitizerScope SanScope(this);
1530   Value *Cond = Builder.CreateICmpNE(
1531       ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1532   EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1533             SanitizerHandler::InvalidBuiltin,
1534             {EmitCheckSourceLocation(E->getExprLoc()),
1535              llvm::ConstantInt::get(Builder.getInt8Ty(), Kind)},
1536             None);
1537   return ArgValue;
1538 }
1539
1540 /// Get the argument type for arguments to os_log_helper.
1541 static CanQualType getOSLogArgType(ASTContext &C, int Size) {
1542   QualType UnsignedTy = C.getIntTypeForBitwidth(Size * 8, /*Signed=*/false);
1543   return C.getCanonicalType(UnsignedTy);
1544 }
1545
1546 llvm::Function *CodeGenFunction::generateBuiltinOSLogHelperFunction(
1547     const analyze_os_log::OSLogBufferLayout &Layout,
1548     CharUnits BufferAlignment) {
1549   ASTContext &Ctx = getContext();
1550
1551   llvm::SmallString<64> Name;
1552   {
1553     raw_svector_ostream OS(Name);
1554     OS << "__os_log_helper";
1555     OS << "_" << BufferAlignment.getQuantity();
1556     OS << "_" << int(Layout.getSummaryByte());
1557     OS << "_" << int(Layout.getNumArgsByte());
1558     for (const auto &Item : Layout.Items)
1559       OS << "_" << int(Item.getSizeByte()) << "_"
1560          << int(Item.getDescriptorByte());
1561   }
1562
1563   if (llvm::Function *F = CGM.getModule().getFunction(Name))
1564     return F;
1565
1566   llvm::SmallVector<QualType, 4> ArgTys;
1567   FunctionArgList Args;
1568   Args.push_back(ImplicitParamDecl::Create(
1569       Ctx, nullptr, SourceLocation(), &Ctx.Idents.get("buffer"), Ctx.VoidPtrTy,
1570       ImplicitParamDecl::Other));
1571   ArgTys.emplace_back(Ctx.VoidPtrTy);
1572
1573   for (unsigned int I = 0, E = Layout.Items.size(); I < E; ++I) {
1574     char Size = Layout.Items[I].getSizeByte();
1575     if (!Size)
1576       continue;
1577
1578     QualType ArgTy = getOSLogArgType(Ctx, Size);
1579     Args.push_back(ImplicitParamDecl::Create(
1580         Ctx, nullptr, SourceLocation(),
1581         &Ctx.Idents.get(std::string("arg") + llvm::to_string(I)), ArgTy,
1582         ImplicitParamDecl::Other));
1583     ArgTys.emplace_back(ArgTy);
1584   }
1585
1586   QualType ReturnTy = Ctx.VoidTy;
1587   QualType FuncionTy = Ctx.getFunctionType(ReturnTy, ArgTys, {});
1588
1589   // The helper function has linkonce_odr linkage to enable the linker to merge
1590   // identical functions. To ensure the merging always happens, 'noinline' is
1591   // attached to the function when compiling with -Oz.
1592   const CGFunctionInfo &FI =
1593       CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, Args);
1594   llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
1595   llvm::Function *Fn = llvm::Function::Create(
1596       FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
1597   Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1598   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, Fn);
1599   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
1600   Fn->setDoesNotThrow();
1601
1602   // Attach 'noinline' at -Oz.
1603   if (CGM.getCodeGenOpts().OptimizeSize == 2)
1604     Fn->addFnAttr(llvm::Attribute::NoInline);
1605
1606   auto NL = ApplyDebugLocation::CreateEmpty(*this);
1607   IdentifierInfo *II = &Ctx.Idents.get(Name);
1608   FunctionDecl *FD = FunctionDecl::Create(
1609       Ctx, Ctx.getTranslationUnitDecl(), SourceLocation(), SourceLocation(), II,
1610       FuncionTy, nullptr, SC_PrivateExtern, false, false);
1611   // Avoid generating debug location info for the function.
1612   FD->setImplicit();
1613
1614   StartFunction(FD, ReturnTy, Fn, FI, Args);
1615
1616   // Create a scope with an artificial location for the body of this function.
1617   auto AL = ApplyDebugLocation::CreateArtificial(*this);
1618
1619   CharUnits Offset;
1620   Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(Args[0]), "buf"),
1621                   BufferAlignment);
1622   Builder.CreateStore(Builder.getInt8(Layout.getSummaryByte()),
1623                       Builder.CreateConstByteGEP(BufAddr, Offset++, "summary"));
1624   Builder.CreateStore(Builder.getInt8(Layout.getNumArgsByte()),
1625                       Builder.CreateConstByteGEP(BufAddr, Offset++, "numArgs"));
1626
1627   unsigned I = 1;
1628   for (const auto &Item : Layout.Items) {
1629     Builder.CreateStore(
1630         Builder.getInt8(Item.getDescriptorByte()),
1631         Builder.CreateConstByteGEP(BufAddr, Offset++, "argDescriptor"));
1632     Builder.CreateStore(
1633         Builder.getInt8(Item.getSizeByte()),
1634         Builder.CreateConstByteGEP(BufAddr, Offset++, "argSize"));
1635
1636     CharUnits Size = Item.size();
1637     if (!Size.getQuantity())
1638       continue;
1639
1640     Address Arg = GetAddrOfLocalVar(Args[I]);
1641     Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset, "argData");
1642     Addr = Builder.CreateBitCast(Addr, Arg.getPointer()->getType(),
1643                                  "argDataCast");
1644     Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1645     Offset += Size;
1646     ++I;
1647   }
1648
1649   FinishFunction();
1650
1651   return Fn;
1652 }
1653
1654 RValue CodeGenFunction::emitBuiltinOSLogFormat(const CallExpr &E) {
1655   assert(E.getNumArgs() >= 2 &&
1656          "__builtin_os_log_format takes at least 2 arguments");
1657   ASTContext &Ctx = getContext();
1658   analyze_os_log::OSLogBufferLayout Layout;
1659   analyze_os_log::computeOSLogBufferLayout(Ctx, &E, Layout);
1660   Address BufAddr = EmitPointerWithAlignment(E.getArg(0));
1661   llvm::SmallVector<llvm::Value *, 4> RetainableOperands;
1662
1663   // Ignore argument 1, the format string. It is not currently used.
1664   CallArgList Args;
1665   Args.add(RValue::get(BufAddr.getPointer()), Ctx.VoidPtrTy);
1666
1667   for (const auto &Item : Layout.Items) {
1668     int Size = Item.getSizeByte();
1669     if (!Size)
1670       continue;
1671
1672     llvm::Value *ArgVal;
1673
1674     if (Item.getKind() == analyze_os_log::OSLogBufferItem::MaskKind) {
1675       uint64_t Val = 0;
1676       for (unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1677         Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
1678       ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
1679     } else if (const Expr *TheExpr = Item.getExpr()) {
1680       ArgVal = EmitScalarExpr(TheExpr, /*Ignore*/ false);
1681
1682       // If a temporary object that requires destruction after the full
1683       // expression is passed, push a lifetime-extended cleanup to extend its
1684       // lifetime to the end of the enclosing block scope.
1685       auto LifetimeExtendObject = [&](const Expr *E) {
1686         E = E->IgnoreParenCasts();
1687         // Extend lifetimes of objects returned by function calls and message
1688         // sends.
1689
1690         // FIXME: We should do this in other cases in which temporaries are
1691         //        created including arguments of non-ARC types (e.g., C++
1692         //        temporaries).
1693         if (isa<CallExpr>(E) || isa<ObjCMessageExpr>(E))
1694           return true;
1695         return false;
1696       };
1697
1698       if (TheExpr->getType()->isObjCRetainableType() &&
1699           getLangOpts().ObjCAutoRefCount && LifetimeExtendObject(TheExpr)) {
1700         assert(getEvaluationKind(TheExpr->getType()) == TEK_Scalar &&
1701                "Only scalar can be a ObjC retainable type");
1702         if (!isa<Constant>(ArgVal)) {
1703           CleanupKind Cleanup = getARCCleanupKind();
1704           QualType Ty = TheExpr->getType();
1705           Address Alloca = Address::invalid();
1706           Address Addr = CreateMemTemp(Ty, "os.log.arg", &Alloca);
1707           ArgVal = EmitARCRetain(Ty, ArgVal);
1708           Builder.CreateStore(ArgVal, Addr);
1709           pushLifetimeExtendedDestroy(Cleanup, Alloca, Ty,
1710                                       CodeGenFunction::destroyARCStrongPrecise,
1711                                       Cleanup & EHCleanup);
1712
1713           // Push a clang.arc.use call to ensure ARC optimizer knows that the
1714           // argument has to be alive.
1715           if (CGM.getCodeGenOpts().OptimizationLevel != 0)
1716             pushCleanupAfterFullExpr<CallObjCArcUse>(Cleanup, ArgVal);
1717         }
1718       }
1719     } else {
1720       ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1721     }
1722
1723     unsigned ArgValSize =
1724         CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
1725     llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
1726                                                      ArgValSize);
1727     ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
1728     CanQualType ArgTy = getOSLogArgType(Ctx, Size);
1729     // If ArgVal has type x86_fp80, zero-extend ArgVal.
1730     ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
1731     Args.add(RValue::get(ArgVal), ArgTy);
1732   }
1733
1734   const CGFunctionInfo &FI =
1735       CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
1736   llvm::Function *F = CodeGenFunction(CGM).generateBuiltinOSLogHelperFunction(
1737       Layout, BufAddr.getAlignment());
1738   EmitCall(FI, CGCallee::forDirect(F), ReturnValueSlot(), Args);
1739   return RValue::get(BufAddr.getPointer());
1740 }
1741
1742 /// Determine if a binop is a checked mixed-sign multiply we can specialize.
1743 static bool isSpecialMixedSignMultiply(unsigned BuiltinID,
1744                                        WidthAndSignedness Op1Info,
1745                                        WidthAndSignedness Op2Info,
1746                                        WidthAndSignedness ResultInfo) {
1747   return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1748          std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
1749          Op1Info.Signed != Op2Info.Signed;
1750 }
1751
1752 /// Emit a checked mixed-sign multiply. This is a cheaper specialization of
1753 /// the generic checked-binop irgen.
1754 static RValue
1755 EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1,
1756                              WidthAndSignedness Op1Info, const clang::Expr *Op2,
1757                              WidthAndSignedness Op2Info,
1758                              const clang::Expr *ResultArg, QualType ResultQTy,
1759                              WidthAndSignedness ResultInfo) {
1760   assert(isSpecialMixedSignMultiply(Builtin::BI__builtin_mul_overflow, Op1Info,
1761                                     Op2Info, ResultInfo) &&
1762          "Not a mixed-sign multipliction we can specialize");
1763
1764   // Emit the signed and unsigned operands.
1765   const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1766   const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1767   llvm::Value *Signed = CGF.EmitScalarExpr(SignedOp);
1768   llvm::Value *Unsigned = CGF.EmitScalarExpr(UnsignedOp);
1769   unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
1770   unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
1771
1772   // One of the operands may be smaller than the other. If so, [s|z]ext it.
1773   if (SignedOpWidth < UnsignedOpWidth)
1774     Signed = CGF.Builder.CreateSExt(Signed, Unsigned->getType(), "op.sext");
1775   if (UnsignedOpWidth < SignedOpWidth)
1776     Unsigned = CGF.Builder.CreateZExt(Unsigned, Signed->getType(), "op.zext");
1777
1778   llvm::Type *OpTy = Signed->getType();
1779   llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1780   Address ResultPtr = CGF.EmitPointerWithAlignment(ResultArg);
1781   llvm::Type *ResTy = ResultPtr.getElementType();
1782   unsigned OpWidth = std::max(Op1Info.Width, Op2Info.Width);
1783
1784   // Take the absolute value of the signed operand.
1785   llvm::Value *IsNegative = CGF.Builder.CreateICmpSLT(Signed, Zero);
1786   llvm::Value *AbsOfNegative = CGF.Builder.CreateSub(Zero, Signed);
1787   llvm::Value *AbsSigned =
1788       CGF.Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
1789
1790   // Perform a checked unsigned multiplication.
1791   llvm::Value *UnsignedOverflow;
1792   llvm::Value *UnsignedResult =
1793       EmitOverflowIntrinsic(CGF, llvm::Intrinsic::umul_with_overflow, AbsSigned,
1794                             Unsigned, UnsignedOverflow);
1795
1796   llvm::Value *Overflow, *Result;
1797   if (ResultInfo.Signed) {
1798     // Signed overflow occurs if the result is greater than INT_MAX or lesser
1799     // than INT_MIN, i.e when |Result| > (INT_MAX + IsNegative).
1800     auto IntMax =
1801         llvm::APInt::getSignedMaxValue(ResultInfo.Width).zextOrSelf(OpWidth);
1802     llvm::Value *MaxResult =
1803         CGF.Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
1804                               CGF.Builder.CreateZExt(IsNegative, OpTy));
1805     llvm::Value *SignedOverflow =
1806         CGF.Builder.CreateICmpUGT(UnsignedResult, MaxResult);
1807     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, SignedOverflow);
1808
1809     // Prepare the signed result (possibly by negating it).
1810     llvm::Value *NegativeResult = CGF.Builder.CreateNeg(UnsignedResult);
1811     llvm::Value *SignedResult =
1812         CGF.Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
1813     Result = CGF.Builder.CreateTrunc(SignedResult, ResTy);
1814   } else {
1815     // Unsigned overflow occurs if the result is < 0 or greater than UINT_MAX.
1816     llvm::Value *Underflow = CGF.Builder.CreateAnd(
1817         IsNegative, CGF.Builder.CreateIsNotNull(UnsignedResult));
1818     Overflow = CGF.Builder.CreateOr(UnsignedOverflow, Underflow);
1819     if (ResultInfo.Width < OpWidth) {
1820       auto IntMax =
1821           llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
1822       llvm::Value *TruncOverflow = CGF.Builder.CreateICmpUGT(
1823           UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
1824       Overflow = CGF.Builder.CreateOr(Overflow, TruncOverflow);
1825     }
1826
1827     // Negate the product if it would be negative in infinite precision.
1828     Result = CGF.Builder.CreateSelect(
1829         IsNegative, CGF.Builder.CreateNeg(UnsignedResult), UnsignedResult);
1830
1831     Result = CGF.Builder.CreateTrunc(Result, ResTy);
1832   }
1833   assert(Overflow && Result && "Missing overflow or result");
1834
1835   bool isVolatile =
1836       ResultArg->getType()->getPointeeType().isVolatileQualified();
1837   CGF.Builder.CreateStore(CGF.EmitToMemory(Result, ResultQTy), ResultPtr,
1838                           isVolatile);
1839   return RValue::get(Overflow);
1840 }
1841
1842 static llvm::Value *dumpRecord(CodeGenFunction &CGF, QualType RType,
1843                                Value *&RecordPtr, CharUnits Align,
1844                                llvm::FunctionCallee Func, int Lvl) {
1845   ASTContext &Context = CGF.getContext();
1846   RecordDecl *RD = RType->castAs<RecordType>()->getDecl()->getDefinition();
1847   std::string Pad = std::string(Lvl * 4, ' ');
1848
1849   Value *GString =
1850       CGF.Builder.CreateGlobalStringPtr(RType.getAsString() + " {\n");
1851   Value *Res = CGF.Builder.CreateCall(Func, {GString});
1852
1853   static llvm::DenseMap<QualType, const char *> Types;
1854   if (Types.empty()) {
1855     Types[Context.CharTy] = "%c";
1856     Types[Context.BoolTy] = "%d";
1857     Types[Context.SignedCharTy] = "%hhd";
1858     Types[Context.UnsignedCharTy] = "%hhu";
1859     Types[Context.IntTy] = "%d";
1860     Types[Context.UnsignedIntTy] = "%u";
1861     Types[Context.LongTy] = "%ld";
1862     Types[Context.UnsignedLongTy] = "%lu";
1863     Types[Context.LongLongTy] = "%lld";
1864     Types[Context.UnsignedLongLongTy] = "%llu";
1865     Types[Context.ShortTy] = "%hd";
1866     Types[Context.UnsignedShortTy] = "%hu";
1867     Types[Context.VoidPtrTy] = "%p";
1868     Types[Context.FloatTy] = "%f";
1869     Types[Context.DoubleTy] = "%f";
1870     Types[Context.LongDoubleTy] = "%Lf";
1871     Types[Context.getPointerType(Context.CharTy)] = "%s";
1872     Types[Context.getPointerType(Context.getConstType(Context.CharTy))] = "%s";
1873   }
1874
1875   for (const auto *FD : RD->fields()) {
1876     Value *FieldPtr = RecordPtr;
1877     if (RD->isUnion())
1878       FieldPtr = CGF.Builder.CreatePointerCast(
1879           FieldPtr, CGF.ConvertType(Context.getPointerType(FD->getType())));
1880     else
1881       FieldPtr = CGF.Builder.CreateStructGEP(CGF.ConvertType(RType), FieldPtr,
1882                                              FD->getFieldIndex());
1883
1884     GString = CGF.Builder.CreateGlobalStringPtr(
1885         llvm::Twine(Pad)
1886             .concat(FD->getType().getAsString())
1887             .concat(llvm::Twine(' '))
1888             .concat(FD->getNameAsString())
1889             .concat(" : ")
1890             .str());
1891     Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1892     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1893
1894     QualType CanonicalType =
1895         FD->getType().getUnqualifiedType().getCanonicalType();
1896
1897     // We check whether we are in a recursive type
1898     if (CanonicalType->isRecordType()) {
1899       TmpRes = dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
1900       Res = CGF.Builder.CreateAdd(TmpRes, Res);
1901       continue;
1902     }
1903
1904     // We try to determine the best format to print the current field
1905     llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1906                              ? Types[Context.VoidPtrTy]
1907                              : Types[CanonicalType];
1908
1909     Address FieldAddress = Address(FieldPtr, Align);
1910     FieldPtr = CGF.Builder.CreateLoad(FieldAddress);
1911
1912     // FIXME Need to handle bitfield here
1913     GString = CGF.Builder.CreateGlobalStringPtr(
1914         Format.concat(llvm::Twine('\n')).str());
1915     TmpRes = CGF.Builder.CreateCall(Func, {GString, FieldPtr});
1916     Res = CGF.Builder.CreateAdd(Res, TmpRes);
1917   }
1918
1919   GString = CGF.Builder.CreateGlobalStringPtr(Pad + "}\n");
1920   Value *TmpRes = CGF.Builder.CreateCall(Func, {GString});
1921   Res = CGF.Builder.CreateAdd(Res, TmpRes);
1922   return Res;
1923 }
1924
1925 static bool
1926 TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty,
1927                               llvm::SmallPtrSetImpl<const Decl *> &Seen) {
1928   if (const auto *Arr = Ctx.getAsArrayType(Ty))
1929     Ty = Ctx.getBaseElementType(Arr);
1930
1931   const auto *Record = Ty->getAsCXXRecordDecl();
1932   if (!Record)
1933     return false;
1934
1935   // We've already checked this type, or are in the process of checking it.
1936   if (!Seen.insert(Record).second)
1937     return false;
1938
1939   assert(Record->hasDefinition() &&
1940          "Incomplete types should already be diagnosed");
1941
1942   if (Record->isDynamicClass())
1943     return true;
1944
1945   for (FieldDecl *F : Record->fields()) {
1946     if (TypeRequiresBuiltinLaunderImp(Ctx, F->getType(), Seen))
1947       return true;
1948   }
1949   return false;
1950 }
1951
1952 /// Determine if the specified type requires laundering by checking if it is a
1953 /// dynamic class type or contains a subobject which is a dynamic class type.
1954 static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty) {
1955   if (!CGM.getCodeGenOpts().StrictVTablePointers)
1956     return false;
1957   llvm::SmallPtrSet<const Decl *, 16> Seen;
1958   return TypeRequiresBuiltinLaunderImp(CGM.getContext(), Ty, Seen);
1959 }
1960
1961 RValue CodeGenFunction::emitRotate(const CallExpr *E, bool IsRotateRight) {
1962   llvm::Value *Src = EmitScalarExpr(E->getArg(0));
1963   llvm::Value *ShiftAmt = EmitScalarExpr(E->getArg(1));
1964
1965   // The builtin's shift arg may have a different type than the source arg and
1966   // result, but the LLVM intrinsic uses the same type for all values.
1967   llvm::Type *Ty = Src->getType();
1968   ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty, false);
1969
1970   // Rotate is a special case of LLVM funnel shift - 1st 2 args are the same.
1971   unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1972   Function *F = CGM.getIntrinsic(IID, Ty);
1973   return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
1974 }
1975
1976 RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
1977                                         const CallExpr *E,
1978                                         ReturnValueSlot ReturnValue) {
1979   const FunctionDecl *FD = GD.getDecl()->getAsFunction();
1980   // See if we can constant fold this builtin.  If so, don't emit it at all.
1981   Expr::EvalResult Result;
1982   if (E->EvaluateAsRValue(Result, CGM.getContext()) &&
1983       !Result.hasSideEffects()) {
1984     if (Result.Val.isInt())
1985       return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1986                                                 Result.Val.getInt()));
1987     if (Result.Val.isFloat())
1988       return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1989                                                Result.Val.getFloat()));
1990   }
1991
1992   // If the builtin has been declared explicitly with an assembler label,
1993   // disable the specialized emitting below. Ideally we should communicate the
1994   // rename in IR, or at least avoid generating the intrinsic calls that are
1995   // likely to get lowered to the renamed library functions.
1996   const unsigned BuiltinIDIfNoAsmLabel =
1997       FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
1998
1999   // There are LLVM math intrinsics/instructions corresponding to math library
2000   // functions except the LLVM op will never set errno while the math library
2001   // might. Also, math builtins have the same semantics as their math library
2002   // twins. Thus, we can transform math library and builtin calls to their
2003   // LLVM counterparts if the call is marked 'const' (known to never set errno).
2004   if (FD->hasAttr<ConstAttr>()) {
2005     switch (BuiltinIDIfNoAsmLabel) {
2006     case Builtin::BIceil:
2007     case Builtin::BIceilf:
2008     case Builtin::BIceill:
2009     case Builtin::BI__builtin_ceil:
2010     case Builtin::BI__builtin_ceilf:
2011     case Builtin::BI__builtin_ceilf16:
2012     case Builtin::BI__builtin_ceill:
2013     case Builtin::BI__builtin_ceilf128:
2014       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2015                                    Intrinsic::ceil,
2016                                    Intrinsic::experimental_constrained_ceil));
2017
2018     case Builtin::BIcopysign:
2019     case Builtin::BIcopysignf:
2020     case Builtin::BIcopysignl:
2021     case Builtin::BI__builtin_copysign:
2022     case Builtin::BI__builtin_copysignf:
2023     case Builtin::BI__builtin_copysignf16:
2024     case Builtin::BI__builtin_copysignl:
2025     case Builtin::BI__builtin_copysignf128:
2026       return RValue::get(emitBinaryBuiltin(*this, E, Intrinsic::copysign));
2027
2028     case Builtin::BIcos:
2029     case Builtin::BIcosf:
2030     case Builtin::BIcosl:
2031     case Builtin::BI__builtin_cos:
2032     case Builtin::BI__builtin_cosf:
2033     case Builtin::BI__builtin_cosf16:
2034     case Builtin::BI__builtin_cosl:
2035     case Builtin::BI__builtin_cosf128:
2036       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2037                                    Intrinsic::cos,
2038                                    Intrinsic::experimental_constrained_cos));
2039
2040     case Builtin::BIexp:
2041     case Builtin::BIexpf:
2042     case Builtin::BIexpl:
2043     case Builtin::BI__builtin_exp:
2044     case Builtin::BI__builtin_expf:
2045     case Builtin::BI__builtin_expf16:
2046     case Builtin::BI__builtin_expl:
2047     case Builtin::BI__builtin_expf128:
2048       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2049                                    Intrinsic::exp,
2050                                    Intrinsic::experimental_constrained_exp));
2051
2052     case Builtin::BIexp2:
2053     case Builtin::BIexp2f:
2054     case Builtin::BIexp2l:
2055     case Builtin::BI__builtin_exp2:
2056     case Builtin::BI__builtin_exp2f:
2057     case Builtin::BI__builtin_exp2f16:
2058     case Builtin::BI__builtin_exp2l:
2059     case Builtin::BI__builtin_exp2f128:
2060       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2061                                    Intrinsic::exp2,
2062                                    Intrinsic::experimental_constrained_exp2));
2063
2064     case Builtin::BIfabs:
2065     case Builtin::BIfabsf:
2066     case Builtin::BIfabsl:
2067     case Builtin::BI__builtin_fabs:
2068     case Builtin::BI__builtin_fabsf:
2069     case Builtin::BI__builtin_fabsf16:
2070     case Builtin::BI__builtin_fabsl:
2071     case Builtin::BI__builtin_fabsf128:
2072       return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::fabs));
2073
2074     case Builtin::BIfloor:
2075     case Builtin::BIfloorf:
2076     case Builtin::BIfloorl:
2077     case Builtin::BI__builtin_floor:
2078     case Builtin::BI__builtin_floorf:
2079     case Builtin::BI__builtin_floorf16:
2080     case Builtin::BI__builtin_floorl:
2081     case Builtin::BI__builtin_floorf128:
2082       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2083                                    Intrinsic::floor,
2084                                    Intrinsic::experimental_constrained_floor));
2085
2086     case Builtin::BIfma:
2087     case Builtin::BIfmaf:
2088     case Builtin::BIfmal:
2089     case Builtin::BI__builtin_fma:
2090     case Builtin::BI__builtin_fmaf:
2091     case Builtin::BI__builtin_fmaf16:
2092     case Builtin::BI__builtin_fmal:
2093     case Builtin::BI__builtin_fmaf128:
2094       return RValue::get(emitTernaryMaybeConstrainedFPBuiltin(*this, E,
2095                                    Intrinsic::fma,
2096                                    Intrinsic::experimental_constrained_fma));
2097
2098     case Builtin::BIfmax:
2099     case Builtin::BIfmaxf:
2100     case Builtin::BIfmaxl:
2101     case Builtin::BI__builtin_fmax:
2102     case Builtin::BI__builtin_fmaxf:
2103     case Builtin::BI__builtin_fmaxf16:
2104     case Builtin::BI__builtin_fmaxl:
2105     case Builtin::BI__builtin_fmaxf128:
2106       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2107                                    Intrinsic::maxnum,
2108                                    Intrinsic::experimental_constrained_maxnum));
2109
2110     case Builtin::BIfmin:
2111     case Builtin::BIfminf:
2112     case Builtin::BIfminl:
2113     case Builtin::BI__builtin_fmin:
2114     case Builtin::BI__builtin_fminf:
2115     case Builtin::BI__builtin_fminf16:
2116     case Builtin::BI__builtin_fminl:
2117     case Builtin::BI__builtin_fminf128:
2118       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2119                                    Intrinsic::minnum,
2120                                    Intrinsic::experimental_constrained_minnum));
2121
2122     // fmod() is a special-case. It maps to the frem instruction rather than an
2123     // LLVM intrinsic.
2124     case Builtin::BIfmod:
2125     case Builtin::BIfmodf:
2126     case Builtin::BIfmodl:
2127     case Builtin::BI__builtin_fmod:
2128     case Builtin::BI__builtin_fmodf:
2129     case Builtin::BI__builtin_fmodf16:
2130     case Builtin::BI__builtin_fmodl:
2131     case Builtin::BI__builtin_fmodf128: {
2132       Value *Arg1 = EmitScalarExpr(E->getArg(0));
2133       Value *Arg2 = EmitScalarExpr(E->getArg(1));
2134       return RValue::get(Builder.CreateFRem(Arg1, Arg2, "fmod"));
2135     }
2136
2137     case Builtin::BIlog:
2138     case Builtin::BIlogf:
2139     case Builtin::BIlogl:
2140     case Builtin::BI__builtin_log:
2141     case Builtin::BI__builtin_logf:
2142     case Builtin::BI__builtin_logf16:
2143     case Builtin::BI__builtin_logl:
2144     case Builtin::BI__builtin_logf128:
2145       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2146                                    Intrinsic::log,
2147                                    Intrinsic::experimental_constrained_log));
2148
2149     case Builtin::BIlog10:
2150     case Builtin::BIlog10f:
2151     case Builtin::BIlog10l:
2152     case Builtin::BI__builtin_log10:
2153     case Builtin::BI__builtin_log10f:
2154     case Builtin::BI__builtin_log10f16:
2155     case Builtin::BI__builtin_log10l:
2156     case Builtin::BI__builtin_log10f128:
2157       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2158                                    Intrinsic::log10,
2159                                    Intrinsic::experimental_constrained_log10));
2160
2161     case Builtin::BIlog2:
2162     case Builtin::BIlog2f:
2163     case Builtin::BIlog2l:
2164     case Builtin::BI__builtin_log2:
2165     case Builtin::BI__builtin_log2f:
2166     case Builtin::BI__builtin_log2f16:
2167     case Builtin::BI__builtin_log2l:
2168     case Builtin::BI__builtin_log2f128:
2169       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2170                                    Intrinsic::log2,
2171                                    Intrinsic::experimental_constrained_log2));
2172
2173     case Builtin::BInearbyint:
2174     case Builtin::BInearbyintf:
2175     case Builtin::BInearbyintl:
2176     case Builtin::BI__builtin_nearbyint:
2177     case Builtin::BI__builtin_nearbyintf:
2178     case Builtin::BI__builtin_nearbyintl:
2179     case Builtin::BI__builtin_nearbyintf128:
2180       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2181                                 Intrinsic::nearbyint,
2182                                 Intrinsic::experimental_constrained_nearbyint));
2183
2184     case Builtin::BIpow:
2185     case Builtin::BIpowf:
2186     case Builtin::BIpowl:
2187     case Builtin::BI__builtin_pow:
2188     case Builtin::BI__builtin_powf:
2189     case Builtin::BI__builtin_powf16:
2190     case Builtin::BI__builtin_powl:
2191     case Builtin::BI__builtin_powf128:
2192       return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(*this, E,
2193                                    Intrinsic::pow,
2194                                    Intrinsic::experimental_constrained_pow));
2195
2196     case Builtin::BIrint:
2197     case Builtin::BIrintf:
2198     case Builtin::BIrintl:
2199     case Builtin::BI__builtin_rint:
2200     case Builtin::BI__builtin_rintf:
2201     case Builtin::BI__builtin_rintf16:
2202     case Builtin::BI__builtin_rintl:
2203     case Builtin::BI__builtin_rintf128:
2204       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2205                                    Intrinsic::rint,
2206                                    Intrinsic::experimental_constrained_rint));
2207
2208     case Builtin::BIround:
2209     case Builtin::BIroundf:
2210     case Builtin::BIroundl:
2211     case Builtin::BI__builtin_round:
2212     case Builtin::BI__builtin_roundf:
2213     case Builtin::BI__builtin_roundf16:
2214     case Builtin::BI__builtin_roundl:
2215     case Builtin::BI__builtin_roundf128:
2216       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2217                                    Intrinsic::round,
2218                                    Intrinsic::experimental_constrained_round));
2219
2220     case Builtin::BIsin:
2221     case Builtin::BIsinf:
2222     case Builtin::BIsinl:
2223     case Builtin::BI__builtin_sin:
2224     case Builtin::BI__builtin_sinf:
2225     case Builtin::BI__builtin_sinf16:
2226     case Builtin::BI__builtin_sinl:
2227     case Builtin::BI__builtin_sinf128:
2228       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2229                                    Intrinsic::sin,
2230                                    Intrinsic::experimental_constrained_sin));
2231
2232     case Builtin::BIsqrt:
2233     case Builtin::BIsqrtf:
2234     case Builtin::BIsqrtl:
2235     case Builtin::BI__builtin_sqrt:
2236     case Builtin::BI__builtin_sqrtf:
2237     case Builtin::BI__builtin_sqrtf16:
2238     case Builtin::BI__builtin_sqrtl:
2239     case Builtin::BI__builtin_sqrtf128:
2240       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2241                                    Intrinsic::sqrt,
2242                                    Intrinsic::experimental_constrained_sqrt));
2243
2244     case Builtin::BItrunc:
2245     case Builtin::BItruncf:
2246     case Builtin::BItruncl:
2247     case Builtin::BI__builtin_trunc:
2248     case Builtin::BI__builtin_truncf:
2249     case Builtin::BI__builtin_truncf16:
2250     case Builtin::BI__builtin_truncl:
2251     case Builtin::BI__builtin_truncf128:
2252       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(*this, E,
2253                                    Intrinsic::trunc,
2254                                    Intrinsic::experimental_constrained_trunc));
2255
2256     case Builtin::BIlround:
2257     case Builtin::BIlroundf:
2258     case Builtin::BIlroundl:
2259     case Builtin::BI__builtin_lround:
2260     case Builtin::BI__builtin_lroundf:
2261     case Builtin::BI__builtin_lroundl:
2262     case Builtin::BI__builtin_lroundf128:
2263       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
2264           *this, E, Intrinsic::lround,
2265           Intrinsic::experimental_constrained_lround));
2266
2267     case Builtin::BIllround:
2268     case Builtin::BIllroundf:
2269     case Builtin::BIllroundl:
2270     case Builtin::BI__builtin_llround:
2271     case Builtin::BI__builtin_llroundf:
2272     case Builtin::BI__builtin_llroundl:
2273     case Builtin::BI__builtin_llroundf128:
2274       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
2275           *this, E, Intrinsic::llround,
2276           Intrinsic::experimental_constrained_llround));
2277
2278     case Builtin::BIlrint:
2279     case Builtin::BIlrintf:
2280     case Builtin::BIlrintl:
2281     case Builtin::BI__builtin_lrint:
2282     case Builtin::BI__builtin_lrintf:
2283     case Builtin::BI__builtin_lrintl:
2284     case Builtin::BI__builtin_lrintf128:
2285       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
2286           *this, E, Intrinsic::lrint,
2287           Intrinsic::experimental_constrained_lrint));
2288
2289     case Builtin::BIllrint:
2290     case Builtin::BIllrintf:
2291     case Builtin::BIllrintl:
2292     case Builtin::BI__builtin_llrint:
2293     case Builtin::BI__builtin_llrintf:
2294     case Builtin::BI__builtin_llrintl:
2295     case Builtin::BI__builtin_llrintf128:
2296       return RValue::get(emitMaybeConstrainedFPToIntRoundBuiltin(
2297           *this, E, Intrinsic::llrint,
2298           Intrinsic::experimental_constrained_llrint));
2299
2300     default:
2301       break;
2302     }
2303   }
2304
2305   switch (BuiltinIDIfNoAsmLabel) {
2306   default: break;
2307   case Builtin::BI__builtin___CFStringMakeConstantString:
2308   case Builtin::BI__builtin___NSStringMakeConstantString:
2309     return RValue::get(ConstantEmitter(*this).emitAbstract(E, E->getType()));
2310   case Builtin::BI__builtin_stdarg_start:
2311   case Builtin::BI__builtin_va_start:
2312   case Builtin::BI__va_start:
2313   case Builtin::BI__builtin_va_end:
2314     return RValue::get(
2315         EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
2316                            ? EmitScalarExpr(E->getArg(0))
2317                            : EmitVAListRef(E->getArg(0)).getPointer(),
2318                        BuiltinID != Builtin::BI__builtin_va_end));
2319   case Builtin::BI__builtin_va_copy: {
2320     Value *DstPtr = EmitVAListRef(E->getArg(0)).getPointer();
2321     Value *SrcPtr = EmitVAListRef(E->getArg(1)).getPointer();
2322
2323     llvm::Type *Type = Int8PtrTy;
2324
2325     DstPtr = Builder.CreateBitCast(DstPtr, Type);
2326     SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
2327     return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
2328                                           {DstPtr, SrcPtr}));
2329   }
2330   case Builtin::BI__builtin_abs:
2331   case Builtin::BI__builtin_labs:
2332   case Builtin::BI__builtin_llabs: {
2333     // X < 0 ? -X : X
2334     // The negation has 'nsw' because abs of INT_MIN is undefined.
2335     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2336     Value *NegOp = Builder.CreateNSWNeg(ArgValue, "neg");
2337     Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
2338     Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero, "abscond");
2339     Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue, "abs");
2340     return RValue::get(Result);
2341   }
2342   case Builtin::BI__builtin_complex: {
2343     Value *Real = EmitScalarExpr(E->getArg(0));
2344     Value *Imag = EmitScalarExpr(E->getArg(1));
2345     return RValue::getComplex({Real, Imag});
2346   }
2347   case Builtin::BI__builtin_conj:
2348   case Builtin::BI__builtin_conjf:
2349   case Builtin::BI__builtin_conjl:
2350   case Builtin::BIconj:
2351   case Builtin::BIconjf:
2352   case Builtin::BIconjl: {
2353     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2354     Value *Real = ComplexVal.first;
2355     Value *Imag = ComplexVal.second;
2356     Imag = Builder.CreateFNeg(Imag, "neg");
2357     return RValue::getComplex(std::make_pair(Real, Imag));
2358   }
2359   case Builtin::BI__builtin_creal:
2360   case Builtin::BI__builtin_crealf:
2361   case Builtin::BI__builtin_creall:
2362   case Builtin::BIcreal:
2363   case Builtin::BIcrealf:
2364   case Builtin::BIcreall: {
2365     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2366     return RValue::get(ComplexVal.first);
2367   }
2368
2369   case Builtin::BI__builtin_dump_struct: {
2370     llvm::Type *LLVMIntTy = getTypes().ConvertType(getContext().IntTy);
2371     llvm::FunctionType *LLVMFuncType = llvm::FunctionType::get(
2372         LLVMIntTy, {llvm::Type::getInt8PtrTy(getLLVMContext())}, true);
2373
2374     Value *Func = EmitScalarExpr(E->getArg(1)->IgnoreImpCasts());
2375     CharUnits Arg0Align = EmitPointerWithAlignment(E->getArg(0)).getAlignment();
2376
2377     const Expr *Arg0 = E->getArg(0)->IgnoreImpCasts();
2378     QualType Arg0Type = Arg0->getType()->getPointeeType();
2379
2380     Value *RecordPtr = EmitScalarExpr(Arg0);
2381     Value *Res = dumpRecord(*this, Arg0Type, RecordPtr, Arg0Align,
2382                             {LLVMFuncType, Func}, 0);
2383     return RValue::get(Res);
2384   }
2385
2386   case Builtin::BI__builtin_preserve_access_index: {
2387     // Only enabled preserved access index region when debuginfo
2388     // is available as debuginfo is needed to preserve user-level
2389     // access pattern.
2390     if (!getDebugInfo()) {
2391       CGM.Error(E->getExprLoc(), "using builtin_preserve_access_index() without -g");
2392       return RValue::get(EmitScalarExpr(E->getArg(0)));
2393     }
2394
2395     // Nested builtin_preserve_access_index() not supported
2396     if (IsInPreservedAIRegion) {
2397       CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported");
2398       return RValue::get(EmitScalarExpr(E->getArg(0)));
2399     }
2400
2401     IsInPreservedAIRegion = true;
2402     Value *Res = EmitScalarExpr(E->getArg(0));
2403     IsInPreservedAIRegion = false;
2404     return RValue::get(Res);
2405   }
2406
2407   case Builtin::BI__builtin_cimag:
2408   case Builtin::BI__builtin_cimagf:
2409   case Builtin::BI__builtin_cimagl:
2410   case Builtin::BIcimag:
2411   case Builtin::BIcimagf:
2412   case Builtin::BIcimagl: {
2413     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
2414     return RValue::get(ComplexVal.second);
2415   }
2416
2417   case Builtin::BI__builtin_clrsb:
2418   case Builtin::BI__builtin_clrsbl:
2419   case Builtin::BI__builtin_clrsbll: {
2420     // clrsb(x) -> clz(x < 0 ? ~x : x) - 1 or
2421     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2422
2423     llvm::Type *ArgType = ArgValue->getType();
2424     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2425
2426     llvm::Type *ResultType = ConvertType(E->getType());
2427     Value *Zero = llvm::Constant::getNullValue(ArgType);
2428     Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero, "isneg");
2429     Value *Inverse = Builder.CreateNot(ArgValue, "not");
2430     Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
2431     Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
2432     Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
2433     Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2434                                    "cast");
2435     return RValue::get(Result);
2436   }
2437   case Builtin::BI__builtin_ctzs:
2438   case Builtin::BI__builtin_ctz:
2439   case Builtin::BI__builtin_ctzl:
2440   case Builtin::BI__builtin_ctzll: {
2441     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CTZPassedZero);
2442
2443     llvm::Type *ArgType = ArgValue->getType();
2444     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
2445
2446     llvm::Type *ResultType = ConvertType(E->getType());
2447     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
2448     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2449     if (Result->getType() != ResultType)
2450       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2451                                      "cast");
2452     return RValue::get(Result);
2453   }
2454   case Builtin::BI__builtin_clzs:
2455   case Builtin::BI__builtin_clz:
2456   case Builtin::BI__builtin_clzl:
2457   case Builtin::BI__builtin_clzll: {
2458     Value *ArgValue = EmitCheckedArgForBuiltin(E->getArg(0), BCK_CLZPassedZero);
2459
2460     llvm::Type *ArgType = ArgValue->getType();
2461     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2462
2463     llvm::Type *ResultType = ConvertType(E->getType());
2464     Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
2465     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
2466     if (Result->getType() != ResultType)
2467       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2468                                      "cast");
2469     return RValue::get(Result);
2470   }
2471   case Builtin::BI__builtin_ffs:
2472   case Builtin::BI__builtin_ffsl:
2473   case Builtin::BI__builtin_ffsll: {
2474     // ffs(x) -> x ? cttz(x) + 1 : 0
2475     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2476
2477     llvm::Type *ArgType = ArgValue->getType();
2478     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
2479
2480     llvm::Type *ResultType = ConvertType(E->getType());
2481     Value *Tmp =
2482         Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
2483                           llvm::ConstantInt::get(ArgType, 1));
2484     Value *Zero = llvm::Constant::getNullValue(ArgType);
2485     Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero, "iszero");
2486     Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp, "ffs");
2487     if (Result->getType() != ResultType)
2488       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2489                                      "cast");
2490     return RValue::get(Result);
2491   }
2492   case Builtin::BI__builtin_parity:
2493   case Builtin::BI__builtin_parityl:
2494   case Builtin::BI__builtin_parityll: {
2495     // parity(x) -> ctpop(x) & 1
2496     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2497
2498     llvm::Type *ArgType = ArgValue->getType();
2499     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
2500
2501     llvm::Type *ResultType = ConvertType(E->getType());
2502     Value *Tmp = Builder.CreateCall(F, ArgValue);
2503     Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
2504     if (Result->getType() != ResultType)
2505       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2506                                      "cast");
2507     return RValue::get(Result);
2508   }
2509   case Builtin::BI__lzcnt16:
2510   case Builtin::BI__lzcnt:
2511   case Builtin::BI__lzcnt64: {
2512     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2513
2514     llvm::Type *ArgType = ArgValue->getType();
2515     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
2516
2517     llvm::Type *ResultType = ConvertType(E->getType());
2518     Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
2519     if (Result->getType() != ResultType)
2520       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2521                                      "cast");
2522     return RValue::get(Result);
2523   }
2524   case Builtin::BI__popcnt16:
2525   case Builtin::BI__popcnt:
2526   case Builtin::BI__popcnt64:
2527   case Builtin::BI__builtin_popcount:
2528   case Builtin::BI__builtin_popcountl:
2529   case Builtin::BI__builtin_popcountll: {
2530     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2531
2532     llvm::Type *ArgType = ArgValue->getType();
2533     Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
2534
2535     llvm::Type *ResultType = ConvertType(E->getType());
2536     Value *Result = Builder.CreateCall(F, ArgValue);
2537     if (Result->getType() != ResultType)
2538       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2539                                      "cast");
2540     return RValue::get(Result);
2541   }
2542   case Builtin::BI__builtin_unpredictable: {
2543     // Always return the argument of __builtin_unpredictable. LLVM does not
2544     // handle this builtin. Metadata for this builtin should be added directly
2545     // to instructions such as branches or switches that use it.
2546     return RValue::get(EmitScalarExpr(E->getArg(0)));
2547   }
2548   case Builtin::BI__builtin_expect: {
2549     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2550     llvm::Type *ArgType = ArgValue->getType();
2551
2552     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
2553     // Don't generate llvm.expect on -O0 as the backend won't use it for
2554     // anything.
2555     // Note, we still IRGen ExpectedValue because it could have side-effects.
2556     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2557       return RValue::get(ArgValue);
2558
2559     Function *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
2560     Value *Result =
2561         Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue}, "expval");
2562     return RValue::get(Result);
2563   }
2564   case Builtin::BI__builtin_expect_with_probability: {
2565     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2566     llvm::Type *ArgType = ArgValue->getType();
2567
2568     Value *ExpectedValue = EmitScalarExpr(E->getArg(1));
2569     llvm::APFloat Probability(0.0);
2570     const Expr *ProbArg = E->getArg(2);
2571     bool EvalSucceed = ProbArg->EvaluateAsFloat(Probability, CGM.getContext());
2572     assert(EvalSucceed && "probability should be able to evaluate as float");
2573     (void)EvalSucceed;
2574     bool LoseInfo = false;
2575     Probability.convert(llvm::APFloat::IEEEdouble(),
2576                         llvm::RoundingMode::Dynamic, &LoseInfo);
2577     llvm::Type *Ty = ConvertType(ProbArg->getType());
2578     Constant *Confidence = ConstantFP::get(Ty, Probability);
2579     // Don't generate llvm.expect.with.probability on -O0 as the backend
2580     // won't use it for anything.
2581     // Note, we still IRGen ExpectedValue because it could have side-effects.
2582     if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2583       return RValue::get(ArgValue);
2584
2585     Function *FnExpect =
2586         CGM.getIntrinsic(Intrinsic::expect_with_probability, ArgType);
2587     Value *Result = Builder.CreateCall(
2588         FnExpect, {ArgValue, ExpectedValue, Confidence}, "expval");
2589     return RValue::get(Result);
2590   }
2591   case Builtin::BI__builtin_assume_aligned: {
2592     const Expr *Ptr = E->getArg(0);
2593     Value *PtrValue = EmitScalarExpr(Ptr);
2594     Value *OffsetValue =
2595       (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) : nullptr;
2596
2597     Value *AlignmentValue = EmitScalarExpr(E->getArg(1));
2598     ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
2599     if (AlignmentCI->getValue().ugt(llvm::Value::MaximumAlignment))
2600       AlignmentCI = ConstantInt::get(AlignmentCI->getType(),
2601                                      llvm::Value::MaximumAlignment);
2602
2603     emitAlignmentAssumption(PtrValue, Ptr,
2604                             /*The expr loc is sufficient.*/ SourceLocation(),
2605                             AlignmentCI, OffsetValue);
2606     return RValue::get(PtrValue);
2607   }
2608   case Builtin::BI__assume:
2609   case Builtin::BI__builtin_assume: {
2610     if (E->getArg(0)->HasSideEffects(getContext()))
2611       return RValue::get(nullptr);
2612
2613     Value *ArgValue = EmitScalarExpr(E->getArg(0));
2614     Function *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
2615     return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
2616   }
2617   case Builtin::BI__builtin_bswap16:
2618   case Builtin::BI__builtin_bswap32:
2619   case Builtin::BI__builtin_bswap64: {
2620     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bswap));
2621   }
2622   case Builtin::BI__builtin_bitreverse8:
2623   case Builtin::BI__builtin_bitreverse16:
2624   case Builtin::BI__builtin_bitreverse32:
2625   case Builtin::BI__builtin_bitreverse64: {
2626     return RValue::get(emitUnaryBuiltin(*this, E, Intrinsic::bitreverse));
2627   }
2628   case Builtin::BI__builtin_rotateleft8:
2629   case Builtin::BI__builtin_rotateleft16:
2630   case Builtin::BI__builtin_rotateleft32:
2631   case Builtin::BI__builtin_rotateleft64:
2632   case Builtin::BI_rotl8: // Microsoft variants of rotate left
2633   case Builtin::BI_rotl16:
2634   case Builtin::BI_rotl:
2635   case Builtin::BI_lrotl:
2636   case Builtin::BI_rotl64:
2637     return emitRotate(E, false);
2638
2639   case Builtin::BI__builtin_rotateright8:
2640   case Builtin::BI__builtin_rotateright16:
2641   case Builtin::BI__builtin_rotateright32:
2642   case Builtin::BI__builtin_rotateright64:
2643   case Builtin::BI_rotr8: // Microsoft variants of rotate right
2644   case Builtin::BI_rotr16:
2645   case Builtin::BI_rotr:
2646   case Builtin::BI_lrotr:
2647   case Builtin::BI_rotr64:
2648     return emitRotate(E, true);
2649
2650   case Builtin::BI__builtin_constant_p: {
2651     llvm::Type *ResultType = ConvertType(E->getType());
2652
2653     const Expr *Arg = E->getArg(0);
2654     QualType ArgType = Arg->getType();
2655     // FIXME: The allowance for Obj-C pointers and block pointers is historical
2656     // and likely a mistake.
2657     if (!ArgType->isIntegralOrEnumerationType() && !ArgType->isFloatingType() &&
2658         !ArgType->isObjCObjectPointerType() && !ArgType->isBlockPointerType())
2659       // Per the GCC documentation, only numeric constants are recognized after
2660       // inlining.
2661       return RValue::get(ConstantInt::get(ResultType, 0));
2662
2663     if (Arg->HasSideEffects(getContext()))
2664       // The argument is unevaluated, so be conservative if it might have
2665       // side-effects.
2666       return RValue::get(ConstantInt::get(ResultType, 0));
2667
2668     Value *ArgValue = EmitScalarExpr(Arg);
2669     if (ArgType->isObjCObjectPointerType()) {
2670       // Convert Objective-C objects to id because we cannot distinguish between
2671       // LLVM types for Obj-C classes as they are opaque.
2672       ArgType = CGM.getContext().getObjCIdType();
2673       ArgValue = Builder.CreateBitCast(ArgValue, ConvertType(ArgType));
2674     }
2675     Function *F =
2676         CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
2677     Value *Result = Builder.CreateCall(F, ArgValue);
2678     if (Result->getType() != ResultType)
2679       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/false);
2680     return RValue::get(Result);
2681   }
2682   case Builtin::BI__builtin_dynamic_object_size:
2683   case Builtin::BI__builtin_object_size: {
2684     unsigned Type =
2685         E->getArg(1)->EvaluateKnownConstInt(getContext()).getZExtValue();
2686     auto *ResType = cast<llvm::IntegerType>(ConvertType(E->getType()));
2687
2688     // We pass this builtin onto the optimizer so that it can figure out the
2689     // object size in more complex cases.
2690     bool IsDynamic = BuiltinID == Builtin::BI__builtin_dynamic_object_size;
2691     return RValue::get(emitBuiltinObjectSize(E->getArg(0), Type, ResType,
2692                                              /*EmittedE=*/nullptr, IsDynamic));
2693   }
2694   case Builtin::BI__builtin_prefetch: {
2695     Value *Locality, *RW, *Address = EmitScalarExpr(E->getArg(0));
2696     // FIXME: Technically these constants should of type 'int', yes?
2697     RW = (E->getNumArgs() > 1) ? EmitScalarExpr(E->getArg(1)) :
2698       llvm::ConstantInt::get(Int32Ty, 0);
2699     Locality = (E->getNumArgs() > 2) ? EmitScalarExpr(E->getArg(2)) :
2700       llvm::ConstantInt::get(Int32Ty, 3);
2701     Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
2702     Function *F = CGM.getIntrinsic(Intrinsic::prefetch, Address->getType());
2703     return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
2704   }
2705   case Builtin::BI__builtin_readcyclecounter: {
2706     Function *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
2707     return RValue::get(Builder.CreateCall(F));
2708   }
2709   case Builtin::BI__builtin___clear_cache: {
2710     Value *Begin = EmitScalarExpr(E->getArg(0));
2711     Value *End = EmitScalarExpr(E->getArg(1));
2712     Function *F = CGM.getIntrinsic(Intrinsic::clear_cache);
2713     return RValue::get(Builder.CreateCall(F, {Begin, End}));
2714   }
2715   case Builtin::BI__builtin_trap:
2716     return RValue::get(EmitTrapCall(Intrinsic::trap));
2717   case Builtin::BI__debugbreak:
2718     return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
2719   case Builtin::BI__builtin_unreachable: {
2720     EmitUnreachable(E->getExprLoc());
2721
2722     // We do need to preserve an insertion point.
2723     EmitBlock(createBasicBlock("unreachable.cont"));
2724
2725     return RValue::get(nullptr);
2726   }
2727
2728   case Builtin::BI__builtin_powi:
2729   case Builtin::BI__builtin_powif:
2730   case Builtin::BI__builtin_powil:
2731     return RValue::get(emitBinaryMaybeConstrainedFPBuiltin(
2732         *this, E, Intrinsic::powi, Intrinsic::experimental_constrained_powi));
2733
2734   case Builtin::BI__builtin_isgreater:
2735   case Builtin::BI__builtin_isgreaterequal:
2736   case Builtin::BI__builtin_isless:
2737   case Builtin::BI__builtin_islessequal:
2738   case Builtin::BI__builtin_islessgreater:
2739   case Builtin::BI__builtin_isunordered: {
2740     // Ordered comparisons: we know the arguments to these are matching scalar
2741     // floating point values.
2742     Value *LHS = EmitScalarExpr(E->getArg(0));
2743     Value *RHS = EmitScalarExpr(E->getArg(1));
2744
2745     switch (BuiltinID) {
2746     default: llvm_unreachable("Unknown ordered comparison");
2747     case Builtin::BI__builtin_isgreater:
2748       LHS = Builder.CreateFCmpOGT(LHS, RHS, "cmp");
2749       break;
2750     case Builtin::BI__builtin_isgreaterequal:
2751       LHS = Builder.CreateFCmpOGE(LHS, RHS, "cmp");
2752       break;
2753     case Builtin::BI__builtin_isless:
2754       LHS = Builder.CreateFCmpOLT(LHS, RHS, "cmp");
2755       break;
2756     case Builtin::BI__builtin_islessequal:
2757       LHS = Builder.CreateFCmpOLE(LHS, RHS, "cmp");
2758       break;
2759     case Builtin::BI__builtin_islessgreater:
2760       LHS = Builder.CreateFCmpONE(LHS, RHS, "cmp");
2761       break;
2762     case Builtin::BI__builtin_isunordered:
2763       LHS = Builder.CreateFCmpUNO(LHS, RHS, "cmp");
2764       break;
2765     }
2766     // ZExt bool to int type.
2767     return RValue::get(Builder.CreateZExt(LHS, ConvertType(E->getType())));
2768   }
2769   case Builtin::BI__builtin_isnan: {
2770     Value *V = EmitScalarExpr(E->getArg(0));
2771     V = Builder.CreateFCmpUNO(V, V, "cmp");
2772     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
2773   }
2774
2775   case Builtin::BI__builtin_matrix_transpose: {
2776     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
2777     Value *MatValue = EmitScalarExpr(E->getArg(0));
2778     MatrixBuilder<CGBuilderTy> MB(Builder);
2779     Value *Result = MB.CreateMatrixTranspose(MatValue, MatrixTy->getNumRows(),
2780                                              MatrixTy->getNumColumns());
2781     return RValue::get(Result);
2782   }
2783
2784   case Builtin::BI__builtin_matrix_column_major_load: {
2785     MatrixBuilder<CGBuilderTy> MB(Builder);
2786     // Emit everything that isn't dependent on the first parameter type
2787     Value *Stride = EmitScalarExpr(E->getArg(3));
2788     const auto *ResultTy = E->getType()->getAs<ConstantMatrixType>();
2789     auto *PtrTy = E->getArg(0)->getType()->getAs<PointerType>();
2790     assert(PtrTy && "arg0 must be of pointer type");
2791     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
2792
2793     Address Src = EmitPointerWithAlignment(E->getArg(0));
2794     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(0)->getType(),
2795                         E->getArg(0)->getExprLoc(), FD, 0);
2796     Value *Result = MB.CreateColumnMajorLoad(
2797         Src.getPointer(), Align(Src.getAlignment().getQuantity()), Stride,
2798         IsVolatile, ResultTy->getNumRows(), ResultTy->getNumColumns(),
2799         "matrix");
2800     return RValue::get(Result);
2801   }
2802
2803   case Builtin::BI__builtin_matrix_column_major_store: {
2804     MatrixBuilder<CGBuilderTy> MB(Builder);
2805     Value *Matrix = EmitScalarExpr(E->getArg(0));
2806     Address Dst = EmitPointerWithAlignment(E->getArg(1));
2807     Value *Stride = EmitScalarExpr(E->getArg(2));
2808
2809     const auto *MatrixTy = E->getArg(0)->getType()->getAs<ConstantMatrixType>();
2810     auto *PtrTy = E->getArg(1)->getType()->getAs<PointerType>();
2811     assert(PtrTy && "arg1 must be of pointer type");
2812     bool IsVolatile = PtrTy->getPointeeType().isVolatileQualified();
2813
2814     EmitNonNullArgCheck(RValue::get(Dst.getPointer()), E->getArg(1)->getType(),
2815                         E->getArg(1)->getExprLoc(), FD, 0);
2816     Value *Result = MB.CreateColumnMajorStore(
2817         Matrix, Dst.getPointer(), Align(Dst.getAlignment().getQuantity()),
2818         Stride, IsVolatile, MatrixTy->getNumRows(), MatrixTy->getNumColumns());
2819     return RValue::get(Result);
2820   }
2821
2822   case Builtin::BIfinite:
2823   case Builtin::BI__finite:
2824   case Builtin::BIfinitef:
2825   case Builtin::BI__finitef:
2826   case Builtin::BIfinitel:
2827   case Builtin::BI__finitel:
2828   case Builtin::BI__builtin_isinf:
2829   case Builtin::BI__builtin_isfinite: {
2830     // isinf(x)    --> fabs(x) == infinity
2831     // isfinite(x) --> fabs(x) != infinity
2832     // x != NaN via the ordered compare in either case.
2833     Value *V = EmitScalarExpr(E->getArg(0));
2834     Value *Fabs = EmitFAbs(*this, V);
2835     Constant *Infinity = ConstantFP::getInfinity(V->getType());
2836     CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
2837                                   ? CmpInst::FCMP_OEQ
2838                                   : CmpInst::FCMP_ONE;
2839     Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity, "cmpinf");
2840     return RValue::get(Builder.CreateZExt(FCmp, ConvertType(E->getType())));
2841   }
2842
2843   case Builtin::BI__builtin_isinf_sign: {
2844     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
2845     Value *Arg = EmitScalarExpr(E->getArg(0));
2846     Value *AbsArg = EmitFAbs(*this, Arg);
2847     Value *IsInf = Builder.CreateFCmpOEQ(
2848         AbsArg, ConstantFP::getInfinity(Arg->getType()), "isinf");
2849     Value *IsNeg = EmitSignBit(*this, Arg);
2850
2851     llvm::Type *IntTy = ConvertType(E->getType());
2852     Value *Zero = Constant::getNullValue(IntTy);
2853     Value *One = ConstantInt::get(IntTy, 1);
2854     Value *NegativeOne = ConstantInt::get(IntTy, -1);
2855     Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
2856     Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
2857     return RValue::get(Result);
2858   }
2859
2860   case Builtin::BI__builtin_isnormal: {
2861     // isnormal(x) --> x == x && fabsf(x) < infinity && fabsf(x) >= float_min
2862     Value *V = EmitScalarExpr(E->getArg(0));
2863     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
2864
2865     Value *Abs = EmitFAbs(*this, V);
2866     Value *IsLessThanInf =
2867       Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
2868     APFloat Smallest = APFloat::getSmallestNormalized(
2869                    getContext().getFloatTypeSemantics(E->getArg(0)->getType()));
2870     Value *IsNormal =
2871       Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
2872                             "isnormal");
2873     V = Builder.CreateAnd(Eq, IsLessThanInf, "and");
2874     V = Builder.CreateAnd(V, IsNormal, "and");
2875     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
2876   }
2877
2878   case Builtin::BI__builtin_flt_rounds: {
2879     Function *F = CGM.getIntrinsic(Intrinsic::flt_rounds);
2880
2881     llvm::Type *ResultType = ConvertType(E->getType());
2882     Value *Result = Builder.CreateCall(F);
2883     if (Result->getType() != ResultType)
2884       Result = Builder.CreateIntCast(Result, ResultType, /*isSigned*/true,
2885                                      "cast");
2886     return RValue::get(Result);
2887   }
2888
2889   case Builtin::BI__builtin_fpclassify: {
2890     Value *V = EmitScalarExpr(E->getArg(5));
2891     llvm::Type *Ty = ConvertType(E->getArg(5)->getType());
2892
2893     // Create Result
2894     BasicBlock *Begin = Builder.GetInsertBlock();
2895     BasicBlock *End = createBasicBlock("fpclassify_end", this->CurFn);
2896     Builder.SetInsertPoint(End);
2897     PHINode *Result =
2898       Builder.CreatePHI(ConvertType(E->getArg(0)->getType()), 4,
2899                         "fpclassify_result");
2900
2901     // if (V==0) return FP_ZERO
2902     Builder.SetInsertPoint(Begin);
2903     Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
2904                                           "iszero");
2905     Value *ZeroLiteral = EmitScalarExpr(E->getArg(4));
2906     BasicBlock *NotZero = createBasicBlock("fpclassify_not_zero", this->CurFn);
2907     Builder.CreateCondBr(IsZero, End, NotZero);
2908     Result->addIncoming(ZeroLiteral, Begin);
2909
2910     // if (V != V) return FP_NAN
2911     Builder.SetInsertPoint(NotZero);
2912     Value *IsNan = Builder.CreateFCmpUNO(V, V, "cmp");
2913     Value *NanLiteral = EmitScalarExpr(E->getArg(0));
2914     BasicBlock *NotNan = createBasicBlock("fpclassify_not_nan", this->CurFn);
2915     Builder.CreateCondBr(IsNan, End, NotNan);
2916     Result->addIncoming(NanLiteral, NotZero);
2917
2918     // if (fabs(V) == infinity) return FP_INFINITY
2919     Builder.SetInsertPoint(NotNan);
2920     Value *VAbs = EmitFAbs(*this, V);
2921     Value *IsInf =
2922       Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
2923                             "isinf");
2924     Value *InfLiteral = EmitScalarExpr(E->getArg(1));
2925     BasicBlock *NotInf = createBasicBlock("fpclassify_not_inf", this->CurFn);
2926     Builder.CreateCondBr(IsInf, End, NotInf);
2927     Result->addIncoming(InfLiteral, NotNan);
2928
2929     // if (fabs(V) >= MIN_NORMAL) return FP_NORMAL else FP_SUBNORMAL
2930     Builder.SetInsertPoint(NotInf);
2931     APFloat Smallest = APFloat::getSmallestNormalized(
2932         getContext().getFloatTypeSemantics(E->getArg(5)->getType()));
2933     Value *IsNormal =
2934       Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
2935                             "isnormal");
2936     Value *NormalResult =
2937       Builder.CreateSelect(IsNormal, EmitScalarExpr(E->getArg(2)),
2938                            EmitScalarExpr(E->getArg(3)));
2939     Builder.CreateBr(End);
2940     Result->addIncoming(NormalResult, NotInf);
2941
2942     // return Result
2943     Builder.SetInsertPoint(End);
2944     return RValue::get(Result);
2945   }
2946
2947   case Builtin::BIalloca:
2948   case Builtin::BI_alloca:
2949   case Builtin::BI__builtin_alloca: {
2950     Value *Size = EmitScalarExpr(E->getArg(0));
2951     const TargetInfo &TI = getContext().getTargetInfo();
2952     // The alignment of the alloca should correspond to __BIGGEST_ALIGNMENT__.
2953     const Align SuitableAlignmentInBytes =
2954         CGM.getContext()
2955             .toCharUnitsFromBits(TI.getSuitableAlign())
2956             .getAsAlign();
2957     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2958     AI->setAlignment(SuitableAlignmentInBytes);
2959     initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
2960     return RValue::get(AI);
2961   }
2962
2963   case Builtin::BI__builtin_alloca_with_align: {
2964     Value *Size = EmitScalarExpr(E->getArg(0));
2965     Value *AlignmentInBitsValue = EmitScalarExpr(E->getArg(1));
2966     auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
2967     unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
2968     const Align AlignmentInBytes =
2969         CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getAsAlign();
2970     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2971     AI->setAlignment(AlignmentInBytes);
2972     initializeAlloca(*this, AI, Size, AlignmentInBytes);
2973     return RValue::get(AI);
2974   }
2975
2976   case Builtin::BIbzero:
2977   case Builtin::BI__builtin_bzero: {
2978     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2979     Value *SizeVal = EmitScalarExpr(E->getArg(1));
2980     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2981                         E->getArg(0)->getExprLoc(), FD, 0);
2982     Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal, false);
2983     return RValue::get(nullptr);
2984   }
2985   case Builtin::BImemcpy:
2986   case Builtin::BI__builtin_memcpy:
2987   case Builtin::BImempcpy:
2988   case Builtin::BI__builtin_mempcpy: {
2989     Address Dest = EmitPointerWithAlignment(E->getArg(0));
2990     Address Src = EmitPointerWithAlignment(E->getArg(1));
2991     Value *SizeVal = EmitScalarExpr(E->getArg(2));
2992     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
2993                         E->getArg(0)->getExprLoc(), FD, 0);
2994     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
2995                         E->getArg(1)->getExprLoc(), FD, 1);
2996     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
2997     if (BuiltinID == Builtin::BImempcpy ||
2998         BuiltinID == Builtin::BI__builtin_mempcpy)
2999       return RValue::get(Builder.CreateInBoundsGEP(Dest.getPointer(), SizeVal));
3000     else
3001       return RValue::get(Dest.getPointer());
3002   }
3003
3004   case Builtin::BI__builtin_memcpy_inline: {
3005     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3006     Address Src = EmitPointerWithAlignment(E->getArg(1));
3007     uint64_t Size =
3008         E->getArg(2)->EvaluateKnownConstInt(getContext()).getZExtValue();
3009     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
3010                         E->getArg(0)->getExprLoc(), FD, 0);
3011     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
3012                         E->getArg(1)->getExprLoc(), FD, 1);
3013     Builder.CreateMemCpyInline(Dest, Src, Size);
3014     return RValue::get(nullptr);
3015   }
3016
3017   case Builtin::BI__builtin_char_memchr:
3018     BuiltinID = Builtin::BI__builtin_memchr;
3019     break;
3020
3021   case Builtin::BI__builtin___memcpy_chk: {
3022     // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
3023     Expr::EvalResult SizeResult, DstSizeResult;
3024     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
3025         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
3026       break;
3027     llvm::APSInt Size = SizeResult.Val.getInt();
3028     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
3029     if (Size.ugt(DstSize))
3030       break;
3031     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3032     Address Src = EmitPointerWithAlignment(E->getArg(1));
3033     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3034     Builder.CreateMemCpy(Dest, Src, SizeVal, false);
3035     return RValue::get(Dest.getPointer());
3036   }
3037
3038   case Builtin::BI__builtin_objc_memmove_collectable: {
3039     Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
3040     Address SrcAddr = EmitPointerWithAlignment(E->getArg(1));
3041     Value *SizeVal = EmitScalarExpr(E->getArg(2));
3042     CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
3043                                                   DestAddr, SrcAddr, SizeVal);
3044     return RValue::get(DestAddr.getPointer());
3045   }
3046
3047   case Builtin::BI__builtin___memmove_chk: {
3048     // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
3049     Expr::EvalResult SizeResult, DstSizeResult;
3050     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
3051         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
3052       break;
3053     llvm::APSInt Size = SizeResult.Val.getInt();
3054     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
3055     if (Size.ugt(DstSize))
3056       break;
3057     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3058     Address Src = EmitPointerWithAlignment(E->getArg(1));
3059     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3060     Builder.CreateMemMove(Dest, Src, SizeVal, false);
3061     return RValue::get(Dest.getPointer());
3062   }
3063
3064   case Builtin::BImemmove:
3065   case Builtin::BI__builtin_memmove: {
3066     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3067     Address Src = EmitPointerWithAlignment(E->getArg(1));
3068     Value *SizeVal = EmitScalarExpr(E->getArg(2));
3069     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
3070                         E->getArg(0)->getExprLoc(), FD, 0);
3071     EmitNonNullArgCheck(RValue::get(Src.getPointer()), E->getArg(1)->getType(),
3072                         E->getArg(1)->getExprLoc(), FD, 1);
3073     Builder.CreateMemMove(Dest, Src, SizeVal, false);
3074     return RValue::get(Dest.getPointer());
3075   }
3076   case Builtin::BImemset:
3077   case Builtin::BI__builtin_memset: {
3078     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3079     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
3080                                          Builder.getInt8Ty());
3081     Value *SizeVal = EmitScalarExpr(E->getArg(2));
3082     EmitNonNullArgCheck(RValue::get(Dest.getPointer()), E->getArg(0)->getType(),
3083                         E->getArg(0)->getExprLoc(), FD, 0);
3084     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
3085     return RValue::get(Dest.getPointer());
3086   }
3087   case Builtin::BI__builtin___memset_chk: {
3088     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
3089     Expr::EvalResult SizeResult, DstSizeResult;
3090     if (!E->getArg(2)->EvaluateAsInt(SizeResult, CGM.getContext()) ||
3091         !E->getArg(3)->EvaluateAsInt(DstSizeResult, CGM.getContext()))
3092       break;
3093     llvm::APSInt Size = SizeResult.Val.getInt();
3094     llvm::APSInt DstSize = DstSizeResult.Val.getInt();
3095     if (Size.ugt(DstSize))
3096       break;
3097     Address Dest = EmitPointerWithAlignment(E->getArg(0));
3098     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
3099                                          Builder.getInt8Ty());
3100     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
3101     Builder.CreateMemSet(Dest, ByteVal, SizeVal, false);
3102     return RValue::get(Dest.getPointer());
3103   }
3104   case Builtin::BI__builtin_wmemcmp: {
3105     // The MSVC runtime library does not provide a definition of wmemcmp, so we
3106     // need an inline implementation.
3107     if (!getTarget().getTriple().isOSMSVCRT())
3108       break;
3109
3110     llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
3111
3112     Value *Dst = EmitScalarExpr(E->getArg(0));
3113     Value *Src = EmitScalarExpr(E->getArg(1));
3114     Value *Size = EmitScalarExpr(E->getArg(2));
3115
3116     BasicBlock *Entry = Builder.GetInsertBlock();
3117     BasicBlock *CmpGT = createBasicBlock("wmemcmp.gt");
3118     BasicBlock *CmpLT = createBasicBlock("wmemcmp.lt");
3119     BasicBlock *Next = createBasicBlock("wmemcmp.next");
3120     BasicBlock *Exit = createBasicBlock("wmemcmp.exit");
3121     Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
3122     Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
3123
3124     EmitBlock(CmpGT);
3125     PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
3126     DstPhi->addIncoming(Dst, Entry);
3127     PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
3128     SrcPhi->addIncoming(Src, Entry);
3129     PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
3130     SizePhi->addIncoming(Size, Entry);
3131     CharUnits WCharAlign =
3132         getContext().getTypeAlignInChars(getContext().WCharTy);
3133     Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
3134     Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
3135     Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
3136     Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
3137
3138     EmitBlock(CmpLT);
3139     Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
3140     Builder.CreateCondBr(DstLtSrc, Exit, Next);
3141
3142     EmitBlock(Next);
3143     Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
3144     Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
3145     Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
3146     Value *NextSizeEq0 =
3147         Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
3148     Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
3149     DstPhi->addIncoming(NextDst, Next);
3150     SrcPhi->addIncoming(NextSrc, Next);
3151     SizePhi->addIncoming(NextSize, Next);
3152
3153     EmitBlock(Exit);
3154     PHINode *Ret = Builder.CreatePHI(IntTy, 4);
3155     Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
3156     Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
3157     Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
3158     Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
3159     return RValue::get(Ret);
3160   }
3161   case Builtin::BI__builtin_dwarf_cfa: {
3162     // The offset in bytes from the first argument to the CFA.
3163     //
3164     // Why on earth is this in the frontend?  Is there any reason at
3165     // all that the backend can't reasonably determine this while
3166     // lowering llvm.eh.dwarf.cfa()?
3167     //
3168     // TODO: If there's a satisfactory reason, add a target hook for
3169     // this instead of hard-coding 0, which is correct for most targets.
3170     int32_t Offset = 0;
3171
3172     Function *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
3173     return RValue::get(Builder.CreateCall(F,
3174                                       llvm::ConstantInt::get(Int32Ty, Offset)));
3175   }
3176   case Builtin::BI__builtin_return_address: {
3177     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
3178                                                    getContext().UnsignedIntTy);
3179     Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
3180     return RValue::get(Builder.CreateCall(F, Depth));
3181   }
3182   case Builtin::BI_ReturnAddress: {
3183     Function *F = CGM.getIntrinsic(Intrinsic::returnaddress);
3184     return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
3185   }
3186   case Builtin::BI__builtin_frame_address: {
3187     Value *Depth = ConstantEmitter(*this).emitAbstract(E->getArg(0),
3188                                                    getContext().UnsignedIntTy);
3189     Function *F = CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy);
3190     return RValue::get(Builder.CreateCall(F, Depth));
3191   }
3192   case Builtin::BI__builtin_extract_return_addr: {
3193     Value *Address = EmitScalarExpr(E->getArg(0));
3194     Value *Result = getTargetHooks().decodeReturnAddress(*this, Address);
3195     return RValue::get(Result);
3196   }
3197   case Builtin::BI__builtin_frob_return_addr: {
3198     Value *Address = EmitScalarExpr(E->getArg(0));
3199     Value *Result = getTargetHooks().encodeReturnAddress(*this, Address);
3200     return RValue::get(Result);
3201   }
3202   case Builtin::BI__builtin_dwarf_sp_column: {
3203     llvm::IntegerType *Ty
3204       = cast<llvm::IntegerType>(ConvertType(E->getType()));
3205     int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
3206     if (Column == -1) {
3207       CGM.ErrorUnsupported(E, "__builtin_dwarf_sp_column");
3208       return RValue::get(llvm::UndefValue::get(Ty));
3209     }
3210     return RValue::get(llvm::ConstantInt::get(Ty, Column, true));
3211   }
3212   case Builtin::BI__builtin_init_dwarf_reg_size_table: {
3213     Value *Address = EmitScalarExpr(E->getArg(0));
3214     if (getTargetHooks().initDwarfEHRegSizeTable(*this, Address))
3215       CGM.ErrorUnsupported(E, "__builtin_init_dwarf_reg_size_table");
3216     return RValue::get(llvm::UndefValue::get(ConvertType(E->getType())));
3217   }
3218   case Builtin::BI__builtin_eh_return: {
3219     Value *Int = EmitScalarExpr(E->getArg(0));
3220     Value *Ptr = EmitScalarExpr(E->getArg(1));
3221
3222     llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
3223     assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
3224            "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
3225     Function *F =
3226         CGM.getIntrinsic(IntTy->getBitWidth() == 32 ? Intrinsic::eh_return_i32
3227                                                     : Intrinsic::eh_return_i64);
3228     Builder.CreateCall(F, {Int, Ptr});
3229     Builder.CreateUnreachable();
3230
3231     // We do need to preserve an insertion point.
3232     EmitBlock(createBasicBlock("builtin_eh_return.cont"));
3233
3234     return RValue::get(nullptr);
3235   }
3236   case Builtin::BI__builtin_unwind_init: {
3237     Function *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
3238     return RValue::get(Builder.CreateCall(F));
3239   }
3240   case Builtin::BI__builtin_extend_pointer: {
3241     // Extends a pointer to the size of an _Unwind_Word, which is
3242     // uint64_t on all platforms.  Generally this gets poked into a
3243     // register and eventually used as an address, so if the
3244     // addressing registers are wider than pointers and the platform
3245     // doesn't implicitly ignore high-order bits when doing
3246     // addressing, we need to make sure we zext / sext based on
3247     // the platform's expectations.
3248     //
3249     // See: http://gcc.gnu.org/ml/gcc-bugs/2002-02/msg00237.html
3250
3251     // Cast the pointer to intptr_t.
3252     Value *Ptr = EmitScalarExpr(E->getArg(0));
3253     Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy, "extend.cast");
3254
3255     // If that's 64 bits, we're done.
3256     if (IntPtrTy->getBitWidth() == 64)
3257       return RValue::get(Result);
3258
3259     // Otherwise, ask the codegen data what to do.
3260     if (getTargetHooks().extendPointerWithSExt())
3261       return RValue::get(Builder.CreateSExt(Result, Int64Ty, "extend.sext"));
3262     else
3263       return RValue::get(Builder.CreateZExt(Result, Int64Ty, "extend.zext"));
3264   }
3265   case Builtin::BI__builtin_setjmp: {
3266     // Buffer is a void**.
3267     Address Buf = EmitPointerWithAlignment(E->getArg(0));
3268
3269     // Store the frame pointer to the setjmp buffer.
3270     Value *FrameAddr = Builder.CreateCall(
3271         CGM.getIntrinsic(Intrinsic::frameaddress, AllocaInt8PtrTy),
3272         ConstantInt::get(Int32Ty, 0));
3273     Builder.CreateStore(FrameAddr, Buf);
3274
3275     // Store the stack pointer to the setjmp buffer.
3276     Value *StackAddr =
3277         Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
3278     Address StackSaveSlot = Builder.CreateConstInBoundsGEP(Buf, 2);
3279     Builder.CreateStore(StackAddr, StackSaveSlot);
3280
3281     // Call LLVM's EH setjmp, which is lightweight.
3282     Function *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
3283     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
3284     return RValue::get(Builder.CreateCall(F, Buf.getPointer()));
3285   }
3286   case Builtin::BI__builtin_longjmp: {
3287     Value *Buf = EmitScalarExpr(E->getArg(0));
3288     Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
3289
3290     // Call LLVM's EH longjmp, which is lightweight.
3291     Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
3292
3293     // longjmp doesn't return; mark this as unreachable.
3294     Builder.CreateUnreachable();
3295
3296     // We do need to preserve an insertion point.
3297     EmitBlock(createBasicBlock("longjmp.cont"));
3298
3299     return RValue::get(nullptr);
3300   }
3301   case Builtin::BI__builtin_launder: {
3302     const Expr *Arg = E->getArg(0);
3303     QualType ArgTy = Arg->getType()->getPointeeType();
3304     Value *Ptr = EmitScalarExpr(Arg);
3305     if (TypeRequiresBuiltinLaunder(CGM, ArgTy))
3306       Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
3307
3308     return RValue::get(Ptr);
3309   }
3310   case Builtin::BI__sync_fetch_and_add:
3311   case Builtin::BI__sync_fetch_and_sub:
3312   case Builtin::BI__sync_fetch_and_or:
3313   case Builtin::BI__sync_fetch_and_and:
3314   case Builtin::BI__sync_fetch_and_xor:
3315   case Builtin::BI__sync_fetch_and_nand:
3316   case Builtin::BI__sync_add_and_fetch:
3317   case Builtin::BI__sync_sub_and_fetch:
3318   case Builtin::BI__sync_and_and_fetch:
3319   case Builtin::BI__sync_or_and_fetch:
3320   case Builtin::BI__sync_xor_and_fetch:
3321   case Builtin::BI__sync_nand_and_fetch:
3322   case Builtin::BI__sync_val_compare_and_swap:
3323   case Builtin::BI__sync_bool_compare_and_swap:
3324   case Builtin::BI__sync_lock_test_and_set:
3325   case Builtin::BI__sync_lock_release:
3326   case Builtin::BI__sync_swap:
3327     llvm_unreachable("Shouldn't make it through sema");
3328   case Builtin::BI__sync_fetch_and_add_1:
3329   case Builtin::BI__sync_fetch_and_add_2:
3330   case Builtin::BI__sync_fetch_and_add_4:
3331   case Builtin::BI__sync_fetch_and_add_8:
3332   case Builtin::BI__sync_fetch_and_add_16:
3333     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Add, E);
3334   case Builtin::BI__sync_fetch_and_sub_1:
3335   case Builtin::BI__sync_fetch_and_sub_2:
3336   case Builtin::BI__sync_fetch_and_sub_4:
3337   case Builtin::BI__sync_fetch_and_sub_8:
3338   case Builtin::BI__sync_fetch_and_sub_16:
3339     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Sub, E);
3340   case Builtin::BI__sync_fetch_and_or_1:
3341   case Builtin::BI__sync_fetch_and_or_2:
3342   case Builtin::BI__sync_fetch_and_or_4:
3343   case Builtin::BI__sync_fetch_and_or_8:
3344   case Builtin::BI__sync_fetch_and_or_16:
3345     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Or, E);
3346   case Builtin::BI__sync_fetch_and_and_1:
3347   case Builtin::BI__sync_fetch_and_and_2:
3348   case Builtin::BI__sync_fetch_and_and_4:
3349   case Builtin::BI__sync_fetch_and_and_8:
3350   case Builtin::BI__sync_fetch_and_and_16:
3351     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::And, E);
3352   case Builtin::BI__sync_fetch_and_xor_1:
3353   case Builtin::BI__sync_fetch_and_xor_2:
3354   case Builtin::BI__sync_fetch_and_xor_4:
3355   case Builtin::BI__sync_fetch_and_xor_8:
3356   case Builtin::BI__sync_fetch_and_xor_16:
3357     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xor, E);
3358   case Builtin::BI__sync_fetch_and_nand_1:
3359   case Builtin::BI__sync_fetch_and_nand_2:
3360   case Builtin::BI__sync_fetch_and_nand_4:
3361   case Builtin::BI__sync_fetch_and_nand_8:
3362   case Builtin::BI__sync_fetch_and_nand_16:
3363     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Nand, E);
3364
3365   // Clang extensions: not overloaded yet.
3366   case Builtin::BI__sync_fetch_and_min:
3367     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Min, E);
3368   case Builtin::BI__sync_fetch_and_max:
3369     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Max, E);
3370   case Builtin::BI__sync_fetch_and_umin:
3371     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMin, E);
3372   case Builtin::BI__sync_fetch_and_umax:
3373     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::UMax, E);
3374
3375   case Builtin::BI__sync_add_and_fetch_1:
3376   case Builtin::BI__sync_add_and_fetch_2:
3377   case Builtin::BI__sync_add_and_fetch_4:
3378   case Builtin::BI__sync_add_and_fetch_8:
3379   case Builtin::BI__sync_add_and_fetch_16:
3380     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Add, E,
3381                                 llvm::Instruction::Add);
3382   case Builtin::BI__sync_sub_and_fetch_1:
3383   case Builtin::BI__sync_sub_and_fetch_2:
3384   case Builtin::BI__sync_sub_and_fetch_4:
3385   case Builtin::BI__sync_sub_and_fetch_8:
3386   case Builtin::BI__sync_sub_and_fetch_16:
3387     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Sub, E,
3388                                 llvm::Instruction::Sub);
3389   case Builtin::BI__sync_and_and_fetch_1:
3390   case Builtin::BI__sync_and_and_fetch_2:
3391   case Builtin::BI__sync_and_and_fetch_4:
3392   case Builtin::BI__sync_and_and_fetch_8:
3393   case Builtin::BI__sync_and_and_fetch_16:
3394     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::And, E,
3395                                 llvm::Instruction::And);
3396   case Builtin::BI__sync_or_and_fetch_1:
3397   case Builtin::BI__sync_or_and_fetch_2:
3398   case Builtin::BI__sync_or_and_fetch_4:
3399   case Builtin::BI__sync_or_and_fetch_8:
3400   case Builtin::BI__sync_or_and_fetch_16:
3401     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Or, E,
3402                                 llvm::Instruction::Or);
3403   case Builtin::BI__sync_xor_and_fetch_1:
3404   case Builtin::BI__sync_xor_and_fetch_2:
3405   case Builtin::BI__sync_xor_and_fetch_4:
3406   case Builtin::BI__sync_xor_and_fetch_8:
3407   case Builtin::BI__sync_xor_and_fetch_16:
3408     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Xor, E,
3409                                 llvm::Instruction::Xor);
3410   case Builtin::BI__sync_nand_and_fetch_1:
3411   case Builtin::BI__sync_nand_and_fetch_2:
3412   case Builtin::BI__sync_nand_and_fetch_4:
3413   case Builtin::BI__sync_nand_and_fetch_8:
3414   case Builtin::BI__sync_nand_and_fetch_16:
3415     return EmitBinaryAtomicPost(*this, llvm::AtomicRMWInst::Nand, E,
3416                                 llvm::Instruction::And, true);
3417
3418   case Builtin::BI__sync_val_compare_and_swap_1:
3419   case Builtin::BI__sync_val_compare_and_swap_2:
3420   case Builtin::BI__sync_val_compare_and_swap_4:
3421   case Builtin::BI__sync_val_compare_and_swap_8:
3422   case Builtin::BI__sync_val_compare_and_swap_16:
3423     return RValue::get(MakeAtomicCmpXchgValue(*this, E, false));
3424
3425   case Builtin::BI__sync_bool_compare_and_swap_1:
3426   case Builtin::BI__sync_bool_compare_and_swap_2:
3427   case Builtin::BI__sync_bool_compare_and_swap_4:
3428   case Builtin::BI__sync_bool_compare_and_swap_8:
3429   case Builtin::BI__sync_bool_compare_and_swap_16:
3430     return RValue::get(MakeAtomicCmpXchgValue(*this, E, true));
3431
3432   case Builtin::BI__sync_swap_1:
3433   case Builtin::BI__sync_swap_2:
3434   case Builtin::BI__sync_swap_4:
3435   case Builtin::BI__sync_swap_8:
3436   case Builtin::BI__sync_swap_16:
3437     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
3438
3439   case Builtin::BI__sync_lock_test_and_set_1:
3440   case Builtin::BI__sync_lock_test_and_set_2:
3441   case Builtin::BI__sync_lock_test_and_set_4:
3442   case Builtin::BI__sync_lock_test_and_set_8:
3443   case Builtin::BI__sync_lock_test_and_set_16:
3444     return EmitBinaryAtomic(*this, llvm::AtomicRMWInst::Xchg, E);
3445
3446   case Builtin::BI__sync_lock_release_1:
3447   case Builtin::BI__sync_lock_release_2:
3448   case Builtin::BI__sync_lock_release_4:
3449   case Builtin::BI__sync_lock_release_8:
3450   case Builtin::BI__sync_lock_release_16: {
3451     Value *Ptr = EmitScalarExpr(E->getArg(0));
3452     QualType ElTy = E->getArg(0)->getType()->getPointeeType();
3453     CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
3454     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
3455                                              StoreSize.getQuantity() * 8);
3456     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
3457     llvm::StoreInst *Store =
3458       Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
3459                                  StoreSize);
3460     Store->setAtomic(llvm::AtomicOrdering::Release);
3461     return RValue::get(nullptr);
3462   }
3463
3464   case Builtin::BI__sync_synchronize: {
3465     // We assume this is supposed to correspond to a C++0x-style
3466     // sequentially-consistent fence (i.e. this is only usable for
3467     // synchronization, not device I/O or anything like that). This intrinsic
3468     // is really badly designed in the sense that in theory, there isn't
3469     // any way to safely use it... but in practice, it mostly works
3470     // to use it with non-atomic loads and stores to get acquire/release
3471     // semantics.
3472     Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
3473     return RValue::get(nullptr);
3474   }
3475
3476   case Builtin::BI__builtin_nontemporal_load:
3477     return RValue::get(EmitNontemporalLoad(*this, E));
3478   case Builtin::BI__builtin_nontemporal_store:
3479     return RValue::get(EmitNontemporalStore(*this, E));
3480   case Builtin::BI__c11_atomic_is_lock_free:
3481   case Builtin::BI__atomic_is_lock_free: {
3482     // Call "bool __atomic_is_lock_free(size_t size, void *ptr)". For the
3483     // __c11 builtin, ptr is 0 (indicating a properly-aligned object), since
3484     // _Atomic(T) is always properly-aligned.
3485     const char *LibCallName = "__atomic_is_lock_free";
3486     CallArgList Args;
3487     Args.add(RValue::get(EmitScalarExpr(E->getArg(0))),
3488              getContext().getSizeType());
3489     if (BuiltinID == Builtin::BI__atomic_is_lock_free)
3490       Args.add(RValue::get(EmitScalarExpr(E->getArg(1))),
3491                getContext().VoidPtrTy);
3492     else
3493       Args.add(RValue::get(llvm::Constant::getNullValue(VoidPtrTy)),
3494                getContext().VoidPtrTy);
3495     const CGFunctionInfo &FuncInfo =
3496         CGM.getTypes().arrangeBuiltinFunctionCall(E->getType(), Args);
3497     llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
3498     llvm::FunctionCallee Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
3499     return EmitCall(FuncInfo, CGCallee::forDirect(Func),
3500                     ReturnValueSlot(), Args);
3501   }
3502
3503   case Builtin::BI__atomic_test_and_set: {
3504     // Look at the argument type to determine whether this is a volatile
3505     // operation. The parameter type is always volatile.
3506     QualType PtrTy = E->getArg(0)->IgnoreImpCasts()->getType();
3507     bool Volatile =
3508         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
3509
3510     Value *Ptr = EmitScalarExpr(E->getArg(0));
3511     unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
3512     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
3513     Value *NewVal = Builder.getInt8(1);
3514     Value *Order = EmitScalarExpr(E->getArg(1));
3515     if (isa<llvm::ConstantInt>(Order)) {
3516       int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3517       AtomicRMWInst *Result = nullptr;
3518       switch (ord) {
3519       case 0:  // memory_order_relaxed
3520       default: // invalid order
3521         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3522                                          llvm::AtomicOrdering::Monotonic);
3523         break;
3524       case 1: // memory_order_consume
3525       case 2: // memory_order_acquire
3526         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3527                                          llvm::AtomicOrdering::Acquire);
3528         break;
3529       case 3: // memory_order_release
3530         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3531                                          llvm::AtomicOrdering::Release);
3532         break;
3533       case 4: // memory_order_acq_rel
3534
3535         Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3536                                          llvm::AtomicOrdering::AcquireRelease);
3537         break;
3538       case 5: // memory_order_seq_cst
3539         Result = Builder.CreateAtomicRMW(
3540             llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
3541             llvm::AtomicOrdering::SequentiallyConsistent);
3542         break;
3543       }
3544       Result->setVolatile(Volatile);
3545       return RValue::get(Builder.CreateIsNotNull(Result, "tobool"));
3546     }
3547
3548     llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3549
3550     llvm::BasicBlock *BBs[5] = {
3551       createBasicBlock("monotonic", CurFn),
3552       createBasicBlock("acquire", CurFn),
3553       createBasicBlock("release", CurFn),
3554       createBasicBlock("acqrel", CurFn),
3555       createBasicBlock("seqcst", CurFn)
3556     };
3557     llvm::AtomicOrdering Orders[5] = {
3558         llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
3559         llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
3560         llvm::AtomicOrdering::SequentiallyConsistent};
3561
3562     Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3563     llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
3564
3565     Builder.SetInsertPoint(ContBB);
3566     PHINode *Result = Builder.CreatePHI(Int8Ty, 5, "was_set");
3567
3568     for (unsigned i = 0; i < 5; ++i) {
3569       Builder.SetInsertPoint(BBs[i]);
3570       AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,