1 //===--- CGCall.cpp - Encapsulate calling convention details --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // These classes wrap the information about a call or function
10 // definition used to handle ABI compliancy.
12 //===----------------------------------------------------------------------===//
18 #include "CGCleanup.h"
19 #include "CGRecordLayout.h"
20 #include "CodeGenFunction.h"
21 #include "CodeGenModule.h"
22 #include "TargetInfo.h"
23 #include "clang/AST/Attr.h"
24 #include "clang/AST/Decl.h"
25 #include "clang/AST/DeclCXX.h"
26 #include "clang/AST/DeclObjC.h"
27 #include "clang/Basic/CodeGenOptions.h"
28 #include "clang/Basic/TargetBuiltins.h"
29 #include "clang/Basic/TargetInfo.h"
30 #include "clang/CodeGen/CGFunctionInfo.h"
31 #include "clang/CodeGen/SwiftCallingConv.h"
32 #include "llvm/ADT/StringExtras.h"
33 #include "llvm/Analysis/ValueTracking.h"
34 #include "llvm/IR/Attributes.h"
35 #include "llvm/IR/CallingConv.h"
36 #include "llvm/IR/DataLayout.h"
37 #include "llvm/IR/InlineAsm.h"
38 #include "llvm/IR/IntrinsicInst.h"
39 #include "llvm/IR/Intrinsics.h"
40 #include "llvm/Transforms/Utils/Local.h"
41 using namespace clang;
42 using namespace CodeGen;
46 unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
48 default: return llvm::CallingConv::C;
49 case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
50 case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
51 case CC_X86RegCall: return llvm::CallingConv::X86_RegCall;
52 case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
53 case CC_Win64: return llvm::CallingConv::Win64;
54 case CC_X86_64SysV: return llvm::CallingConv::X86_64_SysV;
55 case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
56 case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
57 case CC_IntelOclBicc: return llvm::CallingConv::Intel_OCL_BI;
58 // TODO: Add support for __pascal to LLVM.
59 case CC_X86Pascal: return llvm::CallingConv::C;
60 // TODO: Add support for __vectorcall to LLVM.
61 case CC_X86VectorCall: return llvm::CallingConv::X86_VectorCall;
62 case CC_AArch64VectorCall: return llvm::CallingConv::AArch64_VectorCall;
63 case CC_SpirFunction: return llvm::CallingConv::SPIR_FUNC;
64 case CC_OpenCLKernel: return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
65 case CC_PreserveMost: return llvm::CallingConv::PreserveMost;
66 case CC_PreserveAll: return llvm::CallingConv::PreserveAll;
67 case CC_Swift: return llvm::CallingConv::Swift;
71 /// Derives the 'this' type for codegen purposes, i.e. ignoring method CVR
72 /// qualification. Either or both of RD and MD may be null. A null RD indicates
73 /// that there is no meaningful 'this' type, and a null MD can occur when
74 /// calling a method pointer.
75 CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
76 const CXXMethodDecl *MD) {
79 RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
81 RecTy = Context.VoidTy;
84 RecTy = Context.getAddrSpaceQualType(RecTy, MD->getMethodQualifiers().getAddressSpace());
85 return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
88 /// Returns the canonical formal type of the given C++ method.
89 static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
90 return MD->getType()->getCanonicalTypeUnqualified()
91 .getAs<FunctionProtoType>();
94 /// Returns the "extra-canonicalized" return type, which discards
95 /// qualifiers on the return type. Codegen doesn't care about them,
96 /// and it makes ABI code a little easier to be able to assume that
97 /// all parameter and return types are top-level unqualified.
98 static CanQualType GetReturnType(QualType RetTy) {
99 return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
102 /// Arrange the argument and result information for a value of the given
103 /// unprototyped freestanding function type.
104 const CGFunctionInfo &
105 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionNoProtoType> FTNP) {
106 // When translating an unprototyped function type, always use a
108 return arrangeLLVMFunctionInfo(FTNP->getReturnType().getUnqualifiedType(),
109 /*instanceMethod=*/false,
110 /*chainCall=*/false, None,
111 FTNP->getExtInfo(), {}, RequiredArgs(0));
114 static void addExtParameterInfosForCall(
115 llvm::SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos,
116 const FunctionProtoType *proto,
118 unsigned totalArgs) {
119 assert(proto->hasExtParameterInfos());
120 assert(paramInfos.size() <= prefixArgs);
121 assert(proto->getNumParams() + prefixArgs <= totalArgs);
123 paramInfos.reserve(totalArgs);
125 // Add default infos for any prefix args that don't already have infos.
126 paramInfos.resize(prefixArgs);
128 // Add infos for the prototype.
129 for (const auto &ParamInfo : proto->getExtParameterInfos()) {
130 paramInfos.push_back(ParamInfo);
131 // pass_object_size params have no parameter info.
132 if (ParamInfo.hasPassObjectSize())
133 paramInfos.emplace_back();
136 assert(paramInfos.size() <= totalArgs &&
137 "Did we forget to insert pass_object_size args?");
138 // Add default infos for the variadic and/or suffix arguments.
139 paramInfos.resize(totalArgs);
142 /// Adds the formal parameters in FPT to the given prefix. If any parameter in
143 /// FPT has pass_object_size attrs, then we'll add parameters for those, too.
144 static void appendParameterTypes(const CodeGenTypes &CGT,
145 SmallVectorImpl<CanQualType> &prefix,
146 SmallVectorImpl<FunctionProtoType::ExtParameterInfo> ¶mInfos,
147 CanQual<FunctionProtoType> FPT) {
148 // Fast path: don't touch param info if we don't need to.
149 if (!FPT->hasExtParameterInfos()) {
150 assert(paramInfos.empty() &&
151 "We have paramInfos, but the prototype doesn't?");
152 prefix.append(FPT->param_type_begin(), FPT->param_type_end());
156 unsigned PrefixSize = prefix.size();
157 // In the vast majority of cases, we'll have precisely FPT->getNumParams()
158 // parameters; the only thing that can change this is the presence of
159 // pass_object_size. So, we preallocate for the common case.
160 prefix.reserve(prefix.size() + FPT->getNumParams());
162 auto ExtInfos = FPT->getExtParameterInfos();
163 assert(ExtInfos.size() == FPT->getNumParams());
164 for (unsigned I = 0, E = FPT->getNumParams(); I != E; ++I) {
165 prefix.push_back(FPT->getParamType(I));
166 if (ExtInfos[I].hasPassObjectSize())
167 prefix.push_back(CGT.getContext().getSizeType());
170 addExtParameterInfosForCall(paramInfos, FPT.getTypePtr(), PrefixSize,
174 /// Arrange the LLVM function layout for a value of the given function
175 /// type, on top of any implicit parameters already stored.
176 static const CGFunctionInfo &
177 arrangeLLVMFunctionInfo(CodeGenTypes &CGT, bool instanceMethod,
178 SmallVectorImpl<CanQualType> &prefix,
179 CanQual<FunctionProtoType> FTP) {
180 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
181 RequiredArgs Required = RequiredArgs::forPrototypePlus(FTP, prefix.size());
183 appendParameterTypes(CGT, prefix, paramInfos, FTP);
184 CanQualType resultType = FTP->getReturnType().getUnqualifiedType();
186 return CGT.arrangeLLVMFunctionInfo(resultType, instanceMethod,
187 /*chainCall=*/false, prefix,
188 FTP->getExtInfo(), paramInfos,
192 /// Arrange the argument and result information for a value of the
193 /// given freestanding function type.
194 const CGFunctionInfo &
195 CodeGenTypes::arrangeFreeFunctionType(CanQual<FunctionProtoType> FTP) {
196 SmallVector<CanQualType, 16> argTypes;
197 return ::arrangeLLVMFunctionInfo(*this, /*instanceMethod=*/false, argTypes,
201 static CallingConv getCallingConventionForDecl(const Decl *D, bool IsWindows) {
202 // Set the appropriate calling convention for the Function.
203 if (D->hasAttr<StdCallAttr>())
204 return CC_X86StdCall;
206 if (D->hasAttr<FastCallAttr>())
207 return CC_X86FastCall;
209 if (D->hasAttr<RegCallAttr>())
210 return CC_X86RegCall;
212 if (D->hasAttr<ThisCallAttr>())
213 return CC_X86ThisCall;
215 if (D->hasAttr<VectorCallAttr>())
216 return CC_X86VectorCall;
218 if (D->hasAttr<PascalAttr>())
221 if (PcsAttr *PCS = D->getAttr<PcsAttr>())
222 return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
224 if (D->hasAttr<AArch64VectorPcsAttr>())
225 return CC_AArch64VectorCall;
227 if (D->hasAttr<IntelOclBiccAttr>())
228 return CC_IntelOclBicc;
230 if (D->hasAttr<MSABIAttr>())
231 return IsWindows ? CC_C : CC_Win64;
233 if (D->hasAttr<SysVABIAttr>())
234 return IsWindows ? CC_X86_64SysV : CC_C;
236 if (D->hasAttr<PreserveMostAttr>())
237 return CC_PreserveMost;
239 if (D->hasAttr<PreserveAllAttr>())
240 return CC_PreserveAll;
245 /// Arrange the argument and result information for a call to an
246 /// unknown C++ non-static member function of the given abstract type.
247 /// (A null RD means we don't have any meaningful "this" argument type,
248 /// so fall back to a generic pointer type).
249 /// The member function must be an ordinary function, i.e. not a
250 /// constructor or destructor.
251 const CGFunctionInfo &
252 CodeGenTypes::arrangeCXXMethodType(const CXXRecordDecl *RD,
253 const FunctionProtoType *FTP,
254 const CXXMethodDecl *MD) {
255 SmallVector<CanQualType, 16> argTypes;
257 // Add the 'this' pointer.
258 argTypes.push_back(DeriveThisType(RD, MD));
260 return ::arrangeLLVMFunctionInfo(
261 *this, true, argTypes,
262 FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
265 /// Set calling convention for CUDA/HIP kernel.
266 static void setCUDAKernelCallingConvention(CanQualType &FTy, CodeGenModule &CGM,
267 const FunctionDecl *FD) {
268 if (FD->hasAttr<CUDAGlobalAttr>()) {
269 const FunctionType *FT = FTy->getAs<FunctionType>();
270 CGM.getTargetCodeGenInfo().setCUDAKernelCallingConvention(FT);
271 FTy = FT->getCanonicalTypeUnqualified();
275 /// Arrange the argument and result information for a declaration or
276 /// definition of the given C++ non-static member function. The
277 /// member function must be an ordinary function, i.e. not a
278 /// constructor or destructor.
279 const CGFunctionInfo &
280 CodeGenTypes::arrangeCXXMethodDeclaration(const CXXMethodDecl *MD) {
281 assert(!isa<CXXConstructorDecl>(MD) && "wrong method for constructors!");
282 assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
284 CanQualType FT = GetFormalType(MD).getAs<Type>();
285 setCUDAKernelCallingConvention(FT, CGM, MD);
286 auto prototype = FT.getAs<FunctionProtoType>();
288 if (MD->isInstance()) {
289 // The abstract case is perfectly fine.
290 const CXXRecordDecl *ThisType = TheCXXABI.getThisArgumentTypeForMethod(MD);
291 return arrangeCXXMethodType(ThisType, prototype.getTypePtr(), MD);
294 return arrangeFreeFunctionType(prototype);
297 bool CodeGenTypes::inheritingCtorHasParams(
298 const InheritedConstructor &Inherited, CXXCtorType Type) {
299 // Parameters are unnecessary if we're constructing a base class subobject
300 // and the inherited constructor lives in a virtual base.
301 return Type == Ctor_Complete ||
302 !Inherited.getShadowDecl()->constructsVirtualBase() ||
303 !Target.getCXXABI().hasConstructorVariants();
306 const CGFunctionInfo &
307 CodeGenTypes::arrangeCXXStructorDeclaration(GlobalDecl GD) {
308 auto *MD = cast<CXXMethodDecl>(GD.getDecl());
310 SmallVector<CanQualType, 16> argTypes;
311 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
312 argTypes.push_back(DeriveThisType(MD->getParent(), MD));
314 bool PassParams = true;
316 if (auto *CD = dyn_cast<CXXConstructorDecl>(MD)) {
317 // A base class inheriting constructor doesn't get forwarded arguments
318 // needed to construct a virtual base (or base class thereof).
319 if (auto Inherited = CD->getInheritedConstructor())
320 PassParams = inheritingCtorHasParams(Inherited, GD.getCtorType());
323 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
325 // Add the formal parameters.
327 appendParameterTypes(*this, argTypes, paramInfos, FTP);
329 CGCXXABI::AddedStructorArgCounts AddedArgs =
330 TheCXXABI.buildStructorSignature(GD, argTypes);
331 if (!paramInfos.empty()) {
332 // Note: prefix implies after the first param.
333 if (AddedArgs.Prefix)
334 paramInfos.insert(paramInfos.begin() + 1, AddedArgs.Prefix,
335 FunctionProtoType::ExtParameterInfo{});
336 if (AddedArgs.Suffix)
337 paramInfos.append(AddedArgs.Suffix,
338 FunctionProtoType::ExtParameterInfo{});
341 RequiredArgs required =
342 (PassParams && MD->isVariadic() ? RequiredArgs(argTypes.size())
343 : RequiredArgs::All);
345 FunctionType::ExtInfo extInfo = FTP->getExtInfo();
346 CanQualType resultType = TheCXXABI.HasThisReturn(GD)
348 : TheCXXABI.hasMostDerivedReturn(GD)
349 ? CGM.getContext().VoidPtrTy
351 return arrangeLLVMFunctionInfo(resultType, /*instanceMethod=*/true,
352 /*chainCall=*/false, argTypes, extInfo,
353 paramInfos, required);
356 static SmallVector<CanQualType, 16>
357 getArgTypesForCall(ASTContext &ctx, const CallArgList &args) {
358 SmallVector<CanQualType, 16> argTypes;
359 for (auto &arg : args)
360 argTypes.push_back(ctx.getCanonicalParamType(arg.Ty));
364 static SmallVector<CanQualType, 16>
365 getArgTypesForDeclaration(ASTContext &ctx, const FunctionArgList &args) {
366 SmallVector<CanQualType, 16> argTypes;
367 for (auto &arg : args)
368 argTypes.push_back(ctx.getCanonicalParamType(arg->getType()));
372 static llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16>
373 getExtParameterInfosForCall(const FunctionProtoType *proto,
374 unsigned prefixArgs, unsigned totalArgs) {
375 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> result;
376 if (proto->hasExtParameterInfos()) {
377 addExtParameterInfosForCall(result, proto, prefixArgs, totalArgs);
382 /// Arrange a call to a C++ method, passing the given arguments.
384 /// ExtraPrefixArgs is the number of ABI-specific args passed after the `this`
386 /// ExtraSuffixArgs is the number of ABI-specific args passed at the end of
388 /// PassProtoArgs indicates whether `args` has args for the parameters in the
389 /// given CXXConstructorDecl.
390 const CGFunctionInfo &
391 CodeGenTypes::arrangeCXXConstructorCall(const CallArgList &args,
392 const CXXConstructorDecl *D,
393 CXXCtorType CtorKind,
394 unsigned ExtraPrefixArgs,
395 unsigned ExtraSuffixArgs,
396 bool PassProtoArgs) {
398 SmallVector<CanQualType, 16> ArgTypes;
399 for (const auto &Arg : args)
400 ArgTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
402 // +1 for implicit this, which should always be args[0].
403 unsigned TotalPrefixArgs = 1 + ExtraPrefixArgs;
405 CanQual<FunctionProtoType> FPT = GetFormalType(D);
406 RequiredArgs Required = PassProtoArgs
407 ? RequiredArgs::forPrototypePlus(
408 FPT, TotalPrefixArgs + ExtraSuffixArgs)
411 GlobalDecl GD(D, CtorKind);
412 CanQualType ResultType = TheCXXABI.HasThisReturn(GD)
414 : TheCXXABI.hasMostDerivedReturn(GD)
415 ? CGM.getContext().VoidPtrTy
418 FunctionType::ExtInfo Info = FPT->getExtInfo();
419 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> ParamInfos;
420 // If the prototype args are elided, we should only have ABI-specific args,
421 // which never have param info.
422 if (PassProtoArgs && FPT->hasExtParameterInfos()) {
423 // ABI-specific suffix arguments are treated the same as variadic arguments.
424 addExtParameterInfosForCall(ParamInfos, FPT.getTypePtr(), TotalPrefixArgs,
427 return arrangeLLVMFunctionInfo(ResultType, /*instanceMethod=*/true,
428 /*chainCall=*/false, ArgTypes, Info,
429 ParamInfos, Required);
432 /// Arrange the argument and result information for the declaration or
433 /// definition of the given function.
434 const CGFunctionInfo &
435 CodeGenTypes::arrangeFunctionDeclaration(const FunctionDecl *FD) {
436 if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
437 if (MD->isInstance())
438 return arrangeCXXMethodDeclaration(MD);
440 CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
442 assert(isa<FunctionType>(FTy));
443 setCUDAKernelCallingConvention(FTy, CGM, FD);
445 // When declaring a function without a prototype, always use a
446 // non-variadic type.
447 if (CanQual<FunctionNoProtoType> noProto = FTy.getAs<FunctionNoProtoType>()) {
448 return arrangeLLVMFunctionInfo(
449 noProto->getReturnType(), /*instanceMethod=*/false,
450 /*chainCall=*/false, None, noProto->getExtInfo(), {},RequiredArgs::All);
453 return arrangeFreeFunctionType(FTy.castAs<FunctionProtoType>());
456 /// Arrange the argument and result information for the declaration or
457 /// definition of an Objective-C method.
458 const CGFunctionInfo &
459 CodeGenTypes::arrangeObjCMethodDeclaration(const ObjCMethodDecl *MD) {
460 // It happens that this is the same as a call with no optional
461 // arguments, except also using the formal 'self' type.
462 return arrangeObjCMessageSendSignature(MD, MD->getSelfDecl()->getType());
465 /// Arrange the argument and result information for the function type
466 /// through which to perform a send to the given Objective-C method,
467 /// using the given receiver type. The receiver type is not always
468 /// the 'self' type of the method or even an Objective-C pointer type.
469 /// This is *not* the right method for actually performing such a
470 /// message send, due to the possibility of optional arguments.
471 const CGFunctionInfo &
472 CodeGenTypes::arrangeObjCMessageSendSignature(const ObjCMethodDecl *MD,
473 QualType receiverType) {
474 SmallVector<CanQualType, 16> argTys;
475 SmallVector<FunctionProtoType::ExtParameterInfo, 4> extParamInfos(2);
476 argTys.push_back(Context.getCanonicalParamType(receiverType));
477 argTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
479 for (const auto *I : MD->parameters()) {
480 argTys.push_back(Context.getCanonicalParamType(I->getType()));
481 auto extParamInfo = FunctionProtoType::ExtParameterInfo().withIsNoEscape(
482 I->hasAttr<NoEscapeAttr>());
483 extParamInfos.push_back(extParamInfo);
486 FunctionType::ExtInfo einfo;
487 bool IsWindows = getContext().getTargetInfo().getTriple().isOSWindows();
488 einfo = einfo.withCallingConv(getCallingConventionForDecl(MD, IsWindows));
490 if (getContext().getLangOpts().ObjCAutoRefCount &&
491 MD->hasAttr<NSReturnsRetainedAttr>())
492 einfo = einfo.withProducesResult(true);
494 RequiredArgs required =
495 (MD->isVariadic() ? RequiredArgs(argTys.size()) : RequiredArgs::All);
497 return arrangeLLVMFunctionInfo(
498 GetReturnType(MD->getReturnType()), /*instanceMethod=*/false,
499 /*chainCall=*/false, argTys, einfo, extParamInfos, required);
502 const CGFunctionInfo &
503 CodeGenTypes::arrangeUnprototypedObjCMessageSend(QualType returnType,
504 const CallArgList &args) {
505 auto argTypes = getArgTypesForCall(Context, args);
506 FunctionType::ExtInfo einfo;
508 return arrangeLLVMFunctionInfo(
509 GetReturnType(returnType), /*instanceMethod=*/false,
510 /*chainCall=*/false, argTypes, einfo, {}, RequiredArgs::All);
513 const CGFunctionInfo &
514 CodeGenTypes::arrangeGlobalDeclaration(GlobalDecl GD) {
515 // FIXME: Do we need to handle ObjCMethodDecl?
516 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
518 if (isa<CXXConstructorDecl>(GD.getDecl()) ||
519 isa<CXXDestructorDecl>(GD.getDecl()))
520 return arrangeCXXStructorDeclaration(GD);
522 return arrangeFunctionDeclaration(FD);
525 /// Arrange a thunk that takes 'this' as the first parameter followed by
526 /// varargs. Return a void pointer, regardless of the actual return type.
527 /// The body of the thunk will end in a musttail call to a function of the
528 /// correct type, and the caller will bitcast the function to the correct
530 const CGFunctionInfo &
531 CodeGenTypes::arrangeUnprototypedMustTailThunk(const CXXMethodDecl *MD) {
532 assert(MD->isVirtual() && "only methods have thunks");
533 CanQual<FunctionProtoType> FTP = GetFormalType(MD);
534 CanQualType ArgTys[] = {DeriveThisType(MD->getParent(), MD)};
535 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/false,
536 /*chainCall=*/false, ArgTys,
537 FTP->getExtInfo(), {}, RequiredArgs(1));
540 const CGFunctionInfo &
541 CodeGenTypes::arrangeMSCtorClosure(const CXXConstructorDecl *CD,
543 assert(CT == Ctor_CopyingClosure || CT == Ctor_DefaultClosure);
545 CanQual<FunctionProtoType> FTP = GetFormalType(CD);
546 SmallVector<CanQualType, 2> ArgTys;
547 const CXXRecordDecl *RD = CD->getParent();
548 ArgTys.push_back(DeriveThisType(RD, CD));
549 if (CT == Ctor_CopyingClosure)
550 ArgTys.push_back(*FTP->param_type_begin());
551 if (RD->getNumVBases() > 0)
552 ArgTys.push_back(Context.IntTy);
553 CallingConv CC = Context.getDefaultCallingConvention(
554 /*IsVariadic=*/false, /*IsCXXMethod=*/true);
555 return arrangeLLVMFunctionInfo(Context.VoidTy, /*instanceMethod=*/true,
556 /*chainCall=*/false, ArgTys,
557 FunctionType::ExtInfo(CC), {},
561 /// Arrange a call as unto a free function, except possibly with an
562 /// additional number of formal parameters considered required.
563 static const CGFunctionInfo &
564 arrangeFreeFunctionLikeCall(CodeGenTypes &CGT,
566 const CallArgList &args,
567 const FunctionType *fnType,
568 unsigned numExtraRequiredArgs,
570 assert(args.size() >= numExtraRequiredArgs);
572 llvm::SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
574 // In most cases, there are no optional arguments.
575 RequiredArgs required = RequiredArgs::All;
577 // If we have a variadic prototype, the required arguments are the
578 // extra prefix plus the arguments in the prototype.
579 if (const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(fnType)) {
580 if (proto->isVariadic())
581 required = RequiredArgs::forPrototypePlus(proto, numExtraRequiredArgs);
583 if (proto->hasExtParameterInfos())
584 addExtParameterInfosForCall(paramInfos, proto, numExtraRequiredArgs,
587 // If we don't have a prototype at all, but we're supposed to
588 // explicitly use the variadic convention for unprototyped calls,
589 // treat all of the arguments as required but preserve the nominal
590 // possibility of variadics.
591 } else if (CGM.getTargetCodeGenInfo()
592 .isNoProtoCallVariadic(args,
593 cast<FunctionNoProtoType>(fnType))) {
594 required = RequiredArgs(args.size());
598 SmallVector<CanQualType, 16> argTypes;
599 for (const auto &arg : args)
600 argTypes.push_back(CGT.getContext().getCanonicalParamType(arg.Ty));
601 return CGT.arrangeLLVMFunctionInfo(GetReturnType(fnType->getReturnType()),
602 /*instanceMethod=*/false, chainCall,
603 argTypes, fnType->getExtInfo(), paramInfos,
607 /// Figure out the rules for calling a function with the given formal
608 /// type using the given arguments. The arguments are necessary
609 /// because the function might be unprototyped, in which case it's
610 /// target-dependent in crazy ways.
611 const CGFunctionInfo &
612 CodeGenTypes::arrangeFreeFunctionCall(const CallArgList &args,
613 const FunctionType *fnType,
615 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType,
616 chainCall ? 1 : 0, chainCall);
619 /// A block function is essentially a free function with an
620 /// extra implicit argument.
621 const CGFunctionInfo &
622 CodeGenTypes::arrangeBlockFunctionCall(const CallArgList &args,
623 const FunctionType *fnType) {
624 return arrangeFreeFunctionLikeCall(*this, CGM, args, fnType, 1,
625 /*chainCall=*/false);
628 const CGFunctionInfo &
629 CodeGenTypes::arrangeBlockFunctionDeclaration(const FunctionProtoType *proto,
630 const FunctionArgList ¶ms) {
631 auto paramInfos = getExtParameterInfosForCall(proto, 1, params.size());
632 auto argTypes = getArgTypesForDeclaration(Context, params);
634 return arrangeLLVMFunctionInfo(GetReturnType(proto->getReturnType()),
635 /*instanceMethod*/ false, /*chainCall*/ false,
636 argTypes, proto->getExtInfo(), paramInfos,
637 RequiredArgs::forPrototypePlus(proto, 1));
640 const CGFunctionInfo &
641 CodeGenTypes::arrangeBuiltinFunctionCall(QualType resultType,
642 const CallArgList &args) {
644 SmallVector<CanQualType, 16> argTypes;
645 for (const auto &Arg : args)
646 argTypes.push_back(Context.getCanonicalParamType(Arg.Ty));
647 return arrangeLLVMFunctionInfo(
648 GetReturnType(resultType), /*instanceMethod=*/false,
649 /*chainCall=*/false, argTypes, FunctionType::ExtInfo(),
650 /*paramInfos=*/ {}, RequiredArgs::All);
653 const CGFunctionInfo &
654 CodeGenTypes::arrangeBuiltinFunctionDeclaration(QualType resultType,
655 const FunctionArgList &args) {
656 auto argTypes = getArgTypesForDeclaration(Context, args);
658 return arrangeLLVMFunctionInfo(
659 GetReturnType(resultType), /*instanceMethod=*/false, /*chainCall=*/false,
660 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
663 const CGFunctionInfo &
664 CodeGenTypes::arrangeBuiltinFunctionDeclaration(CanQualType resultType,
665 ArrayRef<CanQualType> argTypes) {
666 return arrangeLLVMFunctionInfo(
667 resultType, /*instanceMethod=*/false, /*chainCall=*/false,
668 argTypes, FunctionType::ExtInfo(), {}, RequiredArgs::All);
671 /// Arrange a call to a C++ method, passing the given arguments.
673 /// numPrefixArgs is the number of ABI-specific prefix arguments we have. It
674 /// does not count `this`.
675 const CGFunctionInfo &
676 CodeGenTypes::arrangeCXXMethodCall(const CallArgList &args,
677 const FunctionProtoType *proto,
678 RequiredArgs required,
679 unsigned numPrefixArgs) {
680 assert(numPrefixArgs + 1 <= args.size() &&
681 "Emitting a call with less args than the required prefix?");
682 // Add one to account for `this`. It's a bit awkward here, but we don't count
683 // `this` in similar places elsewhere.
685 getExtParameterInfosForCall(proto, numPrefixArgs + 1, args.size());
688 auto argTypes = getArgTypesForCall(Context, args);
690 FunctionType::ExtInfo info = proto->getExtInfo();
691 return arrangeLLVMFunctionInfo(
692 GetReturnType(proto->getReturnType()), /*instanceMethod=*/true,
693 /*chainCall=*/false, argTypes, info, paramInfos, required);
696 const CGFunctionInfo &CodeGenTypes::arrangeNullaryFunction() {
697 return arrangeLLVMFunctionInfo(
698 getContext().VoidTy, /*instanceMethod=*/false, /*chainCall=*/false,
699 None, FunctionType::ExtInfo(), {}, RequiredArgs::All);
702 const CGFunctionInfo &
703 CodeGenTypes::arrangeCall(const CGFunctionInfo &signature,
704 const CallArgList &args) {
705 assert(signature.arg_size() <= args.size());
706 if (signature.arg_size() == args.size())
709 SmallVector<FunctionProtoType::ExtParameterInfo, 16> paramInfos;
710 auto sigParamInfos = signature.getExtParameterInfos();
711 if (!sigParamInfos.empty()) {
712 paramInfos.append(sigParamInfos.begin(), sigParamInfos.end());
713 paramInfos.resize(args.size());
716 auto argTypes = getArgTypesForCall(Context, args);
718 assert(signature.getRequiredArgs().allowsOptionalArgs());
719 return arrangeLLVMFunctionInfo(signature.getReturnType(),
720 signature.isInstanceMethod(),
721 signature.isChainCall(),
723 signature.getExtInfo(),
725 signature.getRequiredArgs());
730 void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI);
734 /// Arrange the argument and result information for an abstract value
735 /// of a given function type. This is the method which all of the
736 /// above functions ultimately defer to.
737 const CGFunctionInfo &
738 CodeGenTypes::arrangeLLVMFunctionInfo(CanQualType resultType,
741 ArrayRef<CanQualType> argTypes,
742 FunctionType::ExtInfo info,
743 ArrayRef<FunctionProtoType::ExtParameterInfo> paramInfos,
744 RequiredArgs required) {
745 assert(llvm::all_of(argTypes,
746 [](CanQualType T) { return T.isCanonicalAsParam(); }));
748 // Lookup or create unique function info.
749 llvm::FoldingSetNodeID ID;
750 CGFunctionInfo::Profile(ID, instanceMethod, chainCall, info, paramInfos,
751 required, resultType, argTypes);
753 void *insertPos = nullptr;
754 CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, insertPos);
758 unsigned CC = ClangCallConvToLLVMCallConv(info.getCC());
760 // Construct the function info. We co-allocate the ArgInfos.
761 FI = CGFunctionInfo::create(CC, instanceMethod, chainCall, info,
762 paramInfos, resultType, argTypes, required);
763 FunctionInfos.InsertNode(FI, insertPos);
765 bool inserted = FunctionsBeingProcessed.insert(FI).second;
767 assert(inserted && "Recursively being processed?");
769 // Compute ABI information.
770 if (CC == llvm::CallingConv::SPIR_KERNEL) {
771 // Force target independent argument handling for the host visible
773 computeSPIRKernelABIInfo(CGM, *FI);
774 } else if (info.getCC() == CC_Swift) {
775 swiftcall::computeABIInfo(CGM, *FI);
777 getABIInfo().computeInfo(*FI);
780 // Loop over all of the computed argument and return value info. If any of
781 // them are direct or extend without a specified coerce type, specify the
783 ABIArgInfo &retInfo = FI->getReturnInfo();
784 if (retInfo.canHaveCoerceToType() && retInfo.getCoerceToType() == nullptr)
785 retInfo.setCoerceToType(ConvertType(FI->getReturnType()));
787 for (auto &I : FI->arguments())
788 if (I.info.canHaveCoerceToType() && I.info.getCoerceToType() == nullptr)
789 I.info.setCoerceToType(ConvertType(I.type));
791 bool erased = FunctionsBeingProcessed.erase(FI); (void)erased;
792 assert(erased && "Not in set?");
797 CGFunctionInfo *CGFunctionInfo::create(unsigned llvmCC,
800 const FunctionType::ExtInfo &info,
801 ArrayRef<ExtParameterInfo> paramInfos,
802 CanQualType resultType,
803 ArrayRef<CanQualType> argTypes,
804 RequiredArgs required) {
805 assert(paramInfos.empty() || paramInfos.size() == argTypes.size());
806 assert(!required.allowsOptionalArgs() ||
807 required.getNumRequiredArgs() <= argTypes.size());
810 operator new(totalSizeToAlloc<ArgInfo, ExtParameterInfo>(
811 argTypes.size() + 1, paramInfos.size()));
813 CGFunctionInfo *FI = new(buffer) CGFunctionInfo();
814 FI->CallingConvention = llvmCC;
815 FI->EffectiveCallingConvention = llvmCC;
816 FI->ASTCallingConvention = info.getCC();
817 FI->InstanceMethod = instanceMethod;
818 FI->ChainCall = chainCall;
819 FI->CmseNSCall = info.getCmseNSCall();
820 FI->NoReturn = info.getNoReturn();
821 FI->ReturnsRetained = info.getProducesResult();
822 FI->NoCallerSavedRegs = info.getNoCallerSavedRegs();
823 FI->NoCfCheck = info.getNoCfCheck();
824 FI->Required = required;
825 FI->HasRegParm = info.getHasRegParm();
826 FI->RegParm = info.getRegParm();
827 FI->ArgStruct = nullptr;
828 FI->ArgStructAlign = 0;
829 FI->NumArgs = argTypes.size();
830 FI->HasExtParameterInfos = !paramInfos.empty();
831 FI->getArgsBuffer()[0].type = resultType;
832 for (unsigned i = 0, e = argTypes.size(); i != e; ++i)
833 FI->getArgsBuffer()[i + 1].type = argTypes[i];
834 for (unsigned i = 0, e = paramInfos.size(); i != e; ++i)
835 FI->getExtParameterInfosBuffer()[i] = paramInfos[i];
842 // ABIArgInfo::Expand implementation.
844 // Specifies the way QualType passed as ABIArgInfo::Expand is expanded.
845 struct TypeExpansion {
846 enum TypeExpansionKind {
847 // Elements of constant arrays are expanded recursively.
849 // Record fields are expanded recursively (but if record is a union, only
850 // the field with the largest size is expanded).
852 // For complex types, real and imaginary parts are expanded recursively.
854 // All other types are not expandable.
858 const TypeExpansionKind Kind;
860 TypeExpansion(TypeExpansionKind K) : Kind(K) {}
861 virtual ~TypeExpansion() {}
864 struct ConstantArrayExpansion : TypeExpansion {
868 ConstantArrayExpansion(QualType EltTy, uint64_t NumElts)
869 : TypeExpansion(TEK_ConstantArray), EltTy(EltTy), NumElts(NumElts) {}
870 static bool classof(const TypeExpansion *TE) {
871 return TE->Kind == TEK_ConstantArray;
875 struct RecordExpansion : TypeExpansion {
876 SmallVector<const CXXBaseSpecifier *, 1> Bases;
878 SmallVector<const FieldDecl *, 1> Fields;
880 RecordExpansion(SmallVector<const CXXBaseSpecifier *, 1> &&Bases,
881 SmallVector<const FieldDecl *, 1> &&Fields)
882 : TypeExpansion(TEK_Record), Bases(std::move(Bases)),
883 Fields(std::move(Fields)) {}
884 static bool classof(const TypeExpansion *TE) {
885 return TE->Kind == TEK_Record;
889 struct ComplexExpansion : TypeExpansion {
892 ComplexExpansion(QualType EltTy) : TypeExpansion(TEK_Complex), EltTy(EltTy) {}
893 static bool classof(const TypeExpansion *TE) {
894 return TE->Kind == TEK_Complex;
898 struct NoExpansion : TypeExpansion {
899 NoExpansion() : TypeExpansion(TEK_None) {}
900 static bool classof(const TypeExpansion *TE) {
901 return TE->Kind == TEK_None;
906 static std::unique_ptr<TypeExpansion>
907 getTypeExpansion(QualType Ty, const ASTContext &Context) {
908 if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
909 return std::make_unique<ConstantArrayExpansion>(
910 AT->getElementType(), AT->getSize().getZExtValue());
912 if (const RecordType *RT = Ty->getAs<RecordType>()) {
913 SmallVector<const CXXBaseSpecifier *, 1> Bases;
914 SmallVector<const FieldDecl *, 1> Fields;
915 const RecordDecl *RD = RT->getDecl();
916 assert(!RD->hasFlexibleArrayMember() &&
917 "Cannot expand structure with flexible array.");
919 // Unions can be here only in degenerative cases - all the fields are same
920 // after flattening. Thus we have to use the "largest" field.
921 const FieldDecl *LargestFD = nullptr;
922 CharUnits UnionSize = CharUnits::Zero();
924 for (const auto *FD : RD->fields()) {
925 if (FD->isZeroLengthBitField(Context))
927 assert(!FD->isBitField() &&
928 "Cannot expand structure with bit-field members.");
929 CharUnits FieldSize = Context.getTypeSizeInChars(FD->getType());
930 if (UnionSize < FieldSize) {
931 UnionSize = FieldSize;
936 Fields.push_back(LargestFD);
938 if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
939 assert(!CXXRD->isDynamicClass() &&
940 "cannot expand vtable pointers in dynamic classes");
941 for (const CXXBaseSpecifier &BS : CXXRD->bases())
942 Bases.push_back(&BS);
945 for (const auto *FD : RD->fields()) {
946 if (FD->isZeroLengthBitField(Context))
948 assert(!FD->isBitField() &&
949 "Cannot expand structure with bit-field members.");
950 Fields.push_back(FD);
953 return std::make_unique<RecordExpansion>(std::move(Bases),
956 if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
957 return std::make_unique<ComplexExpansion>(CT->getElementType());
959 return std::make_unique<NoExpansion>();
962 static int getExpansionSize(QualType Ty, const ASTContext &Context) {
963 auto Exp = getTypeExpansion(Ty, Context);
964 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
965 return CAExp->NumElts * getExpansionSize(CAExp->EltTy, Context);
967 if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
969 for (auto BS : RExp->Bases)
970 Res += getExpansionSize(BS->getType(), Context);
971 for (auto FD : RExp->Fields)
972 Res += getExpansionSize(FD->getType(), Context);
975 if (isa<ComplexExpansion>(Exp.get()))
977 assert(isa<NoExpansion>(Exp.get()));
982 CodeGenTypes::getExpandedTypes(QualType Ty,
983 SmallVectorImpl<llvm::Type *>::iterator &TI) {
984 auto Exp = getTypeExpansion(Ty, Context);
985 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
986 for (int i = 0, n = CAExp->NumElts; i < n; i++) {
987 getExpandedTypes(CAExp->EltTy, TI);
989 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
990 for (auto BS : RExp->Bases)
991 getExpandedTypes(BS->getType(), TI);
992 for (auto FD : RExp->Fields)
993 getExpandedTypes(FD->getType(), TI);
994 } else if (auto CExp = dyn_cast<ComplexExpansion>(Exp.get())) {
995 llvm::Type *EltTy = ConvertType(CExp->EltTy);
999 assert(isa<NoExpansion>(Exp.get()));
1000 *TI++ = ConvertType(Ty);
1004 static void forConstantArrayExpansion(CodeGenFunction &CGF,
1005 ConstantArrayExpansion *CAE,
1007 llvm::function_ref<void(Address)> Fn) {
1008 CharUnits EltSize = CGF.getContext().getTypeSizeInChars(CAE->EltTy);
1009 CharUnits EltAlign =
1010 BaseAddr.getAlignment().alignmentOfArrayElement(EltSize);
1012 for (int i = 0, n = CAE->NumElts; i < n; i++) {
1013 llvm::Value *EltAddr =
1014 CGF.Builder.CreateConstGEP2_32(nullptr, BaseAddr.getPointer(), 0, i);
1015 Fn(Address(EltAddr, EltAlign));
1019 void CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1020 llvm::Function::arg_iterator &AI) {
1021 assert(LV.isSimple() &&
1022 "Unexpected non-simple lvalue during struct expansion.");
1024 auto Exp = getTypeExpansion(Ty, getContext());
1025 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1026 forConstantArrayExpansion(
1027 *this, CAExp, LV.getAddress(*this), [&](Address EltAddr) {
1028 LValue LV = MakeAddrLValue(EltAddr, CAExp->EltTy);
1029 ExpandTypeFromArgs(CAExp->EltTy, LV, AI);
1031 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1032 Address This = LV.getAddress(*this);
1033 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1034 // Perform a single step derived-to-base conversion.
1036 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1037 /*NullCheckValue=*/false, SourceLocation());
1038 LValue SubLV = MakeAddrLValue(Base, BS->getType());
1040 // Recurse onto bases.
1041 ExpandTypeFromArgs(BS->getType(), SubLV, AI);
1043 for (auto FD : RExp->Fields) {
1044 // FIXME: What are the right qualifiers here?
1045 LValue SubLV = EmitLValueForFieldInitialization(LV, FD);
1046 ExpandTypeFromArgs(FD->getType(), SubLV, AI);
1048 } else if (isa<ComplexExpansion>(Exp.get())) {
1049 auto realValue = &*AI++;
1050 auto imagValue = &*AI++;
1051 EmitStoreOfComplex(ComplexPairTy(realValue, imagValue), LV, /*init*/ true);
1053 // Call EmitStoreOfScalar except when the lvalue is a bitfield to emit a
1055 assert(isa<NoExpansion>(Exp.get()));
1056 if (LV.isBitField())
1057 EmitStoreThroughLValue(RValue::get(&*AI++), LV);
1059 EmitStoreOfScalar(&*AI++, LV);
1063 void CodeGenFunction::ExpandTypeToArgs(
1064 QualType Ty, CallArg Arg, llvm::FunctionType *IRFuncTy,
1065 SmallVectorImpl<llvm::Value *> &IRCallArgs, unsigned &IRCallArgPos) {
1066 auto Exp = getTypeExpansion(Ty, getContext());
1067 if (auto CAExp = dyn_cast<ConstantArrayExpansion>(Exp.get())) {
1068 Address Addr = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1069 : Arg.getKnownRValue().getAggregateAddress();
1070 forConstantArrayExpansion(
1071 *this, CAExp, Addr, [&](Address EltAddr) {
1072 CallArg EltArg = CallArg(
1073 convertTempToRValue(EltAddr, CAExp->EltTy, SourceLocation()),
1075 ExpandTypeToArgs(CAExp->EltTy, EltArg, IRFuncTy, IRCallArgs,
1078 } else if (auto RExp = dyn_cast<RecordExpansion>(Exp.get())) {
1079 Address This = Arg.hasLValue() ? Arg.getKnownLValue().getAddress(*this)
1080 : Arg.getKnownRValue().getAggregateAddress();
1081 for (const CXXBaseSpecifier *BS : RExp->Bases) {
1082 // Perform a single step derived-to-base conversion.
1084 GetAddressOfBaseClass(This, Ty->getAsCXXRecordDecl(), &BS, &BS + 1,
1085 /*NullCheckValue=*/false, SourceLocation());
1086 CallArg BaseArg = CallArg(RValue::getAggregate(Base), BS->getType());
1088 // Recurse onto bases.
1089 ExpandTypeToArgs(BS->getType(), BaseArg, IRFuncTy, IRCallArgs,
1093 LValue LV = MakeAddrLValue(This, Ty);
1094 for (auto FD : RExp->Fields) {
1096 CallArg(EmitRValueForField(LV, FD, SourceLocation()), FD->getType());
1097 ExpandTypeToArgs(FD->getType(), FldArg, IRFuncTy, IRCallArgs,
1100 } else if (isa<ComplexExpansion>(Exp.get())) {
1101 ComplexPairTy CV = Arg.getKnownRValue().getComplexVal();
1102 IRCallArgs[IRCallArgPos++] = CV.first;
1103 IRCallArgs[IRCallArgPos++] = CV.second;
1105 assert(isa<NoExpansion>(Exp.get()));
1106 auto RV = Arg.getKnownRValue();
1107 assert(RV.isScalar() &&
1108 "Unexpected non-scalar rvalue during struct expansion.");
1110 // Insert a bitcast as needed.
1111 llvm::Value *V = RV.getScalarVal();
1112 if (IRCallArgPos < IRFuncTy->getNumParams() &&
1113 V->getType() != IRFuncTy->getParamType(IRCallArgPos))
1114 V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRCallArgPos));
1116 IRCallArgs[IRCallArgPos++] = V;
1120 /// Create a temporary allocation for the purposes of coercion.
1121 static Address CreateTempAllocaForCoercion(CodeGenFunction &CGF, llvm::Type *Ty,
1123 const Twine &Name = "tmp") {
1124 // Don't use an alignment that's worse than what LLVM would prefer.
1125 auto PrefAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(Ty);
1126 CharUnits Align = std::max(MinAlign, CharUnits::fromQuantity(PrefAlign));
1128 return CGF.CreateTempAlloca(Ty, Align, Name + ".coerce");
1131 /// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
1132 /// accessing some number of bytes out of it, try to gep into the struct to get
1133 /// at its inner goodness. Dive as deep as possible without entering an element
1134 /// with an in-memory size smaller than DstSize.
1136 EnterStructPointerForCoercedAccess(Address SrcPtr,
1137 llvm::StructType *SrcSTy,
1138 uint64_t DstSize, CodeGenFunction &CGF) {
1139 // We can't dive into a zero-element struct.
1140 if (SrcSTy->getNumElements() == 0) return SrcPtr;
1142 llvm::Type *FirstElt = SrcSTy->getElementType(0);
1144 // If the first elt is at least as large as what we're looking for, or if the
1145 // first element is the same size as the whole struct, we can enter it. The
1146 // comparison must be made on the store size and not the alloca size. Using
1147 // the alloca size may overstate the size of the load.
1148 uint64_t FirstEltSize =
1149 CGF.CGM.getDataLayout().getTypeStoreSize(FirstElt);
1150 if (FirstEltSize < DstSize &&
1151 FirstEltSize < CGF.CGM.getDataLayout().getTypeStoreSize(SrcSTy))
1154 // GEP into the first element.
1155 SrcPtr = CGF.Builder.CreateStructGEP(SrcPtr, 0, "coerce.dive");
1157 // If the first element is a struct, recurse.
1158 llvm::Type *SrcTy = SrcPtr.getElementType();
1159 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
1160 return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
1165 /// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
1166 /// are either integers or pointers. This does a truncation of the value if it
1167 /// is too large or a zero extension if it is too small.
1169 /// This behaves as if the value were coerced through memory, so on big-endian
1170 /// targets the high bits are preserved in a truncation, while little-endian
1171 /// targets preserve the low bits.
1172 static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
1174 CodeGenFunction &CGF) {
1175 if (Val->getType() == Ty)
1178 if (isa<llvm::PointerType>(Val->getType())) {
1179 // If this is Pointer->Pointer avoid conversion to and from int.
1180 if (isa<llvm::PointerType>(Ty))
1181 return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
1183 // Convert the pointer to an integer so we can play with its width.
1184 Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
1187 llvm::Type *DestIntTy = Ty;
1188 if (isa<llvm::PointerType>(DestIntTy))
1189 DestIntTy = CGF.IntPtrTy;
1191 if (Val->getType() != DestIntTy) {
1192 const llvm::DataLayout &DL = CGF.CGM.getDataLayout();
1193 if (DL.isBigEndian()) {
1194 // Preserve the high bits on big-endian targets.
1195 // That is what memory coercion does.
1196 uint64_t SrcSize = DL.getTypeSizeInBits(Val->getType());
1197 uint64_t DstSize = DL.getTypeSizeInBits(DestIntTy);
1199 if (SrcSize > DstSize) {
1200 Val = CGF.Builder.CreateLShr(Val, SrcSize - DstSize, "coerce.highbits");
1201 Val = CGF.Builder.CreateTrunc(Val, DestIntTy, "coerce.val.ii");
1203 Val = CGF.Builder.CreateZExt(Val, DestIntTy, "coerce.val.ii");
1204 Val = CGF.Builder.CreateShl(Val, DstSize - SrcSize, "coerce.highbits");
1207 // Little-endian targets preserve the low bits. No shifts required.
1208 Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
1212 if (isa<llvm::PointerType>(Ty))
1213 Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
1219 /// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1220 /// a pointer to an object of type \arg Ty, known to be aligned to
1221 /// \arg SrcAlign bytes.
1223 /// This safely handles the case when the src type is smaller than the
1224 /// destination type; in this situation the values of bits which not
1225 /// present in the src are undefined.
1226 static llvm::Value *CreateCoercedLoad(Address Src, llvm::Type *Ty,
1227 CodeGenFunction &CGF) {
1228 llvm::Type *SrcTy = Src.getElementType();
1230 // If SrcTy and Ty are the same, just do a load.
1232 return CGF.Builder.CreateLoad(Src);
1234 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
1236 if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
1237 Src = EnterStructPointerForCoercedAccess(Src, SrcSTy,
1238 DstSize.getFixedSize(), CGF);
1239 SrcTy = Src.getElementType();
1242 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1244 // If the source and destination are integer or pointer types, just do an
1245 // extension or truncation to the desired type.
1246 if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
1247 (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
1248 llvm::Value *Load = CGF.Builder.CreateLoad(Src);
1249 return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
1252 // If load is legal, just bitcast the src pointer.
1253 if (!SrcSize.isScalable() && !DstSize.isScalable() &&
1254 SrcSize.getFixedSize() >= DstSize.getFixedSize()) {
1255 // Generally SrcSize is never greater than DstSize, since this means we are
1256 // losing bits. However, this can happen in cases where the structure has
1257 // additional padding, for example due to a user specified alignment.
1259 // FIXME: Assert that we aren't truncating non-padding bits when have access
1260 // to that information.
1261 Src = CGF.Builder.CreateBitCast(Src,
1262 Ty->getPointerTo(Src.getAddressSpace()));
1263 return CGF.Builder.CreateLoad(Src);
1266 // Otherwise do coercion through memory. This is stupid, but simple.
1268 CreateTempAllocaForCoercion(CGF, Ty, Src.getAlignment(), Src.getName());
1269 CGF.Builder.CreateMemCpy(
1270 Tmp.getPointer(), Tmp.getAlignment().getAsAlign(), Src.getPointer(),
1271 Src.getAlignment().getAsAlign(),
1272 llvm::ConstantInt::get(CGF.IntPtrTy, SrcSize.getKnownMinSize()));
1273 return CGF.Builder.CreateLoad(Tmp);
1276 // Function to store a first-class aggregate into memory. We prefer to
1277 // store the elements rather than the aggregate to be more friendly to
1279 // FIXME: Do we need to recurse here?
1280 void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
1281 bool DestIsVolatile) {
1282 // Prefer scalar stores to first-class aggregate stores.
1283 if (llvm::StructType *STy = dyn_cast<llvm::StructType>(Val->getType())) {
1284 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1285 Address EltPtr = Builder.CreateStructGEP(Dest, i);
1286 llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
1287 Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
1290 Builder.CreateStore(Val, Dest, DestIsVolatile);
1294 /// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1295 /// where the source and destination may have different types. The
1296 /// destination is known to be aligned to \arg DstAlign bytes.
1298 /// This safely handles the case when the src type is larger than the
1299 /// destination type; the upper bits of the src will be lost.
1300 static void CreateCoercedStore(llvm::Value *Src,
1303 CodeGenFunction &CGF) {
1304 llvm::Type *SrcTy = Src->getType();
1305 llvm::Type *DstTy = Dst.getElementType();
1306 if (SrcTy == DstTy) {
1307 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1311 llvm::TypeSize SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
1313 if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
1314 Dst = EnterStructPointerForCoercedAccess(Dst, DstSTy,
1315 SrcSize.getFixedSize(), CGF);
1316 DstTy = Dst.getElementType();
1319 llvm::PointerType *SrcPtrTy = llvm::dyn_cast<llvm::PointerType>(SrcTy);
1320 llvm::PointerType *DstPtrTy = llvm::dyn_cast<llvm::PointerType>(DstTy);
1321 if (SrcPtrTy && DstPtrTy &&
1322 SrcPtrTy->getAddressSpace() != DstPtrTy->getAddressSpace()) {
1323 Src = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Src, DstTy);
1324 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1328 // If the source and destination are integer or pointer types, just do an
1329 // extension or truncation to the desired type.
1330 if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
1331 (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
1332 Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
1333 CGF.Builder.CreateStore(Src, Dst, DstIsVolatile);
1337 llvm::TypeSize DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
1339 // If store is legal, just bitcast the src pointer.
1340 if (isa<llvm::ScalableVectorType>(SrcTy) ||
1341 isa<llvm::ScalableVectorType>(DstTy) ||
1342 SrcSize.getFixedSize() <= DstSize.getFixedSize()) {
1343 Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
1344 CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
1346 // Otherwise do coercion through memory. This is stupid, but
1349 // Generally SrcSize is never greater than DstSize, since this means we are
1350 // losing bits. However, this can happen in cases where the structure has
1351 // additional padding, for example due to a user specified alignment.
1353 // FIXME: Assert that we aren't truncating non-padding bits when have access
1354 // to that information.
1355 Address Tmp = CreateTempAllocaForCoercion(CGF, SrcTy, Dst.getAlignment());
1356 CGF.Builder.CreateStore(Src, Tmp);
1357 CGF.Builder.CreateMemCpy(
1358 Dst.getPointer(), Dst.getAlignment().getAsAlign(), Tmp.getPointer(),
1359 Tmp.getAlignment().getAsAlign(),
1360 llvm::ConstantInt::get(CGF.IntPtrTy, DstSize.getFixedSize()));
1364 static Address emitAddressAtOffset(CodeGenFunction &CGF, Address addr,
1365 const ABIArgInfo &info) {
1366 if (unsigned offset = info.getDirectOffset()) {
1367 addr = CGF.Builder.CreateElementBitCast(addr, CGF.Int8Ty);
1368 addr = CGF.Builder.CreateConstInBoundsByteGEP(addr,
1369 CharUnits::fromQuantity(offset));
1370 addr = CGF.Builder.CreateElementBitCast(addr, info.getCoerceToType());
1377 /// Encapsulates information about the way function arguments from
1378 /// CGFunctionInfo should be passed to actual LLVM IR function.
1379 class ClangToLLVMArgMapping {
1380 static const unsigned InvalidIndex = ~0U;
1381 unsigned InallocaArgNo;
1383 unsigned TotalIRArgs;
1385 /// Arguments of LLVM IR function corresponding to single Clang argument.
1387 unsigned PaddingArgIndex;
1388 // Argument is expanded to IR arguments at positions
1389 // [FirstArgIndex, FirstArgIndex + NumberOfArgs).
1390 unsigned FirstArgIndex;
1391 unsigned NumberOfArgs;
1394 : PaddingArgIndex(InvalidIndex), FirstArgIndex(InvalidIndex),
1398 SmallVector<IRArgs, 8> ArgInfo;
1401 ClangToLLVMArgMapping(const ASTContext &Context, const CGFunctionInfo &FI,
1402 bool OnlyRequiredArgs = false)
1403 : InallocaArgNo(InvalidIndex), SRetArgNo(InvalidIndex), TotalIRArgs(0),
1404 ArgInfo(OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size()) {
1405 construct(Context, FI, OnlyRequiredArgs);
1408 bool hasInallocaArg() const { return InallocaArgNo != InvalidIndex; }
1409 unsigned getInallocaArgNo() const {
1410 assert(hasInallocaArg());
1411 return InallocaArgNo;
1414 bool hasSRetArg() const { return SRetArgNo != InvalidIndex; }
1415 unsigned getSRetArgNo() const {
1416 assert(hasSRetArg());
1420 unsigned totalIRArgs() const { return TotalIRArgs; }
1422 bool hasPaddingArg(unsigned ArgNo) const {
1423 assert(ArgNo < ArgInfo.size());
1424 return ArgInfo[ArgNo].PaddingArgIndex != InvalidIndex;
1426 unsigned getPaddingArgNo(unsigned ArgNo) const {
1427 assert(hasPaddingArg(ArgNo));
1428 return ArgInfo[ArgNo].PaddingArgIndex;
1431 /// Returns index of first IR argument corresponding to ArgNo, and their
1433 std::pair<unsigned, unsigned> getIRArgs(unsigned ArgNo) const {
1434 assert(ArgNo < ArgInfo.size());
1435 return std::make_pair(ArgInfo[ArgNo].FirstArgIndex,
1436 ArgInfo[ArgNo].NumberOfArgs);
1440 void construct(const ASTContext &Context, const CGFunctionInfo &FI,
1441 bool OnlyRequiredArgs);
1444 void ClangToLLVMArgMapping::construct(const ASTContext &Context,
1445 const CGFunctionInfo &FI,
1446 bool OnlyRequiredArgs) {
1447 unsigned IRArgNo = 0;
1448 bool SwapThisWithSRet = false;
1449 const ABIArgInfo &RetAI = FI.getReturnInfo();
1451 if (RetAI.getKind() == ABIArgInfo::Indirect) {
1452 SwapThisWithSRet = RetAI.isSRetAfterThis();
1453 SRetArgNo = SwapThisWithSRet ? 1 : IRArgNo++;
1457 unsigned NumArgs = OnlyRequiredArgs ? FI.getNumRequiredArgs() : FI.arg_size();
1458 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(); ArgNo < NumArgs;
1460 assert(I != FI.arg_end());
1461 QualType ArgType = I->type;
1462 const ABIArgInfo &AI = I->info;
1463 // Collect data about IR arguments corresponding to Clang argument ArgNo.
1464 auto &IRArgs = ArgInfo[ArgNo];
1466 if (AI.getPaddingType())
1467 IRArgs.PaddingArgIndex = IRArgNo++;
1469 switch (AI.getKind()) {
1470 case ABIArgInfo::Extend:
1471 case ABIArgInfo::Direct: {
1472 // FIXME: handle sseregparm someday...
1473 llvm::StructType *STy = dyn_cast<llvm::StructType>(AI.getCoerceToType());
1474 if (AI.isDirect() && AI.getCanBeFlattened() && STy) {
1475 IRArgs.NumberOfArgs = STy->getNumElements();
1477 IRArgs.NumberOfArgs = 1;
1481 case ABIArgInfo::Indirect:
1482 case ABIArgInfo::IndirectAliased:
1483 IRArgs.NumberOfArgs = 1;
1485 case ABIArgInfo::Ignore:
1486 case ABIArgInfo::InAlloca:
1487 // ignore and inalloca doesn't have matching LLVM parameters.
1488 IRArgs.NumberOfArgs = 0;
1490 case ABIArgInfo::CoerceAndExpand:
1491 IRArgs.NumberOfArgs = AI.getCoerceAndExpandTypeSequence().size();
1493 case ABIArgInfo::Expand:
1494 IRArgs.NumberOfArgs = getExpansionSize(ArgType, Context);
1498 if (IRArgs.NumberOfArgs > 0) {
1499 IRArgs.FirstArgIndex = IRArgNo;
1500 IRArgNo += IRArgs.NumberOfArgs;
1503 // Skip over the sret parameter when it comes second. We already handled it
1505 if (IRArgNo == 1 && SwapThisWithSRet)
1508 assert(ArgNo == ArgInfo.size());
1510 if (FI.usesInAlloca())
1511 InallocaArgNo = IRArgNo++;
1513 TotalIRArgs = IRArgNo;
1519 bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
1520 const auto &RI = FI.getReturnInfo();
1521 return RI.isIndirect() || (RI.isInAlloca() && RI.getInAllocaSRet());
1524 bool CodeGenModule::ReturnSlotInterferesWithArgs(const CGFunctionInfo &FI) {
1525 return ReturnTypeUsesSRet(FI) &&
1526 getTargetCodeGenInfo().doesReturnSlotInterfereWithArgs();
1529 bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
1530 if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
1531 switch (BT->getKind()) {
1534 case BuiltinType::Float:
1535 return getTarget().useObjCFPRetForRealType(TargetInfo::Float);
1536 case BuiltinType::Double:
1537 return getTarget().useObjCFPRetForRealType(TargetInfo::Double);
1538 case BuiltinType::LongDouble:
1539 return getTarget().useObjCFPRetForRealType(TargetInfo::LongDouble);
1546 bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
1547 if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
1548 if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
1549 if (BT->getKind() == BuiltinType::LongDouble)
1550 return getTarget().useObjCFP2RetForComplexLongDouble();
1557 llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
1558 const CGFunctionInfo &FI = arrangeGlobalDeclaration(GD);
1559 return GetFunctionType(FI);
1562 llvm::FunctionType *
1563 CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
1565 bool Inserted = FunctionsBeingProcessed.insert(&FI).second;
1567 assert(Inserted && "Recursively being processed?");
1569 llvm::Type *resultType = nullptr;
1570 const ABIArgInfo &retAI = FI.getReturnInfo();
1571 switch (retAI.getKind()) {
1572 case ABIArgInfo::Expand:
1573 case ABIArgInfo::IndirectAliased:
1574 llvm_unreachable("Invalid ABI kind for return argument");
1576 case ABIArgInfo::Extend:
1577 case ABIArgInfo::Direct:
1578 resultType = retAI.getCoerceToType();
1581 case ABIArgInfo::InAlloca:
1582 if (retAI.getInAllocaSRet()) {
1583 // sret things on win32 aren't void, they return the sret pointer.
1584 QualType ret = FI.getReturnType();
1585 llvm::Type *ty = ConvertType(ret);
1586 unsigned addressSpace = Context.getTargetAddressSpace(ret);
1587 resultType = llvm::PointerType::get(ty, addressSpace);
1589 resultType = llvm::Type::getVoidTy(getLLVMContext());
1593 case ABIArgInfo::Indirect:
1594 case ABIArgInfo::Ignore:
1595 resultType = llvm::Type::getVoidTy(getLLVMContext());
1598 case ABIArgInfo::CoerceAndExpand:
1599 resultType = retAI.getUnpaddedCoerceAndExpandType();
1603 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI, true);
1604 SmallVector<llvm::Type*, 8> ArgTypes(IRFunctionArgs.totalIRArgs());
1606 // Add type for sret argument.
1607 if (IRFunctionArgs.hasSRetArg()) {
1608 QualType Ret = FI.getReturnType();
1609 llvm::Type *Ty = ConvertType(Ret);
1610 unsigned AddressSpace = Context.getTargetAddressSpace(Ret);
1611 ArgTypes[IRFunctionArgs.getSRetArgNo()] =
1612 llvm::PointerType::get(Ty, AddressSpace);
1615 // Add type for inalloca argument.
1616 if (IRFunctionArgs.hasInallocaArg()) {
1617 auto ArgStruct = FI.getArgStruct();
1619 ArgTypes[IRFunctionArgs.getInallocaArgNo()] = ArgStruct->getPointerTo();
1622 // Add in all of the required arguments.
1624 CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1625 ie = it + FI.getNumRequiredArgs();
1626 for (; it != ie; ++it, ++ArgNo) {
1627 const ABIArgInfo &ArgInfo = it->info;
1629 // Insert a padding type to ensure proper alignment.
1630 if (IRFunctionArgs.hasPaddingArg(ArgNo))
1631 ArgTypes[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
1632 ArgInfo.getPaddingType();
1634 unsigned FirstIRArg, NumIRArgs;
1635 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
1637 switch (ArgInfo.getKind()) {
1638 case ABIArgInfo::Ignore:
1639 case ABIArgInfo::InAlloca:
1640 assert(NumIRArgs == 0);
1643 case ABIArgInfo::Indirect: {
1644 assert(NumIRArgs == 1);
1645 // indirect arguments are always on the stack, which is alloca addr space.
1646 llvm::Type *LTy = ConvertTypeForMem(it->type);
1647 ArgTypes[FirstIRArg] = LTy->getPointerTo(
1648 CGM.getDataLayout().getAllocaAddrSpace());
1651 case ABIArgInfo::IndirectAliased: {
1652 assert(NumIRArgs == 1);
1653 llvm::Type *LTy = ConvertTypeForMem(it->type);
1654 ArgTypes[FirstIRArg] = LTy->getPointerTo(ArgInfo.getIndirectAddrSpace());
1657 case ABIArgInfo::Extend:
1658 case ABIArgInfo::Direct: {
1659 // Fast-isel and the optimizer generally like scalar values better than
1660 // FCAs, so we flatten them if this is safe to do for this argument.
1661 llvm::Type *argType = ArgInfo.getCoerceToType();
1662 llvm::StructType *st = dyn_cast<llvm::StructType>(argType);
1663 if (st && ArgInfo.isDirect() && ArgInfo.getCanBeFlattened()) {
1664 assert(NumIRArgs == st->getNumElements());
1665 for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
1666 ArgTypes[FirstIRArg + i] = st->getElementType(i);
1668 assert(NumIRArgs == 1);
1669 ArgTypes[FirstIRArg] = argType;
1674 case ABIArgInfo::CoerceAndExpand: {
1675 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1676 for (auto EltTy : ArgInfo.getCoerceAndExpandTypeSequence()) {
1677 *ArgTypesIter++ = EltTy;
1679 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1683 case ABIArgInfo::Expand:
1684 auto ArgTypesIter = ArgTypes.begin() + FirstIRArg;
1685 getExpandedTypes(it->type, ArgTypesIter);
1686 assert(ArgTypesIter == ArgTypes.begin() + FirstIRArg + NumIRArgs);
1691 bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
1692 assert(Erased && "Not in set?");
1694 return llvm::FunctionType::get(resultType, ArgTypes, FI.isVariadic());
1697 llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
1698 const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
1699 const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
1701 if (!isFuncTypeConvertible(FPT))
1702 return llvm::StructType::get(getLLVMContext());
1704 return GetFunctionType(GD);
1707 static void AddAttributesFromFunctionProtoType(ASTContext &Ctx,
1708 llvm::AttrBuilder &FuncAttrs,
1709 const FunctionProtoType *FPT) {
1713 if (!isUnresolvedExceptionSpec(FPT->getExceptionSpecType()) &&
1715 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1718 void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
1720 bool AttrOnCallSite,
1721 llvm::AttrBuilder &FuncAttrs) {
1722 // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
1724 if (CodeGenOpts.OptimizeSize)
1725 FuncAttrs.addAttribute(llvm::Attribute::OptimizeForSize);
1726 if (CodeGenOpts.OptimizeSize == 2)
1727 FuncAttrs.addAttribute(llvm::Attribute::MinSize);
1730 if (CodeGenOpts.DisableRedZone)
1731 FuncAttrs.addAttribute(llvm::Attribute::NoRedZone);
1732 if (CodeGenOpts.IndirectTlsSegRefs)
1733 FuncAttrs.addAttribute("indirect-tls-seg-refs");
1734 if (CodeGenOpts.NoImplicitFloat)
1735 FuncAttrs.addAttribute(llvm::Attribute::NoImplicitFloat);
1737 if (AttrOnCallSite) {
1738 // Attributes that should go on the call site only.
1739 if (!CodeGenOpts.SimplifyLibCalls ||
1740 CodeGenOpts.isNoBuiltinFunc(Name.data()))
1741 FuncAttrs.addAttribute(llvm::Attribute::NoBuiltin);
1742 if (!CodeGenOpts.TrapFuncName.empty())
1743 FuncAttrs.addAttribute("trap-func-name", CodeGenOpts.TrapFuncName);
1746 switch (CodeGenOpts.getFramePointer()) {
1747 case CodeGenOptions::FramePointerKind::None:
1750 case CodeGenOptions::FramePointerKind::NonLeaf:
1751 FpKind = "non-leaf";
1753 case CodeGenOptions::FramePointerKind::All:
1757 FuncAttrs.addAttribute("frame-pointer", FpKind);
1759 FuncAttrs.addAttribute("less-precise-fpmad",
1760 llvm::toStringRef(CodeGenOpts.LessPreciseFPMAD));
1762 if (CodeGenOpts.NullPointerIsValid)
1763 FuncAttrs.addAttribute(llvm::Attribute::NullPointerIsValid);
1765 if (CodeGenOpts.FPDenormalMode != llvm::DenormalMode::getIEEE())
1766 FuncAttrs.addAttribute("denormal-fp-math",
1767 CodeGenOpts.FPDenormalMode.str());
1768 if (CodeGenOpts.FP32DenormalMode != CodeGenOpts.FPDenormalMode) {
1769 FuncAttrs.addAttribute(
1770 "denormal-fp-math-f32",
1771 CodeGenOpts.FP32DenormalMode.str());
1774 FuncAttrs.addAttribute("no-trapping-math",
1775 llvm::toStringRef(LangOpts.getFPExceptionMode() ==
1776 LangOptions::FPE_Ignore));
1778 // Strict (compliant) code is the default, so only add this attribute to
1779 // indicate that we are trying to workaround a problem case.
1780 if (!CodeGenOpts.StrictFloatCastOverflow)
1781 FuncAttrs.addAttribute("strict-float-cast-overflow", "false");
1783 // TODO: Are these all needed?
1784 // unsafe/inf/nan/nsz are handled by instruction-level FastMathFlags.
1785 FuncAttrs.addAttribute("no-infs-fp-math",
1786 llvm::toStringRef(LangOpts.NoHonorInfs));
1787 FuncAttrs.addAttribute("no-nans-fp-math",
1788 llvm::toStringRef(LangOpts.NoHonorNaNs));
1789 FuncAttrs.addAttribute("unsafe-fp-math",
1790 llvm::toStringRef(LangOpts.UnsafeFPMath));
1791 FuncAttrs.addAttribute("use-soft-float",
1792 llvm::toStringRef(CodeGenOpts.SoftFloat));
1793 FuncAttrs.addAttribute("stack-protector-buffer-size",
1794 llvm::utostr(CodeGenOpts.SSPBufferSize));
1795 FuncAttrs.addAttribute("no-signed-zeros-fp-math",
1796 llvm::toStringRef(LangOpts.NoSignedZero));
1798 // TODO: Reciprocal estimate codegen options should apply to instructions?
1799 const std::vector<std::string> &Recips = CodeGenOpts.Reciprocals;
1800 if (!Recips.empty())
1801 FuncAttrs.addAttribute("reciprocal-estimates",
1802 llvm::join(Recips, ","));
1804 if (!CodeGenOpts.PreferVectorWidth.empty() &&
1805 CodeGenOpts.PreferVectorWidth != "none")
1806 FuncAttrs.addAttribute("prefer-vector-width",
1807 CodeGenOpts.PreferVectorWidth);
1809 if (CodeGenOpts.StackRealignment)
1810 FuncAttrs.addAttribute("stackrealign");
1811 if (CodeGenOpts.Backchain)
1812 FuncAttrs.addAttribute("backchain");
1813 if (CodeGenOpts.EnableSegmentedStacks)
1814 FuncAttrs.addAttribute("split-stack");
1816 if (CodeGenOpts.SpeculativeLoadHardening)
1817 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
1820 if (getLangOpts().assumeFunctionsAreConvergent()) {
1821 // Conservatively, mark all functions and calls in CUDA and OpenCL as
1822 // convergent (meaning, they may call an intrinsically convergent op, such
1823 // as __syncthreads() / barrier(), and so can't have certain optimizations
1824 // applied around them). LLVM will remove this attribute where it safely
1826 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1829 if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
1830 // Exceptions aren't supported in CUDA device code.
1831 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1834 for (StringRef Attr : CodeGenOpts.DefaultFunctionAttrs) {
1835 StringRef Var, Value;
1836 std::tie(Var, Value) = Attr.split('=');
1837 FuncAttrs.addAttribute(Var, Value);
1841 void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
1842 llvm::AttrBuilder FuncAttrs;
1843 getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
1844 /* AttrOnCallSite = */ false, FuncAttrs);
1845 // TODO: call GetCPUAndFeaturesAttributes?
1846 F.addAttributes(llvm::AttributeList::FunctionIndex, FuncAttrs);
1849 void CodeGenModule::addDefaultFunctionDefinitionAttributes(
1850 llvm::AttrBuilder &attrs) {
1851 getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
1852 /*for call*/ false, attrs);
1853 GetCPUAndFeaturesAttributes(GlobalDecl(), attrs);
1856 static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
1857 const LangOptions &LangOpts,
1858 const NoBuiltinAttr *NBA = nullptr) {
1859 auto AddNoBuiltinAttr = [&FuncAttrs](StringRef BuiltinName) {
1860 SmallString<32> AttributeName;
1861 AttributeName += "no-builtin-";
1862 AttributeName += BuiltinName;
1863 FuncAttrs.addAttribute(AttributeName);
1866 // First, handle the language options passed through -fno-builtin.
1867 if (LangOpts.NoBuiltin) {
1868 // -fno-builtin disables them all.
1869 FuncAttrs.addAttribute("no-builtins");
1873 // Then, add attributes for builtins specified through -fno-builtin-<name>.
1874 llvm::for_each(LangOpts.NoBuiltinFuncs, AddNoBuiltinAttr);
1876 // Now, let's check the __attribute__((no_builtin("...")) attribute added to
1881 // If there is a wildcard in the builtin names specified through the
1882 // attribute, disable them all.
1883 if (llvm::is_contained(NBA->builtinNames(), "*")) {
1884 FuncAttrs.addAttribute("no-builtins");
1888 // And last, add the rest of the builtin names.
1889 llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
1892 /// Construct the IR attribute list of a function or call.
1894 /// When adding an attribute, please consider where it should be handled:
1896 /// - getDefaultFunctionAttributes is for attributes that are essentially
1897 /// part of the global target configuration (but perhaps can be
1898 /// overridden on a per-function basis). Adding attributes there
1899 /// will cause them to also be set in frontends that build on Clang's
1900 /// target-configuration logic, as well as for code defined in library
1901 /// modules such as CUDA's libdevice.
1903 /// - ConstructAttributeList builds on top of getDefaultFunctionAttributes
1904 /// and adds declaration-specific, convention-specific, and
1905 /// frontend-specific logic. The last is of particular importance:
1906 /// attributes that restrict how the frontend generates code must be
1907 /// added here rather than getDefaultFunctionAttributes.
1909 void CodeGenModule::ConstructAttributeList(
1910 StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
1911 llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
1912 llvm::AttrBuilder FuncAttrs;
1913 llvm::AttrBuilder RetAttrs;
1915 // Collect function IR attributes from the CC lowering.
1916 // We'll collect the paramete and result attributes later.
1917 CallingConv = FI.getEffectiveCallingConvention();
1918 if (FI.isNoReturn())
1919 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1920 if (FI.isCmseNSCall())
1921 FuncAttrs.addAttribute("cmse_nonsecure_call");
1923 // Collect function IR attributes from the callee prototype if we have one.
1924 AddAttributesFromFunctionProtoType(getContext(), FuncAttrs,
1925 CalleeInfo.getCalleeFunctionProtoType());
1927 const Decl *TargetDecl = CalleeInfo.getCalleeDecl().getDecl();
1929 bool HasOptnone = false;
1930 // The NoBuiltinAttr attached to the target FunctionDecl.
1931 const NoBuiltinAttr *NBA = nullptr;
1933 // Collect function IR attributes based on declaration-specific
1935 // FIXME: handle sseregparm someday...
1937 if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
1938 FuncAttrs.addAttribute(llvm::Attribute::ReturnsTwice);
1939 if (TargetDecl->hasAttr<NoThrowAttr>())
1940 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1941 if (TargetDecl->hasAttr<NoReturnAttr>())
1942 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1943 if (TargetDecl->hasAttr<ColdAttr>())
1944 FuncAttrs.addAttribute(llvm::Attribute::Cold);
1945 if (TargetDecl->hasAttr<NoDuplicateAttr>())
1946 FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
1947 if (TargetDecl->hasAttr<ConvergentAttr>())
1948 FuncAttrs.addAttribute(llvm::Attribute::Convergent);
1950 if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
1951 AddAttributesFromFunctionProtoType(
1952 getContext(), FuncAttrs, Fn->getType()->getAs<FunctionProtoType>());
1953 if (AttrOnCallSite && Fn->isReplaceableGlobalAllocationFunction()) {
1954 // A sane operator new returns a non-aliasing pointer.
1955 auto Kind = Fn->getDeclName().getCXXOverloadedOperator();
1956 if (getCodeGenOpts().AssumeSaneOperatorNew &&
1957 (Kind == OO_New || Kind == OO_Array_New))
1958 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1960 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Fn);
1961 const bool IsVirtualCall = MD && MD->isVirtual();
1962 // Don't use [[noreturn]], _Noreturn or [[no_builtin]] for a call to a
1963 // virtual function. These attributes are not inherited by overloads.
1964 if (!(AttrOnCallSite && IsVirtualCall)) {
1965 if (Fn->isNoReturn())
1966 FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
1967 NBA = Fn->getAttr<NoBuiltinAttr>();
1971 // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
1972 if (TargetDecl->hasAttr<ConstAttr>()) {
1973 FuncAttrs.addAttribute(llvm::Attribute::ReadNone);
1974 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1975 } else if (TargetDecl->hasAttr<PureAttr>()) {
1976 FuncAttrs.addAttribute(llvm::Attribute::ReadOnly);
1977 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1978 } else if (TargetDecl->hasAttr<NoAliasAttr>()) {
1979 FuncAttrs.addAttribute(llvm::Attribute::ArgMemOnly);
1980 FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
1982 if (TargetDecl->hasAttr<RestrictAttr>())
1983 RetAttrs.addAttribute(llvm::Attribute::NoAlias);
1984 if (TargetDecl->hasAttr<ReturnsNonNullAttr>() &&
1985 !CodeGenOpts.NullPointerIsValid)
1986 RetAttrs.addAttribute(llvm::Attribute::NonNull);
1987 if (TargetDecl->hasAttr<AnyX86NoCallerSavedRegistersAttr>())
1988 FuncAttrs.addAttribute("no_caller_saved_registers");
1989 if (TargetDecl->hasAttr<AnyX86NoCfCheckAttr>())
1990 FuncAttrs.addAttribute(llvm::Attribute::NoCfCheck);
1992 HasOptnone = TargetDecl->hasAttr<OptimizeNoneAttr>();
1993 if (auto *AllocSize = TargetDecl->getAttr<AllocSizeAttr>()) {
1994 Optional<unsigned> NumElemsParam;
1995 if (AllocSize->getNumElemsParam().isValid())
1996 NumElemsParam = AllocSize->getNumElemsParam().getLLVMIndex();
1997 FuncAttrs.addAllocSizeAttr(AllocSize->getElemSizeParam().getLLVMIndex(),
2001 if (TargetDecl->hasAttr<OpenCLKernelAttr>()) {
2002 if (getLangOpts().OpenCLVersion <= 120) {
2003 // OpenCL v1.2 Work groups are always uniform
2004 FuncAttrs.addAttribute("uniform-work-group-size", "true");
2006 // OpenCL v2.0 Work groups may be whether uniform or not.
2007 // '-cl-uniform-work-group-size' compile option gets a hint
2008 // to the compiler that the global work-size be a multiple of
2009 // the work-group size specified to clEnqueueNDRangeKernel
2010 // (i.e. work groups are uniform).
2011 FuncAttrs.addAttribute("uniform-work-group-size",
2012 llvm::toStringRef(CodeGenOpts.UniformWGSize));
2017 // Attach "no-builtins" attributes to:
2018 // * call sites: both `nobuiltin` and "no-builtins" or "no-builtin-<name>".
2019 // * definitions: "no-builtins" or "no-builtin-<name>" only.
2020 // The attributes can come from:
2021 // * LangOpts: -ffreestanding, -fno-builtin, -fno-builtin-<name>
2022 // * FunctionDecl attributes: __attribute__((no_builtin(...)))
2023 addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
2025 // Collect function IR attributes based on global settiings.
2026 getDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
2028 // Override some default IR attributes based on declaration-specific
2031 if (TargetDecl->hasAttr<NoSpeculativeLoadHardeningAttr>())
2032 FuncAttrs.removeAttribute(llvm::Attribute::SpeculativeLoadHardening);
2033 if (TargetDecl->hasAttr<SpeculativeLoadHardeningAttr>())
2034 FuncAttrs.addAttribute(llvm::Attribute::SpeculativeLoadHardening);
2035 if (TargetDecl->hasAttr<NoSplitStackAttr>())
2036 FuncAttrs.removeAttribute("split-stack");
2038 // Add NonLazyBind attribute to function declarations when -fno-plt
2040 // FIXME: what if we just haven't processed the function definition
2041 // yet, or if it's an external definition like C99 inline?
2042 if (CodeGenOpts.NoPLT) {
2043 if (auto *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
2044 if (!Fn->isDefined() && !AttrOnCallSite) {
2045 FuncAttrs.addAttribute(llvm::Attribute::NonLazyBind);
2051 // Collect non-call-site function IR attributes from declaration-specific
2053 if (!AttrOnCallSite) {
2054 if (TargetDecl && TargetDecl->hasAttr<CmseNSEntryAttr>())
2055 FuncAttrs.addAttribute("cmse_nonsecure_entry");
2057 // Whether tail calls are enabled.
2058 auto shouldDisableTailCalls = [&] {
2059 // Should this be honored in getDefaultFunctionAttributes?
2060 if (CodeGenOpts.DisableTailCalls)
2066 if (TargetDecl->hasAttr<DisableTailCallsAttr>() ||
2067 TargetDecl->hasAttr<AnyX86InterruptAttr>())
2070 if (CodeGenOpts.NoEscapingBlockTailCalls) {
2071 if (const auto *BD = dyn_cast<BlockDecl>(TargetDecl))
2072 if (!BD->doesNotEscape())
2078 FuncAttrs.addAttribute("disable-tail-calls",
2079 llvm::toStringRef(shouldDisableTailCalls()));
2081 // CPU/feature overrides. addDefaultFunctionDefinitionAttributes
2082 // handles these separately to set them based on the global defaults.
2083 GetCPUAndFeaturesAttributes(CalleeInfo.getCalleeDecl(), FuncAttrs);
2086 // Collect attributes from arguments and return values.
2087 ClangToLLVMArgMapping IRFunctionArgs(getContext(), FI);
2089 QualType RetTy = FI.getReturnType();
2090 const ABIArgInfo &RetAI = FI.getReturnInfo();
2091 switch (RetAI.getKind()) {
2092 case ABIArgInfo::Extend:
2093 if (RetAI.isSignExt())
2094 RetAttrs.addAttribute(llvm::Attribute::SExt);
2096 RetAttrs.addAttribute(llvm::Attribute::ZExt);
2098 case ABIArgInfo::Direct:
2099 if (RetAI.getInReg())
2100 RetAttrs.addAttribute(llvm::Attribute::InReg);
2102 case ABIArgInfo::Ignore:
2105 case ABIArgInfo::InAlloca:
2106 case ABIArgInfo::Indirect: {
2107 // inalloca and sret disable readnone and readonly
2108 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2109 .removeAttribute(llvm::Attribute::ReadNone);
2113 case ABIArgInfo::CoerceAndExpand:
2116 case ABIArgInfo::Expand:
2117 case ABIArgInfo::IndirectAliased:
2118 llvm_unreachable("Invalid ABI kind for return argument");
2121 if (const auto *RefTy = RetTy->getAs<ReferenceType>()) {
2122 QualType PTy = RefTy->getPointeeType();
2123 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2124 RetAttrs.addDereferenceableAttr(
2125 getMinimumObjectSize(PTy).getQuantity());
2126 if (getContext().getTargetAddressSpace(PTy) == 0 &&
2127 !CodeGenOpts.NullPointerIsValid)
2128 RetAttrs.addAttribute(llvm::Attribute::NonNull);
2129 if (PTy->isObjectType()) {
2130 llvm::Align Alignment =
2131 getNaturalPointeeTypeAlignment(RetTy).getAsAlign();
2132 RetAttrs.addAlignmentAttr(Alignment);
2136 bool hasUsedSRet = false;
2137 SmallVector<llvm::AttributeSet, 4> ArgAttrs(IRFunctionArgs.totalIRArgs());
2139 // Attach attributes to sret.
2140 if (IRFunctionArgs.hasSRetArg()) {
2141 llvm::AttrBuilder SRETAttrs;
2142 SRETAttrs.addStructRetAttr(getTypes().ConvertTypeForMem(RetTy));
2144 if (RetAI.getInReg())
2145 SRETAttrs.addAttribute(llvm::Attribute::InReg);
2146 SRETAttrs.addAlignmentAttr(RetAI.getIndirectAlign().getQuantity());
2147 ArgAttrs[IRFunctionArgs.getSRetArgNo()] =
2148 llvm::AttributeSet::get(getLLVMContext(), SRETAttrs);
2151 // Attach attributes to inalloca argument.
2152 if (IRFunctionArgs.hasInallocaArg()) {
2153 llvm::AttrBuilder Attrs;
2154 Attrs.addAttribute(llvm::Attribute::InAlloca);
2155 ArgAttrs[IRFunctionArgs.getInallocaArgNo()] =
2156 llvm::AttributeSet::get(getLLVMContext(), Attrs);
2159 // Apply `nonnull` and `dereferencable(N)` to the `this` argument.
2160 if (FI.isInstanceMethod() && !IRFunctionArgs.hasInallocaArg() &&
2161 !FI.arg_begin()->type->isVoidPointerType()) {
2162 auto IRArgs = IRFunctionArgs.getIRArgs(0);
2164 assert(IRArgs.second == 1 && "Expected only a single `this` pointer.");
2166 llvm::AttrBuilder Attrs;
2168 if (!CodeGenOpts.NullPointerIsValid &&
2169 getContext().getTargetAddressSpace(FI.arg_begin()->type) == 0) {
2170 Attrs.addAttribute(llvm::Attribute::NonNull);
2171 Attrs.addDereferenceableAttr(
2172 getMinimumObjectSize(
2173 FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2176 // FIXME dereferenceable should be correct here, regardless of
2177 // NullPointerIsValid. However, dereferenceable currently does not always
2178 // respect NullPointerIsValid and may imply nonnull and break the program.
2179 // See https://reviews.llvm.org/D66618 for discussions.
2180 Attrs.addDereferenceableOrNullAttr(
2181 getMinimumObjectSize(
2182 FI.arg_begin()->type.castAs<PointerType>()->getPointeeType())
2186 ArgAttrs[IRArgs.first] = llvm::AttributeSet::get(getLLVMContext(), Attrs);
2190 for (CGFunctionInfo::const_arg_iterator I = FI.arg_begin(),
2192 I != E; ++I, ++ArgNo) {
2193 QualType ParamType = I->type;
2194 const ABIArgInfo &AI = I->info;
2195 llvm::AttrBuilder Attrs;
2197 // Add attribute for padding argument, if necessary.
2198 if (IRFunctionArgs.hasPaddingArg(ArgNo)) {
2199 if (AI.getPaddingInReg()) {
2200 ArgAttrs[IRFunctionArgs.getPaddingArgNo(ArgNo)] =
2201 llvm::AttributeSet::get(
2203 llvm::AttrBuilder().addAttribute(llvm::Attribute::InReg));
2207 // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
2208 // have the corresponding parameter variable. It doesn't make
2209 // sense to do it here because parameters are so messed up.
2210 switch (AI.getKind()) {
2211 case ABIArgInfo::Extend:
2213 Attrs.addAttribute(llvm::Attribute::SExt);
2215 Attrs.addAttribute(llvm::Attribute::ZExt);
2217 case ABIArgInfo::Direct:
2218 if (ArgNo == 0 && FI.isChainCall())
2219 Attrs.addAttribute(llvm::Attribute::Nest);
2220 else if (AI.getInReg())
2221 Attrs.addAttribute(llvm::Attribute::InReg);
2224 case ABIArgInfo::Indirect: {
2226 Attrs.addAttribute(llvm::Attribute::InReg);
2228 if (AI.getIndirectByVal())
2229 Attrs.addByValAttr(getTypes().ConvertTypeForMem(ParamType));
2231 auto *Decl = ParamType->getAsRecordDecl();
2232 if (CodeGenOpts.PassByValueIsNoAlias && Decl &&
2233 Decl->getArgPassingRestrictions() == RecordDecl::APK_CanPassInRegs)
2234 // When calling the function, the pointer passed in will be the only
2235 // reference to the underlying object. Mark it accordingly.
2236 Attrs.addAttribute(llvm::Attribute::NoAlias);
2238 // TODO: We could add the byref attribute if not byval, but it would
2239 // require updating many testcases.
2241 CharUnits Align = AI.getIndirectAlign();
2243 // In a byval argument, it is important that the required
2244 // alignment of the type is honored, as LLVM might be creating a
2245 // *new* stack object, and needs to know what alignment to give
2246 // it. (Sometimes it can deduce a sensible alignment on its own,
2247 // but not if clang decides it must emit a packed struct, or the
2248 // user specifies increased alignment requirements.)
2250 // This is different from indirect *not* byval, where the object
2251 // exists already, and the align attribute is purely
2253 assert(!Align.isZero());
2255 // For now, only add this when we have a byval argument.
2256 // TODO: be less lazy about updating test cases.
2257 if (AI.getIndirectByVal())
2258 Attrs.addAlignmentAttr(Align.getQuantity());
2260 // byval disables readnone and readonly.
2261 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2262 .removeAttribute(llvm::Attribute::ReadNone);
2266 case ABIArgInfo::IndirectAliased: {
2267 CharUnits Align = AI.getIndirectAlign();
2268 Attrs.addByRefAttr(getTypes().ConvertTypeForMem(ParamType));
2269 Attrs.addAlignmentAttr(Align.getQuantity());
2272 case ABIArgInfo::Ignore:
2273 case ABIArgInfo::Expand:
2274 case ABIArgInfo::CoerceAndExpand:
2277 case ABIArgInfo::InAlloca:
2278 // inalloca disables readnone and readonly.
2279 FuncAttrs.removeAttribute(llvm::Attribute::ReadOnly)
2280 .removeAttribute(llvm::Attribute::ReadNone);
2284 if (const auto *RefTy = ParamType->getAs<ReferenceType>()) {
2285 QualType PTy = RefTy->getPointeeType();
2286 if (!PTy->isIncompleteType() && PTy->isConstantSizeType())
2287 Attrs.addDereferenceableAttr(
2288 getMinimumObjectSize(PTy).getQuantity());
2289 if (getContext().getTargetAddressSpace(PTy) == 0 &&
2290 !CodeGenOpts.NullPointerIsValid)
2291 Attrs.addAttribute(llvm::Attribute::NonNull);
2292 if (PTy->isObjectType()) {
2293 llvm::Align Alignment =
2294 getNaturalPointeeTypeAlignment(ParamType).getAsAlign();
2295 Attrs.addAlignmentAttr(Alignment);
2299 switch (FI.getExtParameterInfo(ArgNo).getABI()) {
2300 case ParameterABI::Ordinary:
2303 case ParameterABI::SwiftIndirectResult: {
2304 // Add 'sret' if we haven't already used it for something, but
2305 // only if the result is void.
2306 if (!hasUsedSRet && RetTy->isVoidType()) {
2307 Attrs.addStructRetAttr(getTypes().ConvertTypeForMem(ParamType));
2311 // Add 'noalias' in either case.
2312 Attrs.addAttribute(llvm::Attribute::NoAlias);
2314 // Add 'dereferenceable' and 'alignment'.
2315 auto PTy = ParamType->getPointeeType();
2316 if (!PTy->isIncompleteType() && PTy->isConstantSizeType()) {
2317 auto info = getContext().getTypeInfoInChars(PTy);
2318 Attrs.addDereferenceableAttr(info.Width.getQuantity());
2319 Attrs.addAlignmentAttr(info.Align.getAsAlign());
2324 case ParameterABI::SwiftErrorResult:
2325 Attrs.addAttribute(llvm::Attribute::SwiftError);
2328 case ParameterABI::SwiftContext:
2329 Attrs.addAttribute(llvm::Attribute::SwiftSelf);
2333 if (FI.getExtParameterInfo(ArgNo).isNoEscape())
2334 Attrs.addAttribute(llvm::Attribute::NoCapture);
2336 if (Attrs.hasAttributes()) {
2337 unsigned FirstIRArg, NumIRArgs;
2338 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2339 for (unsigned i = 0; i < NumIRArgs; i++)
2340 ArgAttrs[FirstIRArg + i] =
2341 llvm::AttributeSet::get(getLLVMContext(), Attrs);
2344 assert(ArgNo == FI.arg_size());
2346 AttrList = llvm::AttributeList::get(
2347 getLLVMContext(), llvm::AttributeSet::get(getLLVMContext(), FuncAttrs),
2348 llvm::AttributeSet::get(getLLVMContext(), RetAttrs), ArgAttrs);
2351 /// An argument came in as a promoted argument; demote it back to its
2353 static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
2355 llvm::Value *value) {
2356 llvm::Type *varType = CGF.ConvertType(var->getType());
2358 // This can happen with promotions that actually don't change the
2359 // underlying type, like the enum promotions.
2360 if (value->getType() == varType) return value;
2362 assert((varType->isIntegerTy() || varType->isFloatingPointTy())
2363 && "unexpected promotion type");
2365 if (isa<llvm::IntegerType>(varType))
2366 return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
2368 return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
2371 /// Returns the attribute (either parameter attribute, or function
2372 /// attribute), which declares argument ArgNo to be non-null.
2373 static const NonNullAttr *getNonNullAttr(const Decl *FD, const ParmVarDecl *PVD,
2374 QualType ArgType, unsigned ArgNo) {
2375 // FIXME: __attribute__((nonnull)) can also be applied to:
2376 // - references to pointers, where the pointee is known to be
2377 // nonnull (apparently a Clang extension)
2378 // - transparent unions containing pointers
2379 // In the former case, LLVM IR cannot represent the constraint. In
2380 // the latter case, we have no guarantee that the transparent union
2381 // is in fact passed as a pointer.
2382 if (!ArgType->isAnyPointerType() && !ArgType->isBlockPointerType())
2384 // First, check attribute on parameter itself.
2386 if (auto ParmNNAttr = PVD->getAttr<NonNullAttr>())
2389 // Check function attributes.
2392 for (const auto *NNAttr : FD->specific_attrs<NonNullAttr>()) {
2393 if (NNAttr->isNonNull(ArgNo))
2400 struct CopyBackSwiftError final : EHScopeStack::Cleanup {
2403 CopyBackSwiftError(Address temp, Address arg) : Temp(temp), Arg(arg) {}
2404 void Emit(CodeGenFunction &CGF, Flags flags) override {
2405 llvm::Value *errorValue = CGF.Builder.CreateLoad(Temp);
2406 CGF.Builder.CreateStore(errorValue, Arg);
2411 void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
2413 const FunctionArgList &Args) {
2414 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>())
2415 // Naked functions don't have prologues.
2418 // If this is an implicit-return-zero function, go ahead and
2419 // initialize the return value. TODO: it might be nice to have
2420 // a more general mechanism for this that didn't require synthesized
2421 // return statements.
2422 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl)) {
2423 if (FD->hasImplicitReturnZero()) {
2424 QualType RetTy = FD->getReturnType().getUnqualifiedType();
2425 llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
2426 llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
2427 Builder.CreateStore(Zero, ReturnValue);
2431 // FIXME: We no longer need the types from FunctionArgList; lift up and
2434 ClangToLLVMArgMapping IRFunctionArgs(CGM.getContext(), FI);
2435 assert(Fn->arg_size() == IRFunctionArgs.totalIRArgs());
2437 // If we're using inalloca, all the memory arguments are GEPs off of the last
2438 // parameter, which is a pointer to the complete memory area.
2439 Address ArgStruct = Address::invalid();
2440 if (IRFunctionArgs.hasInallocaArg()) {
2441 ArgStruct = Address(Fn->getArg(IRFunctionArgs.getInallocaArgNo()),
2442 FI.getArgStructAlignment());
2444 assert(ArgStruct.getType() == FI.getArgStruct()->getPointerTo());
2447 // Name the struct return parameter.
2448 if (IRFunctionArgs.hasSRetArg()) {
2449 auto AI = Fn->getArg(IRFunctionArgs.getSRetArgNo());
2450 AI->setName("agg.result");
2451 AI->addAttr(llvm::Attribute::NoAlias);
2454 // Track if we received the parameter as a pointer (indirect, byval, or
2455 // inalloca). If already have a pointer, EmitParmDecl doesn't need to copy it
2456 // into a local alloca for us.
2457 SmallVector<ParamValue, 16> ArgVals;
2458 ArgVals.reserve(Args.size());
2460 // Create a pointer value for every parameter declaration. This usually
2461 // entails copying one or more LLVM IR arguments into an alloca. Don't push
2462 // any cleanups or do anything that might unwind. We do that separately, so
2463 // we can push the cleanups in the correct order for the ABI.
2464 assert(FI.arg_size() == Args.size() &&
2465 "Mismatch between function signature & arguments.");
2467 CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
2468 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
2469 i != e; ++i, ++info_it, ++ArgNo) {
2470 const VarDecl *Arg = *i;
2471 const ABIArgInfo &ArgI = info_it->info;
2474 isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
2475 // We are converting from ABIArgInfo type to VarDecl type directly, unless
2476 // the parameter is promoted. In this case we convert to
2477 // CGFunctionInfo::ArgInfo type with subsequent argument demotion.
2478 QualType Ty = isPromoted ? info_it->type : Arg->getType();
2479 assert(hasScalarEvaluationKind(Ty) ==
2480 hasScalarEvaluationKind(Arg->getType()));
2482 unsigned FirstIRArg, NumIRArgs;
2483 std::tie(FirstIRArg, NumIRArgs) = IRFunctionArgs.getIRArgs(ArgNo);
2485 switch (ArgI.getKind()) {
2486 case ABIArgInfo::InAlloca: {
2487 assert(NumIRArgs == 0);
2488 auto FieldIndex = ArgI.getInAllocaFieldIndex();
2490 Builder.CreateStructGEP(ArgStruct, FieldIndex, Arg->getName());
2491 if (ArgI.getInAllocaIndirect())
2492 V = Address(Builder.CreateLoad(V),
2493 getContext().getTypeAlignInChars(Ty));
2494 ArgVals.push_back(ParamValue::forIndirect(V));
2498 case ABIArgInfo::Indirect:
2499 case ABIArgInfo::IndirectAliased: {
2500 assert(NumIRArgs == 1);
2502 Address(Fn->getArg(FirstIRArg), ArgI.getIndirectAlign());
2504 if (!hasScalarEvaluationKind(Ty)) {
2505 // Aggregates and complex variables are accessed by reference. All we
2506 // need to do is realign the value, if requested. Also, if the address
2507 // may be aliased, copy it to ensure that the parameter variable is
2508 // mutable and has a unique adress, as C requires.
2509 Address V = ParamAddr;
2510 if (ArgI.getIndirectRealign() || ArgI.isIndirectAliased()) {
2511 Address AlignedTemp = CreateMemTemp(Ty, "coerce");
2513 // Copy from the incoming argument pointer to the temporary with the
2514 // appropriate alignment.
2516 // FIXME: We should have a common utility for generating an aggregate
2518 CharUnits Size = getContext().getTypeSizeInChars(Ty);
2519 Builder.CreateMemCpy(
2520 AlignedTemp.getPointer(), AlignedTemp.getAlignment().getAsAlign(),
2521 ParamAddr.getPointer(), ParamAddr.getAlignment().getAsAlign(),
2522 llvm::ConstantInt::get(IntPtrTy, Size.getQuantity()));
2525 ArgVals.push_back(ParamValue::forIndirect(V));
2527 // Load scalar value from indirect argument.
2529 EmitLoadOfScalar(ParamAddr, false, Ty, Arg->getBeginLoc());
2532 V = emitArgumentDemotion(*this, Arg, V);
2533 ArgVals.push_back(ParamValue::forDirect(V));
2538 case ABIArgInfo::Extend:
2539 case ABIArgInfo::Direct: {
2540 auto AI = Fn->getArg(FirstIRArg);
2541 llvm::Type *LTy = ConvertType(Arg->getType());
2543 // Prepare parameter attributes. So far, only attributes for pointer
2544 // parameters are prepared. See
2545 // http://llvm.org/docs/LangRef.html#paramattrs.
2546 if (ArgI.getDirectOffset() == 0 && LTy->isPointerTy() &&
2547 ArgI.getCoerceToType()->isPointerTy()) {
2548 assert(NumIRArgs == 1);
2550 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(Arg)) {
2551 // Set `nonnull` attribute if any.
2552 if (getNonNullAttr(CurCodeDecl, PVD, PVD->getType(),
2553 PVD->getFunctionScopeIndex()) &&
2554 !CGM.getCodeGenOpts().NullPointerIsValid)
2555 AI->addAttr(llvm::Attribute::NonNull);
2557 QualType OTy = PVD->getOriginalType();
2558 if (const auto *ArrTy =
2559 getContext().getAsConstantArrayType(OTy)) {
2560 // A C99 array parameter declaration with the static keyword also
2561 // indicates dereferenceability, and if the size is constant we can
2562 // use the dereferenceable attribute (which requires the size in
2564 if (ArrTy->getSizeModifier() == ArrayType::Static) {
2565 QualType ETy = ArrTy->getElementType();
2566 llvm::Align Alignment =
2567 CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2568 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2569 uint64_t ArrSize = ArrTy->getSize().getZExtValue();
2570 if (!ETy->isIncompleteType() && ETy->isConstantSizeType() &&
2572 llvm::AttrBuilder Attrs;
2573 Attrs.addDereferenceableAttr(
2574 getContext().getTypeSizeInChars(ETy).getQuantity() *
2576 AI->addAttrs(Attrs);
2577 } else if (getContext().getTargetInfo().getNullPointerValue(
2578 ETy.getAddressSpace()) == 0 &&
2579 !CGM.getCodeGenOpts().NullPointerIsValid) {
2580 AI->addAttr(llvm::Attribute::NonNull);
2583 } else if (const auto *ArrTy =
2584 getContext().getAsVariableArrayType(OTy)) {
2585 // For C99 VLAs with the static keyword, we don't know the size so
2586 // we can't use the dereferenceable attribute, but in addrspace(0)
2587 // we know that it must be nonnull.
2588 if (ArrTy->getSizeModifier() == VariableArrayType::Static) {
2589 QualType ETy = ArrTy->getElementType();
2590 llvm::Align Alignment =
2591 CGM.getNaturalTypeAlignment(ETy).getAsAlign();
2592 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(Alignment));
2593 if (!getContext().getTargetAddressSpace(ETy) &&
2594 !CGM.getCodeGenOpts().NullPointerIsValid)
2595 AI->addAttr(llvm::Attribute::NonNull);
2599 // Set `align` attribute if any.
2600 const auto *AVAttr = PVD->getAttr<AlignValueAttr>();
2602 if (const auto *TOTy = dyn_cast<TypedefType>(OTy))
2603 AVAttr = TOTy->getDecl()->getAttr<AlignValueAttr>();
2604 if (AVAttr && !SanOpts.has(SanitizerKind::Alignment)) {
2605 // If alignment-assumption sanitizer is enabled, we do *not* add
2606 // alignment attribute here, but emit normal alignment assumption,
2607 // so the UBSAN check could function.
2608 llvm::ConstantInt *AlignmentCI =
2609 cast<llvm::ConstantInt>(EmitScalarExpr(AVAttr->getAlignment()));
2610 unsigned AlignmentInt =
2611 AlignmentCI->getLimitedValue(llvm::Value::MaximumAlignment);
2612 if (AI->getParamAlign().valueOrOne() < AlignmentInt) {
2613 AI->removeAttr(llvm::Attribute::AttrKind::Alignment);
2614 AI->addAttrs(llvm::AttrBuilder().addAlignmentAttr(
2615 llvm::Align(AlignmentInt)));
2620 // Set 'noalias' if an argument type has the `restrict` qualifier.
2621 if (Arg->getType().isRestrictQualified())
2622 AI->addAttr(llvm::Attribute::NoAlias);
2625 // Prepare the argument value. If we have the trivial case, handle it
2626 // with no muss and fuss.
2627 if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
2628 ArgI.getCoerceToType() == ConvertType(Ty) &&
2629 ArgI.getDirectOffset() == 0) {
2630 assert(NumIRArgs == 1);
2632 // LLVM expects swifterror parameters to be used in very restricted
2633 // ways. Copy the value into a less-restricted temporary.
2634 llvm::Value *V = AI;
2635 if (FI.getExtParameterInfo(ArgNo).getABI()
2636 == ParameterABI::SwiftErrorResult) {
2637 QualType pointeeTy = Ty->getPointeeType();
2638 assert(pointeeTy->isPointerType());
2640 CreateMemTemp(pointeeTy, getPointerAlign(), "swifterror.temp");
2641 Address arg = Address(V, getContext().getTypeAlignInChars(pointeeTy));
2642 llvm::Value *incomingErrorValue = Builder.CreateLoad(arg);
2643 Builder.CreateStore(incomingErrorValue, temp);
2644 V = temp.getPointer();
2646 // Push a cleanup to copy the value back at the end of the function.
2647 // The convention does not guarantee that the value will be written
2648 // back if the function exits with an unwind exception.
2649 EHStack.pushCleanup<CopyBackSwiftError>(NormalCleanup, temp, arg);
2652 // Ensure the argument is the correct type.
2653 if (V->getType() != ArgI.getCoerceToType())
2654 V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
2657 V = emitArgumentDemotion(*this, Arg, V);
2659 // Because of merging of function types from multiple decls it is
2660 // possible for the type of an argument to not match the corresponding
2661 // type in the function type. Since we are codegening the callee
2662 // in here, add a cast to the argument type.
2663 llvm::Type *LTy = ConvertType(Arg->getType());
2664 if (V->getType() != LTy)
2665 V = Builder.CreateBitCast(V, LTy);
2667 ArgVals.push_back(ParamValue::forDirect(V));
2671 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg),
2674 // Pointer to store into.
2675 Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
2677 // Fast-isel and the optimizer generally like scalar values better than
2678 // FCAs, so we flatten them if this is safe to do for this argument.
2679 llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
2680 if (ArgI.isDirect() && ArgI.getCanBeFlattened() && STy &&
2681 STy->getNumElements() > 1) {
2682 uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
2683 llvm::Type *DstTy = Ptr.getElementType();
2684 uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
2686 Address AddrToStoreInto = Address::invalid();
2687 if (SrcSize <= DstSize) {
2688 AddrToStoreInto = Builder.CreateElementBitCast(Ptr, STy);
2691 CreateTempAlloca(STy, Alloca.getAlignment(), "coerce");
2694 assert(STy->getNumElements() == NumIRArgs);
2695 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
2696 auto AI = Fn->getArg(FirstIRArg + i);
2697 AI->setName(Arg->getName() + ".coerce" + Twine(i));
2698 Address EltPtr = Builder.CreateStructGEP(AddrToStoreInto, i);
2699 Builder.CreateStore(AI, EltPtr);
2702 if (SrcSize > DstSize) {
2703 Builder.CreateMemCpy(Ptr, AddrToStoreInto, DstSize);
2707 // Simple case, just do a coerced store of the argument into the alloca.
2708 assert(NumIRArgs == 1);
2709 auto AI = Fn->getArg(FirstIRArg);
2710 AI->setName(Arg->getName() + ".coerce");
2711 CreateCoercedStore(AI, Ptr, /*DstIsVolatile=*/false, *this);
2714 // Match to what EmitParmDecl is expecting for this type.
2715 if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
2717 EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
2719 V = emitArgumentDemotion(*this, Arg, V);
2720 ArgVals.push_back(ParamValue::forDirect(V));
2722 ArgVals.push_back(ParamValue::forIndirect(Alloca));
2727 case ABIArgInfo::CoerceAndExpand: {
2728 // Reconstruct into a temporary.
2729 Address alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2730 ArgVals.push_back(ParamValue::forIndirect(alloca));
2732 auto coercionType = ArgI.getCoerceAndExpandType();
2733 alloca = Builder.CreateElementBitCast(alloca, coercionType);
2735 unsigned argIndex = FirstIRArg;
2736 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
2737 llvm::Type *eltType = coercionType->getElementType(i);
2738 if (ABIArgInfo::isPaddingForCoerceAndExpand(eltType))
2741 auto eltAddr = Builder.CreateStructGEP(alloca, i);
2742 auto elt = Fn->getArg(argIndex++);
2743 Builder.CreateStore(elt, eltAddr);
2745 assert(argIndex == FirstIRArg + NumIRArgs);
2749 case ABIArgInfo::Expand: {
2750 // If this structure was expanded into multiple arguments then
2751 // we need to create a temporary and reconstruct it from the
2753 Address Alloca = CreateMemTemp(Ty, getContext().getDeclAlign(Arg));
2754 LValue LV = MakeAddrLValue(Alloca, Ty);
2755 ArgVals.push_back(ParamValue::forIndirect(Alloca));
2757 auto FnArgIter = Fn->arg_begin() + FirstIRArg;
2758 ExpandTypeFromArgs(Ty, LV, FnArgIter);
2759 assert(FnArgIter == Fn->arg_begin() + FirstIRArg + NumIRArgs);
2760 for (unsigned i = 0, e = NumIRArgs; i != e; ++i) {
2761 auto AI = Fn->getArg(FirstIRArg + i);
2762 AI->setName(Arg->getName() + "." + Twine(i));
2767 case ABIArgInfo::Ignore:
2768 assert(NumIRArgs == 0);
2769 // Initialize the local variable appropriately.
2770 if (!hasScalarEvaluationKind(Ty)) {
2771 ArgVals.push_back(ParamValue::forIndirect(CreateMemTemp(Ty)));
2773 llvm::Value *U = llvm::UndefValue::get(ConvertType(Arg->getType()));
2774 ArgVals.push_back(ParamValue::forDirect(U));
2780 if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
2781 for (int I = Args.size() - 1; I >= 0; --I)
2782 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2784 for (unsigned I = 0, E = Args.size(); I != E; ++I)
2785 EmitParmDecl(*Args[I], ArgVals[I], I + 1);
2789 static void eraseUnusedBitCasts(llvm::Instruction *insn) {
2790 while (insn->use_empty()) {
2791 llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(insn);
2792 if (!bitcast) return;
2794 // This is "safe" because we would have used a ConstantExpr otherwise.
2795 insn = cast<llvm::Instruction>(bitcast->getOperand(0));
2796 bitcast->eraseFromParent();
2800 /// Try to emit a fused autorelease of a return result.
2801 static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
2802 llvm::Value *result) {
2803 // We must be immediately followed the cast.
2804 llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
2805 if (BB->empty()) return nullptr;
2806 if (&BB->back() != result) return nullptr;
2808 llvm::Type *resultType = result->getType();
2810 // result is in a BasicBlock and is therefore an Instruction.
2811 llvm::Instruction *generator = cast<llvm::Instruction>(result);
2813 SmallVector<llvm::Instruction *, 4> InstsToKill;
2816 // %generator = bitcast %type1* %generator2 to %type2*
2817 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
2818 // We would have emitted this as a constant if the operand weren't
2820 generator = cast<llvm::Instruction>(bitcast->getOperand(0));
2822 // Require the generator to be immediately followed by the cast.
2823 if (generator->getNextNode() != bitcast)
2826 InstsToKill.push_back(bitcast);
2830 // %generator = call i8* @objc_retain(i8* %originalResult)
2832 // %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
2833 llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
2834 if (!call) return nullptr;
2836 bool doRetainAutorelease;
2838 if (call->getCalledOperand() == CGF.CGM.getObjCEntrypoints().objc_retain) {
2839 doRetainAutorelease = true;
2840 } else if (call->getCalledOperand() ==
2841 CGF.CGM.getObjCEntrypoints().objc_retainAutoreleasedReturnValue) {
2842 doRetainAutorelease = false;
2844 // If we emitted an assembly marker for this call (and the
2845 // ARCEntrypoints field should have been set if so), go looking
2846 // for that call. If we can't find it, we can't do this
2847 // optimization. But it should always be the immediately previous
2848 // instruction, unless we needed bitcasts around the call.
2849 if (CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker) {
2850 llvm::Instruction *prev = call->getPrevNode();
2852 if (isa<llvm::BitCastInst>(prev)) {
2853 prev = prev->getPrevNode();
2856 assert(isa<llvm::CallInst>(prev));
2857 assert(cast<llvm::CallInst>(prev)->getCalledOperand() ==
2858 CGF.CGM.getObjCEntrypoints().retainAutoreleasedReturnValueMarker);
2859 InstsToKill.push_back(prev);
2865 result = call->getArgOperand(0);
2866 InstsToKill.push_back(call);
2868 // Keep killing bitcasts, for sanity. Note that we no longer care
2869 // about precise ordering as long as there's exactly one use.
2870 while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
2871 if (!bitcast->hasOneUse()) break;
2872 InstsToKill.push_back(bitcast);
2873 result = bitcast->getOperand(0);
2876 // Delete all the unnecessary instructions, from latest to earliest.
2877 for (auto *I : InstsToKill)
2878 I->eraseFromParent();
2880 // Do the fused retain/autorelease if we were asked to.
2881 if (doRetainAutorelease)
2882 result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
2884 // Cast back to the result type.
2885 return CGF.Builder.CreateBitCast(result, resultType);
2888 /// If this is a +1 of the value of an immutable 'self', remove it.
2889 static llvm::Value *tryRemoveRetainOfSelf(CodeGenFunction &CGF,
2890 llvm::Value *result) {
2891 // This is only applicable to a method with an immutable 'self'.
2892 const ObjCMethodDecl *method =
2893 dyn_cast_or_null<ObjCMethodDecl>(CGF.CurCodeDecl);
2894 if (!method) return nullptr;
2895 const VarDecl *self = method->getSelfDecl();
2896 if (!self->getType().isConstQualified()) return nullptr;
2898 // Look for a retain call.
2899 llvm::CallInst *retainCall =
2900 dyn_cast<llvm::CallInst>(result->stripPointerCasts());
2901 if (!retainCall || retainCall->getCalledOperand() !=
2902 CGF.CGM.getObjCEntrypoints().objc_retain)
2905 // Look for an ordinary load of 'self'.
2906 llvm::Value *retainedValue = retainCall->getArgOperand(0);
2907 llvm::LoadInst *load =
2908 dyn_cast<llvm::LoadInst>(retainedValue->stripPointerCasts());
2909 if (!load || load->isAtomic() || load->isVolatile() ||
2910 load->getPointerOperand() != CGF.GetAddrOfLocalVar(self).getPointer())
2913 // Okay! Burn it all down. This relies for correctness on the
2914 // assumption that the retain is emitted as part of the return and
2915 // that thereafter everything is used "linearly".
2916 llvm::Type *resultType = result->getType();
2917 eraseUnusedBitCasts(cast<llvm::Instruction>(result));
2918 assert(retainCall->use_empty());
2919 retainCall->eraseFromParent();
2920 eraseUnusedBitCasts(cast<llvm::Instruction>(retainedValue));
2922 return CGF.Builder.CreateBitCast(load, resultType);
2925 /// Emit an ARC autorelease of the result of a function.
2927 /// \return the value to actually return from the function
2928 static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
2929 llvm::Value *result) {
2930 // If we're returning 'self', kill the initial retain. This is a
2931 // heuristic attempt to "encourage correctness" in the really unfortunate
2932 // case where we have a return of self during a dealloc and we desperately
2933 // need to avoid the possible autorelease.
2934 if (llvm::Value *self = tryRemoveRetainOfSelf(CGF, result))
2937 // At -O0, try to emit a fused retain/autorelease.
2938 if (CGF.shouldUseFusedARCCalls())
2939 if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
2942 return CGF.EmitARCAutoreleaseReturnValue(result);
2945 /// Heuristically search for a dominating store to the return-value slot.
2946 static llvm::StoreInst *findDominatingStoreToReturnValue(CodeGenFunction &CGF) {
2947 // Check if a User is a store which pointerOperand is the ReturnValue.
2948 // We are looking for stores to the ReturnValue, not for stores of the
2949 // ReturnValue to some other location.
2950 auto GetStoreIfValid = [&CGF](llvm::User *U) -> llvm::StoreInst * {
2951 auto *SI = dyn_cast<llvm::StoreInst>(U);
2952 if (!SI || SI->getPointerOperand() != CGF.ReturnValue.getPointer())
2954 // These aren't actually possible for non-coerced returns, and we
2955 // only care about non-coerced returns on this code path.
2956 assert(!SI->isAtomic() && !SI->isVolatile());
2959 // If there are multiple uses of the return-value slot, just check
2960 // for something immediately preceding the IP. Sometimes this can
2961 // happen with how we generate implicit-returns; it can also happen
2962 // with noreturn cleanups.
2963 if (!CGF.ReturnValue.getPointer()->hasOneUse()) {
2964 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2965 if (IP->empty()) return nullptr;
2966 llvm::Instruction *I = &IP->back();
2968 // Skip lifetime markers
2969 for (llvm::BasicBlock::reverse_iterator II = IP->rbegin(),
2972 if (llvm::IntrinsicInst *Intrinsic =
2973 dyn_cast<llvm::IntrinsicInst>(&*II)) {
2974 if (Intrinsic->getIntrinsicID() == llvm::Intrinsic::lifetime_end) {
2975 const llvm::Value *CastAddr = Intrinsic->getArgOperand(1);
2979 if (isa<llvm::BitCastInst>(&*II) && (CastAddr == &*II))
2987 return GetStoreIfValid(I);
2990 llvm::StoreInst *store =
2991 GetStoreIfValid(CGF.ReturnValue.getPointer()->user_back());
2992 if (!store) return nullptr;
2994 // Now do a first-and-dirty dominance check: just walk up the
2995 // single-predecessors chain from the current insertion point.
2996 llvm::BasicBlock *StoreBB = store->getParent();
2997 llvm::BasicBlock *IP = CGF.Builder.GetInsertBlock();
2998 while (IP != StoreBB) {
2999 if (!(IP = IP->getSinglePredecessor()))
3003 // Okay, the store's basic block dominates the insertion point; we
3004 // can do our thing.
3008 // Helper functions for EmitCMSEClearRecord
3010 // Set the bits corresponding to a field having width `BitWidth` and located at
3011 // offset `BitOffset` (from the least significant bit) within a storage unit of
3012 // `Bits.size()` bytes. Each element of `Bits` corresponds to one target byte.
3013 // Use little-endian layout, i.e.`Bits[0]` is the LSB.
3014 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int BitOffset,
3015 int BitWidth, int CharWidth) {
3016 assert(CharWidth <= 64);
3017 assert(static_cast<unsigned>(BitWidth) <= Bits.size() * CharWidth);
3020 if (BitOffset >= CharWidth) {
3021 Pos += BitOffset / CharWidth;
3022 BitOffset = BitOffset % CharWidth;
3025 const uint64_t Used = (uint64_t(1) << CharWidth) - 1;
3026 if (BitOffset + BitWidth >= CharWidth) {
3027 Bits[Pos++] |= (Used << BitOffset) & Used;
3028 BitWidth -= CharWidth - BitOffset;
3032 while (BitWidth >= CharWidth) {
3034 BitWidth -= CharWidth;
3038 Bits[Pos++] |= (Used >> (CharWidth - BitWidth)) << BitOffset;
3041 // Set the bits corresponding to a field having width `BitWidth` and located at
3042 // offset `BitOffset` (from the least significant bit) within a storage unit of
3043 // `StorageSize` bytes, located at `StorageOffset` in `Bits`. Each element of
3044 // `Bits` corresponds to one target byte. Use target endian layout.
3045 static void setBitRange(SmallVectorImpl<uint64_t> &Bits, int StorageOffset,
3046 int StorageSize, int BitOffset, int BitWidth,
3047 int CharWidth, bool BigEndian) {
3049 SmallVector<uint64_t, 8> TmpBits(StorageSize);
3050 setBitRange(TmpBits, BitOffset, BitWidth, CharWidth);
3053 std::reverse(TmpBits.begin(), TmpBits.end());
3055 for (uint64_t V : TmpBits)
3056 Bits[StorageOffset++] |= V;
3059 static void setUsedBits(CodeGenModule &, QualType, int,
3060 SmallVectorImpl<uint64_t> &);
3062 // Set the bits in `Bits`, which correspond to the value representations of
3063 // the actual members of the record type `RTy`. Note that this function does
3064 // not handle base classes, virtual tables, etc, since they cannot happen in
3065 // CMSE function arguments or return. The bit mask corresponds to the target
3066 // memory layout, i.e. it's endian dependent.
3067 static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
3068 SmallVectorImpl<uint64_t> &Bits) {
3069 ASTContext &Context = CGM.getContext();
3070 int CharWidth = Context.getCharWidth();
3071 const RecordDecl *RD = RTy->getDecl()->getDefinition();
3072 const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
3073 const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
3076 for (auto I = RD->field_begin(), E = RD->field_end(); I != E; ++I, ++Idx) {
3077 const FieldDecl *F = *I;
3079 if (F->isUnnamedBitfield() || F->isZeroLengthBitField(Context) ||
3080 F->getType()->isIncompleteArrayType())
3083 if (F->isBitField()) {
3084 const CGBitFieldInfo &BFI = Layout.getBitFieldInfo(F);
3085 setBitRange(Bits, Offset + BFI.StorageOffset.getQuantity(),
3086 BFI.StorageSize / CharWidth, BFI.Offset,
3087 BFI.Size, CharWidth,
3088 CGM.getDataLayout().isBigEndian());
3092 setUsedBits(CGM, F->getType(),
3093 Offset + ASTLayout.getFieldOffset(Idx) / CharWidth, Bits);
3097 // Set the bits in `Bits`, which correspond to the value representations of
3098 // the elements of an array type `ATy`.
3099 static void setUsedBits(CodeGenModule &CGM, const ConstantArrayType *ATy,
3100 int Offset, SmallVectorImpl<uint64_t> &Bits) {
3101 const ASTContext &Context = CGM.getContext();
3103 QualType ETy = Context.getBaseElementType(ATy);
3104 int Size = Context.getTypeSizeInChars(ETy).getQuantity();
3105 SmallVector<uint64_t, 4> TmpBits(Size);
3106 setUsedBits(CGM, ETy, 0, TmpBits);
3108 for (int I = 0, N = Context.getConstantArrayElementCount(ATy); I < N; ++I) {
3109 auto Src = TmpBits.begin();
3110 auto Dst = Bits.begin() + Offset + I * Size;
3111 for (int J = 0; J < Size; ++J)
3116 // Set the bits in `Bits`, which correspond to the value representations of
3118 static void setUsedBits(CodeGenModule &CGM, QualType QTy, int Offset,
3119 SmallVectorImpl<uint64_t> &Bits) {
3120 if (const auto *RTy = QTy->getAs<RecordType>())
3121 return setUsedBits(CGM, RTy, Offset, Bits);
3123 ASTContext &Context = CGM.getContext();
3124 if (const auto *ATy = Context.getAsConstantArrayType(QTy))
3125 return setUsedBits(CGM, ATy, Offset, Bits);
3127 int Size = Context.getTypeSizeInChars(QTy).getQuantity();
3131 std::fill_n(Bits.begin() + Offset, Size,
3132 (uint64_t(1) << Context.getCharWidth()) - 1);
3135 static uint64_t buildMultiCharMask(const SmallVectorImpl<uint64_t> &Bits,
3136 int Pos, int Size, int CharWidth,
3141 for (auto P = Bits.begin() + Pos, E = Bits.begin() + Pos + Size; P != E;
3143 Mask = (Mask << CharWidth) | *P;
3145 auto P = Bits.begin() + Pos + Size, End = Bits.begin() + Pos;
3147 Mask = (Mask << CharWidth) | *--P;
3153 // Emit code to clear the bits in a record, which aren't a part of any user
3154 // declared member, when the record is a function return.
3155 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3156 llvm::IntegerType *ITy,
3158 assert(Src->getType() == ITy);
3159 assert(ITy->getScalarSizeInBits() <= 64);
3161 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3162 int Size = DataLayout.getTypeStoreSize(ITy);
3163 SmallVector<uint64_t, 4> Bits(Size);
3164 setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3166 int CharWidth = CGM.getContext().getCharWidth();
3168 buildMultiCharMask(Bits, 0, Size, CharWidth, DataLayout.isBigEndian());
3170 return Builder.CreateAnd(Src, Mask, "cmse.clear");
3173 // Emit code to clear the bits in a record, which aren't a part of any user
3174 // declared member, when the record is a function argument.
3175 llvm::Value *CodeGenFunction::EmitCMSEClearRecord(llvm::Value *Src,
3176 llvm::ArrayType *ATy,
3178 const llvm::DataLayout &DataLayout = CGM.getDataLayout();
3179 int Size = DataLayout.getTypeStoreSize(ATy);
3180 SmallVector<uint64_t, 16> Bits(Size);
3181 setUsedBits(CGM, QTy->castAs<RecordType>(), 0, Bits);
3183 // Clear each element of the LLVM array.
3184 int CharWidth = CGM.getContext().getCharWidth();
3186 ATy->getArrayElementType()->getScalarSizeInBits() / CharWidth;
3188 llvm::Value *R = llvm::UndefValue::get(ATy);
3189 for (int I = 0, N = ATy->getArrayNumElements(); I != N; ++I) {
3190 uint64_t Mask = buildMultiCharMask(Bits, MaskIndex, CharsPerElt, CharWidth,
3191 DataLayout.isBigEndian());
3192 MaskIndex += CharsPerElt;
3193 llvm::Value *T0 = Builder.CreateExtractValue(Src, I);
3194 llvm::Value *T1 = Builder.CreateAnd(T0, Mask, "cmse.clear");
3195 R = Builder.CreateInsertValue(R, T1, I);
3201 void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
3203 SourceLocation EndLoc) {
3204 if (FI.isNoReturn()) {
3205 // Noreturn functions don't return.
3206 EmitUnreachable(EndLoc);
3210 if (CurCodeDecl && CurCodeDecl->hasAttr<NakedAttr>()) {
3211 // Naked functions don't have epilogues.
3212 Builder.CreateUnreachable();
3216 // Functions with no result always return void.
3217 if (!ReturnValue.isValid()) {
3218 Builder.CreateRetVoid();
3222 llvm::DebugLoc RetDbgLoc;
3223 llvm::Value *RV = nullptr;
3224 QualType RetTy = FI.getReturnType();
3225 const ABIArgInfo &RetAI = FI.getReturnInfo();
3227 switch (RetAI.getKind()) {
3228 case ABIArgInfo::InAlloca:
3229 // Aggregrates get evaluated directly into the destination. Sometimes we
3230 // need to return the sret value in a register, though.
3231 assert(hasAggregateEvaluationKind(RetTy));
3232 if (RetAI.getInAllocaSRet()) {
3233 llvm::Function::arg_iterator EI = CurFn->arg_end();
3235 llvm::Value *ArgStruct = &*EI;
3236 llvm::Value *SRet = Builder.CreateStructGEP(
3237 nullptr, ArgStruct, RetAI.getInAllocaFieldIndex());
3238 RV = Builder.CreateAlignedLoad(SRet, getPointerAlign(), "sret");
3242 case ABIArgInfo::Indirect: {
3243 auto AI = CurFn->arg_begin();
3244 if (RetAI.isSRetAfterThis())
3246 switch (getEvaluationKind(RetTy)) {
3249 EmitLoadOfComplex(MakeAddrLValue(ReturnValue, RetTy), EndLoc);
3250 EmitStoreOfComplex(RT, MakeNaturalAlignAddrLValue(&*AI, RetTy),
3255 // Do nothing; aggregrates get evaluated directly into the destination.
3258 EmitStoreOfScalar(Builder.CreateLoad(ReturnValue),
3259 MakeNaturalAlignAddrLValue(&*AI, RetTy),
3266 case ABIArgInfo::Extend:
3267 case ABIArgInfo::Direct:
3268 if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
3269 RetAI.getDirectOffset() == 0) {
3270 // The internal return value temp always will have pointer-to-return-type
3271 // type, just do a load.
3273 // If there is a dominating store to ReturnValue, we can elide
3274 // the load, zap the store, and usually zap the alloca.
3275 if (llvm::StoreInst *SI =
3276 findDominatingStoreToReturnValue(*this)) {
3277 // Reuse the debug location from the store unless there is
3278 // cleanup code to be emitted between the store and return
3280 if (EmitRetDbgLoc && !AutoreleaseResult)
3281 RetDbgLoc = SI->getDebugLoc();
3282 // Get the stored value and nuke the now-dead store.
3283 RV = SI->getValueOperand();
3284 SI->eraseFromParent();
3286 // Otherwise, we have to do a simple load.
3288 RV = Builder.CreateLoad(ReturnValue);
3291 // If the value is offset in memory, apply the offset now.
3292 Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
3294 RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
3297 // In ARC, end functions that return a retainable type with a call
3298 // to objc_autoreleaseReturnValue.
3299 if (AutoreleaseResult) {
3301 // Type::isObjCRetainabletype has to be called on a QualType that hasn't
3302 // been stripped of the typedefs, so we cannot use RetTy here. Get the
3303 // original return type of FunctionDecl, CurCodeDecl, and BlockDecl from
3304 // CurCodeDecl or BlockInfo.
3307 if (auto *FD = dyn_cast<FunctionDecl>(CurCodeDecl))
3308 RT = FD->getReturnType();
3309 else if (auto *MD = dyn_cast<ObjCMethodDecl>(CurCodeDecl))
3310 RT = MD->getReturnType();
3311 else if (isa<BlockDecl>(CurCodeDecl))
3312 RT = BlockInfo->BlockExpression->getFunctionType()->getReturnType();
3314 llvm_unreachable("Unexpected function/method type");
3316 assert(getLangOpts().ObjCAutoRefCount &&
3317 !FI.isReturnsRetained() &&
3318 RT->isObjCRetainableType());
3320 RV = emitAutoreleaseOfResult(*this, RV);
3325 case ABIArgInfo::Ignore:
3328 case ABIArgInfo::CoerceAndExpand: {
3329 auto coercionType = RetAI.getCoerceAndExpandType();
3331 // Load all of the coerced elements out into results.
3332 llvm::SmallVector<llvm::Value*, 4> results;
3333 Address addr = Builder.CreateElementBitCast(ReturnValue, coercionType);
3334 for (unsigned i = 0, e = coercionType->getNumElements(); i != e; ++i) {
3335 auto coercedEltType = coercionType->getElementType(i);
3336 if (ABIArgInfo::isPaddingForCoerceAndExpand(coercedEltType))
3339 auto eltAddr = Builder.CreateStructGEP(addr, i);
3340 auto elt = Builder.CreateLoad(eltAddr);
3341 results.push_back(elt);
3344 // If we have one result, it's the single direct result type.
3345 if (results.size() == 1) {
3348 // Otherwise, we need to make a first-class aggregate.
3350 // Construct a return type that lacks padding elements.
3351 llvm::Type *returnType = RetAI.getUnpaddedCoerceAndExpandType();
3353 RV = llvm::UndefValue::get(returnType);
3354 for (unsigned i = 0, e = results.size(); i != e; ++i) {
3355 RV = Builder.CreateInsertValue(RV, results[i], i);
3360 case ABIArgInfo::Expand:
3361 case ABIArgInfo::IndirectAliased:
3362 llvm_unreachable("Invalid ABI kind for return argument");
3365 llvm::Instruction *Ret;
3367 if (CurFuncDecl && CurFuncDecl->hasAttr<CmseNSEntryAttr>()) {
3368 // For certain return types, clear padding bits, as they may reveal
3369 // sensitive information.
3370 // Small struct/union types are passed as integers.
3371 auto *ITy = dyn_cast<llvm::IntegerType>(RV->getType());
3372 if (ITy != nullptr && isa<RecordType>(RetTy.getCanonicalType()))
3373 RV = EmitCMSEClearRecord(RV, ITy, RetTy);
3375 EmitReturnValueCheck(RV);
3376 Ret = Builder.CreateRet(RV);
3378 Ret = Builder.CreateRetVoid();
3382 Ret->setDebugLoc(std::move(RetDbgLoc));
3385 void CodeGenFunction::EmitReturnValueCheck(llvm::Value *RV) {
3386 // A current decl may not be available when emitting vtable thunks.
3390 // If the return block isn't reachable, neither is this check, so don't emit
3392 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty())
3395 ReturnsNonNullAttr *RetNNAttr = nullptr;
3396 if (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute))
3397 RetNNAttr = CurCodeDecl->getAttr<ReturnsNonNullAttr>();
3399 if (!RetNNAttr && !requiresReturnValueNullabilityCheck())
3402 // Prefer the returns_nonnull attribute if it's present.
3403 SourceLocation AttrLoc;
3404 SanitizerMask CheckKind;
3405 SanitizerHandler Handler;
3407 assert(!requiresReturnValueNullabilityCheck() &&
3408 "Cannot check nullability and the nonnull attribute");
3409 AttrLoc = RetNNAttr->getLocation();
3410 CheckKind = SanitizerKind::ReturnsNonnullAttribute;
3411 Handler = SanitizerHandler::NonnullReturn;
3413 if (auto *DD = dyn_cast<DeclaratorDecl>(CurCodeDecl))
3414 if (auto *TSI = DD->getTypeSourceInfo())
3415 if (auto FTL = TSI->getTypeLoc().getAsAdjusted<FunctionTypeLoc>())
3416 AttrLoc = FTL.getReturnLoc().findNullabilityLoc();
3417 CheckKind = SanitizerKind::NullabilityReturn;
3418 Handler = SanitizerHandler::NullabilityReturn;
3421 SanitizerScope SanScope(this);
3423 // Make sure the "return" source location is valid. If we're checking a
3424 // nullability annotation, make sure the preconditions for the check are met.
3425 llvm::BasicBlock *Check = createBasicBlock("nullcheck");
3426 llvm::BasicBlock *NoCheck = createBasicBlock("no.nullcheck");
3427 llvm::Value *SLocPtr = Builder.CreateLoad(ReturnLocation, "return.sloc.load");
3428 llvm::Value *CanNullCheck = Builder.CreateIsNotNull(SLocPtr);
3429 if (requiresReturnValueNullabilityCheck())
3431 Builder.CreateAnd(CanNullCheck, RetValNullabilityPrecondition);
3432 Builder.CreateCondBr(CanNullCheck, Check, NoCheck);
3435 // Now do the null check.
3436 llvm::Value *Cond = Builder.CreateIsNotNull(RV);
3437 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(AttrLoc)};
3438 llvm::Value *DynamicData[] = {SLocPtr};
3439 EmitCheck(std::make_pair(Cond, CheckKind), Handler, StaticData, DynamicData);
3444 // The return location should not be used after the check has been emitted.
3445 ReturnLocation = Address::invalid();
3449 static bool isInAllocaArgument(CGCXXABI &ABI, QualType type) {
3450 const CXXRecordDecl *RD = type->getAsCXXRecordDecl();
3451 return RD && ABI.getRecordArgABI(RD) == CGCXXABI::RAA_DirectInMemory;
3454 static AggValueSlot createPlaceholderSlot(CodeGenFunction &CGF,
3456 // FIXME: Generate IR in one pass, rather than going back and fixing up these
3458 llvm::Type *IRTy = CGF.ConvertTypeForMem(Ty);
3459 llvm::Type *IRPtrTy = IRTy->getPointerTo();
3460 llvm::Value *Placeholder = llvm::UndefValue::get(IRPtrTy->getPointerTo());
3462 // FIXME: When we generate this IR in one pass, we shouldn't need
3463 // this win32-specific alignment hack.
3464 CharUnits Align = CharUnits::fromQuantity(4);
3465 Placeholder = CGF.Builder.CreateAlignedLoad(IRPtrTy, Placeholder, Align);
3467 return AggValueSlot::forAddr(Address(Placeholder, Align),
3469 AggValueSlot::IsNotDestructed,
3470 AggValueSlot::DoesNotNeedGCBarriers,
3471 AggValueSlot::IsNotAliased,
3472 AggValueSlot::DoesNotOverlap);
3475 void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
3476 const VarDecl *param,
3477 SourceLocation loc) {
3478 // StartFunction converted the ABI-lowered parameter(s) into a
3479 // local alloca. We need to turn that into an r-value suitable
3481 Address local = GetAddrOfLocalVar(param);
3483 QualType type = param->getType();
3485 if (isInAllocaArgument(CGM.getCXXABI(), type)) {
3486 CGM.ErrorUnsupported(param, "forwarded non-trivially copyable parameter");
3489 // GetAddrOfLocalVar returns a pointer-to-pointer for references,
3490 // but the argument needs to be the original pointer.
3491 if (type->isReferenceType()) {
3492 args.add(RValue::get(Builder.CreateLoad(local)), type);
3494 // In ARC, move out of consumed arguments so that the release cleanup
3495 // entered by StartFunction doesn't cause an over-release. This isn't
3496 // optimal -O0 code generation, but it should get cleaned up when
3497 // optimization is enabled. This also assumes that delegate calls are
3498 // performed exactly once for a set of arguments, but that should be safe.
3499 } else if (getLangOpts().ObjCAutoRefCount &&
3500 param->hasAttr<NSConsumedAttr>() &&
3501 type->isObjCRetainableType()) {
3502 llvm::Value *ptr = Builder.CreateLoad(local);
3504 llvm::ConstantPointerNull::get(cast<llvm::PointerType>(ptr->getType()));
3505 Builder.CreateStore(null, local);
3506 args.add(RValue::get(ptr), type);
3508 // For the most part, we just need to load the alloca, except that
3509 // aggregate r-values are actually pointers to temporaries.
3511 args.add(convertTempToRValue(local, type, loc), type);
3514 // Deactivate the cleanup for the callee-destructed param that was pushed.
3515 if (hasAggregateEvaluationKind(type) && !CurFuncIsThunk &&
3516 type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
3517 param->needsDestruction(getContext())) {
3518 EHScopeStack::stable_iterator cleanup =
3519 CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
3520 assert(cleanup.isValid() &&
3521 "cleanup for callee-destructed param not recorded");
3522 // This unreachable is a temporary marker which will be removed later.
3523 llvm::Instruction *isActive = Builder.CreateUnreachable();
3524 args.addArgCleanupDeactivation(cleanup, isActive);
3528 static bool isProvablyNull(llvm::Value *addr) {
3529 return isa<llvm::ConstantPointerNull>(addr);
3532 /// Emit the actual writing-back of a writeback.
3533 static void emitWriteback(CodeGenFunction &CGF,
3534 const CallArgList::Writeback &writeback) {
3535 const LValue &srcLV = writeback.Source;
3536 Address srcAddr = srcLV.getAddress(CGF);
3537 assert(!isProvablyNull(srcAddr.getPointer()) &&
3538 "shouldn't have writeback for provably null argument");
3540 llvm::BasicBlock *contBB = nullptr;
3542 // If the argument wasn't provably non-null, we need to null check
3543 // before doing the store.
3544 bool provablyNonNull = llvm::isKnownNonZero(srcAddr.getPointer(),
3545 CGF.CGM.getDataLayout());
3546 if (!provablyNonNull) {
3547 llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
3548 contBB = CGF.createBasicBlock("icr.done");
3550 llvm::Value *isNull =
3551 CGF.Builder.CreateIsNull(srcAddr.getPointer(), "icr.isnull");
3552 CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
3553 CGF.EmitBlock(writebackBB);
3556 // Load the value to writeback.
3557 llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
3559 // Cast it back, in case we're writing an id to a Foo* or something.
3560 value = CGF.Builder.CreateBitCast(value, srcAddr.getElementType(),
3561 "icr.writeback-cast");
3563 // Perform the writeback.
3565 // If we have a "to use" value, it's something we need to emit a use
3566 // of. This has to be carefully threaded in: if it's done after the
3567 // release it's potentially undefined behavior (and the optimizer
3568 // will ignore it), and if it happens before the retain then the
3569 // optimizer could move the release there.
3570 if (writeback.ToUse) {
3571 assert(srcLV.getObjCLifetime() == Qualifiers::OCL_Strong);
3573 // Retain the new value. No need to block-copy here: the block's
3574 // being passed up the stack.
3575 value = CGF.EmitARCRetainNonBlock(value);
3577 // Emit the intrinsic use here.
3578 CGF.EmitARCIntrinsicUse(writeback.ToUse);
3580 // Load the old value (primitively).
3581 llvm::Value *oldValue = CGF.EmitLoadOfScalar(srcLV, SourceLocation());