| //! Set and unset common attributes on LLVM values. |
| |
| use rustc_codegen_ssa::traits::*; |
| use rustc_hir::def_id::DefId; |
| use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; |
| use rustc_middle::ty::{self, TyCtxt}; |
| use rustc_session::config::OptLevel; |
| use rustc_span::symbol::sym; |
| use rustc_target::spec::abi::Abi; |
| use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; |
| use smallvec::SmallVec; |
| |
| use crate::attributes; |
| use crate::errors::{MissingFeatures, SanitizerMemtagRequiresMte, TargetFeatureDisableOrEnable}; |
| use crate::llvm::AttributePlace::Function; |
| use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace, MemoryEffects}; |
| use crate::llvm_util; |
| pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; |
| |
| use crate::context::CodegenCx; |
| use crate::value::Value; |
| |
| pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { |
| if !attrs.is_empty() { |
| llvm::AddFunctionAttributes(llfn, idx, attrs); |
| } |
| } |
| |
| pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { |
| if !attrs.is_empty() { |
| llvm::AddCallSiteAttributes(callsite, idx, attrs); |
| } |
| } |
| |
| /// Get LLVM attribute for the provided inline heuristic. |
| #[inline] |
| fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { |
| if !cx.tcx.sess.opts.unstable_opts.inline_llvm { |
| // disable LLVM inlining |
| return Some(AttributeKind::NoInline.create_attr(cx.llcx)); |
| } |
| match inline { |
| InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), |
| InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), |
| InlineAttr::Never => { |
| if cx.sess().target.arch != "amdgpu" { |
| Some(AttributeKind::NoInline.create_attr(cx.llcx)) |
| } else { |
| None |
| } |
| } |
| InlineAttr::None => None, |
| } |
| } |
| |
| /// Get LLVM sanitize attributes. |
| #[inline] |
| pub fn sanitize_attrs<'ll>( |
| cx: &CodegenCx<'ll, '_>, |
| no_sanitize: SanitizerSet, |
| ) -> SmallVec<[&'ll Attribute; 4]> { |
| let mut attrs = SmallVec::new(); |
| let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; |
| if enabled.contains(SanitizerSet::ADDRESS) || enabled.contains(SanitizerSet::KERNELADDRESS) { |
| attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::MEMORY) { |
| attrs.push(llvm::AttributeKind::SanitizeMemory.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::THREAD) { |
| attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::HWADDRESS) { |
| attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { |
| attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::MEMTAG) { |
| // Check to make sure the mte target feature is actually enabled. |
| let features = cx.tcx.global_backend_features(()); |
| let mte_feature = |
| features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); |
| if let None | Some("-mte") = mte_feature { |
| cx.tcx.sess.emit_err(SanitizerMemtagRequiresMte); |
| } |
| |
| attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); |
| } |
| if enabled.contains(SanitizerSet::SAFESTACK) { |
| attrs.push(llvm::AttributeKind::SanitizeSafeStack.create_attr(cx.llcx)); |
| } |
| attrs |
| } |
| |
| /// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. |
| #[inline] |
| pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { |
| // NOTE: We should determine if we even need async unwind tables, as they |
| // take have more overhead and if we can use sync unwind tables we |
| // probably should. |
| llvm::CreateUWTableAttr(llcx, true) |
| } |
| |
| pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| let mut fp = cx.sess().target.frame_pointer; |
| let opts = &cx.sess().opts; |
| // "mcount" function relies on stack pointer. |
| // See <https://sourceware.org/binutils/docs/gprof/Implementation.html>. |
| if opts.unstable_opts.instrument_mcount || matches!(opts.cg.force_frame_pointers, Some(true)) { |
| fp = FramePointer::Always; |
| } |
| let attr_value = match fp { |
| FramePointer::Always => "all", |
| FramePointer::NonLeaf => "non-leaf", |
| FramePointer::MayOmit => return None, |
| }; |
| Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) |
| } |
| |
| /// Tell LLVM what instrument function to insert. |
| #[inline] |
| fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> { |
| let mut attrs = SmallVec::new(); |
| if cx.sess().opts.unstable_opts.instrument_mcount { |
| // Similar to `clang -pg` behavior. Handled by the |
| // `post-inline-ee-instrument` LLVM pass. |
| |
| // The function name varies on platforms. |
| // See test/CodeGen/mcount.c in clang. |
| let mcount_name = match &cx.sess().target.llvm_mcount_intrinsic { |
| Some(llvm_mcount_intrinsic) => llvm_mcount_intrinsic.as_ref(), |
| None => cx.sess().target.mcount.as_ref(), |
| }; |
| |
| attrs.push(llvm::CreateAttrStringValue( |
| cx.llcx, |
| "instrument-function-entry-inlined", |
| &mcount_name, |
| )); |
| } |
| if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray { |
| // XRay instrumentation is similar to __cyg_profile_func_{enter,exit}. |
| // Function prologue and epilogue are instrumented with NOP sleds, |
| // a runtime library later replaces them with detours into tracing code. |
| if options.always { |
| attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-always")); |
| } |
| if options.never { |
| attrs.push(llvm::CreateAttrStringValue(cx.llcx, "function-instrument", "xray-never")); |
| } |
| if options.ignore_loops { |
| attrs.push(llvm::CreateAttrString(cx.llcx, "xray-ignore-loops")); |
| } |
| // LLVM will not choose the default for us, but rather requires specific |
| // threshold in absence of "xray-always". Use the same default as Clang. |
| let threshold = options.instruction_threshold.unwrap_or(200); |
| attrs.push(llvm::CreateAttrStringValue( |
| cx.llcx, |
| "xray-instruction-threshold", |
| &threshold.to_string(), |
| )); |
| if options.skip_entry { |
| attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-entry")); |
| } |
| if options.skip_exit { |
| attrs.push(llvm::CreateAttrString(cx.llcx, "xray-skip-exit")); |
| } |
| } |
| attrs |
| } |
| |
| fn nojumptables_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| if !cx.sess().opts.unstable_opts.no_jump_tables { |
| return None; |
| } |
| |
| Some(llvm::CreateAttrStringValue(cx.llcx, "no-jump-tables", "true")) |
| } |
| |
| fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| // Currently stack probes seem somewhat incompatible with the address |
| // sanitizer and thread sanitizer. With asan we're already protected from |
| // stack overflow anyway so we don't really need stack probes regardless. |
| if cx |
| .sess() |
| .opts |
| .unstable_opts |
| .sanitizer |
| .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) |
| { |
| return None; |
| } |
| |
| // probestack doesn't play nice either with `-C profile-generate`. |
| if cx.sess().opts.cg.profile_generate.enabled() { |
| return None; |
| } |
| |
| // probestack doesn't play nice either with gcov profiling. |
| if cx.sess().opts.unstable_opts.profile { |
| return None; |
| } |
| |
| let attr_value = match cx.sess().target.stack_probes { |
| StackProbeType::None => return None, |
| // Request LLVM to generate the probes inline. If the given LLVM version does not support |
| // this, no probe is generated at all (even if the attribute is specified). |
| StackProbeType::Inline => "inline-asm", |
| // Flag our internal `__rust_probestack` function as the stack probe symbol. |
| // This is defined in the `compiler-builtins` crate for each architecture. |
| StackProbeType::Call => "__rust_probestack", |
| // Pick from the two above based on the LLVM version. |
| StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { |
| if llvm_util::get_version() < min_llvm_version_for_inline { |
| "__rust_probestack" |
| } else { |
| "inline-asm" |
| } |
| } |
| }; |
| Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) |
| } |
| |
| fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| let sspattr = match cx.sess().stack_protector() { |
| StackProtector::None => return None, |
| StackProtector::All => AttributeKind::StackProtectReq, |
| StackProtector::Strong => AttributeKind::StackProtectStrong, |
| StackProtector::Basic => AttributeKind::StackProtect, |
| }; |
| |
| Some(sspattr.create_attr(cx.llcx)) |
| } |
| |
| pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { |
| let target_cpu = llvm_util::target_cpu(cx.tcx.sess); |
| llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) |
| } |
| |
| pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| llvm_util::tune_cpu(cx.tcx.sess) |
| .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) |
| } |
| |
| /// Get the `NonLazyBind` LLVM attribute, |
| /// if the codegen options allow skipping the PLT. |
| pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { |
| // Don't generate calls through PLT if it's not necessary |
| if !cx.sess().needs_plt() { |
| Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) |
| } else { |
| None |
| } |
| } |
| |
| /// Get the default optimizations attrs for a function. |
| #[inline] |
| pub(crate) fn default_optimisation_attrs<'ll>( |
| cx: &CodegenCx<'ll, '_>, |
| ) -> SmallVec<[&'ll Attribute; 2]> { |
| let mut attrs = SmallVec::new(); |
| match cx.sess().opts.optimize { |
| OptLevel::Size => { |
| attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); |
| } |
| OptLevel::SizeMin => { |
| attrs.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); |
| attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); |
| } |
| _ => {} |
| } |
| attrs |
| } |
| |
| fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { |
| llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") |
| } |
| |
| /// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) |
| /// attributes. |
| pub fn from_fn_attrs<'ll, 'tcx>( |
| cx: &CodegenCx<'ll, 'tcx>, |
| llfn: &'ll Value, |
| instance: ty::Instance<'tcx>, |
| ) { |
| let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); |
| |
| let mut to_add = SmallVec::<[_; 16]>::new(); |
| |
| match codegen_fn_attrs.optimize { |
| OptimizeAttr::None => { |
| to_add.extend(default_optimisation_attrs(cx)); |
| } |
| OptimizeAttr::Size => { |
| to_add.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); |
| to_add.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); |
| } |
| OptimizeAttr::Speed => {} |
| } |
| |
| let inline = |
| if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { |
| InlineAttr::Hint |
| } else { |
| codegen_fn_attrs.inline |
| }; |
| to_add.extend(inline_attr(cx, inline)); |
| |
| // The `uwtable` attribute according to LLVM is: |
| // |
| // This attribute indicates that the ABI being targeted requires that an |
| // unwind table entry be produced for this function even if we can show |
| // that no exceptions passes by it. This is normally the case for the |
| // ELF x86-64 abi, but it can be disabled for some compilation units. |
| // |
| // Typically when we're compiling with `-C panic=abort` (which implies this |
| // `no_landing_pads` check) we don't need `uwtable` because we can't |
| // generate any exceptions! On Windows, however, exceptions include other |
| // events such as illegal instructions, segfaults, etc. This means that on |
| // Windows we end up still needing the `uwtable` attribute even if the `-C |
| // panic=abort` flag is passed. |
| // |
| // You can also find more info on why Windows always requires uwtables here: |
| // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 |
| if cx.sess().must_emit_unwind_tables() { |
| to_add.push(uwtable_attr(cx.llcx)); |
| } |
| |
| if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { |
| to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); |
| } |
| |
| // FIXME: none of these three functions interact with source level attributes. |
| to_add.extend(frame_pointer_type_attr(cx)); |
| to_add.extend(instrument_function_attr(cx)); |
| to_add.extend(nojumptables_attr(cx)); |
| to_add.extend(probestack_attr(cx)); |
| to_add.extend(stackprotector_attr(cx)); |
| |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_BUILTINS) { |
| to_add.push(llvm::CreateAttrString(cx.llcx, "no-builtins")); |
| } |
| |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { |
| to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { |
| to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { |
| to_add.push(MemoryEffects::ReadOnly.create_attr(cx.llcx)); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { |
| to_add.push(MemoryEffects::None.create_attr(cx.llcx)); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { |
| to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); |
| // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. |
| // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. |
| // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 |
| to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); |
| // Need this for AArch64. |
| to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) |
| || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) |
| { |
| to_add.push(create_alloc_family_attr(cx.llcx)); |
| // apply to argument place instead of function |
| let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); |
| to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); |
| let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { |
| flags |= AllocKindFlags::Uninitialized; |
| } else { |
| flags |= AllocKindFlags::Zeroed; |
| } |
| to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); |
| // apply to return place instead of function (unlike all other attributes applied in this function) |
| let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { |
| to_add.push(create_alloc_family_attr(cx.llcx)); |
| to_add.push(llvm::CreateAllocKindAttr( |
| cx.llcx, |
| AllocKindFlags::Realloc | AllocKindFlags::Aligned, |
| )); |
| // applies to argument place instead of function place |
| let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); |
| // apply to argument place instead of function |
| let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); |
| to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); |
| let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { |
| to_add.push(create_alloc_family_attr(cx.llcx)); |
| to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); |
| // applies to argument place instead of function place |
| let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); |
| attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); |
| } |
| if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { |
| to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); |
| } |
| if let Some(align) = codegen_fn_attrs.alignment { |
| llvm::set_alignment(llfn, align as usize); |
| } |
| to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); |
| |
| // Always annotate functions with the target-cpu they are compiled for. |
| // Without this, ThinLTO won't inline Rust functions into Clang generated |
| // functions (because Clang annotates functions this way too). |
| to_add.push(target_cpu_attr(cx)); |
| // tune-cpu is only conveyed through the attribute for our purpose. |
| // The target doesn't care; the subtarget reads our attribute. |
| to_add.extend(tune_cpu_attr(cx)); |
| |
| let function_features = |
| codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::<Vec<&str>>(); |
| |
| if let Some(f) = llvm_util::check_tied_features( |
| cx.tcx.sess, |
| &function_features.iter().map(|f| (*f, true)).collect(), |
| ) { |
| let span = cx |
| .tcx |
| .get_attrs(instance.def_id(), sym::target_feature) |
| .next() |
| .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); |
| cx.tcx |
| .sess |
| .create_err(TargetFeatureDisableOrEnable { |
| features: f, |
| span: Some(span), |
| missing_features: Some(MissingFeatures), |
| }) |
| .emit(); |
| return; |
| } |
| |
| let mut function_features = function_features |
| .iter() |
| .flat_map(|feat| { |
| llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{f}")) |
| }) |
| .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { |
| InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), |
| InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), |
| })) |
| .collect::<Vec<String>>(); |
| |
| if cx.tcx.sess.target.is_like_wasm { |
| // If this function is an import from the environment but the wasm |
| // import has a specific module/name, apply them here. |
| if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { |
| to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", &module)); |
| |
| let name = |
| codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); |
| let name = name.as_str(); |
| to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); |
| } |
| |
| // The `"wasm"` abi on wasm targets automatically enables the |
| // `+multivalue` feature because the purpose of the wasm abi is to match |
| // the WebAssembly specification, which has this feature. This won't be |
| // needed when LLVM enables this `multivalue` feature by default. |
| if !cx.tcx.is_closure(instance.def_id()) { |
| let abi = cx.tcx.fn_sig(instance.def_id()).skip_binder().abi(); |
| if abi == Abi::Wasm { |
| function_features.push("+multivalue".to_string()); |
| } |
| } |
| } |
| |
| let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); |
| let function_features = function_features.iter().map(|s| s.as_str()); |
| let target_features: String = |
| global_features.chain(function_features).intersperse(",").collect(); |
| if !target_features.is_empty() { |
| to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); |
| } |
| |
| attributes::apply_to_llfn(llfn, Function, &to_add); |
| } |
| |
| fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<&String> { |
| tcx.wasm_import_module_map(id.krate).get(&id) |
| } |
