add extern "custom" functions

folkertdev · folkertdev · commit 847b3579efdc · 2025-05-07T22:52:53.000+02:00
diff --git a/compiler/rustc_abi/src/extern_abi.rs b/compiler/rustc_abi/src/extern_abi.rs
@@ -70,6 +70,10 @@ pub enum ExternAbi {
     RustCold,
     RiscvInterruptM,
     RiscvInterruptS,
+    /// An ABI that rustc does not know how to call or define. Functions with this ABI can
+    /// only be created using `#[naked]` functions or `extern "custom"` blocks, and can only
+    /// be called from inline assembly.
+    Custom,
 }
 
 macro_rules! abi_impls {
@@ -141,6 +145,7 @@ abi_impls! {
             Win64 { unwind: false } =><= "win64",
             Win64 { unwind: true } =><= "win64-unwind",
             X86Interrupt =><= "x86-interrupt",
+            Custom =><= "custom",
     }
 }
 
diff --git a/compiler/rustc_ast_lowering/src/stability.rs b/compiler/rustc_ast_lowering/src/stability.rs
@@ -134,5 +134,8 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {
             feature: sym::cmse_nonsecure_entry,
             explain: GateReason::Experimental,
         }),
+        ExternAbi::Custom => {
+            Err(UnstableAbi { abi, feature: sym::abi_custom, explain: GateReason::Experimental })
+        }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -65,6 +65,10 @@ pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: Call
             sess.dcx().fatal("C-cmse-nonsecure-entry call conv is not yet implemented");
         }
 
+        Conv::Custom => {
+            unreachable!("functions with the {c:?} call conv cannot be called");
+        }
+
         Conv::Msp430Intr | Conv::GpuKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => {
             unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
         }
diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs
@@ -272,6 +272,9 @@ pub fn conv_to_fn_attribute<'gcc>(conv: Conv, arch: &str) -> Option<FnAttribute<
         Conv::X86VectorCall => return None,
         Conv::X86_64SysV => FnAttribute::SysvAbi,
         Conv::X86_64Win64 => FnAttribute::MsAbi,
+        Conv::Custom => {
+            panic!("functions with the \"custom\" calling convention cannot be called")
+        }
     };
     Some(attribute)
 }
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -676,6 +676,9 @@ impl llvm::CallConv {
             Conv::X86VectorCall => llvm::X86_VectorCall,
             Conv::X86_64SysV => llvm::X86_64_SysV,
             Conv::X86_64Win64 => llvm::X86_64_Win64,
+            Conv::Custom => {
+                panic!("functions with the \"custom\" calling convention cannot be called")
+            }
         }
     }
 }
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
@@ -354,6 +354,8 @@ declare_features! (
     (unstable, abi_avr_interrupt, "1.45.0", Some(69664)),
     /// Allows `extern "C-cmse-nonsecure-call" fn()`.
     (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391)),
+    /// Allows `extern "custom" fn()`.
+    (unstable, abi_custom, "CURRENT_RUSTC_VERSION", Some(140566)),
     /// Allows `extern "gpu-kernel" fn()`.
     (unstable, abi_gpu_kernel, "1.86.0", Some(135467)),
     /// Allows `extern "msp430-interrupt" fn()`.
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
@@ -960,6 +960,7 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
                     tcx.ensure_ok().check_mod_loops(module);
                     tcx.ensure_ok().check_mod_attrs(module);
                     tcx.ensure_ok().check_mod_naked_functions(module);
+                    tcx.ensure_ok().check_mod_custom_abi(module);
                     tcx.ensure_ok().check_mod_unstable_api_usage(module);
                 });
             },
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
@@ -1117,6 +1117,10 @@ rustc_queries! {
         desc { |tcx| "checking naked functions in {}", describe_as_module(key, tcx) }
     }
 
+    query check_mod_custom_abi(key: LocalModDefId) {
+        desc { |tcx| "checking use of `extern \"custom\"` in {}", describe_as_module(key, tcx) }
+    }
+
     query check_mod_privacy(key: LocalModDefId) {
         desc { |tcx| "checking privacy in {}", describe_as_module(key.to_local_def_id(), tcx) }
     }
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
@@ -1264,6 +1264,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | RiscvInterruptS
         | CCmseNonSecureCall
         | CCmseNonSecureEntry
+        | Custom
         | Unadjusted => false,
         Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
     }
diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
@@ -4,6 +4,22 @@
 -passes_see_issue =
     see issue #{$issue} <https://github.com/rust-lang/rust/issues/{$issue}> for more information
 
+passes_abi_custom_call =
+    functions with the `"custom"` ABI cannot be called
+    .note = an `extern "custom"` function can only be called from within inline assembly
+
+passes_abi_custom_clothed_function =
+    functions with the `"custom"` ABI must be naked
+    .suggestion = add the `#[unsafe(naked)]` attribute to this function
+
+passes_abi_custom_safe_foreign_function =
+    foreign functions with the `"custom"` ABI cannot be safe
+    .suggestion = remove the `safe` keyword from this definition
+
+passes_abi_custom_safe_function =
+    functions with the `"custom"` ABI must be unsafe
+    .suggestion = add the `unsafe` keyword this function definition
+
 passes_abi_invalid_attribute =
     `#[rustc_abi]` can only be applied to function items, type aliases, and associated functions
 passes_abi_ne =
diff --git a/compiler/rustc_passes/src/custom_abi.rs b/compiler/rustc_passes/src/custom_abi.rs
@@ -0,0 +1,153 @@
+use rustc_abi::ExternAbi;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{LocalDefId, LocalModDefId};
+use rustc_hir::intravisit::Visitor;
+use rustc_hir::{self as hir, ExprKind, FnSig};
+use rustc_middle::hir::nested_filter::OnlyBodies;
+use rustc_middle::query::Providers;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::{BytePos, sym};
+
+use crate::errors::{
+    AbiCustomCall, AbiCustomClothedFunction, AbiCustomSafeForeignFunction, AbiCustomSafeFunction,
+};
+
+pub(crate) fn provide(providers: &mut Providers) {
+    *providers = Providers { check_mod_custom_abi, ..*providers };
+}
+
+fn check_mod_custom_abi(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
+    if !tcx.features().abi_custom() {
+        return;
+    }
+
+    let items = tcx.hir_module_items(module_def_id);
+    for def_id in items.definitions() {
+        let def_kind = tcx.def_kind(def_id);
+
+        // An `extern "custom"` function cannot be marked as `safe`.
+        if let DefKind::ForeignMod = def_kind
+            && let hir::Node::Item(item) = tcx.hir_node_by_def_id(def_id)
+            && let hir::ItemKind::ForeignMod { abi: ExternAbi::Custom, items } = item.kind
+        {
+            for item in items {
+                let hir_id = item.id.hir_id();
+                if let hir::Node::ForeignItem(foreign_item) = tcx.hir_node(hir_id)
+                    && let hir::ForeignItemKind::Fn(sig, _, _) = foreign_item.kind
+                    && sig.header.is_safe()
+                {
+                    let len = "safe ".len() as u32;
+                    let safe_span = sig.span.shrink_to_lo().with_hi(sig.span.lo() + BytePos(len));
+                    tcx.dcx().emit_err(AbiCustomSafeForeignFunction { span: sig.span, safe_span });
+                }
+            }
+        }
+
+        // An `extern "custom"` function cannot be a `const fn`, because `naked_asm!` cannot be
+        // evaluated at compile time, and `extern` blocks cannot declare `const fn` functions.
+        // Therefore, to find all calls to `extern "custom"` functions, it suffices to traverse
+        // all function bodies (i.e. we can skip `const` and `static` initializers).
+        if !matches!(def_kind, DefKind::Fn | DefKind::AssocFn) {
+            continue;
+        }
+
+        let body = match tcx.hir_node_by_def_id(def_id) {
+            hir::Node::Item(hir::Item {
+                kind: hir::ItemKind::Fn { sig, body: body_id, .. },
+                ..
+            })
+            | hir::Node::ImplItem(hir::ImplItem {
+                kind: hir::ImplItemKind::Fn(sig, body_id),
+                ..
+            }) => {
+                check_signature(tcx, def_id, sig, true);
+                tcx.hir_body(*body_id)
+            }
+            hir::Node::TraitItem(hir::TraitItem {
+                kind: hir::TraitItemKind::Fn(sig, trait_fn),
+                ..
+            }) => match trait_fn {
+                hir::TraitFn::Required(_) => {
+                    check_signature(tcx, def_id, sig, false);
+                    continue;
+                }
+                hir::TraitFn::Provided(body_id) => {
+                    check_signature(tcx, def_id, sig, true);
+                    tcx.hir_body(*body_id)
+                }
+            },
+            _ => continue,
+        };
+
+        let mut visitor = CheckCustomAbi { tcx };
+        visitor.visit_body(body);
+    }
+}
+
+fn check_signature<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId, sig: &FnSig<'tcx>, has_body: bool) {
+    if sig.header.abi == ExternAbi::Custom {
+        // Function definitions that use `extern "custom"` must be naked functions.
+        if has_body && !tcx.has_attr(def_id, sym::naked) {
+            tcx.dcx().emit_err(AbiCustomClothedFunction {
+                span: sig.span,
+                naked_span: sig.span.shrink_to_lo(),
+            });
+        }
+
+        // Function definitions that use `extern "custom"` must unsafe.
+        if sig.header.is_safe() {
+            tcx.dcx().emit_err(AbiCustomSafeFunction {
+                span: sig.span,
+                unsafe_span: sig.span.shrink_to_lo(),
+            });
+        }
+    }
+}
+
+struct CheckCustomAbi<'tcx> {
+    tcx: TyCtxt<'tcx>,
+}
+
+impl<'tcx> Visitor<'tcx> for CheckCustomAbi<'tcx> {
+    type NestedFilter = OnlyBodies;
+
+    fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt {
+        self.tcx
+    }
+
+    fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
+        let opt_span_and_abi = match expr.kind {
+            ExprKind::Call(fun, _) => {
+                let fun_ty = self.tcx.typeck(fun.hir_id.owner.def_id).node_type(fun.hir_id);
+
+                match *fun_ty.kind() {
+                    ty::FnDef(def_id, _) => {
+                        let sig = self.tcx.fn_sig(def_id).skip_binder().skip_binder();
+                        Some((expr.span, sig.abi))
+                    }
+                    ty::FnPtr(_, header) => Some((expr.span, header.abi)),
+                    _ => None,
+                }
+            }
+
+            ExprKind::MethodCall(_, receiver, _, span) => {
+                let opt_def_id = self
+                    .tcx
+                    .typeck(receiver.hir_id.owner.def_id)
+                    .type_dependent_def_id(expr.hir_id);
+
+                opt_def_id.map(|def_id| {
+                    let sig = self.tcx.fn_sig(def_id).skip_binder().skip_binder();
+                    (span, sig.abi)
+                })
+            }
+            _ => None,
+        };
+
+        if let Some((span, ExternAbi::Custom)) = opt_span_and_abi {
+            self.tcx.dcx().emit_err(AbiCustomCall { span });
+        }
+
+        hir::intravisit::walk_expr(self, expr);
+    }
+}
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
@@ -1044,6 +1044,57 @@ pub(crate) struct AbiInvalidAttribute {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(passes_abi_custom_safe_foreign_function)]
+pub(crate) struct AbiCustomSafeForeignFunction {
+    #[primary_span]
+    pub span: Span,
+
+    #[suggestion(
+        passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "",
+        style = "verbose"
+    )]
+    pub safe_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_abi_custom_safe_function)]
+pub(crate) struct AbiCustomSafeFunction {
+    #[primary_span]
+    pub span: Span,
+
+    #[suggestion(
+        passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "unsafe ",
+        style = "verbose"
+    )]
+    pub unsafe_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_abi_custom_clothed_function)]
+pub(crate) struct AbiCustomClothedFunction {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(
+        passes_suggestion,
+        applicability = "maybe-incorrect",
+        code = "#[unsafe(naked)]\n",
+        style = "short"
+    )]
+    pub naked_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_abi_custom_call)]
+pub(crate) struct AbiCustomCall {
+    #[primary_span]
+    pub span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(passes_unrecognized_argument)]
 pub(crate) struct UnrecognizedArgument {
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
@@ -20,6 +20,7 @@ use rustc_middle::query::Providers;
 pub mod abi_test;
 mod check_attr;
 mod check_export;
+mod custom_abi;
 pub mod dead;
 mod debugger_visualizer;
 mod diagnostic_items;
@@ -51,6 +52,7 @@ pub fn provide(providers: &mut Providers) {
     lib_features::provide(providers);
     loops::provide(providers);
     naked_functions::provide(providers);
+    custom_abi::provide(providers);
     liveness::provide(providers);
     reachable::provide(providers);
     stability::provide(providers);
diff --git a/compiler/rustc_smir/src/rustc_internal/internal.rs b/compiler/rustc_smir/src/rustc_internal/internal.rs
@@ -496,6 +496,7 @@ impl RustcInternal for Abi {
             Abi::RustCold => rustc_abi::ExternAbi::RustCold,
             Abi::RiscvInterruptM => rustc_abi::ExternAbi::RiscvInterruptM,
             Abi::RiscvInterruptS => rustc_abi::ExternAbi::RiscvInterruptS,
+            Abi::Custom => rustc_abi::ExternAbi::Custom,
         }
     }
 }
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
@@ -117,6 +117,7 @@ impl<'tcx> Stable<'tcx> for callconv::Conv {
             Conv::AvrInterrupt => CallConvention::AvrInterrupt,
             Conv::AvrNonBlockingInterrupt => CallConvention::AvrNonBlockingInterrupt,
             Conv::RiscvInterrupt { .. } => CallConvention::RiscvInterrupt,
+            Conv::Custom => CallConvention::Custom,
         }
     }
 }
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -879,6 +879,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::ExternAbi {
             ExternAbi::RustCold => Abi::RustCold,
             ExternAbi::RiscvInterruptM => Abi::RiscvInterruptM,
             ExternAbi::RiscvInterruptS => Abi::RiscvInterruptS,
+            ExternAbi::Custom => Abi::Custom,
         }
     }
 }
diff --git a/compiler/rustc_smir/src/stable_mir/abi.rs b/compiler/rustc_smir/src/stable_mir/abi.rs
@@ -430,6 +430,8 @@ pub enum CallConvention {
     PreserveMost,
     PreserveAll,
 
+    Custom,
+
     // Target-specific calling conventions.
     ArmAapcs,
     CCmseNonSecureCall,
diff --git a/compiler/rustc_smir/src/stable_mir/ty.rs b/compiler/rustc_smir/src/stable_mir/ty.rs
@@ -1098,6 +1098,7 @@ pub enum Abi {
     RustCold,
     RiscvInterruptM,
     RiscvInterruptS,
+    Custom,
 }
 
 /// A binder represents a possibly generic type and its bound vars.
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
@@ -413,6 +413,7 @@ symbols! {
         abi_amdgpu_kernel,
         abi_avr_interrupt,
         abi_c_cmse_nonsecure_call,
+        abi_custom,
         abi_efiapi,
         abi_gpu_kernel,
         abi_msp430_interrupt,
diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs
diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
diff --git a/tests/ui/abi/bad-custom.rs b/tests/ui/abi/bad-custom.rs
diff --git a/tests/ui/abi/bad-custom.stderr b/tests/ui/abi/bad-custom.stderr
diff --git a/tests/ui/abi/custom.rs b/tests/ui/abi/custom.rs
diff --git a/tests/ui/feature-gates/feature-gate-abi-custom.rs b/tests/ui/feature-gates/feature-gate-abi-custom.rs
diff --git a/tests/ui/feature-gates/feature-gate-abi-custom.stderr b/tests/ui/feature-gates/feature-gate-abi-custom.stderr

Original file line number	Diff line number	Diff line change
`@@ -70,6 +70,10 @@ pub enum ExternAbi {`
`70`	`70`	`RustCold,`
`71`	`71`	`RiscvInterruptM,`
`72`	`72`	`RiscvInterruptS,`
	`73`	`+ /// An ABI that rustc does not know how to call or define. Functions with this ABI can`
	`74`	+ /// only be created using `#[naked]` functions or `extern "custom"` blocks, and can only
	`75`	`+ /// be called from inline assembly.`
	`76`	`+ Custom,`
`73`	`77`	`}`
`74`	`78`
`75`	`79`	`macro_rules! abi_impls {`
`@@ -141,6 +145,7 @@ abi_impls! {`
`141`	`145`	`Win64 { unwind: false } =><= "win64",`
`142`	`146`	`Win64 { unwind: true } =><= "win64-unwind",`
`143`	`147`	`X86Interrupt =><= "x86-interrupt",`
	`148`	`+ Custom =><= "custom",`
`144`	`149`	`}`
`145`	`150`	`}`
`146`	`151`
Original file line number	Diff line number	Diff line change
`@@ -134,5 +134,8 @@ pub fn extern_abi_stability(abi: ExternAbi) -> Result<(), UnstableAbi> {`
`134`	`134`	`feature: sym::cmse_nonsecure_entry,`
`135`	`135`	`explain: GateReason::Experimental,`
`136`	`136`	`}),`
	`137`	`+ ExternAbi::Custom => {`
	`138`	`+ Err(UnstableAbi { abi, feature: sym::abi_custom, explain: GateReason::Experimental })`
	`139`	`+ }`
`137`	`140`	`}`
`138`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,10 @@ pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: Call`
`65`	`65`	`sess.dcx().fatal("C-cmse-nonsecure-entry call conv is not yet implemented");`
`66`	`66`	`}`
`67`	`67`
	`68`	`+ Conv::Custom => {`
	`69`	`+ unreachable!("functions with the {c:?} call conv cannot be called");`
	`70`	`+ }`
	`71`	`+`
`68`	`72`	`Conv::Msp430Intr \| Conv::GpuKernel \| Conv::AvrInterrupt \| Conv::AvrNonBlockingInterrupt => {`
`69`	`73`	`unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");`
`70`	`74`	`}`
Original file line number	Diff line number	Diff line change
`@@ -676,6 +676,9 @@ impl llvm::CallConv {`
`676`	`676`	`Conv::X86VectorCall => llvm::X86_VectorCall,`
`677`	`677`	`Conv::X86_64SysV => llvm::X86_64_SysV,`
`678`	`678`	`Conv::X86_64Win64 => llvm::X86_64_Win64,`
	`679`	`+ Conv::Custom => {`
	`680`	`+ panic!("functions with the \"custom\" calling convention cannot be called")`
	`681`	`+ }`
`679`	`682`	`}`
`680`	`683`	`}`
`681`	`684`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1117,6 +1117,10 @@ rustc_queries! {`
`1117`	`1117`	`desc { \|tcx\| "checking naked functions in {}", describe_as_module(key, tcx) }`
`1118`	`1118`	`}`
`1119`	`1119`
	`1120`	`+ query check_mod_custom_abi(key: LocalModDefId) {`
	`1121`	+ desc { \|tcx\| "checking use of `extern \"custom\"` in {}", describe_as_module(key, tcx) }
	`1122`	`+ }`
	`1123`	`+`
`1120`	`1124`	`query check_mod_privacy(key: LocalModDefId) {`
`1121`	`1125`	`desc { \|tcx\| "checking privacy in {}", describe_as_module(key.to_local_def_id(), tcx) }`
`1122`	`1126`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1264,6 +1264,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)`
`1264`	`1264`	`\| RiscvInterruptS`
`1265`	`1265`	`\| CCmseNonSecureCall`
`1266`	`1266`	`\| CCmseNonSecureEntry`
	`1267`	`+ \| Custom`
`1267`	`1268`	`\| Unadjusted => false,`
`1268`	`1269`	`Rust \| RustCall \| RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,`
`1269`	`1270`	`}`