abi: Emit a clean error for non-pointer x86-interrupt first argument

src/abi.rs

@@ -16,6 +16,8 @@ use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
 use rustc_target::spec::Arch;
 
 use crate::builder::Builder;
+#[cfg(feature = "master")]
+use crate::common::type_is_pointer;
 use crate::context::CodegenCx;
 use crate::type_of::LayoutGccExt;
 
@@ -277,3 +279,33 @@ pub fn conv_to_fn_attribute<'gcc>(conv: CanonAbi, arch: &Arch) -> Option<FnAttri
     };
     Some(attribute)
 }
+
+/// GCC's `interrupt` attribute requires the first parameter of an `x86-interrupt`
+/// function to be a pointer. When it is not, libgccjit aborts codegen with an internal
+/// error and leaves no object file behind. Detect that case so the backend can emit a
+/// clean diagnostic and skip the attribute instead of crashing (issue #833).
+///
+/// This works on the already-lowered GCC argument types so callers that have them (e.g.
+/// `declare_fn`) avoid a redundant ABI lowering. `x86-interrupt` functions return `()`,
+/// so there is no struct-return pointer prepended and `arguments_type[0]` is the frame
+/// argument.
+#[cfg(feature = "master")]
+pub fn x86_interrupt_first_arg_is_invalid<'gcc>(
+    conv: CanonAbi,
+    arguments_type: &[Type<'gcc>],
+) -> bool {
+    matches!(conv, CanonAbi::Interrupt(InterruptKind::X86))
+        && arguments_type.first().is_none_or(|ty| !type_is_pointer(*ty))
+}
+
+/// Convenience wrapper around [`x86_interrupt_first_arg_is_invalid`] for callers that only
+/// hold the `FnAbi` and must lower it themselves. Short-circuits before lowering for the
+/// common non-`x86-interrupt` case.
+#[cfg(feature = "master")]
+pub fn x86_interrupt_has_invalid_first_arg<'gcc, 'tcx>(
+    cx: &CodegenCx<'gcc, 'tcx>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+) -> bool {
+    matches!(fn_abi.conv, CanonAbi::Interrupt(InterruptKind::X86))
+        && x86_interrupt_first_arg_is_invalid(fn_abi.conv, &fn_abi.gcc_type(cx).arguments_type)
+}

src/declare.rs

@@ -6,7 +6,9 @@ use rustc_middle::ty::Ty;
 use rustc_span::Symbol;
 use rustc_target::callconv::FnAbi;
 
-use crate::abi::{FnAbiGcc, FnAbiGccExt};
+use crate::abi::FnAbiGccExt;
+#[cfg(feature = "master")]
+use crate::abi::x86_interrupt_first_arg_is_invalid;
 use crate::context::CodegenCx;
 
 impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
@@ -110,22 +112,31 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> {
-        let FnAbiGcc {
-            return_type,
-            arguments_type,
-            is_c_variadic,
-            on_stack_param_indices,
-            #[cfg(feature = "master")]
-            fn_attributes,
-        } = fn_abi.gcc_type(self);
+        // Lower the ABI once and reuse it below, including for the `x86-interrupt` check,
+        // instead of lowering it a second time inside a helper.
+        let fn_abi_gcc = fn_abi.gcc_type(self);
         #[cfg(feature = "master")]
-        let conv = fn_abi.gcc_cconv(self);
+        let conv = if x86_interrupt_first_arg_is_invalid(fn_abi.conv, &fn_abi_gcc.arguments_type) {
+            // GCC rejects an `x86-interrupt` function whose first argument is not a
+            // pointer. Drop the calling-convention attribute so libgccjit does not abort;
+            // a clean error is emitted in `predefine_fn` instead (issue #833).
+            None
+        } else {
+            fn_abi.gcc_cconv(self)
+        };
         #[cfg(not(feature = "master"))]
         let conv = None;
-        let func = declare_raw_fn(self, name, conv, return_type, &arguments_type, is_c_variadic);
-        self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
+        let func = declare_raw_fn(
+            self,
+            name,
+            conv,
+            fn_abi_gcc.return_type,
+            &fn_abi_gcc.arguments_type,
+            fn_abi_gcc.is_c_variadic,
+        );
+        self.on_stack_function_params.borrow_mut().insert(func, fn_abi_gcc.on_stack_param_indices);
         #[cfg(feature = "master")]
-        for fn_attr in fn_attributes {
+        for fn_attr in fn_abi_gcc.fn_attributes {
             func.add_attribute(fn_attr);
         }
         func

src/errors.rs

@@ -30,3 +30,13 @@ pub(crate) struct NulBytesInAsm {
     #[primary_span]
     pub span: Span,
 }
+
+#[cfg(feature = "master")]
+#[derive(Diagnostic)]
+#[diag(
+    "the GCC backend requires the first argument of an `x86-interrupt` function to be a pointer"
+)]
+pub(crate) struct X86InterruptBadFirstArg {
+    #[primary_span]
+    pub span: Span,
+}

src/mono_item.rs

@@ -10,7 +10,11 @@ use rustc_middle::mono::Visibility;
 use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf};
 use rustc_middle::ty::{self, Instance, TypeVisitableExt};
 
+#[cfg(feature = "master")]
+use crate::abi::x86_interrupt_has_invalid_first_arg;
 use crate::context::CodegenCx;
+#[cfg(feature = "master")]
+use crate::errors::X86InterruptBadFirstArg;
 use crate::type_of::LayoutGccExt;
 use crate::{attributes, base};
 
@@ -51,6 +55,12 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         assert!(!instance.args.has_infer());
 
         let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty());
+        #[cfg(feature = "master")]
+        if x86_interrupt_has_invalid_first_arg(self, fn_abi) {
+            self.tcx
+                .dcx()
+                .emit_err(X86InterruptBadFirstArg { span: self.tcx.def_span(instance.def_id()) });
+        }
         self.linkage.set(base::linkage_to_gcc(linkage));
         let decl = self.declare_fn(symbol_name, fn_abi);
         //let attrs = self.tcx.codegen_instance_attrs(instance.def);

tests/compile/x86_interrupt_bad_first_arg.rs

@@ -0,0 +1,17 @@
+// Compiler:
+//   status: error
+//   stderr:
+//     error: the GCC backend requires the first argument of an `x86-interrupt` function to be a pointer
+//     ...
+
+// Test that an `x86-interrupt` function whose first argument is not a pointer emits a
+// clean error instead of an internal libgccjit error (issue #833).
+
+#![feature(abi_x86_interrupt)]
+
+pub extern "x86-interrupt" fn f(_a: i64) {}
+
+fn main() {
+    // Take the function's address so that it is codegened.
+    let _f: extern "x86-interrupt" fn(i64) = f;
+}

tests/lang_tests.rs

@@ -211,7 +211,13 @@ fn compile_tests(tempdir: PathBuf, current_dir: String) {
         "lang compile",
         "tests/compile",
         TestMode::Compile,
-        &["simd-ffi.rs", "asm_nul_byte.rs", "global_asm_nul_byte.rs", "naked_asm_nul_byte.rs"],
+        &[
+            "simd-ffi.rs",
+            "asm_nul_byte.rs",
+            "global_asm_nul_byte.rs",
+            "naked_asm_nul_byte.rs",
+            "x86_interrupt_bad_first_arg.rs",
+        ],
     );
 }