Don't compute FnAbi for LLVM intrinsics

compiler/rustc_const_eval/src/interpret/step.rs

@@ -25,7 +25,8 @@ struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> {
     callee: FnVal<'tcx, M::ExtraFnVal>,
     args: Vec<FnArg<'tcx, M::Provenance>>,
     fn_sig: ty::FnSig<'tcx>,
-    fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
+    /// None if LLVM intrinsic
+    fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
     /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`])
     with_caller_location: bool,
 }
@@ -480,13 +481,17 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             ty::FnPtr(..) => {
                 let fn_ptr = self.read_pointer(&func)?;
                 let fn_val = self.get_ptr_fn(fn_ptr)?;
-                (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false)
+                (fn_val, Some(self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?), false)
             }
             ty::FnDef(def_id, args) => {
                 let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?;
+                // Don't compute FnAbi for LLVM intrinsics. Trying to that would panic.
+                let has_fn_abi = !matches!(instance.def, ty::InstanceKind::LlvmIntrinsic(_));
                 (
                     FnVal::Instance(instance),
-                    self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?,
+                    has_fn_abi
+                        .then(|| self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args))
+                        .transpose()?,
                     instance.def.requires_caller_location(*self.tcx),
                 )
             }
@@ -557,15 +562,34 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
                     self.eval_callee_and_args(terminator, func, args, &destination)?;
 
-                self.init_fn_call(
-                    callee,
-                    (fn_sig.abi(), fn_abi),
-                    &args,
-                    with_caller_location,
-                    &dest_place,
-                    target,
-                    if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
-                )?;
+                match callee {
+                    FnVal::Instance(
+                        instance
+                        @ ty::Instance { def: ty::InstanceKind::LlvmIntrinsic(_), args: _ },
+                    ) => {
+                        // FIXME: Should `InPlace` arguments be reset to uninit?
+                        M::call_llvm_intrinsic(
+                            self,
+                            instance,
+                            &Self::copy_fn_args(&args),
+                            &dest_place,
+                            target,
+                        )?;
+                    }
+                    _ => {
+                        let fn_abi = fn_abi.expect("FnAbi should have been computed for this call");
+                        self.init_fn_call(
+                            callee,
+                            (fn_sig.abi(), fn_abi),
+                            &args,
+                            with_caller_location,
+                            &dest_place,
+                            target,
+                            if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable },
+                        )?;
+                    }
+                };
+
                 // Sanity-check that `eval_fn_call` either pushed a new frame or
                 // did a jump to another block. We disable the sanity check for functions that
                 // can't return, since Miri sometimes does have to keep the location the same
@@ -581,6 +605,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
                     self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?;
+                let fn_abi = fn_abi.expect("FnAbi should have been computed for this call");
 
                 self.init_fn_tail_call(
                     callee,

compiler/rustc_monomorphize/src/mono_checks/abi_check.rs

@@ -222,6 +222,9 @@ fn check_call_site_abi<'tcx>(
                 args.no_bound_vars().unwrap(),
                 DUMMY_SP,
             );
+            if let InstanceKind::LlvmIntrinsic(..) = instance.def {
+                return;
+            }
             tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty())))
         }
         _ => {

compiler/rustc_target/src/callconv/mod.rs

@@ -426,20 +426,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         PassMode::Indirect { attrs, meta_attrs, on_stack: false }
     }
 
-    /// Pass this argument directly instead. Should NOT be used!
-    /// Only exists because of past ABI mistakes that will take time to fix
-    /// (see <https://github.com/rust-lang/rust/issues/115666>).
-    #[track_caller]
-    pub fn make_direct_deprecated(&mut self) {
-        match self.mode {
-            PassMode::Indirect { .. } => {
-                self.mode = PassMode::Direct(ArgAttributes::new());
-            }
-            PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct
-            _ => panic!("Tried to make {:?} direct", self.mode),
-        }
-    }
-
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
     /// This is valid for both sized and unsized arguments.
     #[track_caller]

compiler/rustc_target/src/spec/abi_map.rs

@@ -97,6 +97,8 @@ impl AbiMap {
             // infallible lowerings
             (ExternAbi::C { .. }, _) => CanonAbi::C,
             (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust,
+
+            // Dummy mapping to prevent reporting an error in the frontend
             (ExternAbi::Unadjusted, _) => CanonAbi::C,
 
             (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,

compiler/rustc_ty_utils/src/abi.rs

@@ -433,7 +433,6 @@ fn fn_abi_sanity_check<'tcx>(
 
     fn fn_arg_sanity_check<'tcx>(
         cx: &LayoutCx<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         spec_abi: ExternAbi,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
         is_ret: bool,
@@ -466,28 +465,17 @@ fn fn_abi_sanity_check<'tcx>(
             PassMode::Direct(attrs) => {
                 // Here the Rust type is used to determine the actual ABI, so we have to be very
                 // careful. Scalar/Vector is fine, since backends will generally use
-                // `layout.backend_repr` and ignore everything else. We should just reject
-                //`Aggregate` entirely here, but some targets need to be fixed first.
+                // `layout.backend_repr` and ignore everything else.
                 match arg.layout.backend_repr {
                     BackendRepr::Scalar(_)
                     | BackendRepr::SimdVector { .. }
                     | BackendRepr::SimdScalableVector { .. } => {}
                     BackendRepr::ScalarPair { .. } => {
                         panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty)
                     }
-                    BackendRepr::Memory { sized } => {
-                        // For an unsized type we'd only pass the sized prefix, so there is no universe
-                        // in which we ever want to allow this.
-                        assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi);
-
-                        // This really shouldn't happen even for sized aggregates, since
-                        // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an
-                        // LLVM type. This means all sorts of Rust type details leak into the ABI.
-                        // The unadjusted ABI however uses Direct for all args. It is ill-specified,
-                        // but unfortunately we need it for calling certain LLVM intrinsics.
-                        assert!(
-                            matches!(spec_abi, ExternAbi::Unadjusted),
-                            "`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\
+                    BackendRepr::Memory { .. } => {
+                        panic!(
+                            "`PassMode::Direct` for aggregates not allowed\n\
                              Problematic type: {:#?}",
                             arg.layout,
                         );
@@ -539,9 +527,9 @@ fn fn_abi_sanity_check<'tcx>(
     }
 
     for arg in fn_abi.args.iter() {
-        fn_arg_sanity_check(cx, fn_abi, spec_abi, arg, false);
+        fn_arg_sanity_check(cx, spec_abi, arg, false);
     }
-    fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret, true);
+    fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret, true);
 }
 
 #[tracing::instrument(
@@ -636,26 +624,13 @@ fn fn_abi_adjust_for_abi<'tcx>(
     fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
     abi: ExternAbi,
 ) {
-    if abi == ExternAbi::Unadjusted {
-        // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for
-        // some LLVM intrinsics.
-        fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) {
-            // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended,
-            // but who knows what breaks if we change this now.
-            if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) {
-                assert!(
-                    arg.layout.backend_repr.is_sized(),
-                    "'unadjusted' ABI does not support unsized arguments"
-                );
-            }
-            arg.make_direct_deprecated();
-        }
+    assert_ne!(
+        abi,
+        ExternAbi::Unadjusted,
+        "fn_abi_of_instance should not be called on LLVM intrinsics"
+    );
 
-        unadjust(&mut fn_abi.ret);
-        for arg in fn_abi.args.iter_mut() {
-            unadjust(arg);
-        }
-    } else if abi.is_rustic_abi() {
+    if abi.is_rustic_abi() {
         fn_abi.adjust_for_rust_abi(cx);
     } else {
         fn_abi.adjust_for_foreign_abi(cx, abi);

library/compiler-builtins/compiler-builtins/src/float/conv.rs

@@ -228,16 +228,26 @@ intrinsics! {
         f64::from_bits(int_to_float::u64_to_f64_bits(i))
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floatuntisf(i: u128) -> f32 {
         f32::from_bits(int_to_float::u128_to_f32_bits(i))
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floatuntisf(lo: u64, hi: u64) -> f32 {
+        f32::from_bits(int_to_float::u128_to_f32_bits((u128::from(hi) << 64) | u128::from(lo)))
+    }
+
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floatuntidf(i: u128) -> f64 {
         f64::from_bits(int_to_float::u128_to_f64_bits(i))
     }
 
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floatuntidf(lo: u64, hi: u64) -> f64 {
+        f64::from_bits(int_to_float::u128_to_f64_bits((u128::from(hi) << 64) | u128::from(lo)))
+    }
+
     #[ppc_alias = __floatunsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatunsitf(i: u32) -> f128 {
@@ -279,16 +289,26 @@ intrinsics! {
         int_to_float::signed(i, int_to_float::u64_to_f64_bits)
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floattisf(i: i128) -> f32 {
         int_to_float::signed(i, int_to_float::u128_to_f32_bits)
     }
 
-    #[cfg_attr(target_os = "uefi", unadjusted_on_win64)]
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floattisf(lo: u64, hi: u64) -> f32 {
+        int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f32_bits)
+    }
+
+    #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))]
     pub extern "C" fn __floattidf(i: i128) -> f64 {
         int_to_float::signed(i, int_to_float::u128_to_f64_bits)
     }
 
+    #[cfg(all(target_os = "uefi", target_arch = "x86_64"))]
+    pub extern "C" fn __floattidf(lo: u64, hi: u64) -> f64 {
+        int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f64_bits)
+    }
+
     #[ppc_alias = __floatsikf]
     #[cfg(f128_enabled)]
     pub extern "C" fn __floatsitf(i: i32) -> f128 {

library/compiler-builtins/compiler-builtins/src/int/mul.rs

@@ -173,13 +173,29 @@ intrinsics! {
         mul
     }
 
-    #[unadjusted_on_win64]
+    #[cfg(not(all(
+        any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")),
+        target_pointer_width = "64",
+    )))]
     pub extern "C" fn __muloti4(a: i128, b: i128, oflow: &mut i32) -> i128 {
         let (mul, o) = i128_overflowing_mul(a, b);
         *oflow = o as i32;
         mul
     }
 
+    #[cfg(all(
+        any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")),
+        target_pointer_width = "64",
+    ))]
+    pub extern "C" fn f(a_lo: u64, a_hi: u64, b_lo: u64, b_hi: u64, oflow: &mut i32) -> i128 {
+        let (mul, o) = i128_overflowing_mul(
+            (i128::from(a_hi) << 64) | i128::from(a_lo),
+            (i128::from(b_hi) << 64) | i128::from(b_lo),
+        );
+        *oflow = o as i32;
+        mul
+    }
+
     pub extern "C" fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128 {
         let (mul, o) = i128_overflowing_mul(a, b);
         *oflow = o.into();

library/compiler-builtins/compiler-builtins/src/macros.rs

@@ -108,6 +108,25 @@ macro_rules! intrinsics {
 
         intrinsics!($($rest)*);
     );
+    // Support cfg:
+    (
+        #[cfg($e:meta)]
+        $(#[$($attrs:tt)*])*
+        pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? {
+            $($body:tt)*
+        }
+        $($rest:tt)*
+    ) => (
+        #[cfg($e)]
+        intrinsics! {
+            $(#[$($attrs)*])*
+            pub extern $abi fn $name($($argname: $ty),*) $(-> $ret)? {
+                $($body)*
+            }
+        }
+
+        intrinsics!($($rest)*);
+    );
 
     // Right now there's a bunch of architecture-optimized intrinsics in the
     // stock compiler-rt implementation. Not all of these have been ported over
@@ -182,36 +201,6 @@ macro_rules! intrinsics {
         intrinsics!($($rest)*);
     );
 
-    // Like aapcs above we recognize an attribute for the "unadjusted" abi on
-    // win64 for some methods.
-    (
-        #[unadjusted_on_win64]
-        $(#[$($attr:tt)*])*
-        pub extern $abi:tt fn $name:ident( $($argname:ident:  $ty:ty),* ) $(-> $ret:ty)? {
-            $($body:tt)*
-        }
-
-        $($rest:tt)*
-    ) => (
-        #[cfg(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64"))]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        #[cfg(not(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64")))]
-        intrinsics! {
-            $(#[$($attr)*])*
-            pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? {
-                $($body)*
-            }
-        }
-
-        intrinsics!($($rest)*);
-    );
-
     // `arm_aeabi_alias` would conflict with `f16_apple_{arg,ret}_abi` not handled here. Avoid macro ambiguity by combining in a
     // single `#[]`.
     (