diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index dd2477503fd6d..197a90235d03b 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -25,7 +25,8 @@ struct EvaluatedCalleeAndArgs<'tcx, M: Machine<'tcx>> { callee: FnVal<'tcx, M::ExtraFnVal>, args: Vec>, fn_sig: ty::FnSig<'tcx>, - fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, + /// None if LLVM intrinsic + fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>, /// True if the function is marked as `#[track_caller]` ([`ty::InstanceKind::requires_caller_location`]) with_caller_location: bool, } @@ -480,13 +481,17 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { ty::FnPtr(..) => { let fn_ptr = self.read_pointer(&func)?; let fn_val = self.get_ptr_fn(fn_ptr)?; - (fn_val, self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?, false) + (fn_val, Some(self.fn_abi_of_fn_ptr(fn_sig_binder, extra_args)?), false) } ty::FnDef(def_id, args) => { let instance = self.resolve(def_id, args.no_bound_vars().unwrap())?; + // Don't compute FnAbi for LLVM intrinsics. Trying to that would panic. + let has_fn_abi = !matches!(instance.def, ty::InstanceKind::LlvmIntrinsic(_)); ( FnVal::Instance(instance), - self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?, + has_fn_abi + .then(|| self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)) + .transpose()?, instance.def.requires_caller_location(*self.tcx), ) } @@ -557,15 +562,34 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = self.eval_callee_and_args(terminator, func, args, &destination)?; - self.init_fn_call( - callee, - (fn_sig.abi(), fn_abi), - &args, - with_caller_location, - &dest_place, - target, - if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable }, - )?; + match callee { + FnVal::Instance( + instance + @ ty::Instance { def: ty::InstanceKind::LlvmIntrinsic(_), args: _ }, + ) => { + // FIXME: Should `InPlace` arguments be reset to uninit? + M::call_llvm_intrinsic( + self, + instance, + &Self::copy_fn_args(&args), + &dest_place, + target, + )?; + } + _ => { + let fn_abi = fn_abi.expect("FnAbi should have been computed for this call"); + self.init_fn_call( + callee, + (fn_sig.abi(), fn_abi), + &args, + with_caller_location, + &dest_place, + target, + if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable }, + )?; + } + }; + // Sanity-check that `eval_fn_call` either pushed a new frame or // did a jump to another block. We disable the sanity check for functions that // can't return, since Miri sometimes does have to keep the location the same @@ -581,6 +605,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } = self.eval_callee_and_args(terminator, func, args, &mir::Place::return_place())?; + let fn_abi = fn_abi.expect("FnAbi should have been computed for this call"); self.init_fn_tail_call( callee, diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs index 4479ce2dba08b..4211a2443d8b8 100644 --- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs +++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs @@ -222,6 +222,9 @@ fn check_call_site_abi<'tcx>( args.no_bound_vars().unwrap(), DUMMY_SP, ); + if let InstanceKind::LlvmIntrinsic(..) = instance.def { + return; + } tcx.fn_abi_of_instance(typing_env.as_query_input((instance, ty::List::empty()))) } _ => { diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs index 578a101b5f29a..32ac2d9c7f132 100644 --- a/compiler/rustc_target/src/callconv/mod.rs +++ b/compiler/rustc_target/src/callconv/mod.rs @@ -426,20 +426,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> { PassMode::Indirect { attrs, meta_attrs, on_stack: false } } - /// Pass this argument directly instead. Should NOT be used! - /// Only exists because of past ABI mistakes that will take time to fix - /// (see ). - #[track_caller] - pub fn make_direct_deprecated(&mut self) { - match self.mode { - PassMode::Indirect { .. } => { - self.mode = PassMode::Direct(ArgAttributes::new()); - } - PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => {} // already direct - _ => panic!("Tried to make {:?} direct", self.mode), - } - } - /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead. /// This is valid for both sized and unsized arguments. #[track_caller] diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs index dd69afec47975..3f28b90c9095e 100644 --- a/compiler/rustc_target/src/spec/abi_map.rs +++ b/compiler/rustc_target/src/spec/abi_map.rs @@ -97,6 +97,8 @@ impl AbiMap { // infallible lowerings (ExternAbi::C { .. }, _) => CanonAbi::C, (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust, + + // Dummy mapping to prevent reporting an error in the frontend (ExternAbi::Unadjusted, _) => CanonAbi::C, (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust, diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index ad8918f70a582..91cbe76c64ec4 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -433,7 +433,6 @@ fn fn_abi_sanity_check<'tcx>( fn fn_arg_sanity_check<'tcx>( cx: &LayoutCx<'tcx>, - fn_abi: &FnAbi<'tcx, Ty<'tcx>>, spec_abi: ExternAbi, arg: &ArgAbi<'tcx, Ty<'tcx>>, is_ret: bool, @@ -466,8 +465,7 @@ fn fn_abi_sanity_check<'tcx>( PassMode::Direct(attrs) => { // Here the Rust type is used to determine the actual ABI, so we have to be very // careful. Scalar/Vector is fine, since backends will generally use - // `layout.backend_repr` and ignore everything else. We should just reject - //`Aggregate` entirely here, but some targets need to be fixed first. + // `layout.backend_repr` and ignore everything else. match arg.layout.backend_repr { BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } @@ -475,19 +473,9 @@ fn fn_abi_sanity_check<'tcx>( BackendRepr::ScalarPair { .. } => { panic!("`PassMode::Direct` used for ScalarPair type {}", arg.layout.ty) } - BackendRepr::Memory { sized } => { - // For an unsized type we'd only pass the sized prefix, so there is no universe - // in which we ever want to allow this. - assert!(sized, "`PassMode::Direct` for unsized type in ABI: {:#?}", fn_abi); - - // This really shouldn't happen even for sized aggregates, since - // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an - // LLVM type. This means all sorts of Rust type details leak into the ABI. - // The unadjusted ABI however uses Direct for all args. It is ill-specified, - // but unfortunately we need it for calling certain LLVM intrinsics. - assert!( - matches!(spec_abi, ExternAbi::Unadjusted), - "`PassMode::Direct` for aggregates only allowed for \"unadjusted\"\n\ + BackendRepr::Memory { .. } => { + panic!( + "`PassMode::Direct` for aggregates not allowed\n\ Problematic type: {:#?}", arg.layout, ); @@ -539,9 +527,9 @@ fn fn_abi_sanity_check<'tcx>( } for arg in fn_abi.args.iter() { - fn_arg_sanity_check(cx, fn_abi, spec_abi, arg, false); + fn_arg_sanity_check(cx, spec_abi, arg, false); } - fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret, true); + fn_arg_sanity_check(cx, spec_abi, &fn_abi.ret, true); } #[tracing::instrument( @@ -636,26 +624,13 @@ fn fn_abi_adjust_for_abi<'tcx>( fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>, abi: ExternAbi, ) { - if abi == ExternAbi::Unadjusted { - // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for - // some LLVM intrinsics. - fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) { - // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended, - // but who knows what breaks if we change this now. - if matches!(arg.layout.backend_repr, BackendRepr::Memory { .. }) { - assert!( - arg.layout.backend_repr.is_sized(), - "'unadjusted' ABI does not support unsized arguments" - ); - } - arg.make_direct_deprecated(); - } + assert_ne!( + abi, + ExternAbi::Unadjusted, + "fn_abi_of_instance should not be called on LLVM intrinsics" + ); - unadjust(&mut fn_abi.ret); - for arg in fn_abi.args.iter_mut() { - unadjust(arg); - } - } else if abi.is_rustic_abi() { + if abi.is_rustic_abi() { fn_abi.adjust_for_rust_abi(cx); } else { fn_abi.adjust_for_foreign_abi(cx, abi); diff --git a/library/compiler-builtins/compiler-builtins/src/float/conv.rs b/library/compiler-builtins/compiler-builtins/src/float/conv.rs index 7310cd4ec4f02..6193aa416e222 100644 --- a/library/compiler-builtins/compiler-builtins/src/float/conv.rs +++ b/library/compiler-builtins/compiler-builtins/src/float/conv.rs @@ -228,16 +228,26 @@ intrinsics! { f64::from_bits(int_to_float::u64_to_f64_bits(i)) } - #[cfg_attr(target_os = "uefi", unadjusted_on_win64)] + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floatuntisf(i: u128) -> f32 { f32::from_bits(int_to_float::u128_to_f32_bits(i)) } - #[cfg_attr(target_os = "uefi", unadjusted_on_win64)] + #[cfg(all(target_os = "uefi", target_arch = "x86_64"))] + pub extern "C" fn __floatuntisf(lo: u64, hi: u64) -> f32 { + f32::from_bits(int_to_float::u128_to_f32_bits((u128::from(hi) << 64) | u128::from(lo))) + } + + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floatuntidf(i: u128) -> f64 { f64::from_bits(int_to_float::u128_to_f64_bits(i)) } + #[cfg(all(target_os = "uefi", target_arch = "x86_64"))] + pub extern "C" fn __floatuntidf(lo: u64, hi: u64) -> f64 { + f64::from_bits(int_to_float::u128_to_f64_bits((u128::from(hi) << 64) | u128::from(lo))) + } + #[ppc_alias = __floatunsikf] #[cfg(f128_enabled)] pub extern "C" fn __floatunsitf(i: u32) -> f128 { @@ -279,16 +289,26 @@ intrinsics! { int_to_float::signed(i, int_to_float::u64_to_f64_bits) } - #[cfg_attr(target_os = "uefi", unadjusted_on_win64)] + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floattisf(i: i128) -> f32 { int_to_float::signed(i, int_to_float::u128_to_f32_bits) } - #[cfg_attr(target_os = "uefi", unadjusted_on_win64)] + #[cfg(all(target_os = "uefi", target_arch = "x86_64"))] + pub extern "C" fn __floattisf(lo: u64, hi: u64) -> f32 { + int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f32_bits) + } + + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floattidf(i: i128) -> f64 { int_to_float::signed(i, int_to_float::u128_to_f64_bits) } + #[cfg(all(target_os = "uefi", target_arch = "x86_64"))] + pub extern "C" fn __floattidf(lo: u64, hi: u64) -> f64 { + int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f64_bits) + } + #[ppc_alias = __floatsikf] #[cfg(f128_enabled)] pub extern "C" fn __floatsitf(i: i32) -> f128 { diff --git a/library/compiler-builtins/compiler-builtins/src/int/mul.rs b/library/compiler-builtins/compiler-builtins/src/int/mul.rs index 9331ab60a4fc1..de5ee96867025 100644 --- a/library/compiler-builtins/compiler-builtins/src/int/mul.rs +++ b/library/compiler-builtins/compiler-builtins/src/int/mul.rs @@ -173,13 +173,29 @@ intrinsics! { mul } - #[unadjusted_on_win64] + #[cfg(not(all( + any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), + target_pointer_width = "64", + )))] pub extern "C" fn __muloti4(a: i128, b: i128, oflow: &mut i32) -> i128 { let (mul, o) = i128_overflowing_mul(a, b); *oflow = o as i32; mul } + #[cfg(all( + any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), + target_pointer_width = "64", + ))] + pub extern "C" fn f(a_lo: u64, a_hi: u64, b_lo: u64, b_hi: u64, oflow: &mut i32) -> i128 { + let (mul, o) = i128_overflowing_mul( + (i128::from(a_hi) << 64) | i128::from(a_lo), + (i128::from(b_hi) << 64) | i128::from(b_lo), + ); + *oflow = o as i32; + mul + } + pub extern "C" fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128 { let (mul, o) = i128_overflowing_mul(a, b); *oflow = o.into(); diff --git a/library/compiler-builtins/compiler-builtins/src/macros.rs b/library/compiler-builtins/compiler-builtins/src/macros.rs index c5e49f7780641..25bdbcf3f975e 100644 --- a/library/compiler-builtins/compiler-builtins/src/macros.rs +++ b/library/compiler-builtins/compiler-builtins/src/macros.rs @@ -108,6 +108,25 @@ macro_rules! intrinsics { intrinsics!($($rest)*); ); + // Support cfg: + ( + #[cfg($e:meta)] + $(#[$($attrs:tt)*])* + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + $($rest:tt)* + ) => ( + #[cfg($e)] + intrinsics! { + $(#[$($attrs)*])* + pub extern $abi fn $name($($argname: $ty),*) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); // Right now there's a bunch of architecture-optimized intrinsics in the // stock compiler-rt implementation. Not all of these have been ported over @@ -182,36 +201,6 @@ macro_rules! intrinsics { intrinsics!($($rest)*); ); - // Like aapcs above we recognize an attribute for the "unadjusted" abi on - // win64 for some methods. - ( - #[unadjusted_on_win64] - $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { - $($body:tt)* - } - - $($rest:tt)* - ) => ( - #[cfg(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64"))] - intrinsics! { - $(#[$($attr)*])* - pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? { - $($body)* - } - } - - #[cfg(not(all(any(windows, target_os = "cygwin", all(target_os = "uefi", target_arch = "x86_64")), target_pointer_width = "64")))] - intrinsics! { - $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { - $($body)* - } - } - - intrinsics!($($rest)*); - ); - // `arm_aeabi_alias` would conflict with `f16_apple_{arg,ret}_abi` not handled here. Avoid macro ambiguity by combining in a // single `#[]`. ( diff --git a/tests/codegen-llvm/const-vector.rs b/tests/codegen-llvm/const-vector.rs index 0459d0145e26f..40bfc58f8d584 100644 --- a/tests/codegen-llvm/const-vector.rs +++ b/tests/codegen-llvm/const-vector.rs @@ -1,4 +1,5 @@ //@ revisions: OPT0 OPT0_S390X +//@ min-llvm-version: 22 //@ [OPT0] ignore-s390x //@ [OPT0_S390X] only-s390x //@ [OPT0] compile-flags: -C no-prepopulate-passes -Copt-level=0 @@ -9,6 +10,7 @@ #![crate_type = "lib"] #![feature(abi_unadjusted)] #![feature(const_trait_impl)] +#![feature(link_llvm_intrinsics)] #![feature(repr_simd)] #![feature(rustc_attrs)] #![feature(simd_ffi)] @@ -19,26 +21,26 @@ #[path = "../auxiliary/minisimd.rs"] mod minisimd; -use minisimd::{PackedSimd as Simd, f32x2, i8x2}; +use minisimd::{PackedSimd, Simd, f32x2, i8x2}; // The following functions are required for the tests to ensure // that they are called with a const vector extern "unadjusted" { - fn test_i8x2(a: i8x2); - fn test_i8x2_two_args(a: i8x2, b: i8x2); - fn test_i8x2_mixed_args(a: i8x2, c: i32, b: i8x2); - fn test_i8x2_arr(a: i8x2); - fn test_f32x2(a: f32x2); - fn test_f32x2_arr(a: f32x2); - fn test_simd(a: Simd); - fn test_simd_unaligned(a: Simd); + #[link_name = "llvm.vector.reduce.add.v2i8"] + fn test_i8x2(a: i8x2) -> i8; + #[link_name = "llvm.vector.partial.reduce.add.v2i8.v2i8.v2i8"] + fn test_i8x2_two_args(a: i8x2, b: i8x2) -> i8x2; + #[link_name = "llvm.vector.insert.v2i8.v2i8"] + fn test_i8x2_mixed_args(a: i8x2, b: i8x2, c: u64) -> i8x2; + #[link_name = "llvm.vector.reduce.fadd.v2f32"] + fn test_f32x2(a: f32, b: f32x2) -> f32; + #[link_name = "llvm.vector.reduce.add.v4i32"] + fn test_simd4(a: PackedSimd) -> i32; + #[link_name = "llvm.vector.reduce.add.v3i32"] + fn test_simd3(a: Simd) -> i32; } -// Ensure the packed variant of the simd struct does not become a const vector -// if the size is not a power of 2 -// CHECK: %"minisimd::PackedSimd" = type { [3 x i32] } - #[cfg_attr(target_family = "wasm", target_feature(enable = "simd128"))] #[cfg_attr(target_arch = "arm", target_feature(enable = "neon"))] #[cfg_attr(target_arch = "x86", target_feature(enable = "sse"))] @@ -46,36 +48,29 @@ extern "unadjusted" { #[cfg_attr(target_arch = "riscv64", target_feature(enable = "v"))] pub fn do_call() { unsafe { - // CHECK: call void @test_i8x2(<2 x i8> + // CHECK: call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> test_i8x2(const { i8x2::from_array([32, 64]) }); - // CHECK: call void @test_i8x2_two_args(<2 x i8> , <2 x i8> + // CHECK: call <2 x i8> @llvm.vector.partial.reduce.add.v2i8.v2i8(<2 x i8> , <2 x i8> ) test_i8x2_two_args( const { i8x2::from_array([32, 64]) }, const { i8x2::from_array([8, 16]) }, ); - // CHECK: call void @test_i8x2_mixed_args(<2 x i8> , i32 43, <2 x i8> + // CHECK: call <2 x i8> @llvm.vector.insert.v2i8.v2i8(<2 x i8> , <2 x i8> , i64 0 test_i8x2_mixed_args( const { i8x2::from_array([32, 64]) }, - 43, const { i8x2::from_array([8, 16]) }, + 0, ); - // CHECK: call void @test_i8x2_arr(<2 x i8> - test_i8x2_arr(const { i8x2::from_array([32, 64]) }); - - // CHECK: call void @test_f32x2(<2 x float> - test_f32x2(const { f32x2::from_array([0.32, 0.64]) }); - - // CHECK: void @test_f32x2_arr(<2 x float> - test_f32x2_arr(const { f32x2::from_array([0.32, 0.64]) }); + // CHECK: call float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> + test_f32x2(0.0, const { f32x2::from_array([0.32, 0.64]) }); - // CHECK: call void @test_simd(<4 x i32> - test_simd(const { Simd::([2, 4, 6, 8]) }); + // CHECK: call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> + test_simd4(const { PackedSimd::([2, 4, 6, 8]) }); - // CHECK: [[UNALIGNED_ARG:%.*]] = load %"minisimd::PackedSimd", ptr @anon{{.*}} - // CHECK-NEXT: call void @test_simd_unaligned(%"minisimd::PackedSimd" [[UNALIGNED_ARG]] - test_simd_unaligned(const { Simd::([2, 4, 6]) }); + // CHECK: call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> + test_simd3(const { Simd::([2, 4, 6]) }); } } diff --git a/tests/codegen-llvm/simd/unpadded-simd.rs b/tests/codegen-llvm/simd/unpadded-simd.rs index ef067a15702c3..64f8921b1fd75 100644 --- a/tests/codegen-llvm/simd/unpadded-simd.rs +++ b/tests/codegen-llvm/simd/unpadded-simd.rs @@ -1,19 +1,21 @@ // Make sure that no 0-sized padding is inserted in structs and that // structs are represented as expected by Neon intrinsics in LLVM. // See #87254. +//@ only-aarch64 +//@ compile-flags: -Cno-prepopulate-passes #![crate_type = "lib"] -#![feature(repr_simd, abi_unadjusted)] +#![feature(abi_unadjusted, link_llvm_intrinsics)] -#[derive(Copy, Clone)] -#[repr(simd)] -pub struct int16x4_t(pub [i16; 4]); +use std::arch::aarch64::int16x4x2_t; -#[derive(Copy, Clone)] -pub struct int16x4x2_t(pub int16x4_t, pub int16x4_t); +unsafe extern "unadjusted" { + #[link_name = "llvm.aarch64.neon.ld1x2.v4i16.p0"] + fn vld1_s16_x2(t: *const i16) -> int16x4x2_t; +} -// CHECK: %int16x4x2_t = type { <4 x i16>, <4 x i16> } #[no_mangle] -extern "unadjusted" fn takes_int16x4x2_t(t: int16x4x2_t) -> int16x4x2_t { - t +unsafe extern "C" fn returns_int16x4x2_t(a: *const i16) -> int16x4x2_t { + // CHECK: call { <4 x i16>, <4 x i16> } @llvm.aarch64.neon.ld1x2.v4i16.p0 + vld1_s16_x2(a) }