Skip to content

Commit 6df68f3

Browse files
beetreesbeetrees
committed
Fix RISC-V C function ABI when passing/returning structs containing floats
1 parent 40daf23 commit 6df68f3

File tree

13 files changed

+537
-134
lines changed

13 files changed

+537
-134
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,18 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -
4141
param
4242
}
4343

44-
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
44+
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[(Size, AbiParam); 2]> {
45+
if let Some(offset_from_start) = cast.rest_offset {
46+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
47+
assert_eq!(cast.rest.unit.size, cast.rest.total);
48+
let first = cast.prefix[0].unwrap();
49+
let second = cast.rest.unit;
50+
return smallvec![
51+
(Size::ZERO, reg_to_abi_param(first)),
52+
(offset_from_start, reg_to_abi_param(second))
53+
];
54+
}
55+
4556
let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
4657
(0, 0)
4758
} else {
@@ -56,25 +67,32 @@ fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
5667
// different types in Cranelift IR. Instead a single array of primitive types is used.
5768

5869
// Create list of fields in the main structure
59-
let mut args = cast
70+
let args = cast
6071
.prefix
6172
.iter()
6273
.flatten()
6374
.map(|&reg| reg_to_abi_param(reg))
64-
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
65-
.collect::<SmallVec<_>>();
75+
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)));
76+
77+
let mut res = SmallVec::new();
78+
let mut offset = Size::ZERO;
79+
80+
for arg in args {
81+
res.push((offset, arg));
82+
offset += Size::from_bytes(arg.value_type.bytes());
83+
}
6684

6785
// Append final integer
6886
if rem_bytes != 0 {
6987
// Only integers can be really split further.
7088
assert_eq!(cast.rest.unit.kind, RegKind::Integer);
71-
args.push(reg_to_abi_param(Reg {
72-
kind: RegKind::Integer,
73-
size: Size::from_bytes(rem_bytes),
74-
}));
89+
res.push((
90+
offset,
91+
reg_to_abi_param(Reg { kind: RegKind::Integer, size: Size::from_bytes(rem_bytes) }),
92+
));
7593
}
7694

77-
args
95+
res
7896
}
7997

8098
impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
@@ -105,7 +123,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
105123
},
106124
PassMode::Cast { ref cast, pad_i32 } => {
107125
assert!(!pad_i32, "padding support not yet implemented");
108-
cast_target_to_abi_params(cast)
126+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
109127
}
110128
PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
111129
if on_stack {
@@ -151,9 +169,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
151169
}
152170
_ => unreachable!("{:?}", self.layout.backend_repr),
153171
},
154-
PassMode::Cast { ref cast, .. } => {
155-
(None, cast_target_to_abi_params(cast).into_iter().collect())
156-
}
172+
PassMode::Cast { ref cast, .. } => (
173+
None,
174+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
175+
),
157176
PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => {
158177
assert!(!on_stack);
159178
(Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
@@ -172,12 +191,14 @@ pub(super) fn to_casted_value<'tcx>(
172191
) -> SmallVec<[Value; 2]> {
173192
let (ptr, meta) = arg.force_stack(fx);
174193
assert!(meta.is_none());
175-
let mut offset = 0;
176194
cast_target_to_abi_params(cast)
177195
.into_iter()
178-
.map(|param| {
179-
let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
180-
offset += i64::from(param.value_type.bytes());
196+
.map(|(offset, param)| {
197+
let val = ptr.offset_i64(fx, offset.bytes() as i64).load(
198+
fx,
199+
param.value_type,
200+
MemFlags::new(),
201+
);
181202
val
182203
})
183204
.collect()
@@ -190,7 +211,7 @@ pub(super) fn from_casted_value<'tcx>(
190211
cast: &CastTarget,
191212
) -> CValue<'tcx> {
192213
let abi_params = cast_target_to_abi_params(cast);
193-
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
214+
let abi_param_size: u32 = abi_params.iter().map(|(_, param)| param.value_type.bytes()).sum();
194215
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
195216
let ptr = fx.create_stack_slot(
196217
// Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
@@ -199,16 +220,13 @@ pub(super) fn from_casted_value<'tcx>(
199220
std::cmp::max(abi_param_size, layout_size),
200221
u32::try_from(layout.align.abi.bytes()).unwrap(),
201222
);
202-
let mut offset = 0;
203223
let mut block_params_iter = block_params.iter().copied();
204-
for param in abi_params {
205-
let val = ptr.offset_i64(fx, offset).store(
224+
for (offset, _) in abi_params {
225+
ptr.offset_i64(fx, offset.bytes() as i64).store(
206226
fx,
207227
block_params_iter.next().unwrap(),
208228
MemFlags::new(),
209-
);
210-
offset += i64::from(param.value_type.bytes());
211-
val
229+
)
212230
}
213231
assert_eq!(block_params_iter.next(), None, "Leftover block param");
214232
CValue::by_ref(ptr, layout)

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -626,7 +626,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
626626
bx.lifetime_start(llscratch, scratch_size);
627627

628628
// ... where we first store the value...
629-
bx.store(val, llscratch, scratch_align);
629+
rustc_codegen_ssa::mir::store_cast(bx, cast, val, llscratch, scratch_align);
630630

631631
// ... and then memcpy it to the intended destination.
632632
bx.memcpy(

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
229229
let llscratch = bx.alloca(scratch_size, scratch_align);
230230
bx.lifetime_start(llscratch, scratch_size);
231231
// ...store the value...
232-
bx.store(val, llscratch, scratch_align);
232+
rustc_codegen_ssa::mir::store_cast(bx, cast, val, llscratch, scratch_align);
233233
// ... and then memcpy it to the intended destination.
234234
bx.memcpy(
235235
dst.val.llval,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::cmp;
22

3-
use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
3+
use rustc_abi::{Align, BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
44
use rustc_ast as ast;
55
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
66
use rustc_data_structures::packed::Pu128;
@@ -13,7 +13,7 @@ use rustc_middle::{bug, span_bug};
1313
use rustc_session::config::OptLevel;
1414
use rustc_span::Span;
1515
use rustc_span::source_map::Spanned;
16-
use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
16+
use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode};
1717
use tracing::{debug, info};
1818

1919
use super::operand::OperandRef;
@@ -558,8 +558,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
558558
}
559559
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
560560
};
561-
let ty = bx.cast_backend_type(cast_ty);
562-
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
561+
load_cast(bx, cast_ty, llslot, self.fn_abi.ret.layout.align.abi)
563562
}
564563
};
565564
bx.ret(llval);
@@ -1618,8 +1617,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
16181617
MemFlags::empty(),
16191618
);
16201619
// ...and then load it with the ABI type.
1621-
let cast_ty = bx.cast_backend_type(cast);
1622-
llval = bx.load(cast_ty, llscratch, scratch_align);
1620+
llval = load_cast(bx, cast, llscratch, scratch_align);
16231621
bx.lifetime_end(llscratch, scratch_size);
16241622
} else {
16251623
// We can't use `PlaceRef::load` here because the argument
@@ -1969,3 +1967,47 @@ enum ReturnDest<'tcx, V> {
19691967
/// Store a direct return value to an operand local place.
19701968
DirectOperand(mir::Local),
19711969
}
1970+
1971+
fn load_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1972+
bx: &mut Bx,
1973+
cast: &CastTarget,
1974+
ptr: Bx::Value,
1975+
align: Align,
1976+
) -> Bx::Value {
1977+
let cast_ty = bx.cast_backend_type(cast);
1978+
if let Some(offset_from_start) = cast.rest_offset {
1979+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
1980+
assert_eq!(cast.rest.unit.size, cast.rest.total);
1981+
let first_ty = bx.reg_backend_type(&cast.prefix[0].unwrap());
1982+
let second_ty = bx.reg_backend_type(&cast.rest.unit);
1983+
let first = bx.load(first_ty, ptr, align);
1984+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
1985+
let second = bx.load(second_ty, second_ptr, align.restrict_for_offset(offset_from_start));
1986+
let res = bx.cx().const_poison(cast_ty);
1987+
let res = bx.insert_value(res, first, 0);
1988+
bx.insert_value(res, second, 1)
1989+
} else {
1990+
bx.load(cast_ty, ptr, align)
1991+
}
1992+
}
1993+
1994+
pub fn store_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1995+
bx: &mut Bx,
1996+
cast: &CastTarget,
1997+
value: Bx::Value,
1998+
ptr: Bx::Value,
1999+
align: Align,
2000+
) {
2001+
if let Some(offset_from_start) = cast.rest_offset {
2002+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
2003+
assert_eq!(cast.rest.unit.size, cast.rest.total);
2004+
assert!(cast.prefix[0].is_some());
2005+
let first = bx.extract_value(value, 0);
2006+
let second = bx.extract_value(value, 1);
2007+
bx.store(first, ptr, align);
2008+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
2009+
bx.store(second, second_ptr, align.restrict_for_offset(offset_from_start));
2010+
} else {
2011+
bx.store(value, ptr, align);
2012+
};
2013+
}

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub mod place;
2626
mod rvalue;
2727
mod statement;
2828

29+
pub use self::block::store_cast;
2930
use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
3031
use self::operand::{OperandRef, OperandValue};
3132
use self::place::PlaceRef;
@@ -259,7 +260,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
259260
}
260261
PassMode::Cast { ref cast, .. } => {
261262
debug!("alloc: {:?} (return place) -> place", local);
262-
let size = cast.size(&start_bx);
263+
let size = cast.size(&start_bx).max(layout.size);
263264
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
264265
}
265266
_ => {}

‎compiler/rustc_target/src/callconv/mips64.rs

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ use rustc_abi::{
22
BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
33
};
44

5-
use crate::callconv::{
6-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Uniform,
7-
};
5+
use crate::callconv::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
86

97
fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
108
// Always sign extend u32 values on 64-bit mips
@@ -140,16 +138,7 @@ where
140138

141139
// Extract first 8 chunks as the prefix
142140
let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
143-
arg.cast_to(CastTarget {
144-
prefix,
145-
rest: Uniform::new(Reg::i64(), rest_size),
146-
attrs: ArgAttributes {
147-
regular: ArgAttribute::default(),
148-
arg_ext: ArgExtension::None,
149-
pointee_size: Size::ZERO,
150-
pointee_align: None,
151-
},
152-
});
141+
arg.cast_to(CastTarget::prefixed(prefix, Uniform::new(Reg::i64(), rest_size)));
153142
}
154143

155144
pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)

0 commit comments

Comments
 (0)