Skip to content

Commit 86b24b8

Browse files
beetreesbeetrees
committed
Fix RISC-V C function ABI when passing/returning structs containing floats
1 parent 82eb03e commit 86b24b8

File tree

13 files changed

+528
-134
lines changed

13 files changed

+528
-134
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

Lines changed: 42 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,18 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -
3939
param
4040
}
4141

42-
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
42+
fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[(Size, AbiParam); 2]> {
43+
if let Some(offset_from_start) = cast.rest_offset {
44+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
45+
assert_eq!(cast.rest.unit.size, cast.rest.total);
46+
let first = cast.prefix[0].unwrap();
47+
let second = cast.rest.unit;
48+
return smallvec![
49+
(Size::ZERO, reg_to_abi_param(first)),
50+
(offset_from_start, reg_to_abi_param(second))
51+
];
52+
}
53+
4354
let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
4455
(0, 0)
4556
} else {
@@ -54,25 +65,32 @@ fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
5465
// different types in Cranelift IR. Instead a single array of primitive types is used.
5566

5667
// Create list of fields in the main structure
57-
let mut args = cast
68+
let args = cast
5869
.prefix
5970
.iter()
6071
.flatten()
6172
.map(|&reg| reg_to_abi_param(reg))
62-
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
63-
.collect::<SmallVec<_>>();
73+
.chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)));
74+
75+
let mut res = SmallVec::new();
76+
let mut offset = Size::ZERO;
77+
78+
for arg in args {
79+
res.push((offset, arg));
80+
offset += Size::from_bytes(arg.value_type.bytes());
81+
}
6482

6583
// Append final integer
6684
if rem_bytes != 0 {
6785
// Only integers can be really split further.
6886
assert_eq!(cast.rest.unit.kind, RegKind::Integer);
69-
args.push(reg_to_abi_param(Reg {
70-
kind: RegKind::Integer,
71-
size: Size::from_bytes(rem_bytes),
72-
}));
87+
res.push((
88+
offset,
89+
reg_to_abi_param(Reg { kind: RegKind::Integer, size: Size::from_bytes(rem_bytes) }),
90+
));
7391
}
7492

75-
args
93+
res
7694
}
7795

7896
impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
@@ -103,7 +121,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
103121
},
104122
PassMode::Cast { ref cast, pad_i32 } => {
105123
assert!(!pad_i32, "padding support not yet implemented");
106-
cast_target_to_abi_params(cast)
124+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
107125
}
108126
PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
109127
if on_stack {
@@ -149,9 +167,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
149167
}
150168
_ => unreachable!("{:?}", self.layout.backend_repr),
151169
},
152-
PassMode::Cast { ref cast, .. } => {
153-
(None, cast_target_to_abi_params(cast).into_iter().collect())
154-
}
170+
PassMode::Cast { ref cast, .. } => (
171+
None,
172+
cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
173+
),
155174
PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => {
156175
assert!(!on_stack);
157176
(Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
@@ -170,12 +189,14 @@ pub(super) fn to_casted_value<'tcx>(
170189
) -> SmallVec<[Value; 2]> {
171190
let (ptr, meta) = arg.force_stack(fx);
172191
assert!(meta.is_none());
173-
let mut offset = 0;
174192
cast_target_to_abi_params(cast)
175193
.into_iter()
176-
.map(|param| {
177-
let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
178-
offset += i64::from(param.value_type.bytes());
194+
.map(|(offset, param)| {
195+
let val = ptr.offset_i64(fx, offset.bytes() as i64).load(
196+
fx,
197+
param.value_type,
198+
MemFlags::new(),
199+
);
179200
val
180201
})
181202
.collect()
@@ -188,7 +209,7 @@ pub(super) fn from_casted_value<'tcx>(
188209
cast: &CastTarget,
189210
) -> CValue<'tcx> {
190211
let abi_params = cast_target_to_abi_params(cast);
191-
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
212+
let abi_param_size: u32 = abi_params.iter().map(|(_, param)| param.value_type.bytes()).sum();
192213
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
193214
let ptr = fx.create_stack_slot(
194215
// Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
@@ -197,16 +218,13 @@ pub(super) fn from_casted_value<'tcx>(
197218
std::cmp::max(abi_param_size, layout_size),
198219
u32::try_from(layout.align.abi.bytes()).unwrap(),
199220
);
200-
let mut offset = 0;
201221
let mut block_params_iter = block_params.iter().copied();
202-
for param in abi_params {
203-
let val = ptr.offset_i64(fx, offset).store(
222+
for (offset, _) in abi_params {
223+
ptr.offset_i64(fx, offset.bytes() as i64).store(
204224
fx,
205225
block_params_iter.next().unwrap(),
206226
MemFlags::new(),
207-
);
208-
offset += i64::from(param.value_type.bytes());
209-
val
227+
)
210228
}
211229
assert_eq!(block_params_iter.next(), None, "Leftover block param");
212230
CValue::by_ref(ptr, layout)

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -551,7 +551,7 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
551551
bx.lifetime_start(llscratch, scratch_size);
552552

553553
// ... where we first store the value...
554-
bx.store(val, llscratch, scratch_align);
554+
rustc_codegen_ssa::mir::store_cast(bx, cast, llscratch, scratch_align);
555555

556556
// ... and then memcpy it to the intended destination.
557557
bx.memcpy(

‎compiler/rustc_codegen_llvm/src/abi.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
234234
let llscratch = bx.alloca(scratch_size, scratch_align);
235235
bx.lifetime_start(llscratch, scratch_size);
236236
// ...store the value...
237-
bx.store(val, llscratch, scratch_align);
237+
rustc_codegen_ssa::mir::store_cast(bx, cast, val, llscratch, scratch_align);
238238
// ... and then memcpy it to the intended destination.
239239
bx.memcpy(
240240
dst.val.llval,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use std::cmp;
22

3-
use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
3+
use rustc_abi::{Align, BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
44
use rustc_ast as ast;
55
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
66
use rustc_data_structures::packed::Pu128;
@@ -13,7 +13,7 @@ use rustc_middle::{bug, span_bug};
1313
use rustc_session::config::OptLevel;
1414
use rustc_span::source_map::Spanned;
1515
use rustc_span::{Span, sym};
16-
use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
16+
use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode};
1717
use tracing::{debug, info};
1818

1919
use super::operand::OperandRef;
@@ -558,8 +558,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
558558
}
559559
ZeroSized => bug!("ZST return value shouldn't be in PassMode::Cast"),
560560
};
561-
let ty = bx.cast_backend_type(cast_ty);
562-
bx.load(ty, llslot, self.fn_abi.ret.layout.align.abi)
561+
load_cast(bx, cast_ty, llslot, self.fn_abi.ret.layout.align.abi)
563562
}
564563
};
565564
bx.ret(llval);
@@ -1587,8 +1586,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
15871586
MemFlags::empty(),
15881587
);
15891588
// ...and then load it with the ABI type.
1590-
let cast_ty = bx.cast_backend_type(cast);
1591-
llval = bx.load(cast_ty, llscratch, scratch_align);
1589+
llval = load_cast(bx, cast, llscratch, scratch_align);
15921590
bx.lifetime_end(llscratch, scratch_size);
15931591
} else {
15941592
// We can't use `PlaceRef::load` here because the argument
@@ -1944,3 +1942,47 @@ enum ReturnDest<'tcx, V> {
19441942
/// Store a direct return value to an operand local place.
19451943
DirectOperand(mir::Local),
19461944
}
1945+
1946+
fn load_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1947+
bx: &mut Bx,
1948+
cast: &CastTarget,
1949+
ptr: Bx::Value,
1950+
align: Align,
1951+
) -> Bx::Value {
1952+
let cast_ty = bx.cast_backend_type(cast);
1953+
if let Some(offset_from_start) = cast.rest_offset {
1954+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
1955+
assert_eq!(cast.rest.unit.size, cast.rest.total);
1956+
let first_ty = bx.reg_backend_type(&cast.prefix[0].unwrap());
1957+
let second_ty = bx.reg_backend_type(&cast.rest.unit);
1958+
let first = bx.load(first_ty, ptr, align);
1959+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
1960+
let second = bx.load(second_ty, second_ptr, align.restrict_for_offset(offset_from_start));
1961+
let res = bx.cx().const_poison(cast_ty);
1962+
let res = bx.insert_value(res, first, 0);
1963+
bx.insert_value(res, second, 1)
1964+
} else {
1965+
bx.load(cast_ty, ptr, align)
1966+
}
1967+
}
1968+
1969+
pub fn store_cast<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
1970+
bx: &mut Bx,
1971+
cast: &CastTarget,
1972+
value: Bx::Value,
1973+
ptr: Bx::Value,
1974+
align: Align,
1975+
) {
1976+
if let Some(offset_from_start) = cast.rest_offset {
1977+
assert!(cast.prefix[1..].iter().all(|p| p.is_none()));
1978+
assert_eq!(cast.rest.unit.size, cast.rest.total);
1979+
assert!(cast.prefix[0].is_some());
1980+
let first = bx.extract_value(value, 0);
1981+
let second = bx.extract_value(value, 1);
1982+
bx.store(first, ptr, align);
1983+
let second_ptr = bx.inbounds_ptradd(ptr, bx.const_usize(offset_from_start.bytes()));
1984+
bx.store(second, second_ptr, align.restrict_for_offset(offset_from_start));
1985+
} else {
1986+
bx.store(value, ptr, align);
1987+
};
1988+
}

‎compiler/rustc_codegen_ssa/src/mir/mod.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ pub mod place;
2626
mod rvalue;
2727
mod statement;
2828

29+
pub use self::block::store_cast;
2930
use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
3031
use self::operand::{OperandRef, OperandValue};
3132
use self::place::PlaceRef;
@@ -254,7 +255,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
254255
}
255256
PassMode::Cast { ref cast, .. } => {
256257
debug!("alloc: {:?} (return place) -> place", local);
257-
let size = cast.size(&start_bx);
258+
let size = cast.size(&start_bx).max(layout.size);
258259
return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
259260
}
260261
_ => {}

‎compiler/rustc_target/src/callconv/mips64.rs

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ use rustc_abi::{
22
BackendRepr, FieldsShape, Float, HasDataLayout, Primitive, Reg, Size, TyAbiInterface,
33
};
44

5-
use crate::callconv::{
6-
ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode, Uniform,
7-
};
5+
use crate::callconv::{ArgAbi, ArgExtension, CastTarget, FnAbi, PassMode, Uniform};
86

97
fn extend_integer_width_mips<Ty>(arg: &mut ArgAbi<'_, Ty>, bits: u64) {
108
// Always sign extend u32 values on 64-bit mips
@@ -140,16 +138,7 @@ where
140138

141139
// Extract first 8 chunks as the prefix
142140
let rest_size = size - Size::from_bytes(8) * prefix_index as u64;
143-
arg.cast_to(CastTarget {
144-
prefix,
145-
rest: Uniform::new(Reg::i64(), rest_size),
146-
attrs: ArgAttributes {
147-
regular: ArgAttribute::default(),
148-
arg_ext: ArgExtension::None,
149-
pointee_size: Size::ZERO,
150-
pointee_align: None,
151-
},
152-
});
141+
arg.cast_to(CastTarget::prefixed(prefix, Uniform::new(Reg::i64(), rest_size)));
153142
}
154143

155144
pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)

0 commit comments

Comments
 (0)