Skip to content

Commit 5f6bd6e

Browse files
borsbors
committed
Auto merge of #74850 - TimDiekmann:remove-in-place-alloc, r=Amanieu
Remove in-place allocation and revert to separate methods for zeroed allocations closes rust-lang/wg-allocators#58
2 parents c9b80bb + 6395659 commit 5f6bd6e

File tree

14 files changed

+391
-396
lines changed

14 files changed

+391
-396
lines changed

‎library/alloc/src/alloc.rs

+89-64
Original file line numberDiff line numberDiff line change
@@ -164,25 +164,34 @@ pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
164164
#[unstable(feature = "allocator_api", issue = "32838")]
165165
unsafe impl AllocRef for Global {
166166
#[inline]
167-
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
168-
unsafe {
169-
let size = layout.size();
170-
if size == 0 {
171-
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
172-
} else {
173-
let raw_ptr = match init {
174-
AllocInit::Uninitialized => alloc(layout),
175-
AllocInit::Zeroed => alloc_zeroed(layout),
176-
};
177-
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
178-
Ok(MemoryBlock { ptr, size })
179-
}
180-
}
167+
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
168+
let size = layout.size();
169+
let ptr = if size == 0 {
170+
layout.dangling()
171+
} else {
172+
// SAFETY: `layout` is non-zero in size,
173+
unsafe { NonNull::new(alloc(layout)).ok_or(AllocErr)? }
174+
};
175+
Ok(MemoryBlock { ptr, size })
176+
}
177+
178+
#[inline]
179+
fn alloc_zeroed(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
180+
let size = layout.size();
181+
let ptr = if size == 0 {
182+
layout.dangling()
183+
} else {
184+
// SAFETY: `layout` is non-zero in size,
185+
unsafe { NonNull::new(alloc_zeroed(layout)).ok_or(AllocErr)? }
186+
};
187+
Ok(MemoryBlock { ptr, size })
181188
}
182189

183190
#[inline]
184191
unsafe fn dealloc(&mut self, ptr: NonNull<u8>, layout: Layout) {
185192
if layout.size() != 0 {
193+
// SAFETY: `layout` is non-zero in size,
194+
// other conditions must be upheld by the caller
186195
unsafe { dealloc(ptr.as_ptr(), layout) }
187196
}
188197
}
@@ -193,38 +202,55 @@ unsafe impl AllocRef for Global {
193202
ptr: NonNull<u8>,
194203
layout: Layout,
195204
new_size: usize,
196-
placement: ReallocPlacement,
197-
init: AllocInit,
198205
) -> Result<MemoryBlock, AllocErr> {
199-
let size = layout.size();
200206
debug_assert!(
201-
new_size >= size,
202-
"`new_size` must be greater than or equal to `memory.size()`"
207+
new_size >= layout.size(),
208+
"`new_size` must be greater than or equal to `layout.size()`"
203209
);
204210

205-
if size == new_size {
206-
return Ok(MemoryBlock { ptr, size });
211+
// SAFETY: `new_size` must be non-zero, which is checked in the match expression.
212+
// Other conditions must be upheld by the caller
213+
unsafe {
214+
match layout.size() {
215+
old_size if old_size == new_size => Ok(MemoryBlock { ptr, size: new_size }),
216+
0 => self.alloc(Layout::from_size_align_unchecked(new_size, layout.align())),
217+
old_size => {
218+
// `realloc` probably checks for `new_size > size` or something similar.
219+
intrinsics::assume(new_size > old_size);
220+
let raw_ptr = realloc(ptr.as_ptr(), layout, new_size);
221+
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
222+
Ok(MemoryBlock { ptr, size: new_size })
223+
}
224+
}
207225
}
226+
}
208227

209-
match placement {
210-
ReallocPlacement::InPlace => Err(AllocErr),
211-
ReallocPlacement::MayMove if layout.size() == 0 => {
212-
let new_layout =
213-
unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
214-
self.alloc(new_layout, init)
215-
}
216-
ReallocPlacement::MayMove => {
217-
// `realloc` probably checks for `new_size > size` or something similar.
218-
let ptr = unsafe {
219-
intrinsics::assume(new_size > size);
220-
realloc(ptr.as_ptr(), layout, new_size)
221-
};
222-
let memory =
223-
MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size };
224-
unsafe {
225-
init.init_offset(memory, size);
228+
#[inline]
229+
unsafe fn grow_zeroed(
230+
&mut self,
231+
ptr: NonNull<u8>,
232+
layout: Layout,
233+
new_size: usize,
234+
) -> Result<MemoryBlock, AllocErr> {
235+
debug_assert!(
236+
new_size >= layout.size(),
237+
"`new_size` must be greater than or equal to `layout.size()`"
238+
);
239+
240+
// SAFETY: `new_size` must be non-zero, which is checked in the match expression.
241+
// Other conditions must be upheld by the caller
242+
unsafe {
243+
match layout.size() {
244+
old_size if old_size == new_size => Ok(MemoryBlock { ptr, size: new_size }),
245+
0 => self.alloc_zeroed(Layout::from_size_align_unchecked(new_size, layout.align())),
246+
old_size => {
247+
// `realloc` probably checks for `new_size > size` or something similar.
248+
intrinsics::assume(new_size > old_size);
249+
let raw_ptr = realloc(ptr.as_ptr(), layout, new_size);
250+
raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
251+
let ptr = NonNull::new(raw_ptr).ok_or(AllocErr)?;
252+
Ok(MemoryBlock { ptr, size: new_size })
226253
}
227-
Ok(memory)
228254
}
229255
}
230256
}
@@ -235,35 +261,34 @@ unsafe impl AllocRef for Global {
235261
ptr: NonNull<u8>,
236262
layout: Layout,
237263
new_size: usize,
238-
placement: ReallocPlacement,
239264
) -> Result<MemoryBlock, AllocErr> {
240-
let size = layout.size();
265+
let old_size = layout.size();
241266
debug_assert!(
242-
new_size <= size,
243-
"`new_size` must be smaller than or equal to `memory.size()`"
267+
new_size <= old_size,
268+
"`new_size` must be smaller than or equal to `layout.size()`"
244269
);
245270

246-
if size == new_size {
247-
return Ok(MemoryBlock { ptr, size });
248-
}
249-
250-
match placement {
251-
ReallocPlacement::InPlace => Err(AllocErr),
252-
ReallocPlacement::MayMove if new_size == 0 => {
253-
unsafe {
254-
self.dealloc(ptr, layout);
255-
}
256-
Ok(MemoryBlock { ptr: layout.dangling(), size: 0 })
271+
let ptr = if new_size == old_size {
272+
ptr
273+
} else if new_size == 0 {
274+
// SAFETY: `layout` is non-zero in size as `old_size` != `new_size`
275+
// Other conditions must be upheld by the caller
276+
unsafe {
277+
self.dealloc(ptr, layout);
257278
}
258-
ReallocPlacement::MayMove => {
259-
// `realloc` probably checks for `new_size < size` or something similar.
260-
let ptr = unsafe {
261-
intrinsics::assume(new_size < size);
262-
realloc(ptr.as_ptr(), layout, new_size)
263-
};
264-
Ok(MemoryBlock { ptr: NonNull::new(ptr).ok_or(AllocErr)?, size: new_size })
265-
}
266-
}
279+
layout.dangling()
280+
} else {
281+
// SAFETY: new_size is not zero,
282+
// Other conditions must be upheld by the caller
283+
let raw_ptr = unsafe {
284+
// `realloc` probably checks for `new_size < old_size` or something similar.
285+
intrinsics::assume(new_size < old_size);
286+
realloc(ptr.as_ptr(), layout, new_size)
287+
};
288+
NonNull::new(raw_ptr).ok_or(AllocErr)?
289+
};
290+
291+
Ok(MemoryBlock { ptr, size: new_size })
267292
}
268293
}
269294

@@ -274,7 +299,7 @@ unsafe impl AllocRef for Global {
274299
#[inline]
275300
unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
276301
let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
277-
match Global.alloc(layout, AllocInit::Uninitialized) {
302+
match Global.alloc(layout) {
278303
Ok(memory) => memory.ptr.as_ptr(),
279304
Err(_) => handle_alloc_error(layout),
280305
}

‎library/alloc/src/alloc/tests.rs

+2-3
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@ use test::Bencher;
88
fn allocate_zeroed() {
99
unsafe {
1010
let layout = Layout::from_size_align(1024, 1).unwrap();
11-
let memory = Global
12-
.alloc(layout.clone(), AllocInit::Zeroed)
13-
.unwrap_or_else(|_| handle_alloc_error(layout));
11+
let memory =
12+
Global.alloc_zeroed(layout.clone()).unwrap_or_else(|_| handle_alloc_error(layout));
1413

1514
let mut i = memory.ptr.cast::<u8>().as_ptr();
1615
let end = i.add(layout.size());

‎library/alloc/src/boxed.rs

+4-7
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ use core::pin::Pin;
146146
use core::ptr::{self, Unique};
147147
use core::task::{Context, Poll};
148148

149-
use crate::alloc::{self, AllocInit, AllocRef, Global};
149+
use crate::alloc::{self, AllocRef, Global};
150150
use crate::borrow::Cow;
151151
use crate::raw_vec::RawVec;
152152
use crate::str::from_boxed_utf8_unchecked;
@@ -197,11 +197,8 @@ impl<T> Box<T> {
197197
#[unstable(feature = "new_uninit", issue = "63291")]
198198
pub fn new_uninit() -> Box<mem::MaybeUninit<T>> {
199199
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
200-
let ptr = Global
201-
.alloc(layout, AllocInit::Uninitialized)
202-
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
203-
.ptr
204-
.cast();
200+
let ptr =
201+
Global.alloc(layout).unwrap_or_else(|_| alloc::handle_alloc_error(layout)).ptr.cast();
205202
unsafe { Box::from_raw(ptr.as_ptr()) }
206203
}
207204

@@ -227,7 +224,7 @@ impl<T> Box<T> {
227224
pub fn new_zeroed() -> Box<mem::MaybeUninit<T>> {
228225
let layout = alloc::Layout::new::<mem::MaybeUninit<T>>();
229226
let ptr = Global
230-
.alloc(layout, AllocInit::Zeroed)
227+
.alloc_zeroed(layout)
231228
.unwrap_or_else(|_| alloc::handle_alloc_error(layout))
232229
.ptr
233230
.cast();

‎library/alloc/src/raw_vec.rs

+22-22
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,20 @@ use core::ops::Drop;
88
use core::ptr::{NonNull, Unique};
99
use core::slice;
1010

11-
use crate::alloc::{
12-
handle_alloc_error,
13-
AllocInit::{self, *},
14-
AllocRef, Global, Layout,
15-
ReallocPlacement::{self, *},
16-
};
11+
use crate::alloc::{handle_alloc_error, AllocRef, Global, Layout};
1712
use crate::boxed::Box;
1813
use crate::collections::TryReserveError::{self, *};
1914

2015
#[cfg(test)]
2116
mod tests;
2217

18+
enum AllocInit {
19+
/// The contents of the new memory are uninitialized.
20+
Uninitialized,
21+
/// The new memory is guaranteed to be zeroed.
22+
Zeroed,
23+
}
24+
2325
/// A low-level utility for more ergonomically allocating, reallocating, and deallocating
2426
/// a buffer of memory on the heap without having to worry about all the corner cases
2527
/// involved. This type is excellent for building your own data structures like Vec and VecDeque.
@@ -156,14 +158,14 @@ impl<T, A: AllocRef> RawVec<T, A> {
156158
/// allocator for the returned `RawVec`.
157159
#[inline]
158160
pub fn with_capacity_in(capacity: usize, alloc: A) -> Self {
159-
Self::allocate_in(capacity, Uninitialized, alloc)
161+
Self::allocate_in(capacity, AllocInit::Uninitialized, alloc)
160162
}
161163

162164
/// Like `with_capacity_zeroed`, but parameterized over the choice
163165
/// of allocator for the returned `RawVec`.
164166
#[inline]
165167
pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self {
166-
Self::allocate_in(capacity, Zeroed, alloc)
168+
Self::allocate_in(capacity, AllocInit::Zeroed, alloc)
167169
}
168170

169171
fn allocate_in(capacity: usize, init: AllocInit, mut alloc: A) -> Self {
@@ -180,7 +182,11 @@ impl<T, A: AllocRef> RawVec<T, A> {
180182
Ok(_) => {}
181183
Err(_) => capacity_overflow(),
182184
}
183-
let memory = match alloc.alloc(layout, init) {
185+
let result = match init {
186+
AllocInit::Uninitialized => alloc.alloc(layout),
187+
AllocInit::Zeroed => alloc.alloc_zeroed(layout),
188+
};
189+
let memory = match result {
184190
Ok(memory) => memory,
185191
Err(_) => handle_alloc_error(layout),
186192
};
@@ -358,7 +364,7 @@ impl<T, A: AllocRef> RawVec<T, A> {
358364
///
359365
/// Aborts on OOM.
360366
pub fn shrink_to_fit(&mut self, amount: usize) {
361-
match self.shrink(amount, MayMove) {
367+
match self.shrink(amount) {
362368
Err(CapacityOverflow) => capacity_overflow(),
363369
Err(AllocError { layout, .. }) => handle_alloc_error(layout),
364370
Ok(()) => { /* yay */ }
@@ -450,22 +456,16 @@ impl<T, A: AllocRef> RawVec<T, A> {
450456
Ok(())
451457
}
452458

453-
fn shrink(
454-
&mut self,
455-
amount: usize,
456-
placement: ReallocPlacement,
457-
) -> Result<(), TryReserveError> {
459+
fn shrink(&mut self, amount: usize) -> Result<(), TryReserveError> {
458460
assert!(amount <= self.capacity(), "Tried to shrink to a larger capacity");
459461

460462
let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) };
461463
let new_size = amount * mem::size_of::<T>();
462464

463465
let memory = unsafe {
464-
self.alloc.shrink(ptr, layout, new_size, placement).map_err(|_| {
465-
TryReserveError::AllocError {
466-
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
467-
non_exhaustive: (),
468-
}
466+
self.alloc.shrink(ptr, layout, new_size).map_err(|_| TryReserveError::AllocError {
467+
layout: Layout::from_size_align_unchecked(new_size, layout.align()),
468+
non_exhaustive: (),
469469
})?
470470
};
471471
self.set_memory(memory);
@@ -492,9 +492,9 @@ where
492492

493493
let memory = if let Some((ptr, old_layout)) = current_memory {
494494
debug_assert_eq!(old_layout.align(), new_layout.align());
495-
unsafe { alloc.grow(ptr, old_layout, new_layout.size(), MayMove, Uninitialized) }
495+
unsafe { alloc.grow(ptr, old_layout, new_layout.size()) }
496496
} else {
497-
alloc.alloc(new_layout, Uninitialized)
497+
alloc.alloc(new_layout)
498498
}
499499
.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?;
500500

‎library/alloc/src/raw_vec/tests.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ fn allocator_param() {
2020
fuel: usize,
2121
}
2222
unsafe impl AllocRef for BoundedAlloc {
23-
fn alloc(&mut self, layout: Layout, init: AllocInit) -> Result<MemoryBlock, AllocErr> {
23+
fn alloc(&mut self, layout: Layout) -> Result<MemoryBlock, AllocErr> {
2424
let size = layout.size();
2525
if size > self.fuel {
2626
return Err(AllocErr);
2727
}
28-
match Global.alloc(layout, init) {
28+
match Global.alloc(layout) {
2929
ok @ Ok(_) => {
3030
self.fuel -= size;
3131
ok

‎library/alloc/src/rc.rs

+2-4
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ use core::pin::Pin;
250250
use core::ptr::{self, NonNull};
251251
use core::slice::from_raw_parts_mut;
252252

253-
use crate::alloc::{box_free, handle_alloc_error, AllocInit, AllocRef, Global, Layout};
253+
use crate::alloc::{box_free, handle_alloc_error, AllocRef, Global, Layout};
254254
use crate::borrow::{Cow, ToOwned};
255255
use crate::string::String;
256256
use crate::vec::Vec;
@@ -928,9 +928,7 @@ impl<T: ?Sized> Rc<T> {
928928
let layout = Layout::new::<RcBox<()>>().extend(value_layout).unwrap().0.pad_to_align();
929929

930930
// Allocate for the layout.
931-
let mem = Global
932-
.alloc(layout, AllocInit::Uninitialized)
933-
.unwrap_or_else(|_| handle_alloc_error(layout));
931+
let mem = Global.alloc(layout).unwrap_or_else(|_| handle_alloc_error(layout));
934932

935933
// Initialize the RcBox
936934
let inner = mem_to_rcbox(mem.ptr.as_ptr());

0 commit comments

Comments
 (0)