diff --git a/Cargo.lock b/Cargo.lock index 0229927e9617..e8bfcc01ac7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1518,16 +1518,14 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -3083,9 +3081,9 @@ dependencies = [ [[package]] name = "databend-client" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0819048a792e2eac58b455bbbcf6077c81e2780b3cc4f565a6e72e92dde56bd1" +checksum = "bd8770a1c49fa21e62a768a0de442cc3f77998a357303d56ddd3485cb7c58d3a" dependencies = [ "async-trait", "log", @@ -3225,6 +3223,7 @@ dependencies = [ "fastrace", "futures", "libc", + "libmimalloc-sys", "log", "logcall", "micromarshal", @@ -4781,9 +4780,9 @@ dependencies = [ [[package]] name = "databend-driver" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbea10312fa203003b572b701b6121cecd8eaf260e7ec7687796b552044541c6" +checksum = "393daaf83f32c5685887103b04bd9762d43298f7820b9f888144877fcf1d89e8" dependencies = [ "arrow", "async-compression 0.4.12", @@ -4806,9 +4805,9 @@ dependencies = [ [[package]] name = "databend-driver-core" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b8f3bf7bb87d0f2fce8a1992eea4a0b442c01165a0d68aab2727a386c945ab" +checksum = "7dd1abd8ab1c4200648510204cdfcb538af1eca2c9433e12276dee26a1806144" dependencies = [ "arrow", "chrono", @@ -4829,9 +4828,9 @@ dependencies = [ [[package]] name = "databend-driver-macros" -version = "0.22.0" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40bed5c66f36e79baea7c95d2d0dc8433671606ca7152012562c7145e3b909a1" +checksum = "5038d47b403ce2351b109fb82627959a27a16a53ce16397bf1ca39307e052b5c" dependencies = [ "quote", "syn 2.0.58", @@ -9314,12 +9313,6 @@ dependencies = [ "spin", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128" version = "0.2.5" @@ -9471,6 +9464,16 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libmimalloc-sys" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libredox" version = "0.1.3" diff --git a/Cargo.toml b/Cargo.toml index 7fc23e8b3ad7..598c08ff6394 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -331,6 +331,7 @@ levenshtein_automata = "0.2.1" lexical-core = "1" libc = { version = "0.2.158" } libm = "0.2.6" +libmimalloc-sys = { version = "0.1.39" } limits-rs = "0.2.0" logforth = { version = "0.14", features = [ 'json', diff --git a/src/binaries/Cargo.toml b/src/binaries/Cargo.toml index 63196e4fb017..a2baa7b15f53 100644 --- a/src/binaries/Cargo.toml +++ b/src/binaries/Cargo.toml @@ -8,7 +8,8 @@ publish = { workspace = true } edition = { workspace = true } [features] -default = ["simd", "jemalloc"] +#default = ["simd", "jemalloc"] +default = ["simd", "mimalloc"] memory-profiling = [ "databend-query/memory-profiling", "databend-common-base/memory-profiling", @@ -16,6 +17,7 @@ memory-profiling = [ python-udf = ["databend-query/python-udf"] simd = ["databend-query/simd"] jemalloc = ["databend-common-base/jemalloc", "databend-query/jemalloc"] +mimalloc = ["databend-common-base/mimalloc"] io-uring = [ "databend-query/io-uring", ] diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index 2c71430e8f9a..01525e7145ca 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -13,6 +13,7 @@ test = true [features] tracing = ["tokio/tracing"] jemalloc = ["tikv-jemalloc-sys", "tikv-jemalloc-ctl"] +mimalloc = ["libmimalloc-sys"] disable_initial_exec_tls = ["tikv-jemalloc-sys/disable_initial_exec_tls"] memory-profiling = [ "tikv-jemalloc-sys/stats", @@ -33,6 +34,7 @@ enquote = { workspace = true } fastrace = { workspace = true } futures = { workspace = true } libc = { workspace = true } +libmimalloc-sys = { workspace = true, optional = true } log = { workspace = true } logcall = { workspace = true } micromarshal = { workspace = true } diff --git a/src/common/base/src/mem_allocator/default.rs b/src/common/base/src/mem_allocator/default.rs index fca17168ce4d..0d0042e19e3c 100644 --- a/src/common/base/src/mem_allocator/default.rs +++ b/src/common/base/src/mem_allocator/default.rs @@ -13,7 +13,10 @@ // limitations under the License. // Default allocator is jemalloc, you can change it to std: +#[cfg(not(any(feature = "jemalloc", feature = "mimalloc")))] +pub type DefaultAllocator = crate::mem_allocator::StdAllocator; #[cfg(feature = "jemalloc")] pub type DefaultAllocator = crate::mem_allocator::JEAllocator; -#[cfg(not(feature = "jemalloc"))] -pub type DefaultAllocator = crate::mem_allocator::StdAllocator; + +#[cfg(feature = "mimalloc")] +pub type DefaultAllocator = crate::mem_allocator::MIAllocator; diff --git a/src/common/base/src/mem_allocator/mimalloc.rs b/src/common/base/src/mem_allocator/mimalloc.rs new file mode 100644 index 000000000000..c3a57f023b77 --- /dev/null +++ b/src/common/base/src/mem_allocator/mimalloc.rs @@ -0,0 +1,188 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +/// jemalloc allocator. +#[derive(Debug, Clone, Copy, Default)] +pub struct MIAllocator; + +impl MIAllocator { + pub fn name() -> String { + "mimalloc".to_string() + } + + pub fn conf() -> String { + "".to_string() + } +} + +#[cfg(target_os = "linux")] +pub mod linux { + use std::alloc::AllocError; + use std::alloc::Allocator; + use std::alloc::Layout; + use std::ptr::NonNull; + + use libc::c_void; + use libmimalloc_sys as ffi; + + use super::MIAllocator; + use crate::runtime::ThreadTracker; + + unsafe impl Allocator for MIAllocator { + #[inline(always)] + fn allocate(&self, layout: Layout) -> Result, AllocError> { + ThreadTracker::alloc(layout.size() as i64)?; + + let data_address = if layout.size() == 0 { + unsafe { NonNull::new(layout.align() as *mut ()).unwrap_unchecked() } + } else { + unsafe { + NonNull::new(ffi::mi_malloc_aligned(layout.size(), layout.align()) as *mut ()) + .ok_or(AllocError)? + } + }; + Ok(NonNull::<[u8]>::from_raw_parts(data_address, layout.size())) + } + + #[inline(always)] + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + ThreadTracker::alloc(layout.size() as i64)?; + + let data_address = if layout.size() == 0 { + unsafe { NonNull::new(layout.align() as *mut ()).unwrap_unchecked() } + } else { + unsafe { + NonNull::new(ffi::mi_zalloc_aligned(layout.size(), layout.align()) as *mut ()) + .ok_or(AllocError)? + } + }; + + Ok(NonNull::<[u8]>::from_raw_parts(data_address, layout.size())) + } + + #[inline(always)] + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + ThreadTracker::dealloc(layout.size() as i64); + + ffi::mi_free(ptr.as_ptr() as *mut _); + } + + unsafe fn grow( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert_eq!(old_layout.align(), new_layout.align()); + debug_assert!(old_layout.size() <= new_layout.size()); + + ThreadTracker::dealloc(old_layout.size() as i64); + ThreadTracker::alloc(new_layout.size() as i64)?; + + let data_address = if new_layout.size() == 0 { + NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() + } else if old_layout.size() == 0 { + NonNull::new( + ffi::mi_malloc_aligned(new_layout.size(), new_layout.align()) as *mut (), + ) + .ok_or(AllocError)? + } else { + NonNull::new(ffi::mi_realloc(ptr.cast().as_ptr(), new_layout.size()) as *mut ()) + .unwrap() + }; + + Ok(NonNull::<[u8]>::from_raw_parts( + data_address, + new_layout.size(), + )) + } + + unsafe fn grow_zeroed( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert!(old_layout.size() <= new_layout.size()); + + ThreadTracker::dealloc(old_layout.size() as i64); + ThreadTracker::alloc(new_layout.size() as i64)?; + + let data_address = if new_layout.size() == 0 { + NonNull::new(new_layout.align() as *mut ()).unwrap_unchecked() + } else if old_layout.size() == 0 { + NonNull::new( + ffi::mi_malloc_aligned(new_layout.size(), new_layout.align()) as *mut (), + ) + .ok_or(AllocError)? + } else { + let raw = ffi::mi_realloc_aligned( + ptr.cast().as_ptr(), + new_layout.size(), + new_layout.align(), + ) as *mut u8; + let r = NonNull::new(raw as *mut ()).unwrap(); + raw.add(old_layout.size()) + .write_bytes(0, new_layout.size() - old_layout.size()); + r + }; + + Ok(NonNull::<[u8]>::from_raw_parts( + data_address, + new_layout.size(), + )) + } + + unsafe fn shrink( + &self, + ptr: NonNull, + old_layout: Layout, + new_layout: Layout, + ) -> Result, AllocError> { + debug_assert_eq!(old_layout.align(), new_layout.align()); + debug_assert!(old_layout.size() >= new_layout.size()); + + ThreadTracker::dealloc(old_layout.size() as i64); + ThreadTracker::alloc(new_layout.size() as i64)?; + + if old_layout.size() == 0 { + debug_assert_eq!(ptr.as_ptr() as usize, old_layout.align()); + let slice = std::slice::from_raw_parts_mut(ptr.as_ptr(), 0); + let ptr = NonNull::new(slice).unwrap_unchecked(); + return Ok(ptr); + } + + let new_ptr = if new_layout.size() == 0 { + ffi::mi_free(ptr.as_ptr() as *mut c_void); + let slice = std::slice::from_raw_parts_mut(new_layout.align() as *mut u8, 0); + NonNull::new(slice).unwrap_unchecked() + } else { + let data_address = ffi::mi_realloc_aligned( + ptr.cast().as_ptr(), + new_layout.size(), + new_layout.align(), + ) as *mut u8; + assert!(!data_address.is_null()); + let slice = std::slice::from_raw_parts_mut(data_address, new_layout.size()); + NonNull::new(slice).ok_or(AllocError)? + }; + + Ok(new_ptr) + } + } +} diff --git a/src/common/base/src/mem_allocator/mmap.rs b/src/common/base/src/mem_allocator/mmap.rs index 759c3679fb8b..5f420d689c08 100644 --- a/src/common/base/src/mem_allocator/mmap.rs +++ b/src/common/base/src/mem_allocator/mmap.rs @@ -22,7 +22,9 @@ pub struct MmapAllocator { #[cfg(feature = "jemalloc")] allocator: crate::mem_allocator::JEAllocator, - #[cfg(not(feature = "jemalloc"))] + #[cfg(feature = "mimalloc")] + allocator: crate::mem_allocator::MIAllocator, + #[cfg(not(any(feature = "jemalloc", feature = "mimalloc")))] allocator: crate::mem_allocator::StdAllocator, } @@ -31,7 +33,9 @@ impl MmapAllocator { Self { #[cfg(feature = "jemalloc")] allocator: crate::mem_allocator::JEAllocator, - #[cfg(not(feature = "jemalloc"))] + #[cfg(feature = "mimalloc")] + allocator: crate::mem_allocator::MIAllocator, + #[cfg(not(any(feature = "jemalloc", feature = "mimalloc")))] allocator: crate::mem_allocator::StdAllocator, } } diff --git a/src/common/base/src/mem_allocator/mod.rs b/src/common/base/src/mem_allocator/mod.rs index ca2aa9fe8791..3615ac8d8eb0 100644 --- a/src/common/base/src/mem_allocator/mod.rs +++ b/src/common/base/src/mem_allocator/mod.rs @@ -12,22 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod default; mod global; #[cfg(feature = "jemalloc")] mod jemalloc; +#[cfg(feature = "mimalloc")] +mod mimalloc; mod mmap; +#[cfg(feature = "memory-profiling")] +mod profiling; mod std_; pub use default::DefaultAllocator; pub use global::GlobalAllocator; #[cfg(feature = "jemalloc")] pub use jemalloc::JEAllocator; +#[cfg(feature = "mimalloc")] +pub use mimalloc::MIAllocator; pub use mmap::MmapAllocator; -pub use std_::StdAllocator; - -mod default; -#[cfg(feature = "memory-profiling")] -mod profiling; - #[cfg(feature = "memory-profiling")] pub use profiling::dump_profile; +pub use std_::StdAllocator; diff --git a/src/meta/binaries/Cargo.toml b/src/meta/binaries/Cargo.toml index db188e51d6be..ac5746f1755a 100644 --- a/src/meta/binaries/Cargo.toml +++ b/src/meta/binaries/Cargo.toml @@ -8,13 +8,15 @@ publish = { workspace = true } edition = { workspace = true } [features] -default = ["simd", "jemalloc"] +#default = ["simd", "jemalloc"] +default = ["simd", "mimalloc"] memory-profiling = [ "databend-meta/memory-profiling", "databend-common-base/memory-profiling", ] simd = ["databend-meta/simd"] jemalloc = ["databend-common-base/jemalloc"] +mimalloc = ["databend-common-base/mimalloc"] io-uring = [ "databend-meta/io-uring", "databend-common-meta-store/io-uring",