Skip to content

Commit

Permalink
Fix.
Browse files Browse the repository at this point in the history
  • Loading branch information
small-turtle-1 committed Dec 27, 2024
1 parent abe72da commit 02f98ad
Show file tree
Hide file tree
Showing 4 changed files with 49 additions and 24 deletions.
43 changes: 19 additions & 24 deletions src/storage/buffer/file_worker/file_worker.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -215,43 +215,38 @@ void FileWorker::Mmap() {
}
auto [defer_fn, read_path] = GetFilePathInner(false);
bool use_object_cache = persistence_manager_ != nullptr;
SizeT file_size = VirtualStore::GetFileSize(read_path);
int ret = VirtualStore::MmapFile(read_path, mmap_addr_, file_size);
if (ret < 0) {
UnrecoverableError(fmt::format("Mmap file {} failed. {}", read_path, strerror(errno)));
}
// ret = madvise(mmap_addr_, file_size, MADV_DONTNEED);
// if (ret < 0) {
// UnrecoverableError(fmt::format("Madvise file {} failed. {}", read_path, strerror(errno)));
// }
if (use_object_cache) {
const void *ptr = mmap_addr_ + obj_addr_.part_offset_;
this->ReadFromMmapImpl(ptr, obj_addr_.part_size_);
int ret = VirtualStore::MmapFilePart(read_path, obj_addr_.part_offset_, obj_addr_.part_size_, mmap_addr_);
if (ret < 0) {
UnrecoverableError(fmt::format("Mmap file {} failed. {}", read_path, strerror(errno)));
}
this->ReadFromMmapImpl(mmap_addr_, obj_addr_.part_size_);
} else {
const void *ptr = mmap_addr_;
this->ReadFromMmapImpl(ptr, file_size);
SizeT file_size = VirtualStore::GetFileSize(read_path);
int ret = VirtualStore::MmapFile(read_path, mmap_addr_, file_size);
if (ret < 0) {
UnrecoverableError(fmt::format("Mmap file {} failed. {}", read_path, strerror(errno)));
}
this->ReadFromMmapImpl(mmap_addr_, file_size);
}
}

void FileWorker::Munmap() {
if (mmap_addr_ == nullptr) {
return;
}
auto [defer_fn, read_path] = GetFilePathInner(false);
this->FreeFromMmapImpl();
VirtualStore::MunmapFile(read_path);
auto [defer_fn, read_path] = GetFilePathInner(false);
bool use_object_cache = persistence_manager_ != nullptr;
if (use_object_cache) {
VirtualStore::MunmapFilePart(mmap_addr_, obj_addr_.part_offset_, obj_addr_.part_size_);
} else {
VirtualStore::MunmapFile(read_path);
}
mmap_addr_ = nullptr;
mmap_data_ = nullptr;
}

void FileWorker::MmapNotNeed() {
// this->Munmap();
// auto [defer_fn, read_path] = GetFilePathInner(false);
// SizeT file_size = VirtualStore::GetFileSize(read_path);
// int ret = madvise(mmap_addr_, file_size, MADV_PAGEOUT);
// if (ret < 0) {
// UnrecoverableError(fmt::format("Madvise file {} failed. {}", read_path, strerror(errno)));
// }
}
void FileWorker::MmapNotNeed() {}

} // namespace infinity
24 changes: 24 additions & 0 deletions src/storage/io/virtual_store.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,30 @@ i32 VirtualStore::MunmapFile(const String &file_path) {
return 0;
}

i32 VirtualStore::MmapFilePart(const String &file_path, SizeT offset, SizeT length, u8 *&data_ptr) {
SizeT align_offset = offset / getpagesize() * getpagesize();
SizeT diff = offset - align_offset;
SizeT align_length = length + diff;
int fd = open(file_path.c_str(), O_RDONLY);
if (fd < 0) {
return -1;
}
void *ret = mmap(NULL, align_length, PROT_READ, MAP_SHARED, fd, align_offset);
close(fd);
if (ret == MAP_FAILED) {
return -1;
}
data_ptr = reinterpret_cast<u8 *>(ret) + diff;
return 0;
}

i32 VirtualStore::MunmapFilePart(u8 *data_ptr, SizeT offset, SizeT length) {
SizeT align_offset = offset / getpagesize() * getpagesize();
u8 *aligned_ptr = data_ptr - offset + align_offset;
SizeT align_length = length + data_ptr - aligned_ptr;
return munmap(aligned_ptr, align_length);
}

// Remote storage
StorageType VirtualStore::storage_type_ = StorageType::kInvalid;
String VirtualStore::bucket_ = "infinity";
Expand Down
3 changes: 3 additions & 0 deletions src/storage/io/virtual_store.cppm
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ public:
static i32 MmapFile(const String &file_path, u8 *&data_ptr, SizeT &data_len);
static i32 MunmapFile(const String &file_path);

static i32 MmapFilePart(const String &file_path, SizeT offset, SizeT length, u8 *&data_ptr);
static i32 MunmapFilePart(u8 *data_ptr, SizeT offset, SizeT length);

static Status InitRemoteStore(StorageType storage_type = StorageType::kMinio,
const String &URL = "http://localhost:9000",
bool HTTPS = false,
Expand Down
3 changes: 3 additions & 0 deletions src/storage/meta/entry/chunk_index_entry.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,9 @@ void ChunkIndexEntry::SaveIndexFile() {
return;
}
buffer_obj_->Save();
if (segment_index_entry_->table_index_entry()->index_base()->index_type_ == IndexType::kHnsw) {
buffer_obj_->ToMmap();
}
}

void ChunkIndexEntry::DeprecateChunk(TxnTimeStamp commit_ts) {
Expand Down

0 comments on commit 02f98ad

Please sign in to comment.