Skip to content

Commit

Permalink
current_process_memory_usage: Add system memory stats.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ned14
ned14 committed Sep 8, 2021
1 parent bcc600d commit afbac28
Show file tree
Hide file tree
Showing 6 changed files with 339 additions and 213 deletions.
6 changes: 3 additions & 3 deletions include/llfio/revision.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Note the second line of this file must ALWAYS be the git SHA, third line ALWAYS the git SHA update time
#define LLFIO_PREVIOUS_COMMIT_REF 8cec4ff8fcd601b774c0ed882fe869e73d23d136
#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-30 14:53:35 +00:00"
#define LLFIO_PREVIOUS_COMMIT_UNIQUE 8cec4ff8
#define LLFIO_PREVIOUS_COMMIT_REF bcc600db9feb6aa50e0880422a071d0e39ae74aa
#define LLFIO_PREVIOUS_COMMIT_DATE "2021-08-30 15:08:05 +00:00"
#define LLFIO_PREVIOUS_COMMIT_UNIQUE bcc600db
340 changes: 195 additions & 145 deletions include/llfio/v2.0/detail/impl/posix/utils.ipp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,156 +341,187 @@ namespace utils
private_paged_in = ??? MISSING
*/
if(!(want & process_memory_usage::want::private_committed) || (want & process_memory_usage::want::private_committed_inaccurate))
process_memory_usage ret;
if(!!(want & process_memory_usage::want::this_process))
{
process_memory_usage ret;
if((want & process_memory_usage::want::total_address_space_in_use) || (want & process_memory_usage::want::total_address_space_paged_in) ||
(want & process_memory_usage::want::private_paged_in))
{
std::vector<char> buffer(256);
OUTCOME_TRY(fill_buffer(buffer, "/proc/self/statm"));
if(buffer.size() > 1)
{
size_t file_and_shared_pages_paged_in = 0;
sscanf(buffer.data(), "%zu %zu %zu", &ret.total_address_space_in_use, &ret.total_address_space_paged_in, &file_and_shared_pages_paged_in);
ret.private_paged_in = ret.total_address_space_paged_in - file_and_shared_pages_paged_in;
ret.total_address_space_in_use *= page_size();
ret.total_address_space_paged_in *= page_size();
ret.private_paged_in *= page_size();
//std::cout << string_view(buffer.data(), buffer.size()) << std::endl;
}
}
if(want & process_memory_usage::want::private_committed)
if(!(want & process_memory_usage::want::private_committed) || (want & process_memory_usage::want::private_committed_inaccurate))
{
std::vector<char> smaps_rollup(256), maps(65536);
auto r = fill_buffer(smaps_rollup, "/proc/self/smaps_rollup");
if(!r)
if((want & process_memory_usage::want::total_address_space_in_use) || (want & process_memory_usage::want::total_address_space_paged_in) ||
(want & process_memory_usage::want::private_paged_in))
{
if(r.error() == errc::no_such_file_or_directory)
std::vector<char> buffer(256);
OUTCOME_TRY(fill_buffer(buffer, "/proc/self/statm"));
if(buffer.size() > 1)
{
// Linux kernel is too old
return errc::operation_not_supported;
size_t file_and_shared_pages_paged_in = 0;
sscanf(buffer.data(), "%zu %zu %zu", &ret.total_address_space_in_use, &ret.total_address_space_paged_in, &file_and_shared_pages_paged_in);
ret.private_paged_in = ret.total_address_space_paged_in - file_and_shared_pages_paged_in;
ret.total_address_space_in_use *= page_size();
ret.total_address_space_paged_in *= page_size();
ret.private_paged_in *= page_size();
// std::cout << string_view(buffer.data(), buffer.size()) << std::endl;
}
return std::move(r).error();
}
OUTCOME_TRY(fill_buffer(maps, "/proc/self/maps"));
uint64_t lazyfree = 0;
if(want & process_memory_usage::want::private_committed)
{
string_view i(smaps_rollup.data(), smaps_rollup.size());
OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:"));
std::vector<char> smaps_rollup(256), maps(65536);
auto r = fill_buffer(smaps_rollup, "/proc/self/smaps_rollup");
if(!r)
{
if(r.error() == errc::no_such_file_or_directory)
{
// Linux kernel is too old
return errc::operation_not_supported;
}
return std::move(r).error();
}
OUTCOME_TRY(fill_buffer(maps, "/proc/self/maps"));
uint64_t lazyfree = 0;
{
string_view i(smaps_rollup.data(), smaps_rollup.size());
OUTCOME_TRY(lazyfree, parse(i, "\nLazyFree:"));
}
string_view i(maps.data(), maps.size());
size_t anonymous = 0;
for(size_t idx = 0;;)
{
idx = i.find("\n", idx);
if(idx == i.npos)
{
break;
}
idx++;
size_t start = 0, end = 0, inode = 1;
char read = 0, write = 0, executable = 0, private_ = 0;
sscanf(i.data() + idx, "%zx-%zx %c%c%c%c %*u %*u:%*u %zd", &start, &end, &read, &write, &executable, &private_, &inode);
if(inode == 0 && read == 'r' && write == 'w' && executable == '-' && private_ == 'p')
{
anonymous += end - start;
// std::cout << (end - start) << " " << i.substr(idx, 40) << std::endl;
}
}
if(lazyfree != (uint64_t) -1)
{
anonymous -= (size_t) lazyfree;
}
ret.private_committed = anonymous;
}
string_view i(maps.data(), maps.size());
size_t anonymous = 0;
for(size_t idx = 0;;)
}
else
{
std::vector<char> buffer(1024 * 1024);
OUTCOME_TRY(fill_buffer(buffer, "/proc/self/smaps"));
const string_view totalview(buffer.data(), buffer.size());
// std::cerr << totalview << std::endl;
std::vector<string_view> anon_entries, non_anon_entries;
anon_entries.reserve(32);
non_anon_entries.reserve(32);
auto find_item = [&](size_t idx) -> string_view {
auto x = totalview.rfind("\nSize:", idx);
if(x == string_view::npos)
{
return {};
}
x = totalview.rfind("\n", x - 1);
if(x == string_view::npos)
{
x = 0;
}
else
{
x++;
}
return totalview.substr(x, idx - x);
};
for(string_view item = find_item(totalview.size()); item != string_view(); item = find_item(item.data() - totalview.data()))
{
idx = i.find("\n", idx);
if(idx == i.npos)
// std::cout << "***" << item << "***";
// hexaddr-hexaddr flags offset dev:id inode [path]
size_t inode = 1;
sscanf(item.data(), "%*x-%*x %*c%*c%*c%*c %*x %*c%*c:%*c%*c %zu", &inode);
auto vmflagsidx = item.rfind("\nVmFlags:");
if(vmflagsidx == string_view::npos)
{
break;
return errc::illegal_byte_sequence;
}
// Is there " ac" after vmflagsidx?
if(string_view::npos != item.find(" ac", vmflagsidx) && inode == 0)
{
// std::cerr << "Adding anon entry at offset " << itemtopidx << std::endl;
anon_entries.push_back(item);
}
idx++;
size_t start = 0, end = 0, inode = 1;
char read = 0, write = 0, executable = 0, private_ = 0;
sscanf(i.data() + idx, "%zx-%zx %c%c%c%c %*u %*u:%*u %zd", &start, &end, &read, &write, &executable, &private_, &inode);
if(inode == 0 && read == 'r' && write == 'w' && executable == '-' && private_ == 'p')
else
{
anonymous += end - start;
// std::cout << (end - start) << " " << i.substr(idx, 40) << std::endl;
// std::cerr << "Adding non-anon entry at offset " << itemtopidx << std::endl;
non_anon_entries.push_back(item);
}
}
if(lazyfree != (uint64_t) -1)
// std::cerr << "Anon entries:";
for(auto &i : anon_entries)
{
anonymous -= (size_t) lazyfree;
OUTCOME_TRY(auto &&size, parse(i, "\nSize:"));
OUTCOME_TRY(auto &&rss, parse(i, "\nRss:"));
OUTCOME_TRY(auto &&anonymous, parse(i, "\nAnonymous:"));
OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:"));
if(size != (uint64_t) -1 && rss != (uint64_t) -1 && anonymous != (uint64_t) -1)
{
ret.total_address_space_in_use += size;
ret.total_address_space_paged_in += rss;
ret.private_committed += anonymous;
if(lazyfree != (uint64_t) -1)
{
ret.total_address_space_paged_in -= lazyfree;
ret.private_committed -= lazyfree;
}
ret.private_paged_in += rss;
}
// std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl;
}
ret.private_committed = anonymous;
}
return ret;
}
std::vector<char> buffer(1024 * 1024);
OUTCOME_TRY(fill_buffer(buffer, "/proc/self/smaps"));
const string_view totalview(buffer.data(), buffer.size());
// std::cerr << totalview << std::endl;
std::vector<string_view> anon_entries, non_anon_entries;
anon_entries.reserve(32);
non_anon_entries.reserve(32);
auto find_item = [&](size_t idx) -> string_view {
auto x = totalview.rfind("\nSize:", idx);
if(x == string_view::npos)
{
return {};
}
x = totalview.rfind("\n", x - 1);
if(x == string_view::npos)
{
x = 0;
}
else
{
x++;
}
return totalview.substr(x, idx - x);
};
for(string_view item = find_item(totalview.size()); item != string_view(); item = find_item(item.data() - totalview.data()))
{
// std::cout << "***" << item << "***";
// hexaddr-hexaddr flags offset dev:id inode [path]
size_t inode = 1;
sscanf(item.data(), "%*x-%*x %*c%*c%*c%*c %*x %*c%*c:%*c%*c %zu", &inode);
auto vmflagsidx = item.rfind("\nVmFlags:");
if(vmflagsidx == string_view::npos)
{
return errc::illegal_byte_sequence;
}
// Is there " ac" after vmflagsidx?
if(string_view::npos != item.find(" ac", vmflagsidx) && inode == 0)
{
// std::cerr << "Adding anon entry at offset " << itemtopidx << std::endl;
anon_entries.push_back(item);
}
else
{
// std::cerr << "Adding non-anon entry at offset " << itemtopidx << std::endl;
non_anon_entries.push_back(item);
}
}
process_memory_usage ret;
// std::cerr << "Anon entries:";
for(auto &i : anon_entries)
{
OUTCOME_TRY(auto &&size, parse(i, "\nSize:"));
OUTCOME_TRY(auto &&rss, parse(i, "\nRss:"));
OUTCOME_TRY(auto &&anonymous, parse(i, "\nAnonymous:"));
OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:"));
if(size != (uint64_t) -1 && rss != (uint64_t) -1 && anonymous != (uint64_t) -1)
{
ret.total_address_space_in_use += size;
ret.total_address_space_paged_in += rss;
ret.private_committed += anonymous;
if(lazyfree != (uint64_t) -1)
// std::cerr << "\n\nNon-anon entries:";
for(auto &i : non_anon_entries)
{
ret.total_address_space_paged_in -= lazyfree;
ret.private_committed -= lazyfree;
OUTCOME_TRY(auto &&size, parse(i, "\nSize:"));
OUTCOME_TRY(auto &&rss, parse(i, "\nRss:"));
OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:"));
if(size != (uint64_t) -1 && rss != (uint64_t) -1)
{
ret.total_address_space_in_use += size;
ret.total_address_space_paged_in += rss;
if(lazyfree != (uint64_t) -1)
{
ret.total_address_space_in_use -= lazyfree;
}
}
// std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl;
}
ret.private_paged_in += rss;
}
// std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl;
}
// std::cerr << "\n\nNon-anon entries:";
for(auto &i : non_anon_entries)
if(!!(want & process_memory_usage::want::this_system))
{
OUTCOME_TRY(auto &&size, parse(i, "\nSize:"));
OUTCOME_TRY(auto &&rss, parse(i, "\nRss:"));
OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:"));
if(size != (uint64_t) -1 && rss != (uint64_t) -1)
std::vector<char> buffer(1024);
OUTCOME_TRY(fill_buffer(buffer, "/proc/meminfo"));
if(buffer.size() > 1)
{
ret.total_address_space_in_use += size;
ret.total_address_space_paged_in += rss;
if(lazyfree != (uint64_t) -1)
string_view i(buffer.data(), buffer.size());
OUTCOME_TRY(ret.system_physical_memory_total, parse(i, "MemTotal:"));
OUTCOME_TRY(ret.system_physical_memory_available, parse(i, "\nMemAvailable:"));
if((uint64_t) -1 == ret.system_physical_memory_available)
{
ret.total_address_space_in_use -= lazyfree;
// MemAvailable is >= Linux 3.14, so let's approximate what it would be
OUTCOME_TRY(auto &&memfree, parse(i, "\nMemFree:"));
OUTCOME_TRY(auto &&cached, parse(i, "\nCached:"));
OUTCOME_TRY(auto &&swapcached, parse(i, "\nSwapCached:"));
ret.system_physical_memory_available = memfree + cached + swapcached;
}
OUTCOME_TRY(ret.system_commit_charge_maximum, parse(i, "\nCommitLimit:"));
OUTCOME_TRY(ret.system_commit_charge_available, parse(i, "\nCommitted_AS:"));
OUTCOME_TRY(auto &&lazyfree, parse(i, "\nLazyFree:"));
if(lazyfree == (uint64_t) -1)
{
lazyfree = 0;
}
ret.system_commit_charge_available = ret.system_commit_charge_maximum - ret.system_commit_charge_available + lazyfree;
}
// std::cerr << i << "\nSize = " << size << " Rss = " << rss << std::endl;
}
return ret;
}
Expand All @@ -502,29 +533,48 @@ namespace utils
(void) want;
kern_return_t error;
mach_msg_type_number_t outCount;
task_vm_info_data_t vmInfo;
// task_kernelmemory_info_data_t kmInfo;
process_memory_usage ret;
if(!!(wanted & process_memory_usage::want::this_process))
{
task_vm_info_data_t vmInfo;
// task_kernelmemory_info_data_t kmInfo;

outCount = TASK_VM_INFO_COUNT;
error = task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &vmInfo, &outCount);
if(error != KERN_SUCCESS)
outCount = TASK_VM_INFO_COUNT;
error = task_info(mach_task_self(), TASK_VM_INFO, (task_info_t) &vmInfo, &outCount);
if(error != KERN_SUCCESS)
{
return errc::invalid_argument;
}
// outCount = TASK_KERNELMEMORY_INFO_COUNT;
// error = task_info(mach_task_self(), TASK_KERNELMEMORY_INFO, (task_info_t)&kmInfo, &outCount);
// if (error != KERN_SUCCESS) {
// return errc::invalid_argument;
//}
// std::cout << vmInfo.virtual_size << "\n" << vmInfo.region_count << "\n" << vmInfo.resident_size << "\n" << vmInfo.device << "\n" << vmInfo.internal <<
// "\n" << vmInfo.external << "\n" << vmInfo.reusable << "\n" << vmInfo.purgeable_volatile_pmap<< "\n" << vmInfo.purgeable_volatile_resident << "\n" <<
// vmInfo.purgeable_volatile_virtual << "\n" << vmInfo.compressed << "\n" << vmInfo.phys_footprint << std::endl; std::cout << "\n" << kmInfo.total_palloc
// <<
// "\n" << kmInfo.total_pfree << "\n" << kmInfo.total_salloc << "\n" << kmInfo.total_sfree << std::endl;
ret.total_address_space_in_use = vmInfo.virtual_size;
ret.total_address_space_paged_in = vmInfo.resident_size;
ret.private_committed = vmInfo.internal + vmInfo.compressed;
ret.private_paged_in = vmInfo.phys_footprint;
}
if(!!(wanted & process_memory_usage::want::this_system))
{
return errc::invalid_argument;
vm_statistics_data_t vmStat;
outCount = HOST_VM_INFO_COUNT;
error = host_statistics(mach_host_self(), HOST_VM_INFO, (host_info_t) &vmStat, &outCount);
if(error != KERN_SUCCESS)
{
return errc::invalid_argument;
}
ret.system_physical_memory_total = ((uint64_t) vmStat.free_count + vmStat.active_count + vmStat.inactive_count + vmStat.wire_count) * page_size();
ret.system_physical_memory_available = ((uint64_t) vmStat.free_count + vmStat.inactive_count) * page_size();
// Not sure how to retrieve these on Mac OS
// ret.system_commit_charge_maximum = (uint64_t) pi.CommitLimit * page_size();
// ret.system_commit_charge_available = (uint64_t) pi.CommitTotal * page_size();
}
// outCount = TASK_KERNELMEMORY_INFO_COUNT;
// error = task_info(mach_task_self(), TASK_KERNELMEMORY_INFO, (task_info_t)&kmInfo, &outCount);
// if (error != KERN_SUCCESS) {
// return errc::invalid_argument;
//}
// std::cout << vmInfo.virtual_size << "\n" << vmInfo.region_count << "\n" << vmInfo.resident_size << "\n" << vmInfo.device << "\n" << vmInfo.internal <<
// "\n" << vmInfo.external << "\n" << vmInfo.reusable << "\n" << vmInfo.purgeable_volatile_pmap<< "\n" << vmInfo.purgeable_volatile_resident << "\n" <<
// vmInfo.purgeable_volatile_virtual << "\n" << vmInfo.compressed << "\n" << vmInfo.phys_footprint << std::endl; std::cout << "\n" << kmInfo.total_palloc <<
// "\n" << kmInfo.total_pfree << "\n" << kmInfo.total_salloc << "\n" << kmInfo.total_sfree << std::endl;
process_memory_usage ret;
ret.total_address_space_in_use = vmInfo.virtual_size;
ret.total_address_space_paged_in = vmInfo.resident_size;
ret.private_committed = vmInfo.internal + vmInfo.compressed;
ret.private_paged_in = vmInfo.phys_footprint;
return ret;
#else
#error Unknown platform
Expand Down
Loading

0 comments on commit afbac28

Please sign in to comment.