[ML] Report the "actual" memory usage of the autodetect process (#2846)

Determine the actual memory usgae of the autodetect process as reported by the OS, e.g. on Linux this mould be the value of the maximum resident set size returned by a call to `getrusage`.

Add this value to the model size stats record returned to the ES Java process so it can be included in the `job counts` tab for anomaly detection jobs.

---------

Co-authored-by: Valeriy Khakhutskyy <[email protected]>

Author: edsavage

bin/autodetect/Main.cc

@@ -24,6 +24,7 @@
 #include <core/CJsonOutputStreamWrapper.h>
 #include <core/CLogger.h>
 #include <core/CProcessPriority.h>
+#include <core/CProcessStats.h>
 #include <core/CProgramCounters.h>
 #include <core/CStringUtils.h>
 #include <core/CoreTypes.h>
@@ -83,7 +84,9 @@ int main(int argc, char** argv) {
         ml::counter_t::E_TSADNumberMemoryLimitModelCreationFailures,
         ml::counter_t::E_TSADNumberPrunedItems,
         ml::counter_t::E_TSADAssignmentMemoryBasis,
-        ml::counter_t::E_TSADOutputMemoryAllocatorUsage};
+        ml::counter_t::E_TSADOutputMemoryAllocatorUsage,
+        ml::counter_t::E_TSADSystemMemoryUsage,
+        ml::counter_t::E_TSADMaxSystemMemoryUsage};
 
     ml::core::CProgramCounters::registerProgramCounterTypes(counters);
 
@@ -151,6 +154,8 @@ int main(int argc, char** argv) {
     }
     cancellerThread.stop();
 
+    LOG_DEBUG(<< "Max Resident Set Size: " << ml::core::CProcessStats::maxResidentSetSize());
+    LOG_DEBUG(<< "Resident Set Size: " << ml::core::CProcessStats::residentSetSize());
     // Log the program version immediately after reconfiguring the logger.  This
     // must be done from the program, and NOT a shared library, as each program
     // statically links its own version library.

docs/CHANGELOG.asciidoc

@@ -28,6 +28,12 @@
 
 //=== Regressions
 
+== {es} version 9.2.0
+
+=== Enhancements
+
+* Report the actual memory usage of the autodetect process. (See {ml-pull}2846[#2846])
+
 == {es} version 9.1.0
 
 === Enhancements

include/core/CProgramCounters.h

@@ -112,6 +112,12 @@ enum ECounterTypes {
     //! The memory currently used by the allocators to output JSON documents, in bytes.
     E_TSADOutputMemoryAllocatorUsage = 30,
 
+    //! The resident set size of the process, in bytes.
+    E_TSADSystemMemoryUsage = 31,
+
+    //! The maximum resident set size of the process, in bytes.
+    E_TSADMaxSystemMemoryUsage = 32,
+
     // Data Frame Outlier Detection
 
     //! The estimated peak memory usage for outlier detection in bytes
@@ -146,7 +152,7 @@ enum ECounterTypes {
     // Add any new values here
 
     //! This MUST be last, increment the value for every new enum added
-    E_LastEnumCounter = 31
+    E_LastEnumCounter = 33
 };
 
 static constexpr std::size_t NUM_COUNTERS = static_cast<std::size_t>(E_LastEnumCounter);
@@ -355,6 +361,10 @@ class CORE_EXPORT CProgramCounters {
           "Which option is being used to get model memory for node assignment?"},
          {counter_t::E_TSADOutputMemoryAllocatorUsage, "E_TSADOutputMemoryAllocatorUsage",
           "The amount of memory used to output JSON documents, in bytes."},
+         {counter_t::E_TSADSystemMemoryUsage, "E_TSADSystemMemoryUsage",
+          "The amount of system memory used by the process, in bytes"},
+         {counter_t::E_TSADMaxSystemMemoryUsage, "E_TSADMaxSystemMemoryUsage",
+          "The maximum amount of system memory used by the process, in bytes"},
          {counter_t::E_DFOEstimatedPeakMemoryUsage, "E_DFOEstimatedPeakMemoryUsage",
           "The upfront estimate of the peak memory outlier detection would use"},
          {counter_t::E_DFOPeakMemoryUsage, "E_DFOPeakMemoryUsage", "The peak memory outlier detection used"},

include/model/CProcessMemoryUsage.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the following additional limitation. Functionality enabled by the
+ * files subject to the Elastic License 2.0 may only be used in production when
+ * invoked by an Elasticsearch process with a license key installed that permits
+ * use of machine learning features. You may not use this file except in
+ * compliance with the Elastic License 2.0 and the foregoing additional
+ * limitation.
+ */
+
+#ifndef INCLUDED_ml_model_CSystemMemoryUsage_h
+#define INCLUDED_ml_model_CSystemMemoryUsage_h
+
+#include <model/ImportExport.h>
+
+#include <cstddef>
+
+namespace ml {
+namespace model {
+
+//! \brief Determines how to calculate the memory used by the current process.
+//!
+//! DESCRIPTION:\n
+//! Determines how to calculate the memory used by the current process based on the operating system.
+//! On some OS's (Mac, Windows) we use the estimated memory usage of the models,
+//! while on others (Linux) we use the actual memory of the process as provided by system calls.
+class MODEL_EXPORT CProcessMemoryUsage {
+public:
+    enum class EMemoryStrategy { E_Estimated, E_System };
+
+    static const EMemoryStrategy MEMORY_STRATEGY;
+
+public:
+    CProcessMemoryUsage() = delete;
+};
+}
+}
+
+#endif //INCLUDED_ml_model_CSystemMemoryUsage_h

include/model/CResourceMonitor.h

@@ -180,6 +180,12 @@ class MODEL_EXPORT CResourceMonitor {
     //! Returns the sum of used memory plus any extra memory
     std::size_t totalMemory() const;
 
+    //! Returns the current physical memory of the process (rss) as reported by the system
+    static std::size_t systemMemory();
+
+    //! Returns the maximum physical memory of the process (max rss) as reported by the system
+    static std::size_t maxSystemMemory();
+
 private:
     using TMonitoredResourcePtrSizeUMap =
         boost::unordered_map<CMonitoredResource*, std::size_t>;
@@ -229,6 +235,9 @@ class MODEL_EXPORT CResourceMonitor {
     //! Returns the amount by which reported memory usage is scaled depending on the type of persistence in use
     std::size_t persistenceMemoryIncreaseFactor() const;
 
+    //! Modify the supplied usage value depending on a platform dependent strategy.
+    std::size_t applyMemoryStrategy(std::size_t usage) const;
+
 private:
     //! The registered collection of components
     TMonitoredResourcePtrSizeUMap m_Resources;

lib/api/CAnomalyJob.cc

@@ -205,6 +205,10 @@ bool CAnomalyJob::handleRecord(const TStrStrUMap& dataRowFields, TOptionalTime t
     }
 
     ++core::CProgramCounters::counter(counter_t::E_TSADNumberApiRecordsHandled);
+    core::CProgramCounters::counter(counter_t::E_TSADSystemMemoryUsage) =
+        model::CResourceMonitor::systemMemory();
+    core::CProgramCounters::counter(counter_t::E_TSADMaxSystemMemoryUsage) =
+        model::CResourceMonitor::maxSystemMemory();
 
     ++m_NumRecordsHandled;
     m_LatestRecordTime = std::max(m_LatestRecordTime, *time);

lib/api/CModelSizeStatsJsonWriter.cc

@@ -25,6 +25,8 @@ const std::string JOB_ID{"job_id"};
 const std::string MODEL_SIZE_STATS{"model_size_stats"};
 const std::string MODEL_BYTES{"model_bytes"};
 const std::string PEAK_MODEL_BYTES{"peak_model_bytes"};
+const std::string SYSTEM_MEMORY_BYTES{"system_memory_bytes"};
+const std::string MAX_SYSTEM_MEMORY_BYTES{"max_system_memory_bytes"};
 const std::string MODEL_BYTES_EXCEEDED{"model_bytes_exceeded"};
 const std::string MODEL_BYTES_MEMORY_LIMIT{"model_bytes_memory_limit"};
 const std::string TOTAL_BY_FIELD_COUNT{"total_by_field_count"};

lib/api/unittest/CAnomalyJobLimitTest.cc

@@ -9,6 +9,7 @@
  * limitation.
  */
 #include <core/CJsonOutputStreamWrapper.h>
+#include <core/CProcessStats.h>
 #include <core/CoreTypes.h>
 
 #include <maths/common/CIntegerTools.h>
@@ -105,8 +106,6 @@ BOOST_AUTO_TEST_CASE(testAccuracy) {
         core::CJsonOutputStreamWrapper wrappedOutputStream(outputStrm);
 
         model::CLimits limits;
-        //limits.resourceMonitor().m_ByteLimitHigh = 100000;
-        //limits.resourceMonitor().m_ByteLimitLow = 90000;
 
         {
             LOG_TRACE(<< "Setting up job");
@@ -129,6 +128,7 @@ BOOST_AUTO_TEST_CASE(testAccuracy) {
             nonLimitedUsage = limits.resourceMonitor().totalMemory();
         }
     }
+    LOG_DEBUG(<< "nonLimitedUsage: " << nonLimitedUsage);
     {
         // Now run the data with limiting
         ml::api::CAnomalyJobConfig jobConfig = CTestAnomalyJob::makeSimpleJobConfig(

lib/api/unittest/CJsonOutputWriterTest.cc

@@ -1733,7 +1733,7 @@ BOOST_AUTO_TEST_CASE(testReportMemoryUsage) {
         resourceUsage.s_OverFields = 7;
         resourceUsage.s_AllocationFailures = 8;
         resourceUsage.s_MemoryStatus = ml::model_t::E_MemoryStatusHardLimit;
-        resourceUsage.s_AssignmentMemoryBasis = ml::model_t::E_AssignmentBasisCurrentModelBytes;
+        resourceUsage.s_AssignmentMemoryBasis = ml::model_t::E_AssignmentBasisPeakModelBytes;
         resourceUsage.s_BucketStartTime = 9;
         resourceUsage.s_BytesExceeded = 10;
         resourceUsage.s_BytesMemoryLimit = 11;
@@ -1785,7 +1785,7 @@ BOOST_AUTO_TEST_CASE(testReportMemoryUsage) {
     BOOST_TEST_REQUIRE(sizeStats.contains("memory_status"));
     BOOST_REQUIRE_EQUAL("hard_limit", sizeStats.at("memory_status").as_string());
     BOOST_TEST_REQUIRE(sizeStats.contains("assignment_memory_basis"));
-    BOOST_REQUIRE_EQUAL("current_model_bytes",
+    BOOST_REQUIRE_EQUAL("peak_model_bytes",
                         sizeStats.at("assignment_memory_basis").as_string());
     BOOST_TEST_REQUIRE(sizeStats.contains("log_time"));
     std::int64_t nowMs{ml::core::CTimeUtils::nowMs()};

lib/core/CProcessStats_MacOSX.cc

@@ -8,9 +8,10 @@
  * compliance with the Elastic License 2.0 and the foregoing additional
  * limitation.
  */
-#include <core/CLogger.h>
 #include <core/CProcessStats.h>
 
+#include <core/CLogger.h>
+
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/resource.h>