From 9c6f867752c034ab6971a3083f9cc1a1ed1f82c9 Mon Sep 17 00:00:00 2001
From: Kamil Mankowski <mankowski@cert.at>
Date: Mon, 11 Dec 2023 12:55:45 +0100
Subject: [PATCH 1/2] FIX: Ensure closing log files

During the reloading process, log handlers are not explicitly closed.
This may cause long living open file handlers and keeping cached files
in the memory. If log files are big, the RAM cache usage can be very
high.
---
 CHANGELOG.md       |  1 +
 intelmq/lib/bot.py | 17 ++++++++++++++++-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2613d7e96..e3b8d0e20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
 - `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
+- `intelmq.lib.bot`: Ensure closing log files on reloading (PR by Kamil Mankowski).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).
diff --git a/intelmq/lib/bot.py b/intelmq/lib/bot.py
index 31b6847c1..d04b24b7c 100644
--- a/intelmq/lib/bot.py
+++ b/intelmq/lib/bot.py
@@ -308,10 +308,25 @@ def __handle_sighup(self):
             self.shutdown()  # disconnects, stops threads etc
         except Exception:
             self.logger.exception('Error during shutdown of bot.')
-        self.logger.handlers = []  # remove all existing handlers
+        self.__cleanup_logging_handlers()
         self.__sighup.clear()
         self.__init__(self.__bot_id_full, sighup_event=self.__sighup, standalone=self._standalone)
 
+    def __cleanup_logging_handlers(self):
+        # thread-safe removing of handlers and closing opened files
+        handlers_list = self.logger.handlers[:]
+        for handler in handlers_list:
+            try:
+                self.logger.removeHandler(handler)
+                # ensure all log files are closed to prevent caching them in RAM
+                if isinstance(handler, logging.FileHandler):
+                    handler.close()
+            except:
+                # Logger should still be safe to use even without handlers
+                # In addition, we do not want any side issue to break execution
+                #  - we are about to reinitialize logging.
+                self.logger.exception("Error while cleaning up logging handlers")
+
     def init(self):
         pass
 

From e8f1dc8d53f1173be59513c70bdec7a34ad54dc6 Mon Sep 17 00:00:00 2001
From: Kamil Mankowski <mankowski@cert.at>
Date: Mon, 11 Dec 2023 15:03:05 +0100
Subject: [PATCH 2/2] Add PR number

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e3b8d0e20..25c1e6400 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,7 +33,7 @@
   `intelmq.bots.collectors.stomp.collector` and `intelmq.bots.outputs.stomp.output`
   (PR#2408 and PR#2414 by Jan Kaliszewski).
 - `intelmq.lib.upgrades`: Replace deprecated instances of `url2fqdn` experts by the new `url` expert in runtime configuration (PR#2432 by Sebastian Wagner).
-- `intelmq.lib.bot`: Ensure closing log files on reloading (PR by Kamil Mankowski).
+- `intelmq.lib.bot`: Ensure closing log files on reloading (PR#2435 by Kamil Mankowski).
 
 ### Development
 - Makefile: Add codespell and test commands (PR#2425 by Sebastian Wagner).