Merge pull request #8228 from romayalon/romy-5.16.1-backports

5.16.1 Backports

Author: nimrod-becker

config.js

@@ -14,6 +14,9 @@ const _ = require('lodash');
 const util = require('util');
 const nsfs_schema_utils = require('./src/manage_nsfs/nsfs_schema_utils');
 const range_utils = require('./src/util/range_utils');
+const { EventEmitter } = require('events');
+
+config.event_emitter = new EventEmitter();
 
 /////////////////////////
 // CONTAINER RESOURCES //
@@ -464,6 +467,9 @@ config.EVENT_FACILITY = 'LOG_LOCAL2';
 config.EVENT_LOGGING_ENABLED = true;
 config.EVENT_LEVEL = 5;
 
+config.LOG_TO_STDERR_ENABLED = true;
+config.LOG_TO_SYSLOG_ENABLED = false;
+
 // TEST Mode
 config.test_mode = false;
 
@@ -1075,6 +1081,7 @@ function load_nsfs_nc_config() {
         });
         console.warn(`nsfs: config_dir_path=${config.NSFS_NC_CONF_DIR} config.json= ${util.inspect(merged_config)}`);
         validate_nc_master_keys_config(config);
+        config.event_emitter.emit("config_updated");
     } catch (err) {
         if (err.code !== 'MODULE_NOT_FOUND' && err.code !== 'ENOENT') throw err;
         console.warn('config.load_nsfs_nc_config could not find config.json... skipping');

docs/dev_guide/NonContainerizedDeveloperCustomizations.md

@@ -458,6 +458,43 @@ Example:
 3. systemctl restart noobaa
 ```
 
+## 25. Syslog enable flag -
+**Description -** This flag will enable syslog logging for the application.
+
+**Configuration Key -** LOG_TO_SYSLOG_ENABLED
+
+**Type -** boolean
+
+**Default -** true
+
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"LOG_TO_SYSLOG_ENABLED": true
+3. systemctl restart noobaa
+```
+
+## 26. Stderr enable flag -
+**Description -** This flag will decide whether need to push logs to the stderr or not.
+
+**Configuration Key -** LOG_TO_STDERR_ENABLED
+
+**Type -** boolean
+
+**Default -** false
+
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"LOG_TO_STDERR_ENABLED": false
+3. systemctl restart noobaa
+```
+
+```
 > cat /path/to/config_dir/config.json
 {
     "ENDPOINT_PORT": 80,

src/native/fs/fs_napi.cpp

@@ -2123,7 +2123,18 @@ set_debug_level(const Napi::CallbackInfo& info)
 {
     int level = info[0].As<Napi::Number>();
     DBG_SET_LEVEL(level);
-    DBG1("FS::set_debug_level " << level);
+    LOG("FS::set_debug_level " << level);
+    return info.Env().Undefined();
+}
+
+static Napi::Value
+set_log_config(const Napi::CallbackInfo& info)
+{
+    bool stderr_enabled = info[0].As<Napi::Boolean>();
+    bool syslog_enabled = info[1].As<Napi::Boolean>();
+    LOG_TO_STDERR_ENABLED = stderr_enabled;
+    LOG_TO_SYSLOG_ENABLED = syslog_enabled;
+    LOG("FS::set_log_config: " <<  DVAL(LOG_TO_STDERR_ENABLED) << DVAL(LOG_TO_SYSLOG_ENABLED));
     return info.Env().Undefined();
 }
 
@@ -2260,6 +2271,7 @@ fs_napi(Napi::Env env, Napi::Object exports)
 
     exports_fs["dio_buffer_alloc"] = Napi::Function::New(env, dio_buffer_alloc);
     exports_fs["set_debug_level"] = Napi::Function::New(env, set_debug_level);
+    exports_fs["set_log_config"] = Napi::Function::New(env, set_log_config);
 
     exports["fs"] = exports_fs;
 }

src/native/nb_native.gyp

@@ -66,6 +66,7 @@
             'util/struct_buf.h',
             'util/struct_buf.cpp',
             'util/common.h',
+            'util/common.cpp',
             'util/napi.h',
             'util/napi.cpp',
             'util/os.h',
@@ -104,6 +105,7 @@
             'util/buf.cpp',
             'util/buf.h',
             'util/common.h',
+            'util/common.cpp',
             'util/compression.cpp',
             'util/compression.h',
             'util/gf2.h',

src/native/util/common.cpp

@@ -0,0 +1,7 @@
+#include "common.h"
+
+namespace noobaa
+{
+bool LOG_TO_STDERR_ENABLED = true;
+bool LOG_TO_SYSLOG_ENABLED = false;
+}

src/native/util/common.h

@@ -18,6 +18,7 @@
 
 #include "backtrace.h"
 #include "os.h"
+#include "syslog.h"
 
 namespace noobaa
 {
@@ -28,19 +29,46 @@ namespace noobaa
 
 #define DVAL(x) #x "=" << x << " "
 
-#define LOG(x) std::cerr << LOG_PREFIX() << x << std::endl
+extern bool LOG_TO_STDERR_ENABLED;
+extern bool LOG_TO_SYSLOG_ENABLED;
+
+#define STD_LOG(x)                                      \
+    do {                                                \
+        std::cerr << x << std::endl;                    \
+    } while (0) 
+
+#define SYS_LOG(x)                                      \
+    do {                                                \
+        const char* log_msg = x.c_str();                \
+        int facility = LOG_LOCAL0;                      \
+        int priority = 0;                               \
+        ::syslog(priority | facility, "%s", log_msg);   \
+    } while (0)
+
+#define LOG(x)                                          \
+    do {                                                \
+        std::ostringstream oss;                         \
+        oss << "" << LOG_PREFIX() << x;                 \
+        std::string message = oss.str();                \
+        if (LOG_TO_STDERR_ENABLED) {                    \
+            STD_LOG(message);                           \
+        }                                               \
+        if (LOG_TO_SYSLOG_ENABLED) {                    \
+            SYS_LOG(message);                           \
+        }                                               \
+    } while (0)   
 
 // to use DBG the module/file should use either DBG_INIT or DBG_INIT_VAR.
 #define DBG_INIT(level) static int __module_debug_var__ = level
 #define DBG_INIT_VAR(debug_var) static int& __module_debug_var__ = debug_var
 #define DBG_SET_LEVEL(level) __module_debug_var__ = level
 #define DBG_GET_LEVEL() (__module_debug_var__)
 #define DBG_VISIBLE(level) (level <= __module_debug_var__)
-#define DBG(level, x)                        \
-    do {                                     \
-        if (DBG_VISIBLE(level)) {            \
-            LOG("[L" << level << "] " << x); \
-        }                                    \
+#define DBG(level, x)                                 \
+    do {                                              \
+        if (DBG_VISIBLE(level)) {                     \
+            LOG("[L" << level << "] " << x);          \
+        }                                             \
     } while (0)
 #define DBG0(x) DBG(0, x)
 #define DBG1(x) DBG(1, x)

src/sdk/nb.d.ts

@@ -942,6 +942,7 @@ interface NativeFS {
 
     dio_buffer_alloc(size: number): Buffer;
     set_debug_level(level: number);
+    set_log_config(stderr_enabled: boolean, syslog_enabled: boolean,);
 
     S_IFMT: number;
     S_IFDIR: number;

src/server/bg_services/semaphore_monitor.js

@@ -27,7 +27,6 @@ class SemaphoreMonitor {
     }
 
     async run_batch() {
-        dbg.log1("semaphore_monitor: START");
         if (!this._can_run()) return;
         try {
             this.run_semaphore_monitor();

src/server/system_services/schemas/nsfs_config_schema.js

@@ -139,7 +139,15 @@ const nsfs_node_config_schema = {
         ENDPOINT_PROCESS_TITLE: {
             type: 'string',
             description: 'This flag will set noobaa process title for letting GPFS to identify the noobaa endpoint processes.'
-        }
+        },
+        LOG_TO_SYSLOG_ENABLED: {
+            type: 'boolean',
+            doc: 'This flag will enable syslog logging for the application.'
+        },
+        LOG_TO_STDERR_ENABLED: {
+            type: 'boolean',
+            doc: 'This flag will decide whether need to push logs to the console or not.'
+        },
     }
 };
 

src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js

@@ -4,6 +4,7 @@
 
 const nsfs_schema_utils = require('../../../manage_nsfs/nsfs_schema_utils');
 const RpcError = require('../../../rpc/rpc_error');
+const config = require('../../../../config');
 
 describe('schema validation NC NSFS config', () => {
 
@@ -251,6 +252,31 @@ describe('schema validation NC NSFS config', () => {
             nsfs_schema_utils.validate_nsfs_config_schema(config_data);
         });
     });
+
+    describe('skip/unskip schema check by config test', () => {
+
+        it('unskip schema check - config.LOG_TO_SYSLOG_ENABLED=false nsfs_config.LOG_TO_SYSLOG_ENABLED=bla - invalid config - should fail', () => {
+            config.LOG_TO_SYSLOG_ENABLED = false;
+            const config_data = {
+                LOG_TO_SYSLOG_ENABLED: 'bla',
+            };
+            const reason = 'Test should have failed because of wrong type ' +
+            'LOG_TO_SYSLOG_ENABLED must be boolean';
+            const message = `must be boolean | {"type":"boolean"} | "/LOG_TO_SYSLOG_ENABLED"`;
+            assert_validation(config_data, reason, message);
+        });
+
+        it('unskip schema check - config.LOG_TO_STDERR_ENABLED=false nsfs_config.LOG_TO_STDERR_ENABLED=bla - invalid config - should fail', () => {
+            config.LOG_TO_STDERR_ENABLED = false;
+            const config_data = {
+                LOG_TO_STDERR_ENABLED: 'bla',
+            };
+            const reason = 'Test should have failed because of wrong type ' +
+            'LOG_TO_STDERR_ENABLED must be boolean';
+            const message = `must be boolean | {"type":"boolean"} | "/LOG_TO_STDERR_ENABLED"`;
+            assert_validation(config_data, reason, message);
+        });
+    });
 });
 
 function assert_validation(config_to_validate, reason, basic_message) {

src/util/debug_module.js

@@ -41,7 +41,6 @@ util.inspect.defaultOptions.depth = 10;
 util.inspect.defaultOptions.colors = true;
 util.inspect.defaultOptions.breakLength = Infinity;
 
-
 //Detect our context, node/atom/browser
 //Different context requires different handling, for example rotating file steam usage or console wrapping
 let syslog;
@@ -379,7 +378,7 @@ class InternalDebugLogger {
     }
 
     log_internal(msg_info) {
-        if (syslog) {
+        if (syslog && config.LOG_TO_SYSLOG_ENABLED) {
             // syslog path
             syslog(this._levels_to_syslog[msg_info.level], msg_info.message_syslog, config.DEBUG_FACILITY);
         } else if (this._log_file) {
@@ -389,7 +388,7 @@ class InternalDebugLogger {
         // This is also used in order to log to the console
         // browser workaround, don't use rotating file steam. Add timestamp and level
         const logfunc = LOG_FUNC_PER_LEVEL[msg_info.level] || 'log';
-        if (this._log_console_silent) {
+        if (this._log_console_silent || !config.LOG_TO_STDERR_ENABLED) {
             // noop
         } else if (console_wrapper) {
             process.stderr.write(msg_info.message_console + '\n');
@@ -586,10 +585,12 @@ if (console_wrapper) {
     console_wrapper.register_logger(conlogger);
 }
 
-let native_log_level = LOG_LEVEL;
-if (process.env.NOOBAA_LOG_LEVEL) {
-    native_log_level = dbg_conf.level;
-}
-if (Number(native_log_level)) {
-    nb_native().fs.set_debug_level(Number(native_log_level));
+
+function set_log_config() {
+    const dbg_native_conf = debug_config.get_debug_config(process.env.NOOBAA_LOG_LEVEL);
+    nb_native().fs.set_debug_level(dbg_native_conf.level);
+    nb_native().fs.set_log_config(config.LOG_TO_STDERR_ENABLED, config.LOG_TO_SYSLOG_ENABLED);
 }
+
+config.event_emitter.on("config_updated", set_log_config);
+set_log_config();