Merge pull request #8226 from romayalon/romy-5.16.1-backports

5.16.1 Backports

docs/dev_guide/standalone_with_db.md

@@ -181,7 +181,7 @@ You can use the configuration for `rsyslog` and `logrotate` for RHEL8. The logs
 
 ```
 sudo cp src/deploy/standalone/noobaa_syslog.conf /etc/rsyslog.d/
-sudo cp src/deploy/standalone/logrotate_noobaa.conf /etc/logrotate.d/
+sudo cp src/deploy/standalone/noobaa-logrotate /etc/logrotate.d/
 sudo systemctl restart systemd-journald rsyslog
 ```
 
@@ -192,7 +192,7 @@ Additionally, it would be helpful if you configure to disable the rate limit of
 1. Add the 2 lines below into `/etc/systemd/journald.conf`
 
 ```
-RateLimitInterva]lSec=0s
+RateLimitIntervalSec=0s
 RateLimitBurst=0
 ```
 

docs/non_containerized_NSFS.md

@@ -582,14 +582,14 @@ systemctl status rsyslog
 
 Noobaa logs are pushed to `var/log/noobaa.log` and the log is rotated and compressed daily.
 
-Verify the rsyslog and logrotate rpm configuration is complete by checking the files `etc/rsyslog.d/noobaa_syslog.conf` for rsyslog and `etc/logrotate.d/noobaa/logrotate_noobaa.conf` for logrotate.These files contain the noobaa specific configuration for rsyslog and logrotate.
+Verify the rsyslog and logrotate rpm configuration is complete by checking the files `etc/rsyslog.d/noobaa_syslog.conf` for rsyslog and `etc/logrotate.d/noobaa-logrotate` for logrotate.These files contain the noobaa specific configuration for rsyslog and logrotate.
 
-Logrotate configuration is set up under `/etc/logrotate.d/noobaa/`. In order to trigger rotation when log files reache size threshold, each log file (`/var/log/noobaa.log` and `/var/log/noobaa_events.log`) uses its own rsyslog `outchannel` that triggers logrotate once file size reaches a limit (105MB). Logrotate is also typically called from cron on daily schedule.
+Logrotate configuration is set up under `/etc/logrotate.d/`.  Logrotate is called from cron on daily schedule.
 
 To rotate the logs manually run.
 
 ```
-logrotate /etc/logrotate.d/noobaa/logrotate_noobaa.conf
+logrotate /etc/logrotate.d/noobaa-logrotate
 ```
 
 **Create env file under the configuration directory (OPTIONAL) -**

src/cmd/manage_nsfs.js

@@ -20,7 +20,7 @@ const { print_usage } = require('../manage_nsfs/manage_nsfs_help_utils');
 const { TYPES, ACTIONS, LIST_ACCOUNT_FILTERS, LIST_BUCKET_FILTERS,
     GLACIER_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
 const { throw_cli_error, write_stdout_response, get_config_file_path, get_symlink_config_file_path,
-    get_config_data, get_boolean_or_string_value} = require('../manage_nsfs/manage_nsfs_cli_utils');
+    get_config_data, get_boolean_or_string_value, get_config_data_if_exists } = require('../manage_nsfs/manage_nsfs_cli_utils');
 const { validate_input_types, validate_bucket_args, validate_account_args,
         verify_delete_account, validate_whitelist_arg, verify_whitelist_ips,
         _validate_access_keys } = require('../manage_nsfs/manage_nsfs_validations');
@@ -613,7 +613,8 @@ async function list_config_files(type, config_path, wide, show_secrets, filters)
         if (entry.name.endsWith('.json')) {
             if (wide || should_filter) {
                 const full_path = path.join(config_path, entry.name);
-                const data = await get_config_data(config_root_backend, full_path, show_secrets || should_filter);
+                const data = await get_config_data_if_exists(config_root_backend, full_path, show_secrets || should_filter);
+                if (!data) return undefined;
                 // decryption causing mkm initalization
                 // decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true)
                 if (data.access_keys && show_secrets) data.access_keys = await nc_mkm.decrypt_access_keys(data);
@@ -626,6 +627,7 @@ async function list_config_files(type, config_path, wide, show_secrets, filters)
         }
     });
     // it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it
+    // in case the entry was deleted during the list it also inserts undefined
     config_files_list = config_files_list.filter(item => item);
 
     return config_files_list;

src/deploy/NVA_build/NooBaa.Dockerfile

@@ -110,7 +110,7 @@ COPY ./src/deploy/NVA_build/supervisord.orig /tmp/supervisord
 COPY ./src/deploy/NVA_build/supervisorctl.bash_completion /etc/bash_completion.d/supervisorctl
 COPY ./src/deploy/NVA_build/rsyslog.conf /etc/rsyslog.conf
 COPY ./src/deploy/NVA_build/noobaa_syslog.conf /etc/rsyslog.d/
-COPY ./src/deploy/NVA_build/logrotate_noobaa.conf /etc/logrotate.d/noobaa/
+COPY ./src/deploy/NVA_build/noobaa-logrotate /etc/logrotate.d/
 COPY ./src/deploy/NVA_build/noobaa_init.sh /noobaa_init_files/
 
 COPY ./src/deploy/NVA_build/setup_platform.sh /usr/bin/setup_platform.sh

src/deploy/NVA_build/logrotate.sh

@@ -8,7 +8,7 @@ echo "*/5 * * * * /root/node_modules/noobaa-core/src/deploy/NVA_build/noobaa_log
 while true; do
 
     echo "$(date): =================================== running logrotate ===================================" >/dev/stdout 2>&1
-    /usr/sbin/logrotate -v /etc/logrotate.d/noobaa/logrotate_noobaa.conf >/dev/stdout 2>&1
+    /usr/sbin/logrotate -v /etc/logrotate.d/noobaa-logrotate >/dev/stdout 2>&1
     echo "$(date): =================================== logrotate Done ======================================" >/dev/stdout 2>&1
 
     sleep 60

src/deploy/RPM_build/RPM.Dockerfile

@@ -5,7 +5,6 @@ ARG CENTOS_VER=9
 ARG BUILD_S3SELECT=0
 ARG BUILD_S3SELECT_PARQUET=0
 
-RUN mkdir -p /etc/logrotate.d/noobaa/
 RUN mkdir -p /etc/noobaa.conf.d/
 
 COPY ./src/agent ./src/agent
@@ -35,8 +34,7 @@ COPY ./package*.json ./
 COPY ./binding.gyp .
 COPY ./src/deploy/standalone/noobaa_rsyslog.conf ./src/deploy/standalone/noobaa_rsyslog.conf
 COPY ./src/deploy/standalone/noobaa_syslog.conf ./src/deploy/standalone/noobaa_syslog.conf
-COPY ./src/deploy/standalone/logrotate_noobaa.conf ./src/deploy/standalone/logrotate_noobaa.conf
-COPY ./src/deploy/standalone/logrotate_noobaa.sh ./src/deploy/standalone/logrotate_noobaa.sh
+COPY ./src/deploy/standalone/noobaa-logrotate ./src/deploy/standalone/noobaa-logrotate
 COPY ./src/manage_nsfs ./src/manage_nsfs
 
 WORKDIR /build

src/deploy/RPM_build/noobaa.spec

@@ -87,15 +87,14 @@ mkdir -p $RPM_BUILD_ROOT/etc/noobaa.conf.d/
 mkdir -p $RPM_BUILD_ROOT/etc/rsyslog.d/
 mv $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/standalone/noobaa_syslog.conf $RPM_BUILD_ROOT/etc/rsyslog.d/noobaa_syslog.conf
 
-mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d/noobaa
-ln -s /usr/local/noobaa-core/src/deploy/standalone/logrotate_noobaa.conf $RPM_BUILD_ROOT/etc/logrotate.d/noobaa/logrotate_noobaa.conf
-chmod +x $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/standalone/logrotate_noobaa.sh
+mkdir -p $RPM_BUILD_ROOT/etc/logrotate.d
+mv $RPM_BUILD_ROOT/usr/local/noobaa-core/src/deploy/standalone/noobaa-logrotate $RPM_BUILD_ROOT/etc/logrotate.d/noobaa-logrotate
 
 %files
 /usr/local/noobaa-core
 %{_unitdir}/noobaa.service
-/etc/logrotate.d/noobaa/logrotate_noobaa.conf
-/etc/rsyslog.d/noobaa_syslog.conf
+%config(noreplace) /etc/logrotate.d/noobaa-logrotate
+%config(noreplace) /etc/rsyslog.d/noobaa_syslog.conf
 /etc/noobaa.conf.d/
 /usr/local/bin/noobaa-cli
 %doc

src/deploy/spectrum_archive/deployment_guide.md

@@ -157,12 +157,12 @@ systemctl status rsyslog
 
 Noobaa logs are pushed to `/var/log/noobaa.log` and the log is rotated and compressed daily.
 
-Verify the rsyslog and logrotate rpm configuration is complete by checking the files `/etc/rsyslog.d/noobaa_syslog.conf` and `/etc/rsyslog.d/noobaa_rsyslog.conf` for rsyslog and `/etc/logrotate.d/noobaa/logrotate_noobaa.conf` for logrotate.These files contain the noobaa specific configuration for rsyslog and logrotate.
+Verify the rsyslog and logrotate rpm configuration is complete by checking the files `/etc/rsyslog.d/noobaa_syslog.conf` and `/etc/rsyslog.d/noobaa_rsyslog.conf` for rsyslog and `/etc/logrotate.d/noobaa-logrotate` for logrotate.These files contain the noobaa specific configuration for rsyslog and logrotate.
 
 Rotate the logs manually.
 
 ```
-logrotate /etc/logrotate.d/noobaa/logrotate_noobaa.conf
+logrotate /etc/logrotate.d/noobaa-logrotate
 ```
 
 # FAQ

src/deploy/standalone/logrotate_noobaa.conf → src/deploy/standalone/noobaa-logrotate

@@ -12,6 +12,7 @@
     postrotate 
         killall -HUP rsyslogd || true
         killall -HUP syslogd || true
+        systemctl reload syslog-ng > /dev/null 2>&1 || true
     endscript
 }
 
@@ -29,6 +30,7 @@
     postrotate
         killall -HUP rsyslogd || true
         killall -HUP syslogd || true
+        systemctl reload syslog-ng > /dev/null 2>&1 || true
     endscript
 }
 
@@ -44,5 +46,6 @@
     postrotate
         killall -HUP rsyslogd || true
         killall -HUP syslogd || true
+        systemctl reload syslog-ng > /dev/null 2>&1 || true
     endscript
 }

src/deploy/standalone/noobaa_syslog.conf

@@ -4,6 +4,9 @@ $umask 0000
 $FileCreateMode 0644
 $EscapeControlCharactersOnReceive off
 
+$imjournalRatelimitInterval 0
+$imjournalRatelimitBurst 0
+
 # Provides UDP/TCP forwarding. The IP/DNS is the server's IP/DNS address
 # This is an example of sending everything except NooBaa logs using UDP
 # When changing this format make sure to change the relevant functions in os_utils
@@ -28,14 +31,12 @@ template(name="LogseneFormat" type="list" option.json="on") {
   constant(value="\"}\n")
 }
 
-# 105M limit(110100480), 5MB bigger than logrotate value. When updating this value update logrotate also.
-$outchannel log_rotation,/var/log/noobaa.log,110100480,/usr/local/noobaa-core/src/deploy/standalone/logrotate_noobaa.sh
-local0.* :omfile:$log_rotation
+# For servers
+local0.*        /var/log/noobaa.log;RSYSLOG_FileFormat
 &stop
 
 # For events
-$outchannel event_rotation,/var/log/noobaa_events.log,110100480,/usr/local/noobaa-core/src/deploy/standalone/logrotate_noobaa.sh
-local2.* :omfile:$event_rotation;LogseneFormat
+local2.*        /var/log/noobaa_events.log;LogseneFormat
 &stop
 
 # For clients

src/manage_nsfs/manage_nsfs_cli_utils.js

@@ -1,6 +1,7 @@
 /* Copyright (C) 2024 NooBaa */
 'use strict';
 
+const dbg = require('../util/debug_module')(__filename);
 const _ = require('lodash');
 const path = require('path');
 const nb_native = require('../util/nb_native');
@@ -53,6 +54,23 @@ async function get_config_data(config_root_backend, config_file_path, show_secre
     return config_data;
 }
 
+/**
+ * get_config_data_if_exists will read a config file and return its content 
+ * while omitting secrets if show_secrets flag was not provided
+ * if the config file was deleted (encounter ENOENT error) - continue (returns undefined)
+ * @param {string} config_file_path
+ * @param {boolean} [show_secrets]
+ */
+async function get_config_data_if_exists(config_root_backend, config_file_path, show_secrets = false) {
+    try {
+        const config_data = await get_config_data(config_root_backend, config_file_path, show_secrets);
+        return config_data;
+    } catch (err) {
+        dbg.warn('get_config_data_if_exists: with config_file_path', config_file_path, 'got an error', err);
+        if (err.code !== 'ENOENT') throw err;
+    }
+}
+
 /**
  * get_bucket_owner_account will return the account of the bucket_owner
  * otherwise it would throw an error
@@ -118,3 +136,5 @@ exports.get_boolean_or_string_value = get_boolean_or_string_value;
 exports.get_config_data = get_config_data;
 exports.get_bucket_owner_account = get_bucket_owner_account;
 exports.get_options_from_file = get_options_from_file;
+exports.get_config_data_if_exists = get_config_data_if_exists;
+

src/manage_nsfs/manage_nsfs_validations.js

@@ -12,7 +12,7 @@ const native_fs_utils = require('../util/native_fs_utils');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
 const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
 const { throw_cli_error, get_config_file_path, get_bucket_owner_account,
-    get_config_data, get_options_from_file } = require('../manage_nsfs/manage_nsfs_cli_utils');
+    get_options_from_file, get_config_data_if_exists } = require('../manage_nsfs/manage_nsfs_cli_utils');
 const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES,
     GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS } = require('../manage_nsfs/manage_nsfs_constants');
 
@@ -335,11 +335,12 @@ function _validate_access_keys(access_key, secret_key) {
 async function verify_delete_account(config_root_backend, buckets_dir_path, account_name) {
     const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
     const entries = await nb_native().fs.readdir(fs_context, buckets_dir_path);
+    let data;
     await P.map_with_concurrency(10, entries, async entry => {
         if (entry.name.endsWith('.json')) {
             const full_path = path.join(buckets_dir_path, entry.name);
-            const data = await get_config_data(config_root_backend, full_path);
-            if (data.bucket_owner === account_name) {
+            data = await get_config_data_if_exists(config_root_backend, full_path);
+            if (data && data.bucket_owner === account_name) {
                 const detail_msg = `Account ${account_name} has bucket ${data.name}`;
                 throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasBuckets, detail_msg);
             }

src/sdk/bucketspace_fs.js

@@ -228,7 +228,15 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
                 return;
             }
             const bucket_name = this.get_bucket_name(entry.name);
-            const bucket = await object_sdk.read_bucket_sdk_config_info(bucket_name);
+            let bucket;
+            try {
+                bucket = await object_sdk.read_bucket_sdk_config_info(bucket_name);
+            } catch (err) {
+                dbg.warn('list_buckets: read_bucket_sdk_config_info of bucket', bucket_name, 'got an error', err);
+                // in case the config file was deleted during the bucket list - we will continue
+                if (err.rpc_code !== 'NO_SUCH_BUCKET') throw err;
+            }
+            if (!bucket) return;
             const bucket_policy_accessible = await this.has_bucket_action_permission(bucket, account, 's3:ListBucket');
             if (!bucket_policy_accessible) return;
             const fs_accessible = await this.validate_fs_bucket_access(bucket, object_sdk);

src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js

@@ -925,6 +925,13 @@ describe('manage nsfs cli account flow', () => {
             expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.MissingIdentifier.code);
         });
 
+        it('cli list filter by access key (non existing) and name (of account3) - (none)', async () => {
+            const account_options = { config_root, name: 'account3', access_key: 'non-existing-access-key' };
+            const action = ACTIONS.LIST;
+            const res = await exec_manage_cli(TYPES.ACCOUNT, action, account_options);
+            expect(JSON.parse(res).response.reply.map(item => item.name))
+                .toEqual([]);
+        });
     });
 
     describe('cli delete account', () => {

src/test/unit_tests/test_debug_module.js

@@ -90,7 +90,7 @@ mocha.describe('debug_module', function() {
         if (os_utils.IS_MAC) {
             rotation_command = 'ls';
         } else {
-            rotation_command = '/usr/sbin/logrotate /etc/logrotate.d/noobaa';
+            rotation_command = '/usr/sbin/logrotate /etc/logrotate.d';
         }
         return os_utils.exec(rotation_command).then(function() {
             const dbg = new DebugModule('/web/noise/noobaa-core/src/blabla.asd/lll.asd');