NSFS | Improve list objects performance on top of NS FS

Signed-off-by: naveenpaul1 <[email protected]>

Author: naveenpaul1

config.js

@@ -799,6 +799,9 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;
 // of `manage_nsfs glacier restore`
 config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;
 
+// enable/disable unsorted listing application level
+config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false;
+
 // NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies
 // when NooBaa should allow running glacier expiry process
 // NOTE: This will also be in the same timezone as specified in

src/api/object_api.js

@@ -720,6 +720,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {
@@ -774,6 +777,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {

src/endpoint/s3/ops/s3_get_bucket.js

@@ -41,6 +41,7 @@ async function get_bucket(req) {
         bucket: req.params.bucket,
         prefix: req.query.prefix,
         delimiter: req.query.delimiter,
+        list_type: list_type,
         limit: Math.min(max_keys_received, 1000),
         key_marker: list_type === '2' ?
             (cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,

src/native/fs/fs_napi.cpp

@@ -2061,7 +2061,7 @@ struct TellDir : public FSWrapWorker<DirWrap>
     }
     virtual void OnOK()
     {
-        DBG0("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
+        DBG1("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
         Napi::Env env = Env();
         auto res = Napi::BigInt::New(env, _tell_res);
         _deferred.Resolve(res);

src/sdk/namespace_fs.js

@@ -474,6 +474,26 @@ function get_new_buckets_path_by_test_env(new_buckets_full_path, new_buckets_dir
     return is_nc_coretest ? path.join(new_buckets_full_path, new_buckets_dir) : new_buckets_dir;
 }
 
+
+/**
+ * common dummy SDK for testing
+ */
+function make_dummy_object_sdk() {
+    return {
+        requesting_account: {
+            force_md5_etag: false,
+            nsfs_account_config: {
+                uid: process.getuid(),
+                gid: process.getgid(),
+            }
+        },
+        abort_controller: new AbortController(),
+        throw_if_aborted() {
+            if (this.abort_controller.signal.aborted) throw new Error('request aborted signal');
+        }
+    };
+}
+
 /**
  * write_manual_config_file writes config file directly to the file system without using config FS
  * used for creating backward compatibility tests, invalid config files etc
@@ -627,8 +647,10 @@ exports.get_new_buckets_path_by_test_env = get_new_buckets_path_by_test_env;
 exports.write_manual_config_file = write_manual_config_file;
 exports.write_manual_old_account_config_file = write_manual_old_account_config_file;
 exports.delete_manual_config_file = delete_manual_config_file;
+exports.make_dummy_object_sdk = make_dummy_object_sdk;
 exports.create_redirect_file = create_redirect_file;
 exports.delete_redirect_file = delete_redirect_file;
 exports.fail_test_if_default_config_dir_exists = fail_test_if_default_config_dir_exists;
 exports.create_config_dir = create_config_dir;
 exports.clean_config_dir = clean_config_dir;
+