Skip to content

Commit

Permalink
NC | Config Directory Restructure
Browse files Browse the repository at this point in the history 
Signed-off-by: Romy <[email protected]>
  • Loading branch information
@romayalon
romayalon committed Aug 13, 2024
1 parent a72063b commit d65e8f9
Show file tree
Hide file tree
Showing 24 changed files with 1,127 additions and 804 deletions.
1 change: 1 addition & 0 deletions config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -861,6 +861,7 @@ config.NSFS_NC_CONF_DIR = process.env.NSFS_NC_CONF_DIR || '';
config.NSFS_TEMP_CONF_DIR_NAME = '.noobaa-config-nsfs';
config.NSFS_NC_CONFIG_DIR_BACKEND = '';
config.NSFS_NC_STORAGE_BACKEND = '';
config.CONFIG_DIR_FALLBACK_ENABLED = true;
config.ENDPOINT_PORT = Number(process.env.ENDPOINT_PORT) || 6001;
config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;
Expand Down
55 changes: 29 additions & 26 deletions src/cmd/manage_nsfs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ const minimist = require('minimist');
const config = require('../../config');
const P = require('../util/promise');
const nb_native = require('../util/nb_native');
const { ConfigFS, JSON_SUFFIX } = require('../sdk/config_fs');
const { ConfigFS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../sdk/config_fs');
const cloud_utils = require('../util/cloud_utils');
const native_fs_utils = require('../util/native_fs_utils');
const mongo_utils = require('../util/mongo_utils');
Expand Down Expand Up @@ -387,7 +387,7 @@ async function add_account(data) {
await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.ADD, undefined);

const access_key = has_access_keys(data.access_keys) ? data.access_keys[0].access_key : undefined;
const name_exists = await config_fs.is_account_exists({ name: data.name });
const name_exists = await config_fs.is_account_exists({ name: data.name }, undefined, { fallback: true });
const access_key_exists = access_key && await config_fs.is_account_exists({ access_key });

const event_arg = data.name ? data.name : access_key;
Expand All @@ -405,7 +405,7 @@ async function add_account(data) {
// for validating against the schema we need an object, hence we parse it back to object
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await config_fs.create_account_config_file(data.name, account, true);
await config_fs.create_account_config_file(account, true);
write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg });
}

Expand Down Expand Up @@ -434,7 +434,7 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
// for validating against the schema we need an object, hence we parse it back to object
const account = encrypted_data ? JSON.parse(encrypted_data) : data;
nsfs_schema_utils.validate_account_schema(account);
await config_fs.update_account_config_file(data.name, account, undefined, undefined);
await config_fs.update_account_config_file(account, undefined, undefined, undefined);
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
return;
}
Expand All @@ -446,7 +446,7 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
secret_key: data.access_keys[0].secret_key,
};

const name_exists = update_name && await config_fs.is_account_exists({ name: data.name });
const name_exists = update_name && await config_fs.is_account_exists({ name: data.name }, undefined, { fallback: true });
const access_key_exists = update_access_key &&
await config_fs.is_account_exists({ access_key: data.access_keys[0].access_key.unwrap() });

Expand All @@ -466,23 +466,23 @@ async function update_account(data, is_flag_iam_operate_on_root_account) {
const parsed_data = JSON.parse(encrypted_data);
nsfs_schema_utils.validate_account_schema(parsed_data);
if (update_name) {
await config_fs.create_account_config_file(new_name, parsed_data, true, [cur_access_key]);
await config_fs.delete_account_config_file(cur_name, data.access_keys);
} else if (update_access_key) {
await config_fs.update_account_config_file(cur_name, parsed_data, parsed_data.access_keys, [cur_access_key]);
await config_fs.update_account_config_file(parsed_data, cur_name, undefined, undefined);
}
if (update_access_key) {
await config_fs.update_account_config_file(parsed_data, undefined, parsed_data.access_keys, [{ access_key: cur_access_key }]);
}
write_stdout_response(ManageCLIResponse.AccountUpdated, data);
}

async function delete_account(data) {
await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.DELETE, undefined);
await config_fs.delete_account_config_file(data.name, data.access_keys);
await config_fs.delete_account_config_file(data._id, data.name, data.access_keys);
write_stdout_response(ManageCLIResponse.AccountDeleted, '', { account: data.name });
}

async function get_account_status(data, show_secrets) {
await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.STATUS, undefined);
const options = { show_secrets, decrypt_secret_key: show_secrets };
const options = { show_secrets, decrypt_secret_key: show_secrets, fallback: true };

try {
const config_data = data.name === undefined ?
Expand Down Expand Up @@ -578,39 +578,42 @@ function filter_bucket(bucket, filters) {
* @param {object} [filters]
*/
async function list_config_files(type, wide, show_secrets, filters = {}) {
let entries;
// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
let entries = [];
const suffix = type === TYPES.ACCOUNT ? SYMLINK_SUFFIX : JSON_SUFFIX;
const should_filter = Object.keys(filters).length > 0;
const is_filter_by_name = filters.name !== undefined;
if (is_filter_by_name) {
entries = [{'name': filters.name + JSON_SUFFIX}];
} else {
entries = type === TYPES.ACCOUNT ?
await config_fs.list_root_accounts() :
await config_fs.list_buckets();
}

const should_filter = Object.keys(filters).length > 0;
// decryption causing mkm initalization
// decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true)
const options = {
show_secrets: show_secrets || should_filter,
decrypt_secret_key: show_secrets,
silent_if_missing: true
silent_if_missing: true,
fallback: true
};

// in case we have a filter by name, we don't need to read all the entries and iterate them
// instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance)
if (is_filter_by_name) {
entries = [{'name': filters.name + suffix}];
} else if (type === TYPES.ACCOUNT) {
entries = await config_fs.list_accounts(options);
} else if (type === TYPES.BUCKET) {
entries = await config_fs.list_buckets();
}

let config_files_list = await P.map_with_concurrency(10, entries, async entry => {
if (entry.name.endsWith(JSON_SUFFIX)) {
if (entry.name.endsWith(suffix)) {
if (wide || should_filter) {
const data = type === TYPES.ACCOUNT ?
await config_fs.get_account_by_name(entry.name, options) :
await config_fs.get_bucket_by_name(entry.name, options);
if (!data) return undefined;
if (should_filter && !filter_list_item(type, data, filters)) return undefined;
// remove secrets on !show_secrets && should filter
return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) };
return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf(suffix)) };
} else {
return { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) };
return { name: entry.name.slice(0, entry.name.indexOf(suffix)) };
}
}
});
Expand Down
5 changes: 0 additions & 5 deletions src/deploy/NVA_build/standalone_deploy_nsfs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,8 @@ function execute() {

# Please note that the command we use here are without "sudo" because we are running from the container with Root permissions
function main() {
# Add accounts to run ceph tests
execute "node src/cmd/manage_nsfs account add --name cephalt --new_buckets_path ${FS_ROOT_1} --uid 1000 --gid 1000" nsfs_cephalt.log
execute "node src/cmd/manage_nsfs account add --name cephtenant --new_buckets_path ${FS_ROOT_2} --uid 2000 --gid 2000" nsfs_cephtenant.log

# Start noobaa service
execute "node src/cmd/nsfs" nsfs.log

# Wait for sometime to process to start
sleep 10
}
Expand Down
24 changes: 14 additions & 10 deletions src/manage_nsfs/health.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ const nb_native = require('../util/nb_native');
const native_fs_utils = require('../util/native_fs_utils');
const { read_stream_join } = require('../util/buffer_utils');
const { make_https_request } = require('../util/http_utils');
const { JSON_SUFFIX } = require('../sdk/config_fs');
const { JSON_SUFFIX, SYMLINK_SUFFIX } = require('../sdk/config_fs');
const { TYPES } = require('./manage_nsfs_constants');
const { get_boolean_or_string_value, throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils');
const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
Expand Down Expand Up @@ -327,7 +327,7 @@ class NSFSHealth {
config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
} else if (type === TYPES.ACCOUNT) {
// TODO - handle iam accounts when directory structure changes - read_account_by_id
config_dir_path = this.config_fs.accounts_dir_path;
config_dir_path = this.config_fs.accounts_by_name_dir_path;
config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
}
// TODO - this is not a good handling for that - we need to take it to an upper level
Expand All @@ -339,11 +339,15 @@ class NSFSHealth {
};
}

const entries = type === TYPES.BUCKET ?
await this.config_fs.list_buckets() :
await this.config_fs.list_root_accounts();
let config_files;
if (type === TYPES.BUCKET) {
const entries = await this.config_fs.list_buckets();
config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(JSON_SUFFIX));
} else {
const entries = await this.config_fs.list_accounts({ fallback: true });
config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(SYMLINK_SUFFIX));

const config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(JSON_SUFFIX));
}
for (const config_file of config_files) {
// config_file get data or push error
const { config_data = undefined, err_obj = undefined } =
Expand All @@ -358,7 +362,7 @@ class NSFSHealth {
let res;
const storage_path = type === TYPES.BUCKET ?
config_data.path :
config_data.nsfs_account_config.new_buckets_path;
config_data.nsfs_account_config?.new_buckets_path;

if (type === TYPES.ACCOUNT) {
const config_file_path = this.config_fs.get_account_path_by_name(config_file.name);
Expand Down Expand Up @@ -391,13 +395,13 @@ class NSFSHealth {
config_data = type === TYPES.BUCKET ?
await this.config_fs.get_bucket_by_name(config_file_name) :
// TODO - should be changed to id when moving to new structure for supporting iam accounts
await this.config_fs.get_account_by_name(config_file_name);
await this.config_fs.get_account_by_name(config_file_name, { fallback: true });
} catch (err) {
let err_code;
const config_file_path = type === TYPES.BUCKET ?
await this.config_fs.get_bucket_path_by_name(config_file_name) :
this.config_fs.get_bucket_path_by_name(config_file_name) :
// TODO - should be changed to id when moving to new structure for supporting iam accounts
await this.config_fs.get_account_path_by_name(config_file_name);
this.config_fs.get_account_path_by_name(config_file_name);

if (err.code === 'ENOENT') {
dbg.log1(`Error: Config file path should be a valid path`, config_file_path, err);
Expand Down
16 changes: 7 additions & 9 deletions src/manage_nsfs/manage_nsfs_validations.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@

const config = require('../../config');
const dbg = require('../util/debug_module')(__filename);
const path = require('path');
const net = require('net');
const P = require('../util/promise');
const nb_native = require('../util/nb_native');
const string_utils = require('../util/string_utils');
const { JSON_SUFFIX } = require('../sdk/config_fs');
const { JSON_SUFFIX, SYMLINK_SUFFIX } = require('../sdk/config_fs');
const native_fs_utils = require('../util/native_fs_utils');
const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
Expand Down Expand Up @@ -354,7 +352,7 @@ async function validate_bucket_args(config_fs, data, action) {
if (data.s3_policy) {
try {
await bucket_policy_utils.validate_s3_policy(data.s3_policy, data.name,
async principal => config_fs.is_account_exists({ name: principal })
async principal => config_fs.is_account_exists({ name: principal }, account.owner, { fallback: true })
);
} catch (err) {
dbg.error('validate_bucket_args invalid bucket policy err:', err);
Expand Down Expand Up @@ -503,12 +501,12 @@ async function validate_account_not_owns_buckets(config_fs, account_name) {
* @param {string} action
*/
async function check_if_root_account_does_not_have_IAM_users(config_fs, account_to_check, action) {
const fs_context = config_fs.fs_context;
const entries = await nb_native().fs.readdir(fs_context, config_fs.accounts_dir_path);
// TODO - For supporting IAM, we need to check if {config_dir}/identities/{account_id}/users/ has anything inside
const options = { fallback: true };
const entries = await config_fs.list_accounts(options);
await P.map_with_concurrency(10, entries, async entry => {
if (entry.name.endsWith(JSON_SUFFIX)) {
const full_path = path.join(config_fs.accounts_dir_path, entry.name);
const account_data = await config_fs.get_config_data(full_path);
if (entry.name.endsWith(SYMLINK_SUFFIX)) {
const account_data = await config_fs.get_account_by_name(entry.name, options);
if (entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME)) return undefined;
const is_root_account_owns_user = check_root_account_owns_user(account_to_check, account_data);
if (is_root_account_owns_user) {
Expand Down
Loading

0 comments on commit d65e8f9

Please sign in to comment.