diff --git a/src/tests/ftest/recovery/dlck_basic.py b/src/tests/ftest/recovery/dlck_basic.py new file mode 100644 index 00000000000..7ed5a78038b --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic.py @@ -0,0 +1,51 @@ +""" + Copyright 2026 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os + +from apricot import TestWithServers +from dlck_utils import DlckCommand +from test_utils_pool import add_pool + + +class DlckBasicTest(TestWithServers): + """Test class for dlck command line utility. + + :avocado: recursive + """ + def test_dlck_basic(self): + """Basic Test: Run 'dlck' command + + :avocado: tags=all,daily_regression + :avocado: tags=hw,medium + :avocado: tags=recovery,dlck_cmd + :avocado: tags=DlckBasicTest,test_dlck_basic + """ + errors = [] + dmg = self.get_dmg_command() + self.log_step("Create a pool to run dlck") + pool = add_pool(self) + scm_mount = self.server_managers[0].get_config_value("scm_mount") + host = self.server_managers[0].hosts[0:1] + if self.server_managers[0].manager.job.using_control_metadata: + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + daos_control_dir = os.path.join(control_metadata_dir, "daos_control") + engine_path_dir = os.path.join(daos_control_dir, "engine0") + nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, + storage_mount=scm_mount) + else: + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() + self.log_step("Run dlck command to check the health of the pool and storage") + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + dmg.system_start() + if errors: + self.fail(f"dlck basic test failed with errors: {errors}") + self.log.info("dlck basic test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic.yaml b/src/tests/ftest/recovery/dlck_basic.yaml new file mode 100644 index 00000000000..e59614e4202 --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic.yaml @@ -0,0 +1,19 @@ +hosts: + test_servers: 2 + test_clients: 1 + +timeout: 600 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + storage: auto + 1: + targets: 4 + storage: auto + +pool: + size: 80% diff --git a/src/tests/ftest/recovery/dlck_basic_fault.py b/src/tests/ftest/recovery/dlck_basic_fault.py new file mode 100644 index 00000000000..1792ea8a9af --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic_fault.py @@ -0,0 +1,94 @@ +""" + Copyright 2026 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os + +from apricot import TestWithServers +from dlck_utils import DlckCommand +from fault_config_utils import FaultInjection +from file_utils import distribute_files +from run_utils import run_remote +from test_utils_pool import add_pool + + +class DlckBasicFaultTest(TestWithServers): + """Test class for dlck command line utility. + + :avocado: recursive + """ + def test_dlck_basic_fault(self): + """Basic Fault Test: Run 'dlck' injecting basic faults. + + :avocado: tags=all,full_regression + :avocado: tags=hw,medium + :avocado: tags=recovery,dlck_cmd + :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault + """ + errors = [] + faults_dict = {} + faults_object = FaultInjection() + faults_dict = faults_object.get_faults_dict() + fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*') + dmg = self.get_dmg_command() + pool = add_pool(self) + scm_mount = self.server_managers[0].get_config_value("scm_mount") + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + if fault_inject_file == "None set for now": + self.fail("D_FI_CONFIG environment variable not set") + self.log.info("Fault injection file contents") + cmd = "cat {}".format(fault_inject_file) + host = self.server_managers[0].hosts[0:1] + self.log_step("Run the command to read the fault injection file contents") + run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30) + # Run the testing with the first fault which is injected at the beginning of the test. + if self.server_managers[0].manager.job.using_control_metadata: + log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file")) + control_metadata_dir = os.path.join(log_dir, "control_metadata") + daos_control_dir = os.path.join(control_metadata_dir, "daos_control") + engine_path_dir = os.path.join(daos_control_dir, "engine0") + nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf") + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, + storage_mount=scm_mount) + else: + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + self.log_step("Perform dmg system stop to run dlck command") + dmg.system_stop() + self.log_step("Run dlck command after injecting the first fault") + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + # Now, run the other fault injection flags without rebooting or creating any new pools. + # Rebooting the servers or creating the new pools will result in injecting fault in + # the wrong test code. Fault injections should done only for the dlck alone. + for test_fault in fault_list: + with open(fault_inject_file, 'w') as f: + f.write("fault_config:\n") + count = 0 + for key, value in faults_dict[test_fault].items(): + if count == 0: + f.write(f"- {key}: \'{value}\'\n") + else: + f.write(f" {key}: \'{value}\'\n") + count += 1 + self.log.info("Reading the updated fault injection file contents") + with open(fault_inject_file, 'r') as f: + file_data = f.read() + self.log.info("\n %s", file_data) + distribute_files(self.log, self.hostlist_servers, fault_inject_file, + fault_inject_file) + if self.server_managers[0].manager.job.using_control_metadata: + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf, + storage_mount=scm_mount) + else: + dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount) + self.log_step("Run dlck command after injecting fault") + result = dlck_cmd.run() + if not result.passed: + errors.append(f"dlck failed on {result.failed_hosts}") + self.log_step("Run the dmg start command") + dmg.system_start() + if not errors: + self.fail(f"dlck basic faulttest failed with errors: {errors}") + self.log.info("dlck basic fault test passed with no errors") diff --git a/src/tests/ftest/recovery/dlck_basic_fault.yaml b/src/tests/ftest/recovery/dlck_basic_fault.yaml new file mode 100644 index 00000000000..8fd27367019 --- /dev/null +++ b/src/tests/ftest/recovery/dlck_basic_fault.yaml @@ -0,0 +1,40 @@ +hosts: + test_servers: 2 + test_clients: 1 + +timeout: 600 + +server_config: + name: daos_server + engines_per_host: 1 + engines: + 0: + targets: 4 + storage: auto + +pool: + size: 80% + +faults: + create_log_dir: + fault_list: + - DLCK_FAULT_CREATE_LOG_DIR + +dlck_test_additional_faults: + fault_list: + - DLCK_FAULT_CREATE_POOL_DIR + - DLCK_FAULT_ENGINE_START + - DLCK_FAULT_ENGINE_EXEC + - DLCK_FAULT_ENGINE_JOIN + - DLCK_FAULT_ENGINE_STOP + - DAOS_FAULT_POOL_NVME_HEALTH + - DAOS_FAULT_POOL_OPEN_BIO + - DAOS_FAULT_POOL_OPEN_UMEM + - DAOS_FAULT_POOL_OPEN_MAGIC + - DAOS_FAULT_POOL_OPEN_VERSION + - DAOS_FAULT_POOL_OPEN_UUID + - DAOS_FAULT_BTREE_OPEN_INV_CLASS + - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS + - DAOS_FAULT_BTREE_FEATURES + - DAOS_FAULT_POOL_EXT_PADDING + - DAOS_FAULT_POOL_EXT_RESERVED diff --git a/src/tests/ftest/util/dlck_utils.py b/src/tests/ftest/util/dlck_utils.py new file mode 100644 index 00000000000..341e905295f --- /dev/null +++ b/src/tests/ftest/util/dlck_utils.py @@ -0,0 +1,82 @@ +""" + Copyright 2026 Hewlett Packard Enterprise Development LP + + SPDX-License-Identifier: BSD-2-Clause-Patent +""" +import os +from logging import getLogger + +from command_utils import ExecutableCommand +from command_utils_base import FormattedParameter +from run_utils import run_remote + + +class DlckCommand: + """Defines the basic structures of dlck command.""" + + def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None, + verbose=True, timeout=None, sudo=True): + """Constructor that sets the common variables for sub-commands. + + Args: + server_host (NodeSet): Server host to run the command. + path (str): path to the dlck command. + pool_uuid (str, optional): Pool UUID. Defaults to None. + nvme_conf (str, optional): NVMe config file path. Defaults to None. + storage_mount (str, optional): Storage mount point. Defaults to None. + verbose (bool, optional): Display command output in run. + Defaults to True. + timeout (int, optional): Command timeout (sec) used in run. Defaults to + None. + sudo (bool, optional): Whether to run dlck with sudo. Defaults to True. + """ + self.command = ExecutableCommand("/run/dlck/*", "dlck", path) + self._logger = getLogger(__name__) + + # Add the fault injection file to the environment of the remote command. + fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now") + if fault_inject_file != "None set for now": + self.command.env["D_FI_CONFIG"] = fault_inject_file + self.command.sudo = sudo + + self.host = server_host + + # Members needed for run(). + self.verbose = verbose + self.timeout = timeout + + # Pool UUID. (--file pool_uuid[,target_id]) + if pool_uuid: + self.command.pool_uuid = FormattedParameter("--file={}", pool_uuid) + + # NVMe config file path. (--nvme nvme_conf) + if nvme_conf: + self.command.nvme = FormattedParameter("--nvme={}", nvme_conf) + + # Storage mount point. (--storage storage_mount) + if storage_mount: + self.command.storage_mount = FormattedParameter("--storage={}", storage_mount) + + def __str__(self): + """Return the command with all of its defined parameters as a string. + + Returns: + str: the command with all the defined parameters + """ + return self.command.with_exports + + def run(self): + """Run the dlck command. + Args: + host (NodeSet): Host(s) on which to run the command. + command (str): Environment Variable string + dlck sub-command to run. + verbose (bool, optional): Display command output in run. + Defaults to True. + timeout (int, optional): Command timeout (sec) used in run. Defaults to + None. + + Returns: + CommandResult: groups of command results from the same hosts with the same return status + """ + return run_remote( + self._logger, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout) diff --git a/src/tests/ftest/util/fault_config_utils.py b/src/tests/ftest/util/fault_config_utils.py index 5dd3071271f..4721ec691ac 100644 --- a/src/tests/ftest/util/fault_config_utils.py +++ b/src/tests/ftest/util/fault_config_utils.py @@ -1,5 +1,6 @@ """ - (C) Copyright 2019-2024 Intel Corporation. + Copyright 2019-2024 Intel Corporation. + Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -235,6 +236,66 @@ 'probability_y': '100', 'interval': '1', 'max_faults': '1'}, + 'DLCK_FAULT_CREATE_LOG_DIR': { + 'id': '131328', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_CREATE_POOL_DIR': { + 'id': '131329', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_START': { + 'id': '131330', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_EXEC': { + 'id': '131331', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_JOIN': { + 'id': '131332', + 'interval': '1', + 'max_faults': '1'}, + 'DLCK_FAULT_ENGINE_STOP': { + 'id': '131333', + 'interval': '1', + 'max_faults': '1'}, + 'DAOS_FAULT_POOL_NVME_HEALTH': { + 'id': '131584', + 'interval': '2', # skip sys_db + 'max_faults': '1'}, + 'DAOS_FAULT_POOL_OPEN_BIO': { + 'id': '131585'}, + 'DAOS_FAULT_POOL_OPEN_UMEM': { + 'id': '131586', + 'interval': '2'}, # skip sys_db + 'DAOS_FAULT_POOL_OPEN_MAGIC': { + 'id': '131587', + 'interval': '2'}, # skip sys_db + 'DAOS_FAULT_POOL_OPEN_VERSION': { + 'id': '131588', + 'interval': '2'}, # skip sys_db + 'DAOS_FAULT_POOL_OPEN_UUID': { + 'id': '131589', + 'interval': '2'}, # skip sys_db + 'DAOS_FAULT_BTREE_OPEN_INV_CLASS': { + 'id': '131590', + 'interval': '28', # containers tree fine-tuned + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS': { + 'id': '131591', + 'interval': '29', + 'max_faults': '1'}, + 'DAOS_FAULT_BTREE_FEATURES': { + 'id': '131592', + 'interval': '28', + 'max_faults': '1'}, + 'DAOS_FAULT_POOL_EXT_PADDING': { + 'id': '131593', + 'interval': '1'}, + 'DAOS_FAULT_POOL_EXT_RESERVED': { + 'id': '131594', + 'interval': '1'}, } @@ -255,6 +316,12 @@ def __init__(self): self._test_dir = None self._fault_list = [] + def get_faults_dict(self): + """Get the predefined fault dictionary. + Returns: dict: The fault dictionary + """ + return FAULTS + def write_fault_file(self, on_the_fly_fault=None): """Write out a fault injection config file.