DAOS-17519 test: Automate dlck testing (basic/fault_injection)

src/tests/ftest/recovery/dlck_basic.py

@@ -0,0 +1,51 @@
+"""
+  Copyright 2026 Hewlett Packard Enterprise Development LP
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+import os
+
+from apricot import TestWithServers
+from dlck_utils import DlckCommand
+from test_utils_pool import add_pool
+
+
+class DlckBasicTest(TestWithServers):
+    """Test class for dlck command line utility.
+
+    :avocado: recursive
+    """
+    def test_dlck_basic(self):
+        """Basic Test: Run 'dlck' command
+
+        :avocado: tags=all,daily_regression
+        :avocado: tags=hw,medium
+        :avocado: tags=recovery,dlck_cmd
+        :avocado: tags=DlckBasicTest,test_dlck_basic
+        """
+        errors = []
+        dmg = self.get_dmg_command()
+        self.log_step("Create a pool to run dlck")
+        pool = add_pool(self)
+        scm_mount = self.server_managers[0].get_config_value("scm_mount")
+        host = self.server_managers[0].hosts[0:1]
+        if self.server_managers[0].manager.job.using_control_metadata:
+            log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file"))
+            control_metadata_dir = os.path.join(log_dir, "control_metadata")
+            daos_control_dir = os.path.join(control_metadata_dir, "daos_control")
+            engine_path_dir = os.path.join(daos_control_dir, "engine0")
+            nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf")
+            dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf,
+                                   storage_mount=scm_mount)
+        else:
+            dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount)
+        self.log_step("Perform dmg system stop to run dlck command")
+        dmg.system_stop()
+        self.log_step("Run dlck command to check the health of the pool and storage")
+        result = dlck_cmd.run()
+        if not result.passed:
+            errors.append(f"dlck failed on {result.failed_hosts}")
+        dmg.system_start()
+        if errors:
+            self.fail(f"dlck basic test failed with errors: {errors}")
+        self.log.info("dlck basic test passed with no errors")

src/tests/ftest/recovery/dlck_basic.yaml

@@ -0,0 +1,19 @@
+hosts:
+  test_servers: 2
+  test_clients: 1
+
+timeout: 600
+
+server_config:
+  name: daos_server
+  engines_per_host: 1
+  engines:
+    0:
+      targets: 4
+      storage: auto
+    1:
+      targets: 4
+      storage: auto
+
+pool:
+  size: 80%

src/tests/ftest/recovery/dlck_basic_fault.py

@@ -0,0 +1,94 @@
+"""
+  Copyright 2026 Hewlett Packard Enterprise Development LP
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+import os
+
+from apricot import TestWithServers
+from dlck_utils import DlckCommand
+from fault_config_utils import FaultInjection
+from file_utils import distribute_files
+from run_utils import run_remote
+from test_utils_pool import add_pool
+
+
+class DlckBasicFaultTest(TestWithServers):
+    """Test class for dlck command line utility.
+
+    :avocado: recursive
+    """
+    def test_dlck_basic_fault(self):
+        """Basic Fault Test: Run 'dlck' injecting basic faults.
+
+        :avocado: tags=all,full_regression
+        :avocado: tags=hw,medium
+        :avocado: tags=recovery,dlck_cmd
+        :avocado: tags=DlckBasicFaultTest,test_dlck_basic_fault
+        """
+        errors = []
+        faults_dict = {}
+        faults_object = FaultInjection()
+        faults_dict = faults_object.get_faults_dict()
+        fault_list = self.params.get("fault_list", '/run/dlck_test_additional_faults/*')
+        dmg = self.get_dmg_command()
+        pool = add_pool(self)
+        scm_mount = self.server_managers[0].get_config_value("scm_mount")
+        fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now")
+        if fault_inject_file == "None set for now":
+            self.fail("D_FI_CONFIG environment variable not set")
+        self.log.info("Fault injection file contents")
+        cmd = "cat {}".format(fault_inject_file)
+        host = self.server_managers[0].hosts[0:1]
+        self.log_step("Run the command to read the fault injection file contents")
+        run_remote(self.log, self.hostlist_clients[0], cmd, timeout=30)
+        # Run the testing with the first fault which is injected at the beginning of the test.
+        if self.server_managers[0].manager.job.using_control_metadata:
+            log_dir = os.path.dirname(self.server_managers[0].get_config_value("log_file"))
+            control_metadata_dir = os.path.join(log_dir, "control_metadata")
+            daos_control_dir = os.path.join(control_metadata_dir, "daos_control")
+            engine_path_dir = os.path.join(daos_control_dir, "engine0")
+            nvme_conf = os.path.join(engine_path_dir, "daos_nvme.conf")
+            dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf,
+                                   storage_mount=scm_mount)
+        else:
+            dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount)
+        self.log_step("Perform dmg system stop to run dlck command")
+        dmg.system_stop()
+        self.log_step("Run dlck command after injecting the first fault")
+        result = dlck_cmd.run()
+        if not result.passed:
+            errors.append(f"dlck failed on {result.failed_hosts}")
+        # Now, run the other fault injection flags without rebooting or creating any new pools.
+        # Rebooting the servers or creating the new pools will result in injecting fault in
+        # the wrong test code. Fault injections should done only for the dlck alone.
+        for test_fault in fault_list:
+            with open(fault_inject_file, 'w') as f:
+                f.write("fault_config:\n")
+                count = 0
+                for key, value in faults_dict[test_fault].items():
+                    if count == 0:
+                        f.write(f"- {key}: \'{value}\'\n")
+                    else:
+                        f.write(f"  {key}: \'{value}\'\n")
+                    count += 1
+            self.log.info("Reading the updated fault injection file contents")
+            with open(fault_inject_file, 'r') as f:
+                file_data = f.read()
+                self.log.info("\n %s", file_data)
+            distribute_files(self.log, self.hostlist_servers, fault_inject_file,
+                             fault_inject_file)
+            if self.server_managers[0].manager.job.using_control_metadata:
+                dlck_cmd = DlckCommand(host, self.bin, pool.uuid, nvme_conf=nvme_conf,
+                                       storage_mount=scm_mount)
+            else:
+                dlck_cmd = DlckCommand(host, self.bin, pool.uuid, storage_mount=scm_mount)
+            self.log_step("Run dlck command after injecting fault")
+            result = dlck_cmd.run()
+            if not result.passed:
+                errors.append(f"dlck failed on {result.failed_hosts}")
+        self.log_step("Run the dmg start command")
+        dmg.system_start()
+        if not errors:
+            self.fail(f"dlck basic faulttest failed with errors: {errors}")
+        self.log.info("dlck basic fault test passed with no errors")

src/tests/ftest/recovery/dlck_basic_fault.yaml

@@ -0,0 +1,40 @@
+hosts:
+  test_servers: 2
+  test_clients: 1
+
+timeout: 600
+
+server_config:
+  name: daos_server
+  engines_per_host: 1
+  engines:
+    0:
+      targets: 4
+      storage: auto
+
+pool:
+  size: 80%
+
+faults:
+  create_log_dir:
+    fault_list:
+      - DLCK_FAULT_CREATE_LOG_DIR
+
+dlck_test_additional_faults:
+  fault_list:
+    - DLCK_FAULT_CREATE_POOL_DIR
+    - DLCK_FAULT_ENGINE_START
+    - DLCK_FAULT_ENGINE_EXEC
+    - DLCK_FAULT_ENGINE_JOIN
+    - DLCK_FAULT_ENGINE_STOP
+    - DAOS_FAULT_POOL_NVME_HEALTH
+    - DAOS_FAULT_POOL_OPEN_BIO
+    - DAOS_FAULT_POOL_OPEN_UMEM
+    - DAOS_FAULT_POOL_OPEN_MAGIC
+    - DAOS_FAULT_POOL_OPEN_VERSION
+    - DAOS_FAULT_POOL_OPEN_UUID
+    - DAOS_FAULT_BTREE_OPEN_INV_CLASS
+    - DAOS_FAULT_BTREE_OPEN_UNREG_CLASS
+    - DAOS_FAULT_BTREE_FEATURES
+    - DAOS_FAULT_POOL_EXT_PADDING
+    - DAOS_FAULT_POOL_EXT_RESERVED

src/tests/ftest/util/dlck_utils.py

@@ -0,0 +1,82 @@
+"""
+  Copyright 2026 Hewlett Packard Enterprise Development LP
+
+  SPDX-License-Identifier: BSD-2-Clause-Patent
+"""
+import os
+from logging import getLogger
+
+from command_utils import ExecutableCommand
+from command_utils_base import FormattedParameter
+from run_utils import run_remote
+
+
+class DlckCommand:
+    """Defines the basic structures of dlck command."""
+
+    def __init__(self, server_host, path, pool_uuid=None, nvme_conf=None, storage_mount=None,
+                 verbose=True, timeout=None, sudo=True):
+        """Constructor that sets the common variables for sub-commands.
+
+        Args:
+            server_host (NodeSet): Server host to run the command.
+            path (str): path to the dlck command.
+            pool_uuid (str, optional): Pool UUID. Defaults to None.
+            nvme_conf (str, optional): NVMe config file path. Defaults to None.
+            storage_mount (str, optional): Storage mount point. Defaults to None.
+            verbose (bool, optional): Display command output in run.
+                Defaults to True.
+            timeout (int, optional): Command timeout (sec) used in run. Defaults to
+                None.
+            sudo (bool, optional): Whether to run dlck with sudo. Defaults to True.
+        """
+        self.command = ExecutableCommand("/run/dlck/*", "dlck", path)
+        self._logger = getLogger(__name__)
+
+        # Add the fault injection file to the environment of the remote command.
+        fault_inject_file = os.getenv("D_FI_CONFIG", "None set for now")
+        if fault_inject_file != "None set for now":
+            self.command.env["D_FI_CONFIG"] = fault_inject_file
+        self.command.sudo = sudo
+
+        self.host = server_host
+
+        # Members needed for run().
+        self.verbose = verbose
+        self.timeout = timeout
+
+        # Pool UUID. (--file pool_uuid[,target_id])
+        if pool_uuid:
+            self.command.pool_uuid = FormattedParameter("--file={}", pool_uuid)
+
+        # NVMe config file path. (--nvme nvme_conf)
+        if nvme_conf:
+            self.command.nvme = FormattedParameter("--nvme={}", nvme_conf)
+
+        # Storage mount point. (--storage storage_mount)
+        if storage_mount:
+            self.command.storage_mount = FormattedParameter("--storage={}", storage_mount)
+
+    def __str__(self):
+        """Return the command with all of its defined parameters as a string.
+
+        Returns:
+            str: the command with all the defined parameters
+        """
+        return self.command.with_exports
+
+    def run(self):
+        """Run the dlck command.
+        Args:
+            host (NodeSet): Host(s) on which to run the command.
+            command (str): Environment Variable string + dlck sub-command to run.
+            verbose (bool, optional): Display command output in run.
+                Defaults to True.
+            timeout (int, optional): Command timeout (sec) used in run. Defaults to
+                None.
+
+        Returns:
+            CommandResult: groups of command results from the same hosts with the same return status
+        """
+        return run_remote(
+            self._logger, self.host, command=str(self), verbose=self.verbose, timeout=self.timeout)

src/tests/ftest/util/fault_config_utils.py

@@ -1,5 +1,6 @@
 """
-  (C) Copyright 2019-2024 Intel Corporation.
+  Copyright 2019-2024 Intel Corporation.
+  Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -235,6 +236,66 @@
         'probability_y': '100',
         'interval': '1',
         'max_faults': '1'},
+    'DLCK_FAULT_CREATE_LOG_DIR': {
+        'id': '131328',
+        'interval': '1',
+        'max_faults': '1'},
+    'DLCK_FAULT_CREATE_POOL_DIR': {
+        'id': '131329',
+        'interval': '1',
+        'max_faults': '1'},
+    'DLCK_FAULT_ENGINE_START': {
+        'id': '131330',
+        'interval': '1',
+        'max_faults': '1'},
+    'DLCK_FAULT_ENGINE_EXEC': {
+        'id': '131331',
+        'interval': '1',
+        'max_faults': '1'},
+    'DLCK_FAULT_ENGINE_JOIN': {
+        'id': '131332',
+        'interval': '1',
+        'max_faults': '1'},
+    'DLCK_FAULT_ENGINE_STOP': {
+        'id': '131333',
+        'interval': '1',
+        'max_faults': '1'},
+    'DAOS_FAULT_POOL_NVME_HEALTH': {
+        'id': '131584',
+        'interval': '2',  # skip sys_db
+        'max_faults': '1'},
+    'DAOS_FAULT_POOL_OPEN_BIO': {
+        'id': '131585'},
+    'DAOS_FAULT_POOL_OPEN_UMEM': {
+        'id': '131586',
+        'interval': '2'},  # skip sys_db
+    'DAOS_FAULT_POOL_OPEN_MAGIC': {
+        'id': '131587',
+        'interval': '2'},  # skip sys_db
+    'DAOS_FAULT_POOL_OPEN_VERSION': {
+        'id': '131588',
+        'interval': '2'},  # skip sys_db
+    'DAOS_FAULT_POOL_OPEN_UUID': {
+        'id': '131589',
+        'interval': '2'},  # skip sys_db
+    'DAOS_FAULT_BTREE_OPEN_INV_CLASS': {
+        'id': '131590',
+        'interval': '28',  # containers tree fine-tuned
+        'max_faults': '1'},
+    'DAOS_FAULT_BTREE_OPEN_UNREG_CLASS': {
+        'id': '131591',
+        'interval': '29',
+        'max_faults': '1'},
+    'DAOS_FAULT_BTREE_FEATURES': {
+        'id': '131592',
+        'interval': '28',
+        'max_faults': '1'},
+    'DAOS_FAULT_POOL_EXT_PADDING': {
+        'id': '131593',
+        'interval': '1'},
+    'DAOS_FAULT_POOL_EXT_RESERVED': {
+        'id': '131594',
+        'interval': '1'},
 }
 
 
@@ -255,6 +316,12 @@ def __init__(self):
         self._test_dir = None
         self._fault_list = []
 
+    def get_faults_dict(self):
+        """Get the predefined fault dictionary.
+        Returns: dict: The fault dictionary
+        """
+        return FAULTS
+
     def write_fault_file(self, on_the_fly_fault=None):
         """Write out a fault injection config file.