[New Feature Automation] RHSTOR-4624 ODF used capacity trend

ocs_ci/ocs/cluster.py

@@ -16,8 +16,9 @@
 import os
 import pandas as pd
 import re
+import math
 
-
+from datetime import datetime
 from semantic_version import Version
 from ocs_ci.ocs.utils import thread_init_class
 
@@ -1364,6 +1365,7 @@ def parse_ceph_df_pools(raw_output: str) -> pd.DataFrame:
         "%USED",
         "MAX AVAIL",
         "QUOTA OBJECTS",
+        "QUOTA OBJECTS",
         "QUOTA BYTES",
         "DIRTY",
         "USED COMPR",
@@ -3602,3 +3604,48 @@ def bring_down_mds_memory_usage_gradually():
     assert (
         time_elapsed <= 1800
     ), "Memory usage remained high for more than 30 minutes. Failed to bring down the memory usage of MDS"
+
+
+def get_used_and_total_capacity_in_gibibytes():
+    """
+    Get used capacity and total capacity of the cluster from the ceph tools pod
+    Convert the storage values from bytes to gibibytes
+
+    Returns:
+        tuple: (total_used_in_gibibytes, total_capacity_in_gibibytes) ex: Used capacity, Total capacity
+
+    """
+    ct_pod = pod.get_ceph_tools_pod()
+    output = ct_pod.exec_ceph_cmd(ceph_cmd="ceph df")
+    total_used = output.get("stats").get("total_used_raw_bytes")
+    total_capacity = output.get("stats").get("total_bytes")
+    total_used_in_gibibytes = total_used / (2**30)
+    total_capacity_in_gibibytes = total_capacity / (2**30)
+    return (total_used_in_gibibytes, total_capacity_in_gibibytes)
+
+
+def get_age_of_cluster_in_days():
+    """
+    Get age of the cluster in days.
+    1. Get creation time by executing oc cmd on cluster
+    2. Get current time from the ceph tools pod
+    3. Calculate time difference between two times
+    4. Convert the time into days
+
+    Returns:
+        int: returns number of days the cluster has been running
+
+    """
+    cmd = "get namespace kube-system -o jsonpath='{.metadata.creationTimestamp}'"
+    creation_time = OCP().exec_oc_cmd(command=cmd, out_yaml_format=False)
+    logger.info(f"The cluster creation time is: {creation_time}")
+    ct_pod = pod.get_ceph_tools_pod()
+    cephcmd = 'date -u +"%Y-%m-%dT%H:%M:%SZ"'
+    current_time = ct_pod.exec_cmd_on_pod(command=cephcmd, out_yaml_format=False)
+    logger.info(f"Current time in the cluster is: {current_time}")
+    d1 = datetime.fromisoformat(creation_time[:-1])
+    d2 = datetime.fromisoformat(current_time.strip()[:-1])
+    time_difference_in_sec = (d2 - d1).total_seconds()
+    seconds_per_day = 24 * 60 * 60
+    time_diff_in_days = time_difference_in_sec / seconds_per_day
+    return math.ceil(time_diff_in_days)

ocs_ci/ocs/constants.py

@@ -605,6 +605,7 @@
 RBD_NODEPLUGIN_LABEL = "app=openshift-storage.rbd.csi.ceph.com-nodeplugin"
 CEPHFS_CTRLPLUGIN_LABEL = "app=openshift-storage.cephfs.csi.ceph.com-ctrlplugin"
 RBD_CTRLPLUGIN_LABEL = "app=openshift-storage.rbd.csi.ceph.com-ctrlplugin"
+PROMETHEUS_POD_LABEL = "app.kubernetes.io/name=prometheus"
 
 # Noobaa Deployments and Statefulsets
 NOOBAA_OPERATOR_DEPLOYMENT = "noobaa-operator"

ocs_ci/ocs/resources/pod.py

@@ -3855,7 +3855,7 @@ def get_pod_used_memory_in_mebibytes(podname):
         podname: (str)  name of the pod to get used memory of it
 
     Returns:
-        memory_value: (int) the used memory of the pod in Mebibytes (MiB)
+        int:  the used memory of the pod in Mebibytes (MiB)
 
     """
     logger.info("Retrieve raw resource utilization data using oc adm top command")
@@ -3927,3 +3927,24 @@ def get_container_images(pod_obj):
         raise ValueError(f"Didn't find images for the pod {pod_obj.name} containers")
 
     return images
+
+
+def get_prometheus_pods(
+    prometheus_label=constants.PROMETHEUS_POD_LABEL,
+    namespace=constants.MONITORING_NAMESPACE,
+):
+    """
+    Fetches info about prometheus pods in the cluster
+
+    Args:
+        prometheus_label (str): label associated with prometheus pods
+        namespace (str): Namespace in which prometheus pods lives
+
+    Returns:
+        list : of prometheus pod objects
+
+    """
+    namespace = namespace
+    pods_with_label_match = get_pods_having_label(prometheus_label, namespace)
+    prometheus_pod_objs = [Pod(**prometheus) for prometheus in pods_with_label_match]
+    return prometheus_pod_objs

ocs_ci/ocs/ui/page_objects/block_and_file.py

@@ -1,8 +1,11 @@
+import re
 import time
 
+from ocs_ci.framework import config
 from ocs_ci.ocs.ui.helpers_ui import format_locator, logger
 from ocs_ci.ocs.ui.page_objects.storage_system_details import StorageSystemDetails
 from ocs_ci.ocs.ui.workload_ui import PvcCapacityDeploymentList, compare_mem_usage
+from ocs_ci.utility.utils import TimeoutSampler
 
 
 class BlockAndFile(StorageSystemDetails):
@@ -181,3 +184,78 @@ def get_raw_capacity_card_values(self):
         )
 
         return used, available
+
+    def get_estimated_days_from_consumption_trend(self):
+        """
+        This will fetch information from DataFoundation>>Storage>>Block and File page>>Consumption trend card
+
+        Returns:
+            tuple: (get_est_days_from_element, get_avg_from_element)
+
+        """
+
+        get_est_days_from_element = self.get_element_text(
+            self.validation_loc["locate_estimated_days_along_with_value"]
+        )
+        get_avg_from_element = self.get_element_text(
+            self.validation_loc["locate_average_of_storage_consumption"]
+        )
+        return (get_est_days_from_element, get_avg_from_element)
+
+    def odf_storagesystems_consumption_trend(self):
+        """
+        Function to verify changes and validate elements on ODF storage consumption trend for ODF 4.17
+        This will navigate through below order
+        DataFoundation>>Storage>>storagecluster_storagesystem_details>>Block and File page
+        Further it looks for the Consumption trend card
+
+        Returns:
+            tuple: tpl_of_days_and_avg  ex: (Estimated days, Average)
+
+        """
+
+        if not config.ENV_DATA["mcg_only_deployment"]:
+            for tpl_of_days_and_avg in TimeoutSampler(
+                timeout=300,
+                sleep=30,
+                func=self.get_estimated_days_from_consumption_trend,
+            ):
+
+                if re.search(
+                    r"(?=.*\d)(?=.*[a-zA-Z])", tpl_of_days_and_avg[0]
+                ) and re.search(r"(?=.*\d)(?=.*[a-zA-Z])", tpl_of_days_and_avg[1]):
+                    return tpl_of_days_and_avg
+                else:
+                    logger.warning("Dashboard is not yet ready yet after osd resize")
+        else:
+            logger.error("No data available for MCG-only deployments.")
+            return None
+
+    def get_est_days_from_ui(self):
+        """
+        Get the value of 'Estimated days until full' from the UI
+
+        Returns:
+            int: Estimated days until full from UI
+
+        """
+
+        collected_tpl_of_days_and_avg = self.odf_storagesystems_consumption_trend()
+        est_days = re.search(r"\d+", collected_tpl_of_days_and_avg[0]).group()
+        logger.info(f"'Estimated days until full' from the UI : {est_days}")
+        return int(est_days)
+
+    def get_avg_consumption_from_ui(self):
+        """
+        Get the value of 'Average storage consumption' from the UI
+
+        Returns:
+            float: Average of storage consumption per day
+
+        """
+        collected_tpl_of_days_and_avg = self.odf_storagesystems_consumption_trend()
+        average = float(
+            re.search(r"-?\d+\.*\d*", collected_tpl_of_days_and_avg[1]).group()
+        )
+        logger.info(f"'Average of storage consumption per day' from the UI : {average}")
+        return average

ocs_ci/ocs/ui/validation_ui.py

@@ -1,23 +1,27 @@
 import logging
-import warnings
+import math
+import pytest
 import time
+import warnings
 
 import pandas as pd
-import pytest
-from selenium.common.exceptions import TimeoutException
+from ocs_ci.framework import config
+from ocs_ci.framework.logger_helper import log_step
+from ocs_ci.ocs.cluster import (
+    get_used_and_total_capacity_in_gibibytes,
+    get_age_of_cluster_in_days,
+)
 from ocs_ci.ocs.exceptions import UnexpectedODFAccessException
 from ocs_ci.ocs.ui.page_objects.backing_store_tab import BackingStoreTab
 from ocs_ci.ocs.ui.page_objects.namespace_store_tab import NameSpaceStoreTab
 from ocs_ci.ocs.ui.page_objects.overview_tab import OverviewTab
 from ocs_ci.ocs.ui.page_objects.page_navigator import PageNavigator
 from ocs_ci.ocs.ui.page_objects.storage_system_details import StorageSystemDetails
-from ocs_ci.utility import version
-from ocs_ci.utility.utils import TimeoutSampler
-from ocs_ci.framework import config
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.resources.storage_cluster import StorageCluster
-from ocs_ci.framework.logger_helper import log_step
-
+from ocs_ci.utility import version
+from ocs_ci.utility.utils import TimeoutSampler
+from selenium.common.exceptions import TimeoutException
 
 logger = logging.getLogger(__name__)
 
@@ -677,3 +681,31 @@ def verify_storage_clients_page(self):
             page_name="client_onboarding_token_page",
         )
         return storage_client_obj
+
+    def calculate_est_days_and_average_manually(self):
+        """
+        Calculates the 'Estimated days until full' manually by:
+        1. Get the age of the cluster in days
+        2. Get used capacity of the cluster
+        3. Get total capacity of the cluster
+        4. Calculate average consumption of the storage per day
+        5. Calculate the 'Estimated days until full' by using average and available capacity.
+
+        Returns:
+            tuple: Estimated days until full which calculated manually
+
+        """
+        number_of_days = get_age_of_cluster_in_days()
+        logger.info(f"Age of the cluster in days: {number_of_days}")
+        tpl_of_used_and_total_capacity = get_used_and_total_capacity_in_gibibytes()
+        used_capacity = tpl_of_used_and_total_capacity[0]
+        logger.info(f"The used capacity from the cluster is: {used_capacity}")
+        total_capacity = tpl_of_used_and_total_capacity[1]
+        available_capacity = total_capacity - used_capacity
+        logger.info(f"The available capacity from the cluster is: {available_capacity}")
+        average = used_capacity / number_of_days
+        logger.info(f"Average of storage consumption per day: {average}")
+        estimated_days_calculated = available_capacity / average
+        manual_est = math.floor(estimated_days_calculated)
+        logger.info(f"Estimated days calculated are {manual_est}")
+        return (manual_est, average)

ocs_ci/ocs/ui/views.py

@@ -1773,6 +1773,17 @@
     "onboarding_token": ("//*[@class='odf-onboarding-modal__text-area']", By.XPATH),
 }
 
+validation_4_17 = {
+    "locate_average_of_storage_consumption": (
+        "//div[@class='pf-m-flex-1'][1]",
+        By.XPATH,
+    ),
+    "locate_estimated_days_along_with_value": (
+        "//div[@class='pf-m-flex-1'][2]",
+        By.XPATH,
+    ),
+}
+
 topology = {
     "topology_graph": ("//*[@data-kind='graph']", By.XPATH),
     "node_label": ("//*[@class='pf-topology__node__label']", By.XPATH),
@@ -2013,6 +2024,7 @@
             **validation_4_12,
             **validation_4_13,
             **validation_4_14,
+            **validation_4_17,
         },
         "block_pool": {**block_pool, **block_pool_4_12, **block_pool_4_13},
         "storageclass": {**storageclass, **storageclass_4_9},