Merge branch 'main' into ODSC-55290/update_job_byoc_api

Author: lu-ohai

ads/opctl/operator/lowcode/common/errors.py

@@ -39,3 +39,9 @@ def __init__(self, error: str):
             "complies with the required schema for the operator. \n"
             f"{error}"
         )
+
+
+class InsufficientDataError(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(message)

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -249,20 +249,28 @@ def generate_report(self):
                     train_metrics_sections = [sec9_text, sec9]
 
                 backtest_sections = []
+                output_dir = self.spec.output_directory.url
+                backtest_report_name = "backtest_stats.csv"
+                file_path = f"{output_dir}/{backtest_report_name}"
                 if self.spec.model == AUTO_SELECT:
-                    output_dir = self.spec.output_directory.url
-                    backtest_report_name = "backtest_stats.csv"
-                    backtest_stats = pd.read_csv(f"{output_dir}/{backtest_report_name}")
-                    average_dict = backtest_stats.mean().to_dict()
-                    del average_dict['backtest']
-                    best_model = min(average_dict, key=average_dict.get)
-                    backtest_text = rc.Heading("Back Testing Metrics", level=2)
-                    summary_text = rc.Text(
-                        f"Overall, the average scores for the models are {average_dict}, with {best_model}"
-                        f" being identified as the top-performing model during backtesting.")
-                    backtest_table = rc.DataTable(backtest_stats, index=True)
-                    liner_plot = get_auto_select_plot(backtest_stats)
-                    backtest_sections = [backtest_text, backtest_table, summary_text, liner_plot]
+                    backtest_sections.append(rc.Heading("Auto-select statistics", level=2))
+                    if not os.path.exists(file_path):
+                        failure_msg = rc.Text("auto-select could not be executed. Please check the "
+                                              "logs for more details.")
+                        backtest_sections.append(failure_msg)
+                    else:
+                        backtest_stats = pd.read_csv(file_path)
+                        average_dict = backtest_stats.mean().to_dict()
+                        del average_dict['backtest']
+                        best_model = min(average_dict, key=average_dict.get)
+                        backtest_text = rc.Heading("Back Testing Metrics", level=3)
+                        summary_text = rc.Text(
+                            f"Overall, the average scores for the models are {average_dict}, with {best_model}"
+                            f" being identified as the top-performing model during backtesting.")
+                        backtest_table = rc.DataTable(backtest_stats, index=True)
+                        liner_plot = get_auto_select_plot(backtest_stats)
+                        backtest_sections.extend([backtest_text, backtest_table, summary_text,
+                                                                      liner_plot])
 
 
                 forecast_plots = []

ads/opctl/operator/lowcode/forecast/model_evaluator.py

@@ -12,7 +12,8 @@
 from ads.opctl.operator.lowcode.common.const import DataColumns
 from .model.forecast_datasets import ForecastDatasets
 from .operator_config import ForecastOperatorConfig
-
+from ads.opctl.operator.lowcode.forecast.model.factory import SupportedModels
+from ads.opctl.operator.lowcode.common.errors import InsufficientDataError
 
 class ModelEvaluator:
     """
@@ -61,6 +62,9 @@ def generate_k_fold_data(self, datasets: ForecastDatasets, operator_config: Fore
         unique_dates = min_series_data[date_col].unique()
 
         cut_offs = self.generate_cutoffs(unique_dates, horizon)
+        if not len(cut_offs):
+            raise InsufficientDataError("Insufficient data to evaluate multiple models. Please specify a model "
+                                        "instead of using auto-select.")
         training_datasets = [sampled_historical_data[sampled_historical_data[date_col] <= cut_off_date] for cut_off_date
                              in cut_offs]
         test_datasets = [sampled_historical_data[sampled_historical_data[date_col] > cut_offs[0]]]
@@ -137,7 +141,12 @@ def run_all_models(self, datasets: ForecastDatasets, operator_config: ForecastOp
         return metrics
 
     def find_best_model(self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig):
-        metrics = self.run_all_models(datasets, operator_config)
+        try:
+            metrics = self.run_all_models(datasets, operator_config)
+        except InsufficientDataError as e:
+            model = SupportedModels.Prophet
+            logger.error(f"Running {model} model as auto-select failed with the following error: {e.message}")
+            return model
         avg_backtests_metrics = {key: sum(value.values()) / len(value.values()) for key, value in metrics.items()}
         best_model = min(avg_backtests_metrics, key=avg_backtests_metrics.get)
         logger.info(f"Among models {self.models}, {best_model} model shows better performance during backtesting.")

docs/source/index.rst

@@ -45,6 +45,7 @@ Oracle Accelerated Data Science (ADS)
    user_guide/operators/forecasting_operator/index
    user_guide/operators/anomaly_detection_operator/index
    user_guide/operators/pii_operator/index
+   user_guide/operators/recommender_operator/index
 
 .. toctree::
    :hidden:

docs/source/user_guide/operators/recommender/index.rst renamed to docs/source/user_guide/operators/recommender_operator/index.rst

@@ -1,6 +1,6 @@
-===
+===========
 Recommender
-===
+===========
 
 The Recommender Operator utilizes advanced algorithms to provide personalized recommendations based on user behavior and preferences. This operator streamlines the data science workflow by automating the process of selecting the best recommendation algorithms, tuning hyperparameters, and extracting relevant features, ensuring that users receive the most relevant and effective suggestions for their needs.