Merge branch 'main' into ODSC-66822/allow_cache_delete

Author: VipulMascarenhas

ads/aqua/common/utils.py

@@ -33,6 +33,7 @@
 )
 from oci.data_science.models import JobRun, Model
 from oci.object_storage.models import ObjectSummary
+from pydantic import ValidationError
 
 from ads.aqua.common.enums import (
     InferenceContainerParamType,
@@ -1206,3 +1207,15 @@ def validate_cmd_var(cmd_var: List[str], overrides: List[str]) -> List[str]:
 
     combined_cmd_var = cmd_var + overrides
     return combined_cmd_var
+
+
+def build_pydantic_error_message(ex: ValidationError):
+    """Added to handle error messages from pydantic model validator.
+    Combine both loc and msg for errors where loc (field) is present in error details, else only build error
+    message using msg field."""
+
+    return {
+        ".".join(map(str, e["loc"])): e["msg"]
+        for e in ex.errors()
+        if "loc" in e and e["loc"]
+    } or "; ".join(e["msg"] for e in ex.errors())

ads/aqua/data.py

@@ -1,9 +1,8 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 
 from ads.common.serializer import DataClassSerializable
 
@@ -13,19 +12,3 @@ class AquaResourceIdentifier(DataClassSerializable):
     id: str = ""
     name: str = ""
     url: str = ""
-
-
-@dataclass(repr=False)
-class AquaJobSummary(DataClassSerializable):
-    """Represents an Aqua job summary."""
-
-    id: str
-    name: str
-    console_url: str
-    lifecycle_state: str
-    lifecycle_details: str
-    time_created: str
-    tags: dict
-    experiment: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    source: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    job: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)

ads/aqua/extension/finetune_handler.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
@@ -10,9 +10,7 @@
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.extension.base_handler import AquaAPIhandler
 from ads.aqua.extension.errors import Errors
-from ads.aqua.extension.utils import validate_function_parameters
 from ads.aqua.finetuning import AquaFineTuningApp
-from ads.aqua.finetuning.entities import CreateFineTuningDetails
 
 
 class AquaFineTuneHandler(AquaAPIhandler):
@@ -33,7 +31,7 @@ def get(self, id=""):
             raise HTTPError(400, f"The request {self.request.path} is invalid.")
 
     @handle_exceptions
-    def post(self, *args, **kwargs):
+    def post(self, *args, **kwargs):  # noqa: ARG002
         """Handles post request for the fine-tuning API
 
         Raises
@@ -43,17 +41,13 @@ def post(self, *args, **kwargs):
         """
         try:
             input_data = self.get_json_body()
-        except Exception:
-            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT)
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
 
         if not input_data:
             raise HTTPError(400, Errors.NO_INPUT_DATA)
 
-        validate_function_parameters(
-            data_class=CreateFineTuningDetails, input_data=input_data
-        )
-
-        self.finish(AquaFineTuningApp().create(CreateFineTuningDetails(**input_data)))
+        self.finish(AquaFineTuningApp().create(**input_data))
 
     def get_finetuning_config(self, model_id):
         """Gets the finetuning config for Aqua model."""
@@ -71,7 +65,7 @@ def get(self, model_id):
         )
 
     @handle_exceptions
-    def post(self, *args, **kwargs):
+    def post(self, *args, **kwargs):  # noqa: ARG002
         """Handles post request for the finetuning param handler API.
 
         Raises
@@ -81,8 +75,8 @@ def post(self, *args, **kwargs):
         """
         try:
             input_data = self.get_json_body()
-        except Exception:
-            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT)
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
 
         if not input_data:
             raise HTTPError(400, Errors.NO_INPUT_DATA)

ads/aqua/finetuning/constants.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from ads.common.extended_enum import ExtendedEnumMeta
@@ -17,4 +16,8 @@ class FineTuneCustomMetadata(str, metaclass=ExtendedEnumMeta):
     SERVICE_MODEL_FINE_TUNE_CONTAINER = "finetune-container"
 
 
+class FineTuningRestrictedParams(str, metaclass=ExtendedEnumMeta):
+    OPTIMIZER = "optimizer"
+
+
 ENV_AQUA_FINE_TUNING_CONTAINER = "AQUA_FINE_TUNING_CONTAINER"

ads/aqua/finetuning/entities.py

@@ -1,18 +1,24 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-from dataclasses import dataclass, field
-from typing import List, Optional
 
-from ads.aqua.data import AquaJobSummary
-from ads.common.serializer import DataClassSerializable
+import json
+from typing import List, Literal, Optional, Union
 
+from pydantic import Field, model_validator
 
-@dataclass(repr=False)
-class AquaFineTuningParams(DataClassSerializable):
-    epochs: int
+from ads.aqua.common.errors import AquaValueError
+from ads.aqua.config.utils.serializer import Serializable
+from ads.aqua.data import AquaResourceIdentifier
+from ads.aqua.finetuning.constants import FineTuningRestrictedParams
+
+
+class AquaFineTuningParams(Serializable):
+    """Class for maintaining aqua fine-tuning model parameters"""
+
+    epochs: Optional[int] = None
     learning_rate: Optional[float] = None
-    sample_packing: Optional[bool] = "auto"
+    sample_packing: Union[bool, None, Literal["auto"]] = "auto"
     batch_size: Optional[int] = (
         None  # make it batch_size for user, but internally this is micro_batch_size
     )
@@ -22,21 +28,59 @@ class AquaFineTuningParams(DataClassSerializable):
     lora_alpha: Optional[int] = None
     lora_dropout: Optional[float] = None
     lora_target_linear: Optional[bool] = None
-    lora_target_modules: Optional[List] = None
+    lora_target_modules: Optional[List[str]] = None
     early_stopping_patience: Optional[int] = None
     early_stopping_threshold: Optional[float] = None
 
+    class Config:
+        extra = "allow"
+
+    def to_dict(self) -> dict:
+        return json.loads(super().to_json(exclude_none=True))
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_restricted_fields(cls, data: dict):
+        # we may want to skip validation if loading data from config files instead of user entered parameters
+        validate = data.pop("_validate", True)
+        if not (validate and isinstance(data, dict)):
+            return data
+        restricted_params = [
+            param for param in data if param in FineTuningRestrictedParams.values()
+        ]
+        if restricted_params:
+            raise AquaValueError(
+                f"Found restricted parameter name: {restricted_params}"
+            )
+        return data
 
-@dataclass(repr=False)
-class AquaFineTuningSummary(AquaJobSummary, DataClassSerializable):
-    parameters: AquaFineTuningParams = field(default_factory=AquaFineTuningParams)
 
+class AquaFineTuningSummary(Serializable):
+    """Represents a summary of Aqua Finetuning job."""
 
-@dataclass(repr=False)
-class CreateFineTuningDetails(DataClassSerializable):
-    """Dataclass to create aqua model fine tuning.
+    id: str
+    name: str
+    console_url: str
+    lifecycle_state: str
+    lifecycle_details: str
+    time_created: str
+    tags: dict
+    experiment: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    source: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    job: AquaResourceIdentifier = Field(default_factory=AquaResourceIdentifier)
+    parameters: AquaFineTuningParams = Field(default_factory=AquaFineTuningParams)
 
-    Fields
+    class Config:
+        extra = "ignore"
+
+    def to_dict(self) -> dict:
+        return json.loads(super().to_json(exclude_none=True))
+
+
+class CreateFineTuningDetails(Serializable):
+    """Class to create aqua model fine-tuning instance.
+
+    Properties
     ------
     ft_source_id: str
         The fine tuning source id. Must be model ocid.
@@ -107,3 +151,6 @@ class CreateFineTuningDetails(DataClassSerializable):
     force_overwrite: Optional[bool] = False
     freeform_tags: Optional[dict] = None
     defined_tags: Optional[dict] = None
+
+    class Config:
+        extra = "ignore"