Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 17 additions & 20 deletions ads/model/runtime/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,15 @@
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

import json
import logging
import os
from functools import lru_cache
from typing import Dict, Tuple
import logging

import fsspec
import requests
import yaml
from cerberus import DocumentError, Validator

from ads.common.object_storage_details import ObjectStorageDetails
from ads.common.utils import PAR_LINK

MODEL_PROVENANCE_SCHEMA_PATH = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
Expand Down Expand Up @@ -127,9 +124,10 @@ def _get_index_json_through_bucket(
service_packs = json.loads(f.read())
service_pack_list = service_packs.get(SERVICE_PACKS)
except Exception as e:
logging.warn(e)
logging.warn(
"Failed to retrieve the full conda pack path from slug. Pass conda pack path 'oci://<bucketname>@<namespace>/<path_to_conda>' instead of slug."
logging.error(
f"Error occurred in attempt to extract the list of the service conda environments "
f"from the object storage for bucket '{bucketname}' and namespace '{namespace}'. "
f"Please make sure that you've provided correct bucket and namespace."
)
return service_pack_list

Expand Down Expand Up @@ -159,22 +157,21 @@ def get_service_packs(
"""
service_pack_path_mapping = {}
service_pack_slug_mapping = {}

try:
response = requests.request("GET", PAR_LINK)

# if there is internet
if response.ok:
service_pack_list = response.json().get(SERVICE_PACKS)
# response not good.
else:
service_pack_list = _get_index_json_through_bucket(
namespace=namespace, bucketname=bucketname, auth=auth
)
except Exception as e:
# not internet
service_pack_list = _get_index_json_through_bucket(
namespace=namespace, bucketname=bucketname, auth=auth
namespace=namespace,
bucketname=bucketname,
auth=auth
)
except Exception as e:
logging.error(
"Failed to fetch service packs index from namespace '%s' and bucket '%s': %s",
namespace,
bucketname,
str(e),
)
return service_pack_path_mapping, service_pack_slug_mapping

for service_pack in service_pack_list:
# Here we need to replace the namespace and bucketname
Expand Down
19 changes: 11 additions & 8 deletions tests/unitary/with_extras/evaluator/test_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,23 @@
"""
Contains tests for ads.evaluations.evaluator
"""
import pytest
import unittest
import tempfile
from ads.evaluations.evaluator import Evaluator
import unittest

from sklearn.datasets import make_regression
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
import pandas as pd
import pytest
from lightgbm import LGBMClassifier, LGBMRegressor
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_classification, make_regression
from sklearn.linear_model import LinearRegression
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

from ads.common.model_metadata import UseCaseType
from ads.evaluations.evaluator import Evaluator
from ads.model.framework.lightgbm_model import LightGBMModel
from ads.model.framework.sklearn_model import SklearnModel

DEFAULT_PYTHON_VERSION = "3.12"

def test_model_types():
with pytest.raises(ValueError):
Expand All @@ -39,6 +40,7 @@ def test_pandas_input():
model.prepare(
inference_conda_env="generalml_p38_cpu_v1",
training_conda_env="generalml_p38_cpu_v1",
inference_python_version=DEFAULT_PYTHON_VERSION,
X_sample=X,
y_sample=y,
use_case_type=UseCaseType.MULTINOMIAL_CLASSIFICATION,
Expand Down Expand Up @@ -100,6 +102,7 @@ def train_eval_model(self, data, model, use_case):
my_model.prepare(
inference_conda_env="generalml_p38_cpu_v1",
training_conda_env="generalml_p38_cpu_v1",
inference_python_version=DEFAULT_PYTHON_VERSION,
X_sample=X_test,
y_sample=y_test,
use_case_type=use_case,
Expand Down
12 changes: 7 additions & 5 deletions tests/unitary/with_extras/model/test_generic_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@
from ads.model.deployment.common.utils import State as ModelDeploymentState
from ads.model.generic_model import (
_ATTRIBUTES_TO_SHOW_,
ArtifactsNotAvailableError,
GenericModel,
NotActiveDeploymentError,
ArtifactsNotAvailableError,
SummaryStatus,
_prepare_artifact_dir,
)
Expand Down Expand Up @@ -165,7 +165,7 @@

INFERENCE_CONDA_ENV = "oci://bucket@namespace/<path_to_service_pack>"
TRAINING_CONDA_ENV = "oci://bucket@namespace/<path_to_service_pack>"
DEFAULT_PYTHON_VERSION = "3.8"
DEFAULT_PYTHON_VERSION = "3.12"
MODEL_FILE_NAME = "fake_model_name"
FAKE_MD_URL = "http://<model-deployment-url>"

Expand Down Expand Up @@ -257,7 +257,8 @@ def test__handle_model_file_name_raise_error(self):
def test_prepare(self, mock_signer):
"""prepare a trained model."""
self.generic_model.prepare(
"oci://service-conda-packs@ociodscdev/service_pack/cpu/General_Machine_Learning_for_CPUs/1.0/mlcpuv1",
inference_conda_env="oci://service-conda-packs@ociodscdev/service_pack/cpu/General_Machine_Learning_for_CPUs/1.0/mlcpuv1",
inference_python_version=DEFAULT_PYTHON_VERSION,
model_file_name="fake_model_name",
)

Expand Down Expand Up @@ -1449,7 +1450,8 @@ def test__to_dict_not_prepared(self):
@patch("ads.common.auth.default_signer")
def test__to_dict_prepared(self, moxk_signer):
self.generic_model.prepare(
"oci://service-conda-packs@ociodscdev/service_pack/cpu/General_Machine_Learning_for_CPUs/1.0/mlcpuv1"
inference_conda_env="oci://service-conda-packs@ociodscdev/service_pack/cpu/General_Machine_Learning_for_CPUs/1.0/mlcpuv1",
inference_python_version=DEFAULT_PYTHON_VERSION
)
dictionary = self.generic_model._to_dict()
for key in _ATTRIBUTES_TO_SHOW_:
Expand Down Expand Up @@ -2255,7 +2257,7 @@ class TestCommonMethods:
def test__prepare_artifact_dir(
self, mock_signer, mock_mkdtemp, test_value, expected_value
):
"""Ensures that artifact dir name can be benerated propertly."""
"""Ensures that artifact dir name can be generated propertly."""

assert (
_prepare_artifact_dir(test_value) == expected_value
Expand Down
15 changes: 9 additions & 6 deletions tests/unitary/with_extras/model/test_model_metadata_mixin.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,22 @@
# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

import os
import shutil
from unittest.mock import patch

import numpy as np
import pytest
import sklearn
import xgboost
from sklearn import datasets, linear_model

from ads.model.generic_model import GenericModel
from ads.feature_engineering.schema import Schema
from ads.model.framework.sklearn_model import SklearnModel
from ads.model.framework.xgboost_model import XGBoostModel
from ads.feature_engineering.schema import Schema
import sklearn
import os
import shutil
import xgboost
from ads.model.generic_model import GenericModel

DEFAULT_PYTHON_VERSION = "3.12"

class TestMetadataMixin:
def setup_method(cls):
Expand Down Expand Up @@ -49,6 +51,7 @@ def test_metadata_generic_model(self):
model.prepare(
inference_conda_env="dataexpl_p37_cpu_v3",
namespace="ociodscdev",
inference_python_version=DEFAULT_PYTHON_VERSION,
model_file_name="model.joblib",
force_overwrite=True,
)
Expand Down