From b310de4a9ae68d3af29fd312c625f1dc706feab7 Mon Sep 17 00:00:00 2001 From: Ewe Zi Yi <36802364+deadlycoconuts@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:28:25 +0800 Subject: [PATCH] chore(sdk): Update mlflow version in sdk (#616) # Description This PR is the **first part** (1 out of 3) of a series of PRs to update the version of Mlflow used by Merlin (to `1.26.1`*). These changes are: 1. Update the version of Mlflow used in the Merlin SDK and publish it to PyPI 2. Update - the Merlin pyfunc server and batch predictor to use the updated Merlin SDK version released in step 1 and publish them to PyPI - the Merlin API server to ensure that requests sent to Mlflow reflect the update API endpoint contracts - the Mlflow image so that it's built using version `1.26.1` 3. Update the default pyfunc server and batch predictor version in the Merlin SDK and publish its new version to PyPI I also made some minor refactoring to the requirements files so that they are stored in a separate file and the reading of these files is consistent with what is done in the batch predictor and pyfunc server `setup.py` files. *I've selected `1.26.1` because it's the latest version of Mlflow available that is before Mlflow v2, that still allows `pandas==2.2.0` to be installed (this is a hard requirement needed by the `arize` package). # Modifications - Update Mlflow version to `1.26.1` (and other dependencies) in the SDK as well as observation publisher's requirements file since the unit tests there import the SDK's requirements file's dependencies - ~~Update unit tests to ensure that the updated `PythonModelContext` from Mlflow is initialised with the required arguments~~ (will only be needed for newer versions of Mlflow) # Tests # Checklist - [x] Added PR label - [ ] Added unit test, integration, and/or e2e tests - [x] Tested locally - [ ] Updated documentation - [ ] Update Swagger spec if the PR introduce API changes - [ ] Regenerated Golang and Python client if the PR introduces API changes # Release Notes ```release-note NONE ``` --- python/observation-publisher/requirements.txt | 6 +-- python/sdk/requirements.txt | 20 +++++++ python/sdk/requirements_test.txt | 18 +++++++ python/sdk/setup.py | 52 +++---------------- 4 files changed, 48 insertions(+), 48 deletions(-) create mode 100644 python/sdk/requirements.txt create mode 100644 python/sdk/requirements_test.txt diff --git a/python/observation-publisher/requirements.txt b/python/observation-publisher/requirements.txt index 9a9691a63..e67762d3e 100644 --- a/python/observation-publisher/requirements.txt +++ b/python/observation-publisher/requirements.txt @@ -22,9 +22,9 @@ binaryornot==0.4.4 # via cookiecutter blinker==1.7.0 # via flask -boto3==1.34.45 +boto3==1.35.39 # via merlin-sdk -botocore==1.34.45 +botocore==1.35.39 # via # boto3 # s3transfer @@ -215,7 +215,7 @@ python-dateutil==2.8.2 # pandas python-slugify==8.0.4 # via cookiecutter -pytz==2024.1 +pytz==2022.7.1 # via # mlflow # pandas diff --git a/python/sdk/requirements.txt b/python/sdk/requirements.txt new file mode 100644 index 000000000..3542b7e6f --- /dev/null +++ b/python/sdk/requirements.txt @@ -0,0 +1,20 @@ +boto3>=1.35.39 +caraml-upi-protos>=0.3.1 +certifi>=2017.4.17 +Click>=7.0,<8.1.4 +cloudpickle==2.0.0 # used by mlflow +cookiecutter>=1.7.2 +dataclasses-json>=0.5.2 # allow Flyte version 1.2.0 or above to import Merlin SDK +docker<=6.1.3 +GitPython>=3.1.40 +google-cloud-storage>=1.19.0 +protobuf>=3.12.0,<5.0.0 # Determined by the mlflow dependency +mlflow==1.26.1 +PyPrind>=2.11.2 +python_dateutil>=2.5.3 +PyYAML>=5.4 +six>=1.10 +urllib3>=1.26 +numpy<=1.23.5 # Temporary pin numpy due to https://numpy.org/doc/stable/release/1.20.0-notes.html#numpy-1-20-0-release-notes +caraml-auth-google==0.0.0.post7 +pydantic==2.5.3 \ No newline at end of file diff --git a/python/sdk/requirements_test.txt b/python/sdk/requirements_test.txt new file mode 100644 index 000000000..2eface6ea --- /dev/null +++ b/python/sdk/requirements_test.txt @@ -0,0 +1,18 @@ +google-cloud-bigquery-storage>=0.7.0 +google-cloud-bigquery>=1.18.0 +joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version +mypy>=0.812 +pytest-cov +pytest-dependency +pytest-xdist +pytest +recursive-diff>=1.0.0 +requests +scikit-learn>=1.1.2 +types-python-dateutil +types-PyYAML +types-six +types-protobuf +urllib3-mock>=0.3.3 +xarray +xgboost==1.6.2 \ No newline at end of file diff --git a/python/sdk/setup.py b/python/sdk/setup.py index eb530080c..7431c447f 100644 --- a/python/sdk/setup.py +++ b/python/sdk/setup.py @@ -22,49 +22,11 @@ "merlin.version", os.path.join("merlin", "version.py") ).VERSION -REQUIRES = [ - "boto3>=1.9.84", - "caraml-upi-protos>=0.3.1", - "certifi>=2017.4.17", - "Click>=7.0,<8.1.4", - "cloudpickle==2.0.0", # used by mlflow - "cookiecutter>=1.7.2", - "dataclasses-json>=0.5.2", # allow Flyte version 1.2.0 or above to import Merlin SDK - "docker<=6.1.3", - "GitPython>=3.1.40", - "google-cloud-storage>=1.19.0", - "protobuf>=3.12.0,<5.0.0", # Determined by the mlflow dependency - "mlflow==1.26.1", - "PyPrind>=2.11.2", - "python_dateutil>=2.5.3", - "PyYAML>=5.4", - "six>=1.10", - "urllib3>=1.26", - "numpy<=1.23.5", # Temporary pin numpy due to https://numpy.org/doc/stable/release/1.20.0-notes.html#numpy-1-20-0-release-notes - "caraml-auth-google==0.0.0.post7", - "pydantic==2.5.3" -] +with open("requirements.txt") as f: + REQUIRE = f.read().splitlines() -TEST_REQUIRES = [ - "google-cloud-bigquery-storage>=0.7.0", - "google-cloud-bigquery>=1.18.0", - "joblib>=0.13.0,<1.2.0", # >=1.2.0 upon upgrade of kserve's version - "mypy>=0.812", - "pytest-cov", - "pytest-dependency", - "pytest-xdist", - "pytest", - "recursive-diff>=1.0.0", - "requests", - "scikit-learn>=1.1.2", - "types-python-dateutil", - "types-PyYAML", - "types-six", - "types-protobuf", - "urllib3-mock>=0.3.3", - "xarray", - "xgboost==1.6.2", -] +with open("requirements_test.txt") as f: + TESTS_REQUIRE = f.read().splitlines() setup( name="merlin-sdk", @@ -75,10 +37,10 @@ packages=find_packages(), package_data={"merlin": ["docker/pyfunc.Dockerfile", "docker/standard.Dockerfile"]}, zip_safe=True, - install_requires=REQUIRES, + install_requires=REQUIRE, setup_requires=["setuptools_scm"], - tests_require=TEST_REQUIRES, - extras_require={"test": TEST_REQUIRES}, + tests_require=TESTS_REQUIRE, + extras_require={"test": TESTS_REQUIRE}, python_requires=">=3.8,<3.11", long_description=open("README.md").read(), long_description_content_type="text/markdown",