From 0582a3031a77f466697e8cc3b5920bc765c96f33 Mon Sep 17 00:00:00 2001 From: Tommy Hughes Date: Sun, 1 Dec 2024 15:05:55 -0600 Subject: [PATCH 1/2] operator example runbook & api reference docs Signed-off-by: Tommy Hughes --- examples/README.md | 21 +- examples/operator-quickstart/.gitignore | 1 + examples/operator-quickstart/01-Install.ipynb | 489 +++++++++++++++ examples/operator-quickstart/02-Demo.ipynb | 568 ++++++++++++++++++ .../operator-quickstart/03-Uninstall.ipynb | 97 +++ examples/operator-quickstart/README.md | 7 + examples/operator-quickstart/feast.yaml | 54 ++ examples/operator-quickstart/postgres.yaml | 52 ++ examples/operator-quickstart/redis.yaml | 37 ++ infra/feast-operator/Makefile | 18 +- infra/feast-operator/README.md | 5 +- .../config/samples/kustomization.yaml | 2 +- ...ha1_featurestore_all_services_default.yaml | 3 +- .../v1alpha1_featurestore_db_persistence.yaml | 38 +- infra/feast-operator/docs/api/markdown/ref.md | 541 +++++++++++++++++ .../docs/crd-ref-templates/config.yaml | 8 + .../crd-ref-templates/markdown/gv_details.tpl | 19 + .../crd-ref-templates/markdown/gv_list.tpl | 15 + .../docs/crd-ref-templates/markdown/type.tpl | 33 + .../markdown/type_members.tpl | 8 + 20 files changed, 1989 insertions(+), 27 deletions(-) create mode 100644 examples/operator-quickstart/.gitignore create mode 100644 examples/operator-quickstart/01-Install.ipynb create mode 100644 examples/operator-quickstart/02-Demo.ipynb create mode 100644 examples/operator-quickstart/03-Uninstall.ipynb create mode 100644 examples/operator-quickstart/README.md create mode 100644 examples/operator-quickstart/feast.yaml create mode 100644 examples/operator-quickstart/postgres.yaml create mode 100644 examples/operator-quickstart/redis.yaml create mode 100644 infra/feast-operator/docs/api/markdown/ref.md create mode 100644 infra/feast-operator/docs/crd-ref-templates/config.yaml create mode 100644 infra/feast-operator/docs/crd-ref-templates/markdown/gv_details.tpl create mode 100644 infra/feast-operator/docs/crd-ref-templates/markdown/gv_list.tpl create mode 100644 infra/feast-operator/docs/crd-ref-templates/markdown/type.tpl create mode 100644 infra/feast-operator/docs/crd-ref-templates/markdown/type_members.tpl diff --git a/examples/README.md b/examples/README.md index e796b5000e3..f968b94b5f6 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,18 +1,23 @@ # Feast Examples -1. **[Quickstart Example](https://github.com/feast-dev/feast/tree/master/examples/quickstart)**: This is a step-by-step guide for getting started with Feast. +1. **[Quickstart Example](quickstart)**: This is a step-by-step guide for getting started with Feast. -2. **[Java Demo](https://github.com/feast-dev/feast/tree/master/examples/java-demo)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. +2. **[Java Demo](java-demo)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. -3. **[Python Helm Demo](https://github.com/feast-dev/feast/tree/master/examples/python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. +3. **[Kind Quickstart](kind-quickstart)**: Demonstrates how to install and use Feast on Kind with the Helm chart. -4. **[RBAC Local](https://github.com/feast-dev/feast/tree/master/examples/rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. +4. **[Operator Quickstart](operator-quickstart)**: Demonstrates how to install and use Feast on Kubernetes with the Feast Go Operator. -5. **[RBAC Remote](https://github.com/feast-dev/feast/tree/master/examples/rbac-local)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. +5. **[Credit Risk End-to-End](credit-risk-end-to-end)**: Demonstrates how to use Feast with Java feature server and deployed with Kubernetes. -6. **[Remote Offline Store](https://github.com/feast-dev/feast/tree/master/examples/remote-offline-store)**: Demonstrates how to set up and use remote offline server. +6. **[Python Helm Demo](python-helm-demo)**: Demonstrates Feast with Kubernetes using Helm charts and Python feature server. -7. **[Podman/Podman Compose_local](https://github.com/feast-dev/feast/tree/master/examples/podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. +7. **[RBAC Local](rbac-local)**: Demonstrates using notebooks how configure and test Role-Based Access Control (RBAC) for securing access in Feast using OIDC authorization type with in a local environment. -8. **[RHOAI Feast Demo](https://github.com/feast-dev/feast/tree/master/examples/rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. +8. **[RBAC Remote](rbac-remote)**: Demonstrates how to configure and test Role-Based Access Control (RBAC) for securing access in Feast using Kubernetes or OIDC Authentication type with in Kubernetes environment. +9. **[Remote Offline Store](remote-offline-store)**: Demonstrates how to set up and use remote offline server. + +10. **[Podman/Podman Compose_local](podman_local)**: Demonstrates how to deploy Feast remote server components using Podman Compose locally. + +11. **[RHOAI Feast Demo](rhoai-quickstart)**: Showcases Feast's core functionality using a Jupyter notebook, including fetching online feature data from a remote server and retrieving metadata from a remote registry. diff --git a/examples/operator-quickstart/.gitignore b/examples/operator-quickstart/.gitignore new file mode 100644 index 00000000000..335ec9573de --- /dev/null +++ b/examples/operator-quickstart/.gitignore @@ -0,0 +1 @@ +*.tar.gz diff --git a/examples/operator-quickstart/01-Install.ipynb b/examples/operator-quickstart/01-Install.ipynb new file mode 100644 index 00000000000..d703357f791 --- /dev/null +++ b/examples/operator-quickstart/01-Install.ipynb @@ -0,0 +1,489 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Install Feast on Kubernetes with the Feast Operator\n", + "## Objective\n", + "\n", + "Provide a reference implementation of a runbook to deploy a Feast environment on a Kubernetes cluster using [Kind](https://kind.sigs.k8s.io/docs/user/quick-start) and the [Feast Operator](../../infra/feast-operator/)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "* Kubernetes Cluster\n", + "* [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) Kubernetes CLI tool." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Prerequisites\n", + "\n", + "The following commands install and configure all the prerequisites on a MacOS environment. You can find the\n", + "equivalent instructions on the offical documentation pages:\n", + "* Install the `kubectl` cli.\n", + "* Install Kubernetes and Container runtime (e.g. [Colima](https://github.com/abiosoft/colima)).\n", + " * Alternatively, authenticate to an existing Kubernetes or OpenShift cluster.\n", + " \n", + "```bash\n", + "brew install colima kubectl\n", + "colima start -r containerd -k -m 3 -d 100 -c 2 --cpu-type max -a x86_64\n", + "colima list\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast created\n", + "Context \"colima\" modified.\n" + ] + } + ], + "source": [ + "!kubectl create ns feast\n", + "!kubectl config set-context --current --namespace feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate the cluster setup:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\n", + "feast Active 3s\n" + ] + } + ], + "source": [ + "!kubectl get ns feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deployment Architecture\n", + "The primary objective of this runbook is to guide the deployment of Feast services on a Kubernetes Kind cluster, using the `postgres` template to set up a basic feature store.\n", + "\n", + "In this notebook, we will deploy a distributed topology of Feast services, which includes:\n", + "\n", + "* `Registry Server`: Handles metadata storage for feature definitions.\n", + "* `Online Store Server`: Uses the `Registry Server` to query metadata and is responsible for low-latency serving of features.\n", + "* `Offline Store Server`: Uses the `Registry Server` to query metadata and provides access to batch data for historical feature retrieval.\n", + "\n", + "Each service is backed by a `PostgreSQL` database, which is also deployed within the same Kind cluster." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Postgresql and Redis\n", + "Apply the included [postgres](postgres.yaml) & [redis](redis.yaml) deployments to run simple databases." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/postgres-secret created\n", + "deployment.apps/postgres created\n", + "service/postgres created\n", + "deployment.apps/redis created\n", + "service/redis created\n", + "deployment.apps/redis condition met\n", + "deployment.apps/postgres condition met\n" + ] + } + ], + "source": [ + "!kubectl apply -f postgres.yaml -f redis.yaml\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/redis\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/postgres" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "pod/postgres-ff8d4cf48-bqvfm 1/1 Running 0 99s\n", + "pod/redis-b4756b75d-fxkst 1/1 Running 0 98s\n", + "\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/postgres ClusterIP 10.43.197.94 5432/TCP 99s\n", + "service/redis ClusterIP 10.43.137.235 6379/TCP 98s\n", + "\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "deployment.apps/postgres 1/1 1 1 99s\n", + "deployment.apps/redis 1/1 1 1 98s\n", + "\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/postgres-ff8d4cf48 1 1 1 99s\n", + "replicaset.apps/redis-b4756b75d 1 1 1 98s\n" + ] + } + ], + "source": [ + "!kubectl get all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast Operator" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace/feast-operator-system created\n", + "customresourcedefinition.apiextensions.k8s.io/featurestores.feast.dev created\n", + "serviceaccount/feast-operator-controller-manager created\n", + "role.rbac.authorization.k8s.io/feast-operator-leader-election-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-editor-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-featurestore-viewer-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-manager-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-auth-role created\n", + "clusterrole.rbac.authorization.k8s.io/feast-operator-metrics-reader created\n", + "rolebinding.rbac.authorization.k8s.io/feast-operator-leader-election-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-manager-rolebinding created\n", + "clusterrolebinding.rbac.authorization.k8s.io/feast-operator-metrics-auth-rolebinding created\n", + "service/feast-operator-controller-manager-metrics-service created\n", + "deployment.apps/feast-operator-controller-manager created\n", + "deployment.apps/feast-operator-controller-manager condition met\n" + ] + } + ], + "source": [ + "## Use this install command from a release branch (e.g. 'v0.43-branch')\n", + "!kubectl apply -f ../../infra/feast-operator/dist/install.yaml\n", + "\n", + "## OR, for the latest code/builds, use one the following commands from the 'master' branch\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:develop FS_IMG=quay.io/feastdev-ci/feature-server:develop\n", + "# !make -C ../../infra/feast-operator install deploy IMG=quay.io/feastdev-ci/feast-operator:$(git rev-parse HEAD) FS_IMG=quay.io/feastdev-ci/feature-server:$(git rev-parse HEAD)\n", + "\n", + "!kubectl wait --for=condition=available --timeout=5m deployment/feast-operator-controller-manager -n feast-operator-system" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install the Feast services via FeatureStore CR\n", + "Next, we'll use the running Feast Operator to install the feast services. Apply the included [reference deployment](feast.yaml) to install and configure Feast." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/feast-data-stores created\n", + "featurestore.feast.dev/example created\n" + ] + } + ], + "source": [ + "!kubectl apply -f feast.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validate the running FeatureStore deployment\n", + "Validate the deployment status." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-example-74d5c98984-sr9bs 0/3 Init:0/1 0 4m43s\n", + "pod/postgres-ff8d4cf48-bqvfm 1/1 Running 0 7m36s\n", + "pod/redis-b4756b75d-fxkst 1/1 Running 0 7m35s\n", + "\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "service/feast-example-offline ClusterIP 10.43.81.130 80/TCP 4m44s\n", + "service/feast-example-online ClusterIP 10.43.74.219 80/TCP 4m43s\n", + "service/feast-example-registry ClusterIP 10.43.52.126 80/TCP 4m43s\n", + "service/postgres ClusterIP 10.43.197.94 5432/TCP 7m36s\n", + "service/redis ClusterIP 10.43.137.235 6379/TCP 7m35s\n", + "\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\n", + "deployment.apps/feast-example 0/1 1 0 4m43s\n", + "deployment.apps/postgres 1/1 1 1 7m36s\n", + "deployment.apps/redis 1/1 1 1 7m35s\n", + "\n", + "NAME DESIRED CURRENT READY AGE\n", + "replicaset.apps/feast-example-74d5c98984 1 1 0 4m43s\n", + "replicaset.apps/postgres-ff8d4cf48 1 1 1 7m36s\n", + "replicaset.apps/redis-b4756b75d 1 1 1 7m35s\n", + "deployment.apps/feast-example condition met\n" + ] + } + ], + "source": [ + "!kubectl get all\n", + "!kubectl wait --for=condition=available --timeout=8m deployment/feast-example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Validate that the FeatureStore CR is in a `Ready` state." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE\n", + "example Ready 10m\n" + ] + } + ], + "source": [ + "!kubectl get feast" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the DB includes the expected tables." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " List of relations\n", + " Schema | Name | Type | Owner \n", + "--------+-------------------------+-------+-------\n", + " public | data_sources | table | feast\n", + " public | entities | table | feast\n", + " public | feast_metadata | table | feast\n", + " public | feature_services | table | feast\n", + " public | feature_views | table | feast\n", + " public | managed_infra | table | feast\n", + " public | on_demand_feature_views | table | feast\n", + " public | permissions | table | feast\n", + " public | projects | table | feast\n", + " public | saved_datasets | table | feast\n", + " public | stream_feature_views | table | feast\n", + " public | validation_references | table | feast\n", + "(12 rows)\n", + "\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/postgres -- psql -h localhost -U feast feast -c '\\dt'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify the client `feature_store.yaml` and create the sample feature store definitions." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: credit_scoring_local\n", + "provider: local\n", + "offline_store:\n", + " type: duckdb\n", + "online_store:\n", + " type: redis\n", + " connection_string: redis.feast.svc.cluster.local:6379\n", + "registry:\n", + " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.feast.svc.cluster.local:5432/${POSTGRES_DB}\n", + " registry_type: sql\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "auth:\n", + " type: no_auth\n", + "entity_key_serialization_version: 3\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/feast-data/credit_scoring_local/feature_repo/example_repo.py:27: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " driver = Entity(name=\"driver\", join_keys=[\"driver_id\"])\n", + "Applying changes for project credit_scoring_local\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_store.py:578: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", + " warnings.warn(\n", + "Deploying infrastructure for \u001b[1m\u001b[32mdriver_hourly_stats\u001b[0m\n", + "Deploying infrastructure for \u001b[1m\u001b[32mdriver_hourly_stats_fresh\u001b[0m\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- cat feature_store.yaml\n", + "!kubectl exec deploy/feast-example -itc registry -- feast apply" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List the registered feast projects & feature views." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME DESCRIPTION TAGS OWNER\n", + "credit_scoring_local A project for driver statistics {}\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "NAME ENTITIES TYPE\n", + "driver_hourly_stats {'driver'} FeatureView\n", + "driver_hourly_stats_fresh {'driver'} FeatureView\n", + "transformed_conv_rate {'driver'} OnDemandFeatureView\n", + "transformed_conv_rate_fresh {'driver'} OnDemandFeatureView\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- feast projects list\n", + "!kubectl exec deploy/feast-example -itc registry -- feast feature-views list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let's verify the feast version." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "Feast SDK Version: \"0.43.0\"\n" + ] + } + ], + "source": [ + "!kubectl exec deployment/feast-example -itc registry -- feast version" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-quickstart/02-Demo.ipynb b/examples/operator-quickstart/02-Demo.ipynb new file mode 100644 index 00000000000..51e26ba3616 --- /dev/null +++ b/examples/operator-quickstart/02-Demo.ipynb @@ -0,0 +1,568 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run the \"Real-time Credit Scoring\" tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll use the following tutorial as a demonstration.\n", + "\n", + "https://github.com/feast-dev/feast-credit-score-local-tutorial/tree/f43b44b245ae2632b582f14176392cfe31f98da9" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload the tutorial source code to the running Feast pod." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Upload the tutorial source code to the running feast pod and extract its contents." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2025-01-23 18:09:13-- https://github.com/feast-dev/feast-credit-score-local-tutorial/archive/f43b44b.tar.gz\n", + "Resolving github.com (github.com)... 140.82.112.4\n", + "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n", + "HTTP request sent, awaiting response... 302 Found\n", + "Location: https://codeload.github.com/feast-dev/feast-credit-score-local-tutorial/tar.gz/f43b44b245ae2632b582f14176392cfe31f98da9 [following]\n", + "--2025-01-23 18:09:13-- https://codeload.github.com/feast-dev/feast-credit-score-local-tutorial/tar.gz/f43b44b245ae2632b582f14176392cfe31f98da9\n", + "Resolving codeload.github.com (codeload.github.com)... 140.82.114.10\n", + "Connecting to codeload.github.com (codeload.github.com)|140.82.114.10|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: unspecified [application/x-gzip]\n", + "Saving to: ‘f43b44b.tar.gz’\n", + "\n", + "f43b44b.tar.gz [ <=> ] 45.52M 10.8MB/s in 5.0s \n", + "\n", + "2025-01-23 18:09:19 (9.12 MB/s) - ‘f43b44b.tar.gz’ saved [47734189]\n", + "\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/.gitignore\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/LICENSE\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/README.md\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/app.py\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/credit_model.py\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/credit_history.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/credit_history_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/loan_table.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/loan_table_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/training_dataset_sample.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/zipcode_table.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/data/zipcode_table_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/credit_history.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/credit_history_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/loan_table.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/loan_table_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/training_dataset_sample.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/zipcode_table.parquet\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/data/zipcode_table_sample.csv\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/feature_store.yaml\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/feature_repo/features.py\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/requirements.txt\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/run.py\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/streamlit.png\n", + "feast-credit-score-local-tutorial-f43b44b245ae2632b582f14176392cfe31f98da9/streamlit_app.py\n" + ] + } + ], + "source": [ + "![ -f f43b44b.tar.gz ] || wget https://github.com/feast-dev/feast-credit-score-local-tutorial/archive/f43b44b.tar.gz\n", + "!kubectl cp f43b44b.tar.gz $(kubectl get pods -l 'feast.dev/name=example' -ojsonpath=\"{.items[*].metadata.name}\"):/feast-data -c registry\n", + "!kubectl exec deploy/feast-example -itc registry -- rm -rf /feast-data/feast-credit-score-local-tutorial\n", + "!kubectl exec deploy/feast-example -itc registry -- mkdir /feast-data/feast-credit-score-local-tutorial\n", + "!kubectl exec deploy/feast-example -itc registry -- tar vfx /feast-data/f43b44b.tar.gz -C /feast-data/feast-credit-score-local-tutorial --strip-components 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verify the client `feature_store.yaml`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copy the `feature_store.yaml` to the tutorial directory and verify its contents." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "project: credit_scoring_local\n", + "provider: local\n", + "offline_store:\n", + " type: duckdb\n", + "online_store:\n", + " type: redis\n", + " connection_string: redis.feast.svc.cluster.local:6379\n", + "registry:\n", + " path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.feast.svc.cluster.local:5432/${POSTGRES_DB}\n", + " registry_type: sql\n", + " cache_ttl_seconds: 60\n", + " sqlalchemy_config_kwargs:\n", + " echo: false\n", + " pool_pre_ping: true\n", + "auth:\n", + " type: no_auth\n", + "entity_key_serialization_version: 3\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- cp -f /feast-data/credit_scoring_local/feature_repo/feature_store.yaml /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml\n", + "!kubectl exec deploy/feast-example -itc registry -- cat /feast-data/feast-credit-score-local-tutorial/feature_repo/feature_store.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Apply the tutorial feature store definitions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Update the feature store definitions for the tutorial." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "No project found in the repository. Using project name credit_scoring_local defined in feature_store.yaml\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "Applying changes for project credit_scoring_local\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'driver'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_store.py:578: RuntimeWarning: On demand feature view is an experimental feature. This API is stable, but the functionality does not scale well for offline retrieval\n", + " warnings.warn(\n", + "Deploying infrastructure for \u001b[1m\u001b[32mcredit_history\u001b[0m\n", + "Deploying infrastructure for \u001b[1m\u001b[32mzipcode_features\u001b[0m\n", + "Removing infrastructure for \u001b[1m\u001b[31mdriver_hourly_stats_fresh\u001b[0m\n", + "Removing infrastructure for \u001b[1m\u001b[31mdriver_hourly_stats\u001b[0m\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo apply" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load data from feature views into the online store, beginning from either the previous materialize or materialize-incremental end date, or the beginning of time." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views to \u001b[1m\u001b[32m2025-01-24 00:10:49+00:00\u001b[0m into the \u001b[1m\u001b[32mredis\u001b[0m online store.\n", + "\n", + "\u001b[1m\u001b[32mcredit_history\u001b[0m from \u001b[1m\u001b[32m2024-10-26 00:11:25+00:00\u001b[0m to \u001b[1m\u001b[32m2025-01-24 00:10:49+00:00\u001b[0m:\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "0it [00:00, ?it/s]\n", + "\u001b[1m\u001b[32mzipcode_features\u001b[0m from \u001b[1m\u001b[32m2015-01-27 00:11:41+00:00\u001b[0m to \u001b[1m\u001b[32m2025-01-24 00:10:49+00:00\u001b[0m:\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "100%|███████████████████████████████████████████████████████| 28844/28844 [00:26<00:00, 1090.11it/s]\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial/feature_repo && feast materialize-incremental $(date -u +\"%Y-%m-%dT%H:%M:%S\")'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Execute feast commands inside the client Pod" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "List the registered feast feature views & entities." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "NAME ENTITIES TYPE\n", + "credit_history {'dob_ssn'} FeatureView\n", + "zipcode_features {'zipcode'} FeatureView\n", + "total_debt_calc {'dob_ssn'} OnDemandFeatureView\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "NAME DESCRIPTION TYPE\n", + "zipcode ValueType.INT64\n", + "dob_ssn Date of birth and last four digits of social security number ValueType.STRING\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo feature-views list\n", + "!kubectl exec deploy/feast-example -itc registry -- feast -c /feast-data/feast-credit-score-local-tutorial/feature_repo entities list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train and test the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install the required packages." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting streamlit (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading streamlit-1.41.1-py2.py3-none-any.whl.metadata (8.5 kB)\n", + "Collecting shap (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + " Downloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (24 kB)\n", + "Requirement already satisfied: pandas in /opt/app-root/lib64/python3.11/site-packages (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2.2.3)\n", + "Collecting scikit-learn (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + " Downloading scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (18 kB)\n", + "Collecting matplotlib (from -r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n", + "Collecting altair<6,>=4.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading altair-5.5.0-py3-none-any.whl.metadata (11 kB)\n", + "Collecting blinker<2,>=1.0.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading blinker-1.9.0-py3-none-any.whl.metadata (1.6 kB)\n", + "Requirement already satisfied: cachetools<6,>=4.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (5.5.1)\n", + "Requirement already satisfied: click<9,>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (8.1.8)\n", + "Requirement already satisfied: numpy<3,>=1.23 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (1.26.4)\n", + "Requirement already satisfied: packaging<25,>=20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (24.2)\n", + "Collecting pillow<12,>=7.1.0 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (9.1 kB)\n", + "Requirement already satisfied: protobuf<6,>=3.20 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (4.25.5)\n", + "Requirement already satisfied: pyarrow>=7.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (17.0.0)\n", + "Requirement already satisfied: requests<3,>=2.27 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.32.3)\n", + "Requirement already satisfied: rich<14,>=10.14.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (13.9.4)\n", + "Requirement already satisfied: tenacity<10,>=8.1.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (8.5.0)\n", + "Requirement already satisfied: toml<2,>=0.10.1 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.10.2)\n", + "Requirement already satisfied: typing-extensions<5,>=4.3.0 in /opt/app-root/lib64/python3.11/site-packages (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (4.12.2)\n", + "Collecting watchdog<7,>=2.1.5 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl.metadata (44 kB)\n", + "Collecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading GitPython-3.1.44-py3-none-any.whl.metadata (13 kB)\n", + "Collecting pydeck<1,>=0.8.0b4 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading pydeck-0.9.1-py2.py3-none-any.whl.metadata (4.1 kB)\n", + "Collecting tornado<7,>=6.0.3 (from streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (2.5 kB)\n", + "Collecting scipy (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + " Downloading scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", + "Requirement already satisfied: tqdm>=4.27.0 in /opt/app-root/lib64/python3.11/site-packages (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2)) (4.67.1)\n", + "Collecting slicer==0.0.8 (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + " Downloading slicer-0.0.8-py3-none-any.whl.metadata (4.0 kB)\n", + "Collecting numba (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + " Downloading numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: cloudpickle in /opt/app-root/lib64/python3.11/site-packages (from shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2)) (3.1.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/app-root/lib64/python3.11/site-packages (from pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (2025.1)\n", + "Collecting joblib>=1.2.0 (from scikit-learn->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + " Downloading joblib-1.4.2-py3-none-any.whl.metadata (5.4 kB)\n", + "Collecting threadpoolctl>=3.1.0 (from scikit-learn->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 4))\n", + " Downloading threadpoolctl-3.5.0-py3-none-any.whl.metadata (13 kB)\n", + "Collecting contourpy>=1.0.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (5.4 kB)\n", + "Collecting cycler>=0.10 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading cycler-0.12.1-py3-none-any.whl.metadata (3.8 kB)\n", + "Collecting fonttools>=4.22.0 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading fonttools-4.55.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (165 kB)\n", + "Collecting kiwisolver>=1.3.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.2 kB)\n", + "Collecting pyparsing>=2.3.1 (from matplotlib->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 5))\n", + " Downloading pyparsing-3.2.1-py3-none-any.whl.metadata (5.0 kB)\n", + "Requirement already satisfied: jinja2 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.1.5)\n", + "Requirement already satisfied: jsonschema>=3.0 in /opt/app-root/lib64/python3.11/site-packages (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (4.23.0)\n", + "Collecting narwhals>=1.14.2 (from altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading narwhals-1.23.0-py3-none-any.whl.metadata (10.0 kB)\n", + "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading gitdb-4.0.12-py3-none-any.whl.metadata (1.2 kB)\n", + "Requirement already satisfied: six>=1.5 in /opt/app-root/lib64/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 3)) (1.17.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/app-root/lib64/python3.11/site-packages (from requests<3,>=2.27->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2024.12.14)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/app-root/lib64/python3.11/site-packages (from rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2.19.1)\n", + "Collecting llvmlite<0.45,>=0.44.0dev0 (from numba->shap->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 2))\n", + " Downloading llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.8 kB)\n", + "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1))\n", + " Downloading smmap-5.0.2-py3-none-any.whl.metadata (4.3 kB)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/app-root/lib64/python3.11/site-packages (from jinja2->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (3.0.2)\n", + "Requirement already satisfied: attrs>=22.2.0 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (24.3.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.36.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/app-root/lib64/python3.11/site-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.22.3)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/app-root/lib64/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit->-r /feast-data/feast-credit-score-local-tutorial/requirements.txt (line 1)) (0.1.2)\n", + "Downloading streamlit-1.41.1-py2.py3-none-any.whl (9.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.1/9.1 MB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading shap-0.46.0-cp311-cp311-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (540 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m540.2/540.2 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading slicer-0.0.8-py3-none-any.whl (15 kB)\n", + "Downloading scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (13.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.5/13.5 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\u001b[36m0:00:01\u001b[0mm\n", + "\u001b[?25hDownloading matplotlib-3.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (8.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.6/8.6 MB\u001b[0m \u001b[31m11.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading altair-5.5.0-py3-none-any.whl (731 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.2/731.2 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading blinker-1.9.0-py3-none-any.whl (8.5 kB)\n", + "Downloading contourpy-1.3.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (326 kB)\n", + "Downloading cycler-0.12.1-py3-none-any.whl (8.3 kB)\n", + "Downloading fonttools-4.55.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.9 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.9/4.9 MB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading GitPython-3.1.44-py3-none-any.whl (207 kB)\n", + "Downloading joblib-1.4.2-py3-none-any.whl (301 kB)\n", + "Downloading kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.4/1.4 MB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pillow-11.1.0-cp311-cp311-manylinux_2_28_x86_64.whl (4.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading pydeck-0.9.1-py2.py3-none-any.whl (6.9 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.9/6.9 MB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n", + "\u001b[?25hDownloading pyparsing-3.2.1-py3-none-any.whl (107 kB)\n", + "Downloading scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (40.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.6/40.6 MB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading threadpoolctl-3.5.0-py3-none-any.whl (18 kB)\n", + "Downloading tornado-6.4.2-cp38-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (437 kB)\n", + "Downloading watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl (79 kB)\n", + "Downloading numba-0.61.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading gitdb-4.0.12-py3-none-any.whl (62 kB)\n", + "Downloading llvmlite-0.44.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (42.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.4/42.4 MB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n", + "\u001b[?25hDownloading narwhals-1.23.0-py3-none-any.whl (306 kB)\n", + "Downloading smmap-5.0.2-py3-none-any.whl (24 kB)\n", + "Installing collected packages: watchdog, tornado, threadpoolctl, smmap, slicer, scipy, pyparsing, pillow, narwhals, llvmlite, kiwisolver, joblib, fonttools, cycler, contourpy, blinker, scikit-learn, pydeck, numba, matplotlib, gitdb, shap, gitpython, altair, streamlit\n", + "Successfully installed altair-5.5.0 blinker-1.9.0 contourpy-1.3.1 cycler-0.12.1 fonttools-4.55.5 gitdb-4.0.12 gitpython-3.1.44 joblib-1.4.2 kiwisolver-1.4.8 llvmlite-0.44.0 matplotlib-3.10.0 narwhals-1.23.0 numba-0.61.0 pillow-11.1.0 pydeck-0.9.1 pyparsing-3.2.1 scikit-learn-1.6.1 scipy-1.15.1 shap-0.46.0 slicer-0.0.8 smmap-5.0.2 streamlit-1.41.1 threadpoolctl-3.5.0 tornado-6.4.2 watchdog-6.0.0\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- bash -c 'pip install -r /feast-data/feast-credit-score-local-tutorial/requirements.txt'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Train and test the model." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/opt/app-root/lib64/python3.11/site-packages/feast/feature_view.py:48: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " DUMMY_ENTITY = Entity(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity '__dummy'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'zipcode'.\n", + " entity = cls(\n", + "/opt/app-root/lib64/python3.11/site-packages/feast/entity.py:173: DeprecationWarning: Entity value_type will be mandatory in the next release. Please specify a value_type for entity 'dob_ssn'.\n", + " entity = cls(\n", + "Loan rejected!\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && python run.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Interactive demo (using Streamlit)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In a new terminal, run the following command and leave it active.\n", + "\n", + "```bash\n", + "$ kubectl port-forward deploy/feast-example 8501:8501\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start the Streamlit application" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Collecting usage statistics. To deactivate, set browser.gatherUsageStats to false.\n", + "\u001b[0m\n", + "\u001b[0m\n", + "\u001b[34m\u001b[1m You can now view your Streamlit app in your browser.\u001b[0m\n", + "\u001b[0m\n", + "\u001b[34m Local URL: \u001b[0m\u001b[1mhttp://localhost:8501\u001b[0m\n", + "\u001b[34m Network URL: \u001b[0m\u001b[1mhttp://10.42.0.8:8501\u001b[0m\n", + "\u001b[34m External URL: \u001b[0m\u001b[1mhttp://23.112.66.217:8501\u001b[0m\n", + "\u001b[0m\n" + ] + } + ], + "source": [ + "!kubectl exec deploy/feast-example -itc registry -- bash -c 'cd /feast-data/feast-credit-score-local-tutorial && streamlit run --server.port 8501 streamlit_app.py'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then navigate to the local URL on which Streamlit is being served.\n", + "\n", + "http://localhost:8501" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-quickstart/03-Uninstall.ipynb b/examples/operator-quickstart/03-Uninstall.ipynb new file mode 100644 index 00000000000..02ebedcb50b --- /dev/null +++ b/examples/operator-quickstart/03-Uninstall.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Uninstall the Operator and all Feast related objects" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret \"feast-data-stores\" deleted\n", + "featurestore.feast.dev \"example\" deleted\n", + "secret \"postgres-secret\" deleted\n", + "deployment.apps \"postgres\" deleted\n", + "service \"postgres\" deleted\n", + "deployment.apps \"redis\" deleted\n", + "service \"redis\" deleted\n", + "namespace \"feast-operator-system\" deleted\n", + "customresourcedefinition.apiextensions.k8s.io \"featurestores.feast.dev\" deleted\n", + "serviceaccount \"feast-operator-controller-manager\" deleted\n", + "role.rbac.authorization.k8s.io \"feast-operator-leader-election-role\" deleted\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-featurestore-editor-role\" deleted\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-featurestore-viewer-role\" deleted\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-manager-role\" deleted\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-metrics-auth-role\" deleted\n", + "clusterrole.rbac.authorization.k8s.io \"feast-operator-metrics-reader\" deleted\n", + "rolebinding.rbac.authorization.k8s.io \"feast-operator-leader-election-rolebinding\" deleted\n", + "clusterrolebinding.rbac.authorization.k8s.io \"feast-operator-manager-rolebinding\" deleted\n", + "clusterrolebinding.rbac.authorization.k8s.io \"feast-operator-metrics-auth-rolebinding\" deleted\n", + "service \"feast-operator-controller-manager-metrics-service\" deleted\n", + "deployment.apps \"feast-operator-controller-manager\" deleted\n" + ] + } + ], + "source": [ + "!kubectl delete -f feast.yaml\n", + "!kubectl delete -f postgres.yaml\n", + "!kubectl delete -f redis.yaml\n", + "!kubectl delete -f ../../infra/feast-operator/dist/install.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ensure everything has been removed, or is in the process of being terminated." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "pod/feast-example-74d5c98984-sr9bs 2/3 Terminating 2 (18m ago) 27m\n" + ] + } + ], + "source": [ + "!kubectl get all" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/operator-quickstart/README.md b/examples/operator-quickstart/README.md new file mode 100644 index 00000000000..56ba7173b56 --- /dev/null +++ b/examples/operator-quickstart/README.md @@ -0,0 +1,7 @@ +# Install and run a Feature Store on Kubernetes with the Feast Operator + +The following notebooks will guide you through how to install and use Feast on Kubernetes with the Feast Go Operator. + +* [01-Install.ipynb](./01-Install.ipynb): Install and configure a Feature Store in Kubernetes with the Operator. +* [02-Demo.ipynb](./02-Demo.ipynb): Validate the feature store with demo application. +* [03-Uninstall.ipynb](./03-Uninstall.ipynb): Clear the installed deployments. diff --git a/examples/operator-quickstart/feast.yaml b/examples/operator-quickstart/feast.yaml new file mode 100644 index 00000000000..e4eb20b4c0c --- /dev/null +++ b/examples/operator-quickstart/feast.yaml @@ -0,0 +1,54 @@ +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores +stringData: + redis: | + connection_string: redis.feast.svc.cluster.local:6379 + sql: | + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.feast.svc.cluster.local:5432/${POSTGRES_DB} + cache_ttl_seconds: 60 + sqlalchemy_config_kwargs: + echo: false + pool_pre_ping: true +--- +apiVersion: feast.dev/v1alpha1 +kind: FeatureStore +metadata: + name: example +spec: + feastProject: credit_scoring_local + services: + offlineStore: + persistence: + file: + type: duckdb + envFrom: + - secretRef: + name: postgres-secret + onlineStore: + persistence: + store: + type: redis + secretRef: + name: feast-data-stores + envFrom: + - secretRef: + name: postgres-secret + registry: + local: + persistence: + store: + type: sql + secretRef: + name: feast-data-stores + envFrom: + - secretRef: + name: postgres-secret + env: + - name: MPLCONFIGDIR + value: /tmp + resources: + requests: + cpu: 150m + memory: 128Mi diff --git a/examples/operator-quickstart/postgres.yaml b/examples/operator-quickstart/postgres.yaml new file mode 100644 index 00000000000..4baa0092575 --- /dev/null +++ b/examples/operator-quickstart/postgres.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Secret +metadata: + name: postgres-secret +stringData: + POSTGRES_DB: feast + POSTGRES_USER: feast + POSTGRES_PASSWORD: feast +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: 'postgres:16-alpine' + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: postgres-secret + volumeMounts: + - mountPath: /var/lib/postgresql + name: postgresdata + volumes: + - name: postgresdata + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + labels: + app: postgres +spec: + type: ClusterIP + ports: + - port: 5432 + targetPort: 5432 + protocol: TCP + selector: + app: postgres \ No newline at end of file diff --git a/examples/operator-quickstart/redis.yaml b/examples/operator-quickstart/redis.yaml new file mode 100644 index 00000000000..f39dfc9768d --- /dev/null +++ b/examples/operator-quickstart/redis.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: 'bitnami/redis:latest' + ports: + - containerPort: 6379 + env: + - name: ALLOW_EMPTY_PASSWORD + value: "yes" +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + labels: + app: redis +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + protocol: TCP + selector: + app: redis \ No newline at end of file diff --git a/infra/feast-operator/Makefile b/infra/feast-operator/Makefile index dee7c3068b6..de8164b7f27 100644 --- a/infra/feast-operator/Makefile +++ b/infra/feast-operator/Makefile @@ -112,7 +112,7 @@ vet: ## Run go vet against code. go vet ./... .PHONY: test -test: build-installer fmt vet lint envtest ## Run tests. +test: build-installer vet lint envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. @@ -173,7 +173,7 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform rm Dockerfile.cross .PHONY: build-installer -build-installer: manifests generate related-image-fs kustomize ## Generate a consolidated YAML with CRDs and deployment. +build-installer: manifests generate-ref kustomize related-image-fs ## Generate a consolidated YAML with CRDs and deployment. mkdir -p dist cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/default > dist/install.yaml @@ -204,7 +204,8 @@ undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/. ##@ Dependencies ## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin +LOCALDIR ?= $(shell pwd) +LOCALBIN ?= $(LOCALDIR)/bin $(LOCALBIN): mkdir -p $(LOCALBIN) @@ -212,6 +213,7 @@ $(LOCALBIN): KUBECTL ?= kubectl KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +CRD_REF_DOCS ?= $(LOCALBIN)/crd-ref-docs ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ENVSUBST = $(LOCALBIN)/envsubst @@ -219,6 +221,7 @@ ENVSUBST = $(LOCALBIN)/envsubst ## Tool Versions KUSTOMIZE_VERSION ?= v5.4.2 CONTROLLER_TOOLS_VERSION ?= v0.15.0 +CRD_REF_DOCS_VERSION ?= v0.1.0 ENVTEST_VERSION ?= release-0.18 GOLANGCI_LINT_VERSION ?= v1.59.1 ENVSUBST_VERSION ?= v1.4.2 @@ -282,6 +285,15 @@ OPERATOR_SDK = $(shell which operator-sdk) endif endif +.PHONY: crd-ref-docs +crd-ref-docs: $(CRD_REF_DOCS) ## Download crd-ref-docs locally if necessary. +$(CRD_REF_DOCS): $(LOCALBIN) + $(call go-install-tool,$(CRD_REF_DOCS),github.com/elastic/crd-ref-docs,$(CRD_REF_DOCS_VERSION)) + +.PHONY: generate-ref +generate-ref: generate fmt crd-ref-docs + $(CRD_REF_DOCS) --log-level=WARN --config=$(LOCALDIR)/docs/crd-ref-templates/config.yaml --source-path=$(LOCALDIR)/api/v1alpha1 --renderer=markdown --templates-dir=$(LOCALDIR)/docs/crd-ref-templates/markdown --output-path=$(LOCALDIR)/docs/api/markdown/ref.md + .PHONY: bundle bundle: manifests kustomize related-image-fs operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q diff --git a/infra/feast-operator/README.md b/infra/feast-operator/README.md index 9d4b51f37f4..b4c3ed6565b 100644 --- a/infra/feast-operator/README.md +++ b/infra/feast-operator/README.md @@ -1,6 +1,8 @@ # Feast Operator This is a K8s Operator that can be used to deploy and manage **Feast**, an open source feature store for machine learning. +### **[FeatureStore CR API Reference](docs/api/markdown/ref.md)** + ## Getting Started ### Prerequisites @@ -153,6 +155,3 @@ make test-e2e # delete cluster once you are done. kind delete cluster ``` - - - diff --git a/infra/feast-operator/config/samples/kustomization.yaml b/infra/feast-operator/config/samples/kustomization.yaml index 4869cc7b245..ec9b395f9dc 100644 --- a/infra/feast-operator/config/samples/kustomization.yaml +++ b/infra/feast-operator/config/samples/kustomization.yaml @@ -1,4 +1,4 @@ ## Append samples of your project ## resources: -- v1alpha1_featurestore.yaml +- v1alpha1_featurestore_all_services_default.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_all_services_default.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_all_services_default.yaml index 04b6b3dd10d..1eef0874fff 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_all_services_default.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_all_services_default.yaml @@ -1,7 +1,7 @@ apiVersion: feast.dev/v1alpha1 kind: FeatureStore metadata: - name: sample-all-default + name: sample-all-services spec: feastProject: my_project services: @@ -10,4 +10,3 @@ spec: ui: {} registry: local: {} - diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_db_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_db_persistence.yaml index fd6feb79f2b..eaeca893491 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_db_persistence.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_db_persistence.yaml @@ -3,20 +3,32 @@ kind: Secret metadata: name: postgres-secret namespace: test + labels: + app: postgres +stringData: + POSTGRES_DB: feast + POSTGRES_USER: feast + POSTGRES_PASSWORD: feast +--- +apiVersion: v1 +kind: Secret +metadata: + name: feast-data-stores + namespace: test stringData: postgres-secret-parameters: | - path: postgresql+psycopg://postgres:mysecretpassword@127.0.0.1:55001/feast + path: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres.test.svc.cluster.local:5432/${POSTGRES_DB} cache_ttl_seconds: 60 sqlalchemy_config_kwargs: - echo: false - pool_pre_ping: true + echo: false + pool_pre_ping: true postgres: | - host: 127.0.0.1 - port: 55001 - database: feast + host: postgres.test.svc.cluster.local + port: 5432 + database: ${POSTGRES_DB} db_schema: public - user: postgres - password: mysecretpassword + user: ${POSTGRES_USER} + password: ${POSTGRES_PASSWORD} --- apiVersion: feast.dev/v1alpha1 kind: FeatureStore @@ -31,12 +43,18 @@ spec: store: type: postgres secretRef: - name: postgres-secret + name: feast-data-stores + envFrom: + - secretRef: + name: postgres-secret registry: local: persistence: store: type: sql secretRef: - name: postgres-secret + name: feast-data-stores secretKeyName: postgres-secret-parameters # optional, will use store.type by default as the SecretKeyName if none is specified, in this case that's "sql" + envFrom: + - secretRef: + name: postgres-secret diff --git a/infra/feast-operator/docs/api/markdown/ref.md b/infra/feast-operator/docs/api/markdown/ref.md new file mode 100644 index 00000000000..9d903d050a3 --- /dev/null +++ b/infra/feast-operator/docs/api/markdown/ref.md @@ -0,0 +1,541 @@ +# API Reference + +## Packages +- [feast.dev/v1alpha1](#feastdevv1alpha1) + + +## feast.dev/v1alpha1 + +Package v1alpha1 contains API Schema definitions for the v1alpha1 API group + +### Resource Types +- [FeatureStore](#featurestore) + + + +#### AuthzConfig + + + +AuthzConfig defines the authorization settings for the deployed Feast services. + +_Appears in:_ +- [FeatureStoreSpec](#featurestorespec) + +| Field | Description | +| --- | --- | +| `kubernetes` _[KubernetesAuthz](#kubernetesauthz)_ | | +| `oidc` _[OidcAuthz](#oidcauthz)_ | | + + +#### DefaultConfigs + + + +DefaultConfigs k8s container settings that are applied by default + +_Appears in:_ +- [LocalRegistryConfig](#localregistryconfig) +- [OfflineStore](#offlinestore) +- [OnlineStore](#onlinestore) +- [ServiceConfigs](#serviceconfigs) +- [UIService](#uiservice) + +| Field | Description | +| --- | --- | +| `image` _string_ | | + + +#### FeatureStore + + + +FeatureStore is the Schema for the featurestores API + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `feast.dev/v1alpha1` +| `kind` _string_ | `FeatureStore` +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[FeatureStoreSpec](#featurestorespec)_ | | +| `status` _[FeatureStoreStatus](#featurestorestatus)_ | | + + +#### FeatureStoreRef + + + +FeatureStoreRef defines which existing FeatureStore's registry should be used + +_Appears in:_ +- [RemoteRegistryConfig](#remoteregistryconfig) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name of the FeatureStore | +| `namespace` _string_ | Namespace of the FeatureStore | + + +#### FeatureStoreServices + + + +FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. + +_Appears in:_ +- [FeatureStoreSpec](#featurestorespec) + +| Field | Description | +| --- | --- | +| `offlineStore` _[OfflineStore](#offlinestore)_ | | +| `onlineStore` _[OnlineStore](#onlinestore)_ | | +| `registry` _[Registry](#registry)_ | | +| `ui` _[UIService](#uiservice)_ | | +| `deploymentStrategy` _[DeploymentStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#deploymentstrategy-v1-apps)_ | | +| `disableInitContainers` _boolean_ | Disable the 'feast repo initialization' initContainer | + + +#### FeatureStoreSpec + + + +FeatureStoreSpec defines the desired state of FeatureStore + +_Appears in:_ +- [FeatureStore](#featurestore) +- [FeatureStoreStatus](#featurestorestatus) + +| Field | Description | +| --- | --- | +| `feastProject` _string_ | FeastProject is the Feast project id. This can be any alphanumeric string with underscores, but it cannot start with an underscore. Required. | +| `services` _[FeatureStoreServices](#featurestoreservices)_ | | +| `authz` _[AuthzConfig](#authzconfig)_ | | + + +#### FeatureStoreStatus + + + +FeatureStoreStatus defines the observed state of FeatureStore + +_Appears in:_ +- [FeatureStore](#featurestore) + +| Field | Description | +| --- | --- | +| `applied` _[FeatureStoreSpec](#featurestorespec)_ | Shows the currently applied feast configuration, including any pertinent defaults | +| `clientConfigMap` _string_ | ConfigMap in this namespace containing a client `feature_store.yaml` for this feast deployment | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#condition-v1-meta) array_ | | +| `feastVersion` _string_ | | +| `phase` _string_ | | +| `serviceHostnames` _[ServiceHostnames](#servicehostnames)_ | | + + +#### KubernetesAuthz + + + +KubernetesAuthz provides a way to define the authorization settings using Kubernetes RBAC resources. +https://kubernetes.io/docs/reference/access-authn-authz/rbac/ + +_Appears in:_ +- [AuthzConfig](#authzconfig) + +| Field | Description | +| --- | --- | +| `roles` _string array_ | The Kubernetes RBAC roles to be deployed in the same namespace of the FeatureStore. +Roles are managed by the operator and created with an empty list of rules. +See the Feast permission model at https://docs.feast.dev/getting-started/concepts/permission +The feature store admin is not obligated to manage roles using the Feast operator, roles can be managed independently. +This configuration option is only providing a way to automate this procedure. +Important note: the operator cannot ensure that these roles will match the ones used in the configured Feast permissions. | + + +#### LocalRegistryConfig + + + +LocalRegistryConfig configures the deployed registry service + +_Appears in:_ +- [Registry](#registry) + +| Field | Description | +| --- | --- | +| `image` _string_ | | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `persistence` _[RegistryPersistence](#registrypersistence)_ | | +| `tls` _[TlsConfigs](#tlsconfigs)_ | | +| `logLevel` _string_ | LogLevel sets the logging level for the registry service +Allowed values: "debug", "info", "warning", "error", "critical". | + + +#### OfflineStore + + + +OfflineStore configures the deployed offline store service + +_Appears in:_ +- [FeatureStoreServices](#featurestoreservices) + +| Field | Description | +| --- | --- | +| `image` _string_ | | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `persistence` _[OfflineStorePersistence](#offlinestorepersistence)_ | | +| `tls` _[TlsConfigs](#tlsconfigs)_ | | +| `logLevel` _string_ | LogLevel sets the logging level for the offline store service +Allowed values: "debug", "info", "warning", "error", "critical". | + + +#### OfflineStoreDBStorePersistence + + + +OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service + +_Appears in:_ +- [OfflineStorePersistence](#offlinestorepersistence) + +| Field | Description | +| --- | --- | +| `type` _string_ | | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. | +| `secretKeyName` _string_ | By default, the selected store "type" is used as the SecretKeyName | + + +#### OfflineStoreFilePersistence + + + +OfflineStoreFilePersistence configures the file-based persistence for the offline store service + +_Appears in:_ +- [OfflineStorePersistence](#offlinestorepersistence) + +| Field | Description | +| --- | --- | +| `type` _string_ | | +| `pvc` _[PvcConfig](#pvcconfig)_ | | + + +#### OfflineStorePersistence + + + +OfflineStorePersistence configures the persistence settings for the offline store service + +_Appears in:_ +- [OfflineStore](#offlinestore) + +| Field | Description | +| --- | --- | +| `file` _[OfflineStoreFilePersistence](#offlinestorefilepersistence)_ | | +| `store` _[OfflineStoreDBStorePersistence](#offlinestoredbstorepersistence)_ | | + + +#### OidcAuthz + + + +OidcAuthz defines the authorization settings for deployments using an Open ID Connect identity provider. +https://auth0.com/docs/authenticate/protocols/openid-connect-protocol + +_Appears in:_ +- [AuthzConfig](#authzconfig) + +| Field | Description | +| --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | | + + +#### OnlineStore + + + +OnlineStore configures the deployed online store service + +_Appears in:_ +- [FeatureStoreServices](#featurestoreservices) + +| Field | Description | +| --- | --- | +| `image` _string_ | | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `persistence` _[OnlineStorePersistence](#onlinestorepersistence)_ | | +| `tls` _[TlsConfigs](#tlsconfigs)_ | | +| `logLevel` _string_ | LogLevel sets the logging level for the online store service +Allowed values: "debug", "info", "warning", "error", "critical". | + + +#### OnlineStoreDBStorePersistence + + + +OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service + +_Appears in:_ +- [OnlineStorePersistence](#onlinestorepersistence) + +| Field | Description | +| --- | --- | +| `type` _string_ | | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | Data store parameters should be placed as-is from the "feature_store.yaml" under the secret key. "registry_type" & "type" fields should be removed. | +| `secretKeyName` _string_ | By default, the selected store "type" is used as the SecretKeyName | + + +#### OnlineStoreFilePersistence + + + +OnlineStoreFilePersistence configures the file-based persistence for the offline store service + +_Appears in:_ +- [OnlineStorePersistence](#onlinestorepersistence) + +| Field | Description | +| --- | --- | +| `path` _string_ | | +| `pvc` _[PvcConfig](#pvcconfig)_ | | + + +#### OnlineStorePersistence + + + +OnlineStorePersistence configures the persistence settings for the online store service + +_Appears in:_ +- [OnlineStore](#onlinestore) + +| Field | Description | +| --- | --- | +| `file` _[OnlineStoreFilePersistence](#onlinestorefilepersistence)_ | | +| `store` _[OnlineStoreDBStorePersistence](#onlinestoredbstorepersistence)_ | | + + +#### OptionalConfigs + + + +OptionalConfigs k8s container settings that are optional + +_Appears in:_ +- [LocalRegistryConfig](#localregistryconfig) +- [OfflineStore](#offlinestore) +- [OnlineStore](#onlinestore) +- [ServiceConfigs](#serviceconfigs) +- [UIService](#uiservice) + +| Field | Description | +| --- | --- | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | + + +#### PvcConfig + +_Underlying type:_ `[struct{Ref *k8s.io/api/core/v1.LocalObjectReference "json:\"ref,omitempty\""; Create *PvcCreate "json:\"create,omitempty\""; MountPath string "json:\"mountPath\""}](#struct{ref-*k8sioapicorev1localobjectreference-"json:\"ref,omitempty\"";-create-*pvccreate-"json:\"create,omitempty\"";-mountpath-string-"json:\"mountpath\""})` + +PvcConfig defines the settings for a persistent file store based on PVCs. +We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field. + +_Appears in:_ +- [OfflineStoreFilePersistence](#offlinestorefilepersistence) +- [OnlineStoreFilePersistence](#onlinestorefilepersistence) + + + + + +#### Registry + + + +Registry configures the registry service. One selection is required. Local is the default setting. + +_Appears in:_ +- [FeatureStoreServices](#featurestoreservices) + +| Field | Description | +| --- | --- | +| `local` _[LocalRegistryConfig](#localregistryconfig)_ | | +| `remote` _[RemoteRegistryConfig](#remoteregistryconfig)_ | | + + +#### RegistryDBStorePersistence + +_Underlying type:_ `[struct{Type string "json:\"type\""; SecretRef k8s.io/api/core/v1.LocalObjectReference "json:\"secretRef\""; SecretKeyName string "json:\"secretKeyName,omitempty\""}](#struct{type-string-"json:\"type\"";-secretref-k8sioapicorev1localobjectreference-"json:\"secretref\"";-secretkeyname-string-"json:\"secretkeyname,omitempty\""})` + +RegistryDBStorePersistence configures the DB store persistence for the registry service + +_Appears in:_ +- [RegistryPersistence](#registrypersistence) + + + +#### RegistryFilePersistence + +_Underlying type:_ `[struct{Path string "json:\"path,omitempty\""; PvcConfig *PvcConfig "json:\"pvc,omitempty\""; S3AdditionalKwargs *map[string]string "json:\"s3_additional_kwargs,omitempty\""}](#struct{path-string-"json:\"path,omitempty\"";-pvcconfig-*pvcconfig-"json:\"pvc,omitempty\"";-s3additionalkwargs-*map[string]string-"json:\"s3_additional_kwargs,omitempty\""})` + +RegistryFilePersistence configures the file-based persistence for the registry service + +_Appears in:_ +- [RegistryPersistence](#registrypersistence) + + + +#### RegistryPersistence + + + +RegistryPersistence configures the persistence settings for the registry service + +_Appears in:_ +- [LocalRegistryConfig](#localregistryconfig) + +| Field | Description | +| --- | --- | +| `file` _[RegistryFilePersistence](#registryfilepersistence)_ | | +| `store` _[RegistryDBStorePersistence](#registrydbstorepersistence)_ | | + + +#### RemoteRegistryConfig + + + +RemoteRegistryConfig points to a remote feast registry server. When set, the operator will not deploy a registry for this FeatureStore CR. +Instead, this FeatureStore CR's online/offline services will use a remote registry. One selection is required. + +_Appears in:_ +- [Registry](#registry) + +| Field | Description | +| --- | --- | +| `hostname` _string_ | Host address of the remote registry service - :, e.g. `registry..svc.cluster.local:80` | +| `feastRef` _[FeatureStoreRef](#featurestoreref)_ | Reference to an existing `FeatureStore` CR in the same k8s cluster. | +| `tls` _[TlsRemoteRegistryConfigs](#tlsremoteregistryconfigs)_ | | + + +#### SecretKeyNames + + + +SecretKeyNames defines the secret key names for the TLS key and cert. + +_Appears in:_ +- [TlsConfigs](#tlsconfigs) + +| Field | Description | +| --- | --- | +| `tlsCrt` _string_ | defaults to "tls.crt" | +| `tlsKey` _string_ | defaults to "tls.key" | + + +#### ServiceConfigs + + + +ServiceConfigs k8s container settings + +_Appears in:_ +- [LocalRegistryConfig](#localregistryconfig) +- [OfflineStore](#offlinestore) +- [OnlineStore](#onlinestore) +- [UIService](#uiservice) + +| Field | Description | +| --- | --- | +| `image` _string_ | | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | + + +#### ServiceHostnames + + + +ServiceHostnames defines the service hostnames in the format of :, e.g. example.svc.cluster.local:80 + +_Appears in:_ +- [FeatureStoreStatus](#featurestorestatus) + +| Field | Description | +| --- | --- | +| `offlineStore` _string_ | | +| `onlineStore` _string_ | | +| `registry` _string_ | | +| `ui` _string_ | | + + +#### TlsConfigs + + + +TlsConfigs configures server TLS for a feast service. in an openshift cluster, this is configured by default using service serving certificates. + +_Appears in:_ +- [LocalRegistryConfig](#localregistryconfig) +- [OfflineStore](#offlinestore) +- [OnlineStore](#onlinestore) +- [UIService](#uiservice) + +| Field | Description | +| --- | --- | +| `secretRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | references the local k8s secret where the TLS key and cert reside | +| `secretKeyNames` _[SecretKeyNames](#secretkeynames)_ | | +| `disable` _boolean_ | will disable TLS for the feast service. useful in an openshift cluster, for example, where TLS is configured by default | + + +#### TlsRemoteRegistryConfigs + + + +TlsRemoteRegistryConfigs configures client TLS for a remote feast registry. in an openshift cluster, this is configured by default when the remote feast registry is using service serving certificates. + +_Appears in:_ +- [RemoteRegistryConfig](#remoteregistryconfig) + +| Field | Description | +| --- | --- | +| `configMapRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#localobjectreference-v1-core)_ | references the local k8s configmap where the TLS cert resides | +| `certName` _string_ | defines the configmap key name for the client TLS cert. | + + +#### UIService + + + +UIService configures the deployed Feast UI service + +_Appears in:_ +- [FeatureStoreServices](#featurestoreservices) + +| Field | Description | +| --- | --- | +| `image` _string_ | | +| `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envvar-v1-core)_ | | +| `envFrom` _[EnvFromSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#envfromsource-v1-core)_ | | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#pullpolicy-v1-core)_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#resourcerequirements-v1-core)_ | | +| `tls` _[TlsConfigs](#tlsconfigs)_ | | +| `logLevel` _string_ | LogLevel sets the logging level for the UI service +Allowed values: "debug", "info", "warning", "error", "critical". | + + diff --git a/infra/feast-operator/docs/crd-ref-templates/config.yaml b/infra/feast-operator/docs/crd-ref-templates/config.yaml new file mode 100644 index 00000000000..42d10e08b03 --- /dev/null +++ b/infra/feast-operator/docs/crd-ref-templates/config.yaml @@ -0,0 +1,8 @@ +processor: + ignoreTypes: + - "(FeatureStore)List$" + ignoreFields: + - "TypeMeta$" + +render: + kubernetesVersion: "1.30" \ No newline at end of file diff --git a/infra/feast-operator/docs/crd-ref-templates/markdown/gv_details.tpl b/infra/feast-operator/docs/crd-ref-templates/markdown/gv_details.tpl new file mode 100644 index 00000000000..30ad0d75184 --- /dev/null +++ b/infra/feast-operator/docs/crd-ref-templates/markdown/gv_details.tpl @@ -0,0 +1,19 @@ +{{- define "gvDetails" -}} +{{- $gv := . -}} + +## {{ $gv.GroupVersionString }} + +{{ $gv.Doc }} + +{{- if $gv.Kinds }} +### Resource Types +{{- range $gv.SortedKinds }} +- {{ $gv.TypeForKind . | markdownRenderTypeLink }} +{{- end }} +{{ end }} + +{{ range $gv.SortedTypes }} +{{ template "type" . }} +{{ end }} + +{{- end -}} diff --git a/infra/feast-operator/docs/crd-ref-templates/markdown/gv_list.tpl b/infra/feast-operator/docs/crd-ref-templates/markdown/gv_list.tpl new file mode 100644 index 00000000000..a4d3dadf18c --- /dev/null +++ b/infra/feast-operator/docs/crd-ref-templates/markdown/gv_list.tpl @@ -0,0 +1,15 @@ +{{- define "gvList" -}} +{{- $groupVersions := . -}} + +# API Reference + +## Packages +{{- range $groupVersions }} +- {{ markdownRenderGVLink . }} +{{- end }} + +{{ range $groupVersions }} +{{ template "gvDetails" . }} +{{ end }} + +{{- end -}} diff --git a/infra/feast-operator/docs/crd-ref-templates/markdown/type.tpl b/infra/feast-operator/docs/crd-ref-templates/markdown/type.tpl new file mode 100644 index 00000000000..c0ac2e03539 --- /dev/null +++ b/infra/feast-operator/docs/crd-ref-templates/markdown/type.tpl @@ -0,0 +1,33 @@ +{{- define "type" -}} +{{- $type := . -}} +{{- if markdownShouldRenderType $type -}} + +#### {{ $type.Name }} + +{{ if $type.IsAlias }}_Underlying type:_ `{{ markdownRenderTypeLink $type.UnderlyingType }}`{{ end }} + +{{ $type.Doc }} + +{{ if $type.References -}} +_Appears in:_ +{{- range $type.SortedReferences }} +- {{ markdownRenderTypeLink . }} +{{- end }} +{{- end }} + +{{ if $type.Members -}} +| Field | Description | +| --- | --- | +{{ if $type.GVK -}} +| `apiVersion` _string_ | `{{ $type.GVK.Group }}/{{ $type.GVK.Version }}` +| `kind` _string_ | `{{ $type.GVK.Kind }}` +{{ end -}} + +{{ range $type.Members -}} +| `{{ .Name }}` _{{ markdownRenderType .Type }}_ | {{ template "type_members" . }} | +{{ end -}} + +{{ end -}} + +{{- end -}} +{{- end -}} diff --git a/infra/feast-operator/docs/crd-ref-templates/markdown/type_members.tpl b/infra/feast-operator/docs/crd-ref-templates/markdown/type_members.tpl new file mode 100644 index 00000000000..182fa182166 --- /dev/null +++ b/infra/feast-operator/docs/crd-ref-templates/markdown/type_members.tpl @@ -0,0 +1,8 @@ +{{- define "type_members" -}} +{{- $field := . -}} +{{- if eq $field.Name "metadata" -}} +Refer to Kubernetes API documentation for fields of `metadata`. +{{- else -}} +{{ $field.Doc }} +{{- end -}} +{{- end -}} From c81448c887b2e6b48d01efa2ebc0b42f50c1c81d Mon Sep 17 00:00:00 2001 From: Tommy Hughes Date: Fri, 24 Jan 2025 12:09:26 -0600 Subject: [PATCH 2/2] add namespace to operator example objects Signed-off-by: Tommy Hughes --- examples/operator-quickstart/feast.yaml | 2 ++ examples/operator-quickstart/postgres.yaml | 3 +++ examples/operator-quickstart/redis.yaml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/examples/operator-quickstart/feast.yaml b/examples/operator-quickstart/feast.yaml index e4eb20b4c0c..def24bd9a4c 100644 --- a/examples/operator-quickstart/feast.yaml +++ b/examples/operator-quickstart/feast.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: feast-data-stores + namespace: feast stringData: redis: | connection_string: redis.feast.svc.cluster.local:6379 @@ -16,6 +17,7 @@ apiVersion: feast.dev/v1alpha1 kind: FeatureStore metadata: name: example + namespace: feast spec: feastProject: credit_scoring_local services: diff --git a/examples/operator-quickstart/postgres.yaml b/examples/operator-quickstart/postgres.yaml index 4baa0092575..e37caa16da4 100644 --- a/examples/operator-quickstart/postgres.yaml +++ b/examples/operator-quickstart/postgres.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Secret metadata: name: postgres-secret + namespace: feast stringData: POSTGRES_DB: feast POSTGRES_USER: feast @@ -11,6 +12,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: postgres + namespace: feast spec: replicas: 1 selector: @@ -40,6 +42,7 @@ apiVersion: v1 kind: Service metadata: name: postgres + namespace: feast labels: app: postgres spec: diff --git a/examples/operator-quickstart/redis.yaml b/examples/operator-quickstart/redis.yaml index f39dfc9768d..5d70b6bd5d6 100644 --- a/examples/operator-quickstart/redis.yaml +++ b/examples/operator-quickstart/redis.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: redis + namespace: feast spec: replicas: 1 selector: @@ -25,6 +26,7 @@ apiVersion: v1 kind: Service metadata: name: redis + namespace: feast labels: app: redis spec: