From d64c8e5dc14a8db00ae57df0c6677040944985ef Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 9 Oct 2025 12:51:16 +0200 Subject: [PATCH 1/3] feat: replace KeyDB with Valkey in infrastructure and update related configurations --- README.md | 2 +- infrastructure/README.md | 20 ++++++------- infrastructure/rag/Chart.lock | 10 +++---- infrastructure/rag/Chart.yaml | 10 +++---- .../_admin_backend_and_extractor_helpers.tpl | 4 +++ .../templates/admin-backend/deployment.yaml | 2 ++ .../rag/templates/admin-backend/secrets.yaml | 8 +++++ infrastructure/rag/values.yaml | 29 +++++++++++++------ libs/README.md | 2 +- .../src/admin_api_lib/dependency_container.py | 2 +- .../api_endpoints/default_document_deleter.py | 2 +- .../default_documents_status_retriever.py | 2 +- .../api_endpoints/default_file_uploader.py | 2 +- .../api_endpoints/default_source_uploader.py | 2 +- .../{key_db => key_value_store}/__init__.py | 0 .../file_status_key_value_store.py | 2 +- .../impl/settings/key_value_settings.py | 3 ++ 17 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 infrastructure/rag/templates/admin-backend/secrets.yaml rename libs/admin-api-lib/src/admin_api_lib/impl/{key_db => key_value_store}/__init__.py (100%) rename libs/admin-api-lib/src/admin_api_lib/impl/{key_db => key_value_store}/file_status_key_value_store.py (97%) diff --git a/README.md b/README.md index 3081b63f..55bc8295 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ This can be done with the following command from the root of the git-repository: cd infrastructure/rag;helm dependency update; cd ../.. ``` -> 📝 NOTE: The configuration of the `Tiltfile` requires `features.frontend.enabled=true`, `features.keydb.enabled=true`, `features.langfuse.enabled=true` and `features.qdrant.enabled=true`. +> 📝 NOTE: The configuration of the `Tiltfile` requires `features.frontend.enabled=true`, `features.valkey.enabled=true`, `features.langfuse.enabled=true` and `features.qdrant.enabled=true`. After the initial build of the helm chart *Tilt* is able to update the files. diff --git a/infrastructure/README.md b/infrastructure/README.md index 887e7ea5..7c2b783f 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -8,7 +8,7 @@ The documentation is structured as follows: - [1. Components and Configuration Values to Adjust](#1-components-and-configuration-values-to-adjust) - [1.1 Langfuse](#11-langfuse) - [1.2 Qdrant](#12-qdrant) - - [1.3 KeyDB](#13-keydb) + - [1.3 Valkey](#13-valkey) - [1.4 Frontend](#14-frontend) - [1.5 Backend](#15-backend) - [1.6 MCP Server](#16-mcp-server) @@ -79,7 +79,7 @@ This directory contains the Helm chart for the following RAG components: - [Langfuse](https://langfuse.com/) (dependency) - [Qdrant](https://qdrant.tech/) (dependency) -- [KeyDB](https://docs.keydb.dev/) (dependency) +- [Valkey](https://valkey.io/) (dependency) - Frontend - Backend - MCP Server @@ -101,7 +101,7 @@ features: enabled: true frontend: enabled: true - keydb: + valkey: enabled: true mcp: enabled: true @@ -201,14 +201,14 @@ qdrant: replicaCount: 3 ``` -### 1.3 KeyDB +### 1.3 Valkey -The usage of the KeyDB is **only recommended for development** purposes. KeyDB is used as alternative to Redis to store the state of each uploaded document. The Admin Backend uses the key-value-pairs of the KeyDB to keep track of the current state of the RAG sources. Note, sources include documents as well as non-document sources like confluence. +KeyDB has been replaced by Valkey due to incompatibilities with Langfuse. Valkey is a Redis-compatible key-value store and is used as the in-cluster default for development to store the state of uploaded documents and non-document sources (e.g., Confluence). The Admin Backend uses these key-value pairs to track the current state of RAG sources. Langfuse also utilizes Valkey. In **production**, the usage of a fully-managed Redis instance (e.g. provided by STACKIT) is recommended. The following parameters need to be adjusted in the `values.yaml` file: ```yaml -# For production: Use external Redis instead of KeyDB +# For production: Use managed Redis (or external Valkey) instead of the in-cluster Valkey adminBackend: envs: keyValueStore: @@ -216,12 +216,12 @@ adminBackend: USECASE_KEYVALUE_PORT: 6379 features: - keydb: - enabled: false # Disable KeyDB for production + valkey: + enabled: false # Disable in-cluster Valkey for production langfuse: valkey: - deploy: false # Use Redis instead of KeyDB + deploy: false # Use managed Redis instead of in-cluster Valkey langfuse: additionalEnv: - name: REDIS_CONNECTION_STRING @@ -499,7 +499,7 @@ For deployment of the *NGINX Ingress Controller* and a cert-manager, the followi [base-setup](server-setup/base-setup/Chart.yaml) -The email [here](server-setup/base-setup/templates/cert-issuer.yaml) should be changed from `` to a real email address. +Update the email in the cert issuer template: [server-setup/base-setup/templates/cert-issuer.yaml](server-setup/base-setup/templates/cert-issuer.yaml) — replace `` with a real email address. ## 3. Contributing diff --git a/infrastructure/rag/Chart.lock b/infrastructure/rag/Chart.lock index aac22911..d9793238 100644 --- a/infrastructure/rag/Chart.lock +++ b/infrastructure/rag/Chart.lock @@ -8,11 +8,11 @@ dependencies: - name: minio repository: https://charts.bitnami.com/bitnami version: 15.0.7 -- name: keydb - repository: https://enapter.github.io/charts/ - version: 0.48.0 +- name: valkey + repository: https://valkey.io/valkey-helm/ + version: 0.7.4 - name: ollama repository: https://otwld.github.io/ollama-helm/ version: 1.27.0 -digest: sha256:86652599791c7f3ae4e66e6ed4404630d733cf35503c4b94b90da9c3c950c7fb -generated: "2025-10-07T10:23:59.709679+02:00" +digest: sha256:81d1c5160c0ac99701f31f2b72a57aca1a87ae0cce0e9168506df22ffc8a4f25 +generated: "2025-10-09T12:19:11.826488+02:00" diff --git a/infrastructure/rag/Chart.yaml b/infrastructure/rag/Chart.yaml index b0b0f3bf..d154e9a1 100644 --- a/infrastructure/rag/Chart.yaml +++ b/infrastructure/rag/Chart.yaml @@ -20,11 +20,11 @@ dependencies: repository: https://charts.bitnami.com/bitnami version: "15.0.7" condition: features.minio.enabled -- name: keydb - alias: keydb - repository: https://enapter.github.io/charts/ - version: "0.48.0" - condition: features.keydb.enabled +- name: valkey + repository: https://valkey.io/valkey-helm/ + version: "0.7.4" + appVersion: "8.1.3" + condition: features.valkey.enabled - name: ollama alias: ollama version: 1.27.0 diff --git a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl index be4c54b1..e1dfb8cf 100644 --- a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl +++ b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl @@ -15,6 +15,10 @@ {{- printf "%s-stackit-vllm-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- define "secret.keyvalName" -}} +{{- printf "%s-keyval-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + # configmaps {{- define "configmap.s3Name" -}} {{- printf "%s-s3-configmap" .Release.Name | trunc 63 | trimSuffix "-" -}} diff --git a/infrastructure/rag/templates/admin-backend/deployment.yaml b/infrastructure/rag/templates/admin-backend/deployment.yaml index 53797b8c..d53a55ea 100644 --- a/infrastructure/rag/templates/admin-backend/deployment.yaml +++ b/infrastructure/rag/templates/admin-backend/deployment.yaml @@ -116,6 +116,8 @@ spec: name: {{ template "secret.s3Name" . }} - secretRef: name: {{ template "secret.stackitVllmName" . }} + - secretRef: + name: {{ template "secret.keyvalName" . }} env: - name: PYTHONPATH value: {{ .Values.adminBackend.pythonPathEnv.PYTHONPATH }} diff --git a/infrastructure/rag/templates/admin-backend/secrets.yaml b/infrastructure/rag/templates/admin-backend/secrets.yaml new file mode 100644 index 00000000..e2631519 --- /dev/null +++ b/infrastructure/rag/templates/admin-backend/secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "secret.keyvalName" . }} +type: Opaque +data: + USECASE_KEYVALUE_USERNAME: {{ .Values.adminBackend.secrets.keyvalue.username | b64enc }} + USECASE_KEYVALUE_PASSWORD: {{ .Values.adminBackend.secrets.keyvalue.password | b64enc }} diff --git a/infrastructure/rag/values.yaml b/infrastructure/rag/values.yaml index a1bca824..eb1c0d06 100644 --- a/infrastructure/rag/values.yaml +++ b/infrastructure/rag/values.yaml @@ -14,7 +14,7 @@ features: enabled: true frontend: enabled: true - keydb: + valkey: enabled: true mcp: enabled: true @@ -323,6 +323,11 @@ adminBackend: minio: enabled: true + secrets: + keyvalue: + username: "default" + password: "changeme" + envs: summarizer: SUMMARIZER_MAXIMUM_INPUT_SIZE: "8000" @@ -342,7 +347,7 @@ adminBackend: CHUNKER_OVERLAP: 300 keyValueStore: USECASE_KEYVALUE_PORT: 6379 - USECASE_KEYVALUE_HOST: "rag-keydb" + USECASE_KEYVALUE_HOST: "rag-valkey" sourceUploader: SOURCE_UPLOADER_TIMEOUT: 3600 @@ -522,6 +527,11 @@ langfuse: value: "" - name: LANGFUSE_INIT_USER_PASSWORD value: "" + # if authentication is enabled in redis + # - name: REDIS_CONNECTION_STRING + # value: "redis://default:changeme@rag-valkey:6379" + # - name: LANGFUSE_REDIS_URL + # value: "redis://default:changeme@rag-valkey:6379" # Additional init containers extraInitContainers: @@ -548,13 +558,13 @@ langfuse: password: postgres database: langfuse - # Redis Configuration (external KeyDB) + # Redis Configuration (external ValKey or managed Redis (recommended for production)) redis: deploy: false - host: "rag-keydb" + host: "rag-valkey" port: 6379 auth: - username: "default" + username: "" password: "" # ClickHouse Configuration (external ClickHouse) @@ -644,7 +654,8 @@ qdrant: image: tag: v1.15.4 -keydb: - multiMaster: "no" - activeReplicas: "no" - nodes: 1 + +valkey: + #acl by default disabled. can be enabled. + dataStorage: + enabled: true diff --git a/libs/README.md b/libs/README.md index 960e7cfb..a41cbbd0 100644 --- a/libs/README.md +++ b/libs/README.md @@ -183,7 +183,7 @@ The extracted information will be summarized using LLM. The summary, as well as |----------|---------|--------------|--------------| | file_service | [`admin_api_lib.file_services.file_service.FileService`](./admin-api-lib/src/admin_api_lib/file_services/file_service.py) | [`admin_api_lib.impl.file_services.s3_service.S3Service`](./admin-api-lib/src/admin_api_lib/impl/file_services/s3_service.py) | Handles operations on the connected storage. | | large_language_model | `langchain_core.language_models.llms.BaseLLM` | `langchain_community.llms.vllm.VLLMOpenAI` or `langchain_community.llms.Ollama` | The LLm that is used for all LLM tasks. The default depends on the value of `rag_core_lib.impl.settings.rag_class_types_settings.RAGClassTypeSettings.llm_type` | -| key_value_store | [`admin_api_lib.impl.key_db.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py) | [`admin_api_lib.impl.key_db.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py) | Is used for storing the available sources and their current state. | +| key_value_store | [`admin_api_lib.impl.key_value_store.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py) | [`admin_api_lib.impl.key_value_store.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py) | Is used for storing the available sources and their current state. | | chunker | [`admin_api_lib.chunker.chunker.Chunker`](./admin-api-lib/src/admin_api_lib/chunker/chunker.py) | [`admin_api_lib.impl.chunker.text_chunker.TextChunker`](./admin-api-lib/src/admin_api_lib/impl/chunker/text_chunker.py) | Used for splitting the documents in managable chunks. | | document_extractor | [`admin_api_lib.extractor_api_client.openapi_client.api.extractor_api.ExtractorApi`](./admin-api-lib/src/admin_api_lib/extractor_api_client/openapi_client/api/extractor_api.py) | [`admin_api_lib.extractor_api_client.openapi_client.api.extractor_api.ExtractorApi`](./admin-api-lib/src/admin_api_lib/extractor_api_client/openapi_client/api/extractor_api.py) | Needs to be replaced if adjustments to the `extractor-api` is made. | | rag_api | [`admin_api_lib.rag_backend_client.openapi_client.api.rag_api.RagApi`](./admin-api-lib/src/admin_api_lib/rag_backend_client/openapi_client/api/rag_api.py) | [`admin_api_lib.rag_backend_client.openapi_client.api.rag_api.RagApi`](./admin-api-lib/src/admin_api_lib/rag_backend_client/openapi_client/api/rag_api.py) | Needs to be replaced if changes to the `/information_pieces/remove` or `/information_pieces/upload` of the [`rag-core-api`](#1-rag-core-api) are made. | diff --git a/libs/admin-api-lib/src/admin_api_lib/dependency_container.py b/libs/admin-api-lib/src/admin_api_lib/dependency_container.py index ee86cb5f..deb792cb 100644 --- a/libs/admin-api-lib/src/admin_api_lib/dependency_container.py +++ b/libs/admin-api-lib/src/admin_api_lib/dependency_container.py @@ -35,7 +35,7 @@ from admin_api_lib.impl.information_enhancer.page_summary_enhancer import ( PageSummaryEnhancer, ) -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.impl.mapper.informationpiece2document import ( diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py index 3cf671ff..7c3258dd 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py @@ -7,7 +7,7 @@ from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter from admin_api_lib.file_services.file_service import FileService -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.rag_backend_client.openapi_client.api.rag_api import RagApi diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py index c2e21a04..4e5c5560 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py @@ -5,7 +5,7 @@ from admin_api_lib.api_endpoints.documents_status_retriever import ( DocumentsStatusRetriever, ) -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.models.document_status import DocumentStatus diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py index 668cacd5..e17c242e 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py @@ -18,7 +18,7 @@ from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter from admin_api_lib.chunker.chunker import Chunker from admin_api_lib.models.status import Status -from admin_api_lib.impl.key_db.file_status_key_value_store import FileStatusKeyValueStore +from admin_api_lib.impl.key_value_store.file_status_key_value_store import FileStatusKeyValueStore from admin_api_lib.information_enhancer.information_enhancer import InformationEnhancer from admin_api_lib.utils.utils import sanitize_document_name diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py index f0fe7932..ffe5faf6 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py @@ -17,7 +17,7 @@ from admin_api_lib.api_endpoints.source_uploader import SourceUploader from admin_api_lib.chunker.chunker import Chunker from admin_api_lib.models.status import Status -from admin_api_lib.impl.key_db.file_status_key_value_store import FileStatusKeyValueStore +from admin_api_lib.impl.key_value_store.file_status_key_value_store import FileStatusKeyValueStore from admin_api_lib.information_enhancer.information_enhancer import InformationEnhancer from admin_api_lib.utils.utils import sanitize_document_name from admin_api_lib.rag_backend_client.openapi_client.models.information_piece import ( diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/__init__.py b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/__init__.py similarity index 100% rename from libs/admin-api-lib/src/admin_api_lib/impl/key_db/__init__.py rename to libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/__init__.py diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py similarity index 97% rename from libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py rename to libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py index ef27b65a..ab6e47db 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py @@ -39,7 +39,7 @@ def __init__(self, settings: KeyValueSettings): settings : KeyValueSettings The settings object containing the host and port information for the Redis connection. """ - self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True) + self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True, username=settings.username, password=settings.password) @staticmethod def _to_str(file_name: str, file_status: Status) -> str: diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py index 5acc6371..e4bae16b 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py @@ -1,5 +1,6 @@ """Contains settings regarding the key values store.""" +from typing import Optional from pydantic import Field from pydantic_settings import BaseSettings @@ -24,3 +25,5 @@ class Config: host: str = Field() port: int = Field() + username: Optional[str] = Field(default=None) + password: Optional[str] = Field(default=None) From 479fcc32d01576b5c87c17a730baa2eececfc47f Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 9 Oct 2025 12:51:26 +0200 Subject: [PATCH 2/3] fix: remove commented-out LANGFUSE_REDIS_URL from values.yaml --- infrastructure/rag/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/infrastructure/rag/values.yaml b/infrastructure/rag/values.yaml index eb1c0d06..6b4b3018 100644 --- a/infrastructure/rag/values.yaml +++ b/infrastructure/rag/values.yaml @@ -530,8 +530,6 @@ langfuse: # if authentication is enabled in redis # - name: REDIS_CONNECTION_STRING # value: "redis://default:changeme@rag-valkey:6379" - # - name: LANGFUSE_REDIS_URL - # value: "redis://default:changeme@rag-valkey:6379" # Additional init containers extraInitContainers: From cccea515527d09a27a7bfd55a8080c7fe9ef040c Mon Sep 17 00:00:00 2001 From: Andreas Klos Date: Thu, 9 Oct 2025 13:13:15 +0200 Subject: [PATCH 3/3] refactor: replace KeyDB with Valkey in documentation and codebase --- README.md | 2 +- infrastructure/README.md | 20 +++++++------- infrastructure/rag/Chart.lock | 10 +++---- infrastructure/rag/Chart.yaml | 10 +++---- .../_admin_backend_and_extractor_helpers.tpl | 4 +++ .../templates/admin-backend/deployment.yaml | 2 ++ .../rag/templates/admin-backend/secrets.yaml | 8 ++++++ infrastructure/rag/values.yaml | 27 ++++++++++++------- libs/README.md | 2 +- .../src/admin_api_lib/dependency_container.py | 2 +- .../api_endpoints/default_document_deleter.py | 2 +- .../default_documents_status_retriever.py | 2 +- .../api_endpoints/default_file_uploader.py | 2 +- .../api_endpoints/default_source_uploader.py | 2 +- .../{key_db => key_value_store}/__init__.py | 0 .../file_status_key_value_store.py | 2 +- .../impl/settings/key_value_settings.py | 3 +++ 17 files changed, 63 insertions(+), 37 deletions(-) create mode 100644 infrastructure/rag/templates/admin-backend/secrets.yaml rename libs/admin-api-lib/src/admin_api_lib/impl/{key_db => key_value_store}/__init__.py (100%) rename libs/admin-api-lib/src/admin_api_lib/impl/{key_db => key_value_store}/file_status_key_value_store.py (97%) diff --git a/README.md b/README.md index 3081b63f..55bc8295 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ This can be done with the following command from the root of the git-repository: cd infrastructure/rag;helm dependency update; cd ../.. ``` -> 📝 NOTE: The configuration of the `Tiltfile` requires `features.frontend.enabled=true`, `features.keydb.enabled=true`, `features.langfuse.enabled=true` and `features.qdrant.enabled=true`. +> 📝 NOTE: The configuration of the `Tiltfile` requires `features.frontend.enabled=true`, `features.valkey.enabled=true`, `features.langfuse.enabled=true` and `features.qdrant.enabled=true`. After the initial build of the helm chart *Tilt* is able to update the files. diff --git a/infrastructure/README.md b/infrastructure/README.md index 887e7ea5..7c2b783f 100644 --- a/infrastructure/README.md +++ b/infrastructure/README.md @@ -8,7 +8,7 @@ The documentation is structured as follows: - [1. Components and Configuration Values to Adjust](#1-components-and-configuration-values-to-adjust) - [1.1 Langfuse](#11-langfuse) - [1.2 Qdrant](#12-qdrant) - - [1.3 KeyDB](#13-keydb) + - [1.3 Valkey](#13-valkey) - [1.4 Frontend](#14-frontend) - [1.5 Backend](#15-backend) - [1.6 MCP Server](#16-mcp-server) @@ -79,7 +79,7 @@ This directory contains the Helm chart for the following RAG components: - [Langfuse](https://langfuse.com/) (dependency) - [Qdrant](https://qdrant.tech/) (dependency) -- [KeyDB](https://docs.keydb.dev/) (dependency) +- [Valkey](https://valkey.io/) (dependency) - Frontend - Backend - MCP Server @@ -101,7 +101,7 @@ features: enabled: true frontend: enabled: true - keydb: + valkey: enabled: true mcp: enabled: true @@ -201,14 +201,14 @@ qdrant: replicaCount: 3 ``` -### 1.3 KeyDB +### 1.3 Valkey -The usage of the KeyDB is **only recommended for development** purposes. KeyDB is used as alternative to Redis to store the state of each uploaded document. The Admin Backend uses the key-value-pairs of the KeyDB to keep track of the current state of the RAG sources. Note, sources include documents as well as non-document sources like confluence. +KeyDB has been replaced by Valkey due to incompatibilities with Langfuse. Valkey is a Redis-compatible key-value store and is used as the in-cluster default for development to store the state of uploaded documents and non-document sources (e.g., Confluence). The Admin Backend uses these key-value pairs to track the current state of RAG sources. Langfuse also utilizes Valkey. In **production**, the usage of a fully-managed Redis instance (e.g. provided by STACKIT) is recommended. The following parameters need to be adjusted in the `values.yaml` file: ```yaml -# For production: Use external Redis instead of KeyDB +# For production: Use managed Redis (or external Valkey) instead of the in-cluster Valkey adminBackend: envs: keyValueStore: @@ -216,12 +216,12 @@ adminBackend: USECASE_KEYVALUE_PORT: 6379 features: - keydb: - enabled: false # Disable KeyDB for production + valkey: + enabled: false # Disable in-cluster Valkey for production langfuse: valkey: - deploy: false # Use Redis instead of KeyDB + deploy: false # Use managed Redis instead of in-cluster Valkey langfuse: additionalEnv: - name: REDIS_CONNECTION_STRING @@ -499,7 +499,7 @@ For deployment of the *NGINX Ingress Controller* and a cert-manager, the followi [base-setup](server-setup/base-setup/Chart.yaml) -The email [here](server-setup/base-setup/templates/cert-issuer.yaml) should be changed from `` to a real email address. +Update the email in the cert issuer template: [server-setup/base-setup/templates/cert-issuer.yaml](server-setup/base-setup/templates/cert-issuer.yaml) — replace `` with a real email address. ## 3. Contributing diff --git a/infrastructure/rag/Chart.lock b/infrastructure/rag/Chart.lock index aac22911..d9793238 100644 --- a/infrastructure/rag/Chart.lock +++ b/infrastructure/rag/Chart.lock @@ -8,11 +8,11 @@ dependencies: - name: minio repository: https://charts.bitnami.com/bitnami version: 15.0.7 -- name: keydb - repository: https://enapter.github.io/charts/ - version: 0.48.0 +- name: valkey + repository: https://valkey.io/valkey-helm/ + version: 0.7.4 - name: ollama repository: https://otwld.github.io/ollama-helm/ version: 1.27.0 -digest: sha256:86652599791c7f3ae4e66e6ed4404630d733cf35503c4b94b90da9c3c950c7fb -generated: "2025-10-07T10:23:59.709679+02:00" +digest: sha256:81d1c5160c0ac99701f31f2b72a57aca1a87ae0cce0e9168506df22ffc8a4f25 +generated: "2025-10-09T12:19:11.826488+02:00" diff --git a/infrastructure/rag/Chart.yaml b/infrastructure/rag/Chart.yaml index b0b0f3bf..d154e9a1 100644 --- a/infrastructure/rag/Chart.yaml +++ b/infrastructure/rag/Chart.yaml @@ -20,11 +20,11 @@ dependencies: repository: https://charts.bitnami.com/bitnami version: "15.0.7" condition: features.minio.enabled -- name: keydb - alias: keydb - repository: https://enapter.github.io/charts/ - version: "0.48.0" - condition: features.keydb.enabled +- name: valkey + repository: https://valkey.io/valkey-helm/ + version: "0.7.4" + appVersion: "8.1.3" + condition: features.valkey.enabled - name: ollama alias: ollama version: 1.27.0 diff --git a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl index be4c54b1..e1dfb8cf 100644 --- a/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl +++ b/infrastructure/rag/templates/_admin_backend_and_extractor_helpers.tpl @@ -15,6 +15,10 @@ {{- printf "%s-stackit-vllm-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{- define "secret.keyvalName" -}} +{{- printf "%s-keyval-secret" .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + # configmaps {{- define "configmap.s3Name" -}} {{- printf "%s-s3-configmap" .Release.Name | trunc 63 | trimSuffix "-" -}} diff --git a/infrastructure/rag/templates/admin-backend/deployment.yaml b/infrastructure/rag/templates/admin-backend/deployment.yaml index 53797b8c..d53a55ea 100644 --- a/infrastructure/rag/templates/admin-backend/deployment.yaml +++ b/infrastructure/rag/templates/admin-backend/deployment.yaml @@ -116,6 +116,8 @@ spec: name: {{ template "secret.s3Name" . }} - secretRef: name: {{ template "secret.stackitVllmName" . }} + - secretRef: + name: {{ template "secret.keyvalName" . }} env: - name: PYTHONPATH value: {{ .Values.adminBackend.pythonPathEnv.PYTHONPATH }} diff --git a/infrastructure/rag/templates/admin-backend/secrets.yaml b/infrastructure/rag/templates/admin-backend/secrets.yaml new file mode 100644 index 00000000..e2631519 --- /dev/null +++ b/infrastructure/rag/templates/admin-backend/secrets.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "secret.keyvalName" . }} +type: Opaque +data: + USECASE_KEYVALUE_USERNAME: {{ .Values.adminBackend.secrets.keyvalue.username | b64enc }} + USECASE_KEYVALUE_PASSWORD: {{ .Values.adminBackend.secrets.keyvalue.password | b64enc }} diff --git a/infrastructure/rag/values.yaml b/infrastructure/rag/values.yaml index a1bca824..6b4b3018 100644 --- a/infrastructure/rag/values.yaml +++ b/infrastructure/rag/values.yaml @@ -14,7 +14,7 @@ features: enabled: true frontend: enabled: true - keydb: + valkey: enabled: true mcp: enabled: true @@ -323,6 +323,11 @@ adminBackend: minio: enabled: true + secrets: + keyvalue: + username: "default" + password: "changeme" + envs: summarizer: SUMMARIZER_MAXIMUM_INPUT_SIZE: "8000" @@ -342,7 +347,7 @@ adminBackend: CHUNKER_OVERLAP: 300 keyValueStore: USECASE_KEYVALUE_PORT: 6379 - USECASE_KEYVALUE_HOST: "rag-keydb" + USECASE_KEYVALUE_HOST: "rag-valkey" sourceUploader: SOURCE_UPLOADER_TIMEOUT: 3600 @@ -522,6 +527,9 @@ langfuse: value: "" - name: LANGFUSE_INIT_USER_PASSWORD value: "" + # if authentication is enabled in redis + # - name: REDIS_CONNECTION_STRING + # value: "redis://default:changeme@rag-valkey:6379" # Additional init containers extraInitContainers: @@ -548,13 +556,13 @@ langfuse: password: postgres database: langfuse - # Redis Configuration (external KeyDB) + # Redis Configuration (external ValKey or managed Redis (recommended for production)) redis: deploy: false - host: "rag-keydb" + host: "rag-valkey" port: 6379 auth: - username: "default" + username: "" password: "" # ClickHouse Configuration (external ClickHouse) @@ -644,7 +652,8 @@ qdrant: image: tag: v1.15.4 -keydb: - multiMaster: "no" - activeReplicas: "no" - nodes: 1 + +valkey: + #acl by default disabled. can be enabled. + dataStorage: + enabled: true diff --git a/libs/README.md b/libs/README.md index 960e7cfb..a41cbbd0 100644 --- a/libs/README.md +++ b/libs/README.md @@ -183,7 +183,7 @@ The extracted information will be summarized using LLM. The summary, as well as |----------|---------|--------------|--------------| | file_service | [`admin_api_lib.file_services.file_service.FileService`](./admin-api-lib/src/admin_api_lib/file_services/file_service.py) | [`admin_api_lib.impl.file_services.s3_service.S3Service`](./admin-api-lib/src/admin_api_lib/impl/file_services/s3_service.py) | Handles operations on the connected storage. | | large_language_model | `langchain_core.language_models.llms.BaseLLM` | `langchain_community.llms.vllm.VLLMOpenAI` or `langchain_community.llms.Ollama` | The LLm that is used for all LLM tasks. The default depends on the value of `rag_core_lib.impl.settings.rag_class_types_settings.RAGClassTypeSettings.llm_type` | -| key_value_store | [`admin_api_lib.impl.key_db.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py) | [`admin_api_lib.impl.key_db.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py) | Is used for storing the available sources and their current state. | +| key_value_store | [`admin_api_lib.impl.key_value_store.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py) | [`admin_api_lib.impl.key_value_store.file_status_key_value_store.FileStatusKeyValueStore`](./admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py) | Is used for storing the available sources and their current state. | | chunker | [`admin_api_lib.chunker.chunker.Chunker`](./admin-api-lib/src/admin_api_lib/chunker/chunker.py) | [`admin_api_lib.impl.chunker.text_chunker.TextChunker`](./admin-api-lib/src/admin_api_lib/impl/chunker/text_chunker.py) | Used for splitting the documents in managable chunks. | | document_extractor | [`admin_api_lib.extractor_api_client.openapi_client.api.extractor_api.ExtractorApi`](./admin-api-lib/src/admin_api_lib/extractor_api_client/openapi_client/api/extractor_api.py) | [`admin_api_lib.extractor_api_client.openapi_client.api.extractor_api.ExtractorApi`](./admin-api-lib/src/admin_api_lib/extractor_api_client/openapi_client/api/extractor_api.py) | Needs to be replaced if adjustments to the `extractor-api` is made. | | rag_api | [`admin_api_lib.rag_backend_client.openapi_client.api.rag_api.RagApi`](./admin-api-lib/src/admin_api_lib/rag_backend_client/openapi_client/api/rag_api.py) | [`admin_api_lib.rag_backend_client.openapi_client.api.rag_api.RagApi`](./admin-api-lib/src/admin_api_lib/rag_backend_client/openapi_client/api/rag_api.py) | Needs to be replaced if changes to the `/information_pieces/remove` or `/information_pieces/upload` of the [`rag-core-api`](#1-rag-core-api) are made. | diff --git a/libs/admin-api-lib/src/admin_api_lib/dependency_container.py b/libs/admin-api-lib/src/admin_api_lib/dependency_container.py index ee86cb5f..deb792cb 100644 --- a/libs/admin-api-lib/src/admin_api_lib/dependency_container.py +++ b/libs/admin-api-lib/src/admin_api_lib/dependency_container.py @@ -35,7 +35,7 @@ from admin_api_lib.impl.information_enhancer.page_summary_enhancer import ( PageSummaryEnhancer, ) -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.impl.mapper.informationpiece2document import ( diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py index 3cf671ff..7c3258dd 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_document_deleter.py @@ -7,7 +7,7 @@ from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter from admin_api_lib.file_services.file_service import FileService -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.rag_backend_client.openapi_client.api.rag_api import RagApi diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py index c2e21a04..4e5c5560 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_documents_status_retriever.py @@ -5,7 +5,7 @@ from admin_api_lib.api_endpoints.documents_status_retriever import ( DocumentsStatusRetriever, ) -from admin_api_lib.impl.key_db.file_status_key_value_store import ( +from admin_api_lib.impl.key_value_store.file_status_key_value_store import ( FileStatusKeyValueStore, ) from admin_api_lib.models.document_status import DocumentStatus diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py index 668cacd5..e17c242e 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_file_uploader.py @@ -18,7 +18,7 @@ from admin_api_lib.api_endpoints.document_deleter import DocumentDeleter from admin_api_lib.chunker.chunker import Chunker from admin_api_lib.models.status import Status -from admin_api_lib.impl.key_db.file_status_key_value_store import FileStatusKeyValueStore +from admin_api_lib.impl.key_value_store.file_status_key_value_store import FileStatusKeyValueStore from admin_api_lib.information_enhancer.information_enhancer import InformationEnhancer from admin_api_lib.utils.utils import sanitize_document_name diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py index f0fe7932..ffe5faf6 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/api_endpoints/default_source_uploader.py @@ -17,7 +17,7 @@ from admin_api_lib.api_endpoints.source_uploader import SourceUploader from admin_api_lib.chunker.chunker import Chunker from admin_api_lib.models.status import Status -from admin_api_lib.impl.key_db.file_status_key_value_store import FileStatusKeyValueStore +from admin_api_lib.impl.key_value_store.file_status_key_value_store import FileStatusKeyValueStore from admin_api_lib.information_enhancer.information_enhancer import InformationEnhancer from admin_api_lib.utils.utils import sanitize_document_name from admin_api_lib.rag_backend_client.openapi_client.models.information_piece import ( diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/__init__.py b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/__init__.py similarity index 100% rename from libs/admin-api-lib/src/admin_api_lib/impl/key_db/__init__.py rename to libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/__init__.py diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py similarity index 97% rename from libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py rename to libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py index ef27b65a..ab6e47db 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/key_db/file_status_key_value_store.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/key_value_store/file_status_key_value_store.py @@ -39,7 +39,7 @@ def __init__(self, settings: KeyValueSettings): settings : KeyValueSettings The settings object containing the host and port information for the Redis connection. """ - self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True) + self._redis = Redis(host=settings.host, port=settings.port, decode_responses=True, username=settings.username, password=settings.password) @staticmethod def _to_str(file_name: str, file_status: Status) -> str: diff --git a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py index 5acc6371..e4bae16b 100644 --- a/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py +++ b/libs/admin-api-lib/src/admin_api_lib/impl/settings/key_value_settings.py @@ -1,5 +1,6 @@ """Contains settings regarding the key values store.""" +from typing import Optional from pydantic import Field from pydantic_settings import BaseSettings @@ -24,3 +25,5 @@ class Config: host: str = Field() port: int = Field() + username: Optional[str] = Field(default=None) + password: Optional[str] = Field(default=None)