Add observability support to existing vector stores

 Add observability support to:
 - Cassandra
 - Chroma
 - Elasticsearch
 - Milvus
 - Neo4j
 - OpenSearch
 - Qdrant
 - Redis
 - Typesense
 - Weaviate
 - Pinecone
 - Oracle
 - Gemifire
 - MongoDB
 - HanaDB

 Add autoconfiguration obsrvability for the above vector stores.
 Add integration tests for all vector stores.

Author: tzolov

spring-ai-core/src/main/java/org/springframework/ai/observation/conventions/VectorStoreProvider.java

@@ -23,6 +23,22 @@ public enum VectorStoreProvider {
 
 	// @formatter:off
         PG_VECTOR("pg_vector"),
+        AZURE_VECTOR_STORE("azure_vector_store"),
+        CASSANDRA_VECTOR_STORE("cassandra_vector_store"),
+        CHROMA_VECTOR_STORE("chroma_vector_store"),
+        ELASTICSEARCH_VECTOR_STORE("elasticsearch_vector_store"),
+        MILVUS_VECTOR_STORE("milvus_vector_store"),
+        NEO4J_VECTOR_STORE("neo4j_vector_store"),
+        OPENSEARCH_VECTOR_STORE("opensearch_vector_store"),
+        QDRANT_VECTOR_STORE("qdrant_vector_store"),
+        REDIS_VECTOR_STORE("redis_vector_store"),
+        TYPESENSE_VECTOR_STORE("typesense_vector_store"),
+        WEAVIATE_VECTOR_STORE("weaviate_vector_store"),
+        PINECONE_VECTOR_STORE("pinecone_vector_store"),
+        ORACLE_VECTOR_STORE("oracle_vector_store"),
+        MONGODB_VECTOR_STORE("mongodb_vector_store"),
+        GEMFIRE_VECTOR_STORE("gemfire_vector_store"),
+        HANA_VECTOR_STORE("hana_vector_store"),
         SIMPLE_VECTOR_STORE("simple_vector_store");
 
         // @formatter:on

spring-ai-core/src/main/java/org/springframework/ai/vectorstore/SimpleVectorStore.java

@@ -40,13 +40,16 @@
 import org.springframework.ai.observation.conventions.VectorStoreSimilarityMetric;
 import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
 import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
 import org.springframework.core.io.Resource;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 
+import io.micrometer.observation.ObservationRegistry;
+
 /**
  * SimpleVectorStore is a simple implementation of the VectorStore interface.
  *
@@ -71,6 +74,14 @@ public class SimpleVectorStore extends AbstractObservationVectorStore {
 	protected EmbeddingModel embeddingModel;
 
 	public SimpleVectorStore(EmbeddingModel embeddingModel) {
+		this(embeddingModel, ObservationRegistry.NOOP, null);
+	}
+
+	public SimpleVectorStore(EmbeddingModel embeddingModel, ObservationRegistry observationRegistry,
+			VectorStoreObservationConvention customObservationConvention) {
+
+		super(observationRegistry, customObservationConvention);
+
 		Objects.requireNonNull(embeddingModel, "EmbeddingModel must not be null");
 		this.embeddingModel = embeddingModel;
 	}

spring-ai-core/src/main/java/org/springframework/ai/vectorstore/observation/AbstractObservationVectorStore.java

@@ -38,18 +38,10 @@ public abstract class AbstractObservationVectorStore implements VectorStore {
 	@Nullable
 	private final VectorStoreObservationConvention customObservationConvention;
 
-	public AbstractObservationVectorStore() {
-		this(ObservationRegistry.NOOP, null);
-	}
-
-	public AbstractObservationVectorStore(ObservationRegistry observationRegistry) {
-		this(observationRegistry, null);
-	}
-
 	public AbstractObservationVectorStore(ObservationRegistry observationRegistry,
-			VectorStoreObservationConvention customSearchObservationConvention) {
+			VectorStoreObservationConvention customObservationConvention) {
 		this.observationRegistry = observationRegistry;
-		this.customObservationConvention = customSearchObservationConvention;
+		this.customObservationConvention = customObservationConvention;
 	}
 
 	@Override

spring-ai-core/src/main/java/org/springframework/ai/vectorstore/observation/DefaultVectorStoreObservationConvention.java

@@ -44,6 +44,9 @@ public class DefaultVectorStoreObservationConvention implements VectorStoreObser
 
 	private static final KeyValue TOP_K_NONE = KeyValue.of(HighCardinalityKeyNames.TOP_K, KeyValue.NONE_VALUE);
 
+	private static final KeyValue SIMILARITY_THRESHOLD_NONE = KeyValue.of(HighCardinalityKeyNames.SIMILARITY_THRESHOLD,
+			KeyValue.NONE_VALUE);
+
 	private static final KeyValue SIMILARITY_METRIC_NONE = KeyValue.of(HighCardinalityKeyNames.SIMILARITY_METRIC,
 			KeyValue.NONE_VALUE);
 
@@ -89,7 +92,7 @@ public KeyValues getLowCardinalityKeyValues(VectorStoreObservationContext contex
 	public KeyValues getHighCardinalityKeyValues(VectorStoreObservationContext context) {
 		return KeyValues.of(query(context), metadataFilter(context), topK(context), dimensions(context),
 				similarityMetric(context), collectionName(context), namespace(context), fieldName(context),
-				indexName(context));
+				indexName(context), similarityThreshold(context));
 	}
 
 	protected KeyValue springAiKind() {
@@ -133,6 +136,14 @@ protected KeyValue topK(VectorStoreObservationContext context) {
 		return TOP_K_NONE;
 	}
 
+	protected KeyValue similarityThreshold(VectorStoreObservationContext context) {
+		if (context.getQueryRequest() != null && context.getQueryRequest().getSimilarityThreshold() >= 0) {
+			return KeyValue.of(HighCardinalityKeyNames.SIMILARITY_THRESHOLD,
+					"" + context.getQueryRequest().getSimilarityThreshold());
+		}
+		return SIMILARITY_THRESHOLD_NONE;
+	}
+
 	protected KeyValue similarityMetric(VectorStoreObservationContext context) {
 		if (StringUtils.hasText(context.getSimilarityMetric())) {
 			return KeyValue.of(HighCardinalityKeyNames.SIMILARITY_METRIC, context.getSimilarityMetric());

spring-ai-core/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationContext.java

@@ -27,7 +27,6 @@
  * @author Christian Tzolov
  * @since 1.0.0
  */
-
 public class VectorStoreObservationContext extends Observation.Context {
 
 	public enum Operation {

spring-ai-core/src/main/java/org/springframework/ai/vectorstore/observation/VectorStoreObservationDocumentation.java

@@ -128,6 +128,17 @@ public String asString() {
 				return "db.vector.query.top_k";
 			}
 		},
+		/**
+		 * Similarity threshold that accepts all search scores. A threshold value of 0.0
+		 * means any similarity is accepted or disable the similarity threshold filtering.
+		 * A threshold value of 1.0 means an exact match is required.
+		 */
+		SIMILARITY_THRESHOLD {
+			@Override
+			public String asString() {
+				return "db.vector.query.similarity_threshold";
+			}
+		},
 		/**
 		 * The dimension of the vector.
 		 */

spring-ai-spring-boot-autoconfigure/pom.xml

@@ -513,6 +513,12 @@
 			<scope>test</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>io.micrometer</groupId>
+			<artifactId>micrometer-observation-test</artifactId>
+			<scope>test</scope>
+		</dependency>
+
 	</dependencies>
 
 </project>

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/azure/AzureVectorStoreAutoConfiguration.java

@@ -19,8 +19,14 @@
 import com.azure.search.documents.indexes.SearchIndexClient;
 import com.azure.search.documents.indexes.SearchIndexClientBuilder;
 
+import io.micrometer.observation.ObservationRegistry;
+
+import java.util.List;
+
 import org.springframework.ai.embedding.EmbeddingModel;
 import org.springframework.ai.vectorstore.azure.AzureVectorStore;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -48,9 +54,12 @@ public SearchIndexClient searchIndexClient(AzureVectorStoreProperties properties
 	@Bean
 	@ConditionalOnMissingBean
 	public AzureVectorStore vectorStore(SearchIndexClient searchIndexClient, EmbeddingModel embeddingModel,
-			AzureVectorStoreProperties properties) {
+			AzureVectorStoreProperties properties, ObjectProvider<ObservationRegistry> observationRegistry,
+			ObjectProvider<VectorStoreObservationConvention> customObservationConvention) {
 
-		var vectorStore = new AzureVectorStore(searchIndexClient, embeddingModel, properties.isInitializeSchema());
+		var vectorStore = new AzureVectorStore(searchIndexClient, embeddingModel, properties.isInitializeSchema(),
+				List.of(), observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP),
+				customObservationConvention.getIfAvailable(() -> null));
 
 		vectorStore.setIndexName(properties.getIndexName());
 

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/cassandra/CassandraVectorStoreAutoConfiguration.java

@@ -20,9 +20,13 @@
 import com.datastax.oss.driver.api.core.CqlSession;
 import com.datastax.oss.driver.api.core.config.DefaultDriverOption;
 
+import io.micrometer.observation.ObservationRegistry;
+
 import org.springframework.ai.embedding.EmbeddingModel;
 import org.springframework.ai.vectorstore.CassandraVectorStore;
 import org.springframework.ai.vectorstore.CassandraVectorStoreConfig;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
 import org.springframework.boot.autoconfigure.cassandra.DriverConfigLoaderBuilderCustomizer;
@@ -33,6 +37,7 @@
 
 /**
  * @author Mick Semb Wever
+ * @author Christian Tzolov
  * @since 1.0.0
  */
 @AutoConfiguration(after = CassandraAutoConfiguration.class)
@@ -43,7 +48,8 @@ public class CassandraVectorStoreAutoConfiguration {
 	@Bean
 	@ConditionalOnMissingBean
 	public CassandraVectorStore vectorStore(EmbeddingModel embeddingModel, CassandraVectorStoreProperties properties,
-			CqlSession cqlSession) {
+			CqlSession cqlSession, ObjectProvider<ObservationRegistry> observationRegistry,
+			ObjectProvider<VectorStoreObservationConvention> customObservationConvention) {
 
 		var builder = CassandraVectorStoreConfig.builder().withCqlSession(cqlSession);
 
@@ -61,7 +67,9 @@ public CassandraVectorStore vectorStore(EmbeddingModel embeddingModel, Cassandra
 			builder = builder.returnEmbeddings();
 		}
 
-		return CassandraVectorStore.create(builder.build(), embeddingModel);
+		return new CassandraVectorStore(builder.build(), embeddingModel,
+				observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP),
+				customObservationConvention.getIfAvailable(() -> null));
 	}
 
 	@Bean

spring-ai-spring-boot-autoconfigure/src/main/java/org/springframework/ai/autoconfigure/vectorstore/chroma/ChromaVectorStoreAutoConfiguration.java

@@ -18,6 +18,8 @@
 import org.springframework.ai.chroma.ChromaApi;
 import org.springframework.ai.embedding.EmbeddingModel;
 import org.springframework.ai.vectorstore.ChromaVectorStore;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -29,6 +31,8 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import io.micrometer.observation.ObservationRegistry;
+
 /**
  * @author Christian Tzolov
  * @author Eddú Meléndez
@@ -72,9 +76,11 @@ else if (StringUtils.hasText(apiProperties.getUsername()) && StringUtils.hasText
 	@Bean
 	@ConditionalOnMissingBean
 	public ChromaVectorStore vectorStore(EmbeddingModel embeddingModel, ChromaApi chromaApi,
-			ChromaVectorStoreProperties storeProperties) {
+			ChromaVectorStoreProperties storeProperties, ObjectProvider<ObservationRegistry> observationRegistry,
+			ObjectProvider<VectorStoreObservationConvention> customObservationConvention) {
 		return new ChromaVectorStore(embeddingModel, chromaApi, storeProperties.getCollectionName(),
-				storeProperties.isInitializeSchema());
+				storeProperties.isInitializeSchema(), observationRegistry.getIfUnique(() -> ObservationRegistry.NOOP),
+				customObservationConvention.getIfAvailable(() -> null));
 	}
 
 	static class PropertiesChromaConnectionDetails implements ChromaConnectionDetails {