Skip to content

Commit 3e713b8

Browse files
jhuynh1jhuynh1
committed
Add filter query support for GemFireVectorStore
Signed-off-by: Jason Huynh <[email protected]>
1 parent 53a7af5 commit 3e713b8

File tree

4 files changed

+492
-15
lines changed

4 files changed

+492
-15
lines changed

‎vector-stores/spring-ai-gemfire-store/src/main/java/org/springframework/ai/vectorstore/GemFireAiSearchFilterExpressionConverter.java

+160
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright 2023 - 2025 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package org.springframework.ai.vectorstore;
17+
18+
import java.text.ParseException;
19+
import java.text.SimpleDateFormat;
20+
import java.util.Date;
21+
import java.util.List;
22+
import java.util.TimeZone;
23+
import java.util.regex.Pattern;
24+
25+
import org.springframework.ai.vectorstore.filter.Filter;
26+
import org.springframework.ai.vectorstore.filter.Filter.Expression;
27+
import org.springframework.ai.vectorstore.filter.Filter.Key;
28+
import org.springframework.ai.vectorstore.filter.converter.AbstractFilterExpressionConverter;
29+
30+
/**
31+
* GemFireAiSearchFilterExpressionConverter is a class that converts Filter.Expression
32+
* objects into GemFire VectorDB query string representation. It extends the
33+
* AbstractFilter ExpressionConverter class.
34+
*
35+
* @author Jason Huynh
36+
*/
37+
public class GemFireAiSearchFilterExpressionConverter extends AbstractFilterExpressionConverter {
38+
39+
private static final Pattern DATE_FORMAT_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z");
40+
41+
private final SimpleDateFormat dateFormat;
42+
43+
public GemFireAiSearchFilterExpressionConverter() {
44+
this.dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
45+
this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
46+
}
47+
48+
@Override
49+
protected void doExpression(Expression expression, StringBuilder context) {
50+
if (expression.type() == Filter.ExpressionType.IN || expression.type() == Filter.ExpressionType.NIN) {
51+
context.append(getOperationSymbol(expression));
52+
this.convertOperand(expression.left(), context);
53+
context.append("(");
54+
this.convertOperand(expression.right(), context);
55+
context.append(")");
56+
}
57+
else if (expression.type() == Filter.ExpressionType.GT || expression.type() == Filter.ExpressionType.GTE) {
58+
this.convertOperand(expression.left(), context);
59+
context.append(getOperationSymbol(expression));
60+
this.convertOperand(expression.right(), context);
61+
context.append(" TO *]");
62+
}
63+
else if (expression.type() == Filter.ExpressionType.LT || expression.type() == Filter.ExpressionType.LTE) {
64+
this.convertOperand(expression.left(), context);
65+
context.append("[* TO ");
66+
this.convertOperand(expression.right(), context);
67+
context.append(getOperationSymbol(expression));
68+
}
69+
else {
70+
this.convertOperand(expression.left(), context);
71+
context.append(getOperationSymbol(expression));
72+
this.convertOperand(expression.right(), context);
73+
}
74+
}
75+
76+
@Override
77+
protected void doStartValueRange(Filter.Value listValue, StringBuilder context) {
78+
}
79+
80+
@Override
81+
protected void doEndValueRange(Filter.Value listValue, StringBuilder context) {
82+
}
83+
84+
@Override
85+
protected void doAddValueRangeSpitter(Filter.Value listValue, StringBuilder context) {
86+
context.append(" OR ");
87+
}
88+
89+
private String getOperationSymbol(Expression exp) {
90+
return switch (exp.type()) {
91+
case AND -> " AND ";
92+
case OR -> " OR ";
93+
case EQ, IN -> "";
94+
case NE -> " NOT ";
95+
case LT -> "}";
96+
case LTE -> "]";
97+
case GT -> "{";
98+
case GTE -> "[";
99+
case NIN -> "NOT ";
100+
default -> throw new RuntimeException("Not supported expression type: " + exp.type());
101+
};
102+
}
103+
104+
@Override
105+
public void doKey(Key key, StringBuilder context) {
106+
var identifier = hasOuterQuotes(key.key()) ? removeOuterQuotes(key.key()) : key.key();
107+
context.append(identifier.trim()).append(":");
108+
}
109+
110+
@Override
111+
protected void doValue(Filter.Value filterValue, StringBuilder context) {
112+
if (filterValue.value() instanceof List list) {
113+
int c = 0;
114+
for (Object v : list) {
115+
context.append(v);
116+
if (c++ < list.size() - 1) {
117+
this.doAddValueRangeSpitter(filterValue, context);
118+
}
119+
}
120+
}
121+
else {
122+
this.doSingleValue(filterValue.value(), context);
123+
}
124+
}
125+
126+
@Override
127+
protected void doSingleValue(Object value, StringBuilder context) {
128+
if (value instanceof Date date) {
129+
context.append(this.dateFormat.format(date));
130+
}
131+
else if (value instanceof String text) {
132+
if (DATE_FORMAT_PATTERN.matcher(text).matches()) {
133+
try {
134+
Date date = this.dateFormat.parse(text);
135+
context.append(this.dateFormat.format(date));
136+
}
137+
catch (ParseException e) {
138+
throw new IllegalArgumentException("Invalid date type:" + text, e);
139+
}
140+
}
141+
else {
142+
context.append(text);
143+
}
144+
}
145+
else {
146+
context.append(value);
147+
}
148+
}
149+
150+
@Override
151+
public void doStartGroup(Filter.Group group, StringBuilder context) {
152+
context.append("(");
153+
}
154+
155+
@Override
156+
public void doEndGroup(Filter.Group group, StringBuilder context) {
157+
context.append(")");
158+
}
159+
160+
}

‎vector-stores/spring-ai-gemfire-store/src/main/java/org/springframework/ai/vectorstore/gemfire/GemFireVectorStore.java

+25-7
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@
1616

1717
package org.springframework.ai.vectorstore.gemfire;
1818

19-
import java.util.HashMap;
20-
import java.util.List;
21-
import java.util.Map;
22-
2319
import com.fasterxml.jackson.annotation.JsonCreator;
2420
import com.fasterxml.jackson.annotation.JsonInclude;
2521
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -28,15 +24,16 @@
2824
import com.fasterxml.jackson.databind.json.JsonMapper;
2925
import org.slf4j.Logger;
3026
import org.slf4j.LoggerFactory;
31-
3227
import org.springframework.ai.document.Document;
3328
import org.springframework.ai.document.DocumentMetadata;
3429
import org.springframework.ai.embedding.EmbeddingModel;
3530
import org.springframework.ai.embedding.EmbeddingOptionsBuilder;
3631
import org.springframework.ai.observation.conventions.VectorStoreProvider;
3732
import org.springframework.ai.util.JacksonUtils;
3833
import org.springframework.ai.vectorstore.AbstractVectorStoreBuilder;
34+
import org.springframework.ai.vectorstore.GemFireAiSearchFilterExpressionConverter;
3935
import org.springframework.ai.vectorstore.SearchRequest;
36+
import org.springframework.ai.vectorstore.filter.FilterExpressionConverter;
4037
import org.springframework.ai.vectorstore.observation.AbstractObservationVectorStore;
4138
import org.springframework.ai.vectorstore.observation.VectorStoreObservationContext;
4239
import org.springframework.beans.factory.InitializingBean;
@@ -50,6 +47,10 @@
5047
import org.springframework.web.reactive.function.client.WebClientResponseException;
5148
import org.springframework.web.util.UriComponentsBuilder;
5249

50+
import java.util.HashMap;
51+
import java.util.List;
52+
import java.util.Map;
53+
5354
/**
5455
* A VectorStore implementation backed by GemFire. This store supports creating, updating,
5556
* deleting, and similarity searching of documents in a GemFire index.
@@ -114,6 +115,8 @@ public class GemFireVectorStore extends AbstractObservationVectorStore implement
114115

115116
private final String[] fields;
116117

118+
private final FilterExpressionConverter filterExpressionConverter;
119+
117120
/**
118121
* Protected constructor that accepts a builder instance. This is the preferred way to
119122
* create new GemFireVectorStore instances.
@@ -134,6 +137,7 @@ protected GemFireVectorStore(Builder builder) {
134137
.build(builder.sslEnabled ? "s" : "", builder.host, builder.port)
135138
.toString();
136139
this.client = WebClient.create(base);
140+
this.filterExpressionConverter = new GemFireAiSearchFilterExpressionConverter();
137141
this.objectMapper = JsonMapper.builder().addModules(JacksonUtils.instantiateAvailableModules()).build();
138142
}
139143

@@ -245,15 +249,16 @@ public void doDelete(List<String> idList) {
245249
@Override
246250
@Nullable
247251
public List<Document> doSimilaritySearch(SearchRequest request) {
252+
String filterQuery = null;
248253
if (request.hasFilterExpression()) {
249-
throw new UnsupportedOperationException("GemFire currently does not support metadata filter expressions.");
254+
filterQuery = filterExpressionConverter.convertExpression(request.getFilterExpression());
250255
}
251256
float[] floatVector = this.embeddingModel.embed(request.getQuery());
252257
return this.client.post()
253258
.uri("/" + this.indexName + QUERY)
254259
.contentType(MediaType.APPLICATION_JSON)
255260
.bodyValue(new QueryRequest(floatVector, request.getTopK(), request.getTopK(), // TopKPerBucket
256-
true))
261+
true, filterQuery))
257262
.retrieve()
258263
.bodyToFlux(QueryResponse.class)
259264
.filter(r -> r.score >= request.getSimilarityThreshold())
@@ -474,11 +479,20 @@ private static final class QueryRequest {
474479
@JsonProperty("include-metadata")
475480
private final boolean includeMetadata;
476481

482+
@JsonProperty("filter-query")
483+
@JsonInclude(JsonInclude.Include.NON_NULL)
484+
private final String filterQuery;
485+
477486
QueryRequest(float[] vector, int k, int kPerBucket, boolean includeMetadata) {
487+
this(vector, k, kPerBucket, includeMetadata, null);
488+
}
489+
490+
QueryRequest(float[] vector, int k, int kPerBucket, boolean includeMetadata, String filterQuery) {
478491
this.vector = vector;
479492
this.k = k;
480493
this.kPerBucket = kPerBucket;
481494
this.includeMetadata = includeMetadata;
495+
this.filterQuery = filterQuery;
482496
}
483497

484498
public float[] getVector() {
@@ -497,6 +511,10 @@ public boolean isIncludeMetadata() {
497511
return this.includeMetadata;
498512
}
499513

514+
public String getFilterQuery() {
515+
return filterQuery;
516+
}
517+
500518
}
501519

502520
private static final class QueryResponse {

0 commit comments

Comments
 (0)