Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-17669: Reduce Memory Consumption by 80-90% when using Dynamic fields (DocumentObjectBinder) #3179

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

SOLR-17669: Reduce Memory Consumption by 80-90% when using Dynamic fields (DocumentObjectBinder) #3179

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2b684cd
Reduce Memory Consumption by 90% when Mapping Data to Objects
ds-manzinger Feb 12, 2025
fb0de61
Add Pattern Matcher if the case occours that a Wildcard is in the mid…
ds-manzinger Feb 16, 2025
b77a1cc
Tidy Code
ds-manzinger Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions solr/solrj/src/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ private class DocField {
* is set to <code>TRUE</code> as well as <code>isList</code> is set to <code>TRUE</code>
*/
private boolean isContainedInMap;
private Pattern dynamicFieldNamePatternMatcher;
private DynamicFieldNameMatcher dynamicFieldNamePatternMatcher;

public DocField(AccessibleObject member) {
if (member instanceof java.lang.reflect.Field) {
Expand Down Expand Up @@ -235,10 +235,24 @@ private void storeName(Field annotation) {
} else if (annotation.value().indexOf('*') >= 0) {
// dynamic fields are annotated as @Field("categories_*")
// if the field was annotated as a dynamic field, convert the name into a pattern
// the wildcard (*) is supposed to be either a prefix or a suffix, hence the use of
// replaceFirst
name = annotation.value().replaceFirst("\\*", "\\.*");
dynamicFieldNamePatternMatcher = Pattern.compile("^" + name + "$");
// the wildcard (*) is supposed to be either a prefix or a suffix

if (annotation.value().startsWith("*")) {
// handle suffix
name = annotation.value();
String pattern = annotation.value().substring(1);
dynamicFieldNamePatternMatcher = fieldName -> fieldName.endsWith(pattern);
} else if (annotation.value().endsWith("*")) {
// handle prefix
name = annotation.value();
String pattern = annotation.value().substring(0, annotation.value().length() - 1);
dynamicFieldNamePatternMatcher = fieldName -> fieldName.startsWith(pattern);
} else {
// handle when wildcard is in the middle
name = annotation.value().replaceFirst("\\*", "\\.*");
Pattern compiledPattern = Pattern.compile("^" + name + "$");
dynamicFieldNamePatternMatcher = fieldName -> compiledPattern.matcher(fieldName).find();
}
} else {
name = annotation.value();
}
Expand Down Expand Up @@ -397,7 +411,7 @@ private Object getFieldValue(SolrDocument solrDocument) {
}

for (String field : solrDocument.getFieldNames()) {
if (dynamicFieldNamePatternMatcher.matcher(field).find()) {
if (dynamicFieldNamePatternMatcher.matches(field)) {
Object val = solrDocument.getFieldValue(field);
if (val == null) {
continue;
Expand Down Expand Up @@ -508,4 +522,8 @@ public Object get(final Object obj) {
}

public static final String DEFAULT = "#default";

private interface DynamicFieldNameMatcher {
boolean matches(String name);
}
}
29 changes: 29 additions & 0 deletions solr/solrj/src/test/org/apache/solr/client/solrj/beans/TestDocumentObjectBinder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,12 @@ public void testDynamicFieldBinding() {

List<String> supplierTwo = l.get(3).supplier.get("supplier_2");
assertEquals("CCTV Store", supplierTwo.get(0));

String supplierMiddle = l.get(3).supplier_middle.get("supmiddle_mobile_middle");
assertEquals("Mobile Store in Middle", supplierMiddle);

String supplierBeginning = l.get(3).sup_beginning.get("mobile_supbeginning");
assertEquals("Mobile Store with Beginning Wildcard", supplierBeginning);
}

public void testChild() throws Exception {
Expand Down Expand Up @@ -182,6 +188,12 @@ public static class Item {
@Field("sup_simple_*")
Map<String, String> supplier_simple;

@Field("supmiddle_*_middle")
Map<String, String> supplier_middle;

@Field("*_supbeginning")
Map<String, String> sup_beginning;

private String[] allSuppliers;

@Field("supplier_*")
Expand All @@ -193,6 +205,22 @@ public String[] getAllSuppliers() {
return this.allSuppliers;
}

public Map<String, String> getSupplier_middle() {
return supplier_middle;
}

public void setSupplier_middle(Map<String, String> supplier_middle) {
this.supplier_middle = supplier_middle;
}

public Map<String, String> getSup_beginning() {
return sup_beginning;
}

public void setSup_beginning(Map<String, String> sup_beginning) {
this.sup_beginning = sup_beginning;
}

@Field
public void setInStock(Boolean b) {
inStock = b;
Expand Down Expand Up @@ -275,6 +303,7 @@ public void setAaa(String aaa) {
+ "<str name=\"manu\">Belkin</str><str name=\"name\">iPod &amp; iPod Mini USB 2.0 Cable</str>"
+ "<int name=\"popularity\">1</int><float name=\"price\">11.5</float><str name=\"sku\">IW-02</str>"
+ "<str name=\"supplier_1\">Mobile Store</str><str name=\"supplier_1\">iPod Store</str><str name=\"supplier_2\">CCTV Store</str>"
+ "<str name=\"supmiddle_mobile_middle\">Mobile Store in Middle</str><str name=\"mobile_supbeginning\">Mobile Store with Beginning Wildcard</str>"
+ "<date name=\"timestamp\">2008-04-16T10:35:57.140Z</date><float name=\"weight\">2.0</float></doc></result>\n"
+ "</response>";
}
Loading