Skip to content

Commit

Permalink
Sparse string faceting fix #267
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-austin
patrick-austin committed Jun 15, 2022
1 parent b228aa3 commit 87ee1a3
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 9 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/icatproject/core/manager/search/LuceneApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public JsonObject buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) t
// irrespective of the sort, override the default implementation
JsonObjectBuilder builder = Json.createObjectBuilder();
builder.add("doc", lastBean.getEngineDocId());
builder.add("shardIndex", -1);
builder.add("shardIndex", lastBean.getShardIndex());
float score = lastBean.getScore();
if (!Float.isNaN(score)) {
builder.add("score", score);
Expand Down
33 changes: 27 additions & 6 deletions src/main/java/org/icatproject/core/manager/search/OpensearchApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ public ParentRelation(RelationType relationType, String parentName, String joinF
.build();
private static Map<String, List<ParentRelation>> relations = new HashMap<>();
private static Map<String, List<String>> defaultFieldsMap = new HashMap<>();
private static Map<String, List<String>> defaultFacetsMap = new HashMap<>();
protected static final Set<String> indices = new HashSet<>(
Arrays.asList("datafile", "dataset", "investigation", "instrumentscientist"));

Expand Down Expand Up @@ -175,6 +176,10 @@ public ParentRelation(RelationType relationType, String parentName, String joinF
Arrays.asList("name", "description", "doi", "sample.name", "sample.type.name", "type.name"));
defaultFieldsMap.put("investigation",
Arrays.asList("name", "visitId", "title", "summary", "doi", "facility.name"));

defaultFacetsMap.put("datafile", Arrays.asList("datafileFormat.name.keyword"));
defaultFacetsMap.put("dataset", Arrays.asList("type.name.keyword"));
defaultFacetsMap.put("investigation", Arrays.asList("type.name.keyword"));
}

public OpensearchApi(URI server) throws IcatException {
Expand Down Expand Up @@ -298,10 +303,6 @@ public void commit() throws IcatException {
public List<FacetDimension> facetSearch(String target, JsonObject facetQuery, Integer maxResults,
Integer maxLabels) throws IcatException {
List<FacetDimension> results = new ArrayList<>();
if (!facetQuery.containsKey("dimensions")) {
// If no dimensions were specified, return early
return results;
}
String dimensionPrefix = null;
String index = target.toLowerCase();
if (!indices.contains(index) && relations.containsKey(index)) {
Expand All @@ -311,10 +312,15 @@ public List<FacetDimension> facetSearch(String target, JsonObject facetQuery, In
}

JsonObject queryObject = facetQuery.getJsonObject("query");
JsonArray dimensions = facetQuery.getJsonArray("dimensions");
List<String> defaultFields = defaultFieldsMap.get(index);
JsonObjectBuilder bodyBuilder = parseQuery(Json.createObjectBuilder(), queryObject, index, defaultFields);
bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix);
if (facetQuery.containsKey("dimensions")) {
JsonArray dimensions = facetQuery.getJsonArray("dimensions");
bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix);
} else {
List<String> dimensions = defaultFacetsMap.get(index);
bodyBuilder = parseFacets(bodyBuilder, dimensions, maxLabels, dimensionPrefix);
}
String body = bodyBuilder.build().toString();

Map<String, String> parameterMap = new HashMap<>();
Expand Down Expand Up @@ -345,6 +351,21 @@ private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, JsonArray d
aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels));
}
}
return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder);
}

private JsonObjectBuilder parseFacets(JsonObjectBuilder bodyBuilder, List<String> dimensions, int maxLabels,
String dimensionPrefix) {
JsonObjectBuilder aggsBuilder = Json.createObjectBuilder();
for (String dimensionString : dimensions) {
String field = dimensionPrefix == null ? dimensionString : dimensionPrefix + "." + dimensionString;
aggsBuilder.add(dimensionString, QueryBuilder.buildStringFacet(field, maxLabels));
}
return buildFacetRequestJson(bodyBuilder, dimensionPrefix, aggsBuilder);
}

private JsonObjectBuilder buildFacetRequestJson(JsonObjectBuilder bodyBuilder, String dimensionPrefix,
JsonObjectBuilder aggsBuilder) {
if (dimensionPrefix == null) {
bodyBuilder.add("aggs", aggsBuilder);
} else {
Expand Down
10 changes: 8 additions & 2 deletions src/test/java/org/icatproject/core/manager/TestSearchApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,8 +553,9 @@ private void populate() throws IcatException {
/**
* Queues creation of an Instrument and a corresponding instrument scientist.
*
* @param queue Queue to add create operations to.
* @param instrumentId ICAT entity Id to use for the instrument/instrument scientist.
* @param queue Queue to add create operations to.
* @param instrumentId ICAT entity Id to use for the instrument/instrument
* scientist.
* @return The Instrument entity created.
* @throws IcatException
*/
Expand Down Expand Up @@ -984,6 +985,7 @@ public void modifyDatafile() throws IcatException {
JsonObject highRange = buildFacetRangeObject("high", 2L, 4L);
JsonObject rangeFacetRequest = buildFacetRangeRequest(buildFacetIdQuery("42"), "date", lowRange, highRange);
JsonObject stringFacetRequest = buildFacetStringRequest("42", "datafileFormat.name.keyword");
JsonObject sparseFacetRequest = Json.createObjectBuilder().add("query", buildFacetIdQuery("42")).build();
FacetDimension lowFacet = new FacetDimension("", "date", new FacetLabel("low", 1L), new FacetLabel("high", 0L));
FacetDimension highFacet = new FacetDimension("", "date", new FacetLabel("low", 0L),
new FacetLabel("high", 1L));
Expand All @@ -997,6 +999,7 @@ public void modifyDatafile() throws IcatException {
checkResults(searchApi.getResults(pdfQuery, 5));
checkResults(searchApi.getResults(pngQuery, 5));
checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5));
checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5));
checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), lowFacet);

// Change name and add a format
Expand All @@ -1006,6 +1009,7 @@ public void modifyDatafile() throws IcatException {
checkResults(searchApi.getResults(pdfQuery, 5), 42L);
checkResults(searchApi.getResults(pngQuery, 5));
checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pdfFacet);
checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pdfFacet);
checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet);

// Change just the format
Expand All @@ -1015,6 +1019,7 @@ public void modifyDatafile() throws IcatException {
checkResults(searchApi.getResults(pdfQuery, 5));
checkResults(searchApi.getResults(pngQuery, 5), 42L);
checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5), pngFacet);
checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5), pngFacet);
checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet);

// Remove the format
Expand All @@ -1024,6 +1029,7 @@ public void modifyDatafile() throws IcatException {
checkResults(searchApi.getResults(pdfQuery, 5));
checkResults(searchApi.getResults(pngQuery, 5));
checkFacets(searchApi.facetSearch("Datafile", stringFacetRequest, 5, 5));
checkFacets(searchApi.facetSearch("Datafile", sparseFacetRequest, 5, 5));
checkFacets(searchApi.facetSearch("Datafile", rangeFacetRequest, 5, 5), highFacet);

// Remove the file
Expand Down

0 comments on commit 87ee1a3

Please sign in to comment.