Skip to content

Commit

Permalink
Refactors in EntityBeanManager and SearchManager #267
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-austin
patrick-austin committed Oct 3, 2022
1 parent de8fd2d commit 60f30a6
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 109 deletions.
173 changes: 87 additions & 86 deletions src/main/java/org/icatproject/core/manager/EntityBeanManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1128,9 +1128,6 @@ public String getUserName(String sessionId, EntityManager manager) throws IcatEx
Session session = getSession(sessionId, manager);
String userName = session.getUserName();
logger.debug("user: " + userName + " is associated with: " + sessionId);
// TODO RM
// logger.trace("userName: {}", userName);
// userName = userName.equals("db/notroot") ? "notroot" : userName;
return userName;
} catch (IcatException e) {
logger.debug("sessionId " + sessionId + " is not associated with valid session " + e.getMessage());
Expand Down Expand Up @@ -1408,7 +1405,7 @@ public void searchCommit() throws IcatException {
*
* @param userName User performing the search, used for authorisation.
* @param jo JsonObject containing the details of the query to be used.
* @param limit The maximum number of results to collect before returning. If
* @param maxCount The maximum number of results to collect before returning. If
* a batch from the search engine has more than this many
* authorised results, then the excess results will be
* discarded.
Expand All @@ -1418,56 +1415,15 @@ public void searchCommit() throws IcatException {
* @return SearchResult for the query.
* @throws IcatException
*/
public List<ScoredEntityBaseBean> freeTextSearch(String userName, JsonObject jo, int limit, String sort,
public List<ScoredEntityBaseBean> freeTextSearch(String userName, JsonObject jo, int maxCount,
EntityManager manager, String ip, Class<? extends EntityBaseBean> klass) throws IcatException {
long startMillis = System.currentTimeMillis();
List<ScoredEntityBaseBean> results = new ArrayList<>();
JsonValue searchAfter = null;
JsonValue lastSearchAfter = null;
if (searchActive) {
SearchResult lastSearchResult = null;
List<ScoredEntityBaseBean> allResults = Collections.emptyList();
/*
* As results may be rejected and maxCount may be 1 ensure that we
* don't make a huge number of calls to search engine
*/
int blockSize = Math.max(1000, limit);

do {
lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, Arrays.asList("id"));
allResults = lastSearchResult.getResults();
ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, limit, userName, manager, klass);
if (lastBean == null) {
// Haven't stopped early, so use the Lucene provided searchAfter document
lastSearchAfter = lastSearchResult.getSearchAfter();
if (lastSearchAfter == null) {
break; // If searchAfter is null, we ran out of results so stop here
}
searchAfter = lastSearchAfter;
} else {
// Have stopped early by reaching the limit, so build a searchAfter document
lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort);
break;
}
if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) {
String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds";
throw new IcatException(IcatExceptionType.INTERNAL, msg);
}
} while (results.size() < limit);
}

if (logRequests.contains("R")) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) {
gen.write("userName", userName);
if (results.size() > 0) {
gen.write("entityId", results.get(0).getEntityBaseBeanId());
}
gen.writeEnd();
}
transmitter.processMessage("freeTextSearch", ip, baos.toString(), startMillis);
searchDocuments(userName, jo, null, maxCount, maxCount, null, manager, klass,
startMillis, results, Arrays.asList("id"));
}
logger.debug("Returning {} results", results.size());
logSearch(userName, ip, startMillis, results, "freeTextSearch");
return results;
}

Expand All @@ -1494,44 +1450,16 @@ public List<ScoredEntityBaseBean> freeTextSearch(String userName, JsonObject jo,
* @throws IcatException
*/
public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue searchAfter, int minCount,
int maxCount, String sort, EntityManager manager, String ip,
Class<? extends EntityBaseBean> klass) throws IcatException {
int maxCount, String sort, EntityManager manager, String ip, Class<? extends EntityBaseBean> klass)
throws IcatException {
long startMillis = System.currentTimeMillis();
List<ScoredEntityBaseBean> results = new ArrayList<>();
JsonValue lastSearchAfter = null;
List<ScoredEntityBaseBean> results = new ArrayList<>();
List<FacetDimension> dimensions = new ArrayList<>();
if (searchActive) {
SearchResult lastSearchResult = null;
List<ScoredEntityBaseBean> allResults = Collections.emptyList();
List<String> fields = SearchManager.getPublicSearchFields(gateKeeper, klass.getSimpleName());
/*
* As results may be rejected and maxCount may be 1 ensure that we
* don't make a huge number of calls to search engine
*/
int blockSize = Math.max(1000, maxCount);

do {
lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields);
allResults = lastSearchResult.getResults();
ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager,
klass);
if (lastBean == null) {
// Haven't stopped early, so use the Lucene provided searchAfter document
lastSearchAfter = lastSearchResult.getSearchAfter();
if (lastSearchAfter == null) {
break; // If searchAfter is null, we ran out of results so stop here
}
searchAfter = lastSearchAfter;
} else {
// Have stopped early by reaching the limit, so build a searchAfter document
lastSearchAfter = searchManager.buildSearchAfter(lastBean, sort);
break;
}
if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) {
String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds";
throw new IcatException(IcatExceptionType.INTERNAL, msg);
}
} while (results.size() < minCount);
lastSearchAfter = searchDocuments(userName, jo, searchAfter, maxCount, minCount, sort, manager, klass,
startMillis, results, fields);

if (jo.containsKey("facets")) {
List<JsonObject> jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class);
Expand All @@ -1544,6 +1472,21 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue
}
}
}
logSearch(userName, ip, startMillis, results, "freeTextSearchDocs");
return new SearchResult(lastSearchAfter, results, dimensions);
}

/**
* Performs logging dependent on the value of logRequests.
*
* @param userName User performing the search
* @param ip Used for logging only
* @param startMillis The start time of the search in milliseconds
* @param results List of authorised search results
* @param operation Name of the calling function
*/
private void logSearch(String userName, String ip, long startMillis, List<ScoredEntityBaseBean> results,
String operation) {
if (logRequests.contains("R")) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (JsonGenerator gen = Json.createGenerator(baos).writeStartObject()) {
Expand All @@ -1553,10 +1496,69 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue
}
gen.writeEnd();
}
transmitter.processMessage("freeTextSearchDocs", ip, baos.toString(), startMillis);
transmitter.processMessage(operation, ip, baos.toString(), startMillis);
}
logger.debug("Returning {} results", results.size());
return new SearchResult(lastSearchAfter, results, dimensions);
}

/**
* Performs batches of searches, the results of which are authorised. Results
* are collected until they run out, minCount is reached, or too much time
* elapses.
*
* @param userName User performing the search, used for authorisation.
* @param jo JsonObject containing the details of the query to be used.
* @param searchAfter JsonValue representation of the final result from a
* previous search.
* @param minCount The minimum number of results to collect before returning.
* If a batch from the search engine has at least this many
* authorised results, no further batches will be requested.
* @param maxCount The maximum number of results to collect before returning.
* If a batch from the search engine has more than this many
* authorised results, then the excess results will be
* discarded.
* @param sort String of Json representing sort criteria.
* @param manager EntityManager for finding entities from their Id.
* @param klass Class of the entity to search.
* @param startMillis The start time of the search in milliseconds
* @param results List of results from the search. Authorised results will
* be appended to this List.
* @param fields Fields to include in the returned Documents.
* @return JsonValue representing the last result of the search, formatted to
* allow future searches to "search after" this result. May be null.
* @throws IcatException If the search exceeds the maximum allowed time.
*/
private JsonValue searchDocuments(String userName, JsonObject jo, JsonValue searchAfter, int maxCount, int minCount,
String sort, EntityManager manager, Class<? extends EntityBaseBean> klass, long startMillis,
List<ScoredEntityBaseBean> results, List<String> fields) throws IcatException {
/*
* As results may be rejected and maxCount may be 1 ensure that we
* don't make a huge number of calls to search engine
*/
int blockSize = Math.max(1000, maxCount);
JsonValue lastSearchAfter;
do {
SearchResult lastSearchResult = searchManager.freeTextSearch(jo, searchAfter, blockSize, sort, fields);
List<ScoredEntityBaseBean> allResults = lastSearchResult.getResults();
ScoredEntityBaseBean lastBean = filterReadAccess(results, allResults, maxCount, userName, manager,
klass);
if (lastBean == null) {
// Haven't stopped early, so use the Lucene provided searchAfter document
lastSearchAfter = lastSearchResult.getSearchAfter();
if (lastSearchAfter == null) {
return null; // If searchAfter is null, we ran out of results so stop here
}
searchAfter = lastSearchAfter;
} else {
// Have stopped early by reaching the limit, so build a searchAfter document
return searchManager.buildSearchAfter(lastBean, sort);
}
if (System.currentTimeMillis() - startMillis > searchMaxSearchTimeMillis) {
String msg = "Search cancelled for exceeding " + searchMaxSearchTimeMillis / 1000 + " seconds";
throw new IcatException(IcatExceptionType.INTERNAL, msg);
}
} while (results.size() < minCount);
return lastSearchAfter;
}

/**
Expand All @@ -1569,7 +1571,6 @@ public SearchResult freeTextSearchDocs(String userName, JsonObject jo, JsonValue
* @throws IcatException
*/
public SearchResult facetDocs(JsonObject jo, Class<? extends EntityBaseBean> klass) throws IcatException {
JsonValue lastSearchAfter = null;
List<FacetDimension> dimensions = new ArrayList<>();
if (searchActive && jo.containsKey("facets")) {
List<JsonObject> jsonFacets = jo.getJsonArray("facets").getValuesAs(JsonObject.class);
Expand All @@ -1582,7 +1583,7 @@ public SearchResult facetDocs(JsonObject jo, Class<? extends EntityBaseBean> kla
}
}
}
return new SearchResult(lastSearchAfter, new ArrayList<>(), dimensions);
return new SearchResult(dimensions);
}

/**
Expand Down
5 changes: 2 additions & 3 deletions src/main/java/org/icatproject/core/manager/search/FacetDimension.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.icatproject.core.manager.search;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

/**
Expand All @@ -23,9 +24,7 @@ public FacetDimension(String target, String dimension) {
public FacetDimension(String target, String dimension, FacetLabel... labels) {
this.target = target;
this.dimension = dimension;
for (FacetLabel label : labels) {
facets.add(label);
}
Collections.addAll(facets, labels);
}

public List<FacetLabel> getFacets() {
Expand Down
9 changes: 1 addition & 8 deletions src/main/java/org/icatproject/core/manager/search/FacetLabel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,7 @@ public FacetLabel(String label, Long value) {
}

public FacetLabel(JsonObject jsonObject) {
label = jsonObject.getString("key");
value = jsonObject.getJsonNumber("doc_count").longValueExact();
if (jsonObject.containsKey("from")) {
from = jsonObject.getJsonNumber("from");
}
if (jsonObject.containsKey("to")) {
to = jsonObject.getJsonNumber("to");
}
this(jsonObject.getString("key"), jsonObject);
}

public FacetLabel(String label, JsonObject jsonObject) {
Expand Down
14 changes: 3 additions & 11 deletions src/main/java/org/icatproject/core/manager/search/SearchManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,11 +538,7 @@ public static JsonObject buildFacetQuery(List<ScoredEntityBaseBean> results, Str
JsonArrayBuilder arrayBuilder = Json.createArrayBuilder();
results.forEach(r -> arrayBuilder.add(Long.toString(r.getEntityBaseBeanId())));
JsonObject terms = Json.createObjectBuilder().add(idField, arrayBuilder.build()).build();
JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms);
if (facetJson.containsKey("dimensions")) {
objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions"));
}
return objectBuilder.build();
return buildFacetQuery(terms, facetJson);
}

/**
Expand Down Expand Up @@ -570,11 +566,7 @@ public static JsonObject buildSampleFacetQuery(List<ScoredEntityBaseBean> result
}
});
JsonObject terms = Json.createObjectBuilder().add("sample.id", arrayBuilder.build()).build();
JsonObjectBuilder objectBuilder = Json.createObjectBuilder().add("query", terms);
if (facetJson.containsKey("dimensions")) {
objectBuilder.add("dimensions", facetJson.getJsonArray("dimensions"));
}
return objectBuilder.build();
return buildFacetQuery(terms, facetJson);
}

/**
Expand Down Expand Up @@ -605,7 +597,7 @@ public static JsonObject buildFacetQuery(JsonObject filterObject, JsonObject fac
private static List<String> buildPublicSearchFields(GateKeeper gateKeeper, Map<String, Relationship[]> map) {
List<String> fields = new ArrayList<>();
for (Entry<String, Relationship[]> entry : map.entrySet()) {
Boolean includeField = true;
boolean includeField = true;
if (entry.getValue() != null) {
for (Relationship relationship : entry.getValue()) {
if (!gateKeeper.allowed(relationship)) {
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/icatproject/core/manager/search/SearchResult.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ public class SearchResult {
public SearchResult() {
}

public SearchResult(List<FacetDimension> dimensions) {
this.dimensions = dimensions;
}

public SearchResult(JsonValue searchAfter, List<ScoredEntityBaseBean> results, List<FacetDimension> dimensions) {
this.searchAfter = searchAfter;
this.results = results;
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/icatproject/exposed/ICATRest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1091,7 +1091,7 @@ public String lucene(@Context HttpServletRequest request, @QueryParam("sessionId
throw new IcatException(IcatExceptionType.BAD_PARAMETER, "target:" + target + " is not expected");
}
logger.debug("Free text search with query: {}", jo.toString());
objects = beanManager.freeTextSearch(userName, jo, maxCount, null, manager, request.getRemoteAddr(), klass);
objects = beanManager.freeTextSearch(userName, jo, maxCount, manager, request.getRemoteAddr(), klass);
JsonGenerator gen = Json.createGenerator(baos);
gen.writeStartArray();
for (ScoredEntityBaseBean sb : objects) {
Expand Down

0 comments on commit 60f30a6

Please sign in to comment.