Skip to content

Commit

Permalink
Integration test fixes and text analysis #267
Browse files Browse the repository at this point in the history
  • Loading branch information
@patrick-austin
patrick-austin committed Apr 28, 2022
1 parent 7eeb840 commit 9450c4b
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 35 deletions.
47 changes: 32 additions & 15 deletions src/main/java/org/icatproject/core/manager/SearchApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,20 @@ public ParentRelationship(String parentName, String joinName, List<String> field
protected static String basePath = "";
protected static JsonObject matchAllQuery = Json.createObjectBuilder().add("query", Json.createObjectBuilder()
.add("match_all", Json.createObjectBuilder())).build();
// TODO synonym filter in the default_search ONLY
private static JsonObject indexSettings = Json.createObjectBuilder().add("analysis", Json.createObjectBuilder()
.add("analyzer", Json.createObjectBuilder().add("default", Json.createObjectBuilder()
.add("tokenizer", "lowercase").add("filter", Json.createArrayBuilder().add("porter_stem"))))).build();
// private static JsonObject mappingsBuilder;
.add("analyzer", Json.createObjectBuilder()
.add("default", Json.createObjectBuilder()
.add("tokenizer", "classic").add("filter", Json.createArrayBuilder()
.add("possessive_english").add("lowercase").add("porter_stem")))
.add("default_search", Json.createObjectBuilder()
.add("tokenizer", "classic").add("filter", Json.createArrayBuilder()
.add("possessive_english").add("lowercase").add("porter_stem").add("synonym"))))
.add("filter", Json.createObjectBuilder()
.add("synonym", Json.createObjectBuilder()
.add("type", "synonym").add("synonyms_path", "synonym.txt"))
.add("possessive_english", Json.createObjectBuilder()
.add("type", "stemmer").add("langauge", "possessive_english")))).build();
protected static Set<String> indices = new HashSet<>();
protected static Map<String, String> scripts = new HashMap<>();
private static Map<String, List<ParentRelationship>> relationships = new HashMap<>();
Expand Down Expand Up @@ -263,10 +273,14 @@ private static JsonObject buildNestedQuery(String path, JsonObject... queryObjec
return Json.createObjectBuilder().add("nested", nestedBuilder).build();
}

private static JsonObject buildStringQuery(String field, String value) {
private static JsonObject buildStringQuery(String value, String... fields) {
JsonObjectBuilder queryStringBuilder = Json.createObjectBuilder().add("query", value);
if (field != null) {
queryStringBuilder.add("fields", Json.createArrayBuilder().add(field));
if (fields.length > 0) {
JsonArrayBuilder fieldsBuilder = Json.createArrayBuilder();
for (String field : fields) {
fieldsBuilder.add(field);
}
queryStringBuilder.add("fields", fieldsBuilder);
}
return Json.createObjectBuilder().add("query_string", queryStringBuilder).build();
}
Expand Down Expand Up @@ -447,9 +461,10 @@ public JsonValue buildSearchAfter(ScoredEntityBaseBean lastBean, String sort) th
throw new IcatException(IcatExceptionType.INTERNAL,
"Cannot build searchAfter document from source as sorted field " + key + " missing.");
}
String value = lastBean.getSource().getString(key);
JsonValue value = lastBean.getSource().get(key);
builder.add(value);
}
builder.add(lastBean.getEntityBaseBeanId());
return builder.build();
}
} else {
Expand Down Expand Up @@ -632,6 +647,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer
JsonObject jsonObject = jsonReader.readObject();
JsonArray hits = jsonObject.getJsonObject("hits").getJsonArray("hits");
for (JsonObject hit : hits.getValuesAs(JsonObject.class)) {
logger.trace("Hit: {}", hit.toString());
Float score = Float.NaN;
if (!hit.isNull("_score")) {
score = hit.getJsonNumber("_score").bigDecimalValue().floatValue();
Expand Down Expand Up @@ -660,7 +676,7 @@ public SearchResult getResults(JsonObject query, JsonValue searchAfter, Integer
private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, String sort, String index,
Set<String> queryFields) throws IcatException, ParseException {
JsonObjectBuilder builder = Json.createObjectBuilder();
if (sort == null) {
if (sort == null || sort.equals("")) {
builder.add("sort", Json.createArrayBuilder()
.add(Json.createObjectBuilder().add("_score", "desc"))
.add(Json.createObjectBuilder().add("id", "asc")).build());
Expand All @@ -685,11 +701,11 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St
JsonArrayBuilder filterBuilder = Json.createArrayBuilder();
if (queryFields.contains("text")) {
JsonArrayBuilder mustBuilder = Json.createArrayBuilder();
mustBuilder.add(buildStringQuery(null, query.getString("text")));
mustBuilder.add(buildStringQuery(query.getString("text")));
boolBuilder.add("must", mustBuilder);
}
Long lowerTime = parseDate(query, "lower", 0, Long.MIN_VALUE);
Long upperTime = parseDate(query, "upper", 0, Long.MAX_VALUE);
Long upperTime = parseDate(query, "upper", 59999, Long.MAX_VALUE);
if (lowerTime != Long.MIN_VALUE || upperTime != Long.MAX_VALUE) {
if (index.equals("datafile")) {
// datafile has only one date field
Expand All @@ -705,7 +721,7 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St
investigationUserQueries.add(buildMatchQuery("investigationuser.user.name", query.getString("user")));
}
if (queryFields.contains("userFullName")) {
investigationUserQueries.add(buildStringQuery("investigationuser.user.fullName", query.getString("userFullName")));
investigationUserQueries.add(buildStringQuery(query.getString("userFullName"), "investigationuser.user.fullName"));
}
if (investigationUserQueries.size() > 0) {
filterBuilder.add(buildNestedQuery("investigationuser", investigationUserQueries.toArray(new JsonObject[0])));
Expand All @@ -715,7 +731,7 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St
JsonArray samples = query.getJsonArray("samples");
for (int i = 0; i < samples.size(); i++) {
String sample = samples.getString(i);
filterBuilder.add(buildNestedQuery("sample", buildStringQuery("sample.name", sample)));
filterBuilder.add(buildNestedQuery("sample", buildStringQuery(sample, "sample.name", "sample.type.name")));
}
}
if (queryFields.contains("parameters")) {
Expand All @@ -724,8 +740,8 @@ private JsonObjectBuilder parseQuery(JsonObject query, JsonValue searchAfter, St
String name = parameterObject.getString("name", null);
String units = parameterObject.getString("units", null);
String stringValue = parameterObject.getString("stringValue", null);
Long lowerDate = decodeTime(parameterObject.getString("lowerDateValue", null));
Long upperDate = decodeTime(parameterObject.getString("upperDateValue", null));
Long lowerDate = parseDate(parameterObject, "lowerDateValue", 0, null);
Long upperDate = parseDate(parameterObject, "upperDateValue", 59999, null);
JsonNumber lowerNumeric = parameterObject.getJsonNumber("lowerNumericValue");
JsonNumber upperNumeric = parameterObject.getJsonNumber("upperNumericValue");

Expand Down Expand Up @@ -770,6 +786,7 @@ public void initMappings() throws IcatException {
// If the index isn't present, we should get 404 and create the index
if (statusCode == 200) {
// If the index already exists (200), do not attempt to create it
logger.trace("{} index already exists, continue", index);
continue;
} else if (statusCode != 404) {
// If the code isn't 200 or 404, something has gone wrong
Expand All @@ -782,10 +799,10 @@ public void initMappings() throws IcatException {

try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
URI uri = new URIBuilder(server).setPath(basePath + "/" + index).build();
logger.trace("Making call {}", uri);
HttpPut httpPut = new HttpPut(uri);
String body = Json.createObjectBuilder()
.add("settings", indexSettings).add("mappings", buildMappings(index)).build().toString();
logger.trace("Making call {} with body {}", uri, body);
httpPut.setEntity(new StringEntity(body, ContentType.APPLICATION_JSON));
try (CloseableHttpResponse response = httpclient.execute(httpPut)) {
Rest.checkStatus(response, IcatExceptionType.INTERNAL);
Expand Down
50 changes: 30 additions & 20 deletions src/test/java/org/icatproject/integration/TestRS.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,11 +552,16 @@ public void testLuceneDatasets() throws Exception {

// Try parameters
List<ParameterForLucene> parameters = new ArrayList<>();
parameters.add(new ParameterForLucene("colour", "name", "green"));
parameters.add(new ParameterForLucene("birthday", "date", dft.parse("2014-05-16T16:58:26+0000"),
dft.parse("2014-05-16T16:58:26+0000")));
parameters.add(new ParameterForLucene("current", "amps", 140, 165));

ParameterForLucene stringParameter = new ParameterForLucene("colour", "name", "green");
ParameterForLucene dateParameter = new ParameterForLucene("birthday", "date",
dft.parse("2014-05-16T16:58:26+0000"), dft.parse("2014-05-16T16:58:26+0000"));
ParameterForLucene numericParameter = new ParameterForLucene("current", "amps", 140, 165);
array = searchDatasets(session, null, null, null, null, Arrays.asList(stringParameter), 20, 1);
array = searchDatasets(session, null, null, null, null, Arrays.asList(dateParameter), 20, 1);
array = searchDatasets(session, null, null, null, null, Arrays.asList(numericParameter), 20, 1);
parameters.add(stringParameter);
parameters.add(dateParameter);
parameters.add(numericParameter);
array = searchDatasets(session, null, null, null, null, parameters, 20, 1);

array = searchDatasets(session, null, "gamma AND ds3", dft.parse("2014-05-16T05:09:03+0000"),
Expand Down Expand Up @@ -716,7 +721,8 @@ public void testSearchDatafiles() throws Exception {
responseObject.containsKey("dimensions"));

// Test no facets match on DatafileParameters due to lack of READ access
target = Json.createObjectBuilder().add("target", "DatafileParameter");
target = Json.createObjectBuilder()
.add("target", "DatafileParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name")));
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3);
assertFalse(responseObject.containsKey("search_after"));
Expand All @@ -725,8 +731,6 @@ public void testSearchDatafiles() throws Exception {

// Test facets match on DatafileParameters
wSession.addRule(null, "DatafileParameter", "R");
target = Json.createObjectBuilder().add("target", "DatafileParameter");
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchDatafiles(session, null, null, null, null, null, null, 10, null, facets, 3);
assertFalse(responseObject.containsKey("search_after"));
checkFacets(responseObject, "DatafileParameter.type.name", Arrays.asList("colour"), Arrays.asList(1L));
Expand Down Expand Up @@ -774,9 +778,16 @@ public void testSearchDatasets() throws Exception {
List<ParameterForLucene> parameters = new ArrayList<>();
Date parameterDate = dft.parse("2014-05-16T16:58:26+0000");
parameters.add(new ParameterForLucene("colour", "name", "green"));
responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1);
assertFalse(responseObject.containsKey("search_after"));
checkResultsSource(responseObject, Arrays.asList(expectation), true);

parameters.add(new ParameterForLucene("birthday", "date", parameterDate, parameterDate));
parameters.add(new ParameterForLucene("current", "amps", 140, 165));
responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1);
assertFalse(responseObject.containsKey("search_after"));
checkResultsSource(responseObject, Arrays.asList(expectation), true);

parameters.add(new ParameterForLucene("current", "amps", 140, 165));
responseObject = searchDatasets(session, null, null, null, null, parameters, null, 10, null, null, 1);
assertFalse(responseObject.containsKey("search_after"));
checkResultsSource(responseObject, Arrays.asList(expectation), true);
Expand Down Expand Up @@ -837,15 +848,17 @@ public void testSearchDatasets() throws Exception {
Session piSession = icat.login("db", credentials);
searchDatasets(piSession, null, null, null, null, null, null, 10, null, null, 0);

// Test no facets match on Datasets
JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Dataset");
// Test facets match on Datasets
JsonObjectBuilder target = Json.createObjectBuilder()
.add("target", "Dataset").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name")));
String facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5);
assertFalse(responseObject.containsKey("search_after"));
checkFacets(responseObject, "Dataset.type.name", Arrays.asList("calibration"), Arrays.asList(5L));

// Test no facets match on DatasetParameters due to lack of READ access
target = Json.createObjectBuilder().add("target", "DatasetParameter");
target = Json.createObjectBuilder()
.add("target", "DatasetParameter").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name")));
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5);
assertFalse(responseObject.containsKey("search_after"));
Expand All @@ -854,8 +867,6 @@ public void testSearchDatasets() throws Exception {

// Test facets match on DatasetParameters
wSession.addRule(null, "DatasetParameter", "R");
target = Json.createObjectBuilder().add("target", "DatasetParameter");
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchDatasets(session, null, null, null, null, null, null, 10, null, facets, 5);
assertFalse(responseObject.containsKey("search_after"));
checkFacets(responseObject, "DatasetParameter.type.name", Arrays.asList("colour", "birthday", "current"),
Expand Down Expand Up @@ -891,7 +902,6 @@ public void testSearchInvestigations() throws Exception {

List<ParameterForLucene> parameters = new ArrayList<>();
parameters.add(new ParameterForLucene("colour", "name", "green"));
// TODO remove additional checks here
responseObject = searchInvestigations(session, "db/tr", null, lowerOrigin, upperOrigin, null,
null, null, null, 10, null, null, 1);
responseObject = searchInvestigations(session, "db/tr", textAnd, lowerOrigin, upperOrigin, null,
Expand Down Expand Up @@ -987,16 +997,18 @@ public void testSearchInvestigations() throws Exception {
Session piSession = icat.login("db", credentials);
searchInvestigations(piSession, null, null, null, null, null, null, null, null, 10, null, null, 0);

// Test no facets match on Investigations
JsonObjectBuilder target = Json.createObjectBuilder().add("target", "Investigation");
// Test facets match on Investigations
JsonObjectBuilder target = Json.createObjectBuilder()
.add("target", "Investigation").add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name")));
String facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets,
3);
assertFalse(responseObject.containsKey("search_after"));
checkFacets(responseObject, "Investigation.type.name", Arrays.asList("atype"), Arrays.asList(3L));

// Test no facets match on InvestigationParameters due to lack of READ access
target = Json.createObjectBuilder().add("target", "InvestigationParameter");
target = Json.createObjectBuilder().add("target", "InvestigationParameter")
.add("dimensions", Json.createArrayBuilder().add(Json.createObjectBuilder().add("dimension", "type.name")));
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets,
3);
Expand All @@ -1006,8 +1018,6 @@ public void testSearchInvestigations() throws Exception {

// Test facets match on InvestigationParameters
wSession.addRule(null, "InvestigationParameter", "R");
target = Json.createObjectBuilder().add("target", "InvestigationParameter");
facets = Json.createArrayBuilder().add(target).build().toString();
responseObject = searchInvestigations(session, null, null, null, null, null, null, null, null, 10, null, facets,
3);
assertFalse(responseObject.containsKey("search_after"));
Expand Down

0 comments on commit 9450c4b

Please sign in to comment.