Implement Elasticsearch Java client functions #267

icatproject · Mar 9, 2022 · 44a9d80 · 44a9d80
1 parent 8f369fc
commit 44a9d80
Show file tree

Hide file tree

Showing 30 changed files with 2,043 additions and 771 deletions.
diff --git a/pom.xml b/pom.xml
@@ -124,7 +124,7 @@
 		<dependency>
 			<groupId>org.apache.httpcomponents</groupId>
 			<artifactId>httpclient</artifactId>
-			<version>4.3.6</version>
+			<version>4.5.13</version>
 		</dependency>
 
 		<dependency>
@@ -133,6 +133,17 @@
 			<version>4.3.4</version>
 		</dependency>
 
+		<dependency>
+			<groupId>co.elastic.clients</groupId>
+			<artifactId>elasticsearch-java</artifactId>
+			<version>7.16.3</version>
+		  </dependency>
+		  <dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>2.12.3</version>
+		  </dependency>
+
 	</dependencies>
 
 	<build>
@@ -225,6 +236,7 @@
 					<systemPropertyVariables>
 						<javax.net.ssl.trustStore>${javax.net.ssl.trustStore}</javax.net.ssl.trustStore>
 						<luceneUrl>${luceneUrl}</luceneUrl>
+						<searchUrls>${searchUrls}</searchUrls>
 					</systemPropertyVariables>
 					<testFailureIgnore>false</testFailureIgnore>
 				</configuration>
@@ -243,6 +255,7 @@
 						<javax.net.ssl.trustStore>${javax.net.ssl.trustStore}</javax.net.ssl.trustStore>
 						<serverUrl>${serverUrl}</serverUrl>
 						<luceneUrl>${luceneUrl}</luceneUrl>
+						<searchUrls>${searchUrls}</searchUrls>
 					</systemPropertyVariables>
 				</configuration>
 				<executions>
@@ -322,6 +335,7 @@
 								<argument>${containerHome}</argument>
 								<argument>${serverUrl}</argument>
 								<argument>${luceneUrl}</argument>
+								<argument>${searchUrls}</argument>
 							</arguments>
 						</configuration>
 						<goals>
@@ -399,6 +413,3 @@
 	</reporting>
 
 </project>
-
-
-
diff --git a/src/main/java/org/icatproject/core/entity/Datafile.java b/src/main/java/org/icatproject/core/entity/Datafile.java
@@ -20,7 +20,7 @@
 import javax.persistence.UniqueConstraint;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("A data file")
 @SuppressWarnings("serial")
@@ -194,7 +194,7 @@ public void setSourceDatafiles(List<RelatedDatafile> sourceDatafiles) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
 		StringBuilder sb = new StringBuilder(name);
 		if (description != null) {
 			sb.append(" " + description);
@@ -205,15 +205,17 @@ public void getDoc(JsonGenerator gen) {
 		if (datafileFormat != null) {
 			sb.append(" " + datafileFormat.getName());
 		}
-		LuceneApi.encodeTextfield(gen, "text", sb.toString());
+		searchApi.encodeTextField(gen, "text", sb.toString());
 		if (datafileModTime != null) {
-			LuceneApi.encodeStringField(gen, "date", datafileModTime);
+			searchApi.encodeStringField(gen, "date", datafileModTime);
 		} else if (datafileCreateTime != null) {
-			LuceneApi.encodeStringField(gen, "date", datafileCreateTime);
+			searchApi.encodeStringField(gen, "date", datafileCreateTime);
 		} else {
-			LuceneApi.encodeStringField(gen, "date", modTime);
+			searchApi.encodeStringField(gen, "date", modTime);
 		}
-		LuceneApi.encodeStoredId(gen, id);
-		LuceneApi.encodeStringField(gen, "dataset", dataset.id);
+		searchApi.encodeStoredId(gen, id);
+		searchApi.encodeStringField(gen, "dataset", dataset.id);
+
+		// TODO User and Parameter support for Elasticsearch
 	}
 }
diff --git a/src/main/java/org/icatproject/core/entity/DatafileParameter.java b/src/main/java/org/icatproject/core/entity/DatafileParameter.java
@@ -14,7 +14,7 @@
 import org.icatproject.core.IcatException;
 import org.icatproject.core.manager.EntityBeanManager.PersistMode;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("A parameter associated with a data file")
 @SuppressWarnings("serial")
@@ -54,9 +54,9 @@ public void setDatafile(Datafile datafile) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
-		super.getDoc(gen);
-		LuceneApi.encodeSortedDocValuesField(gen, "datafile", datafile.id);
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
+		super.getDoc(gen, searchApi);
+		searchApi.encodeSortedDocValuesField(gen, "datafile", datafile.id);
 	}
 
 }
diff --git a/src/main/java/org/icatproject/core/entity/Dataset.java b/src/main/java/org/icatproject/core/entity/Dataset.java
@@ -19,7 +19,7 @@
 import javax.persistence.UniqueConstraint;
 import javax.xml.bind.annotation.XmlRootElement;
 
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("A collection of data files and part of an investigation")
 @SuppressWarnings("serial")
@@ -182,7 +182,7 @@ public void setType(DatasetType type) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
 
 		StringBuilder sb = new StringBuilder(name + " " + type.getName() + " " + type.getName());
 		if (description != null) {
@@ -200,25 +200,26 @@ public void getDoc(JsonGenerator gen) {
 			}
 		}
 
-		LuceneApi.encodeTextfield(gen, "text", sb.toString());
+		searchApi.encodeTextField(gen, "text", sb.toString());
 
 		if (startDate != null) {
-			LuceneApi.encodeStringField(gen, "startDate", startDate);
+			searchApi.encodeStringField(gen, "startDate", startDate);
 		} else {
-			LuceneApi.encodeStringField(gen, "startDate", createTime);
+			searchApi.encodeStringField(gen, "startDate", createTime);
 		}
 
 		if (endDate != null) {
-			LuceneApi.encodeStringField(gen, "endDate", endDate);
+			searchApi.encodeStringField(gen, "endDate", endDate);
 		} else {
-			LuceneApi.encodeStringField(gen, "endDate", modTime);
+			searchApi.encodeStringField(gen, "endDate", modTime);
 		}
-		LuceneApi.encodeStoredId(gen, id);
+		searchApi.encodeStoredId(gen, id);
 
-		LuceneApi.encodeSortedDocValuesField(gen, "id", id);
+		searchApi.encodeSortedDocValuesField(gen, "id", id);
 
-		LuceneApi.encodeStringField(gen, "investigation", investigation.id);
+		searchApi.encodeStringField(gen, "investigation", investigation.id);
 
+		// TODO User, Parameter and Sample support for Elasticsearch
 	}
 
 }
diff --git a/src/main/java/org/icatproject/core/entity/DatasetParameter.java b/src/main/java/org/icatproject/core/entity/DatasetParameter.java
@@ -14,7 +14,7 @@
 import org.icatproject.core.IcatException;
 import org.icatproject.core.manager.EntityBeanManager.PersistMode;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("A parameter associated with a data set")
 @SuppressWarnings("serial")
@@ -54,8 +54,8 @@ public void setDataset(Dataset dataset) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
-		super.getDoc(gen);
-		LuceneApi.encodeSortedDocValuesField(gen, "dataset", dataset.id);
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
+		super.getDoc(gen, searchApi);
+		searchApi.encodeSortedDocValuesField(gen, "dataset", dataset.id);
 	}
 }
diff --git a/src/main/java/org/icatproject/core/entity/EntityBaseBean.java b/src/main/java/org/icatproject/core/entity/EntityBaseBean.java
@@ -31,7 +31,8 @@
 import org.icatproject.core.manager.EntityInfoHandler;
 import org.icatproject.core.manager.EntityInfoHandler.Relationship;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneManager;
+import org.icatproject.core.manager.SearchApi;
+import org.icatproject.core.manager.SearchManager;
 import org.icatproject.core.parser.IncludeClause.Step;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -81,8 +82,8 @@ void addToClone(EntityBaseBean clone) {
 
 	// This is only used by the older create and createMany calls and not by the
 	// new Restful write call
-	public void addToLucene(LuceneManager lucene) throws IcatException {
-		lucene.addDocument(this);
+	public void addToSearch(SearchManager searchManager) throws IcatException {
+		searchManager.addDocument(this);
 		Class<? extends EntityBaseBean> klass = this.getClass();
 		Set<Relationship> rs = eiHandler.getRelatedEntities(klass);
 		Map<Field, Method> getters = eiHandler.getGetters(klass);
@@ -94,7 +95,7 @@ public void addToLucene(LuceneManager lucene) throws IcatException {
 					List<EntityBaseBean> collection = (List<EntityBaseBean>) m.invoke(this);
 					if (!collection.isEmpty()) {
 						for (EntityBaseBean bean : collection) {
-							bean.addToLucene(lucene);
+							bean.addToSearch(searchManager);
 						}
 					}
 				} catch (Exception e) {
@@ -434,8 +435,8 @@ public String toString() {
 		return this.getClass().getSimpleName() + ":" + id;
 	}
 
-	/* This should be overridden by classes wishing to index things in lucene */
-	public void getDoc(JsonGenerator gen) {
+	/* This should be overridden by classes wishing to index things in a search engine */
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
 	}
 
 }
diff --git a/src/main/java/org/icatproject/core/entity/Investigation.java b/src/main/java/org/icatproject/core/entity/Investigation.java
@@ -18,7 +18,7 @@
 import javax.persistence.TemporalType;
 import javax.persistence.UniqueConstraint;
 
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("An investigation or experiment")
 @SuppressWarnings("serial")
@@ -258,7 +258,7 @@ public void setVisitId(String visitId) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
 		StringBuilder sb = new StringBuilder(visitId + " " + name + " " + facility.getName() + " " + type.getName());
 		if (summary != null) {
 			sb.append(" " + summary);
@@ -269,22 +269,49 @@ public void getDoc(JsonGenerator gen) {
 		if (title != null) {
 			sb.append(" " + title);
 		}
-		LuceneApi.encodeTextfield(gen, "text", sb.toString());
+		searchApi.encodeTextField(gen, "text", sb.toString());
 
 		if (startDate != null) {
-			LuceneApi.encodeStringField(gen, "startDate", startDate);
+			searchApi.encodeStringField(gen, "startDate", startDate);
 		} else {
-			LuceneApi.encodeStringField(gen, "startDate", createTime);
+			searchApi.encodeStringField(gen, "startDate", createTime);
 		}
 
 		if (endDate != null) {
-			LuceneApi.encodeStringField(gen, "endDate", endDate);
+			searchApi.encodeStringField(gen, "endDate", endDate);
 		} else {
-			LuceneApi.encodeStringField(gen, "endDate", modTime);
+			searchApi.encodeStringField(gen, "endDate", modTime);
 		}
 
-		LuceneApi.encodeSortedDocValuesField(gen, "id", id);
+		investigationUsers.forEach((investigationUser) -> {
+				searchApi.encodeStringField(gen, "userName", investigationUser.getUser().getName());
+				searchApi.encodeTextField(gen, "userFullName", investigationUser.getUser().getFullName());
+		});
+
+
+		samples.forEach((sample) -> {
+				searchApi.encodeSortedSetDocValuesFacetField(gen, "sampleName", sample.getName());
+				searchApi.encodeTextField(gen, "sampleText", sample.getDocText());
+		});
+
+		for (InvestigationParameter parameter : parameters) {
+			ParameterType type = parameter.type;
+			String parameterName = type.getName();
+			String parameterUnits = type.getUnits();
+			searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterName", parameterName);
+			searchApi.encodeStringField(gen, "parameterUnits", parameterUnits);
+			// TODO make all value types facetable...
+			if (type.getValueType() == ParameterValueType.STRING) {
+				searchApi.encodeSortedSetDocValuesFacetField(gen, "parameterStringValue", parameter.getStringValue());
+			} else if (type.getValueType() == ParameterValueType.DATE_AND_TIME) {
+				searchApi.encodeStringField(gen, "parameterDateValue", parameter.getDateTimeValue());
+			} else if (type.getValueType() == ParameterValueType.NUMERIC) {
+				searchApi.encodeDoublePoint(gen, "parameterNumericValue", parameter.getNumericValue());
+			}
+		}
+
+		searchApi.encodeSortedDocValuesField(gen, "id", id);
 
-		LuceneApi.encodeStoredId(gen, id);
+		searchApi.encodeStoredId(gen, id);
 	}
 }
diff --git a/src/main/java/org/icatproject/core/entity/InvestigationParameter.java b/src/main/java/org/icatproject/core/entity/InvestigationParameter.java
@@ -14,7 +14,7 @@
 import org.icatproject.core.IcatException;
 import org.icatproject.core.manager.EntityBeanManager.PersistMode;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("A parameter associated with an investigation")
 @SuppressWarnings("serial")
@@ -55,8 +55,8 @@ public void setInvestigation(Investigation investigation) {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
-		super.getDoc(gen);
-		LuceneApi.encodeSortedDocValuesField(gen, "investigation", investigation.id);
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
+		super.getDoc(gen, searchApi);
+		searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id);
 	}
 }
diff --git a/src/main/java/org/icatproject/core/entity/InvestigationUser.java b/src/main/java/org/icatproject/core/entity/InvestigationUser.java
@@ -10,7 +10,7 @@
 import javax.persistence.Table;
 import javax.persistence.UniqueConstraint;
 
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 
 @Comment("Many to many relationship between investigation and user. It is expected that this will show the association of "
 		+ "individual users with an investigation which might be derived from the proposal. It may also be used as the "
@@ -38,12 +38,12 @@ public InvestigationUser() {
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
 		if (user.getFullName() != null) {
-			LuceneApi.encodeTextfield(gen, "text", user.getFullName());
+			searchApi.encodeTextField(gen, "text", user.getFullName());
 		}
-		LuceneApi.encodeStringField(gen, "name", user.getName());
-		LuceneApi.encodeSortedDocValuesField(gen, "investigation", investigation.id);
+		searchApi.encodeStringField(gen, "name", user.getName());
+		searchApi.encodeSortedDocValuesField(gen, "investigation", investigation.id);
 	}
 
 	public String getRole() {

diff --git a/src/main/java/org/icatproject/core/entity/Parameter.java b/src/main/java/org/icatproject/core/entity/Parameter.java
@@ -17,7 +17,7 @@
 import org.icatproject.core.IcatException;
 import org.icatproject.core.manager.EntityBeanManager.PersistMode;
 import org.icatproject.core.manager.GateKeeper;
-import org.icatproject.core.manager.LuceneApi;
+import org.icatproject.core.manager.SearchApi;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -162,15 +162,15 @@ public void postMergeFixup(EntityManager manager, GateKeeper gateKeeper) throws
 	}
 
 	@Override
-	public void getDoc(JsonGenerator gen) {
-		LuceneApi.encodeStringField(gen, "name", type.getName());
-		LuceneApi.encodeStringField(gen, "units", type.getUnits());
+	public void getDoc(JsonGenerator gen, SearchApi searchApi) {
+		searchApi.encodeStringField(gen, "name", type.getName());
+		searchApi.encodeStringField(gen, "units", type.getUnits());
 		if (stringValue != null) {
-			LuceneApi.encodeStringField(gen, "stringValue", stringValue);
+			searchApi.encodeStringField(gen, "stringValue", stringValue);
 		} else if (numericValue != null) {
-			LuceneApi.encodeDoublePoint(gen, "numericValue", numericValue);
+			searchApi.encodeDoublePoint(gen, "numericValue", numericValue);
 		} else if (dateTimeValue != null) {
-			LuceneApi.encodeStringField(gen, "dateTimeValue", dateTimeValue);
+			searchApi.encodeStringField(gen, "dateTimeValue", dateTimeValue);
 		}
 	}