diff --git a/pom.xml b/pom.xml index e2d6607e40..2bf6950af2 100644 --- a/pom.xml +++ b/pom.xml @@ -293,16 +293,19 @@ org.testcontainers testcontainers + 2.0.2 test org.testcontainers - elasticsearch + testcontainers-elasticsearch + 2.0.2 test org.testcontainers - junit-jupiter + testcontainers-junit-jupiter + 2.0.2 test diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java index 1c1d2c7253..1e3bc6a8e2 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java @@ -50,6 +50,7 @@ import ca.uhn.fhir.jpa.starter.annotations.OnCorsPresent; import ca.uhn.fhir.jpa.starter.annotations.OnImplementationGuidesPresent; import ca.uhn.fhir.jpa.starter.common.validation.IRepositoryValidationInterceptorFactory; +import ca.uhn.fhir.jpa.starter.elastic.ElasticsearchBootSvcImpl; import ca.uhn.fhir.jpa.starter.ig.ExtendedPackageInstallationSpec; import ca.uhn.fhir.jpa.starter.ig.IImplementationGuideOperationProvider; import ca.uhn.fhir.jpa.subscription.util.SubscriptionDebugLogInterceptor; @@ -149,6 +150,7 @@ public ResourceCountCache resourceCountsCache(IFhirSystemDao theSystemDao) @Primary @Bean public LocalContainerEntityManagerFactoryBean entityManagerFactory( + Optional elasticsearchSvc, JpaProperties theJpaProperties, DataSource myDataSource, ConfigurableListableBeanFactory myConfigurableListableBeanFactory, diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/elastic/ElasticsearchBootSvcImpl.java b/src/main/java/ca/uhn/fhir/jpa/starter/elastic/ElasticsearchBootSvcImpl.java index 82d8511330..7cf220ada6 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/elastic/ElasticsearchBootSvcImpl.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/elastic/ElasticsearchBootSvcImpl.java @@ -49,14 +49,14 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc { private static final String OBSERVATION_RESOURCE_NAME = "Observation"; - private final ElasticsearchClient myRestHighLevelClient; + private final ElasticsearchClient elasticsearchClient; private final FhirContext myContext; public ElasticsearchBootSvcImpl(ElasticsearchClient client, FhirContext fhirContext) { myContext = fhirContext; - myRestHighLevelClient = client; + elasticsearchClient = client; try { createObservationIndexIfMissing(); @@ -100,7 +100,7 @@ private void createObservationCodeIndexIfMissing() throws IOException { } private boolean createIndex(String theIndexName, String theMapping) throws IOException { - return myRestHighLevelClient + return elasticsearchClient .indices() .create(cir -> cir.index(theIndexName).withJson(new StringReader(theMapping))) .acknowledged(); @@ -108,7 +108,7 @@ private boolean createIndex(String theIndexName, String theMapping) throws IOExc private boolean indexExists(String theIndexName) throws IOException { ExistsRequest request = new ExistsRequest.Builder().index(theIndexName).build(); - return myRestHighLevelClient.indices().exists(request).value(); + return elasticsearchClient.indices().exists(request).value(); } @Override @@ -121,7 +121,7 @@ public List getObservationResources(Collection observationDocumentResponse = - myRestHighLevelClient.search(searchRequest, ObservationJson.class); + elasticsearchClient.search(searchRequest, ObservationJson.class); List> observationDocumentHits = observationDocumentResponse.hits().hits(); IParser parser = TolerantJsonParser.createWithLenientErrorHandling(myContext, null); @@ -158,6 +158,6 @@ private SearchRequest buildObservationResourceSearchRequest(Collection fn.index(theIndexName)); + elasticsearchClient.indices().refresh(fn -> fn.index(theIndexName)); } } diff --git a/src/main/resources/application-elastic.yaml b/src/main/resources/application-elastic.yaml new file mode 100644 index 0000000000..68b07beb00 --- /dev/null +++ b/src/main/resources/application-elastic.yaml @@ -0,0 +1,50 @@ +spring: + elasticsearch: + uris: http://localhost:9200 + username: elastic + password: elastic + + autoconfigure: + # This empty exclude is needed to override the default exclusion of the Elasticsearch configuration. + exclude: + + jpa: + properties: + hibernate: + # --- Hibernate Search (Lucene/Elasticsearch) --- + # Note: the following values should be kept in sync with ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder + search: + schema_management: + strategy: CREATE + enabled: true + backend: + layout: + strategy: ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy + type: elasticsearch + protocol: http + analysis: + configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticsearchAnalysisConfigurer + scroll_timeout: 60 + schema_management: + settings_file: ca/uhn/fhir/jpa/elastic/index-settings.json + minimal_required_status_wait_timeout: 10000 + minimal_required_status: YELLOW + + dynamic_mapping: true + indexing: + plan: + synchronization: + strategy: async + +# ------------------------------------------------------------------------------------- +# HAPI FHIR — grouped by domain +# ------------------------------------------------------------------------------------- +hapi: + fhir: + # ------------------------------------------------------------------------------- + # D. Search & Indexing + # ------------------------------------------------------------------------------- + # NOTE: Extended Lucene/Elasticsearch indexing is experimental. + # See https://hapifhir.io/hapi-fhir/docs/server_jpa/elastic.html + advanced_lucene_indexing: true + search_index_full_text_enabled: true diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 0b96494c46..bb571cf9a7 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -37,6 +37,10 @@ management: enabled: true spring: +# elasticsearch: +# uris: http://localhost:9200 +# username: elastic +# password: elastic # ------------------------------------------------------------------------------- # A. Spring AI — Model Context Protocol (MCP) # ------------------------------------------------------------------------------- @@ -123,8 +127,10 @@ spring: use_minimal_puts: false # --- Hibernate Search (Lucene/Elasticsearch) --- - search: - enabled: false + #search: + # schema_management: + # strategy: CREATE + # enabled: true # Lucene backend (default example) # backend: # type: lucene @@ -135,10 +141,25 @@ spring: # root: target/lucenefiles # lucene_version: lucene_current # Elasticsearch backend (alternative) — see also hapi.fhir.elasticsearch section in docs - # backend: - # type: elasticsearch - # analysis: - # configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticAnalysisConfigurer +# backend: +# layout: +# strategy: ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy +# type: elasticsearch +# protocol: http +# analysis: +# configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticsearchAnalysisConfigurer +# scroll_timeout: 60 +# schema_management: +# settings_file: ca/uhn/fhir/jpa/elastic/index-settings.json +# minimal_required_status_wait_timeout: 10000 +# minimal_required_status: YELLOW +# +# dynamic_mapping: true +# indexing: +# plan: +# synchronization: +# strategy: async + # ------------------------------------------------------------------------------------- # HAPI FHIR — grouped by domain @@ -242,8 +263,8 @@ hapi: # ------------------------------------------------------------------------------- # NOTE: Extended Lucene/Elasticsearch indexing is experimental. # See https://hapifhir.io/hapi-fhir/docs/server_jpa/elastic.html - advanced_lucene_indexing: false - search_index_full_text_enabled: false + # advanced_lucene_indexing: true + # search_index_full_text_enabled: true # language_search_parameter_enabled: true # upliftedRefchains_enabled: true # index_storage_optimized: false diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index 77d5d6b9aa..128183be16 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -9,12 +9,12 @@ import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; -import java.io.IOException; +import jakarta.annotation.PreDestroy; +import java.io.IOException; import java.util.Date; import java.util.GregorianCalendar; -import jakarta.annotation.PreDestroy; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.DateTimeType; @@ -23,19 +23,15 @@ import org.hl7.fhir.r4.model.Parameters; import org.hl7.fhir.r4.model.Patient; import org.hl7.fhir.r4.model.StringType; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.util.TestPropertyValues; import org.springframework.boot.test.web.server.LocalServerPort; -import org.springframework.context.ApplicationContextInitializer; -import org.springframework.context.ConfigurableApplicationContext; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.junit.jupiter.Container; @@ -43,126 +39,69 @@ @ExtendWith(SpringExtension.class) @Testcontainers -@Disabled -@ActiveProfiles("test") -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = {Application.class}, properties = - { - "spring.datasource.url=jdbc:h2:mem:dbr4", - "hapi.fhir.fhir_version=r4", - "hapi.fhir.lastn_enabled=true", - "hapi.fhir.store_resource_in_lucene_index_enabled=true", - "hapi.fhir.advanced_lucene_indexing=true", - "hapi.fhir.search_index_full_text_enabled=true", - "hapi.fhir.cr_enabled=false", - // Because the port is set randomly, we will set the rest_url using the Initializer. - // "elasticsearch.rest_url='http://localhost:9200'", - - "spring.elasticsearch.uris=http://localhost:9200", - "spring.elasticsearch.username=elastic", - "spring.elasticsearch.password=changeme", - "spring.main.allow-bean-definition-overriding=true", - "spring.jpa.properties.hibernate.search.enabled=true", - "spring.jpa.properties.hibernate.search.backend.type=elasticsearch", - "spring.jpa.properties.hibernate.search.backend.hosts=localhost:9200", - "spring.jpa.properties.hibernate.search.backend.protocol=http", - "spring.jpa.properties.hibernate.search.backend.analysis.configurer=ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticsearchAnalysisConfigurer" - }) -@ContextConfiguration(initializers = ElasticsearchLastNR4IT.Initializer.class) +@ActiveProfiles("test,elastic") +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = {Application.class}, properties = {"spring.datasource.url=jdbc:h2:mem:dbr4", + // Override the default exclude configuration for the Elasticsearch client. + "spring.autoconfigure.exclude=", "hapi.fhir.fhir_version=r4" +}) class ElasticsearchLastNR4IT { - private IGenericClient ourClient; - private FhirContext ourCtx; - - @Container - public static ElasticsearchContainer embeddedElastic = TestElasticsearchContainerHelper.getEmbeddedElasticSearch(); - - @Autowired - private ElasticsearchBootSvcImpl myElasticsearchSvc; - - @BeforeAll - public static void beforeClass() throws IOException { - //Given - // ElasticsearchClient elasticsearchHighLevelRestClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient( -// "http", embeddedElastic.getHost() + ":" + embeddedElastic.getMappedPort(9200), "", ""); - - /* As of 2023-08-10, HAPI FHIR sets SubscriptionConstants.MAX_SUBSCRIPTION_RESULTS to 50000 - which is in excess of elastic's default max_result_window. If MAX_SUBSCRIPTION_RESULTS is changed - to a value <= 10000, the following will no longer be necessary. - dotasek - */ - - /* elasticsearchHighLevelRestClient.indices().putTemplate(t->{ - t.name("hapi_fhir_template"); - t.indexPatterns("*"); - t.settings(new IndexSettings.Builder().maxResultWindow(50000).build()); - return t; - }); -*/ - } - - @PreDestroy - public void stop() { - embeddedElastic.stop(); - } - - @LocalServerPort - private int port; - - @Test - void testLastN() throws IOException, InterruptedException { - Thread.sleep(2000); - - Patient pt = new Patient(); - pt.addName().setFamily("Lastn").addGiven("Arthur"); - IIdType id = ourClient.create().resource(pt).execute().getId().toUnqualifiedVersionless(); - - Observation obs = new Observation(); - obs.getSubject().setReferenceElement(id); - String observationCode = "testobservationcode"; - String codeSystem = "http://testobservationcodesystem"; - - obs.getCode().addCoding().setCode(observationCode).setSystem(codeSystem); - obs.setValue(new StringType(observationCode)); - - Date effectiveDtm = new GregorianCalendar().getTime(); - obs.setEffective(new DateTimeType(effectiveDtm)); - obs.getCategoryFirstRep().addCoding().setCode("testcategorycode").setSystem("http://testcategorycodesystem"); - IIdType obsId = ourClient.create().resource(obs).execute().getId().toUnqualifiedVersionless(); - - myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX); - Thread.sleep(2000); - - Parameters output = ourClient.operation().onType(Observation.class).named("lastn") - .withParameter(Parameters.class, "max", new IntegerType(1)) - .andParameter("subject", new StringType("Patient/" + id.getIdPart())) - .execute(); - Bundle b = (Bundle) output.getParameter().get(0).getResource(); - assertEquals(1, b.getTotal()); - assertEquals(obsId, b.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless()); - } - - @BeforeEach - void beforeEach() throws IOException { - - ourCtx = FhirContext.forR4(); - ourCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); - ourCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000); - String ourServerBase = "http://localhost:" + port + "/fhir/"; - ourClient = ourCtx.newRestfulGenericClient(ourServerBase); - ourClient.registerInterceptor(new LoggingInterceptor(true)); - - } - - static class Initializer - implements ApplicationContextInitializer { - - @Override - public void initialize( - ConfigurableApplicationContext configurableApplicationContext) { - // Since the port is dynamically generated, replace the URL with one that has the correct port - TestPropertyValues.of("spring.elasticsearch.uris=" + embeddedElastic.getHost() +":" + embeddedElastic.getMappedPort(9200)) - .applyTo(configurableApplicationContext.getEnvironment()); - TestPropertyValues.of("spring.jpa.properties.hibernate.search.backend.hosts=" + embeddedElastic.getHost() +":" + embeddedElastic.getMappedPort(9200)) - .applyTo(configurableApplicationContext.getEnvironment()); - } - - } + private IGenericClient ourClient; + + @Container + public static ElasticsearchContainer embeddedElastic = TestElasticsearchContainerHelper.getEmbeddedElasticSearch(); + + @Autowired + private ElasticsearchBootSvcImpl myElasticsearchSvc; + + @PreDestroy + public void stop() { + embeddedElastic.stop(); + } + + @LocalServerPort + private int port; + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("spring.elasticsearch.uris", embeddedElastic::getHttpHostAddress); + } + + @Test + void testLastN() throws IOException, InterruptedException { + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + IIdType id = ourClient.create().resource(pt).execute().getId().toUnqualifiedVersionless(); + + Observation obs = new Observation(); + obs.getSubject().setReferenceElement(id); + String observationCode = "testobservationcode"; + String codeSystem = "http://testobservationcodesystem"; + + obs.getCode().addCoding().setCode(observationCode).setSystem(codeSystem); + obs.setValue(new StringType(observationCode)); + + Date effectiveDtm = new GregorianCalendar().getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + obs.getCategoryFirstRep().addCoding().setCode("testcategorycode").setSystem("http://testcategorycodesystem"); + IIdType obsId = ourClient.create().resource(obs).execute().getId().toUnqualifiedVersionless(); + + myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX); + + Thread.sleep(2000); + Parameters output = ourClient.operation().onType(Observation.class).named("lastn").withParameter(Parameters.class, "max", new IntegerType(1)).andParameter("subject", new StringType("Patient/" + id.getIdPart())).execute(); + Bundle b = (Bundle) output.getParameter().get(0).getResource(); + assertEquals(1, b.getTotal()); + assertEquals(obsId, b.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless()); + } + + @BeforeEach + void beforeEach() { + + FhirContext ourCtx = FhirContext.forR4(); + ourCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); + ourCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000); + String ourServerBase = "http://localhost:" + port + "/fhir/"; + ourClient = ourCtx.newRestfulGenericClient(ourServerBase); + ourClient.registerInterceptor(new LoggingInterceptor(true)); + } }