Feature/elastic back in green (#893)

* Getting automated tests back into green

* using native API's

* formatting

* refactoring

* getting 8.X ES working

* Removed sleep

* Update src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Reintroduced sleep as indicies are a bit slow ?

* Using the production bean instead of a test one

* updating default parameters for es

* Making separate profile for ES

* works from here

* remove all not needed code anymore

* removing parent defined version

* fixed tests

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

pom.xml

@@ -307,25 +307,18 @@
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
-            <version>2.0.3</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>elasticsearch</artifactId>
+            <artifactId>testcontainers-elasticsearch</artifactId>
-            <version>1.21.4</version>
+            <version>2.0.2</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
-            <artifactId>postgresql</artifactId>
+            <artifactId>testcontainers-junit-jupiter</artifactId>
-            <version>1.21.4</version>
+            <version>2.0.2</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.testcontainers</groupId>
-            <artifactId>junit-jupiter</artifactId>
-            <version>1.21.4</version>
             <scope>test</scope>
         </dependency>
 

src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java

@@ -50,6 +50,7 @@ import ca.uhn.fhir.jpa.starter.AppProperties;
 import ca.uhn.fhir.jpa.starter.annotations.OnCorsPresent;
 import ca.uhn.fhir.jpa.starter.annotations.OnImplementationGuidesPresent;
 import ca.uhn.fhir.jpa.starter.common.validation.IRepositoryValidationInterceptorFactory;
+import ca.uhn.fhir.jpa.starter.elastic.ElasticsearchBootSvcImpl;
 import ca.uhn.fhir.jpa.starter.ig.ExtendedPackageInstallationSpec;
 import ca.uhn.fhir.jpa.starter.ig.IImplementationGuideOperationProvider;
 import ca.uhn.fhir.jpa.subscription.util.SubscriptionDebugLogInterceptor;
@@ -149,6 +150,7 @@ public class StarterJpaConfig {
 	@Primary
 	@Bean
 	public LocalContainerEntityManagerFactoryBean entityManagerFactory(
+			Optional<ElasticsearchBootSvcImpl> elasticsearchSvc,
 			JpaProperties theJpaProperties,
 			DataSource myDataSource,
 			ConfigurableListableBeanFactory myConfigurableListableBeanFactory,

src/main/java/ca/uhn/fhir/jpa/starter/elastic/ElasticsearchBootSvcImpl.java

@@ -50,7 +50,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 
 	private static final String OBSERVATION_RESOURCE_NAME = "Observation";
 
-	private final ElasticsearchClient myRestHighLevelClient;
+	private final ElasticsearchClient myElasticsearchClient;
 
 	private final FhirContext myContext;
 
@@ -61,7 +61,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 	public ElasticsearchBootSvcImpl(ElasticsearchClient client, FhirContext fhirContext, AppProperties appProperties) {
 
 		myContext = fhirContext;
-		myRestHighLevelClient = client;
+		myElasticsearchClient = client;
 
 		// Determine index prefix from configuration
 		if (appProperties.getElasticsearch() != null) {
@@ -144,7 +144,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 	}
 
 	private boolean createIndex(String theIndexName, String theMapping) throws IOException {
-		return myRestHighLevelClient
+		return myElasticsearchClient
 				.indices()
 				.create(cir -> cir.index(theIndexName).withJson(new StringReader(theMapping)))
 				.acknowledged();
@@ -152,7 +152,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 
 	private boolean indexExists(String theIndexName) throws IOException {
 		ExistsRequest request = new ExistsRequest.Builder().index(theIndexName).build();
-		return myRestHighLevelClient.indices().exists(request).value();
+		return myElasticsearchClient.indices().exists(request).value();
 	}
 
 	@Override
@@ -165,7 +165,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 		SearchRequest searchRequest = buildObservationResourceSearchRequest(thePids);
 		try {
 			SearchResponse<ObservationJson> observationDocumentResponse =
-					myRestHighLevelClient.search(searchRequest, ObservationJson.class);
+					myElasticsearchClient.search(searchRequest, ObservationJson.class);
 			List<Hit<ObservationJson>> observationDocumentHits =
 					observationDocumentResponse.hits().hits();
 			IParser parser = TolerantJsonParser.createWithLenientErrorHandling(myContext, null);
@@ -202,7 +202,7 @@ public class ElasticsearchBootSvcImpl implements IElasticsearchSvc {
 
 	@VisibleForTesting
 	public void refreshIndex(String theIndexName) throws IOException {
-		myRestHighLevelClient.indices().refresh(fn -> fn.index(theIndexName));
+		myElasticsearchClient.indices().refresh(fn -> fn.index(theIndexName));
 	}
 
 	/**

src/main/resources/application-elastic.yaml

@@ -0,0 +1,50 @@
+spring:
+  elasticsearch:
+    uris: http://localhost:9200
+    username: elastic
+    password: elastic
+
+  autoconfigure:
+    # This empty exclude is needed to override the default exclusion of the Elasticsearch configuration.
+    exclude:
+
+  jpa:
+    properties:
+      hibernate:
+        # --- Hibernate Search (Lucene/Elasticsearch) ---
+        # Note: the following values should be kept in sync with ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder
+        search:
+          schema_management:
+            strategy: CREATE
+          enabled: true
+          backend:
+            layout:
+              strategy: ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy
+            type: elasticsearch
+            protocol: http
+            analysis:
+              configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticsearchAnalysisConfigurer
+            scroll_timeout: 60
+            schema_management:
+              settings_file: ca/uhn/fhir/jpa/elastic/index-settings.json
+              minimal_required_status_wait_timeout: 10000
+              minimal_required_status: YELLOW
+
+            dynamic_mapping: true
+          indexing:
+            plan:
+              synchronization:
+                strategy: async
+
+# -------------------------------------------------------------------------------------
+# HAPI FHIR — grouped by domain
+# -------------------------------------------------------------------------------------
+hapi:
+  fhir:
+    # -------------------------------------------------------------------------------
+    # D. Search & Indexing
+    # -------------------------------------------------------------------------------
+    # NOTE: Extended Lucene/Elasticsearch indexing is experimental.
+    # See https://hapifhir.io/hapi-fhir/docs/server_jpa/elastic.html
+    advanced_lucene_indexing: true
+    search_index_full_text_enabled: true

src/main/resources/application.yaml

@@ -37,6 +37,10 @@ management:
         enabled: true
 
 spring:
+#  elasticsearch:
+#    uris: http://localhost:9200
+#    username: elastic
+#    password: elastic
   # -------------------------------------------------------------------------------
   # Application Name
   # -------------------------------------------------------------------------------
@@ -130,8 +134,10 @@ spring:
           use_minimal_puts: false
 
         # --- Hibernate Search (Lucene/Elasticsearch) ---
-        search:
+        #search:
-          enabled: false
+        #  schema_management:
+        #    strategy: CREATE
+        #  enabled: true
           # Lucene backend (default example)
           # backend:
           #   type: lucene
@@ -142,10 +148,25 @@ spring:
           #     root: target/lucenefiles
           #   lucene_version: lucene_current
           # Elasticsearch backend (alternative) — see also hapi.fhir.elasticsearch section in docs
-          # backend:
+#          backend:
-          #   type: elasticsearch
+#            layout:
-          #   analysis:
+#              strategy: ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy
-          #     configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticAnalysisConfigurer
+#            type: elasticsearch
+#            protocol: http
+#            analysis:
+#              configurer: ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers$HapiElasticsearchAnalysisConfigurer
+#            scroll_timeout: 60
+#            schema_management:
+#              settings_file: ca/uhn/fhir/jpa/elastic/index-settings.json
+#              minimal_required_status_wait_timeout: 10000
+#              minimal_required_status: YELLOW
+#
+#            dynamic_mapping: true
+#          indexing:
+#            plan:
+#              synchronization:
+#                strategy: async
+
 
 # -------------------------------------------------------------------------------------
 # HAPI FHIR — grouped by domain
@@ -249,8 +270,8 @@ hapi:
     # -------------------------------------------------------------------------------
     # NOTE: Extended Lucene/Elasticsearch indexing is experimental.
     # See https://hapifhir.io/hapi-fhir/docs/server_jpa/elastic.html
-    advanced_lucene_indexing: false
+    # advanced_lucene_indexing: true
-    search_index_full_text_enabled: false
+    # search_index_full_text_enabled: true
     # language_search_parameter_enabled: true
     # upliftedRefchains_enabled: true
     # index_storage_optimized: false

src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java

@@ -1,7 +1,5 @@
 package ca.uhn.fhir.jpa.starter;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
 import ca.uhn.fhir.jpa.starter.common.TestContainerHelper;
@@ -36,6 +34,8 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
 @Testcontainers
 @ActiveProfiles("test")
 @TestPropertySource(locations = "classpath:test-elasticsearch-lastn.yaml")
@@ -53,6 +53,11 @@ class ElasticsearchLastNR4IT {
 		TestContainerHelper.registerElasticsearchProperties(registry, ELASTICSEARCH);
 		// Also register spring.elasticsearch.uris for ElasticConfigCondition to enable ElasticsearchBootSvcImpl
 		registry.add("spring.elasticsearch.uris", () -> TestContainerHelper.getElasticsearchHttpUrl(ELASTICSEARCH));
+
+		registry.add("spring.jpa.properties.hibernate.search.backend.hosts", ELASTICSEARCH::getHttpHostAddress);
+		registry.add("spring.jpa.properties.hibernate.search.backend.protocol", () -> "http");
+		registry.add("spring.jpa.properties.hibernate.search.backend.username", () -> "");
+		registry.add("spring.jpa.properties.hibernate.search.backend.password", () -> "");
 	}
 
 	@Autowired
@@ -63,8 +68,6 @@ class ElasticsearchLastNR4IT {
 
 	@Test
 	void testLastN() throws IOException, InterruptedException {
-    Thread.sleep(2000);
-
 		Patient pt = new Patient();
 		pt.addName().setFamily("Lastn").addGiven("Arthur");
 		IIdType id = ourClient.create().resource(pt).execute().getId().toUnqualifiedVersionless();
@@ -73,6 +76,7 @@ class ElasticsearchLastNR4IT {
 		obs.getSubject().setReferenceElement(id);
 		String observationCode = "testobservationcode";
 
+
 		obs.getCode().addCoding().setCode(observationCode).setSystem("http://testobservationcodesystem");
 		obs.setValue(new StringType(observationCode));
 
@@ -82,12 +86,9 @@ class ElasticsearchLastNR4IT {
 		IIdType obsId = ourClient.create().resource(obs).execute().getId().toUnqualifiedVersionless();
 
 		myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
-	  Thread.sleep(2000);
 
-    Parameters output = ourClient.operation().onType(Observation.class).named("lastn")
+		Thread.sleep(2000);
-      .withParameter(Parameters.class, "max", new IntegerType(1))
+		Parameters output = ourClient.operation().onType(Observation.class).named("lastn").withParameter(Parameters.class, "max", new IntegerType(1)).andParameter("subject", new StringType("Patient/" + id.getIdPart())).execute();
-      .andParameter("subject", new StringType("Patient/" + id.getIdPart()))
-      .execute();
 		Bundle b = (Bundle) output.getParameter().get(0).getResource();
 		assertEquals(1, b.getTotal());
 		assertEquals(obsId, b.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless());