From e9f29af1aac64f9a0453bd95b602e6b019a7388f Mon Sep 17 00:00:00 2001 From: dotasek Date: Mon, 31 Jul 2023 14:34:47 -0400 Subject: [PATCH 1/9] Bump to hapi 6.8.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1fc9a25..67495df 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.6.0 + 6.8.0-SNAPSHOT hapi-fhir-jpaserver-starter From aafd4fa0d9484214ffd49f6e68ef665551ccc117 Mon Sep 17 00:00:00 2001 From: dotasek Date: Mon, 31 Jul 2023 14:43:58 -0400 Subject: [PATCH 2/9] Switch to new ca.uhn.fhir.batch2.jobs.export to fix StarterJpaConfig --- .../java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java index c095e27..4788d59 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java @@ -18,7 +18,7 @@ import ca.uhn.fhir.jpa.api.dao.DaoRegistry; import ca.uhn.fhir.jpa.api.dao.IFhirSystemDao; import ca.uhn.fhir.jpa.binary.interceptor.BinaryStorageInterceptor; import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider; -import ca.uhn.fhir.jpa.bulk.export.provider.BulkDataExportProvider; +import ca.uhn.fhir.batch2.jobs.export.BulkDataExportProvider; import ca.uhn.fhir.jpa.config.util.HapiEntityManagerFactoryUtil; import ca.uhn.fhir.jpa.config.util.ResourceCountCacheUtil; import ca.uhn.fhir.jpa.config.util.ValidationSupportConfigUtil; From 3c395e9dbe23ee7de43869f3e2ad8dfc3ad06521 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 10 Aug 2023 12:08:57 -0400 Subject: [PATCH 3/9] WIP try to expand default max_result_window --- .../jpa/starter/ElasticsearchLastNR4IT.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index 5cb0a4e..f8728bb 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -8,6 +8,10 @@ import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Date; @@ -43,6 +47,9 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer; "hapi.fhir.lastn_enabled=true", "hapi.fhir.store_resource_in_lucene_index_enabled=true", "hapi.fhir.advanced_lucene_indexing=true", + "hapi.fhir.subscription.websocket_enabled=false", + "hapi.fhir.subscription.resthook_enabled=false", + "hapi.fhir.subscription.email_enabled=false", "elasticsearch.enabled=true", // Because the port is set randomly, we will set the rest_url using the Initializer. // "elasticsearch.rest_url='http://localhost:9200'", @@ -69,9 +76,21 @@ public class ElasticsearchLastNR4IT { private ElasticsearchSvcImpl myElasticsearchSvc; @BeforeAll - public static void beforeClass() { + public static void beforeClass() throws IOException { embeddedElastic = new ElasticsearchContainer(ELASTIC_IMAGE).withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)); embeddedElastic.start(); + + URL url = new URL("http://" + embeddedElastic.getHost() + ":9200/_settings"); + HttpURLConnection httpCon = (HttpURLConnection) url.openConnection(); + httpCon.setDoOutput(true); + httpCon.setRequestMethod("PUT"); + OutputStreamWriter out = new OutputStreamWriter( + httpCon.getOutputStream()); + out.write("{\n" + + " \"index.max_result_window\": 50000\n" + + "}"); + out.close(); + httpCon.getInputStream(); } @PreDestroy From f7853f5e19afd1f16854c3fd352fe1fc423ef04e Mon Sep 17 00:00:00 2001 From: Tadgh Date: Thu, 10 Aug 2023 09:30:01 -0700 Subject: [PATCH 4/9] Fix up test --- pom.xml | 6 +++ .../jpa/starter/ElasticsearchLastNR4IT.java | 45 ++++++++++++------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 67495df..e84aeed 100644 --- a/pom.xml +++ b/pom.xml @@ -247,6 +247,12 @@ + + ca.uhn.hapi.fhir + hapi-fhir-jpaserver-test-utilities + ${project.version} + test + org.eclipse.jetty jetty-servlets diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index f8728bb..240cd29 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -3,20 +3,28 @@ package ca.uhn.fhir.jpa.starter; import static org.junit.jupiter.api.Assertions.assertEquals; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchRestClientFactory; import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.jpa.test.config.TestElasticsearchContainerHelper; import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import java.io.IOException; import java.io.OutputStreamWriter; import java.net.HttpURLConnection; -import java.net.MalformedURLException; import java.net.URL; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.GregorianCalendar; +import java.util.List; import javax.annotation.PreDestroy; + +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.indices.PutIndexTemplateRequest; +import org.elasticsearch.common.settings.Settings; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Bundle; import org.hl7.fhir.r4.model.DateTimeType; @@ -38,8 +46,11 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; @ExtendWith(SpringExtension.class) +@Testcontainers @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = {Application.class, JpaStarterWebsocketDispatcherConfig.class}, properties = { "spring.datasource.url=jdbc:h2:mem:dbr4", @@ -70,29 +81,28 @@ public class ElasticsearchLastNR4IT { private static final String ELASTIC_VERSION = "7.16.3"; private static final String ELASTIC_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:" + ELASTIC_VERSION; - private static ElasticsearchContainer embeddedElastic; + + @Container + public static ElasticsearchContainer embeddedElastic = TestElasticsearchContainerHelper.getEmbeddedElasticSearch(); @Autowired private ElasticsearchSvcImpl myElasticsearchSvc; @BeforeAll public static void beforeClass() throws IOException { - embeddedElastic = new ElasticsearchContainer(ELASTIC_IMAGE).withStartupTimeout(Duration.of(300, ChronoUnit.SECONDS)); - embeddedElastic.start(); + //Given + RestHighLevelClient elasticsearchHighLevelRestClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient( + "http", embeddedElastic.getHost() + ":" + embeddedElastic.getMappedPort(9200), "", ""); + + PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("hapi_fhir_template"); + putIndexTemplateRequest.patterns(List.of("*")); + Settings settings = Settings.builder().put("index.max_result_window", 50000).build(); + putIndexTemplateRequest.settings(settings); + elasticsearchHighLevelRestClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); + - URL url = new URL("http://" + embeddedElastic.getHost() + ":9200/_settings"); - HttpURLConnection httpCon = (HttpURLConnection) url.openConnection(); - httpCon.setDoOutput(true); - httpCon.setRequestMethod("PUT"); - OutputStreamWriter out = new OutputStreamWriter( - httpCon.getOutputStream()); - out.write("{\n" + - " \"index.max_result_window\": 50000\n" + - "}"); - out.close(); - httpCon.getInputStream(); } - + @PreDestroy public void stop() { embeddedElastic.stop(); @@ -134,7 +144,7 @@ public class ElasticsearchLastNR4IT { } @BeforeEach - void beforeEach() { + void beforeEach() throws IOException { ourCtx = FhirContext.forR4(); ourCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); @@ -142,6 +152,7 @@ public class ElasticsearchLastNR4IT { String ourServerBase = "http://localhost:" + port + "/fhir/"; ourClient = ourCtx.newRestfulGenericClient(ourServerBase); ourClient.registerInterceptor(new LoggingInterceptor(true)); + } static class Initializer From bdf593f151e9210d0c7736f022fa4fe9ae7ab335 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 10 Aug 2023 12:34:45 -0400 Subject: [PATCH 5/9] Clean up code --- .../fhir/jpa/starter/ElasticsearchLastNR4IT.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index 240cd29..f65a572 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -10,17 +10,12 @@ import ca.uhn.fhir.rest.client.api.IGenericClient; import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.time.Duration; -import java.time.temporal.ChronoUnit; + import java.util.Date; import java.util.GregorianCalendar; import java.util.List; import javax.annotation.PreDestroy; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.indices.PutIndexTemplateRequest; @@ -58,9 +53,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; "hapi.fhir.lastn_enabled=true", "hapi.fhir.store_resource_in_lucene_index_enabled=true", "hapi.fhir.advanced_lucene_indexing=true", - "hapi.fhir.subscription.websocket_enabled=false", - "hapi.fhir.subscription.resthook_enabled=false", - "hapi.fhir.subscription.email_enabled=false", + "elasticsearch.enabled=true", // Because the port is set randomly, we will set the rest_url using the Initializer. // "elasticsearch.rest_url='http://localhost:9200'", @@ -79,10 +72,7 @@ public class ElasticsearchLastNR4IT { private IGenericClient ourClient; private FhirContext ourCtx; - private static final String ELASTIC_VERSION = "7.16.3"; - private static final String ELASTIC_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:" + ELASTIC_VERSION; - - @Container + @Container public static ElasticsearchContainer embeddedElastic = TestElasticsearchContainerHelper.getEmbeddedElasticSearch(); @Autowired From 1907536d28ec527cc57a6f9536f1742af0fa6e60 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 10 Aug 2023 12:51:34 -0400 Subject: [PATCH 6/9] Fix one more failing IT config --- src/test/java/ca/uhn/fhir/jpa/starter/ExampleServerR4IT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ExampleServerR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ExampleServerR4IT.java index d08770a..26f27f6 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ExampleServerR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ExampleServerR4IT.java @@ -1,6 +1,7 @@ package ca.uhn.fhir.jpa.starter; import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.searchparam.config.NicknameServiceConfig; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.EncodingEnum; import ca.uhn.fhir.rest.api.MethodOutcome; @@ -30,7 +31,7 @@ import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = {Application.class, JpaStarterWebsocketDispatcherConfig.class}, properties = { +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = {Application.class, JpaStarterWebsocketDispatcherConfig.class, NicknameServiceConfig.class}, properties = { "spring.datasource.url=jdbc:h2:mem:dbr4", "hapi.fhir.enable_repository_validating_interceptor=true", "hapi.fhir.fhir_version=r4", From 7265b115dd83664d972235f677d898e89a1d3c48 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 10 Aug 2023 12:55:30 -0400 Subject: [PATCH 7/9] Leave comment in case of updates --- .../java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index f65a572..6f358d4 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -84,6 +84,10 @@ public class ElasticsearchLastNR4IT { RestHighLevelClient elasticsearchHighLevelRestClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient( "http", embeddedElastic.getHost() + ":" + embeddedElastic.getMappedPort(9200), "", ""); + /* As of 2023-08-10, HAPI FHIR sets SubscriptionConstants.MAX_SUBSCRIPTION_RESULTS to 50000 + which is in excess of elastic's default max_result_window. If MAX_SUBSCRIPTION_RESULTS is changed + to a value <= 10000, the following will no longer be necessary. - dotasek + */ PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("hapi_fhir_template"); putIndexTemplateRequest.patterns(List.of("*")); Settings settings = Settings.builder().put("index.max_result_window", 50000).build(); From 69ff8c4d0675ba88521b6050e0975634381de7f4 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 17 Aug 2023 09:54:46 -0400 Subject: [PATCH 8/9] Bump to HAPI 6.8.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e84aeed..3b38277 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ ca.uhn.hapi.fhir hapi-fhir - 6.8.0-SNAPSHOT + 6.8.0 hapi-fhir-jpaserver-starter From b88ffdada00cb16d19e9dad68289cd07d1473f88 Mon Sep 17 00:00:00 2001 From: dotasek Date: Thu, 17 Aug 2023 13:02:02 -0400 Subject: [PATCH 9/9] Fix breakages due to https://github.com/hapifhir/hapi-fhir/pull/5180 --- .../ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java | 5 ++--- .../ca/uhn/fhir/jpa/starter/cr/StarterCrDstu3Config.java | 2 +- .../java/ca/uhn/fhir/jpa/starter/cr/StarterCrR4Config.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java index 33b498b..7a1934f 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java @@ -9,7 +9,7 @@ import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.FhirVersionEnum; import ca.uhn.fhir.context.support.IValidationSupport; -import ca.uhn.fhir.cr.config.CrProviderLoader; + import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.jpa.api.IDaoRegistry; import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; @@ -111,8 +111,7 @@ public class StarterJpaConfig { @Autowired private ConfigurableEnvironment configurableEnvironment; - @Autowired(required=false) - private CrProviderLoader crProviderLoader; + /** * Customize the default/max page sizes for search results. You can set these however diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrDstu3Config.java b/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrDstu3Config.java index 9efc086..06d95f4 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrDstu3Config.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrDstu3Config.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.starter.cr; -import ca.uhn.fhir.cr.config.CrDstu3Config; +import ca.uhn.fhir.cr.config.dstu3.CrDstu3Config; import ca.uhn.fhir.jpa.starter.annotations.OnDSTU3Condition; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrR4Config.java b/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrR4Config.java index a97cea7..586a324 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrR4Config.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/cr/StarterCrR4Config.java @@ -1,6 +1,6 @@ package ca.uhn.fhir.jpa.starter.cr; -import ca.uhn.fhir.cr.config.CrR4Config; +import ca.uhn.fhir.cr.config.r4.CrR4Config; import ca.uhn.fhir.jpa.starter.annotations.OnR4Condition; import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Import;