From 2e2bdaed6749914f451dac966bde547726214ec8 Mon Sep 17 00:00:00 2001 From: ianmarshall Date: Fri, 4 Dec 2020 15:50:43 -0500 Subject: [PATCH] Added support for $lastn operation and fixed Elasticsearch indexing. --- README.md | 5 + .../uhn/fhir/jpa/starter/AppProperties.java | 10 ++ .../jpa/starter/BaseJpaRestfulServer.java | 6 + .../fhir/jpa/starter/EnvironmentHelper.java | 60 ++++++++ .../jpa/starter/FhirServerConfigDstu3.java | 15 ++ .../fhir/jpa/starter/FhirServerConfigR4.java | 15 ++ .../fhir/jpa/starter/FhirServerConfigR5.java | 16 ++ src/main/resources/application.yaml | 1 + .../jpa/starter/ElasticsearchLastNR4IT.java | 144 ++++++++++++++++++ 9 files changed, 272 insertions(+) create mode 100644 src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java diff --git a/README.md b/README.md index 13d8fe4..a6e5244 100644 --- a/README.md +++ b/README.md @@ -321,3 +321,8 @@ elasticsearch.password=SomePassword elasticsearch.required_index_status=YELLOW elasticsearch.schema_management_strategy=CREATE ``` + +## Enabling LastN + +Set `hapi.fhir.lastn_enabled=true` in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file to enable the $lastn operation on this server. Note that the $lastn operation relies on Elasticsearch, so for $lastn to work, indexing must be enabled using Elasticsearch. + diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java b/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java index 722f517..9c4b767 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java @@ -62,6 +62,8 @@ public class AppProperties { private Partitioning partitioning = null; private List implementationGuides = null; + private Boolean lastn_enabled = false; + public Integer getDefer_indexing_for_codesystems_of_size() { return defer_indexing_for_codesystems_of_size; } @@ -382,6 +384,14 @@ public class AppProperties { this.narrative_enabled = narrative_enabled; } + public Boolean getLastn_enabled() { + return lastn_enabled; + } + + public void setLastn_enabled(Boolean lastn_enabled) { + this.lastn_enabled = lastn_enabled; + } + public static class Cors { private Boolean allow_Credentials = true; private List allowed_origin = ImmutableList.of("*"); diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java b/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java index 99fe658..d60f8bc 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java @@ -360,6 +360,12 @@ public class BaseJpaRestfulServer extends RestfulServer { .setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL)); } } + + if (appProperties.getLastn_enabled()) { + daoConfig.setLastNEnabled(true); + } + + } diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java b/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java index 4dbc122..6dd8311 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java @@ -1,5 +1,8 @@ package ca.uhn.fhir.jpa.starter; +import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; +import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus; +import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy; import org.springframework.core.env.CompositePropertySource; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.EnumerablePropertySource; @@ -35,9 +38,66 @@ public class EnvironmentHelper { properties.put(values[0], values[1]); }); } + + if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null + && environment.getProperty("elasticsearch.enabled", Boolean.class) == true ){ + ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder(); + ElasticsearchIndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", ElasticsearchIndexStatus.class); + if (requiredIndexStatus == null) { + builder.setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW); + } else { + builder.setRequiredIndexStatus(requiredIndexStatus); + } + + builder.setRestUrl(getElasticsearchServerUrl(environment)); + builder.setUsername(getElasticsearchServerUsername(environment)); + builder.setPassword(getElasticsearchServerPassword(environment)); + IndexSchemaManagementStrategy indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", IndexSchemaManagementStrategy.class); + if (indexSchemaManagementStrategy == null) { + builder.setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE); + } else { + builder.setIndexSchemaManagementStrategy(indexSchemaManagementStrategy); + } + // pretty_print_json_log: false + Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class); + if (refreshAfterWrite == null) { + builder.setDebugRefreshAfterWrite(false); + } else { + builder.setDebugRefreshAfterWrite(refreshAfterWrite); + } + // pretty_print_json_log: false + Boolean prettyPrintJsonLog = environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class); + if (prettyPrintJsonLog == null) { + builder.setDebugPrettyPrintJsonLog(false); + } else { + builder.setDebugPrettyPrintJsonLog(prettyPrintJsonLog); + } + builder.apply(properties); + } + return properties; } + public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) { + return environment.getProperty("elasticsearch.rest_url", String.class); + } + + public static String getElasticsearchServerUsername(ConfigurableEnvironment environment) { + return environment.getProperty("elasticsearch.username"); + } + + public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) { + return environment.getProperty("elasticsearch.password"); + } + + public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) { + if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) { + return environment.getProperty("elasticsearch.enabled", Boolean.class); + } else { + return false; + } + } + public static Map getPropertiesStartingWith(ConfigurableEnvironment aEnv, String aKeyPrefix) { Map result = new HashMap<>(); diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigDstu3.java b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigDstu3.java index 02d51ea..07406b2 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigDstu3.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigDstu3.java @@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.config.BaseJavaConfigDstu3; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import ca.uhn.fhir.jpa.starter.annotations.OnDSTU3Condition; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.*; @@ -63,4 +64,18 @@ public class FhirServerConfigDstu3 extends BaseJavaConfigDstu3 { return retVal; } + @Bean() + public ElasticsearchSvcImpl elasticsearchSvc() { + if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) { + String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment); + String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":")); + String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment); + String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment); + int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1)); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword); + } else { + return null; + } + } + } diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR4.java b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR4.java index 6734723..e415fea 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR4.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR4.java @@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.config.BaseJavaConfigR4; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import ca.uhn.fhir.jpa.starter.annotations.OnR4Condition; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -66,4 +67,18 @@ public class FhirServerConfigR4 extends BaseJavaConfigR4 { return retVal; } + @Bean() + public ElasticsearchSvcImpl elasticsearchSvc() { + if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) { + String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment); + String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":")); + String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment); + String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment); + int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1)); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword); + } else { + return null; + } + } + } diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR5.java b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR5.java index 28e5ef5..6e5bd98 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR5.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR5.java @@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.jpa.config.BaseJavaConfigR5; import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; import ca.uhn.fhir.jpa.starter.annotations.OnR5Condition; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -66,4 +67,19 @@ public class FhirServerConfigR5 extends BaseJavaConfigR5 { return retVal; } + @Bean() + public ElasticsearchSvcImpl elasticsearchSvc() { + if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) { + String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment); + String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":")); + String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment); + String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment); + int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1)); + return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword); + } else { + return null; + } + } + + } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index cc33d00..51e25ae 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -120,6 +120,7 @@ hapi: # startTlsEnable: # startTlsRequired: # quitWait: +# lastn_enabled: true # diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java new file mode 100644 index 0000000..c3fb7f7 --- /dev/null +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -0,0 +1,144 @@ +package ca.uhn.fhir.jpa.starter; + +import ca.uhn.fhir.context.ConfigurationException; +import ca.uhn.fhir.context.FhirContext; +import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl; +import ca.uhn.fhir.rest.client.api.IGenericClient; +import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum; +import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor; +import org.hl7.fhir.instance.model.api.IIdType; +import org.hl7.fhir.r4.model.Bundle; +import org.hl7.fhir.r4.model.DateTimeType; +import org.hl7.fhir.r4.model.IntegerType; +import org.hl7.fhir.r4.model.Observation; +import org.hl7.fhir.r4.model.Parameters; +import org.hl7.fhir.r4.model.Patient; +import org.hl7.fhir.r4.model.StringType; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic; +import pl.allegro.tech.embeddedelasticsearch.PopularProperties; + +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = Application.class, properties = + { + "spring.batch.job.enabled=false", + "spring.datasource.url=jdbc:h2:mem:dbr4", + "hapi.fhir.fhir_version=r4", + "hapi.fhir.lastn_enabled=true", + "elasticsearch.enabled=true", + // Because the port is set randomly, we will set the rest_url using the Initializer. + // "elasticsearch.rest_url='http://localhost:9200'", + "elasticsearch.username=SomeUsername", + "elasticsearch.password=SomePassword" + }) +@ContextConfiguration(initializers = ElasticsearchLastNR4IT.Initializer.class) +public class ElasticsearchLastNR4IT { + + private IGenericClient ourClient; + private FhirContext ourCtx; + + private static final String ELASTIC_VERSION = "6.5.4"; + private static EmbeddedElastic embeddedElastic; + + @Autowired + private ElasticsearchSvcImpl myElasticsearchSvc; + + @BeforeAll + public static void beforeClass() { + + embeddedElastic = null; + try { + embeddedElastic = EmbeddedElastic.builder() + .withElasticVersion(ELASTIC_VERSION) + .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0) + .withSetting(PopularProperties.HTTP_PORT, 0) + .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID()) + .withStartTimeout(60, TimeUnit.SECONDS) + .build() + .start(); + } catch (IOException | InterruptedException e) { + throw new ConfigurationException(e); + } + } + + @PreDestroy + public void stop() { + embeddedElastic.stop(); + } + + @LocalServerPort + private int port; + + @Test + void testLastN() throws IOException { + + Patient pt = new Patient(); + pt.addName().setFamily("Lastn").addGiven("Arthur"); + IIdType id = ourClient.create().resource(pt).execute().getId().toUnqualifiedVersionless(); + + Observation obs = new Observation(); + obs.getSubject().setReferenceElement(id); + String observationCode = "testobservationcode"; + String codeSystem = "http://testobservationcodesystem"; + obs.getCode().addCoding().setCode(observationCode).setSystem(codeSystem); + obs.setValue(new StringType(observationCode)); + Date effectiveDtm = new GregorianCalendar().getTime(); + obs.setEffective(new DateTimeType(effectiveDtm)); + obs.getCategoryFirstRep().addCoding().setCode("testcategorycode").setSystem("http://testcategorycodesystem"); + IIdType obsId = ourClient.create().resource(obs).execute().getId().toUnqualifiedVersionless(); + + myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX); + + Parameters output = ourClient.operation().onType(Observation.class).named("lastn") + .withParameter(Parameters.class, "max", new IntegerType(1)) + .andParameter("subject", new StringType("Patient/" + id.getIdPart())) + .execute(); + Bundle b = (Bundle) output.getParameter().get(0).getResource(); + assertEquals(1, b.getTotal()); + assertEquals(obsId, b.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless()); + } + + @BeforeEach + void beforeEach() { + + ourCtx = FhirContext.forR4(); + ourCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER); + ourCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000); + String ourServerBase = "http://localhost:" + port + "/fhir/"; + ourClient = ourCtx.newRestfulGenericClient(ourServerBase); + ourClient.registerInterceptor(new LoggingInterceptor(true)); + } + + static class Initializer + implements ApplicationContextInitializer { + + @Override + public void initialize( + ConfigurableApplicationContext configurableApplicationContext) { + // Since the port is dynamically generated, replace the URL with one that has the correct port + TestPropertyValues.of("elasticsearch.rest_url=http://localhost:" + embeddedElastic.getHttpPort()) + .applyTo(configurableApplicationContext.getEnvironment()); + } + + } +}