Merge branch 'master' into kbd-20201125-cql-initial-impl

src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java

@@ -7,8 +7,10 @@ import ca.uhn.fhir.rest.api.EncodingEnum;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import org.hl7.fhir.r4.model.Bundle;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
@@ -56,12 +58,14 @@ public class AppProperties {
   private Boolean narrative_enabled = true;
 
   private Validation validation = new Validation();
-  private List<Tester> tester = ImmutableList.of(new Tester());
+  private Map<String, Tester> tester = ImmutableMap.of("home", new Tester());
   private Logger logger = new Logger();
   private Subscription subscription = new Subscription();
   private Cors cors = null;
   private Partitioning partitioning = null;
-  private List<ImplementationGuide> implementationGuides = null;
+  private Map<String, ImplementationGuide> implementationGuides = null;
+
+  private Boolean lastn_enabled = false;
 
   public Integer getDefer_indexing_for_codesystems_of_size() {
     return defer_indexing_for_codesystems_of_size;
@@ -71,11 +75,11 @@ public class AppProperties {
     this.defer_indexing_for_codesystems_of_size = defer_indexing_for_codesystems_of_size;
   }
 
-  public List<ImplementationGuide> getImplementationGuides() {
+  public Map<String, ImplementationGuide> getImplementationGuides() {
     return implementationGuides;
   }
 
-  public void setImplementationGuides(List<ImplementationGuide> implementationGuides) {
+  public void setImplementationGuides(Map<String, ImplementationGuide> implementationGuides) {
     this.implementationGuides = implementationGuides;
   }
 
@@ -372,11 +376,11 @@ public class AppProperties {
     this.reuse_cached_search_results_millis = reuse_cached_search_results_millis;
   }
 
-  public List<Tester> getTester() {
+  public Map<String, Tester> getTester() {
     return tester;
   }
 
-  public void setTester(List<Tester> tester) {
+  public void setTester(Map<String, Tester> tester) {
     this.tester = tester;
   }
 
@@ -390,6 +394,14 @@ public class AppProperties {
     this.narrative_enabled = narrative_enabled;
   }
 
+  public Boolean getLastn_enabled() {
+    return lastn_enabled;
+  }
+
+  public void setLastn_enabled(Boolean lastn_enabled) {
+    this.lastn_enabled = lastn_enabled;
+  }
+
   public static class Cors {
     private Boolean allow_Credentials = true;
     private List<String> allowed_origin = ImmutableList.of("*");
@@ -456,7 +468,6 @@ public class AppProperties {
 
   public static class Tester {
 
-    private String id = "home";
     private String name = "Local Tester";
     private String server_address = "http://localhost:8080/fhir";
     private Boolean refuse_to_fetch_third_party_urls = true;
@@ -470,14 +481,6 @@ public class AppProperties {
       this.fhir_version = fhir_version;
     }
 
-    public String getId() {
-      return id;
-    }
-
-    public void setId(String id) {
-      this.id = id;
-    }
-
     public String getName() {
       return name;
     }

src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java

@@ -42,10 +42,7 @@ import org.springframework.http.HttpHeaders;
 import org.springframework.web.cors.CorsConfiguration;
 
 import javax.servlet.ServletException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
 import java.util.stream.Collectors;
 
 public class BaseJpaRestfulServer extends RestfulServer {
@@ -357,15 +354,21 @@ public class BaseJpaRestfulServer extends RestfulServer {
     }
 
     if (appProperties.getImplementationGuides() != null) {
-      List<AppProperties.ImplementationGuide> guides = appProperties.getImplementationGuides();
-      for (AppProperties.ImplementationGuide guide : guides) {
+      Map<String, AppProperties.ImplementationGuide> guides = appProperties.getImplementationGuides();
+      for (Map.Entry<String, AppProperties.ImplementationGuide> guide : guides.entrySet()) {
         packageInstallerSvc.install(new PackageInstallationSpec()
-          .setPackageUrl(guide.getUrl())
-          .setName(guide.getName())
-          .setVersion(guide.getVersion())
+          .setPackageUrl(guide.getValue().getUrl())
+          .setName(guide.getValue().getName())
+          .setVersion(guide.getValue().getVersion())
           .setInstallMode(PackageInstallationSpec.InstallModeEnum.STORE_AND_INSTALL));
       }
     }
+
+    if (appProperties.getLastn_enabled()) {
+      daoConfig.setLastNEnabled(true);
+    }
+
+
   }
 
 

src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java

@@ -1,5 +1,8 @@
 package ca.uhn.fhir.jpa.starter;
 
+import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder;
+import org.hibernate.search.elasticsearch.cfg.ElasticsearchIndexStatus;
+import org.hibernate.search.elasticsearch.cfg.IndexSchemaManagementStrategy;
 import org.springframework.core.env.CompositePropertySource;
 import org.springframework.core.env.ConfigurableEnvironment;
 import org.springframework.core.env.EnumerablePropertySource;
@@ -35,9 +38,66 @@ public class EnvironmentHelper {
         properties.put(values[0], values[1]);
       });
     }
+
+    if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null
+          && environment.getProperty("elasticsearch.enabled", Boolean.class) == true ){
+      ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder();
+      ElasticsearchIndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", ElasticsearchIndexStatus.class);
+      if (requiredIndexStatus == null) {
+        builder.setRequiredIndexStatus(ElasticsearchIndexStatus.YELLOW);
+      } else {
+        builder.setRequiredIndexStatus(requiredIndexStatus);
+      }
+
+      builder.setRestUrl(getElasticsearchServerUrl(environment));
+      builder.setUsername(getElasticsearchServerUsername(environment));
+      builder.setPassword(getElasticsearchServerPassword(environment));
+      IndexSchemaManagementStrategy indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", IndexSchemaManagementStrategy.class);
+      if (indexSchemaManagementStrategy == null) {
+        builder.setIndexSchemaManagementStrategy(IndexSchemaManagementStrategy.CREATE);
+      } else {
+        builder.setIndexSchemaManagementStrategy(indexSchemaManagementStrategy);
+      }
+      //    pretty_print_json_log: false
+      Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class);
+      if (refreshAfterWrite == null) {
+        builder.setDebugRefreshAfterWrite(false);
+      } else {
+        builder.setDebugRefreshAfterWrite(refreshAfterWrite);
+      }
+      //    pretty_print_json_log: false
+      Boolean prettyPrintJsonLog = environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class);
+      if (prettyPrintJsonLog == null) {
+        builder.setDebugPrettyPrintJsonLog(false);
+      } else {
+        builder.setDebugPrettyPrintJsonLog(prettyPrintJsonLog);
+      }
+      builder.apply(properties);
+    }
+
     return properties;
   }
 
+  public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) {
+    return environment.getProperty("elasticsearch.rest_url", String.class);
+  }
+
+  public static String getElasticsearchServerUsername(ConfigurableEnvironment environment) {
+    return environment.getProperty("elasticsearch.username");
+  }
+
+  public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) {
+    return environment.getProperty("elasticsearch.password");
+  }
+
+  public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) {
+    if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) {
+      return environment.getProperty("elasticsearch.enabled", Boolean.class);
+    } else {
+      return false;
+    }
+  }
+
   public static Map<String, Object> getPropertiesStartingWith(ConfigurableEnvironment aEnv,
                                                               String aKeyPrefix) {
     Map<String, Object> result = new HashMap<>();

src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigDstu3.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter;
 import ca.uhn.fhir.context.ConfigurationException;
 import ca.uhn.fhir.jpa.config.BaseJavaConfigDstu3;
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
+import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
 import ca.uhn.fhir.jpa.starter.annotations.OnDSTU3Condition;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.*;
@@ -63,4 +64,18 @@ public class FhirServerConfigDstu3 extends BaseJavaConfigDstu3 {
     return retVal;
   }
 
+  @Bean()
+  public ElasticsearchSvcImpl elasticsearchSvc() {
+    if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) {
+      String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment);
+      String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":"));
+      String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment);
+      String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment);
+      int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1));
+      return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword);
+    } else {
+      return null;
+    }
+  }
+
 }

src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR4.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter;
 import ca.uhn.fhir.context.ConfigurationException;
 import ca.uhn.fhir.jpa.config.BaseJavaConfigR4;
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
+import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
 import ca.uhn.fhir.jpa.starter.annotations.OnR4Condition;
 import ca.uhn.fhir.jpa.starter.cql.StarterCqlR4Config;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -65,4 +66,18 @@ public class FhirServerConfigR4 extends BaseJavaConfigR4 {
     return retVal;
   }
 
+  @Bean()
+  public ElasticsearchSvcImpl elasticsearchSvc() {
+    if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) {
+      String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment);
+      String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":"));
+      String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment);
+      String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment);
+      int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1));
+      return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword);
+    } else {
+      return null;
+    }
+  }
+
 }

src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigR5.java

@@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.starter;
 import ca.uhn.fhir.context.ConfigurationException;
 import ca.uhn.fhir.jpa.config.BaseJavaConfigR5;
 import ca.uhn.fhir.jpa.search.DatabaseBackedPagingProvider;
+import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
 import ca.uhn.fhir.jpa.starter.annotations.OnR5Condition;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
@@ -66,4 +67,19 @@ public class FhirServerConfigR5 extends BaseJavaConfigR5 {
     return retVal;
   }
 
+  @Bean()
+  public ElasticsearchSvcImpl elasticsearchSvc() {
+    if (EnvironmentHelper.isElasticsearchEnabled(configurableEnvironment)) {
+      String elasticsearchUrl = EnvironmentHelper.getElasticsearchServerUrl(configurableEnvironment);
+      String elasticsearchHost = elasticsearchUrl.substring(elasticsearchUrl.indexOf("://")+3, elasticsearchUrl.lastIndexOf(":"));
+      String elasticsearchUsername = EnvironmentHelper.getElasticsearchServerUsername(configurableEnvironment);
+      String elasticsearchPassword = EnvironmentHelper.getElasticsearchServerPassword(configurableEnvironment);
+      int elasticsearchPort = Integer.parseInt(elasticsearchUrl.substring(elasticsearchUrl.lastIndexOf(":")+1));
+      return new ElasticsearchSvcImpl(elasticsearchHost, elasticsearchPort, elasticsearchUsername, elasticsearchPassword);
+    } else {
+      return null;
+    }
+  }
+
+
 }

src/main/java/ca/uhn/fhir/jpa/starter/FhirTesterConfig.java

@@ -36,15 +36,15 @@ public class FhirTesterConfig {
   @Bean
   public TesterConfig testerConfig(AppProperties appProperties) {
     TesterConfig retVal = new TesterConfig();
-    appProperties.getTester().stream().forEach(t -> {
+    appProperties.getTester().entrySet().stream().forEach(t -> {
       retVal
         .addServer()
-        .withId(t.getId())
-        .withFhirVersion(t.getFhir_version())
-        .withBaseUrl(t.getServer_address())
-        .withName(t.getName());
+        .withId(t.getKey())
+        .withFhirVersion(t.getValue().getFhir_version())
+        .withBaseUrl(t.getValue().getServer_address())
+        .withName(t.getValue().getName());
       retVal.setRefuseToFetchThirdPartyUrls(
-        t.getRefuse_to_fetch_third_party_urls());
+        t.getValue().getRefuse_to_fetch_third_party_urls());
 
     });
     return retVal;

src/main/resources/application.yaml

@@ -1,6 +1,7 @@
 spring:
   datasource:
     url: 'jdbc:h2:file:./target/database/h2'
+    #url: jdbc:h2:mem:test_mem
     username: sa
     password: null
     driverClassName: org.h2.Driver
@@ -32,14 +33,16 @@ hapi:
     ### This is the FHIR version. Choose between, DSTU2, DSTU3, R4 or R5
     fhir_version: R4
 #    defer_indexing_for_codesystems_of_size: 101
-#    implementationguides:
-#      -
-#        url: https://build.fhir.org/ig/hl7dk/dk-medcom/branches/corrections/package.tgz
-#        name: dk.fhir.ig.medcom-core
-#        version: 0.8.0
-#      -
-#        name: hl7.fhir.uv.ips
-#        version: 0.3.0
+    #implementationguides:
+      #example from registry (packages.fhir.org)
+      #swiss:
+        #name: swiss.mednet.fhir
+        #version: 0.8.0
+      #example not from registry
+      #ips_1_0_0:
+        #url: https://build.fhir.org/ig/HL7/fhir-ips/package.tgz
+        #name: hl7.fhir.uv.ips
+        #version: 1.0.0
 
     #supported_resource_types:
     #  - Patient
@@ -91,14 +94,14 @@ hapi:
 #    retain_cached_searches_mins: 60
 #    reuse_cached_search_results_millis: 60000
     tester:
-      -
-          id: home
+
+        home:
           name: Local Tester
           server_address: 'http://localhost:8080/fhir'
           refuse_to_fetch_third_party_urls: false
           fhir_version: R4
-      -
-          id: global
+
+        global:
           name: Global Tester
           server_address: "http://hapi.fhir.org/baseR4"
           refuse_to_fetch_third_party_urls: false
@@ -122,6 +125,7 @@ hapi:
 #        startTlsEnable:
 #        startTlsRequired:
 #        quitWait:
+#    lastn_enabled: true
 
 
 #

src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java

@@ -0,0 +1,144 @@
+package ca.uhn.fhir.jpa.starter;
+
+import ca.uhn.fhir.context.ConfigurationException;
+import ca.uhn.fhir.context.FhirContext;
+import ca.uhn.fhir.jpa.search.lastn.ElasticsearchSvcImpl;
+import ca.uhn.fhir.rest.client.api.IGenericClient;
+import ca.uhn.fhir.rest.client.api.ServerValidationModeEnum;
+import ca.uhn.fhir.rest.client.interceptor.LoggingInterceptor;
+import org.hl7.fhir.instance.model.api.IIdType;
+import org.hl7.fhir.r4.model.Bundle;
+import org.hl7.fhir.r4.model.DateTimeType;
+import org.hl7.fhir.r4.model.IntegerType;
+import org.hl7.fhir.r4.model.Observation;
+import org.hl7.fhir.r4.model.Parameters;
+import org.hl7.fhir.r4.model.Patient;
+import org.hl7.fhir.r4.model.StringType;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.util.TestPropertyValues;
+import org.springframework.boot.web.server.LocalServerPort;
+import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import pl.allegro.tech.embeddedelasticsearch.EmbeddedElastic;
+import pl.allegro.tech.embeddedelasticsearch.PopularProperties;
+
+import javax.annotation.PreDestroy;
+import java.io.IOException;
+import java.util.Date;
+import java.util.GregorianCalendar;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, classes = Application.class, properties =
+  {
+    "spring.batch.job.enabled=false",
+    "spring.datasource.url=jdbc:h2:mem:dbr4",
+    "hapi.fhir.fhir_version=r4",
+    "hapi.fhir.lastn_enabled=true",
+    "elasticsearch.enabled=true",
+    // Because the port is set randomly, we will set the rest_url using the Initializer.
+    // "elasticsearch.rest_url='http://localhost:9200'",
+    "elasticsearch.username=SomeUsername",
+    "elasticsearch.password=SomePassword"
+  })
+@ContextConfiguration(initializers = ElasticsearchLastNR4IT.Initializer.class)
+public class ElasticsearchLastNR4IT {
+
+  private IGenericClient ourClient;
+  private FhirContext ourCtx;
+
+  private static final String ELASTIC_VERSION = "6.5.4";
+  private static EmbeddedElastic embeddedElastic;
+
+  @Autowired
+  private ElasticsearchSvcImpl myElasticsearchSvc;
+
+  @BeforeAll
+  public static void beforeClass() {
+
+    embeddedElastic = null;
+    try {
+      embeddedElastic = EmbeddedElastic.builder()
+        .withElasticVersion(ELASTIC_VERSION)
+        .withSetting(PopularProperties.TRANSPORT_TCP_PORT, 0)
+        .withSetting(PopularProperties.HTTP_PORT, 0)
+        .withSetting(PopularProperties.CLUSTER_NAME, UUID.randomUUID())
+        .withStartTimeout(60, TimeUnit.SECONDS)
+        .build()
+        .start();
+    } catch (IOException | InterruptedException e) {
+      throw new ConfigurationException(e);
+    }
+  }
+
+  @PreDestroy
+  public void stop() {
+    embeddedElastic.stop();
+  }
+
+  @LocalServerPort
+  private int port;
+
+  @Test
+  void testLastN() throws IOException {
+
+    Patient pt = new Patient();
+    pt.addName().setFamily("Lastn").addGiven("Arthur");
+    IIdType id = ourClient.create().resource(pt).execute().getId().toUnqualifiedVersionless();
+
+    Observation obs = new Observation();
+    obs.getSubject().setReferenceElement(id);
+    String observationCode = "testobservationcode";
+    String codeSystem = "http://testobservationcodesystem";
+    obs.getCode().addCoding().setCode(observationCode).setSystem(codeSystem);
+    obs.setValue(new StringType(observationCode));
+    Date effectiveDtm = new GregorianCalendar().getTime();
+    obs.setEffective(new DateTimeType(effectiveDtm));
+    obs.getCategoryFirstRep().addCoding().setCode("testcategorycode").setSystem("http://testcategorycodesystem");
+    IIdType obsId = ourClient.create().resource(obs).execute().getId().toUnqualifiedVersionless();
+
+    myElasticsearchSvc.refreshIndex(ElasticsearchSvcImpl.OBSERVATION_INDEX);
+
+    Parameters output = ourClient.operation().onType(Observation.class).named("lastn")
+      .withParameter(Parameters.class, "max", new IntegerType(1))
+      .andParameter("subject", new StringType("Patient/" + id.getIdPart()))
+      .execute();
+    Bundle b = (Bundle) output.getParameter().get(0).getResource();
+    assertEquals(1, b.getTotal());
+    assertEquals(obsId, b.getEntry().get(0).getResource().getIdElement().toUnqualifiedVersionless());
+  }
+
+  @BeforeEach
+  void beforeEach() {
+
+    ourCtx = FhirContext.forR4();
+    ourCtx.getRestfulClientFactory().setServerValidationMode(ServerValidationModeEnum.NEVER);
+    ourCtx.getRestfulClientFactory().setSocketTimeout(1200 * 1000);
+    String ourServerBase = "http://localhost:" + port + "/fhir/";
+    ourClient = ourCtx.newRestfulGenericClient(ourServerBase);
+    ourClient.registerInterceptor(new LoggingInterceptor(true));
+  }
+
+  static class Initializer
+    implements ApplicationContextInitializer<ConfigurableApplicationContext> {
+
+    @Override
+    public void initialize(
+      ConfigurableApplicationContext configurableApplicationContext) {
+      // Since the port is dynamically generated, replace the URL with one that has the correct port
+      TestPropertyValues.of("elasticsearch.rest_url=http://localhost:" + embeddedElastic.getHttpPort())
+        .applyTo(configurableApplicationContext.getEnvironment());
+    }
+
+  }
+}