Merge branch 'master' into rel_5_4_0

# Conflicts:
#	src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java
This commit is contained in:
Frank Tao
2021-03-18 18:02:58 -04:00
4 changed files with 244 additions and 141 deletions

81
.github/workflows/build-images.yaml vendored Normal file
View File

@@ -0,0 +1,81 @@
name: Build Container Images
on:
push:
tags:
- "image/v*"
pull_request:
branches: [master]
jobs:
build:
name: Build
runs-on: ubuntu-20.04
steps:
- name: Docker meta
id: docker_meta
uses: crazy-max/ghaction-docker-meta@v1
with:
images: |
ghcr.io/hapifhir/hapi
docker.io/hapiproject/hapi
tag-sha: false
tag-match: "v(.*)"
# waiting for https://github.com/crazy-max/ghaction-docker-meta/issues/13 for a cleaner solution
- name: Docker distroless meta
id: docker_distroless_meta
uses: crazy-max/ghaction-docker-meta@v1
with:
images: |
ghcr.io/hapifhir/hapi
docker.io/hapiproject/hapi
tag-sha: false
tag-match: "v(.*)"
sep-tags: -distroless,
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v1
if: github.event_name != 'pull_request'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v1
if: github.event_name != 'pull_request'
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
- name: Build and push distroless
id: docker_build_distroless
uses: docker/build-push-action@v2
with:
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_distroless_meta.outputs.tags }}-distroless
labels: ${{ steps.docker_distroless_meta.outputs.labels }}
target: release-distroless
- name: Print image digests
run: |
echo ${{ steps.docker_build.outputs.digest }}
echo ${{ steps.docker_build_distroless.outputs.digest }}

View File

@@ -7,6 +7,21 @@ RUN mvn -ntp dependency:go-offline
COPY src/ /tmp/hapi-fhir-jpaserver-starter/src/ COPY src/ /tmp/hapi-fhir-jpaserver-starter/src/
RUN mvn clean install -DskipTests RUN mvn clean install -DskipTests
FROM build-hapi AS build-distroless
RUN mvn package spring-boot:repackage -Pboot
RUN mkdir /app && \
cp /tmp/hapi-fhir-jpaserver-starter/target/ROOT.war /app/main.war
FROM gcr.io/distroless/java-debian10:11 AS release-distroless
COPY --chown=nonroot:nonroot --from=build-distroless /app /app
EXPOSE 8080
# 65532 is the nonroot user's uid
# used here instead of the name to allow Kubernetes to easily detect that the container
# is running as a non-root (uid != 0) user.
USER 65532:65532
WORKDIR /app
CMD ["/app/main.war"]
FROM tomcat:9.0.38-jdk11-openjdk-slim-buster FROM tomcat:9.0.38-jdk11-openjdk-slim-buster
RUN mkdir -p /data/hapi/lucenefiles && chmod 775 /data/hapi/lucenefiles RUN mkdir -p /data/hapi/lucenefiles && chmod 775 /data/hapi/lucenefiles

View File

@@ -322,7 +322,7 @@ Set `hapi.fhir.cql_enabled=true` in the [application.yaml](https://github.com/ha
## Enabling MDM (EMPI) ## Enabling MDM (EMPI)
Set `hapi.fhir.mdm_enabled=true` in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file to enable MDM on this server. The MDM matching rules are configured in [mdm-rules.json](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/mdm-rules.json). The rules in this example file should be replaced with actual matching rules appropriate to your data. Note that MDM relies on subscriptions, so for MDM to work, subscriptions must be enabled. Set `hapi.fhir.mdm_enabled=true` in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file to enable MDM on this server. The MDM matching rules are configured in [mdm-rules.json](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/mdm-rules.json). The rules in this example file should be replaced with actual matching rules appropriate to your data. Note that MDM relies on subscriptions, so for MDM to work, subscriptions must be enabled.
## Using Elasticsearch ## Using Elasticsearch
@@ -344,23 +344,14 @@ elasticsearch.schema_management_strategy=CREATE
Set `hapi.fhir.lastn_enabled=true` in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file to enable the $lastn operation on this server. Note that the $lastn operation relies on Elasticsearch, so for $lastn to work, indexing must be enabled using Elasticsearch. Set `hapi.fhir.lastn_enabled=true` in the [application.yaml](https://github.com/hapifhir/hapi-fhir-jpaserver-starter/blob/master/src/main/resources/application.yaml) file to enable the $lastn operation on this server. Note that the $lastn operation relies on Elasticsearch, so for $lastn to work, indexing must be enabled using Elasticsearch.
## Example of a Dockerfile based on distroless images (for lower footprint and improved security) ## Build the distroless variant of the image (for lower footprint and improved security)
```code The default Dockerfile contains a `release-distroless` stage to build a variant of the image
FROM maven:3.6.3-jdk-11-slim as build-hapi using the `gcr.io/distroless/java-debian10:11` base image:
WORKDIR /tmp/hapi-fhir-jpaserver-starter
COPY pom.xml . ```sh
RUN mvn -ntp dependency:go-offline docker build --target=release-distroless -t hapi-fhir:distroless .
COPY src/ /tmp/hapi-fhir-jpaserver-starter/src/
RUN mvn clean package spring-boot:repackage -Pboot
FROM gcr.io/distroless/java:11
COPY --from=build-hapi /tmp/hapi-fhir-jpaserver-starter/target/ROOT.war /app/main.war
EXPOSE 8080
WORKDIR /app
CMD ["main.war"]
``` ```
Note that distroless images are also automatically build and pushed to the container registry,
see the `-distroless` suffix in the image tags.

View File

@@ -1,170 +1,186 @@
package ca.uhn.fhir.jpa.starter; package ca.uhn.fhir.jpa.starter;
import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
import ca.uhn.fhir.jpa.search.HapiLuceneAnalysisConfigurer; import ca.uhn.fhir.jpa.search.HapiLuceneAnalysisConfigurer;
import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder;
import org.apache.commons.lang3.StringUtils; import org.apache.lucene.util.Version;
import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.hibernate.cfg.AvailableSettings;
import org.hibernate.search.backend.elasticsearch.cfg.ElasticsearchBackendSettings;
import org.hibernate.search.backend.elasticsearch.index.IndexStatus; import org.hibernate.search.backend.elasticsearch.index.IndexStatus;
import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings; import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings;
import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings; import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings;
import org.hibernate.search.backend.lucene.lowlevel.directory.impl.LocalFileSystemDirectoryProvider;
import org.hibernate.search.engine.cfg.BackendSettings; import org.hibernate.search.engine.cfg.BackendSettings;
import org.hibernate.search.mapper.orm.automaticindexing.session.AutomaticIndexingSynchronizationStrategyNames; import org.hibernate.search.mapper.orm.automaticindexing.session.AutomaticIndexingSynchronizationStrategyNames;
import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings; import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings;
import org.hibernate.search.mapper.orm.schema.management.SchemaManagementStrategyName; import org.hibernate.search.mapper.orm.schema.management.SchemaManagementStrategyName;
import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy;
import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy;
import org.springframework.core.env.CompositePropertySource; import org.springframework.core.env.CompositePropertySource;
import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.ConfigurableEnvironment;
import org.springframework.core.env.EnumerablePropertySource; import org.springframework.core.env.EnumerablePropertySource;
import org.springframework.core.env.PropertySource; import org.springframework.core.env.PropertySource;
import java.util.Arrays; import java.util.*;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
public class EnvironmentHelper { public class EnvironmentHelper {
public static Properties getHibernateProperties(ConfigurableEnvironment environment) { public static Properties getHibernateProperties(ConfigurableEnvironment environment) {
Properties properties = new Properties(); Properties properties = new Properties();
Map<String, Object> jpaProps = getPropertiesStartingWith(environment, "spring.jpa.properties");
for (Map.Entry<String, Object> entry : jpaProps.entrySet()) {
String strippedKey = entry.getKey().replace("spring.jpa.properties.", "");
properties.put(strippedKey, entry.getValue().toString());
}
Map<String, Object> jpaProps = getPropertiesStartingWith(environment, "spring.jpa.properties"); //Spring Boot Autoconfiguration defaults
properties.putIfAbsent("hibernate.format_sql", "false"); properties.putIfAbsent(AvailableSettings.SCANNER, "org.hibernate.boot.archive.scan.internal.DisabledScanner");
properties.putIfAbsent("hibernate.show_sql", "false"); properties.putIfAbsent(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName());
properties.putIfAbsent("hibernate.hbm2ddl.auto", "update"); properties.putIfAbsent(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName());
properties.putIfAbsent("hibernate.jdbc.batch_size", "20"); //TODO The bean factory should be added as parameter but that requires that it can be injected from the entityManagerFactory bean from xBaseConfig
properties.putIfAbsent("hibernate.cache.use_query_cache", "false"); //properties.putIfAbsent(AvailableSettings.BEAN_CONTAINER, new SpringBeanContainer(beanFactory));
properties.putIfAbsent("hibernate.cache.use_second_level_cache", "false");
properties.putIfAbsent("hibernate.cache.use_structured_entries", "false");
properties.putIfAbsent("hibernate.cache.use_minimal_puts", "false");
if (jpaProps.getOrDefault("spring.jpa.properties.hibernate.search.enabled", "false").toString() == "true") { //hapi-fhir-jpaserver-base "sensible defaults"
properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, true); Map<String, Object> hapiJpaPropertyMap = new HapiFhirLocalContainerEntityManagerFactoryBean().getJpaPropertyMap();
properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_TYPE), "local-filesystem"); hapiJpaPropertyMap.forEach(properties::putIfAbsent);
properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_ROOT), "target/lucenefiles");
properties.putIfAbsent(BackendSettings.backendKey(BackendSettings.TYPE), "lucene");
properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.ANALYSIS_CONFIGURER), HapiLuceneAnalysisConfigurer.class.getName());
properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.LUCENE_VERSION), "LUCENE_CURRENT");
} else {
properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, false);
}
for (Map.Entry<String, Object> entry : jpaProps.entrySet()) { //hapi-fhir-jpaserver-starter defaults
String strippedKey = entry.getKey().replace("spring.jpa.properties.", ""); properties.putIfAbsent(AvailableSettings.FORMAT_SQL, false);
properties.put(strippedKey, entry.getValue().toString()); properties.putIfAbsent(AvailableSettings.SHOW_SQL, false);
} properties.putIfAbsent(AvailableSettings.HBM2DDL_AUTO, "update");
properties.putIfAbsent(AvailableSettings.STATEMENT_BATCH_SIZE, 20);
properties.putIfAbsent(AvailableSettings.USE_QUERY_CACHE, false);
properties.putIfAbsent(AvailableSettings.USE_SECOND_LEVEL_CACHE, false);
properties.putIfAbsent(AvailableSettings.USE_STRUCTURED_CACHE, false);
properties.putIfAbsent(AvailableSettings.USE_MINIMAL_PUTS, false);
//Hibernate Search defaults
properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, false);
if (Boolean.parseBoolean(String.valueOf(properties.get(HibernateOrmMapperSettings.ENABLED)))) {
if (isElasticsearchEnabled(environment)) {
properties.putIfAbsent(BackendSettings.backendKey(BackendSettings.TYPE), ElasticsearchBackendSettings.TYPE_NAME);
} else {
properties.putIfAbsent(BackendSettings.backendKey(BackendSettings.TYPE), LuceneBackendSettings.TYPE_NAME);
}
if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null if (properties.get(BackendSettings.backendKey(BackendSettings.TYPE)).equals(LuceneBackendSettings.TYPE_NAME)) {
&& environment.getProperty("elasticsearch.enabled", Boolean.class) == true) { properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_TYPE), LocalFileSystemDirectoryProvider.NAME);
ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder(); properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_ROOT), "target/lucenefiles");
IndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", IndexStatus.class); properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.ANALYSIS_CONFIGURER), HapiLuceneAnalysisConfigurer.class.getName());
if (requiredIndexStatus == null) { properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.LUCENE_VERSION), Version.LATEST);
builder.setRequiredIndexStatus(IndexStatus.YELLOW);
} else {
builder.setRequiredIndexStatus(requiredIndexStatus);
}
builder.setRestUrl(getElasticsearchServerUrl(environment)); } else if (properties.get(BackendSettings.backendKey(BackendSettings.TYPE)).equals(ElasticsearchBackendSettings.TYPE_NAME)) {
builder.setUsername(getElasticsearchServerUsername(environment)); ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder();
builder.setPassword(getElasticsearchServerPassword(environment)); IndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", IndexStatus.class);
builder.setProtocol(getElasticsearchServerProtocol(environment)); builder.setRequiredIndexStatus(requireNonNullElse(requiredIndexStatus, IndexStatus.YELLOW));
SchemaManagementStrategyName indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", SchemaManagementStrategyName.class); builder.setRestUrl(getElasticsearchServerUrl(environment));
if (indexSchemaManagementStrategy == null) { builder.setUsername(getElasticsearchServerUsername(environment));
builder.setIndexSchemaManagementStrategy(SchemaManagementStrategyName.CREATE); builder.setPassword(getElasticsearchServerPassword(environment));
} else { builder.setProtocol(getElasticsearchServerProtocol(environment));
builder.setIndexSchemaManagementStrategy(indexSchemaManagementStrategy); SchemaManagementStrategyName indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", SchemaManagementStrategyName.class);
} builder.setIndexSchemaManagementStrategy(requireNonNullElse(indexSchemaManagementStrategy, SchemaManagementStrategyName.CREATE));
// pretty_print_json_log: false Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class);
Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class); if (refreshAfterWrite == null || !refreshAfterWrite) {
if (refreshAfterWrite == null || refreshAfterWrite == false) { builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.ASYNC);
builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.ASYNC); } else {
} else { builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.READ_SYNC);
builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.READ_SYNC); }
} builder.setDebugPrettyPrintJsonLog(requireNonNullElse(environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class), false));
// pretty_print_json_log: false builder.apply(properties);
Boolean prettyPrintJsonLog = environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class);
if (prettyPrintJsonLog == null) {
builder.setDebugPrettyPrintJsonLog(false);
} else {
builder.setDebugPrettyPrintJsonLog(prettyPrintJsonLog);
}
builder.apply(properties);
}
return properties;
}
public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) { } else {
return environment.getProperty("elasticsearch.rest_url", String.class); throw new UnsupportedOperationException("Unsupported Hibernate Search backend: " + properties.get(BackendSettings.backendKey(BackendSettings.TYPE)));
} }
}
return properties;
}
//TODO Removed when we're up on Java 11
private static <T> T requireNonNullElse(T obj, T defaultObj) {
return (obj != null) ? obj : requireNonNull(defaultObj, "defaultObj");
}
//TODO Removed when we're up on Java 11
private static <T> T requireNonNull(T obj, String message) {
if (obj == null)
throw new NullPointerException(message);
return obj;
}
public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) {
return environment.getProperty("elasticsearch.rest_url", String.class);
}
public static String getElasticsearchServerProtocol(ConfigurableEnvironment environment) { public static String getElasticsearchServerProtocol(ConfigurableEnvironment environment) {
return environment.getProperty("elasticsearch.protocol", String.class, "http"); return environment.getProperty("elasticsearch.protocol", String.class, "http");
} }
public static String getElasticsearchServerUsername(ConfigurableEnvironment environment) { public static String getElasticsearchServerUsername(ConfigurableEnvironment environment) {
return environment.getProperty("elasticsearch.username"); return environment.getProperty("elasticsearch.username");
} }
public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) { public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) {
return environment.getProperty("elasticsearch.password"); return environment.getProperty("elasticsearch.password");
} }
public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) { public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) {
if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) { if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) {
return environment.getProperty("elasticsearch.enabled", Boolean.class); return environment.getProperty("elasticsearch.enabled", Boolean.class);
} else { } else {
return false; return false;
} }
} }
public static Map<String, Object> getPropertiesStartingWith(ConfigurableEnvironment aEnv, public static Map<String, Object> getPropertiesStartingWith(ConfigurableEnvironment aEnv,
String aKeyPrefix) { String aKeyPrefix) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
Map<String, Object> map = getAllProperties(aEnv); Map<String, Object> map = getAllProperties(aEnv);
for (Map.Entry<String, Object> entry : map.entrySet()) { for (Map.Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey(); String key = entry.getKey();
if (key.startsWith(aKeyPrefix)) { if (key.startsWith(aKeyPrefix)) {
result.put(key, entry.getValue()); result.put(key, entry.getValue());
} }
} }
return result; return result;
} }
public static Map<String, Object> getAllProperties(ConfigurableEnvironment aEnv) { public static Map<String, Object> getAllProperties(ConfigurableEnvironment aEnv) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
aEnv.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); aEnv.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps)));
return result; return result;
} }
public static Map<String, Object> getAllProperties(PropertySource<?> aPropSource) { public static Map<String, Object> getAllProperties(PropertySource<?> aPropSource) {
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
if (aPropSource instanceof CompositePropertySource) { if (aPropSource instanceof CompositePropertySource) {
CompositePropertySource cps = (CompositePropertySource) aPropSource; CompositePropertySource cps = (CompositePropertySource) aPropSource;
cps.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); cps.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps)));
return result; return result;
} }
if (aPropSource instanceof EnumerablePropertySource<?>) { if (aPropSource instanceof EnumerablePropertySource<?>) {
EnumerablePropertySource<?> ps = (EnumerablePropertySource<?>) aPropSource; EnumerablePropertySource<?> ps = (EnumerablePropertySource<?>) aPropSource;
Arrays.asList(ps.getPropertyNames()).forEach(key -> result.put(key, ps.getProperty(key))); Arrays.asList(ps.getPropertyNames()).forEach(key -> result.put(key, ps.getProperty(key)));
return result; return result;
} }
return result; return result;
} }
private static void addAll(Map<String, Object> aBase, Map<String, Object> aToBeAdded) { private static void addAll(Map<String, Object> aBase, Map<String, Object> aToBeAdded) {
for (Map.Entry<String, Object> entry : aToBeAdded.entrySet()) { for (Map.Entry<String, Object> entry : aToBeAdded.entrySet()) {
if (aBase.containsKey(entry.getKey())) { if (aBase.containsKey(entry.getKey())) {
continue; continue;
} }
aBase.put(entry.getKey(), entry.getValue()); aBase.put(entry.getKey(), entry.getValue());
} }
} }
} }