From da326875dd5691883d96878cdb8eb74cf9139075 Mon Sep 17 00:00:00 2001 From: Frank Tao Date: Sat, 13 Mar 2021 21:49:50 -0500 Subject: [PATCH 1/3] Prepare for R5.4 --- pom.xml | 2 +- .../fhir/jpa/starter/EnvironmentHelper.java | 256 +++++++++--------- .../jpa/starter/ElasticsearchLastNR4IT.java | 2 +- 3 files changed, 124 insertions(+), 136 deletions(-) diff --git a/pom.xml b/pom.xml index 5d8d98a..218d077 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.3.0 + 5.4.0-PRE1-SNAPSHOT hapi-fhir-jpaserver-starter diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java b/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java index 082e3b4..3df2c53 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/EnvironmentHelper.java @@ -1,182 +1,170 @@ package ca.uhn.fhir.jpa.starter; -import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean; import ca.uhn.fhir.jpa.search.HapiLuceneAnalysisConfigurer; import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; -import org.apache.lucene.util.Version; -import org.hibernate.cfg.AvailableSettings; -import org.hibernate.search.backend.elasticsearch.cfg.ElasticsearchBackendSettings; +import org.apache.commons.lang3.StringUtils; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.hibernate.search.backend.elasticsearch.index.IndexStatus; import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings; import org.hibernate.search.backend.lucene.cfg.LuceneIndexSettings; -import org.hibernate.search.backend.lucene.lowlevel.directory.impl.LocalFileSystemDirectoryProvider; import org.hibernate.search.engine.cfg.BackendSettings; import org.hibernate.search.mapper.orm.automaticindexing.session.AutomaticIndexingSynchronizationStrategyNames; import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings; import org.hibernate.search.mapper.orm.schema.management.SchemaManagementStrategyName; -import org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy; -import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy; import org.springframework.core.env.CompositePropertySource; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.EnumerablePropertySource; import org.springframework.core.env.PropertySource; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; public class EnvironmentHelper { - public static Properties getHibernateProperties(ConfigurableEnvironment environment) { - Properties properties = new Properties(); - Map jpaProps = getPropertiesStartingWith(environment, "spring.jpa.properties"); - for (Map.Entry entry : jpaProps.entrySet()) { - String strippedKey = entry.getKey().replace("spring.jpa.properties.", ""); - properties.put(strippedKey, entry.getValue().toString()); - } + public static Properties getHibernateProperties(ConfigurableEnvironment environment) { + Properties properties = new Properties(); - //Spring Boot Autoconfiguration defaults - properties.putIfAbsent(AvailableSettings.SCANNER, "org.hibernate.boot.archive.scan.internal.DisabledScanner"); - properties.putIfAbsent(AvailableSettings.IMPLICIT_NAMING_STRATEGY, SpringImplicitNamingStrategy.class.getName()); - properties.putIfAbsent(AvailableSettings.PHYSICAL_NAMING_STRATEGY, SpringPhysicalNamingStrategy.class.getName()); - //TODO The bean factory should be added as parameter but that requires that it can be injected from the entityManagerFactory bean from xBaseConfig - //properties.putIfAbsent(AvailableSettings.BEAN_CONTAINER, new SpringBeanContainer(beanFactory)); + Map jpaProps = getPropertiesStartingWith(environment, "spring.jpa.properties"); + properties.putIfAbsent("hibernate.format_sql", "false"); + properties.putIfAbsent("hibernate.show_sql", "false"); + properties.putIfAbsent("hibernate.hbm2ddl.auto", "update"); + properties.putIfAbsent("hibernate.jdbc.batch_size", "20"); + properties.putIfAbsent("hibernate.cache.use_query_cache", "false"); + properties.putIfAbsent("hibernate.cache.use_second_level_cache", "false"); + properties.putIfAbsent("hibernate.cache.use_structured_entries", "false"); + properties.putIfAbsent("hibernate.cache.use_minimal_puts", "false"); - //hapi-fhir-jpaserver-base "sensible defaults" - Map hapiJpaPropertyMap = new HapiFhirLocalContainerEntityManagerFactoryBean().getJpaPropertyMap(); - hapiJpaPropertyMap.forEach(properties::putIfAbsent); + if (jpaProps.getOrDefault("spring.jpa.properties.hibernate.search.enabled", "false").toString() == "true") { + properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, true); + properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_TYPE), "local-filesystem"); + properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_ROOT), "target/lucenefiles"); + properties.putIfAbsent(BackendSettings.backendKey(BackendSettings.TYPE), "lucene"); + properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.ANALYSIS_CONFIGURER), HapiLuceneAnalysisConfigurer.class.getName()); + properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.LUCENE_VERSION), "LUCENE_CURRENT"); + } else { + properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, false); + } - //hapi-fhir-jpaserver-starter defaults - properties.putIfAbsent(AvailableSettings.FORMAT_SQL, false); - properties.putIfAbsent(AvailableSettings.SHOW_SQL, false); - properties.putIfAbsent(AvailableSettings.HBM2DDL_AUTO, "update"); - properties.putIfAbsent(AvailableSettings.STATEMENT_BATCH_SIZE, 20); - properties.putIfAbsent(AvailableSettings.USE_QUERY_CACHE, false); - properties.putIfAbsent(AvailableSettings.USE_SECOND_LEVEL_CACHE, false); - properties.putIfAbsent(AvailableSettings.USE_STRUCTURED_CACHE, false); - properties.putIfAbsent(AvailableSettings.USE_MINIMAL_PUTS, false); + for (Map.Entry entry : jpaProps.entrySet()) { + String strippedKey = entry.getKey().replace("spring.jpa.properties.", ""); + properties.put(strippedKey, entry.getValue().toString()); + } - //Hibernate Search defaults - properties.putIfAbsent(HibernateOrmMapperSettings.ENABLED, true); - if (Boolean.parseBoolean(String.valueOf(properties.get(HibernateOrmMapperSettings.ENABLED)))) { - properties.putIfAbsent(BackendSettings.backendKey(BackendSettings.TYPE), LuceneBackendSettings.TYPE_NAME); - if (properties.get(BackendSettings.backendKey(BackendSettings.TYPE)).equals(LuceneBackendSettings.TYPE_NAME)) { - properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_TYPE), LocalFileSystemDirectoryProvider.NAME); - properties.putIfAbsent(BackendSettings.backendKey(LuceneIndexSettings.DIRECTORY_ROOT), "target/lucenefiles"); - properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.ANALYSIS_CONFIGURER), HapiLuceneAnalysisConfigurer.class.getName()); - properties.putIfAbsent(BackendSettings.backendKey(LuceneBackendSettings.LUCENE_VERSION), Version.LATEST); + if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null + && environment.getProperty("elasticsearch.enabled", Boolean.class) == true) { + ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder(); + IndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", IndexStatus.class); + if (requiredIndexStatus == null) { + builder.setRequiredIndexStatus(IndexStatus.YELLOW); + } else { + builder.setRequiredIndexStatus(requiredIndexStatus); + } - } else if (properties.get(BackendSettings.backendKey(BackendSettings.TYPE)).equals(ElasticsearchBackendSettings.TYPE_NAME)) { - ElasticsearchHibernatePropertiesBuilder builder = new ElasticsearchHibernatePropertiesBuilder(); - IndexStatus requiredIndexStatus = environment.getProperty("elasticsearch.required_index_status", IndexStatus.class); - builder.setRequiredIndexStatus(requireNonNullElse(requiredIndexStatus, IndexStatus.YELLOW)); - builder.setRestUrl(getElasticsearchServerUrl(environment)); - builder.setUsername(getElasticsearchServerUsername(environment)); - builder.setPassword(getElasticsearchServerPassword(environment)); - builder.setProtocol(getElasticsearchServerProtocol(environment)); - SchemaManagementStrategyName indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", SchemaManagementStrategyName.class); - builder.setIndexSchemaManagementStrategy(requireNonNullElse(indexSchemaManagementStrategy, SchemaManagementStrategyName.CREATE)); - Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class); - if (refreshAfterWrite == null || !refreshAfterWrite) { - builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.ASYNC); - } else { - builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.READ_SYNC); - } - builder.setDebugPrettyPrintJsonLog(requireNonNullElse(environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class), false)); - builder.apply(properties); + builder.setRestUrl(getElasticsearchServerUrl(environment)); + builder.setUsername(getElasticsearchServerUsername(environment)); + builder.setPassword(getElasticsearchServerPassword(environment)); + builder.setProtocol(getElasticsearchServerProtocol(environment)); + SchemaManagementStrategyName indexSchemaManagementStrategy = environment.getProperty("elasticsearch.schema_management_strategy", SchemaManagementStrategyName.class); + if (indexSchemaManagementStrategy == null) { + builder.setIndexSchemaManagementStrategy(SchemaManagementStrategyName.CREATE); + } else { + builder.setIndexSchemaManagementStrategy(indexSchemaManagementStrategy); + } + // pretty_print_json_log: false + Boolean refreshAfterWrite = environment.getProperty("elasticsearch.debug.refresh_after_write", Boolean.class); + if (refreshAfterWrite == null || refreshAfterWrite == false) { + builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.ASYNC); + } else { + builder.setDebugIndexSyncStrategy(AutomaticIndexingSynchronizationStrategyNames.READ_SYNC); + } + // pretty_print_json_log: false + Boolean prettyPrintJsonLog = environment.getProperty("elasticsearch.debug.pretty_print_json_log", Boolean.class); + if (prettyPrintJsonLog == null) { + builder.setDebugPrettyPrintJsonLog(false); + } else { + builder.setDebugPrettyPrintJsonLog(prettyPrintJsonLog); + } + builder.apply(properties); + } + return properties; + } - } else { - throw new UnsupportedOperationException("Unsupported Hibernate Search backend: " + properties.get(BackendSettings.backendKey(BackendSettings.TYPE))); - } - } - - return properties; - } - - //TODO Removed when we're up on Java 11 - private static T requireNonNullElse(T obj, T defaultObj) { - return (obj != null) ? obj : requireNonNull(defaultObj, "defaultObj"); - } - - //TODO Removed when we're up on Java 11 - private static T requireNonNull(T obj, String message) { - if (obj == null) - throw new NullPointerException(message); - return obj; - } - - public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) { - return environment.getProperty("elasticsearch.rest_url", String.class); - } + public static String getElasticsearchServerUrl(ConfigurableEnvironment environment) { + return environment.getProperty("elasticsearch.rest_url", String.class); + } public static String getElasticsearchServerProtocol(ConfigurableEnvironment environment) { return environment.getProperty("elasticsearch.protocol", String.class, "http"); } public static String getElasticsearchServerUsername(ConfigurableEnvironment environment) { - return environment.getProperty("elasticsearch.username"); - } + return environment.getProperty("elasticsearch.username"); + } - public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) { - return environment.getProperty("elasticsearch.password"); - } + public static String getElasticsearchServerPassword(ConfigurableEnvironment environment) { + return environment.getProperty("elasticsearch.password"); + } - public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) { - if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) { - return environment.getProperty("elasticsearch.enabled", Boolean.class); - } else { - return false; - } - } + public static Boolean isElasticsearchEnabled(ConfigurableEnvironment environment) { + if (environment.getProperty("elasticsearch.enabled", Boolean.class) != null) { + return environment.getProperty("elasticsearch.enabled", Boolean.class); + } else { + return false; + } + } - public static Map getPropertiesStartingWith(ConfigurableEnvironment aEnv, - String aKeyPrefix) { - Map result = new HashMap<>(); + public static Map getPropertiesStartingWith(ConfigurableEnvironment aEnv, + String aKeyPrefix) { + Map result = new HashMap<>(); - Map map = getAllProperties(aEnv); + Map map = getAllProperties(aEnv); - for (Map.Entry entry : map.entrySet()) { - String key = entry.getKey(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); - if (key.startsWith(aKeyPrefix)) { - result.put(key, entry.getValue()); - } - } + if (key.startsWith(aKeyPrefix)) { + result.put(key, entry.getValue()); + } + } - return result; - } + return result; + } - public static Map getAllProperties(ConfigurableEnvironment aEnv) { - Map result = new HashMap<>(); - aEnv.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); - return result; - } + public static Map getAllProperties(ConfigurableEnvironment aEnv) { + Map result = new HashMap<>(); + aEnv.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); + return result; + } - public static Map getAllProperties(PropertySource aPropSource) { - Map result = new HashMap<>(); + public static Map getAllProperties(PropertySource aPropSource) { + Map result = new HashMap<>(); - if (aPropSource instanceof CompositePropertySource) { - CompositePropertySource cps = (CompositePropertySource) aPropSource; - cps.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); - return result; - } + if (aPropSource instanceof CompositePropertySource) { + CompositePropertySource cps = (CompositePropertySource) aPropSource; + cps.getPropertySources().forEach(ps -> addAll(result, getAllProperties(ps))); + return result; + } - if (aPropSource instanceof EnumerablePropertySource) { - EnumerablePropertySource ps = (EnumerablePropertySource) aPropSource; - Arrays.asList(ps.getPropertyNames()).forEach(key -> result.put(key, ps.getProperty(key))); - return result; - } + if (aPropSource instanceof EnumerablePropertySource) { + EnumerablePropertySource ps = (EnumerablePropertySource) aPropSource; + Arrays.asList(ps.getPropertyNames()).forEach(key -> result.put(key, ps.getProperty(key))); + return result; + } - return result; + return result; - } + } - private static void addAll(Map aBase, Map aToBeAdded) { - for (Map.Entry entry : aToBeAdded.entrySet()) { - if (aBase.containsKey(entry.getKey())) { - continue; - } + private static void addAll(Map aBase, Map aToBeAdded) { + for (Map.Entry entry : aToBeAdded.entrySet()) { + if (aBase.containsKey(entry.getKey())) { + continue; + } - aBase.put(entry.getKey(), entry.getValue()); - } - } + aBase.put(entry.getKey(), entry.getValue()); + } + } } diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java index 902abbe..2dcf32d 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/ElasticsearchLastNR4IT.java @@ -60,7 +60,7 @@ public class ElasticsearchLastNR4IT { private IGenericClient ourClient; private FhirContext ourCtx; - private static final String ELASTIC_VERSION = "7.10.1"; + private static final String ELASTIC_VERSION = "7.10.2"; private static final String ELASTIC_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:" + ELASTIC_VERSION; private static ElasticsearchContainer embeddedElastic; From 8585a121c44e43a7c73656ca257397b4c7d79163 Mon Sep 17 00:00:00 2001 From: Frank Tao Date: Thu, 18 Mar 2021 21:01:13 -0400 Subject: [PATCH 2/3] Added enable_index_contained_resource --- src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java | 9 +++++++++ .../ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java | 2 ++ .../ca/uhn/fhir/jpa/starter/FhirServerConfigCommon.java | 6 ++++++ src/main/resources/application.yaml | 1 + 4 files changed, 18 insertions(+) diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java b/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java index 3955156..e361776 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java @@ -31,6 +31,7 @@ public class AppProperties { private Boolean allow_placeholder_references = true; private Boolean auto_create_placeholder_reference_targets = false; private Boolean enable_index_missing_fields = false; + private Boolean enable_index_contained_resource = false; private Boolean enable_repository_validating_interceptor = false; private Boolean enforce_referential_integrity_on_delete = true; private Boolean enforce_referential_integrity_on_write = true; @@ -273,6 +274,14 @@ public class AppProperties { this.enable_index_missing_fields = enable_index_missing_fields; } + public Boolean getEnable_index_contained_resource() { + return enable_index_contained_resource; + } + + public void setEnable_index_contained_resource(Boolean enable_index_contained_resource) { + this.enable_index_contained_resource = enable_index_contained_resource; + } + public Boolean getEnable_repository_validating_interceptor() { return enable_repository_validating_interceptor; } diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java b/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java index 8154714..95708e7 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/BaseJpaRestfulServer.java @@ -384,5 +384,7 @@ public class BaseJpaRestfulServer extends RestfulServer { } daoConfig.getModelConfig().setNormalizedQuantitySearchLevel(appProperties.getNormalized_quantity_search_level()); + + daoConfig.getModelConfig().setIndexOnContainedResources(appProperties.getEnable_index_contained_resource()); } } diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigCommon.java b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigCommon.java index e73ce18..55eae25 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigCommon.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/FhirServerConfigCommon.java @@ -61,6 +61,10 @@ public class FhirServerConfigCommon { if (appProperties.getSubscription().getEmail() != null) { ourLog.info("Email subscriptions enabled"); } + + if (appProperties.getEnable_index_contained_resource() == Boolean.TRUE) { + ourLog.info("Indexed on contained resource enabled"); + } } /** @@ -163,6 +167,8 @@ public class FhirServerConfigCommon { } modelConfig.setNormalizedQuantitySearchLevel(appProperties.getNormalized_quantity_search_level()); + + modelConfig.setIndexOnContainedResources(appProperties.getEnable_index_contained_resource()); return modelConfig; } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 6d685bd..b348225 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -70,6 +70,7 @@ hapi: # default_page_size: 20 # enable_repository_validating_interceptor: false # enable_index_missing_fields: false +# enable_index_contained_resource: false # enforce_referential_integrity_on_delete: false # enforce_referential_integrity_on_write: false # etag_support_enabled: true From e452468a931faf901f734816d228e26d38f49a58 Mon Sep 17 00:00:00 2001 From: Frank Tao Date: Mon, 22 Mar 2021 20:04:49 -0400 Subject: [PATCH 3/3] Bump up the version to 5.4.0-PRE2-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 218d077..20b3fc7 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ ca.uhn.hapi.fhir hapi-fhir - 5.4.0-PRE1-SNAPSHOT + 5.4.0-PRE2-SNAPSHOT hapi-fhir-jpaserver-starter