Feat/extra resource loading from npm (#784)

* Logic to do the basics

* Added configurable folders

* Scoped additional resources to - pr. IG

* Updated telemetry version

* Changed to set

* Changed PostConstruct to Component and introduced use of PartitionCondition
Added try-load of HAPI optimized dialect

* Satisfying spotless

* Reverted to default config

* Apply spotless

* Use core-provided feature
Addling the binary content as its needed for example resource inspection

* Update src/main/java/ca/uhn/fhir/jpa/starter/util/JpaHibernatePropertiesProvider.java

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Reverting dialect change - fit for a potential other PR

* no message

* Formatting

* Update src/main/java/ca/uhn/fhir/jpa/starter/ig/ExtendedPackageInstallationSpec.java

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Bad AI ... bad bad AI ...

* Formatting

* bump of feature revision flag

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

src/main/java/ca/uhn/fhir/jpa/starter/AppProperties.java

@@ -5,7 +5,7 @@ import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings.ClientIdStrategyEnum;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings.IdStrategyEnum;
 import ca.uhn.fhir.jpa.model.entity.NormalizedQuantitySearchLevel;
-import ca.uhn.fhir.jpa.packages.PackageInstallationSpec;
+import ca.uhn.fhir.jpa.starter.ig.ExtendedPackageInstallationSpec;
 import ca.uhn.fhir.rest.api.EncodingEnum;
 import org.hl7.fhir.r4.model.Bundle;
 import org.springframework.boot.context.properties.ConfigurationProperties;
@@ -88,7 +88,9 @@ public class AppProperties {
 	private Partitioning partitioning = null;
 	private Boolean validate_resource_status_for_package_upload = true;
 	private Boolean install_transitive_ig_dependencies = true;
-	private Map<String, PackageInstallationSpec> implementationGuides = null;
+
+	private List<String> install_additional_resources_from_ig_folders = new ArrayList<>();
+	private Map<String, ExtendedPackageInstallationSpec> implementationGuides = null;
 	private String custom_content_path = null;
 	private String app_content_path = null;
 	private Boolean lastn_enabled = false;
@@ -157,11 +159,11 @@ public class AppProperties {
 		this.defer_indexing_for_codesystems_of_size = defer_indexing_for_codesystems_of_size;
 	}
 
-	public Map<String, PackageInstallationSpec> getImplementationGuides() {
+	public Map<String, ExtendedPackageInstallationSpec> getImplementationGuides() {
 		return implementationGuides;
 	}
 
-	public void setImplementationGuides(Map<String, PackageInstallationSpec> implementationGuides) {
+	public void setImplementationGuides(Map<String, ExtendedPackageInstallationSpec> implementationGuides) {
 		this.implementationGuides = implementationGuides;
 	}
 
@@ -710,6 +712,15 @@ public class AppProperties {
 		this.resource_dbhistory_enabled = resource_dbhistory_enabled;
 	}
 
+	public List<String> getInstall_additional_resources_from_ig_folders() {
+		return install_additional_resources_from_ig_folders;
+	}
+
+	public void setInstall_additional_resources_from_ig_folders(
+			List<String> install_additional_resources_from_ig_folders) {
+		this.install_additional_resources_from_ig_folders = install_additional_resources_from_ig_folders;
+	}
+
 	public Boolean getPre_expand_value_sets() {
 		return this.pre_expand_value_sets;
 	}

src/main/java/ca/uhn/fhir/jpa/starter/common/FhirServerConfigCommon.java

@@ -325,11 +325,6 @@ public class FhirServerConfigCommon {
 		return retVal;
 	}
 
-	@Bean
-	public PartitionModeConfigurer partitionModeConfigurer() {
-		return new PartitionModeConfigurer();
-	}
-
 	@Primary
 	@Bean
 	public HibernatePropertiesProvider jpaStarterDialectProvider(

src/main/java/ca/uhn/fhir/jpa/starter/common/OnPartitionModeEnabled.java

@@ -0,0 +1,18 @@
+package ca.uhn.fhir.jpa.starter.common;
+
+import ca.uhn.fhir.jpa.starter.AppProperties;
+import org.springframework.boot.context.properties.bind.Binder;
+import org.springframework.context.annotation.Condition;
+import org.springframework.context.annotation.ConditionContext;
+import org.springframework.core.type.AnnotatedTypeMetadata;
+
+public class OnPartitionModeEnabled implements Condition {
+	@Override
+	public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
+		var appProperties = Binder.get(context.getEnvironment())
+				.bind("hapi.fhir", AppProperties.class)
+				.orElse(null);
+		if (appProperties == null) return false;
+		return appProperties.getPartitioning() != null;
+	}
+}

src/main/java/ca/uhn/fhir/jpa/starter/common/PartitionModeConfigurer.java

@@ -1,6 +1,5 @@
 package ca.uhn.fhir.jpa.starter.common;
 
-import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.jpa.interceptor.PatientIdPartitionInterceptor;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.partition.PartitionManagementProvider;
@@ -9,48 +8,36 @@ import ca.uhn.fhir.jpa.starter.AppProperties;
 import ca.uhn.fhir.rest.server.RestfulServer;
 import ca.uhn.fhir.rest.server.interceptor.partition.RequestTenantPartitionInterceptor;
 import ca.uhn.fhir.rest.server.tenant.UrlBaseTenantIdentificationStrategy;
-import jakarta.annotation.PostConstruct;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
 
+@Component
+@Conditional({OnPartitionModeEnabled.class})
 public class PartitionModeConfigurer {
 	private static final Logger ourLog = LoggerFactory.getLogger(PartitionModeConfigurer.class);
 
-	@Autowired
-	private AppProperties myAppProperties;
+	public PartitionModeConfigurer(
+			AppProperties myAppProperties,
+			ISearchParamExtractor mySearchParamExtractor,
+			PartitionSettings myPartitionSettings,
+			RestfulServer myRestfulServer,
+			PartitionManagementProvider myPartitionManagementProvider) {
 
-	@Autowired
-	private FhirContext myFhirContext;
-
-	@Autowired
-	private ISearchParamExtractor mySearchParamExtractor;
-
-	@Autowired
-	private PartitionSettings myPartitionSettings;
-
-	@Autowired
-	private RestfulServer myRestfulServer;
-
-	@Autowired
-	private PartitionManagementProvider myPartitionManagementProvider;
-
-	@PostConstruct
-	public void start() {
-		if (myAppProperties.getPartitioning() != null) {
-			if (myAppProperties.getPartitioning().getPatient_id_partitioning_mode() == Boolean.TRUE) {
-				ourLog.info("Partitioning mode enabled in: Patient ID partitioning mode");
-				PatientIdPartitionInterceptor patientIdInterceptor =
-						new PatientIdPartitionInterceptor(myFhirContext, mySearchParamExtractor, myPartitionSettings);
-				myRestfulServer.registerInterceptor(patientIdInterceptor);
-				myPartitionSettings.setUnnamedPartitionMode(true);
-			} else if (myAppProperties.getPartitioning().getRequest_tenant_partitioning_mode() == Boolean.TRUE) {
-				ourLog.info("Partitioning mode enabled in: Request tenant partitioning mode");
-				myRestfulServer.registerInterceptor(new RequestTenantPartitionInterceptor());
-				myRestfulServer.setTenantIdentificationStrategy(new UrlBaseTenantIdentificationStrategy());
-			}
-
-			myRestfulServer.registerProviders(myPartitionManagementProvider);
+		var partitioning = myAppProperties.getPartitioning();
+		if (partitioning.getPatient_id_partitioning_mode()) {
+			ourLog.info("Partitioning mode enabled in: Patient ID partitioning mode");
+			var patientIdInterceptor = new PatientIdPartitionInterceptor(
+					myRestfulServer.getFhirContext(), mySearchParamExtractor, myPartitionSettings);
+			myRestfulServer.registerInterceptor(patientIdInterceptor);
+			myPartitionSettings.setUnnamedPartitionMode(true);
+		} else if (partitioning.getRequest_tenant_partitioning_mode()) {
+			ourLog.info("Partitioning mode enabled in: Request tenant partitioning mode");
+			myRestfulServer.registerInterceptor(new RequestTenantPartitionInterceptor());
+			myRestfulServer.setTenantIdentificationStrategy(new UrlBaseTenantIdentificationStrategy());
 		}
+
+		myRestfulServer.registerProviders(myPartitionManagementProvider);
 	}
 }

src/main/java/ca/uhn/fhir/jpa/starter/common/StarterJpaConfig.java

@@ -9,6 +9,7 @@ import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.context.FhirVersionEnum;
 import ca.uhn.fhir.context.support.IValidationSupport;
 import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
+import ca.uhn.fhir.interceptor.model.RequestPartitionId;
 import ca.uhn.fhir.jpa.api.IDaoRegistry;
 import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
 import ca.uhn.fhir.jpa.api.config.ThreadPoolFactoryConfig;
@@ -20,6 +21,7 @@ import ca.uhn.fhir.jpa.config.util.HapiEntityManagerFactoryUtil;
 import ca.uhn.fhir.jpa.config.util.ResourceCountCacheUtil;
 import ca.uhn.fhir.jpa.dao.FulltextSearchSvcImpl;
 import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
+import ca.uhn.fhir.jpa.dao.TransactionProcessor;
 import ca.uhn.fhir.jpa.dao.search.HSearchSortHelperImpl;
 import ca.uhn.fhir.jpa.dao.search.IHSearchSortHelper;
 import ca.uhn.fhir.jpa.delete.ThreadSafeResourceDeleterSvc;
@@ -29,8 +31,9 @@ import ca.uhn.fhir.jpa.interceptor.UserRequestRetryVersionConflictsInterceptor;
 import ca.uhn.fhir.jpa.interceptor.validation.RepositoryValidatingInterceptor;
 import ca.uhn.fhir.jpa.ips.provider.IpsOperationProvider;
 import ca.uhn.fhir.jpa.model.config.SubscriptionSettings;
+import ca.uhn.fhir.jpa.packages.AdditionalResourcesParser;
+import ca.uhn.fhir.jpa.packages.IHapiPackageCacheManager;
 import ca.uhn.fhir.jpa.packages.IPackageInstallerSvc;
-import ca.uhn.fhir.jpa.packages.PackageInstallationSpec;
 import ca.uhn.fhir.jpa.provider.DaoRegistryResourceSupportedSvc;
 import ca.uhn.fhir.jpa.provider.DiffProvider;
 import ca.uhn.fhir.jpa.provider.IJpaSystemProvider;
@@ -47,6 +50,7 @@ import ca.uhn.fhir.jpa.starter.AppProperties;
 import ca.uhn.fhir.jpa.starter.annotations.OnCorsPresent;
 import ca.uhn.fhir.jpa.starter.annotations.OnImplementationGuidesPresent;
 import ca.uhn.fhir.jpa.starter.common.validation.IRepositoryValidationInterceptorFactory;
+import ca.uhn.fhir.jpa.starter.ig.ExtendedPackageInstallationSpec;
 import ca.uhn.fhir.jpa.starter.ig.IImplementationGuideOperationProvider;
 import ca.uhn.fhir.jpa.starter.util.EnvironmentHelper;
 import ca.uhn.fhir.jpa.subscription.util.SubscriptionDebugLogInterceptor;
@@ -55,6 +59,7 @@ import ca.uhn.fhir.mdm.provider.MdmProviderLoader;
 import ca.uhn.fhir.narrative.DefaultThymeleafNarrativeGenerator;
 import ca.uhn.fhir.narrative2.NullNarrativeGenerator;
 import ca.uhn.fhir.rest.api.IResourceSupportedSvc;
+import ca.uhn.fhir.rest.api.server.SystemRequestDetails;
 import ca.uhn.fhir.rest.openapi.OpenApiInterceptor;
 import ca.uhn.fhir.rest.server.ApacheProxyAddressStrategy;
 import ca.uhn.fhir.rest.server.ETagSupportEnum;
@@ -74,6 +79,7 @@ import ca.uhn.fhir.validation.IValidatorModule;
 import ca.uhn.fhir.validation.ResultSeverityEnum;
 import com.google.common.base.Strings;
 import jakarta.persistence.EntityManagerFactory;
+import org.hl7.fhir.instance.model.api.IBaseBundle;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.NoSuchBeanDefinitionException;
@@ -93,11 +99,8 @@ import org.springframework.orm.jpa.JpaTransactionManager;
 import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
 import org.springframework.web.cors.CorsConfiguration;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.io.IOException;
+import java.util.*;
 import javax.sql.DataSource;
 
 import static ca.uhn.fhir.jpa.starter.common.validation.IRepositoryValidationInterceptorFactory.ENABLE_REPOSITORY_VALIDATING_INTERCEPTOR;
@@ -207,14 +210,18 @@ public class StarterJpaConfig {
 	public IPackageInstallerSvc packageInstaller(
 			AppProperties appProperties,
 			IPackageInstallerSvc packageInstallerSvc,
-			Batch2JobRegisterer batch2JobRegisterer) {
+			Batch2JobRegisterer batch2JobRegisterer,
+			FhirContext fhirContext,
+			TransactionProcessor transactionProcessor,
+			IHapiPackageCacheManager iHapiPackageCacheManager)
+			throws IOException {
 
 		batch2JobRegisterer.start();
 
 		if (appProperties.getImplementationGuides() != null) {
-			Map<String, PackageInstallationSpec> guides = appProperties.getImplementationGuides();
-			for (Map.Entry<String, PackageInstallationSpec> guidesEntry : guides.entrySet()) {
-				PackageInstallationSpec packageInstallationSpec = guidesEntry.getValue();
+			Map<String, ExtendedPackageInstallationSpec> guides = appProperties.getImplementationGuides();
+			for (Map.Entry<String, ExtendedPackageInstallationSpec> guidesEntry : guides.entrySet()) {
+				ExtendedPackageInstallationSpec packageInstallationSpec = guidesEntry.getValue();
 				if (appProperties.getInstall_transitive_ig_dependencies()) {
 
 					packageInstallationSpec
@@ -223,7 +230,22 @@ public class StarterJpaConfig {
 							.addDependencyExclude("hl7.fhir.r4.core")
 							.addDependencyExclude("hl7.fhir.r5.core");
 				}
+
 				packageInstallerSvc.install(packageInstallationSpec);
+
+				Set<String> extraResources = packageInstallationSpec.getAdditionalResourceFolders();
+				packageInstallationSpec.setPackageContents(iHapiPackageCacheManager
+						.loadPackageContents(packageInstallationSpec.getName(), packageInstallationSpec.getVersion())
+						.getBytes());
+
+				if (extraResources != null && !extraResources.isEmpty()) {
+					IBaseBundle transaction = AdditionalResourcesParser.bundleAdditionalResources(
+							extraResources, packageInstallationSpec, fhirContext);
+					transactionProcessor.transaction(
+							new SystemRequestDetails().setRequestPartitionId(RequestPartitionId.defaultPartition()),
+							transaction,
+							false);
+				}
 			}
 		}
 		return packageInstallerSvc;

src/main/java/ca/uhn/fhir/jpa/starter/ig/ExtendedPackageInstallationSpec.java

@@ -0,0 +1,24 @@
+package ca.uhn.fhir.jpa.starter.ig;
+
+import ca.uhn.fhir.jpa.packages.PackageInstallationSpec;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import java.util.Set;
+
+public class ExtendedPackageInstallationSpec extends PackageInstallationSpec {
+
+	public Set<String> getAdditionalResourceFolders() {
+		return additionalResourceFolders;
+	}
+
+	public void setAdditionalResourceFolders(Set<String> additionalResourceFolders) {
+		this.additionalResourceFolders = additionalResourceFolders;
+	}
+
+	@Schema(
+			description =
+					"Specifies folder names containing additional resources to load. These folders will be scanned for resources to include during installation.")
+	@JsonProperty("additionalResourceFolders")
+	private Set<String> additionalResourceFolders;
+}

src/main/resources/application.yaml

@@ -171,10 +171,14 @@ hapi:
     #    reloadExisting: false
     #    installMode: STORE_AND_INSTALL
     #      example not from registry
-    #      ips_1_0_0:
-    #        packageUrl: https://build.fhir.org/ig/HL7/fhir-ips/package.tgz
-    #        name: hl7.fhir.uv.ips
-    #        version: 1.0.0
+    #  ips_1_0_0:
+    #    packageUrl: https://costateixeira.github.io/smart-ips-pilgrimage-fulltest/package.tgz
+    #    name: smart.who.int.ips-pilgrimage-test
+    #    version: 0.1.0
+    #    installMode: STORE_AND_INSTALL
+    #    additionalResourceFolders:
+    #      - example
+        #  - example2
     #    supported_resource_types:
     #      - Patient
     #      - Observation