Merge cebf9b5 into 4482622

Author: michaelabuckley

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4915-mb-include-canonical-url-performance.yaml

@@ -0,0 +1,4 @@
+---
+type: perf
+issue: 4915
+title: "Includes by canonical url now use an indexed query, and are much faster."

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/SearchBuilder.java

@@ -48,6 +48,7 @@
 import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
 import ca.uhn.fhir.jpa.model.config.PartitionSettings;
 import ca.uhn.fhir.jpa.model.dao.JpaPid;
+import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
 import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
 import ca.uhn.fhir.jpa.model.entity.ResourceTag;
 import ca.uhn.fhir.jpa.model.search.SearchBuilderLoadIncludesParameters;
@@ -97,6 +98,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.hl7.fhir.instance.model.api.IAnyResource;
 import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.slf4j.Logger;
@@ -107,6 +109,7 @@
 import org.springframework.transaction.support.TransactionSynchronizationManager;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.persistence.EntityManager;
 import javax.persistence.PersistenceContext;
 import javax.persistence.PersistenceContextType;
@@ -1305,73 +1308,53 @@ public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theP
 					paths = param.getPathsSplitForResourceType(resType);
 					// end replace
 
-					String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
+					Set<String> targetResourceTypes = computeTargetResourceTypes(nextInclude, param);
+
 					for (String nextPath : paths) {
-						boolean haveTargetTypesDefinedByParam = param.hasTargets();
 						String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID) ? "src_resource_id" : "target_resource_id";
 						String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS;
 						if (findVersionFieldName != null) {
 							fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS;
 						}
-
-						// Query for includes lookup has consider 2 cases
+						
+						// Query for includes lookup has 2 cases
 						// Case 1: Where target_resource_id is available in hfj_res_link table for local references
 						// Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical url in target_resource_url
 
 						// Case 1:
+						Map<String, Object> localReferenceQueryParams = new HashMap<>();
+
 						String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID) ? "target_resource_id" : "src_resource_id";
-						StringBuilder resourceIdBasedQuery = new StringBuilder("SELECT " + fieldsToLoad +
+						StringBuilder localReferenceQuery = new StringBuilder("SELECT " + fieldsToLoad +
 							" FROM hfj_res_link r " +
 							" WHERE r.src_path = :src_path AND " +
 							" r.target_resource_id IS NOT NULL AND " +
 							" r." + searchPidFieldSqlColumn + " IN (:target_pids) ");
-						if (targetResourceType != null) {
-							resourceIdBasedQuery.append(" AND r.target_resource_type = :target_resource_type ");
-						} else if (haveTargetTypesDefinedByParam) {
-							resourceIdBasedQuery.append(" AND r.target_resource_type in (:target_resource_types) ");
-						}
-
-						// Case 2:
-						String fieldsToLoadFromSpidxUriTable = "rUri.res_id";
-						// to match the fields loaded in union
-						if (fieldsToLoad.split(",").length > 1) {
-							for (int i = 0; i < fieldsToLoad.split(",").length - 1; i++) {
-								fieldsToLoadFromSpidxUriTable += ", NULL";
+						localReferenceQueryParams.put("src_path", nextPath);
+						// we loop over target_pids later.
+						if (targetResourceTypes != null) {
+							if (targetResourceTypes.size() == 1) {
+								localReferenceQuery.append(" AND r.target_resource_type = :target_resource_type ");
+								localReferenceQueryParams.put("target_resource_type", targetResourceTypes.iterator().next());
+							} else {
+								localReferenceQuery.append(" AND r.target_resource_type in (:target_resource_types) ");
+								localReferenceQueryParams.put("target_resource_types", targetResourceTypes);
 							}
 						}
-						//@formatter:off
-						StringBuilder resourceUrlBasedQuery = new StringBuilder("SELECT " + fieldsToLoadFromSpidxUriTable +
-							" FROM hfj_res_link r " +
-							" JOIN hfj_spidx_uri rUri ON ( " +
-							"   r.target_resource_url = rUri.sp_uri AND " +
-							"   rUri.sp_name = 'url' ");
-
-						if (targetResourceType != null) {
-							resourceUrlBasedQuery.append(" AND rUri.res_type = :target_resource_type ");
 
-						} else if (haveTargetTypesDefinedByParam) {
-							resourceUrlBasedQuery.append(" AND rUri.res_type IN (:target_resource_types) ");
-						}
+						// Case 2:
+						Pair<String, Map<String, Object>> canonicalQuery = buildCanonicalUrlQuery(findVersionFieldName, searchPidFieldSqlColumn, targetResourceTypes);
 
-						resourceUrlBasedQuery.append(" ) ");
-						resourceUrlBasedQuery.append(
-							" WHERE r.src_path = :src_path AND " +
-								" r.target_resource_id IS NULL AND " +
-								" r." + searchPidFieldSqlColumn + " IN (:target_pids) ");
 						//@formatter:on
 
-						String sql = resourceIdBasedQuery + " UNION " + resourceUrlBasedQuery;
+						String sql = localReferenceQuery + " UNION " + canonicalQuery.getLeft();
 
 						List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize());
 						for (Collection<JpaPid> nextPartition : partitions) {
 							Query q = entityManager.createNativeQuery(sql, Tuple.class);
-							q.setParameter("src_path", nextPath);
 							q.setParameter("target_pids", JpaPid.toLongList(nextPartition));
-							if (targetResourceType != null) {
-								q.setParameter("target_resource_type", targetResourceType);
-							} else if (haveTargetTypesDefinedByParam) {
-								q.setParameter("target_resource_types", param.getTargets());
-							}
+							localReferenceQueryParams.forEach(q::setParameter);
+							canonicalQuery.getRight().forEach(q::setParameter);
 
 							if (maxCount != null) {
 								q.setMaxResults(maxCount);
@@ -1395,7 +1378,7 @@ public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theP
 
 			nextRoundMatches.clear();
 			for (JpaPid next : pidsToInclude) {
-				if (original.contains(next) == false && allAdded.contains(next) == false) {
+				if ( !original.contains(next) && !allAdded.contains(next) ) {
 					nextRoundMatches.add(next);
 				}
 			}
@@ -1406,7 +1389,7 @@ public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theP
 				break;
 			}
 
-		} while (includes.size() > 0 && nextRoundMatches.size() > 0 && addedSomeThisRound);
+		} while (!includes.isEmpty() && !nextRoundMatches.isEmpty() && addedSomeThisRound);
 
 		allAdded.removeAll(original);
 
@@ -1415,7 +1398,7 @@ public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theP
 		// Interceptor call: STORAGE_PREACCESS_RESOURCES
 		// This can be used to remove results from the search result details before
 		// the user has a chance to know that they were in the results
-		if (allAdded.size() > 0) {
+		if (!allAdded.isEmpty()) {
 
 			if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, request)) {
 				List<JpaPid> includedPidList = new ArrayList<>(allAdded);
@@ -1440,6 +1423,62 @@ public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theP
 		return allAdded;
 	}
 
+	@Nullable
+	private static Set<String> computeTargetResourceTypes(Include nextInclude, RuntimeSearchParam param) {
+		String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
+		boolean haveTargetTypesDefinedByParam = param.hasTargets();
+		Set<String> targetResourceTypes;
+		if (targetResourceType != null) {
+			targetResourceTypes = Set.of(targetResourceType);
+		} else if (haveTargetTypesDefinedByParam) {
+			targetResourceTypes = param.getTargets();
+		} else {
+			// all types!
+			targetResourceTypes = null;
+		}
+		return targetResourceTypes;
+	}
+
+	@Nonnull
+	private Pair<String, Map<String, Object>> buildCanonicalUrlQuery(String theVersionFieldName, String thePidFieldSqlColumn, Set<String> theTargetResourceTypes) {
+		String fieldsToLoadFromSpidxUriTable = "rUri.res_id";
+		if (theVersionFieldName != null) {
+			// canonical-uri references aren't versioned, but we need to match the column count for the UNION
+			fieldsToLoadFromSpidxUriTable += ", NULL";
+		}
+		// The logical join will be by hfj_spidx_uri on sp_name='uri' and sp_uri=target_resource_url.
+		// But sp_name isn't indexed, so we use hash_identity instead.
+		if (theTargetResourceTypes == null) {
+			// hash_identity includes the resource type.  So a null wildcard must be replaced with a list of all types.
+			theTargetResourceTypes = myDaoRegistry.getRegisteredDaoTypes();
+		}
+		assert !theTargetResourceTypes.isEmpty();
+
+		Set<Long> identityHashesForTypes = theTargetResourceTypes.stream()
+			.map(type-> BaseResourceIndexedSearchParam.calculateHashIdentity(myPartitionSettings, myRequestPartitionId, type, "url"))
+			.collect(Collectors.toSet());
+
+		Map<String, Object> canonicalUriQueryParams = new HashMap<>();
+		StringBuilder canonicalUrlQuery = new StringBuilder(
+			"SELECT " + fieldsToLoadFromSpidxUriTable +
+			" FROM hfj_res_link r " +
+			" JOIN hfj_spidx_uri rUri ON ( ");
+		// join on hash_identity and sp_uri - indexed in IDX_SP_URI_HASH_IDENTITY_V2
+		if (theTargetResourceTypes.size() == 1) {
+			canonicalUrlQuery.append("   rUri.hash_identity = :uri_identity_hash ");
+			canonicalUriQueryParams.put("uri_identity_hash", identityHashesForTypes.iterator().next());
+		} else {
+			canonicalUrlQuery.append("   rUri.hash_identity in (:uri_identity_hashes) ");
+			canonicalUriQueryParams.put("uri_identity_hashes", identityHashesForTypes);
+		}
+
+		canonicalUrlQuery.append("  AND r.target_resource_url = rUri.sp_uri  )" +
+			" WHERE r.src_path = :src_path AND " +
+				" r.target_resource_id IS NULL AND " +
+				" r." + thePidFieldSqlColumn + " IN (:target_pids) ");
+		return Pair.of(canonicalUrlQuery.toString(), canonicalUriQueryParams);
+	}
+
 	private List<Collection<JpaPid>> partition(Collection<JpaPid> theNextRoundMatches, int theMaxLoad) {
 		if (theNextRoundMatches.size() <= theMaxLoad) {
 			return Collections.singletonList(theNextRoundMatches);