[9.0] [Security Solution] Fix "too many clauses" error on prebuilt rules installation page (#223240) (#224282)

nikitaindik · web-flow · commit 5e02f5e97293 · 2025-06-18T11:59:54.000-04:00
# Backport This will backport the following commits from `main` to `9.0`: - [[Security Solution] Fix "too many clauses" error on prebuilt rules installation page (#223240)](#223240)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)
diff --git a/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts b/x-pack/solutions/security/plugins/security_solution/server/lib/detection_engine/prebuilt_rules/logic/rule_assets/prebuilt_rule_assets_client.ts
@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-import { uniqBy } from 'lodash';
+import { chunk, uniqBy } from 'lodash';
+import pMap from 'p-map';
 import type {
   AggregationsMultiBucketAggregateBase,
   AggregationsTopHitsAggregate,
@@ -18,7 +19,10 @@ import { validatePrebuiltRuleAssets } from './prebuilt_rule_assets_validation';
 import { PREBUILT_RULE_ASSETS_SO_TYPE } from './prebuilt_rule_assets_type';
 import type { RuleVersionSpecifier } from '../rule_versions/rule_version_specifier';
 
+const RULE_ASSET_ATTRIBUTES = `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes`;
 const MAX_PREBUILT_RULES_COUNT = 10_000;
+const ES_MAX_CLAUSE_COUNT = 1024;
+const ES_MAX_CONCURRENT_REQUESTS = 2;
 
 export interface IPrebuiltRuleAssetsClient {
   fetchLatestAssets: () => Promise<PrebuiltRuleAsset[]>;
@@ -75,52 +79,66 @@ export const createPrebuiltRuleAssetsClient = (
       });
     },
 
-    fetchLatestVersions: (ruleIds: string[] = []): Promise<RuleVersionSpecifier[]> => {
+    fetchLatestVersions: (ruleIds?: string[]): Promise<RuleVersionSpecifier[]> => {
       return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchLatestVersions', async () => {
-        const filter = ruleIds
-          .map((ruleId) => `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id: ${ruleId}`)
-          .join(' OR ');
+        if (ruleIds && ruleIds.length === 0) {
+          return [];
+        }
 
-        const findResult = await savedObjectsClient.find<
-          PrebuiltRuleAsset,
-          {
-            rules: AggregationsMultiBucketAggregateBase<{
-              latest_version: AggregationsTopHitsAggregate;
-            }>;
-          }
-        >({
-          type: PREBUILT_RULE_ASSETS_SO_TYPE,
-          filter,
-          aggs: {
-            rules: {
-              terms: {
-                field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`,
-                size: MAX_PREBUILT_RULES_COUNT,
-              },
-              aggs: {
-                latest_version: {
-                  top_hits: {
-                    size: 1,
-                    sort: [
-                      {
-                        [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc',
-                      },
-                    ],
-                    _source: [
-                      `${PREBUILT_RULE_ASSETS_SO_TYPE}.rule_id`,
-                      `${PREBUILT_RULE_ASSETS_SO_TYPE}.version`,
-                    ],
+        const fetchLatestVersionInfo = async (filter?: string) => {
+          const findResult = await savedObjectsClient.find<
+            PrebuiltRuleAsset,
+            {
+              rules: AggregationsMultiBucketAggregateBase<{
+                latest_version: AggregationsTopHitsAggregate;
+              }>;
+            }
+          >({
+            type: PREBUILT_RULE_ASSETS_SO_TYPE,
+            filter,
+            aggs: {
+              rules: {
+                terms: {
+                  field: `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id`,
+                  size: MAX_PREBUILT_RULES_COUNT,
+                },
+                aggs: {
+                  latest_version: {
+                    top_hits: {
+                      size: 1,
+                      sort: [
+                        {
+                          [`${PREBUILT_RULE_ASSETS_SO_TYPE}.version`]: 'desc',
+                        },
+                      ],
+                      _source: [
+                        `${PREBUILT_RULE_ASSETS_SO_TYPE}.rule_id`,
+                        `${PREBUILT_RULE_ASSETS_SO_TYPE}.version`,
+                      ],
+                    },
                   },
                 },
               },
             },
-          },
-        });
+          });
 
-        const buckets = findResult.aggregations?.rules?.buckets ?? [];
-        invariant(Array.isArray(buckets), 'Expected buckets to be an array');
+          const aggregatedBuckets = findResult.aggregations?.rules?.buckets ?? [];
+          invariant(Array.isArray(aggregatedBuckets), 'Expected buckets to be an array');
+
+          return aggregatedBuckets;
+        };
+
+        const filters = ruleIds
+          ? createChunkedFilters({
+              items: ruleIds,
+              mapperFn: (ruleId) => `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes.rule_id: ${ruleId}`,
+              clausesPerItem: 2,
+            })
+          : undefined;
 
-        return buckets.map((bucket) => {
+        const buckets = await chunkedFetch(fetchLatestVersionInfo, filters);
+
+        const latestVersions = buckets.map((bucket) => {
           const hit = bucket.latest_version.hits.hits[0];
           const soAttributes = hit._source[PREBUILT_RULE_ASSETS_SO_TYPE];
           const versionInfo: RuleVersionSpecifier = {
@@ -129,31 +147,39 @@ export const createPrebuiltRuleAssetsClient = (
           };
           return versionInfo;
         });
+
+        return latestVersions;
       });
     },
 
     fetchAssetsByVersion: (versions: RuleVersionSpecifier[]): Promise<PrebuiltRuleAsset[]> => {
       return withSecuritySpan('IPrebuiltRuleAssetsClient.fetchAssetsByVersion', async () => {
+        // debugger;
         if (versions.length === 0) {
           // NOTE: without early return it would build incorrect filter and fetch all existing saved objects
           return [];
         }
 
-        const attr = `${PREBUILT_RULE_ASSETS_SO_TYPE}.attributes`;
-        const filter = versions
-          .map((v) => `(${attr}.rule_id: ${v.rule_id} AND ${attr}.version: ${v.version})`)
-          .join(' OR ');
-
-        // Usage of savedObjectsClient.bulkGet() is ~25% more performant and
-        // simplifies deduplication but too many tests get broken.
-        // See https://github.com/elastic/kibana/issues/218198
-        const findResult = await savedObjectsClient.find<PrebuiltRuleAsset>({
-          type: PREBUILT_RULE_ASSETS_SO_TYPE,
-          filter,
-          perPage: MAX_PREBUILT_RULES_COUNT,
+        const filters = createChunkedFilters({
+          items: versions,
+          mapperFn: (versionSpecifier) =>
+            `(${RULE_ASSET_ATTRIBUTES}.rule_id: ${versionSpecifier.rule_id} AND ${RULE_ASSET_ATTRIBUTES}.version: ${versionSpecifier.version})`,
+          clausesPerItem: 4,
         });
 
-        const ruleAssets = findResult.saved_objects.map((so) => so.attributes);
+        const ruleAssets = await chunkedFetch(async (filter) => {
+          // Usage of savedObjectsClient.bulkGet() is ~25% more performant and
+          // simplifies deduplication but too many tests get broken.
+          // See https://github.com/elastic/kibana/issues/218198
+          const findResult = await savedObjectsClient.find<PrebuiltRuleAsset>({
+            type: PREBUILT_RULE_ASSETS_SO_TYPE,
+            filter,
+            perPage: MAX_PREBUILT_RULES_COUNT,
+          });
+
+          return findResult.saved_objects.map((so) => so.attributes);
+        }, filters);
+
         // Rule assets may have duplicates we have to get rid of.
         // In particular prebuilt rule assets package v8.17.1 has duplicates.
         const uniqueRuleAssets = uniqBy(ruleAssets, 'rule_id');
@@ -163,3 +189,50 @@ export const createPrebuiltRuleAssetsClient = (
     },
   };
 };
+
+/**
+ * Creates an array of KQL filter strings for a collection of items.
+ * Uses chunking to ensure that the number of filter clauses does not exceed the ES "too_many_clauses" limit.
+ * See: https://github.com/elastic/kibana/pull/223240
+ *
+ * @param {object} options
+ * @param {T[]} options.items - Array of items to create filters for.
+ * @param {(item: T) => string} options.mapperFn - A function that maps an item to a filter string.
+ * @param {number} options.clausesPerItem - Number of Elasticsearch clauses generated per item. Determined empirically by converting a KQL filter into a Query DSL query.
+ * More complex filters will result in more clauses. Info about clauses in docs: https://www.elastic.co/docs/explore-analyze/query-filter/languages/querydsl#query-dsl
+ * @returns {string[]} An array of filter strings
+ */
+function createChunkedFilters<T>({
+  items,
+  mapperFn,
+  clausesPerItem,
+}: {
+  items: T[];
+  mapperFn: (item: T) => string;
+  clausesPerItem: number;
+}): string[] {
+  return chunk(items, ES_MAX_CLAUSE_COUNT / clausesPerItem).map((singleChunk) =>
+    singleChunk.map(mapperFn).join(' OR ')
+  );
+}
+
+/**
+ * Fetches objects using a provided function.
+ * If filters are provided fetches concurrently in chunks.
+ *
+ * @param {(filter?: string) => Promise<T[]>} chunkFetchFn - Function that fetches a chunk.
+ * @param {string[]} [filters] - An optional array of filter strings. If provided, `chunkFetchFn` will be called for each filter concurrently.
+ * @returns {Promise<T[]>} A promise that resolves to an array of fetched objects.
+ */
+function chunkedFetch<T>(
+  chunkFetchFn: (filter?: string) => Promise<T[]>,
+  filters?: string[]
+): Promise<T[]> {
+  if (filters?.length) {
+    return pMap(filters, chunkFetchFn, {
+      concurrency: ES_MAX_CONCURRENT_REQUESTS,
+    }).then((results) => results.flat());
+  }
+
+  return chunkFetchFn();
+}
