Make the indexing batch size configurable (#20543)

* Introduce configurable batch size for indexing

* Stop using Examine indexing events for reporting index rebuild operation completeness (it is volatile)

Author: kjac

src/Umbraco.Core/Configuration/Models/IndexingSettings.cs

@@ -9,10 +9,16 @@ namespace Umbraco.Cms.Core.Configuration.Models;
 public class IndexingSettings
 {
     private const bool StaticExplicitlyIndexEachNestedProperty = false;
+    private const int StaticBatchSize = 10000;
 
     /// <summary>
     /// Gets or sets a value for whether each nested property should have it's own indexed value. Requires a rebuild of indexes when changed.
     /// </summary>
     [DefaultValue(StaticExplicitlyIndexEachNestedProperty)]
     public bool ExplicitlyIndexEachNestedProperty { get; set; } = StaticExplicitlyIndexEachNestedProperty;
+
+    /// <summary>
+    /// Gets or sets a value for how many items to index at a time.
+    /// </summary>
+    public int BatchSize { get; set; } = StaticBatchSize;
 }

src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs

@@ -1,5 +1,9 @@
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Querying;
 using Umbraco.Cms.Core.Services;
@@ -20,20 +24,45 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
     private readonly bool _publishedValuesOnly;
     private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory;
 
+    private IndexingSettings _indexingSettings;
+
     /// <summary>
     ///     This is a static query, it's parameters don't change so store statically
     /// </summary>
     private IQuery<IContent>? _publishedQuery;
 
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IContentValueSetBuilder contentValueSetBuilder)
+        : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
+    {
+    }
+
     /// <summary>
     ///     Default constructor to lookup all content data
     /// </summary>
     public ContentIndexPopulator(
         ILogger<ContentIndexPopulator> logger,
         IContentService contentService,
         IUmbracoDatabaseFactory umbracoDatabaseFactory,
-        IContentValueSetBuilder contentValueSetBuilder)
-        : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder)
+        IContentValueSetBuilder contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
+        : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, indexingSettings)
+    {
+    }
+
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        bool publishedValuesOnly,
+        int? parentId,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IValueSetBuilder<IContent> contentValueSetBuilder)
+        : this(logger, publishedValuesOnly, parentId, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
     {
     }
 
@@ -46,14 +75,21 @@ public ContentIndexPopulator(
         int? parentId,
         IContentService contentService,
         IUmbracoDatabaseFactory umbracoDatabaseFactory,
-        IValueSetBuilder<IContent> contentValueSetBuilder)
+        IValueSetBuilder<IContent> contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
     {
         _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
         _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
         _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _publishedValuesOnly = publishedValuesOnly;
         _parentId = parentId;
+        _indexingSettings = indexingSettings.CurrentValue;
+
+        indexingSettings.OnChange(change =>
+        {
+            _indexingSettings = change;
+        });
     }
 
     private IQuery<IContent> PublishedQuery => _publishedQuery ??=
@@ -75,7 +111,6 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
             return;
         }
 
-        const int pageSize = 10000;
         var pageIndex = 0;
 
         var contentParentId = -1;
@@ -86,11 +121,11 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
 
         if (_publishedValuesOnly)
         {
-            IndexPublishedContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexPublishedContent(contentParentId, pageIndex, _indexingSettings.BatchSize, indexes);
         }
         else
         {
-            IndexAllContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexAllContent(contentParentId, pageIndex, _indexingSettings.BatchSize, indexes);
         }
     }
 

src/Umbraco.Infrastructure/Examine/DeliveryApiContentIndexHelper.cs

@@ -1,5 +1,7 @@
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
 using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Querying;
 using Umbraco.Cms.Core.Services;
@@ -14,22 +16,33 @@ internal sealed class DeliveryApiContentIndexHelper : IDeliveryApiContentIndexHe
     private readonly IUmbracoDatabaseFactory _umbracoDatabaseFactory;
     private DeliveryApiSettings _deliveryApiSettings;
 
+    private IndexingSettings _indexingSettings;
+
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
     public DeliveryApiContentIndexHelper(
         IContentService contentService,
         IUmbracoDatabaseFactory umbracoDatabaseFactory,
         IOptionsMonitor<DeliveryApiSettings> deliveryApiSettings)
+        : this(contentService,  umbracoDatabaseFactory, deliveryApiSettings, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
+    {
+    }
+
+    public DeliveryApiContentIndexHelper(
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IOptionsMonitor<DeliveryApiSettings> deliveryApiSettings,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
     {
         _contentService = contentService;
         _umbracoDatabaseFactory = umbracoDatabaseFactory;
         _deliveryApiSettings = deliveryApiSettings.CurrentValue;
+        _indexingSettings = indexingSettings.CurrentValue;
         deliveryApiSettings.OnChange(settings => _deliveryApiSettings = settings);
+        indexingSettings.OnChange(settings => _indexingSettings = settings);
     }
 
     public void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform)
-    {
-        const int pageSize = 10000;
-        EnumerateApplicableDescendantsForContentIndex(rootContentId, actionToPerform, pageSize);
-    }
+        => EnumerateApplicableDescendantsForContentIndex(rootContentId, actionToPerform, _indexingSettings.BatchSize);
 
     internal void EnumerateApplicableDescendantsForContentIndex(int rootContentId, Action<IContent[]> actionToPerform, int pageSize)
     {

src/Umbraco.Infrastructure/Examine/MediaIndexPopulator.cs

@@ -1,5 +1,9 @@
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Services;
 
@@ -15,23 +19,43 @@ public class MediaIndexPopulator : IndexPopulator<IUmbracoContentIndex>
     private readonly IValueSetBuilder<IMedia> _mediaValueSetBuilder;
     private readonly int? _parentId;
 
+    private IndexingSettings _indexingSettings;
+
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
+    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
+        : this(logger, null, mediaService, mediaValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
+    {
+    }
+
     /// <summary>
     ///     Default constructor to lookup all content data
     /// </summary>
-    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
-        : this(logger, null, mediaService, mediaValueSetBuilder)
+    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder, IOptionsMonitor<IndexingSettings> indexingSettings)
+        : this(logger, null, mediaService, mediaValueSetBuilder, indexingSettings)
+    {
+    }
+
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
+    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
+        : this(logger, parentId, mediaService, mediaValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
     {
     }
 
     /// <summary>
     ///     Optional constructor allowing specifying custom query parameters
     /// </summary>
-    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder)
+    public MediaIndexPopulator(ILogger<MediaIndexPopulator> logger, int? parentId, IMediaService mediaService, IValueSetBuilder<IMedia> mediaValueSetBuilder, IOptionsMonitor<IndexingSettings> indexingSettings)
     {
         _logger = logger;
         _parentId = parentId;
         _mediaService = mediaService;
         _mediaValueSetBuilder = mediaValueSetBuilder;
+        _indexingSettings = indexingSettings.CurrentValue;
+
+        indexingSettings.OnChange(change =>
+        {
+            _indexingSettings = change;
+        });
     }
 
     protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
@@ -46,7 +70,6 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
             return;
         }
 
-        const int pageSize = 10000;
         var pageIndex = 0;
 
         var mediaParentId = -1;
@@ -60,7 +83,7 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
 
         do
         {
-            media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, pageSize, out _).ToArray();
+            media = _mediaService.GetPagedDescendants(mediaParentId, pageIndex, _indexingSettings.BatchSize, out _).ToArray();
 
             // ReSharper disable once PossibleMultipleEnumeration
             foreach (IIndex index in indexes)
@@ -70,6 +93,6 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
 
             pageIndex++;
         }
-        while (media.Length == pageSize);
+        while (media.Length == _indexingSettings.BatchSize);
     }
 }

src/Umbraco.Infrastructure/Examine/PublishedContentIndexPopulator.cs

@@ -1,4 +1,6 @@
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Infrastructure.Persistence;
 
@@ -15,6 +17,7 @@ namespace Umbraco.Cms.Infrastructure.Examine;
 /// </remarks>
 public class PublishedContentIndexPopulator : ContentIndexPopulator
 {
+    [Obsolete("Please use the non-obsolete constructor. Scheduled for removal in V19.")]
     public PublishedContentIndexPopulator(
         ILogger<PublishedContentIndexPopulator> logger,
         IContentService contentService,
@@ -23,4 +26,14 @@ public PublishedContentIndexPopulator(
         : base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder)
     {
     }
+
+    public PublishedContentIndexPopulator(
+        ILogger<PublishedContentIndexPopulator> logger,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IPublishedContentValueSetBuilder contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
+        : base(logger, true, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder, indexingSettings)
+    {
+    }
 }

src/Umbraco.Infrastructure/Services/IndexingRebuilderService.cs

@@ -42,21 +42,13 @@ public bool TryRebuild(IIndex index, string indexName)
     /// <inheritdoc />
     public async Task<bool> TryRebuildAsync(IIndex index, string indexName)
     {
-        // Remove it in case there's a handler there already
-        index.IndexOperationComplete -= Indexer_IndexOperationComplete;
-
-        // Now add a single handler
-        index.IndexOperationComplete += Indexer_IndexOperationComplete;
-
         try
         {
             Attempt<IndexRebuildResult> attempt = await _indexRebuilder.RebuildIndexAsync(indexName);
             return attempt.Success;
         }
         catch (Exception exception)
         {
-            // Ensure it's not listening
-            index.IndexOperationComplete -= Indexer_IndexOperationComplete;
             _logger.LogError(exception, "An error occurred rebuilding index");
             return false;
         }
@@ -70,19 +62,4 @@ public bool IsRebuilding(string indexName)
     /// <inheritdoc />
     public Task<bool> IsRebuildingAsync(string indexName)
         => _indexRebuilder.IsRebuildingAsync(indexName);
-
-    private void Indexer_IndexOperationComplete(object? sender, EventArgs e)
-    {
-        var indexer = (IIndex?)sender;
-
-        _logger.LogDebug("Logging operation completed for index {IndexName}", indexer?.Name);
-
-        if (indexer is not null)
-        {
-            //ensure it's not listening anymore
-            indexer.IndexOperationComplete -= Indexer_IndexOperationComplete;
-        }
-
-        _logger.LogInformation("Rebuilding index '{IndexerName}' done.", indexer?.Name);
-    }
 }

tests/Umbraco.Tests.Integration/Umbraco.Examine.Lucene/UmbracoExamine/IndexInitializer.cs

@@ -39,6 +39,7 @@ public class IndexInitializer
     private readonly IContentTypeService _contentTypeService;
     private readonly IDocumentUrlService _documentUrlService;
     private readonly ILanguageService _languageService;
+    private readonly IOptionsMonitor<IndexingSettings> _indexSettings;
 
     public IndexInitializer(
         IShortStringHelper shortStringHelper,
@@ -50,7 +51,8 @@ public IndexInitializer(
         ILocalizationService localizationService,
         IContentTypeService contentTypeService,
         IDocumentUrlService documentUrlService,
-        ILanguageService languageService)
+        ILanguageService languageService,
+        IOptionsMonitor<IndexingSettings> indexSettings)
     {
         _shortStringHelper = shortStringHelper;
         _propertyEditors = propertyEditors;
@@ -62,6 +64,7 @@ public IndexInitializer(
         _contentTypeService = contentTypeService;
         _documentUrlService = documentUrlService;
         _languageService = languageService;
+        _indexSettings = indexSettings;
     }
 
     public ContentValueSetBuilder GetContentValueSetBuilder(bool publishedValuesOnly)
@@ -91,7 +94,8 @@ public ContentIndexPopulator GetContentIndexRebuilder(IContentService contentSer
             null,
             contentService,
             umbracoDatabaseFactory,
-            contentValueSetBuilder);
+            contentValueSetBuilder,
+            _indexSettings);
         return contentIndexDataSource;
     }
 
@@ -105,7 +109,7 @@ public MediaIndexPopulator GetMediaIndexRebuilder(IMediaService mediaService)
             _shortStringHelper,
             _contentSettings,
             StaticServiceProvider.Instance.GetRequiredService<IContentTypeService>());
-        var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder);
+        var mediaIndexDataSource = new MediaIndexPopulator(null, mediaService, mediaValueSetBuilder, _indexSettings);
         return mediaIndexDataSource;
     }