POC: Feat/make indexing more resiliant

src/Umbraco.Core/Configuration/Models/IndexingSettings.cs

@@ -15,4 +15,8 @@ public class IndexingSettings
     /// </summary>
     [DefaultValue(StaticExplicitlyIndexEachNestedProperty)]
     public bool ExplicitlyIndexEachNestedProperty { get; set; } = StaticExplicitlyIndexEachNestedProperty;
+    /// <summary>
+    /// Defines how many items to index in a single page
+    /// </summary>
+    public int IndexingPageSize { get; set; } = 10000;
 }

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs

@@ -57,7 +57,7 @@ public static IUmbracoBuilder AddExamine(this IUmbracoBuilder builder)
         builder.Services.AddUnique<IDeliveryApiContentIndexFieldDefinitionBuilder, DeliveryApiContentIndexFieldDefinitionBuilder>();
         builder.Services.AddUnique<IDeliveryApiContentIndexHelper, DeliveryApiContentIndexHelper>();
         builder.Services.AddSingleton<IDeliveryApiIndexingHandler, DeliveryApiIndexingHandler>();
-
+        builder.Services.AddSingleton<IIndexRebuildStatusManager, IndexRebuildStatusManager>();
         builder.Services.AddSingleton<ExamineIndexRebuilder>();  //TODO remove in Umbraco 15. Only the interface should be in the service provider
 
         builder.Services.AddUnique<IDeliveryApiCompositeIdHandler, DeliveryApiCompositeIdHandler>();

src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs

@@ -1,5 +1,9 @@
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Querying;
 using Umbraco.Cms.Core.Services;
@@ -14,6 +18,7 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
 {
     private readonly IContentService _contentService;
     private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
+    private IndexingSettings _indexingSettings;
     private readonly ILogger<ContentIndexPopulator> _logger;
     private readonly int? _parentId;
 
@@ -28,6 +33,7 @@ public class ContentIndexPopulator : IndexPopulator<IUmbracoContentIndex>
     /// <summary>
     ///     Default constructor to lookup all content data
     /// </summary>
+    [Obsolete("Use the constructor with IOptionsMonitor<IndexingSettings>")]
     public ContentIndexPopulator(
         ILogger<ContentIndexPopulator> logger,
         IContentService contentService,
@@ -40,13 +46,14 @@ public ContentIndexPopulator(
     /// <summary>
     ///     Optional constructor allowing specifying custom query parameters
     /// </summary>
+    [Obsolete("Use the constructor with IOptionsMonitor<IndexingSettings>")]
     public ContentIndexPopulator(
         ILogger<ContentIndexPopulator> logger,
         bool publishedValuesOnly,
         int? parentId,
         IContentService contentService,
         IUmbracoDatabaseFactory umbracoDatabaseFactory,
-        IValueSetBuilder<IContent> contentValueSetBuilder)
+        IValueSetBuilder<IContent> contentValueSetBuilder) : this(logger, publishedValuesOnly, parentId, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
     {
         _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
         _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
@@ -55,7 +62,37 @@ public ContentIndexPopulator(
         _publishedValuesOnly = publishedValuesOnly;
         _parentId = parentId;
     }
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IContentValueSetBuilder contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
+        : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder,indexingSettings)
+    {
+    }
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        bool publishedValuesOnly,
+        int? parentId,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IValueSetBuilder<IContent> contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
+    {
+        _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
+        _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
+        _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
+        _indexingSettings = indexingSettings.CurrentValue;
+        indexingSettings.OnChange(change =>
+        {
+            _indexingSettings = change;
+        });
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _publishedValuesOnly = publishedValuesOnly;
+        _parentId = parentId;
 
+    }
     private IQuery<IContent> PublishedQuery => _publishedQuery ??=
         _umbracoDatabaseFactory.SqlContext.Query<IContent>().Where(x => x.Published);
 
@@ -75,7 +112,7 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
             return;
         }
 
-        const int pageSize = 10000;
+
         var pageIndex = 0;
 
         var contentParentId = -1;
@@ -86,11 +123,11 @@ protected override void PopulateIndexes(IReadOnlyList<IIndex> indexes)
 
         if (_publishedValuesOnly)
         {
-            IndexPublishedContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexPublishedContent(contentParentId, pageIndex, _indexingSettings.IndexingPageSize, indexes);
         }
         else
         {
-            IndexAllContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexAllContent(contentParentId, pageIndex, _indexingSettings.IndexingPageSize, indexes);
         }
     }
 

src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs

@@ -2,8 +2,10 @@
 // See LICENSE for more details.
 
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Infrastructure.HostedServices;
@@ -16,6 +18,7 @@ public class ExamineIndexRebuilder : IIndexRebuilder
     private readonly IBackgroundTaskQueue _backgroundTaskQueue;
     private readonly IExamineManager _examineManager;
     private readonly ILogger<ExamineIndexRebuilder> _logger;
+    private readonly IIndexRebuildStatusManager _indexRebuildStatusManager;
     private readonly IMainDom _mainDom;
     private readonly IEnumerable<IIndexPopulator> _populators;
     private readonly object _rebuildLocker = new();
@@ -24,17 +27,29 @@ public class ExamineIndexRebuilder : IIndexRebuilder
     /// <summary>
     ///     Initializes a new instance of the <see cref="ExamineIndexRebuilder" /> class.
     /// </summary>
+    [Obsolete("Use constructor with IIndexRebuildStatusManager. This is scheduled for removal in Umbraco 15.")]
     public ExamineIndexRebuilder(
         IMainDom mainDom,
         IRuntimeState runtimeState,
         ILogger<ExamineIndexRebuilder> logger,
         IExamineManager examineManager,
         IEnumerable<IIndexPopulator> populators,
+        IBackgroundTaskQueue backgroundTaskQueue) : this(mainDom, runtimeState, logger, StaticServiceProvider.Instance.GetRequiredService<IIndexRebuildStatusManager>(), examineManager, populators, backgroundTaskQueue)
+    {
+    }
+    public ExamineIndexRebuilder(
+        IMainDom mainDom,
+        IRuntimeState runtimeState,
+        ILogger<ExamineIndexRebuilder> logger,
+        IIndexRebuildStatusManager indexRebuildStatusManager,
+        IExamineManager examineManager,
+        IEnumerable<IIndexPopulator> populators,
         IBackgroundTaskQueue backgroundTaskQueue)
     {
         _mainDom = mainDom;
         _runtimeState = runtimeState;
         _logger = logger;
+        _indexRebuildStatusManager = indexRebuildStatusManager;
         _examineManager = examineManager;
         _populators = populators;
         _backgroundTaskQueue = backgroundTaskQueue;
@@ -147,7 +162,7 @@ private void RebuildIndex(string indexName, TimeSpan delay, CancellationToken ca
                 {
                     throw new InvalidOperationException($"No index found with name {indexName}");
                 }
-
+                _indexRebuildStatusManager.SetRebuildingIndexStatus([indexName], true);
                 index.CreateIndex(); // clear the index
                 foreach (IIndexPopulator populator in _populators)
                 {
@@ -158,6 +173,7 @@ private void RebuildIndex(string indexName, TimeSpan delay, CancellationToken ca
 
                     populator.Populate(index);
                 }
+                _indexRebuildStatusManager.SetRebuildingIndexStatus([indexName], false);
             }
         }
         finally
@@ -175,6 +191,7 @@ private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationT
         {
             Thread.Sleep(delay);
         }
+        IEnumerable<string>? rebuildedIndex = null;
 
         try
         {
@@ -190,7 +207,8 @@ private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationT
                     ? _examineManager.Indexes.Where(x =>
                         !x.IndexExists() || (x is IIndexStats stats && stats.GetDocumentCount() == 0))
                     : _examineManager.Indexes).ToArray();
-
+                rebuildedIndex = indexes.Select(x => x.Name);
+                _indexRebuildStatusManager.SetRebuildingIndexStatus(rebuildedIndex, true);
                 if (indexes.Length == 0)
                 {
                     return;
@@ -218,10 +236,15 @@ private void RebuildIndexes(bool onlyEmptyIndexes, TimeSpan delay, CancellationT
                         _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType());
                     }
                 }
+
             }
         }
         finally
         {
+            if (rebuildedIndex != null && rebuildedIndex.Any())
+            {
+                _indexRebuildStatusManager.SetRebuildingIndexStatus(rebuildedIndex, false);
+            }
             if (Monitor.IsEntered(_rebuildLocker))
             {
                 Monitor.Exit(_rebuildLocker);

src/Umbraco.Infrastructure/Examine/IIndexRebuildStatusManager.cs

@@ -0,0 +1,7 @@
+namespace Umbraco.Cms.Infrastructure.Examine;
+
+public interface IIndexRebuildStatusManager
+{
+    void SetRebuildingIndexStatus(IEnumerable<string> indexes, bool b);
+    bool GetRebuildingIndexStatus(string index);
+}

src/Umbraco.Infrastructure/Examine/IndexRebuildStatusManager.cs

@@ -0,0 +1,28 @@
+using Examine;
+
+namespace Umbraco.Cms.Infrastructure.Examine;
+
+/// <summary>
+///
+/// </summary>
+public class IndexRebuildStatusManager : IIndexRebuildStatusManager
+{
+    IDictionary<string,bool> _rebuildingStatus = new Dictionary<string, bool>();
+    public IndexRebuildStatusManager(IExamineManager examineManager)
+    {
+        foreach (var index in examineManager.Indexes)
+        {
+            _rebuildingStatus.Add(index.Name, false);
+        }
+    }
+
+    public void SetRebuildingIndexStatus(IEnumerable<string> indexes, bool isRebuilding)
+    {
+        foreach (var index in indexes)
+        {
+            _rebuildingStatus[index] = isRebuilding;
+        }
+    }
+
+    public bool GetRebuildingIndexStatus(string index) => _rebuildingStatus.TryGetValue(index, out var isRebuilding) && isRebuilding;
+}

src/Umbraco.Web.BackOffice/Controllers/ExamineManagementController.cs

@@ -3,9 +3,11 @@
 using Lucene.Net.QueryParsers.Classic;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Cache;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models.ContentEditing;
 using Umbraco.Cms.Infrastructure.Examine;
 using Umbraco.Cms.Web.Common.Attributes;
@@ -22,23 +24,32 @@ public class ExamineManagementController : UmbracoAuthorizedJsonController
     private readonly IExamineManager _examineManager;
     private readonly IIndexDiagnosticsFactory _indexDiagnosticsFactory;
     private readonly IIndexRebuilder _indexRebuilder;
+    private readonly IIndexRebuildStatusManager _rebuildStatusManager;
     private readonly ILogger<ExamineManagementController> _logger;
     private readonly IAppPolicyCache _runtimeCache;
-
+    [Obsolete("Use the ctor with IOptionsMonitor<IIndexRebuildOptions>>")]
     public ExamineManagementController(
         IExamineManager examineManager,
         ILogger<ExamineManagementController> logger,
         IIndexDiagnosticsFactory indexDiagnosticsFactory,
         AppCaches appCaches,
-        IIndexRebuilder indexRebuilder)
+        IIndexRebuilder indexRebuilder) : this(examineManager, logger, indexDiagnosticsFactory, appCaches, indexRebuilder, StaticServiceProvider.Instance.GetRequiredService<IIndexRebuildStatusManager>())
+    {
+    }
+    public ExamineManagementController(
+        IExamineManager examineManager,
+        ILogger<ExamineManagementController> logger,
+        IIndexDiagnosticsFactory indexDiagnosticsFactory,
+        AppCaches appCaches,
+        IIndexRebuilder indexRebuilder, IIndexRebuildStatusManager rebuildStatusManager)
     {
         _examineManager = examineManager;
         _logger = logger;
         _indexDiagnosticsFactory = indexDiagnosticsFactory;
         _runtimeCache = appCaches.RuntimeCache;
         _indexRebuilder = indexRebuilder;
+        _rebuildStatusManager = rebuildStatusManager;
     }
-
     /// <summary>
     ///     Get the details for indexers
     /// </summary>
@@ -129,11 +140,8 @@ public ActionResult<SearchResults> GetSearchResults(string searcherName, string?
             return validate;
         }
 
-        var cacheKey = "temp_indexing_op_" + indexName;
-        var found = _runtimeCache.Get(cacheKey);
-
         //if its still there then it's not done
-        return found != null
+        return _rebuildStatusManager.GetRebuildingIndexStatus(indexName)
             ? null
             : CreateModel(index!);
     }
@@ -167,12 +175,7 @@ public IActionResult PostRebuildIndex(string indexName)
 
         try
         {
-            var cacheKey = "temp_indexing_op_" + index.Name;
-            //put temp val in cache which is used as a rudimentary way to know when the indexing is done
-            _runtimeCache.Insert(cacheKey, () => "tempValue", TimeSpan.FromMinutes(5));
-
             _indexRebuilder.RebuildIndex(indexName);
-
             return new OkResult();
         }
         catch (Exception ex)
@@ -307,8 +310,5 @@ private void Indexer_IndexOperationComplete(object? sender, EventArgs e)
         }
 
         _logger.LogInformation("Rebuilding index '{indexerName}' done.", indexer?.Name);
-
-        var cacheKey = "temp_indexing_op_" + indexer?.Name;
-        _runtimeCache.Clear(cacheKey);
     }
 }