diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 6d84103e1..6e558b1b7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -27,8 +27,6 @@ public interface IConversationService /// If not null, delete messages while input a new message; otherwise delete messages only /// Task TruncateConversation(string conversationId, string messageId, string? newMessageId = null); - Task> GetConversationContentLogs(string conversationId); - Task> GetConversationStateLogs(string conversationId); /// /// Send message to LLM @@ -72,4 +70,6 @@ Task SendMessage(string agentId, /// if pre-loading, then keys are not filter by the search query /// Task> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false); + + Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs index 9d1540521..6e1926191 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs @@ -1,5 +1,3 @@ -using System.Collections.Generic; - namespace BotSharp.Abstraction.Instructs.Models; public class InstructLogFilter : Pagination @@ -8,6 +6,7 @@ public class InstructLogFilter : Pagination public List? Providers { get; set; } public List? Models { get; set; } public List? TemplateNames { get; set; } + public List? UserIds { get; set; } public static InstructLogFilter Empty() { diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs index 844d79424..de8430231 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs @@ -39,6 +39,9 @@ public class InstructionLogModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserId { get; set; } + [JsonIgnore] + public string? UserName { get; set; } + [JsonIgnore] public Dictionary States { get; set; } = []; diff --git a/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs new file mode 100644 index 000000000..23a4809fb --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs @@ -0,0 +1,16 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; + +namespace BotSharp.Abstraction.Loggers.Services; + +public interface ILoggerService +{ + #region Conversation + Task> GetConversationContentLogs(string conversationId); + Task> GetConversationStateLogs(string conversationId); + #endregion + + #region Instruction + Task> GetInstructionLogs(InstructLogFilter filter); + #endregion +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 6f8de07d2..d523dd09c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -151,6 +151,10 @@ List TruncateConversation(string conversationId, string messageId, bool => throw new NotImplementedException(); List GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100) => throw new NotImplementedException(); + List GetConversationsToMigrate(int batchSize = 100) + => throw new NotImplementedException(); + bool MigrateConvsersationLatestStates(string conversationId) + => throw new NotImplementedException(); #endregion #region LLM Completion Log diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 044149f54..00e1f30a5 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs new file mode 100644 index 000000000..230127060 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs @@ -0,0 +1,98 @@ +using NetTopologySuite.Algorithm; +using System.Diagnostics; + +namespace BotSharp.Core.Conversations.Services; + +public partial class ConversationService +{ + public async Task MigrateLatestStates(int batchSize = 100, int errorLimit = 10) + { + var db = _services.GetRequiredService(); + var isSuccess = true; + var errorCount = 0; + var batchNum = 0; + var info = string.Empty; + var error = string.Empty; + +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States..."); +#endif + var sw = Stopwatch.StartNew(); + + var convIds = db.GetConversationsToMigrate(batchSize); + + while (!convIds.IsNullOrEmpty()) + { + batchNum++; + var innerSw = Stopwatch.StartNew(); +#if DEBUG + Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n"); +#else + _logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})"); +#endif + + for (int i = 0; i < convIds.Count; i++) + { + var convId = convIds.ElementAt(i); + try + { + var done = db.MigrateConvsersationLatestStates(convId); + info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + } + catch (Exception ex) + { + errorCount++; + error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}"; +#if DEBUG + Console.WriteLine($"\r\n{error}\r\n"); +#else + _logger.LogError($"{error}"); +#endif + } + } + + if (errorCount >= errorLimit) + { + error = $"\r\nErrors exceed limit => stop the migration!\r\n"; +#if DEBUG + Console.WriteLine($"{error}"); +#else + _logger.LogError($"{error}"); +#endif + innerSw.Stop(); + isSuccess = false; + break; + } + + innerSw.Stop(); + info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " + + $"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + await Task.Delay(100); + convIds = db.GetConversationsToMigrate(batchSize); + } + + sw.Stop(); + info = $"#Done with migrating Conversation Latest States! " + + $"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)"; +#if DEBUG + Console.WriteLine($"\r\n{info}\r\n"); +#else + _logger.LogInformation($"{info}"); +#endif + + return isSuccess; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs index 663c710ad..47f6b2455 100644 --- a/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs @@ -1,5 +1,4 @@ using BotSharp.Abstraction.Plugins.Models; -using BotSharp.Abstraction.Users.Enums; using Microsoft.Extensions.Configuration; namespace BotSharp.Core.Instructs; @@ -22,8 +21,8 @@ public bool AttachMenu(List menu) { SubMenu = new List { - new PluginMenuDef("Instruction", link: "page/instruction"), - new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] } + new PluginMenuDef("Testing", link: "page/instruction/testing"), + new PluginMenuDef("Log", link: "page/instruction/log") } }); diff --git a/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs new file mode 100644 index 000000000..bc99b9c82 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Loggers; + +public class LoggerPlugin : IBotSharpPlugin +{ + public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea"; + public string Name => "Logger"; + public string Description => "Provide log service"; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs similarity index 83% rename from src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs rename to src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs index 7b583f3f0..5ce057c38 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs @@ -1,9 +1,8 @@ using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Repositories; -namespace BotSharp.Core.Conversations.Services; +namespace BotSharp.Core.Loggers.Services; -public partial class ConversationService +public partial class LoggerService { public async Task> GetConversationContentLogs(string conversationId) { diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs new file mode 100644 index 000000000..84efae995 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs @@ -0,0 +1,64 @@ +using BotSharp.Abstraction.Instructs.Models; +using BotSharp.Abstraction.Loggers.Models; +using BotSharp.Abstraction.Users.Enums; +using BotSharp.Abstraction.Users.Models; + +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService +{ + public async Task> GetInstructionLogs(InstructLogFilter filter) + { + if (filter == null) + { + filter = InstructLogFilter.Empty(); + } + + var userService = _services.GetRequiredService(); + var user = await userService.GetUser(_user.Id); + var isAdmin = UserConstant.AdminRoles.Contains(user?.Role); + if (!isAdmin && user?.Id == null) return new(); + + filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : []; + + var agents = new List(); + var users = new List(); + + var db = _services.GetRequiredService(); + var logs = db.GetInstructionLogs(filter); + var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); + var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList(); + agents = db.GetAgents(new AgentFilter + { + AgentIds = agentIds, + Pager = new Pagination { Size = filter.Size } + }); + + if (isAdmin) + { + users = db.GetUserByIds(userIds); + } + + var items = logs.Items.Select(x => + { + x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; + + if (!isAdmin) + { + x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null; + } + else + { + var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null; + x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null; + } + return x; + }).ToList(); + + return new PagedItems + { + Items = items, + Count = logs.Count + }; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs new file mode 100644 index 000000000..bd9a5147b --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Core.Loggers.Services; + +public partial class LoggerService : ILoggerService +{ + private readonly IServiceProvider _services; + private readonly IUserIdentity _user; + private readonly ILogger _logger; + + public LoggerService( + IServiceProvider services, + IUserIdentity user, + ILogger logger) + { + _services = services; + _user = user; + _logger = logger; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 08a534d48..8880a4e0d 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -687,6 +687,56 @@ public List GetConversationStateSearchKeys(int messageLowerLimit = 2, in } + + public List GetConversationsToMigrate(int batchSize = 100) + { + var baseDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); + if (!Directory.Exists(baseDir)) return []; + + var convIds = new List(); + var dirs = Directory.GetDirectories(baseDir); + + foreach (var dir in dirs) + { + var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE); + if (File.Exists(latestStateFile)) continue; + + var convId = dir.Split(Path.DirectorySeparatorChar).Last(); + if (string.IsNullOrEmpty(convId)) continue; + + convIds.Add(convId); + if (convIds.Count >= batchSize) + { + break; + } + } + + return convIds; + } + + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) + { + return false; + } + + var stateFile = Path.Combine(convDir, STATE_FILE); + var states = CollectConversationStates(stateFile); + var latestStates = BuildLatestStates(states); + + var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE); + var stateStr = JsonSerializer.Serialize(latestStates, _options); + File.WriteAllText(latestStateFile, stateStr); + + return true; + } + + #region Private methods private string? FindConversationDirectory(string conversationId) { @@ -883,6 +933,11 @@ private Dictionary CollectConversationLatestStates(string private Dictionary BuildLatestStates(List states) { var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } + foreach (var pair in states) { var value = pair.Values?.LastOrDefault(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs index ea1f27b5d..398fb03aa 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs @@ -182,6 +182,10 @@ public PagedItems GetInstructionLogs(InstructLogFilter filt { matched = matched && filter.TemplateNames.Contains(log.TemplateName); } + if (!filter.UserIds.IsNullOrEmpty()) + { + matched = matched && filter.UserIds.Contains(log.UserId); + } if (!matched) continue; @@ -190,12 +194,6 @@ public PagedItems GetInstructionLogs(InstructLogFilter filt } var records = logs.OrderByDescending(x => x.CreatedTime).Skip(filter.Offset).Take(filter.Size); - var agentIds = records.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); - var agents = GetAgents(new AgentFilter - { - AgentIds = agentIds - }); - records = records.Select(x => { var states = x.InnerStates.ToDictionary(p => p.Key, p => @@ -203,7 +201,6 @@ public PagedItems GetInstructionLogs(InstructLogFilter filt var data = p.Value.RootElement.GetProperty("data"); return data.ValueKind != JsonValueKind.Null ? data.ToString() : null; }); - x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; x.States = states ?? []; return x; }).ToList(); diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs index 8655afb13..0afac090e 100644 --- a/src/Infrastructure/BotSharp.Core/Using.cs +++ b/src/Infrastructure/BotSharp.Core/Using.cs @@ -39,6 +39,8 @@ global using BotSharp.Abstraction.Statistics.Models; global using BotSharp.Abstraction.Statistics.Enums; global using BotSharp.Abstraction.Statistics.Services; +global using BotSharp.Abstraction.Loggers.Services; +global using BotSharp.Abstraction.Infrastructures.Events; global using BotSharp.Core.Repository; global using BotSharp.Core.Routing; global using BotSharp.Core.Agents.Services; @@ -46,4 +48,4 @@ global using BotSharp.Core.Infrastructures; global using BotSharp.Core.Users.Services; global using BotSharp.Core.Statistics.Services; -global using BotSharp.Abstraction.Infrastructures.Events; \ No newline at end of file +global using BotSharp.Core.Loggers.Services; \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 75dc11133..10317815a 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -563,6 +563,16 @@ public async Task> GetConversationStateKeys([FromQuery] string quer } #endregion + #region Migrate Latest States + [HttpPost("/conversation/latest-state/migrate")] + public async Task MigrateConversationLatestStates([FromBody] MigrateLatestStateRequest request) + { + var convService = _services.GetRequiredService(); + var res = await convService.MigrateLatestStates(request.BatchSize, request.ErrorLimit); + return res; + } + #endregion + #region Private methods private void SetStates(IConversationService conv, NewMessageModel input) { diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs index 2c46b5b4d..d67ba8cd4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs @@ -1,6 +1,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Abstraction.Loggers.Models; -using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Loggers.Services; using BotSharp.OpenAPI.ViewModels.Instructs; using Microsoft.AspNetCore.Hosting; @@ -11,10 +11,14 @@ namespace BotSharp.OpenAPI.Controllers; public class LoggerController : ControllerBase { private readonly IServiceProvider _services; + private readonly IUserIdentity _user; - public LoggerController(IServiceProvider services) + public LoggerController( + IServiceProvider services, + IUserIdentity user) { _services = services; + _user = user; } [HttpGet("/logger/full-log")] @@ -38,22 +42,23 @@ public async Task GetFullLog() [HttpGet("/logger/conversation/{conversationId}/content-log")] public async Task> GetConversationContentLogs([FromRoute] string conversationId) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationContentLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationContentLogs(conversationId); } [HttpGet("/logger/conversation/{conversationId}/state-log")] public async Task> GetConversationStateLogs([FromRoute] string conversationId) { - var conversationService = _services.GetRequiredService(); - return await conversationService.GetConversationStateLogs(conversationId); + var logging = _services.GetRequiredService(); + return await logging.GetConversationStateLogs(conversationId); } [HttpGet("/logger/instruction/log")] public async Task> GetInstructionLogs([FromQuery] InstructLogFilter request) { - var db = _services.GetRequiredService(); - var logs = db.GetInstructionLogs(request); + var logging = _services.GetRequiredService(); + var logs = await logging.GetInstructionLogs(request); + return new PagedItems { Items = logs.Items.Select(x => InstructionLogViewModel.From(x)), diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs new file mode 100644 index 000000000..99085b0a7 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/MigrateLatestStateRequest.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class MigrateLatestStateRequest +{ + public int BatchSize { get; set; } = 1000; + public int ErrorLimit { get; set; } = 10; +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs index 88fc27e2a..aca73f762 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructionLogViewModel.cs @@ -40,6 +40,10 @@ public class InstructionLogViewModel [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? UserId { get; set; } + [JsonPropertyName("user_name")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? UserName { get; set; } + [JsonPropertyName("states")] public Dictionary States { get; set; } = []; @@ -60,6 +64,7 @@ public static InstructionLogViewModel From(InstructionLogModel log) SystemInstruction = log.SystemInstruction, CompletionText = log.CompletionText, UserId = log.UserId, + UserName = log.UserName, States = log.States, CreatedTime = log.CreatedTime }; diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 1c0eeb2f3..b9931bd41 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Repositories.Filters; +using MongoDB.Driver; using System.Text.Json; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -661,6 +662,37 @@ public List GetConversationStateSearchKeys(int messageLowerLimit = 2, in return keys; } + + + public List GetConversationsToMigrate(int batchSize = 100) + { + var convFilter = Builders.Filter.Exists(x => x.LatestStates, false); + var sortDef = Builders.Sort.Ascending(x => x.CreatedTime); + var convIds = _dc.Conversations.Find(convFilter).Sort(sortDef) + .Limit(batchSize).ToEnumerable() + .Select(x => x.Id).ToList(); + return convIds ?? []; + } + + public bool MigrateConvsersationLatestStates(string conversationId) + { + if (string.IsNullOrEmpty(conversationId)) return false; + + var stateFilter = Builders.Filter.Eq(x => x.ConversationId, conversationId); + var foundStates = _dc.ConversationStates.Find(stateFilter).FirstOrDefault(); + if (foundStates?.States == null) return false; + + var states = foundStates.States.ToList(); + var latestStates = BuildLatestStates(states); + + var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); + var convUpdate = Builders.Update.Set(x => x.LatestStates, latestStates); + _dc.Conversations.UpdateOne(convFilter, convUpdate); + + return true; + } + + #region Private methods private string ConvertSnakeCaseToPascalCase(string snakeCase) { string[] words = snakeCase.Split('_'); @@ -682,6 +714,11 @@ private string ConvertSnakeCaseToPascalCase(string snakeCase) private Dictionary BuildLatestStates(List states) { var endNodes = new Dictionary(); + if (states.IsNullOrEmpty()) + { + return endNodes; + } + foreach (var pair in states) { var value = pair.Values?.LastOrDefault(); @@ -703,4 +740,5 @@ private Dictionary BuildLatestStates(List GetInstructionLogs(InstructLogFilter filt var docs = _dc.InstructionLogs.Find(filterDef).Sort(sortDef).Skip(filter.Offset).Limit(filter.Size).ToList(); var count = _dc.InstructionLogs.CountDocuments(filterDef); - var agentIds = docs.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList(); - var agents = GetAgents(new AgentFilter - { - AgentIds = agentIds - }); - var logs = docs.Select(x => { var log = InstructionLogBetaDocument.ToDomainModel(x); - log.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null; log.States = x.States.ToDictionary(p => p.Key, p => { var jsonStr = p.Value.ToJson();