From 7f15dcd8a375ae386bb3e59f9338ef5fd048f002 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 25 Mar 2024 14:12:08 -0500 Subject: [PATCH 1/2] refine json converter --- .../RichContentJsonConverter .cs | 2 +- .../TemplateMessageJsonConverter.cs | 2 +- .../Messaging/MessageParser.cs | 26 +++++++++++++++++-- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs index 95c00037d..20b0d949b 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/RichContentJsonConverter .cs @@ -16,7 +16,7 @@ public class RichContentJsonConverter : JsonConverter if (root.TryGetProperty("rich_type", out JsonElement element)) { var richType = element.GetString(); - res = parser.ParseRichMessage(richType, jsonText, options); + res = parser.ParseRichMessage(richType, jsonText, root, options); } return res; diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs index fbd07e3f9..7729f40f6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/JsonConverters/TemplateMessageJsonConverter.cs @@ -16,7 +16,7 @@ public class TemplateMessageJsonConverter : JsonConverter if (root.TryGetProperty("template_type", out JsonElement element)) { var templateType = element.GetString(); - res = parser.ParseTemplateMessage(templateType, jsonText, options); + res = parser.ParseTemplateMessage(templateType, jsonText, root, options); } return res; diff --git a/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs b/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs index abc2f621b..3707de4a1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Messaging/MessageParser.cs @@ -12,7 +12,7 @@ public MessageParser() { } - public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonSerializerOptions options) + public IRichMessage? ParseRichMessage(string richType, string jsonText, JsonElement root, JsonSerializerOptions options) { IRichMessage? res = null; @@ -36,11 +36,22 @@ public MessageParser() { res = JsonSerializer.Deserialize(jsonText, options); } + else if (richType == RichTypeEnum.GenericTemplate) + { + if (root.TryGetProperty("element_type", out var element)) + { + var elementType = element.GetString(); + if (elementType == typeof(GenericElement).Name) + { + res = JsonSerializer.Deserialize>(jsonText, options); + } + } + } return res; } - public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonSerializerOptions options) + public ITemplateMessage? ParseTemplateMessage(string templateType, string jsonText, JsonElement root, JsonSerializerOptions options) { ITemplateMessage? res = null; @@ -60,6 +71,17 @@ public MessageParser() { res = JsonSerializer.Deserialize(jsonText, options); } + else if (templateType == TemplateTypeEnum.Generic) + { + if (root.TryGetProperty("element_type", out var element)) + { + var elementType = element.GetString(); + if (elementType == typeof(GenericElement).Name) + { + res = JsonSerializer.Deserialize>(jsonText, options); + } + } + } return res; } From a918b88937265cb84841825a76ebe859f7613f6b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 25 Mar 2024 17:29:08 -0500 Subject: [PATCH 2/2] refine background service --- .../Conversations/Models/Conversation.cs | 2 + .../Services/ConversationStorage.cs | 4 +- .../FileRepository.Conversation.cs | 36 ++++++++++++++--- .../ConversationTimeoutService.cs | 13 +++++- .../Collections/ConversationDocument.cs | 1 + .../MongoRepository.Conversation.cs | 40 ++++++++++++++----- 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs index 4ef8792bf..2c28259ec 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/Conversation.cs @@ -24,6 +24,8 @@ public class Conversation public string Channel { get; set; } = ConversationChannel.OpenAPI; + public int DialogCount { get; set; } + public DateTime UpdatedTime { get; set; } = DateTime.UtcNow; public DateTime CreatedTime { get; set; } = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 337174108..d66f7129d 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -46,7 +46,7 @@ public void Append(string conversationId, RoleDialogModel dialog) AgentId = agentId, MessageId = dialog.MessageId, FunctionName = dialog.FunctionName, - CreateTime = dialog.CreatedAt + CreateTime = DateTime.UtcNow }; var content = dialog.Content.RemoveNewLine(); @@ -65,7 +65,7 @@ public void Append(string conversationId, RoleDialogModel dialog) MessageId = dialog.MessageId, SenderId = dialog.SenderId, FunctionName = dialog.FunctionName, - CreateTime = dialog.CreatedAt + CreateTime = DateTime.UtcNow }; var content = dialog.Content.RemoveNewLine(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 13746a6d9..59119a37e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -117,6 +117,7 @@ public void AppendConversationDialogs(string conversationId, List var conv = JsonSerializer.Deserialize(json, _options); if (conv != null) { + conv.DialogCount += dialogs.Count(); conv.UpdatedTime = DateTime.UtcNow; File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); } @@ -353,23 +354,40 @@ public List GetIdleConversations(int batchSize, int messageLimit, int bu batchSize = batchLimit; } + if (bufferHours <= 0) + { + bufferHours = 12; + } + + if (messageLimit <= 0) + { + messageLimit = 2; + } + foreach (var d in Directory.GetDirectories(dir)) { var convFile = Path.Combine(d, CONVERSATION_FILE); if (!File.Exists(convFile)) { + Directory.Delete(d, true); continue; } var json = File.ReadAllText(convFile); var conv = JsonSerializer.Deserialize(json, _options); - if (conv == null || conv.UpdatedTime > utcNow.AddHours(-bufferHours)) + + if (conv == null) { + Directory.Delete(d, true); continue; } - var dialogs = GetConversationDialogs(conv.Id); - if (dialogs.Count <= messageLimit) + if (conv.UpdatedTime > utcNow.AddHours(-bufferHours)) + { + continue; + } + + if (conv.DialogCount <= messageLimit) { ids.Add(conv.Id); if (ids.Count >= batchSize) @@ -398,7 +416,7 @@ public bool TruncateConversation(string conversationId, string messageId, bool c if (foundIdx < 0) return false; // Handle truncated dialogs - var isSaved = HandleTruncatedDialogs(dialogDir, dialogs, foundIdx); + var isSaved = HandleTruncatedDialogs(convDir, dialogDir, dialogs, foundIdx); if (!isSaved) return false; // Handle truncated states @@ -489,10 +507,18 @@ private List CollectConversationStates(string stateFile) return states ?? new List(); } - private bool HandleTruncatedDialogs(string dialogDir, List dialogs, int foundIdx) + private bool HandleTruncatedDialogs(string convDir, string dialogDir, List dialogs, int foundIdx) { var truncatedDialogs = dialogs.Where((x, idx) => idx < foundIdx).ToList(); var isSaved = SaveTruncatedDialogs(dialogDir, truncatedDialogs); + var convFile = Path.Combine(convDir, CONVERSATION_FILE); + var convJson = File.ReadAllText(convFile); + var conv = JsonSerializer.Deserialize(convJson, _options); + if (conv != null) + { + conv.DialogCount = truncatedDialogs.Count; + File.WriteAllText(convFile, JsonSerializer.Serialize(conv, _options)); + } return isSaved; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs index 088d0f37e..f4cfda7d4 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/BackgroundServices/ConversationTimeoutService.cs @@ -15,7 +15,18 @@ public ConversationTimeoutService(IServiceProvider services, ILogger + { + await DoWork(stoppingToken); + }); + } + + private async Task DoWork(CancellationToken stoppingToken) + { + _logger.LogInformation("Conversation Timeout Service is doing work..."); + try { while (true) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs index 46db2ae56..295d713b5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/ConversationDocument.cs @@ -8,6 +8,7 @@ public class ConversationDocument : MongoBase public string Title { get; set; } public string Channel { get; set; } public string Status { get; set; } + public int DialogCount { get; set; } public DateTime CreatedTime { get; set; } public DateTime UpdatedTime { get; set; } public DateTime Breakpoint { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 3e55dbeae..40d23052e 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -123,7 +123,8 @@ public void AppendConversationDialogs(string conversationId, List var filterDialog = Builders.Filter.Eq(x => x.ConversationId, conversationId); var dialogElements = dialogs.Select(x => DialogMongoElement.ToMongoElement(x)).ToList(); var updateDialog = Builders.Update.PushEach(x => x.Dialogs, dialogElements); - var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); + var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) + .Inc(x => x.DialogCount, dialogs.Count); _dc.ConversationDialogs.UpdateOne(filterDialog, updateDialog); _dc.Conversations.UpdateOne(filterConv, updateConv); @@ -168,13 +169,16 @@ public ConversationState GetConversationStates(string conversationId) public void UpdateConversationStates(string conversationId, List states) { - if (string.IsNullOrEmpty(conversationId) || states.IsNullOrEmpty()) return; + if (string.IsNullOrEmpty(conversationId) || states == null) return; + var filterConv = Builders.Filter.Eq(x => x.Id, conversationId); var filterStates = Builders.Filter.Eq(x => x.ConversationId, conversationId); var saveStates = states.Select(x => StateMongoElement.ToMongoElement(x)).ToList(); var updateStates = Builders.Update.Set(x => x.States, saveStates); + var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow); _dc.ConversationStates.UpdateOne(filterStates, updateStates); + _dc.Conversations.UpdateOne(filterConv, updateConv); } public void UpdateConversationStatus(string conversationId, string status) @@ -220,6 +224,7 @@ public Conversation GetConversation(string conversationId) Status = conv.Status, Dialogs = dialogElements, States = curStates, + DialogCount = conv.DialogCount, CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime, Breakpoint = conv.Breakpoint @@ -308,6 +313,7 @@ public PagedItems GetConversations(ConversationFilter filter) Title = conv.Title, Channel = conv.Channel, Status = conv.Status, + DialogCount = conv.DialogCount, CreatedTime = conv.CreatedTime, UpdatedTime = conv.UpdatedTime, Breakpoint = conv.Breakpoint @@ -335,6 +341,7 @@ public List GetLastConversations() Title = c.Title, Channel = c.Channel, Status = c.Status, + DialogCount = c.DialogCount, CreatedTime = c.CreatedTime, UpdatedTime = c.UpdatedTime, Breakpoint = c.Breakpoint @@ -353,11 +360,21 @@ public List GetIdleConversations(int batchSize, int messageLimit, int bu batchSize = batchLimit; } + if (bufferHours <= 0) + { + bufferHours = 12; + } + + if (messageLimit <= 0) + { + messageLimit = 2; + } + while (true) { var skip = (page - 1) * batchSize; var candidates = _dc.Conversations.AsQueryable() - .Where(x => x.UpdatedTime <= utcNow.AddHours(-bufferHours)) + .Where(x => (x.DialogCount <= messageLimit) && x.UpdatedTime <= utcNow.AddHours(-bufferHours)) .Skip(skip) .Take(batchSize) .Select(x => x.Id) @@ -367,13 +384,8 @@ public List GetIdleConversations(int batchSize, int messageLimit, int bu { break; } - - var targets = _dc.ConversationDialogs.AsQueryable() - .Where(x => candidates.Contains(x.ConversationId) && x.Dialogs != null && x.Dialogs.Count <= messageLimit) - .Select(x => x.ConversationId) - .ToList(); - - conversationIds = conversationIds.Concat(targets).ToList(); + + conversationIds = conversationIds.Concat(candidates).Distinct().ToList(); if (conversationIds.Count >= batchSize) { break; @@ -420,10 +432,16 @@ public bool TruncateConversation(string conversationId, string messageId, bool c _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); } - // Save + // Save dialogs foundDialog.Dialogs = truncatedDialogs; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); + // Update conversation + var convFilter = Builders.Filter.Eq(x => x.Id, conversationId); + var updateConv = Builders.Update.Set(x => x.UpdatedTime, DateTime.UtcNow) + .Set(x => x.DialogCount, truncatedDialogs.Count); + _dc.Conversations.UpdateOne(convFilter, updateConv); + // Remove logs if (cleanLog) {