diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index af407d253..a94e3c607 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -59,7 +59,7 @@ public interface IBotSharpRepository PagedItems GetConversations(ConversationFilter filter); void UpdateConversationTitle(string conversationId, string title); List GetLastConversations(); - bool TruncateConversation(string conversationId, string messageId); + bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false); #endregion #region Execution Log diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs index ca2719d3e..1f1cc8fde 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.TruncateMessage.cs @@ -7,7 +7,7 @@ public partial class ConversationService : IConversationService public async Task TruncateConversation(string conversationId, string messageId) { var db = _services.GetRequiredService(); - var isSaved = db.TruncateConversation(conversationId, messageId); + var isSaved = db.TruncateConversation(conversationId, messageId, true); return await Task.FromResult(isSaved); } } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index 48f0c1590..232c8a71e 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -217,7 +217,7 @@ public void UpdateConversationStatus(string conversationId, string status) throw new NotImplementedException(); } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { throw new NotImplementedException(); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index e8609f6fc..057580e56 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Loggers.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; using System.Globalization; @@ -263,7 +264,7 @@ public List GetLastConversations() } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false; @@ -288,6 +289,12 @@ public bool TruncateConversation(string conversationId, string messageId) var states = CollectConversationStates(stateDir); isSaved = HandleTruncatedStates(stateDir, states, refTime); + // Remove logs + if (cleanLog) + { + HandleTruncatedLogs(convDir, refTime); + } + return isSaved; } @@ -392,6 +399,44 @@ private bool HandleTruncatedStates(string stateDir, List states, return isSaved; } + private bool HandleTruncatedLogs(string convDir, DateTime refTime) + { + var contentLogDir = Path.Combine(convDir, "content_log"); + var stateLogDir = Path.Combine(convDir, "state_log"); + + if (Directory.Exists(contentLogDir)) + { + foreach (var file in Directory.GetFiles(contentLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + if (Directory.Exists(stateLogDir)) + { + foreach (var file in Directory.GetFiles(stateLogDir)) + { + var text = File.ReadAllText(file); + var log = JsonSerializer.Deserialize(text); + if (log == null) continue; + + if (log.CreateTime >= refTime) + { + File.Delete(file); + } + } + } + + return true; + } + private bool SaveTruncatedDialogs(string dialogDir, List dialogs) { if (string.IsNullOrEmpty(dialogDir) || dialogs == null) return false; diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs index 90fbcefad..c34ced0d7 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/Hooks/StreamingLogHook.cs @@ -101,12 +101,10 @@ public async Task AfterGenerated(RoleDialogModel message, TokenStatsModel tokenS var conversationId = _state.GetConversationId(); var agent = await _agentService.LoadAgent(message.CurrentAgentId); - var logSource = string.Empty; var log = tokenStats.Prompt; - logSource = ContentLogSource.Prompt; await _chatHub.Clients.User(_user.Id).SendAsync("OnConversationContentLogGenerated", - BuildContentLog(conversationId, agent?.Name, log, logSource, message)); + BuildContentLog(conversationId, agent?.Name, log, ContentLogSource.Prompt, message)); } /// diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index aa0deda36..98fb11e33 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Repositories.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; +using MongoDB.Driver; namespace BotSharp.Plugin.MongoStorage.Repository; @@ -273,7 +274,7 @@ public List GetLastConversations() }).ToList(); } - public bool TruncateConversation(string conversationId, string messageId) + public bool TruncateConversation(string conversationId, string messageId, bool cleanLog = false) { if (string.IsNullOrEmpty(conversationId) || string.IsNullOrEmpty(messageId)) return false; @@ -308,6 +309,28 @@ public bool TruncateConversation(string conversationId, string messageId) foundStates.States = truncatedStates; _dc.ConversationDialogs.ReplaceOne(dialogFilter, foundDialog); _dc.ConversationStates.ReplaceOne(stateFilter, foundStates); + + // Remove logs + if (cleanLog) + { + var contentLogBuilder = Builders.Filter; + var stateLogBuilder = Builders.Filter; + + var contentLogFilters = new List>() + { + contentLogBuilder.Eq(x => x.ConversationId, conversationId), + contentLogBuilder.Gte(x => x.CreateTime, refTime) + }; + var stateLogFilters = new List>() + { + stateLogBuilder.Eq(x => x.ConversationId, conversationId), + stateLogBuilder.Gte(x => x.CreateTime, refTime) + }; + + _dc.ContentLogs.DeleteMany(contentLogBuilder.And(contentLogFilters)); + _dc.StateLogs.DeleteMany(stateLogBuilder.And(stateLogFilters)); + } + return true; } }