add content log and state log

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 
 namespace BotSharp.Abstraction.Conversations;
@@ -14,6 +15,8 @@ public interface IConversationService
     Task<List<Conversation>> GetLastConversations();
     Task<bool> DeleteConversation(string id);
     Task<bool> TruncateConversation(string conversationId, string messageId);
+    Task<List<ConversationContentLogModel>> GetConversationContentLogs(string conversationId);
+    Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
 
     /// <summary>
     /// Send message to LLM

src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs

@@ -9,4 +9,6 @@ public class ConversationSetting
     public int MaxRecursiveDepth { get; set; } = 3;
     public bool EnableLlmCompletionLog { get; set; }
     public bool EnableExecutionLog { get; set; }
+    public bool EnableContentLog { get; set; }
+    public bool EnableStateLog { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Loggers/Models/StreamingLogModel.cs → src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationContentLogModel.cs

@@ -1,9 +1,11 @@
 namespace BotSharp.Abstraction.Loggers.Models;
 
-public class StreamingLogModel
+public class ConversationContentLogModel
 {
     [JsonPropertyName("conversation_id")]
     public string ConversationId { get; set; }
+    [JsonPropertyName("message_id")]
+    public string MessageId { get; set; }
     [JsonPropertyName("name")]
     public string? Name { get; set; }
     [JsonPropertyName("role")]
@@ -13,5 +15,5 @@ public class StreamingLogModel
     public string Content { get; set; }
 
     [JsonPropertyName("created_at")]
-    public DateTime CreateTime { get; set; }
+    public DateTime CreateTime { get; set; } = DateTime.UtcNow;
 }

src/Infrastructure/BotSharp.Abstraction/Loggers/Models/ConversationStateLogModel.cs

@@ -0,0 +1,13 @@
+namespace BotSharp.Abstraction.Loggers.Models;
+
+public class ConversationStateLogModel
+{
+    [JsonPropertyName("conversation_id")]
+    public string ConversationId { get; set; }
+    [JsonPropertyName("message_id")]
+    public string MessageId { get; set; }
+    [JsonPropertyName("states")]
+    public Dictionary<string, string> States { get; set; }
+    [JsonPropertyName("created_at")]
+    public DateTime CreateTime { get; set; } = DateTime.UtcNow;
+}

src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs → src/Infrastructure/BotSharp.Abstraction/Loggers/Models/LlmCompletionLog.cs

@@ -1,4 +1,4 @@
-namespace BotSharp.Abstraction.Conversations.Models;
+namespace BotSharp.Abstraction.Loggers.Models;
 
 public class LlmCompletionLog
 {

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories.Filters;
 using BotSharp.Abstraction.Repositories.Models;
@@ -70,6 +71,16 @@ public interface IBotSharpRepository
     void SaveLlmCompletionLog(LlmCompletionLog log);
     #endregion
 
+    #region Conversation Content Log
+    void SaveConversationContentLog(ConversationContentLogModel log);
+    List<ConversationContentLogModel> GetConversationContentLogs(string conversationId);
+    #endregion
+
+    #region Conversation State Log
+    void SaveConversationStateLog(ConversationStateLogModel log);
+    List<ConversationStateLogModel> GetConversationStateLogs(string conversationId);
+    #endregion
+
     #region Statistics
     void IncrementConversationCount();
     #endregion

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs

@@ -0,0 +1,22 @@
+using BotSharp.Abstraction.Loggers.Models;
+using BotSharp.Abstraction.Repositories;
+
+namespace BotSharp.Core.Conversations.Services;
+
+public partial class ConversationService
+{
+    public async Task<List<ConversationContentLogModel>> GetConversationContentLogs(string conversationId)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var logs = db.GetConversationContentLogs(conversationId);
+        return await Task.FromResult(logs);
+    }
+
+
+    public async Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var logs = db.GetConversationStateLogs(conversationId);
+        return await Task.FromResult(logs);
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Plugins.Models;
 using BotSharp.Abstraction.Repositories;
 using BotSharp.Abstraction.Repositories.Filters;
@@ -255,6 +256,30 @@ public void SaveLlmCompletionLog(LlmCompletionLog log)
     }
     #endregion
 
+    #region Conversation Content Log
+    public void SaveConversationContentLog(ConversationContentLogModel log)
+    {
+        throw new NotImplementedException();
+    }
+
+    public List<ConversationContentLogModel> GetConversationContentLogs(string conversationId)
+    {
+        throw new NotImplementedException();
+    }
+    #endregion
+
+    #region Conversation State Log
+    public void SaveConversationStateLog(ConversationStateLogModel log)
+    {
+        throw new NotImplementedException();
+    }
+
+    public List<ConversationStateLogModel> GetConversationStateLogs(string conversationId)
+    {
+        throw new NotImplementedException();
+    }
+    #endregion
+
     #region Stats
     public void IncrementConversationCount()
     {

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Log.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Loggers.Models;
+using Serilog;
 using System.IO;
 
 namespace BotSharp.Core.Repository
@@ -54,14 +56,112 @@ public void SaveLlmCompletionLog(LlmCompletionLog log)
                 Directory.CreateDirectory(logDir);
             }
 
-            var index = GetNextLlmCompletionLogIndex(logDir, log.MessageId);
+            var index = GetNextLogIndex(logDir, log.MessageId);
             var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log");
             File.WriteAllText(file, JsonSerializer.Serialize(log, _options));
         }
         #endregion
 
+        #region Conversation Content Log
+        public void SaveConversationContentLog(ConversationContentLogModel log)
+        {
+            if (log == null) return;
+
+            log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+            log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+
+            var convDir = FindConversationDirectory(log.ConversationId);
+            if (string.IsNullOrEmpty(convDir))
+            {
+                convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId);
+                Directory.CreateDirectory(convDir);
+            }
+
+            var logDir = Path.Combine(convDir, "content_log");
+            if (!Directory.Exists(logDir))
+            {
+                Directory.CreateDirectory(logDir);
+            }
+
+            var index = GetNextLogIndex(logDir, log.MessageId);
+            var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log");
+            File.WriteAllText(file, JsonSerializer.Serialize(log, _options));
+        }
+
+        public List<ConversationContentLogModel> GetConversationContentLogs(string conversationId)
+        {
+            var logs = new List<ConversationContentLogModel>();
+            if (string.IsNullOrEmpty(conversationId)) return logs;
+
+            var convDir = FindConversationDirectory(conversationId);
+            if (string.IsNullOrEmpty(convDir)) return logs;
+
+            var logDir = Path.Combine(convDir, "content_log");
+            if (!Directory.Exists(logDir)) return logs;
+
+            foreach (var file in Directory.GetFiles(logDir))
+            {
+                var text = File.ReadAllText(file);
+                var log = JsonSerializer.Deserialize<ConversationContentLogModel>(text);
+                if (log == null) continue;
+
+                logs.Add(log);
+            }
+            return logs.OrderBy(x => x.CreateTime).ToList();
+        }
+        #endregion
+
+        #region Conversation State Log
+        public void SaveConversationStateLog(ConversationStateLogModel log)
+        {
+            if (log == null) return;
+
+            log.ConversationId = log.ConversationId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+            log.MessageId = log.MessageId.IfNullOrEmptyAs(Guid.NewGuid().ToString());
+
+            var convDir = FindConversationDirectory(log.ConversationId);
+            if (string.IsNullOrEmpty(convDir))
+            {
+                convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId);
+                Directory.CreateDirectory(convDir);
+            }
+
+            var logDir = Path.Combine(convDir, "state_log");
+            if (!Directory.Exists(logDir))
+            {
+                Directory.CreateDirectory(logDir);
+            }
+
+            var index = GetNextLogIndex(logDir, log.MessageId);
+            var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log");
+            File.WriteAllText(file, JsonSerializer.Serialize(log, _options));
+        }
+
+        public List<ConversationStateLogModel> GetConversationStateLogs(string conversationId)
+        {
+            var logs = new List<ConversationStateLogModel>();
+            if (string.IsNullOrEmpty(conversationId)) return logs;
+
+            var convDir = FindConversationDirectory(conversationId);
+            if (string.IsNullOrEmpty(convDir)) return logs;
+
+            var logDir = Path.Combine(convDir, "state_log");
+            if (!Directory.Exists(logDir)) return logs;
+
+            foreach (var file in Directory.GetFiles(logDir))
+            {
+                var text = File.ReadAllText(file);
+                var log = JsonSerializer.Deserialize<ConversationStateLogModel>(text);
+                if (log == null) continue;
+
+                logs.Add(log);
+            }
+            return logs.OrderBy(x => x.CreateTime).ToList();
+        }
+        #endregion
+
         #region Private methods
-        private int GetNextLlmCompletionLogIndex(string logDir, string id)
+        private int GetNextLogIndex(string logDir, string id)
         {
             var files = Directory.GetFiles(logDir);
             if (files.IsNullOrEmpty())

src/Infrastructure/BotSharp.Logger/Hooks/CommonContentGeneratingHook.cs

@@ -1,3 +1,5 @@
+using BotSharp.Abstraction.Loggers.Models;
+
 public class CommonContentGeneratingHook : IContentGeneratingHook
 {
     private readonly IServiceProvider _services;

src/Infrastructure/BotSharp.OpenAPI/Controllers/LoggerController.cs

@@ -1,3 +1,4 @@
+using BotSharp.Abstraction.Loggers.Models;
 using Microsoft.AspNetCore.Hosting;
 using SharpCompress.Compressors.Xz;
 using System;
@@ -34,4 +35,18 @@ public async Task<IActionResult> GetFullLog()
             return NotFound();
         }
     }
+
+    [HttpGet("/logger/conversation/{conversationId}/content-log")]
+    public async Task<List<ConversationContentLogModel>> GetConversationContentLogs([FromRoute] string conversationId)
+    {
+        var conversationService = _services.GetRequiredService<IConversationService>();
+        return await conversationService.GetConversationContentLogs(conversationId);
+    }
+
+    [HttpGet("/logger/conversation/{conversationId}/state-log")]
+    public async Task<List<ConversationStateLogModel>> GetConversationStateLogs([FromRoute] string conversationId)
+    {
+        var conversationService = _services.GetRequiredService<IConversationService>();
+        return await conversationService.GetConversationStateLogs(conversationId);
+    }
 }

src/Plugins/BotSharp.Plugin.ChatHub/Hooks/ChatHubConversationHook.cs

@@ -1,6 +1,8 @@
+using BotSharp.Abstraction.Loggers.Models;
 using BotSharp.Abstraction.Messaging;
 using BotSharp.Abstraction.Messaging.JsonConverters;
 using BotSharp.Abstraction.Messaging.Models.RichContent;
+using BotSharp.Abstraction.Repositories;
 using Microsoft.AspNetCore.SignalR;
 
 namespace BotSharp.Plugin.ChatHub.Hooks;
@@ -117,20 +119,28 @@ public override async Task OnResponseGenerated(RoleDialogModel message)
             }
         }, _serializerOptions);
         await _chatHub.Clients.User(_user.Id).SendAsync("OnMessageReceivedFromAssistant", json);
-        await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates()));
+        await _chatHub.Clients.User(_user.Id).SendAsync("OnConversateStatesGenerated", BuildConversationStates(conv.ConversationId, state.GetStates(), message));
 
         await base.OnResponseGenerated(message);
     }
 
-    private string BuildConversationStates(string conversationId, Dictionary<string, string> states)
+    private string BuildConversationStates(string conversationId, Dictionary<string, string> states, RoleDialogModel message)
     {
-        var model = new ConversationStateLogModel
+        var log = new ConversationStateLogModel
         {
-            ConvsersationId = conversationId,
-            States = JsonSerializer.Serialize(states, _serializerOptions),
+            ConversationId = conversationId,
+            MessageId = message.MessageId,
+            States = states,
             CreateTime = DateTime.UtcNow
         };
 
-        return JsonSerializer.Serialize(model, _serializerOptions);
+        var convSettings = _services.GetRequiredService<ConversationSetting>();
+        if (convSettings.EnableStateLog)
+        {
+            var db = _services.GetRequiredService<IBotSharpRepository>();
+            db.SaveConversationStateLog(log);
+        }
+
+        return JsonSerializer.Serialize(log, _serializerOptions);
     }
 }