refine instruction log

src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs

@@ -27,8 +27,6 @@ public interface IConversationService
     /// <param name="newMessageId">If not null, delete messages while input a new message; otherwise delete messages only</param>
     /// <returns></returns>
     Task<bool> TruncateConversation(string conversationId, string messageId, string? newMessageId = null);
-    Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
-    Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
 
     /// <summary>
     /// Send message to LLM
@@ -72,4 +70,6 @@ Task<bool> SendMessage(string agentId,
     /// <param name="preLoad">if pre-loading, then keys are not filter by the search query</param>
     /// <returns></returns>
     Task<List<string>> GetConversationStateSearhKeys(string query, int convLimit = 100, int keyLimit = 10, bool preload = false);
+
+    Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10);
 }

src/Infrastructure/BotSharp.Abstraction/Instructs/Models/InstructLogFilter.cs

@@ -1,5 +1,3 @@
-using System.Collections.Generic;
-
 namespace BotSharp.Abstraction.Instructs.Models;
 
 public class InstructLogFilter : Pagination
@@ -8,6 +6,7 @@ public class InstructLogFilter : Pagination
     public List<string>? Providers { get; set; }
     public List<string>? Models { get; set; }
     public List<string>? TemplateNames { get; set; }
+    public List<string>? UserIds { get; set; }
 
     public static InstructLogFilter Empty()
     {

src/Infrastructure/BotSharp.Abstraction/Loggers/Models/InstructionLogModel.cs

@@ -39,6 +39,9 @@ public class InstructionLogModel
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? UserId { get; set; }
 
+    [JsonIgnore]
+    public string? UserName { get; set; }
+
     [JsonIgnore]
     public Dictionary<string, string> States { get; set; } = [];
 

src/Infrastructure/BotSharp.Abstraction/Loggers/Services/ILoggerService.cs

@@ -0,0 +1,16 @@
+using BotSharp.Abstraction.Instructs.Models;
+using BotSharp.Abstraction.Loggers.Models;
+
+namespace BotSharp.Abstraction.Loggers.Services;
+
+public interface ILoggerService
+{
+    #region Conversation
+    Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId);
+    Task<List<ConversationStateLogModel>> GetConversationStateLogs(string conversationId);
+    #endregion
+
+    #region Instruction
+    Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter);
+    #endregion
+}

src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs

@@ -44,7 +44,7 @@
     List<User> GetUsersByAffiliateId(string affiliateId) => throw new NotImplementedException();
     User? GetUserByUserName(string userName) => throw new NotImplementedException();
     void UpdateUserName(string userId, string userName) => throw new NotImplementedException();
     Dashboard? GetDashboard(string id = null) => throw new NotImplementedException();
     void CreateUser(User user) => throw new NotImplementedException();
     void UpdateExistUser(string userId, User user) => throw new NotImplementedException();
     void UpdateUserVerified(string userId) => throw new NotImplementedException();
@@ -151,6 +151,10 @@
          => throw new NotImplementedException();
     List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, int convUpperLimit = 100)
          => throw new NotImplementedException();
+    List<string> GetConversationsToMigrate(int batchSize = 100)
+        => throw new NotImplementedException();
+    bool MigrateConvsersationLatestStates(string conversationId)
+         => throw new NotImplementedException();
     #endregion
 
     #region LLM Completion Log

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>$(TargetFramework)</TargetFramework>

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Migration.cs

@@ -0,0 +1,98 @@
+using NetTopologySuite.Algorithm;
+using System.Diagnostics;
+
+namespace BotSharp.Core.Conversations.Services;
+
+public partial class ConversationService
+{
+    public async Task<bool> MigrateLatestStates(int batchSize = 100, int errorLimit = 10)
+    {
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var isSuccess = true;
+        var errorCount = 0;
+        var batchNum = 0;
+        var info = string.Empty;
+        var error = string.Empty;
+
+#if DEBUG
+        Console.WriteLine($"\r\n#Start migrating Conversation Latest States...\r\n");
+#else
+        _logger.LogInformation($"#Start migrating Conversation Latest States...");
+#endif
+        var sw = Stopwatch.StartNew();
+
+        var convIds = db.GetConversationsToMigrate(batchSize);
+
+        while (!convIds.IsNullOrEmpty())
+        {
+            batchNum++;
+            var innerSw = Stopwatch.StartNew();
+#if DEBUG
+            Console.WriteLine($"\r\n#Start migrating Conversation Latest States (batch number: {batchNum})\r\n");
+#else
+            _logger.LogInformation($"#Start migrating Conversation Latest States (batch number: {batchNum})");
+#endif
+
+            for (int i = 0; i < convIds.Count; i++)
+            {
+                var convId = convIds.ElementAt(i);
+                try
+                {
+                    var done = db.MigrateConvsersationLatestStates(convId);
+                    info = $"Conversation {convId} latest states have been migrated ({i + 1}/{convIds.Count})!";
+#if DEBUG
+                    Console.WriteLine($"\r\n{info}\r\n");
+#else
+                    _logger.LogInformation($"{info}");
+#endif
+                }
+                catch (Exception ex)
+                {
+                    errorCount++;
+                    error = $"Conversation {convId} latest states fail to be migrated! ({i + 1}/{convIds.Count})\r\n{ex.Message}\r\n{ex.InnerException}";
+#if DEBUG
+                    Console.WriteLine($"\r\n{error}\r\n");
+#else
+                    _logger.LogError($"{error}");
+#endif
+                }
+            }
+
+            if (errorCount >= errorLimit)
+            {
+                error = $"\r\nErrors exceed limit => stop the migration!\r\n";
+#if DEBUG
+                Console.WriteLine($"{error}");
+#else
+                _logger.LogError($"{error}");
+#endif
+                innerSw.Stop();
+                isSuccess = false;
+                break;
+            }
+
+            innerSw.Stop();
+            info = $"#Done migrating Conversation Latest States (batch number: {batchNum}) " +
+                $"(Total time: {innerSw.Elapsed.Hours} hrs, {innerSw.Elapsed.Minutes} mins, {innerSw.Elapsed.Seconds} seconds)";
+#if DEBUG
+            Console.WriteLine($"\r\n{info}\r\n");
+#else
+            _logger.LogInformation($"{info}");
+#endif
+
+            await Task.Delay(100);
+            convIds = db.GetConversationsToMigrate(batchSize);
+        }
+
+        sw.Stop();
+        info = $"#Done with migrating Conversation Latest States! " +
+            $"(Total time: {sw.Elapsed.Days} days, {sw.Elapsed.Hours} hrs, {sw.Elapsed.Minutes} mins, {sw.Elapsed.Seconds} seconds)";
+#if DEBUG
+        Console.WriteLine($"\r\n{info}\r\n");
+#else
+        _logger.LogInformation($"{info}");
+#endif
+
+        return isSuccess;
+    }
+}

src/Infrastructure/BotSharp.Core/Instructs/InsturctionPlugin.cs

@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Plugins.Models;
-using BotSharp.Abstraction.Users.Enums;
 using Microsoft.Extensions.Configuration;
 
 namespace BotSharp.Core.Instructs;
@@ -22,8 +21,8 @@ public bool AttachMenu(List<PluginMenuDef> menu)
         {
             SubMenu = new List<PluginMenuDef>
             {
-                new PluginMenuDef("Instruction", link: "page/instruction"),
-                new PluginMenuDef("Log", link: "page/instruction/log") { Roles = [UserRole.Root, UserRole.Admin] }
+                new PluginMenuDef("Testing", link: "page/instruction/testing"),
+                new PluginMenuDef("Log", link: "page/instruction/log")
             }
         });
 

src/Infrastructure/BotSharp.Core/Loggers/LoggerPlugin.cs

@@ -0,0 +1,15 @@
+using Microsoft.Extensions.Configuration;
+
+namespace BotSharp.Core.Loggers;
+
+public class LoggerPlugin : IBotSharpPlugin
+{
+    public string Id => "ea1aade7-7e29-4f13-a78b-2b1835aa4fea";
+    public string Name => "Logger";
+    public string Description => "Provide log service";
+
+    public void RegisterDI(IServiceCollection services, IConfiguration config)
+    {
+        services.AddScoped<ILoggerService, LoggerService>();
+    }
+}

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.Log.cs → src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Conversation.cs

@@ -1,9 +1,8 @@
 using BotSharp.Abstraction.Loggers.Models;
-using BotSharp.Abstraction.Repositories;
 
-namespace BotSharp.Core.Conversations.Services;
+namespace BotSharp.Core.Loggers.Services;
 
-public partial class ConversationService
+public partial class LoggerService
 {
     public async Task<List<ContentLogOutputModel>> GetConversationContentLogs(string conversationId)
     {

src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.Instruction.cs

@@ -0,0 +1,64 @@
+using BotSharp.Abstraction.Instructs.Models;
+using BotSharp.Abstraction.Loggers.Models;
+using BotSharp.Abstraction.Users.Enums;
+using BotSharp.Abstraction.Users.Models;
+
+namespace BotSharp.Core.Loggers.Services;
+
+public partial class LoggerService
+{
+    public async Task<PagedItems<InstructionLogModel>> GetInstructionLogs(InstructLogFilter filter)
+    {
+        if (filter == null)
+        {
+            filter = InstructLogFilter.Empty();
+        }
+
+        var userService = _services.GetRequiredService<IUserService>();
+        var user = await userService.GetUser(_user.Id);
+        var isAdmin = UserConstant.AdminRoles.Contains(user?.Role);
+        if (!isAdmin && user?.Id == null) return new();
+
+        filter.UserIds = isAdmin ? [] : user?.Id != null ? [user.Id] : [];
+
+        var agents = new List<Agent>();
+        var users = new List<User>();
+
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var logs = db.GetInstructionLogs(filter);
+        var agentIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.AgentId)).Select(x => x.AgentId).ToList();
+        var userIds = logs.Items.Where(x => !string.IsNullOrEmpty(x.UserId)).Select(x => x.UserId).ToList();
+        agents = db.GetAgents(new AgentFilter
+        {
+            AgentIds = agentIds,
+            Pager = new Pagination { Size = filter.Size }
+        });
+
+        if (isAdmin)
+        {
+            users = db.GetUserByIds(userIds);
+        }
+
+        var items = logs.Items.Select(x =>
+        {
+            x.AgentName = !string.IsNullOrEmpty(x.AgentId) ? agents.FirstOrDefault(a => a.Id == x.AgentId)?.Name : null;
+
+            if (!isAdmin)
+            {
+                x.UserName = user != null ? $"{user.FirstName} {user.LastName}" : null;
+            }
+            else
+            {
+                var found = !string.IsNullOrEmpty(x.UserId) ? users.FirstOrDefault(u => u.Id == x.UserId) : null;
+                x.UserName = found != null ? $"{found.FirstName} {found.LastName}" : null;
+            }
+            return x;
+        }).ToList();
+
+        return new PagedItems<InstructionLogModel>
+        {
+            Items = items,
+            Count = logs.Count
+        };
+    }
+}

src/Infrastructure/BotSharp.Core/Loggers/Services/LoggerService.cs

@@ -0,0 +1,18 @@
+namespace BotSharp.Core.Loggers.Services;
+
+public partial class LoggerService : ILoggerService
+{
+    private readonly IServiceProvider _services;
+    private readonly IUserIdentity _user;
+    private readonly ILogger<LoggerService> _logger;
+
+    public LoggerService(
+        IServiceProvider services,
+        IUserIdentity user,
+        ILogger<LoggerService> logger)
+    {
+        _services = services;
+        _user = user;
+        _logger = logger;
+    }
+}

src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs

@@ -687,6 +687,56 @@ public List<string> GetConversationStateSearchKeys(int messageLowerLimit = 2, in
     }
 
 
+
+    public List<string> GetConversationsToMigrate(int batchSize = 100)
+    {
+        var baseDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir);
+        if (!Directory.Exists(baseDir)) return [];
+
+        var convIds = new List<string>();
+        var dirs = Directory.GetDirectories(baseDir);
+
+        foreach (var dir in dirs)
+        {
+            var latestStateFile = Path.Combine(dir, CONV_LATEST_STATE_FILE);
+            if (File.Exists(latestStateFile)) continue;
+
+            var convId = dir.Split(Path.DirectorySeparatorChar).Last();
+            if (string.IsNullOrEmpty(convId)) continue;
+
+            convIds.Add(convId);
+            if (convIds.Count >= batchSize)
+            {
+                break;
+            }
+        }
+
+        return convIds;
+    }
+
+
+    public bool MigrateConvsersationLatestStates(string conversationId)
+    {
+        if (string.IsNullOrEmpty(conversationId)) return false;
+
+        var convDir = FindConversationDirectory(conversationId);
+        if (string.IsNullOrEmpty(convDir))
+        {
+            return false;
+        }
+
+        var stateFile = Path.Combine(convDir, STATE_FILE);
+        var states = CollectConversationStates(stateFile);
+        var latestStates = BuildLatestStates(states);
+
+        var latestStateFile = Path.Combine(convDir, CONV_LATEST_STATE_FILE);
+        var stateStr = JsonSerializer.Serialize(latestStates, _options);
+        File.WriteAllText(latestStateFile, stateStr);
+
+        return true;
+    }
+
+
     #region Private methods
     private string? FindConversationDirectory(string conversationId)
     {
@@ -883,6 +933,11 @@ private Dictionary<string, JsonDocument> CollectConversationLatestStates(string
     private Dictionary<string, JsonDocument> BuildLatestStates(List<StateKeyValue> states)
     {
         var endNodes = new Dictionary<string, JsonDocument>();
+        if (states.IsNullOrEmpty())
+        {
+            return endNodes;
+        }
+
         foreach (var pair in states)
         {
             var value = pair.Values?.LastOrDefault();