From 7b365d7b847730262ebf3b7a5f5ce99cd88969b2 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 27 Nov 2023 17:49:24 -0600 Subject: [PATCH 1/2] add llm completion log --- .../Conversations/Models/LlmCompletionLog.cs | 12 +++++ .../Repositories/IBotSharpRepository.cs | 4 ++ .../Repository/BotSharpDbContext.cs | 7 +++ .../Repository/FileRepository.cs | 44 +++++++++++-------- .../Collections/LlmCompletionLogCollection.cs | 11 +++++ .../BotSharp.Plugin.MongoStorage/MongoBase.cs | 2 - .../MongoDbContext.cs | 3 ++ .../Repository/MongoRepository.cs | 18 ++++++++ 8 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs create mode 100644 src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogCollection.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs new file mode 100644 index 000000000..26f554945 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/LlmCompletionLog.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Abstraction.Conversations.Models; + +public class LlmCompletionLog +{ + public string Id { get; set; } = string.Empty; + public string ConversationId { get; set; } = string.Empty; + public string MessageId { get; set; } = string.Empty; + public string AgentId { get; set; } = string.Empty; + public string Prompt { get; set; } = string.Empty; + public string? Response { get; set; } + public DateTime CreateDateTime { get; set; } = DateTime.UtcNow; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index 1598537e7..ba8e9d843 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -41,4 +41,8 @@ public interface IBotSharpRepository void AddExectionLogs(string conversationId, List logs); List GetExectionLogs(string conversationId); #endregion + + #region LLM Completion Log + void SaveLlmCompletionLog(LlmCompletionLog log); + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index af1fd0143..48c1a7803 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -197,4 +197,11 @@ public void CreateUser(User user) throw new NotImplementedException(); } #endregion + + #region LLM Completion Log + public void SaveLlmCompletionLog(LlmCompletionLog log) + { + throw new NotImplementedException(); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index aae9455be..306acc4b2 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -7,6 +7,7 @@ using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Utilities; +using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.Core.Repository; @@ -617,10 +618,10 @@ public bool DeleteConversation(string conversationId) { if (string.IsNullOrEmpty(conversationId)) return false; - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); - if (!Directory.Exists(dir)) return false; + var convDir = FindConversationDirectory(conversationId); + if (string.IsNullOrEmpty(convDir)) return false; - Directory.Delete(dir, true); + Directory.Delete(convDir, true); return true; } @@ -842,6 +843,25 @@ public void CreateUser(User user) } #endregion + #region LLM Completion Log + public void SaveLlmCompletionLog(LlmCompletionLog log) + { + var convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + if (!Directory.Exists(convDir)) return; + + var logDir = Path.Combine(convDir, "llm_prompt_log"); + if (!Directory.Exists(logDir)) + { + Directory.CreateDirectory(logDir); + } + + var fileName = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id; + var file = Path.Combine(logDir, $"{fileName}.log"); + File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); + } + #endregion + + #region Private methods private string GetAgentDataDir(string agentId) { @@ -934,22 +954,10 @@ private List FetchResponses(string fileDir) private string? FindConversationDirectory(string conversationId) { - var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir); - - foreach (var d in Directory.GetDirectories(dir)) - { - var path = Path.Combine(d, "conversation.json"); - if (!File.Exists(path)) continue; - - var json = File.ReadAllText(path); - var conv = JsonSerializer.Deserialize(json, _options); - if (conv != null && conv.Id == conversationId) - { - return d; - } - } + var dir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, conversationId); + if (!Directory.Exists(dir)) return null; - return null; + return dir; } private List CollectDialogElements(string dialogDir) diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogCollection.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogCollection.cs new file mode 100644 index 000000000..266742626 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/LlmCompletionLogCollection.cs @@ -0,0 +1,11 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class LlmCompletionLogCollection : MongoBase +{ + public string ConversationId { get; set; } + public string MessageId { get; set; } + public string AgentId { get; set; } + public string Prompt { get; set; } + public string? Response { get; set; } + public DateTime CreateDateTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoBase.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoBase.cs index 6d8842bc3..0af038a5f 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoBase.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoBase.cs @@ -1,5 +1,3 @@ -using MongoDB.Bson.Serialization.Attributes; - namespace BotSharp.Plugin.MongoStorage; [BsonIgnoreExtraElements(Inherited = true)] diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index c8fb7e281..869920941 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -45,4 +45,7 @@ public IMongoCollection Users public IMongoCollection UserAgents => Database.GetCollection($"{_collectionPrefix}_UserAgents"); + + public IMongoCollection LlmCompletionLogs + => Database.GetCollection($"{_collectionPrefix}_Llm_Completion_Logs"); } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs index 61f3ffd31..553ca34e5 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.cs @@ -862,4 +862,22 @@ public void CreateUser(User user) _dc.Users.InsertOne(userCollection); } #endregion + + #region LLM Completion Log + public void SaveLlmCompletionLog(LlmCompletionLog log) + { + var completiongLog = new LlmCompletionLogCollection + { + Id = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id, + ConversationId = log.ConversationId, + MessageId = log.MessageId, + AgentId = log.AgentId, + Prompt = log.Prompt, + Response = log.Response, + CreateDateTime = log.CreateDateTime + }; + + _dc.LlmCompletionLogs.InsertOne(completiongLog); + } + #endregion } From 258fcb91c55a2533b4300534e40caf66df4cb62c Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 27 Nov 2023 19:20:46 -0600 Subject: [PATCH 2/2] add log index --- .../Repository/FileRepository.cs | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs index 306acc4b2..d03dd4762 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository.cs @@ -560,7 +560,7 @@ public string GetAgentTemplate(string agentId, string templateName) foreach (var file in Directory.GetFiles(dir)) { var fileName = file.Split(Path.DirectorySeparatorChar).Last(); - var splits = fileName.ToLower().Split('.'); + var splits = ParseFileNameByPath(fileName.ToLower()); var name = splits[0]; var extension = splits[1]; if (name.IsEqualTo(templateName) && extension.IsEqualTo(_agentSettings.TemplateFormat)) @@ -846,7 +846,7 @@ public void CreateUser(User user) #region LLM Completion Log public void SaveLlmCompletionLog(LlmCompletionLog log) { - var convDir = Path.Combine(_dbSettings.FileRepository, _conversationSettings.DataDir, log.ConversationId); + var convDir = FindConversationDirectory(log.ConversationId); if (!Directory.Exists(convDir)) return; var logDir = Path.Combine(convDir, "llm_prompt_log"); @@ -855,8 +855,8 @@ public void SaveLlmCompletionLog(LlmCompletionLog log) Directory.CreateDirectory(logDir); } - var fileName = string.IsNullOrEmpty(log.Id) ? Guid.NewGuid().ToString() : log.Id; - var file = Path.Combine(logDir, $"{fileName}.log"); + var index = GetLlmCompletionLogIndex(logDir, log.MessageId); + var file = Path.Combine(logDir, $"{log.MessageId}.{index}.log"); File.WriteAllText(file, JsonSerializer.Serialize(log, _options)); } #endregion @@ -1008,5 +1008,30 @@ private List CollectConversationStates(string stateDir) } return states; } + + private int GetLlmCompletionLogIndex(string logDir, string id) + { + var files = Directory.GetFiles(logDir); + if (files.IsNullOrEmpty()) + return 0; + + var logIndexes = files.Where(file => + { + var fileName = ParseFileNameByPath(file); + return fileName[0].IsEqualTo(id); + }).Select(file => + { + var fileName = ParseFileNameByPath(file); + return int.Parse(fileName[1]); + }).ToList(); + + return logIndexes.IsNullOrEmpty() ? 0 : logIndexes.Max() + 1; + } + + private string[] ParseFileNameByPath(string path, string separator = ".") + { + var name = path.Split(Path.DirectorySeparatorChar).Last(); + return name.Split(separator); + } #endregion }