diff --git a/BotSharp.sln b/BotSharp.sln index 8dda48e83..d281450e8 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -449,6 +451,14 @@ Global {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -503,6 +513,7 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} + {F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 6e8a2ae32..2a2705db0 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,7 +10,7 @@ - + diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 94c9e27c5..697f94a56 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 2018d6a79..8484c5fb0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Conversations.Models; @@ -75,6 +74,9 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? RichContent { get; set; } + /// + /// Rich content for secondary language + /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? SecondaryRichContent { get; set; } @@ -86,6 +88,9 @@ public class RoleDialogModel : ITrackableMessage public FunctionCallFromLlm Instruction { get; set; } + /// + /// Files to be used in conversation + /// public List Files { get; set; } = new List(); /// @@ -94,7 +99,8 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - public float KnowledgeConfidence { get; set; } = 0.5f; + + private RoleDialogModel() { } @@ -134,8 +140,7 @@ public static RoleDialogModel From(RoleDialogModel source, Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, - Data = source.Data, - KnowledgeConfidence = source.KnowledgeConfidence + Data = source.Data }; } } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 87df61374..24b284029 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { - public string Name { get; } + public string Provider { get; } /// /// Convert pdf pages to images, and return a list of image file paths diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index 10ccd1a09..dc4ac40bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public string Pdf2TextConverter { get; set; } - public string Pdf2ImageConverter { get; set; } + public SettingBase Pdf2TextConverter { get; set; } + public SettingBase Pdf2ImageConverter { get; set; } } + +public class SettingBase +{ + public string Provider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 78a400e1d..f8f4c605a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + Task VaryImage(string? provider, string? model, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask); #endregion #region Pdf @@ -17,7 +17,11 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + #endregion + + #region Audio + Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index a3e5a32dc..373466c41 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,6 +6,7 @@ public interface IFileStorageService { #region Common string GetDirectory(string conversationId); + IEnumerable GetFiles(string relativePath, string? searchQuery = null); byte[] GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); @@ -16,7 +17,6 @@ public interface IFileStorageService string BuildDirectory(params string[] segments); #endregion - #region Conversation /// /// Get the message file screenshots for specific content types, e.g., pdf @@ -24,7 +24,7 @@ public interface IFileStorageService /// /// /// - Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -37,7 +37,7 @@ public interface IFileStorageService IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// /// Delete files under messages @@ -54,10 +54,11 @@ public interface IFileStorageService #region User string GetUserAvatar(); - bool SaveUserAvatar(BotSharpFile file); + bool SaveUserAvatar(InputFileModel file); #endregion + #region Speech - Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); - Task RetrieveSpeechFileAsync(string conversationId, string fileName); + bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); + BinaryData GetSpeechFile(string conversationId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 11a11e469..bc6706269 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,12 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileBase +public class BotSharpFile : FileInformation { - + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index 3483921af..f952e7f90 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models; public class FileBase { - /// - /// External file url - /// - [JsonPropertyName("file_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileUrl { get; set; } = string.Empty; - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileStorageUrl { get; set; } = string.Empty; - /// /// File name without extension /// @@ -29,18 +15,4 @@ public class FileBase [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; - - /// - /// File content type - /// - [JsonPropertyName("content_type")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ContentType { get; set; } = string.Empty; - - /// - /// File extension without dot - /// - [JsonPropertyName("file_extension")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs new file mode 100644 index 000000000..fe767e163 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileInformation +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs new file mode 100644 index 000000000..7eb350e3d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InputFileModel : FileBase +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public new string FileName { get; set; } = string.Empty; + + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public new string FileData { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs deleted file mode 100644 index 29749bce3..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class InputMessageFiles -{ - public List Files { get; set; } = new List(); - public BotSharpFile? Mask { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs new file mode 100644 index 000000000..7eaddd5af --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InstructFileModel : FileBase +{ + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; + + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 2a0128b66..da2ddec66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileBase +public class MessageFileModel : FileInformation { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906df..bba60b101 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,6 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Abstraction.Files.Utilities; @@ -26,11 +28,30 @@ public static (string, byte[]) GetFileInfoFromData(string data) return (contentType, Convert.FromBase64String(base64Str)); } - public static string GetFileContentType(string filePath) + public static string BuildFileDataFromFile(string fileName, byte[] bytes) + { + var contentType = GetFileContentType(fileName); + var base64 = Convert.ToBase64String(bytes); + return $"data:{contentType};base64,{base64}"; + } + + public static string BuildFileDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var contentType = GetFileContentType(file.FileName); + var base64 = Convert.ToBase64String(stream.ToArray()); + stream.Close(); + + return $"data:{contentType};base64,{base64}"; + } + + public static string GetFileContentType(string fileName) { string contentType; var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) + if (!provider.TryGetContentType(fileName, out contentType)) { contentType = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs index 64c5388b9..b15503e22 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph; public interface IGraphDb { - public string Name { get; } + public string Provider { get; } Task Search(string query, GraphSearchOptions options); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 20825791f..1021ea1c0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + #region Vector + Task CreateVectorCollection(string collectionName, int dimension); + Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); @@ -12,6 +15,10 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + #endregion + + #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index b8f2d47b1..d6edad54c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - public string Name { get; } + public string Provider { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs deleted file mode 100644 index 7d06a8a41..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs +++ /dev/null @@ -1,11 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; - -namespace BotSharp.Abstraction.Knowledges; - -/// -/// Chop large content into chunks -/// -public interface ITextChopper -{ - List Chop(string content, ChunkOption option); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 963a2b3b3..217dafb9e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { - public string VectorDb { get; set; } - public string GraphDb { get; set; } + public SettingBase VectorDb { get; set; } + public SettingBase GraphDb { get; set; } public DefaultKnowledgeBaseSetting Default { get; set; } public List Collections { get; set; } = new(); @@ -23,9 +23,13 @@ public class VectorCollectionSetting public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } } -public class KnowledgeTextEmbeddingSetting +public class KnowledgeTextEmbeddingSetting : SettingBase { - public string Provider { get; set; } public string Model { get; set; } public int Dimension { get; set; } +} + +public class SettingBase +{ + public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs new file mode 100644 index 000000000..54ed2d7d3 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IAudioCompletion +{ + string Provider { get; } + + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); + Task GenerateAudioFromTextAsync(string text); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs deleted file mode 100644 index 9e0dd574e..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks; - -public interface ISpeechToText -{ - string Provider { get; } - - Task GenerateTextFromAudioAsync(string filePath); - // Task AudioToTextTranscript(Stream stream); - Task SetModelName(string modelType); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs deleted file mode 100644 index 344fad0ea..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks -{ - public interface ITextToSpeech - { - /// - /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI - /// - string Provider { get; } - - /// - /// Set model name, one provider can consume different model or version(s) - /// - /// deployment name - void SetModelName(string model); - - Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); - } - - public interface ITextToSpeechOptions - { - - } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index cc7cfba07..008f528bc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -68,5 +68,6 @@ public enum LlmModelType Text = 1, Chat = 2, Image = 3, - Embedding = 4 + Embedding = 4, + Audio = 5 } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs new file mode 100644 index 000000000..f981ba6e9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 24e4152c4..9ea5917cc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class MessageConfig : InputMessageFiles +public class MessageConfig { /// /// Completion Provider @@ -15,7 +15,7 @@ public class MessageConfig : InputMessageFiles public virtual string? Model { get; set; } = null; /// - /// Model name + /// Model id /// [JsonPropertyName("model_id")] public virtual string? ModelId { get; set; } = null; @@ -34,7 +34,7 @@ public class MessageConfig : InputMessageFiles /// /// Conversation states from input /// - public List States { get; set; } = new List(); + public List States { get; set; } = new(); /// /// Agent task id diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs new file mode 100644 index 000000000..ec519f046 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.VectorStorage.Extensions; + +public static class VectorStorageExtension +{ + public static string ToQuestionAnswer(this VectorSearchResult data) + { + if (data?.Data == null) return string.Empty; + + return $"Question: {data.Data[KnowledgePayloadName.Text]}\r\nAnswer: {data.Data[KnowledgePayloadName.Answer]}"; + } + + public static string ToPayloadPair(this VectorSearchResult data, IList payloads) + { + if (data?.Data == null || payloads.IsNullOrEmpty()) return string.Empty; + + var results = data.Data.Where(x => payloads.Contains(x.Key)) + .OrderBy(x => payloads.IndexOf(x.Key)) + .Select(x => + { + return $"{x.Key}: {x.Value}"; + }) + .ToList(); + + return string.Join("\r\n", results.Where(x => !string.IsNullOrWhiteSpace(x))); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 53b921a1a..cedacb46c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,12 +4,13 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - string Name { get; } + string Provider { get; } Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dim); + Task CreateCollection(string collectionName, int dimension); + Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, Guid id); diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 91abcc25f..85b9dec29 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -4,4 +4,7 @@ public class VectorFilter : StringIdPagination { [JsonPropertyName("with_vector")] public bool WithVector { get; set; } -} + + [JsonPropertyName("search_pairs")] + public IEnumerable? SearchPairs { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs index ce39edbf9..f2c582ce1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchResult : VectorCollectionData diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 05d485b74..ff10a5176 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index f06e2ba2b..05653d9c6 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -6,6 +6,7 @@ using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; using BotSharp.Core.Instructs; +using BotSharp.Core.Knowledges.Services; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; using BotSharp.Core.Templating; @@ -54,6 +55,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } public bool AttachMenu(List menu) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs new file mode 100644 index 000000000..e2aa844cd --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -0,0 +1,20 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) + { + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var audioBytes = await DownloadFile(audio); + using var stream = new MemoryStream(); + stream.Write(audioBytes, 0, audioBytes.Length); + stream.Position = 0; + + var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; + var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); + stream.Close(); + return content; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index c9d35cb7e..cd65c16a9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -14,10 +14,10 @@ public async Task ReadImages(string? provider, string? model, s { new RoleDialogModel(AgentRole.User, text) { - Files = images?.ToList() ?? new List() + Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); - return message; + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text) @@ -30,7 +30,7 @@ public async Task GenerateImage(string? provider, string? model return message; } - public async Task VaryImage(string? provider, string? model, BotSharpFile image) + public async Task VaryImage(string? provider, string? model, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -43,16 +43,17 @@ public async Task VaryImage(string? provider, string? model, Bo stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -65,16 +66,17 @@ public async Task EditImage(string? provider, string? model, st stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -94,10 +96,12 @@ public async Task EditImage(string? provider, string? model, st maskStream.Write(maskBytes, 0, maskBytes.Length); maskStream.Position = 0; + var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); @@ -105,7 +109,7 @@ public async Task EditImage(string? provider, string? model, st } #region Private methods - private async Task DownloadFile(BotSharpFile file) + private async Task DownloadFile(InstructFileModel file) { var bytes = new byte[0]; if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2aec257b4..29dfdaf9a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { var content = string.Empty; @@ -50,7 +50,7 @@ public async Task ReadPdf(string? provider, string? model, string? model } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { @@ -80,9 +80,9 @@ private async Task> DownloadFiles(string dir, List> ConvertPdfToImages(IEnumerable f { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs index acd0ddaae..416d9f307 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Core.Files.Services; public partial class FileInstructService : IFileInstructService diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index e90c7812d..1aaba5755 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -1,28 +1,38 @@ using System.IO; -namespace BotSharp.Core.Files.Services +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService { - public partial class LocalFileStorageService + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + try { var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } + var filePath = Path.Combine(dir, fileName); - if (File.Exists(filePath)) return; - using var file = File.Create(filePath); - using var input = data.ToStream(); - await input.CopyToAsync(file); - } + if (File.Exists(filePath)) return false; - public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + using var fs = File.Create(filePath); + using var ds = data.ToStream(); + ds.CopyTo(fs); + return true; + } + catch (Exception ex) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); - using var file = new FileStream(path, FileMode.Open, FileAccess.Read); - return await BinaryData.FromStreamAsync(file); + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; } } + + public BinaryData GetSpeechFile(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return BinaryData.FromStream(file); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c49edbce6..b16c2c2cf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,6 +14,22 @@ public string GetDirectory(string conversationId) return dir; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Enumerable.Empty(); + } + + var path = Path.Combine(_baseDir, relativePath); + + if (!string.IsNullOrWhiteSpace(searchPattern)) + { + return Directory.GetFiles(path, searchPattern); + } + return Directory.GetFiles(path); + } + public byte[] GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 8c2ed8310..5638179cf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) @@ -118,7 +118,7 @@ public IEnumerable GetMessagesWithFile(string conversationId, return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; @@ -276,7 +276,7 @@ private async Task> ConvertPdfToImages(string pdfLoc, string private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 43ff9eed3..d1e962cd4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -16,7 +16,7 @@ public string GetUserAvatar() return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 94ee2beb9..750803c42 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _audioTypes = new List - { - "mp3", - "wav" - }; - private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 874a063e1..d6746e09a 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -28,6 +28,10 @@ public static object GetCompletion(IServiceProvider services, { return GetImageCompletion(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Audio) + { + return GetAudioCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -108,7 +112,7 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services, if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); + logger.LogError($"Can't resolve text-embedding provider by {provider}"); } @@ -120,35 +124,19 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services, return completer; } - public static ITextToSpeech GetTextToSpeech( + public static IAudioCompletion GetAudioCompletion( IServiceProvider services, string provider, string model) { - var completions = services.GetServices(); + var completions = services.GetServices(); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve text2speech provider by {provider}"); + logger.LogError($"Can't resolve audio-completion provider by {provider}"); } - completer.SetModelName(model); - return completer; - } - public static ISpeechToText GetSpeechToText( - IServiceProvider services, - string provider, - string model - ) - { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve speech2text provider by {provider}"); - } completer.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs new file mode 100644 index 000000000..a278d7bff --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -0,0 +1,36 @@ +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; + +public static class KnowledgeSettingHelper +{ + public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) + { + var settings = services.GetRequiredService(); + var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding; + if (found == null) + { + found = settings.Default.TextEmbedding; + } + + var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); + var dimension = found.Dimension; + + if (found.Dimension <= 0) + { + dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model); + } + + embedding.SetModelName(found.Model); + embedding.SetDimension(dimension); + return embedding; + } + + private static int GetLlmTextEmbeddingDimension(IServiceProvider services, string provider, string model) + { + var settings = services.GetRequiredService(); + var found = settings.GetSetting(provider, model); + return found?.Dimension ?? 0; + } +} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs similarity index 78% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs index 88c77641a..68b585393 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs @@ -1,17 +1,18 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Helpers; -public class TextChopperService : ITextChopper +public static class TextChopper { - public List Chop(string content, ChunkOption option) + public static List Chop(string content, ChunkOption option) { content = Regex.Replace(content, @"\.{2,}", " "); content = Regex.Replace(content, @"_{2,}", " "); return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option); } - private List ChopByWord(string content, ChunkOption option) + private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); @@ -34,7 +35,7 @@ private List ChopByWord(string content, ChunkOption option) return chunks; } - private List ChopByChar(string content, ChunkOption option) + private static List ChopByChar(string content, ChunkOption option) { var chunks = new List(); var currentPos = 0; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 65% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 9b185867e..d8b82fb51 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,11 +1,15 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + var lines = TextChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, @@ -25,6 +29,25 @@ public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationMo } } + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 51% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index e78990a79..404b4e2ca 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,7 +1,26 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task DeleteVectorCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780ec..69093659a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd8..b9652f617 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 64% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index eec498910..2c0853d0a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,38 +1,41 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ITextChopper _textChopper; private readonly ILogger _logger; public KnowledgeService( IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper, ILogger logger) { _services = services; _settings = settings; - _textChopper = textChopper; _logger = logger; } private IVectorDb GetVectorDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); return db; } private IGraphDb GetGraphDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider); return db; } private ITextEmbedding GetTextEmbedding(string collection) { - return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection); + return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs index 72549e517..0fcaa5a16 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Core.Routing.Planning; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid index 0332af76c..4c9f51c37 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid @@ -1,8 +1,10 @@ -You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name. -the summarized question/answer should: +You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name. + +The summarized question/answer should: 1. help user to identify the location of tables to find further information 2. identify the table structure and data relationship based on the task description -3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer +3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer + Go through all the columns and generate multiple question & answer pairs. The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n. diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index cf6985e9e..afb8dd019 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -23,10 +23,13 @@ public async Task BeforeGenerating(Agent agent, List conversati { if (!_convSettings.ShowVerboseLog) return; - var dialog = conversations.Last(); - var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; - _logger.LogInformation(log); - + var dialog = conversations.LastOrDefault(); + if (dialog != null) + { + var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; + _logger.LogInformation(log); + } + await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 240022258..9094107da 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 9b964dffd..565677f7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -294,9 +294,7 @@ await conv.SendMessage(agentId, inputMsg, } [HttpPost("/conversation/{agentId}/{conversationId}/sse")] - public async Task SendMessageSse([FromRoute] string agentId, - [FromRoute] string conversationId, - [FromBody] NewMessageModel input) + public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { var conv = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) @@ -391,7 +389,7 @@ public IActionResult UploadAttachments([FromRoute] string conversationId, } [HttpPost("/agent/{agentId}/conversation/{conversationId}/upload")] - public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) + public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] InputMessageFiles input) { var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 48fdafe19..bc02e8464 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -20,8 +21,7 @@ public InstructModeController(IServiceProvider services, ILogger InstructCompletion([FromRoute] string agentId, - [FromBody] InstructMessageModel input) + public async Task InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -79,7 +79,7 @@ public async Task ChatCompletion([FromBody] IncomingMessageModel input) #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + public async Task MultiModalCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -87,21 +87,50 @@ public async Task MultiModalCompletion([FromBody] IncomingMessageModel i try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); - return message.Content; + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + return content; } catch (Exception ex) { - var error = $"Error in analyzing files. {ex.Message}"; + var error = $"Error in reading images. {ex.Message}"; _logger.LogError(error); return error; } } + + [HttpPost("/instruct/multi-modal/upload")] + public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new MultiModalViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadImages(provider, model, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in reading image upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] IncomingMessageModel input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -127,7 +156,7 @@ public async Task ImageGeneration([FromBody] IncomingM #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] IncomingMessageModel input) + public async Task ImageVariation([FromBody] ImageVariationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -135,16 +164,16 @@ public async Task ImageVariation([FromBody] IncomingMe try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; } catch (Exception ex) @@ -156,8 +185,43 @@ public async Task ImageVariation([FromBody] IncomingMe } } + [HttpPost("/instruct/image-variation/upload")] + public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageVariation(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] IncomingMessageModel input) + public async Task ImageEdit([FromBody] ImageEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -166,12 +230,11 @@ public async Task ImageEdit([FromBody] IncomingMessage try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -185,8 +248,44 @@ public async Task ImageEdit([FromBody] IncomingMessage } } + [HttpPost("/instruct/image-edit/upload")] + public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -195,11 +294,11 @@ public async Task ImageMaskEdit([FromBody] IncomingMes try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var image = input.File; var mask = input.Mask; if (image == null || mask == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; @@ -214,11 +313,52 @@ public async Task ImageMaskEdit([FromBody] IncomingMes return imageViewModel; } } + + [HttpPost("/instruct/image-mask-edit/upload")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + using var imageStream = new MemoryStream(); + image.CopyTo(imageStream); + imageStream.Position = 0; + + using var maskStream = new MemoryStream(); + mask.CopyTo(maskStream); + maskStream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() + { + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); + + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageStream.Close(); + maskStream.Close(); + + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } #endregion #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] IncomingMessageModel input) + public async Task PdfCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -239,5 +379,109 @@ public async Task PdfCompletion([FromBody] IncomingMessa return viewModel; } } + + [HttpPost("/instruct/pdf-completion/upload")] + public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + #endregion + + #region Audio + [HttpPost("/instruct/speech-to-text")] + public async Task SpeechToText([FromBody] SpeechToTextRequest input) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + var audio = input.File; + if (audio == null) + { + return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; + } + var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech to text. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/speech-to-text/upload")] + public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); + viewModel.Content = content; + stream.Close(); + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech-to-text upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/text-to-speech")] + public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); + var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var stream = binaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index deb21378a..827c2d7ce 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -18,12 +18,25 @@ public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvi _services = services; } + #region Vector [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections() { return await _knowledgeService.GetVectorCollections(); } + [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] + public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + { + return await _knowledgeService.CreateVectorCollection(collection, dimension); + } + + [HttpDelete("knowledge/vector/{collection}/delete-collection")] + public async Task GetVectorCollections([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollection(collection); + } + [HttpPost("/knowledge/vector/{collection}/search")] public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { @@ -91,7 +104,7 @@ public async Task DeleteVectorCollectionData([FromRoute] string collection public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); var filePath = Path.GetTempFileName(); using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) @@ -109,7 +122,10 @@ public async Task UploadVectorKnowledge([FromRoute] string collec System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + #endregion + + #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { @@ -124,7 +140,10 @@ public async Task SearchGraphKnowledge([FromBody] Searc Result = result.Result }; } + #endregion + + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) { @@ -148,4 +167,5 @@ public async Task SearchKnowledge([FromBody] SearchKno GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null }; } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 5571d9fa9..02282b858 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -25,6 +25,6 @@ public IEnumerable GetLlmProviders() public IEnumerable GetLlmProviderModels([FromRoute] string provider) { var list = _llmProvider.GetProviderModels(provider); - return list.Where(x => !x.ImageGeneration); + return list.Where(x => x.Type == LlmModelType.Chat); } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a7d4c449f..163993349 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -135,9 +135,14 @@ public async Task ModifyUserPhone([FromQuery] string phone) #region Avatar [HttpPost("/user/avatar")] - public bool UploadUserAvatar([FromBody] BotSharpFile file) + public bool UploadUserAvatar([FromBody] UserAvatarModel input) { var fileStorage = _services.GetRequiredService(); + var file = new InputFileModel + { + FileName = input.FileName, + FileData = input.FileData, + }; return fileStorage.SaveUserAvatar(file); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs new file mode 100644 index 000000000..fdf088bbc --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class InputMessageFiles +{ + public List States { get; set; } = new(); + public List Files { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 0050de015..ea8dc0763 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -2,18 +2,11 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class ImageGenerationViewModel +public class ImageGenerationViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable Images { get; set; } = new List(); - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } public class ImageViewModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs new file mode 100644 index 000000000..39a6439d9 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -0,0 +1,75 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructBaseRequest +{ + [JsonPropertyName("provider")] + public virtual string? Provider { get; set; } = null; + + [JsonPropertyName("model")] + public virtual string? Model { get; set; } = null; + + [JsonPropertyName("model_id")] + public virtual string? ModelId { get; set; } = null; + + [JsonPropertyName("states")] + public List States { get; set; } = new(); +} + +public class MultiModalRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("files")] + public List Files { get; set; } = new(); +} + +public class ImageGenerationRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} + +public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class ImageEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class ImageMaskEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } + + [JsonPropertyName("mask")] + public InstructFileModel Mask { get; set; } +} + +public class SpeechToTextRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string? Text { get; set; } + + [JsonPropertyName("file")] + public InstructFileModel File { get; set; } +} + +public class TextToSpeechRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs new file mode 100644 index 000000000..0b20fca36 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructBaseViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b265e044..b736330fc 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.OpenAPI.ViewModels.Instructs; public class InstructMessageModel : IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs new file mode 100644 index 000000000..a7b15262d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class MultiModalViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs index 13ed3eb90..7ac594e61 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -1,13 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class PdfCompletionViewModel +public class PdfCompletionViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs new file mode 100644 index 000000000..d681986d3 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class SpeechToTextViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs new file mode 100644 index 000000000..0eab88e50 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Users; + +public class UserAvatarModel +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public string FileData { get; set; } = string.Empty; +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 304aa97ba..2c289907e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,31 +1,24 @@ using BotSharp.Plugin.AudioHandler.Settings; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Abstraction.Settings; -namespace BotSharp.Plugin.AudioHandler +namespace BotSharp.Plugin.AudioHandler; + +public class AudioHandlerPlugin : IBotSharpPlugin { - public class AudioHandlerPlugin : IBotSharpPlugin + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) { - public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; - public string Name => "Audio Handler"; - public string Description => "Process audio input and transform it into text output."; - public void RegisterDI(IServiceCollection services, IConfiguration config) + services.AddScoped(provider => { - //var settings = new AudioHandlerSettings(); - //config.Bind("AudioHandler", settings); - //services.AddSingleton(x => settings); - - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settingService.Bind("AudioHandler"); - }); + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index 7218d40c1..4ae8c7d27 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -22,5 +22,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs deleted file mode 100644 index 45f4c8a49..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Provider; -using BotSharp.Core.Infrastructures; - -namespace BotSharp.Plugin.AudioHandler.Controllers -{ -#if DEBUG - [AllowAnonymous] -#endif - [ApiController] - public class AudioController : ControllerBase - { - private readonly ISpeechToText _nativeWhisperProvider; - private readonly IServiceProvider _services; - - public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) - { - _nativeWhisperProvider = nativeWhisperProvider; - _services = service; - } - - [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString, string modelType = "") - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - await _nativeWhisperProvider.SetModelName(modelType); - - var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - - [HttpPost("openai/audio/transcript")] - public async Task GetTextFromAudioOpenAiController(string filePath) - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); - var result = await client.GenerateTextFromAudioAsync(filePath); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 436b29227..7b299a76e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,32 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Whisper.net.Wave; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public enum AudioType { - public enum AudioType - { - wav, - mp3, - } + wav, + mp3, +} - public static class AudioTypeExtensions +public static class AudioTypeExtensions +{ + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) { - public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; - public static string ToFileType(this AudioType audioType) + string type = audioType switch { - string type = audioType switch - { - AudioType.mp3 => "audio/mpeg", - AudioType.wav => "audio/wav", - _ => throw new NotImplementedException($"No support found for {audioType}") - }; - return type; - } + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs index 0deab6e9f..d11bd65ac 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public class UtilityName { - public class UtilityName - { - public const string AudioHandler = "audio-handler"; - } + public const string AudioHandler = "audio-handler"; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs deleted file mode 100644 index a65443596..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ /dev/null @@ -1,68 +0,0 @@ -using BotSharp.Plugin.AudioHandler.Enums; -using NAudio; -using NAudio.Wave; -using NAudio.Wave.SampleProviders; - -namespace BotSharp.Plugin.AudioHandler.Functions; - -public class AudioProcessUtilities : IAudioProcessUtilities -{ - public AudioProcessUtilities() - { - } - - public Stream ConvertMp3ToStream(string mp3FileName) - { - var fileStream = File.OpenRead(mp3FileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - - } - - public Stream ConvertWavToStream(string wavFileName) - { - var fileStream = File.OpenRead(wavFileName); - using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) - { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; - } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; - } - - public Stream ConvertToStream(string fileName) - { - if (string.IsNullOrEmpty(fileName)) - { - throw new ArgumentNullException("fileName is Null"); - } - string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); - if (!Enum.TryParse(fileExtension, out AudioType fileType)) - { - throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); - } - - var stream = fileType switch - { - AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), - }; - - return stream; - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 0c2298c60..34ec530c6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using Microsoft.AspNetCore.StaticFiles; @@ -10,9 +9,9 @@ public class HandleAudioRequestFn : IFunctionCallback public string Indication => "Handling audio request"; private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private Agent? _agent; private readonly IEnumerable _audioContentType = new List { @@ -20,13 +19,13 @@ public class HandleAudioRequestFn : IFunctionCallback AudioType.wav.ToFileType(), }; - public HandleAudioRequestFn( + IFileStorageService fileStorage, IServiceProvider serviceProvider, ILogger logger, - BotSharpOptions options - ) + BotSharpOptions options) { + _fileStorage = fileStorage; _serviceProvider = serviceProvider; _logger = logger; _options = options; @@ -36,32 +35,31 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); - var isNeedSummary = args?.IsNeedSummary ?? false; var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + var response = await GetResponeFromDialogs(dialogs); message.Content = response; return true; } - private async Task> AssembleFiles(string convId, List dialogs) + private List AssembleFiles(string convId, List dialogs) { if (dialogs.IsNullOrEmpty()) + { return new List(); + } - var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); foreach (var dialog in dialogs) { var found = audioMessageFiles.Where(x => x.MessageId == dialog.MessageId).ToList(); - if (found.IsNullOrEmpty()) - continue; + if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { @@ -74,53 +72,48 @@ private async Task> AssembleFiles(string convId, List(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); - return canParse; - } - private async Task GetResponeFromDialogs(List dialogs) { - var whisperService = await PrepareModel("native"); // openai, native + var audioCompletion = PrepareModel(); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - int transcribedCount = 0; + var transcripts = new List(); + foreach (var file in dialog.Files) { - if (file == null) - continue; + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - string extension = Path.GetExtension(file?.FileStorageUrl); - if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) - { - file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); - transcribedCount++; - } + var extension = Path.GetExtension(file.FileStorageUrl); + + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!ParseAudioFileType(fileName)) continue; + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); } - if (transcribedCount == 0) + if (transcripts.IsNullOrEmpty()) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } - var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); - return string.Join("\n\r", resList); + + return string.Join("\r\n\r\n", transcripts); } - private async Task PrepareModel(string modelName = "native") + private IAudioCompletion PrepareModel() { - var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); - if (whisperService == null) - { - throw new Exception($"Can't resolve speech2text provider by {modelName}"); - } + return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); + } - if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) - { - return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); - } - await whisperService.SetModelName("Tiny"); - return whisperService; + private bool ParseAudioFileType(string fileName) + { + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); + return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs deleted file mode 100644 index a3c8243b3..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace BotSharp.Plugin.AudioHandler.Functions -{ - public interface IAudioProcessUtilities - { - Stream ConvertMp3ToStream(string mp3FileName); - Stream ConvertWavToStream(string wavFileName); - Stream ConvertToStream(string fileName); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs new file mode 100644 index 000000000..4737a5e9d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -0,0 +1,85 @@ +using NAudio.Wave; +using NAudio.Wave.SampleProviders; + +namespace BotSharp.Plugin.AudioHandler.Helpers; + +public static class AudioHelper +{ + private const int DEFAULT_SAMPLE_RATE = 16000; + + public static Stream ConvertToStream(string fileName) + { + if (string.IsNullOrEmpty(fileName)) + { + throw new ArgumentNullException("fileName is Null when converting to stream in audio processor"); + } + + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); + } + + var stream = fileType switch + { + AudioType.mp3 => ConvertMp3ToStream(fileName), + _ => ConvertWavToStream(fileName) + }; + + return stream; + } + + public static Stream Transform(Stream stream, string fileName) + { + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); + } + + Stream resultStream = new MemoryStream(); + stream.CopyTo(resultStream); + resultStream.Seek(0, SeekOrigin.Begin); + + WaveStream reader = fileType switch + { + AudioType.mp3 => new Mp3FileReader(resultStream), + _ => new WaveFileReader(resultStream) + }; + + resultStream = ChangeSampleRate(reader); + reader.Close(); + return resultStream; + } + + private static Stream ConvertMp3ToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ConvertWavToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); + using var reader = new WaveFileReader(fileStream); + return ChangeSampleRate(reader); + } + + private static Stream ChangeSampleRate(WaveStream ws) + { + var ms = new MemoryStream(); + if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE) + { + var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE); + WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16()); + } + else + { + ws.CopyTo(ms); + } + + ms.Seek(0, SeekOrigin.Begin); + return ms; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index c3ff10d0b..80acb149a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,73 +1,60 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; -namespace BotSharp.Plugin.AudioHandler.Hooks +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerHook : AgentHookBase, IAgentHook { - public class AudioHandlerHook : AgentHookBase, IAgentHook + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - private const string HANDLER_AUDIO = "handle_audio_request"; + } - public override string SelfId => string.Empty; + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + if (isEnabled && isConvMode) { + AddUtility(agent, HANDLER_AUDIO); } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - - if (isEnabled && isConvMode) - { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); - } + base.OnAgentLoaded(agent); + } - base.OnAgentLoaded(agent); - } + private void AddUtility(Agent agent, string functionName) + { + var (prompt, fn) = GetPromptAndFunction(functionName); - private void AddUtility(Agent agent, string utility, string functionName) + if (fn != null) { - if (!IsEnableUtility(agent, utility)) return; - - var (prompt, fn) = GetPromptAndFunction(functionName); - if (fn != null) + if (!string.IsNullOrWhiteSpace(prompt)) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } - - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; } - } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); + } } + } - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); - } + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs index f220acdfa..ac3f0ed73 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -1,10 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Abstraction.Agents; - namespace BotSharp.Plugin.AudioHandler.Hooks; public class AudioHandlerUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs index 5132ef3a9..281305e8f 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs index 3db02f717..ba75464d8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs index 1b58f455d..f8f0bf51d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -1,19 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Whisper.net; -namespace BotSharp.Plugin.AudioHandler.Models +namespace BotSharp.Plugin.AudioHandler.Models; + +public class AudioOutput { - public class AudioOutput - { - public List Segments { get; set; } + public List Segments { get; set; } = new(); - public override string ToString() - { - return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; - } + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 7afae1fcb..e6ac230e9 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -7,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : ISpeechToText +public class NativeWhisperProvider : IAudioCompletion { - public string Provider => "native"; - private readonly IAudioProcessUtilities _audioProcessUtilities; - private static WhisperProcessor _processor; - private readonly ILogger _logger; + private static WhisperProcessor _whisperProcessor; + + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; + private readonly ILogger _logger; - private string MODEL_DIR = "model"; - private string? _currentModelPath; - private Dictionary _modelPathDict = new Dictionary(); - private GgmlType? _modelType; + public string Provider => "native"; public NativeWhisperProvider( - IAudioProcessUtilities audioProcessUtilities, + BotSharpDatabaseSettings dbSettings, + IFileStorageService fileStorage, + IServiceProvider services, ILogger logger) { - _audioProcessUtilities = audioProcessUtilities; + _fileStorage = fileStorage; + _services = services; _logger = logger; } - public async Task GenerateTextFromAudioAsync(string filePath) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) { - string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - - using var stream = _audioProcessUtilities.ConvertToStream(filePath); - - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - var textResult = new List(); - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + using var stream = AudioHelper.Transform(audio, audioFileName); + await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } - _processor.Dispose(); + _whisperProcessor.Dispose(); + stream.Close(); var audioOutput = new AudioOutput { @@ -57,71 +46,69 @@ public async Task GenerateTextFromAudioAsync(string filePath) }; return audioOutput.ToString(); } - private async Task LoadWhisperModel(GgmlType modelType) + + public async Task GenerateAudioFromTextAsync(string text) { - try - { - if (!Directory.Exists(MODEL_DIR)) - Directory.CreateDirectory(MODEL_DIR); + throw new NotImplementedException(); + } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin") - .ToArray(); + public void SetModelName(string model) + { + if (Enum.TryParse(model, true, out GgmlType ggmlType)) + { + LoadWhisperModel(ggmlType); + } + else + { + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + LoadWhisperModel(GgmlType.Tiny); + } + } - if (!availableModelPaths.Any()) + private void LoadWhisperModel(GgmlType modelType) + { + try + { + var modelDir = _fileStorage.BuildDirectory("models", "whisper"); + var exist = _fileStorage.ExistDirectory(modelDir); + if (!exist) { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); + _fileStorage.CreateDirectory(modelDir); } - else + + var files = _fileStorage.GetFiles("models/whisper", "*.bin"); + var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType)); + if (string.IsNullOrEmpty(modelLoc)) { - var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin"); - if (modelFilePath == null) - { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - _currentModelPath = modelFilePath; - } + modelLoc = BuildModelPath(modelType); + DownloadModel(modelType, modelLoc); } - _processor = WhisperFactory - .FromPath(path: _currentModelPath) - .CreateBuilder() - .WithLanguage("auto") - .Build(); - - _modelType = modelType; + var bytes = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { - throw new Exception($"Failed to load whisper model: {ex.Message}"); + var error = "Failed to load whisper model"; + _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}"); + throw new Exception($"{error}: {ex.Message}"); } } - private async Task DownloadModel(GgmlType modelType, string modelDir) + private void DownloadModel(GgmlType modelType, string modelDir) { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType); - using var fileWriter = File.OpenWrite(modelDir); - await modelStream.CopyToAsync(fileWriter); + using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); + modelStream.Close(); } - private string SetModelPath(string rootPath, GgmlType modelType) + private string BuildModelPath(GgmlType modelType) { - string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); - return currentModelPath; + return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType)); } - public async Task SetModelName(string modelType) + private string BuildModelFile(GgmlType modelType) { - if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) - { - await LoadWhisperModel(ggmlType); - return; - } - - _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); + return $"ggml-{modelType}.bin"; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index 4ace63db4..fedf123ca 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,6 +1,5 @@ -namespace BotSharp.Plugin.AudioHandler.Settings +namespace BotSharp.Plugin.AudioHandler.Settings; + +public class AudioHandlerSettings { - public class AudioHandlerSettings - { - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index c15e6cc4a..1a1188fd8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -5,6 +5,7 @@ global using System.Text.Json; global using System.Threading.Tasks; +global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -20,7 +21,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.AudioHandler.Enums; -global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Helpers; global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json index b223c7ae9..c0603b1af 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -1,18 +1,18 @@ { - "name": "handle_audio_request", - "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" - }, - "is_need_summary": { - "type": "boolean", - "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" - } - }, - "required": [ "user_request" ] - } + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index dba74d274..eba22bfa4 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.AzureOpenAI.Providers.Audio; using BotSharp.Plugin.AzureOpenAI.Providers.Chat; using BotSharp.Plugin.AzureOpenAI.Providers.Embedding; using BotSharp.Plugin.AzureOpenAI.Providers.Image; @@ -30,5 +31,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 000000000..5a436feff --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,90 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 000000000..4e5fc3fa9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 000000000..8cc75ad8e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index af82b0374..678ee82be 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public async Task GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model; protected int _dimension; diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 43272f8eb..52e1a845a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -53,7 +53,7 @@ public async Task GetImageEdits(Agent agent, RoleDialogModel me var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7d608fd14..f5d8c1f92 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public async Task GetImageGeneration(Agent agent, RoleDialogMod var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 88f393e06..b2e4c5fe1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public async Task GetImageVariation(Agent agent, RoleDialogMode { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 7d0641547..967f3e082 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ private GeneratedImageSize GetImageSize(string size) case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ private GeneratedImageQuality GetImageQuality(string quality) GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ private GeneratedImageStyle GetImageStyle(string style) GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ private GeneratedImageFormat GetImageFormat(string format) GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b4f9a6e4e..fa038811d 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -84,11 +84,12 @@ private void BuildEmailAttachments(BodyBuilder builder, IEnumerable(); + foreach (var file in files) { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs index 222c27b91..beece9e28 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs index d4ae8c16a..274910157 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 759f94b60..dc4ddef03 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -43,7 +43,7 @@ private void Init(RoleDialogModel message) private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } @@ -99,9 +99,9 @@ private void SaveGeneratedImage(ImageGeneration? image) { if (image == null) return; - var files = new List() + var files = new List() { - new BotSharpFile + new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4102ff7b2..9592ef80c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,7 +50,7 @@ private void Init(RoleDialogModel message) private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } @@ -77,7 +77,7 @@ private void SaveGeneratedImages(List? images) { if (images.IsNullOrEmpty()) return; - var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile + var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index e2b465c8e..814f943e3 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -52,7 +52,7 @@ private async Task> AssembleFiles(string conversationId, L var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); if (screenshots.IsNullOrEmpty()) return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9e0ceaee0..0465dc4e0 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public GraphDb( _settings = settings; } - public string Name => "Default"; + public string Provider => "Remote"; public async Task Search(string query, GraphSearchOptions options) { @@ -44,7 +44,7 @@ public async Task Search(string query, GraphSearchOptions optio return new GraphSearchData(); } - var url = $"{_settings.BaseUrl}/query"; + var url = $"{_settings.BaseUrl}{_settings.SearchPath}"; var request = new GraphQueryRequest { Query = query, diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs index 893513989..34e0b5fd8 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Plugin.Graph; public class GraphDbSettings { public string BaseUrl { get; set; } + public string SearchPath { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445e..29c81ec73 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs index 09401f207..0e39bef0d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs @@ -1,11 +1,11 @@ using UglyToad.PdfPig; using UglyToad.PdfPig.Content; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Plugin.KnowledgeBase.Converters; public class PigPdf2TextConverter : IPdf2TextConverter { - public string Name => "Pig"; + public string Provider => "Pig"; public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 060e21500..ffbef7708 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,9 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Extensions; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { public string Name => "knowledge_retrieval"; + public string Indication => "searching my brain"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,15 +22,16 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); + var knowledgeService = _services.GetRequiredService(); + var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions + { + Fields = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }, + Confidence = 0.2f + }); if (!knowledges.IsNullOrEmpty()) { - var answers = knowledges.Select(x => $"Question: {x.Data[KnowledgePayloadName.Text]}\r\nAnswer: {x.Data[KnowledgePayloadName.Answer]}").ToList(); + var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList(); message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f709e3b89..62388f197 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -4,6 +4,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { public string Name => "memorize_knowledge"; + public string Indication => "remembering knowledge"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,23 +20,16 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorsAsync(new List + var knowledgeService = _services.GetRequiredService(); + var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { - args.Question + Text = args.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, args.Answer } + } }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); - await vectorDb.CreateCollection(collectionName, vector[0].Length); - - var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], - args.Question, - new Dictionary - { - { KnowledgePayloadName.Answer, args.Answer } - }); - message.Content = result ? "Saved to my brain" : "I forgot it"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs index f0538af0e..88a6ebb5a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs @@ -2,9 +2,9 @@ using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class VectorUtility +public static class VectorHelper { public static float[] CalEuclideanDistance(float[] vec, List records) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 213825b5a..ffe4d8f14 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.KnowledgeBase.Converters; using BotSharp.Plugin.KnowledgeBase.Hooks; using Microsoft.Extensions.Configuration; @@ -20,8 +21,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); @@ -35,7 +34,7 @@ public bool AttachMenu(List menu) SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relations", link: "page/knowledge-base/relations") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0d4be8198..17a2c7572 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -8,12 +8,18 @@ public class MemoryVectorDb : IVectorDb private readonly Dictionary> _vectors = new Dictionary>(); - public string Name => "MemoryVector"; + public string Provider => "MemoryVector"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { - _collections[collectionName] = dim; + _collections[collectionName] = dimension; _vectors[collectionName] = new List(); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + return false; } public async Task> GetCollections() @@ -40,8 +46,7 @@ public async Task> Search(string collectionNam return new List(); } - var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); + var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() .Reverse() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 7eeea4f08..1437f41e1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -31,6 +31,5 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; -global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; -global using BotSharp.Plugin.KnowledgeBase.Utilities; \ No newline at end of file +global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs deleted file mode 100644 index 770e1efaa..000000000 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace BotSharp.Plugin.KnowledgeBase.Utilities; - -public static class KnowledgeSettingUtility -{ - public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) - { - var settings = services.GetRequiredService(); - var found = settings.Collections.FirstOrDefault(x => x.Name == collectionName)?.TextEmbedding; - if (found == null) - { - found = settings.Default.TextEmbedding; - } - - var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); - embedding.SetModelName(found.Model); - embedding.SetDimension(found.Dimension); - return embedding; - } -} diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 902087d78..58fb18512 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,9 +9,14 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { - public string Name => "Faiss"; + public string Provider => "Faiss"; - public Task CreateCollection(string collectionName, int dim) + public Task CreateCollection(string collectionName, int dimension) + { + throw new NotImplementedException(); + } + + public Task DeleteCollection(string collectionName) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 4b41ec466..e6d0637c5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; -using Microsoft.Extensions.Configuration; using BotSharp.Plugin.OpenAI.Providers.Audio; +using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.OpenAI; @@ -31,7 +31,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 000000000..9f2480865 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,89 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 000000000..09acf389f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 000000000..3311fd43f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,21 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs deleted file mode 100644 index 2314b431f..000000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Text; -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public class SpeechToTextProvider : ISpeechToText -{ - public string Provider => "openai"; - private readonly IServiceProvider _services; - private string? _modelName; - private AudioTranscriptionOptions? _options; - - public SpeechToTextProvider(IServiceProvider service) - { - _services = service; - } - - public async Task GenerateTextFromAudioAsync(string filePath) - { - var client = ProviderHelper - .GetClient(Provider, _modelName, _services) - .GetAudioClient(_modelName); - SetOptions(); - - var transcription = await client.TranscribeAudioAsync(filePath); - - return transcription.Value.Text; - } - - public async Task SetModelName(string modelName) - { - _modelName = modelName; - } - - public void SetOptions(AudioTranscriptionOptions? options = null) - { - if (_options == null) - { - _options = options ?? new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs deleted file mode 100644 index e559e1096..000000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ /dev/null @@ -1,30 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio -{ - public partial class TextToSpeechProvider : ITextToSpeech - { - public string Provider => "openai"; - private readonly IServiceProvider _services; - private string? _model; - - public TextToSpeechProvider( - IServiceProvider services) - { - _services = services; - } - - public void SetModelName(string model) - { - _model = model; - } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper - .GetClient(Provider, _model, _services) - .GetAudioClient(_model); - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs index 00f47149d..57f7ac13a 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public async Task GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List _logger; - private const int DEFAULT_DIMENSION = 3072; - protected string _model = "text-embedding-3-large"; + private const int DEFAULT_DIMENSION = 1536; + protected string _model = "text-embedding-3-small"; protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 717002fba..47df077ef 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -54,7 +54,7 @@ public async Task GetImageEdits(Agent agent, RoleDialogModel me var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 85c156864..2f55d9172 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public async Task GetImageGeneration(Agent agent, RoleDialogMod var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 8a93df83a..30c8d68fb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public async Task GetImageVariation(Agent agent, RoleDialogMode { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 185cf17c9..0e45e7892 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ private GeneratedImageSize GetImageSize(string size) case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ private GeneratedImageQuality GetImageQuality(string quality) GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ private GeneratedImageStyle GetImageStyle(string style) GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ private GeneratedImageFormat GetImageFormat(string format) GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 81c53f835..37ea35c43 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -29,7 +29,7 @@ public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) _paddleSharpSettings = paddleSharpSettings; } - public string Name => "Paddle"; + public string Provider => "Paddle"; public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 585dc1746..c3e0a7372 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -1,21 +1,13 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; + private readonly ILogger _logger; public PrimaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -25,34 +17,26 @@ public PrimaryStagePlanFn(IServiceProvider services, ILogger public async Task Execute(RoleDialogModel message) { - //debug + // Debug + var agentService = _services.GetRequiredService(); var state = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var fn = _services.GetRequiredService(); + state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - //get knowledge from vectordb - var fn = _services.GetRequiredService(); - - var msg = new ExtractedKnowledge + // Get knowledge from vectordb + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; ; + var knowledges = await knowledgeService.SearchVectorKnowledge(task.Requirements, collectionName, new VectorSearchOptions { - Question = task.Question, - }; - var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) - { - FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = task.Requirements - }), - KnowledgeConfidence = 0.1f, - Content = "" - }; - await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); - message.Content = retrievalMessage.Content; + Confidence = 0.1f + }); + message.Content = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); - var agentService = _services.GetRequiredService(); + // Send knowledge to AI to refine and summarize the primary planning var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - - //send knowledge to AI to refine and summarize the primary planning var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message); var plannerAgent = new Agent { @@ -62,7 +46,7 @@ public async Task Execute(RoleDialogModel message) TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; /*await fn.InvokeFunction("plan_secondary_stage", message); @@ -99,12 +83,14 @@ public async Task Execute(RoleDialogModel message) message.StopCompletion = true;*/ return true; } + private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.1st.plan").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.1st.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -118,13 +104,15 @@ private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest tas { "response_format", responseFormat } }); } + private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "planner_prompt.two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -139,10 +127,12 @@ private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element if(plannerAgent.Name == "planner_summary") { @@ -150,6 +140,7 @@ private async Task GetAIResponse(Agent plannerAgent) wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index d6e022996..211eea53a 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,23 +1,13 @@ -using Azure; -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using NetTopologySuite.Index.HPRtree; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; public SecondaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -28,45 +18,49 @@ public SecondaryStagePlanFn(IServiceProvider services, ILogger Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); - var msg_secondary = RoleDialogModel.From(message); - var task_primary = JsonSerializer.Deserialize(message.FunctionArgs); - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var msgSecondary = RoleDialogModel.From(message); + var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); + + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask { - TaskDescription = task_primary.Requirements + TaskDescription = taskPrimary.Requirements }); - var task_secondary = JsonSerializer.Deserialize(msg_secondary.FunctionArgs); - var items = msg_secondary.Content.JsonArrayContent(); - msg_secondary.KnowledgeConfidence = 0.5f; + var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); + var items = msgSecondary.Content.JsonArrayContent(); + foreach (var item in items) { - if (item.NeedAdditionalInformation) + if (!item.NeedAdditionalInformation) continue; + + var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions { - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = item.Task - }); - await fn.InvokeFunction("knowledge_retrieval", msg_secondary); - message.Content += msg_secondary.Content; - } + Confidence = 0.5f + }); + message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); } + // load agent var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message); + var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary, message); _logger.LogInformation(secondPlanningPrompt); var plannerAgent = new Agent { - Id = "", + Id = string.Empty, Name = "test", Instruction = secondPlanningPrompt, TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; _logger.LogInformation(response.Content); return true; @@ -74,9 +68,10 @@ public async Task Execute(RoleDialogModel message) private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var planner = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = planner.Templates.First(x => x.Name == "two_stage.2nd.plan").Content; + + var planner = await agentService.GetAgent(message.CurrentAgentId); + var template = planner.Templates.FirstOrDefault(x => x.Name == "two_stage.2nd.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new SecondStagePlan { Tool = "tool name if task solution provided", @@ -91,7 +86,7 @@ private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 4ec2a45d4..eda46f50a 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,23 +1,17 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Agents.Services; namespace BotSharp.Plugin.Planner.Functions; public class SummaryPlanFn : IFunctionCallback { public string Name => "plan_summary"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; + private readonly ILogger _logger; - public SummaryPlanFn(IServiceProvider services, ILogger logger) + public SummaryPlanFn( + IServiceProvider services, + ILogger logger) { _services = services; _logger = logger; @@ -27,55 +21,56 @@ public async Task Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); var agentService = _services.GetRequiredService(); - var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - //debug var state = _services.GetRequiredService(); + + var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); state.SetState("max_tokens", "4096"); var task = state.GetState("requirement_detail"); - //get DDL + // Get DDL var steps = message.Content.JsonArrayContent(); - //get all the related tables - List allTables = new List(); + // Get all the related tables + var allTables = new List(); foreach (var step in steps) { allTables.AddRange(step.Tables); } message.Data = allTables.Distinct().ToList(); - //get table DDL and stores in content - var msg2 = RoleDialogModel.From(message); - await fn.InvokeFunction("get_table_definition", msg2); - message.SecondaryContent = msg2.Content; + // Get table DDL and stores in content + var msgCopy = RoleDialogModel.From(message); + await fn.InvokeFunction("get_table_definition", msgCopy); + var ddlStatements = msgCopy.Content; + + // Summarize and generate query + var summaryPlanPrompt = await GetPlanSummaryPrompt(task, message.Content, ddlStatements); + _logger.LogInformation($"Summary plan prompt:\r\n{summaryPlanPrompt}"); - // summarize and generate query - var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message); - _logger.LogInformation(summaryPlanningPrompt); var plannerAgent = new Agent { Id = BuiltInAgentId.Planner, - Name = "planner_summary", - Instruction = summaryPlanningPrompt, - TemplateDict = new Dictionary(), + Name = "Planner Summary", + Instruction = summaryPlanPrompt, LlmConfig = currentAgent.LlmConfig }; - var response_summary = await GetAIResponse(plannerAgent); - message.Content = response_summary.Content; + var summary = await GetAiResponse(plannerAgent); + message.Content = summary.Content; message.StopCompletion = true; return true; } - private async Task GetPlanSummaryPrompt(string task, RoleDialogModel message) + private async Task GetPlanSummaryPrompt(string task, string knowledge, string ddlStatement) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -84,26 +79,28 @@ private async Task GetPlanSummaryPrompt(string task, RoleDialogModel mes return render.Render(template, new Dictionary { - { "table_structure", message.SecondaryContent }, ////check - { "task_description", task}, - { "relevant_knowledges", message.Content }, + { "table_structure", ddlStatement }, + { "task_description", task }, + { "relevant_knowledges", knowledge }, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - //add "test" to wholeDialogs' last element - if(plannerAgent.Name == "planner_summary") + + // Add "test" to wholeDialogs' last element + if (plannerAgent.Name == "planner_summary") { - //add "test" to wholeDialogs' last element in a new paragraph + // Add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { - //add "test" to wholeDialogs' last element in a new paragraph + // Add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nYou must analyze the table description to infer the table relations."; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index be627c752..c6f819e5e 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -1,13 +1,6 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Planning; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Core.Routing.Planning; -using Microsoft.Extensions.Logging; -using System.Threading.Tasks; namespace BotSharp.Plugin.Planner.TwoStaging; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index 20880e816..1e6737dfa 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -3,12 +3,13 @@ global using System.Text.Json.Serialization; global using System.Collections.Generic; global using System.Linq; +global using System.Threading.Tasks; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; global using BotSharp.Abstraction.Plugins; -global using BotSharp.Abstraction.Planning; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -18,5 +19,18 @@ global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Templating; + +global using BotSharp.Abstraction.Knowledges; +global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.VectorStorage.Models; +global using BotSharp.Abstraction.VectorStorage.Extensions; + global using BotSharp.Plugin.Planner.Hooks; -global using BotSharp.Plugin.Planner.Enums; \ No newline at end of file +global using BotSharp.Plugin.Planner.Enums; + +global using BotSharp.Core.Infrastructures; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 0a1f62928..fb13b0c14 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -3,7 +3,7 @@ You are a Task Planner. you will breakdown user business requirements into excut Thinking process: 1. Reference to "Task Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - - The subtask should contains all needed parameters for subsequent steps. + - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true. - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. @@ -22,9 +22,7 @@ Task Knowledge: {{ k }} {% endfor %} {%- endif %} - ===== Task description: -{{ task_description }} - +{{ task_description }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index ccec8033b..d5340e2e4 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -1,5 +1,5 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps. -* the step should contains all needed parameters. +* The step should contains all needed parameters. * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 99d917d44..be89fec05 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -19,7 +19,7 @@ public QdrantDb( _services = services; } - public string Name => "Qdrant"; + public string Provider => "Qdrant"; private QdrantClient GetClient() { @@ -35,6 +35,34 @@ private QdrantClient GetClient() return _client; } + public async Task CreateCollection(string collectionName, int dim) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (exist) return false; + + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dim, + Distance = Distance.Cosine + }); + + return true; + } + + public async Task DeleteCollection(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (!exist) return false; + + await client.DeleteCollectionAsync(collectionName); + return true; + } + public async Task> GetCollections() { // List all the collections @@ -51,10 +79,34 @@ public async Task> GetPagedCollectionDa return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(collectionName); + // Build query filter + Filter? queryFilter = null; + if (!filter.SearchPairs.IsNullOrEmpty()) + { + var conditions = filter.SearchPairs.Select(x => new Condition + { + Field = new FieldCondition + { + Key = x.Key, + Match = new Match { Text = x.Value }, + } + }); + + queryFilter = new Filter + { + Should = + { + conditions + } + }; + } + + var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, - offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, + filter: queryFilter, vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, @@ -93,27 +145,7 @@ public async Task> GetCollectionData(string co }); } - public async Task CreateCollection(string collectionName, int dim) - { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); - if (!exist) - { - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() - { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - } - - // Get collection info - var collectionInfo = await client.GetCollectionInfoAsync(collectionName); - if (collectionInfo == null) - { - throw new Exception($"Create {collectionName} failed."); - } - } + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { @@ -160,43 +192,26 @@ public async Task> Search(string collectionNam return results; } + var payloadSelector = new WithPayloadSelector { Enable = true }; + if (fields != null) + { + payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; + } + var points = await client.SearchAsync(collectionName, - vector, - limit: (ulong)limit, - scoreThreshold: confidence, - vectorsSelector: new WithVectorsSelector { Enable = withVector }); + vector, + limit: (ulong)limit, + scoreThreshold: confidence, + payloadSelector: payloadSelector, + vectorsSelector: withVector); - var pickFields = fields != null; - foreach (var point in points) + results = points.Select(x => new VectorCollectionData { - var data = new Dictionary(); - if (pickFields) - { - foreach (var field in fields) - { - if (point.Payload.ContainsKey(field)) - { - data[field] = point.Payload[field].StringValue; - } - else - { - data[field] = ""; - } - } - } - else - { - data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); - } - - results.Add(new VectorCollectionData - { - Id = point.Id.Uuid, - Data = data, - Score = point.Score, - Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null - }); - } + Id = x.Id.Uuid, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Score = x.Score, + Vector = x.Vectors?.Vector?.Data?.ToArray() + }).ToList(); return results; } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index a797b47fe..4912e1276 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -22,16 +22,23 @@ public SemanticKernelMemoryStoreProvider(IMemoryStore memoryStore) } - public string Name => "SemanticKernel"; + public string Provider => "SemanticKernel"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + await _memoryStore.DeleteCollectionAsync(collectionName); + return false; } public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { - throw new System.NotImplementedException(); + throw new NotImplementedException(); } public Task> GetCollectionData(string collectionName, IEnumerable ids, diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index 6d2b4b056..266abbecb 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -53,6 +53,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs new file mode 100644 index 000000000..f863adb79 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -0,0 +1,24 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.SqlDriver.Controllers; + +[Authorize] +[ApiController] +public class SqlDriverController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SqlDriverController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/knowledge/database/import")] + public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) + { + var dbKnowledge = _services.GetRequiredService(); + return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs deleted file mode 100644 index 5c2bfa456..000000000 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs +++ /dev/null @@ -1,109 +0,0 @@ -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; -using MySqlConnector; -using static Dapper.SqlMapper; -using BotSharp.Abstraction.Agents.Enums; - - -namespace BotSharp.Plugin.Planner.Functions; -public class AddDatabaseKnowledgeFn : IFunctionCallback -{ - public string Name => "add_database_knowledge"; - private readonly IServiceProvider _services; - private object aiAssistant; - - public AddDatabaseKnowledgeFn(IServiceProvider services) - { - _services = services; - } - public async Task Execute(RoleDialogModel message) - { - var agentService = _services.GetRequiredService(); - var sqlDriver = _services.GetRequiredService(); - var fn = _services.GetRequiredService(); - var settings = _services.GetRequiredService(); - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); - - List allTables = new List(); - var sql = $"select table_name from information_schema.tables;"; - var result = connection.Query(sql: sql,dictionary); - foreach (var item in result) - { - allTables.Add(item.TABLE_NAME); - } - message.Data = allTables.Distinct().ToList(); - - var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var note = ""; - foreach (var item in allTables) - { - message.Data = new List { item }; - - await fn.InvokeFunction("get_table_definition", message); - var PlanningPrompt = await GetPrompt(message); - var plannerAgent = new Agent - { - Id = "", - Name = "database_knowledge", - Instruction = PlanningPrompt, - TemplateDict = new Dictionary(), - LlmConfig = currentAgent.LlmConfig - }; - var response = await GetAIResponse(plannerAgent); - try - { - var knowledge = response.Content.JsonArrayContent(); - foreach (var k in knowledge) - { - try - { - message.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = k.Question, - Answer = k.Answer - }); - await fn.InvokeFunction("memorize_knowledge", message); - message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n"; - } - catch (Exception e) - { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; - } - } - } - catch (Exception e) - { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; - } - } - return true; - } - private async Task GetAIResponse(Agent plannerAgent) - { - var conv = _services.GetRequiredService(); - var wholeDialogs = conv.GetDialogHistory(); - var completion = CompletionProvider.GetChatCompletion(_services, - provider: plannerAgent.LlmConfig.Provider, - model: plannerAgent.LlmConfig.Model); - - return await completion.GetChatCompletions(plannerAgent, wholeDialogs); - } - private async Task GetPrompt(RoleDialogModel message) - { - var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); - var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content; - var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = "question", - Answer = "answer" - }); - - return render.Render(template, new Dictionary - { - { "table_structure", message.Content } - }); - } -} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 74ae8f729..14314a8f7 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,5 +1,4 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.Extensions.Logging; using MySqlConnector; using static Dapper.SqlMapper; @@ -9,45 +8,64 @@ public class GetTableDefinitionFn : IFunctionCallback { public string Name => "get_table_definition"; private readonly IServiceProvider _services; + private readonly ILogger _logger; - public GetTableDefinitionFn(IServiceProvider services) + public GetTableDefinitionFn( + IServiceProvider services, + ILogger logger) { _services = services; + _logger = logger; } public async Task Execute(RoleDialogModel message) { - // get agent service var agentService = _services.GetRequiredService(); - - // var args = JsonSerializer.Deserialize(message.FunctionArgs); var sqlDriver = _services.GetRequiredService(); - - //get table DDL from database var settings = _services.GetRequiredService(); + + // Get table DDL from database + var tables = message.Data as List; + if (tables.IsNullOrEmpty()) return false; + + var tableDdls = new List(); using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); + connection.Open(); - var table_ddl = ""; - foreach (var p in (List)message.Data) + foreach (var table in tables) { - dictionary["@" + "table_name"] = p; - var escapedTableName = MySqlHelper.EscapeString(p); - dictionary["table_name"] = escapedTableName; - // can you replace this with a parameterized query? - var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'"; - - var result = connection.QueryFirstOrDefault(sql: sql, dictionary); - if (result != null) + try + { + var escapedTableName = MySqlHelper.EscapeString(table); + + var sql = $"select * from information_schema.tables where table_name = @tableName"; + var result = connection.QueryFirstOrDefault(sql, new + { + tableName = escapedTableName + }); + + if (result == null) continue; + + sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + tableDdls.Add(result); + } + + reader.Close(); + command.Dispose(); + } + catch (Exception ex) { - sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - table_ddl += "\r\n" + result; + _logger.LogWarning($"Error when getting ddl statement of table {table}."); } - } - message.Content = table_ddl; + connection.Close(); + message.Content = string.Join("\r\n\r\n", tableDdls); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs new file mode 100644 index 000000000..48ba820cf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class RequestBase +{ + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } +} + + +public class ImportDbKnowledgeRequest : RequestBase +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs new file mode 100644 index 000000000..746798c9f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -0,0 +1,140 @@ +using static Dapper.SqlMapper; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; +using MySqlConnector; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class DbKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public DbKnowledgeService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Import(string provider, string model, string schema) + { + var sqlDriverSettings = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var tables = new HashSet(); + using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); + + var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema"; + var results = connection.Query(sql, new + { + tableSchema = schema + }); + + foreach (var item in results) + { + if (item == null) continue; + + tables.Add(item.TABLE_NAME); + } + + foreach (var table in tables) + { + try + { + _logger.LogInformation($"Start processing table {table}\r\n"); + + var ddl = GetTableStructure(table); + if (string.IsNullOrEmpty(ddl)) continue; + + var prompt = await GetPrompt(ddl); + var response = await GetAiResponse(prompt, provider, model); + var knowledges = response.Content.JsonArrayContent(); + + if (knowledges.IsNullOrEmpty()) + { + _logger.LogInformation($"No knowledge for table {table}"); + continue; + } + + foreach (var item in knowledges) + { + await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + { + Text = item.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, item.Answer } + } + }); + + _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n"); + } + } + catch (Exception ex) + { + var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}"; + _logger.LogWarning(note); + } + } + + return true; + } + + private string GetTableStructure(string table) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); + + var ddl = string.Empty; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + ddl = reader.GetString(1); + } + + reader.Close(); + command.Dispose(); + connection.Close(); + return ddl; + } + + private async Task GetPrompt(string content) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "table_structure", content } + }); + } + + private async Task GetAiResponse(string prompt, string provider, string model) + { + var agent = new Agent + { + Id = string.Empty, + Name = "Db knowledge", + Instruction = prompt, + }; + + var completion = CompletionProvider.GetChatCompletion(_services, provider, model); + return await completion.GetChatCompletions(agent, new List()); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index aaad1a0fa..c059edfb0 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -16,6 +16,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) }); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index 1886cfa07..c0c2d0766 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -12,6 +12,7 @@ public class BucketClient private readonly string _fullBucketName; private readonly string _appId; private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) { _cosXml = cosXml; @@ -141,7 +142,6 @@ public List GetDirFiles(string dir) var objects = info.contentsList; return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); - } catch (CosClientException clientEx) { @@ -153,6 +153,11 @@ public List GetDirFiles(string dir) } } + public string? GetDirFile(string dir, string key) + { + return GetDirFiles(dir).FirstOrDefault(x => x == key); + } + public List GetDirectories(string dir) { var dirs = new List(); diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs index 4d803628d..ec1a41617 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - throw new NotImplementedException(); + try + { + var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}"; + var exist = _cosClient.BucketClient.DoesObjectExist(file); + if (exist) + { + return false; + } + + return _cosClient.BucketClient.UploadBytes(file, data.ToArray()); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + public BinaryData GetSpeechFile(string conversationId, string fileName) { - throw new NotImplementedException(); + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}"; + var key = $"{dir}/{fileName}"; + var file = _cosClient.BucketClient.GetDirFile(dir, key); + + if (string.IsNullOrWhiteSpace(file)) + { + return BinaryData.Empty; + } + + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + return BinaryData.FromBytes(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e94ab6ce3..d21a4c1af 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,6 +7,24 @@ public string GetDirectory(string conversationId) return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrEmpty(relativePath)) + { + return Enumerable.Empty(); + } + + try + { + return _cosClient.BucketClient.GetDirFiles(relativePath); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public byte[] GetFileBytes(string fileStorageUrl) { try @@ -15,9 +33,9 @@ public byte[] GetFileBytes(string fileStorageUrl) } catch (Exception ex) { - _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}"); + return Array.Empty(); } - return Array.Empty(); } public bool SaveFileStreamToPath(string filePath, Stream stream) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 7e2462de4..ff8dfa17c 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) @@ -115,7 +115,7 @@ public IEnumerable GetMessagesWithFile(string conversationId, return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; @@ -254,7 +254,7 @@ private async Task> ConvertPdfToImages(string pdfLoc, string private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index de222c439..a23085396 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -16,7 +16,7 @@ public string GetUserAvatar() return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index ddb5e031c..c48812f1f 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public TencentCosService( TencentCosSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 068fdbb69..8304e3b27 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -32,7 +32,11 @@ public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, [HttpPost("twilio/voice/welcome")] public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; @@ -47,8 +51,10 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); @@ -66,6 +72,7 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) { var kvp = states.Split(':'); @@ -74,10 +81,8 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat callerMessage.States.Add(kvp[0], kvp[1]); } } - await messageQueue.EnqueueAsync(callerMessage); - response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); } else { @@ -111,12 +116,16 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); + if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; + if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); @@ -133,11 +142,11 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(seg); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(seg); + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); segIndex++; } @@ -162,21 +171,26 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio } else { - response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + response = twilio.ReturnInstructions(new List + { + $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } - } + return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] - public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); - var result = new FileContentResult(data.ToArray(), "audio/mpeg"); - result.FileDownloadName = fileName; + var fileStorage = _services.GetRequiredService(); + var data = fileStorage.GetSpeechFile(conversationId, fileName); + var result = new FileContentResult(data.ToArray(), "audio/mpeg") + { + FileDownloadName = fileName + }; return result; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 455e314ce..dfdc66c33 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -7,6 +7,7 @@ public class TwilioMessageQueue { private readonly Channel _queue; internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() { BoundedChannelOptions options = new(100) @@ -18,7 +19,11 @@ public TwilioMessageQueue() public async ValueTask EnqueueAsync(CallerMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); await _queue.Writer.WriteAsync(request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 3338c46d8..39eac1cf0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -3,7 +3,6 @@ using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -58,22 +57,26 @@ private async Task ProcessUserMessageAsync(CallerMessage message) { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }; + foreach (var kvp in message.States) { states.Add(new MessageState(kvp.Key, kvp.Value)); } conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, @@ -94,14 +97,15 @@ private async Task ProcessUserMessageAsync(CallerMessage message) await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); } }, - async functionExecuted => - { } + async functionExecuted => { } ); - var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1"); - var fileService = sp.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data); + fileStorage.SaveSpeechFile(message.ConversationId, fileName, data); + reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index ee1a31e77..cecbd3c8a 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Utilities; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -59,6 +60,7 @@ public VoiceResponse ReturnInstructions(string message) }, Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}") }; + gather.Say(message); response.Append(gather); return response; @@ -80,7 +82,8 @@ public VoiceResponse ReturnInstructions(List speechPaths, string callbac Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; - if (speechPaths != null && speechPaths.Any()) + + if (!speechPaths.IsNullOrEmpty()) { foreach (var speechPath in speechPaths) { @@ -144,6 +147,7 @@ public VoiceResponse HoldOn(int interval, string message = null) Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), ActionOnEmptyResult = true }; + if (!string.IsNullOrEmpty(message)) { gather.Say(message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index 40924fbaf..f52316352 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -25,9 +25,7 @@ public async Task> RetrieveStagedCallerMessagesAsync(string convers { var db = _redis.GetDatabase(); var key = $"{conversationId}:Caller:{seqNum}"; - return (await db.ListRangeAsync(key)) - .Select(x => (string)x) - .ToList(); + return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList(); } public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index fa971a7c5..2a6d7c221 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,9 +17,12 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); + services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); services.AddSingleton(); services.AddHostedService(); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c34fc7911..4fcc745ca 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -233,8 +233,12 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": "", - "Pdf2ImageConverter": "" + "Pdf2TextConverter": { + "Provider": "" + }, + "Pdf2ImageConverter": { + "Provider": "" + } }, "TencentCos": { @@ -250,7 +254,8 @@ }, "Graph": { - "BaseUrl": "" + "BaseUrl": "", + "SearchPath": "" }, "WeChat": { @@ -262,8 +267,12 @@ }, "KnowledgeBase": { - "VectorDb": "Qdrant", - "GraphDb": "Default", + "VectorDb": { + "Provider": "Qdrant" + }, + "GraphDb": { + "Provider": "Remote" + }, "Default": { "CollectionName": "BotSharp", "TextEmbedding": { @@ -345,6 +354,7 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.TencentCos", "BotSharp.Plugin.PythonInterpreter" ]