diff --git a/Directory.Build.props b/Directory.Build.props index 758141188..351950201 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,9 +1,9 @@ - - net8.0 - 10.0 - 0.22.0 - false - false - + + net8.0 + 10.0 + 0.22.0 + false + false + \ No newline at end of file diff --git a/README.md b/README.md index aa892ce0d..12d0dc68c 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ BotSharp uses component design, the kernel is kept to a minimum, and business fu - BotSharp.Plugin.PaddleSharp #### Tools +- BotSharp.Plugin.Dashboard - BotSharp.Plugin.RoutingSpeeder - BotSharp.Plugin.WebDriver - BotSharp.Plugin.PizzaBot diff --git a/docs/conf.py b/docs/conf.py index a69f2133f..2b1636ce6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -56,7 +56,7 @@ # General information about the project. project = 'BotSharp' -copyright = 'Since 2018, Haiping Chen' +copyright = 'Since 2018, SciSharp STACK' author = 'Haiping Chen' # The version info for the project you're documenting, acts as replacement for @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = '0.21' +version = '0.23' # The full version, including alpha/beta/rc tags. -release = '0.21.0' +release = '0.23.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs index 81aacb85d..61cb0f4dc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs @@ -18,3 +18,13 @@ public enum AgentField Sample, LlmConfig } + +public enum AgentTaskField +{ + All = 1, + Name, + Description, + Enabled, + Content, + DirectAgentId +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 68e636b83..7b48f452f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Routing.Models; +using BotSharp.Abstraction.Tasks.Models; namespace BotSharp.Abstraction.Agents.Models; @@ -36,6 +37,13 @@ public class Agent public List Templates { get; set; } = new List(); + /// + /// Agent tasks + /// + [JsonIgnore] + public List Tasks { get; set; } + = new List(); + /// /// Samples /// @@ -136,6 +144,12 @@ public Agent SetTemplates(List templates) return this; } + public Agent SetTasks(List tasks) + { + Tasks = tasks ?? new List(); + return this; + } + public Agent SetFunctions(List functions) { Functions = functions ?? new List(); diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs index 230a3f2da..0d0cb67e7 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/ConversationHookBase.cs @@ -36,6 +36,11 @@ public virtual Task OnStateChanged(string name, string preValue, string currentV return Task.CompletedTask; } + public virtual Task OnDialogRecordLoaded(RoleDialogModel dialog) + { + return Task.CompletedTask; + } + public virtual Task OnDialogsLoaded(List dialogs) { _dialogs = dialogs; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs index e25314387..4c7168a24 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationHook.cs @@ -32,6 +32,14 @@ public interface IConversationHook /// Task OnDialogsLoaded(List dialogs); + /// + /// Triggered when every dialog record is loaded + /// It can be used to populate extra data point before presenting to user. + /// + /// + /// + Task OnDialogRecordLoaded(RoleDialogModel dialog); + Task OnStateLoaded(ConversationState state); Task OnStateChanged(string name, string preValue, string currentValue); diff --git a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs index dd9e2f1af..b98f62e42 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Functions/Models/FunctionCallFromLlm.cs @@ -15,6 +15,9 @@ public class FunctionCallFromLlm : RoutingArgs [JsonIgnore(Condition = JsonIgnoreCondition.Always)] public bool ExecutingDirectly { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public bool HideDialogContext { get; set; } + /// /// Router routed to a wrong agent. /// Set this flag as True will force router to re-route current request to a new agent. diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index ce602c231..75fd60e65 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,5 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); + LlmModelSetting GetProviderModel(string provider, string id); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/AgentTaskFilter.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/AgentTaskFilter.cs new file mode 100644 index 000000000..2d8ba477b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/Filters/AgentTaskFilter.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Repositories.Filters; + +public class AgentTaskFilter +{ + public Pagination Pager { get; set; } = new Pagination(); + public string? AgentId { get; set; } + public bool? Enabled { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs index bf69bec39..77d37d308 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Repositories/IBotSharpRepository.cs @@ -1,6 +1,7 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; +using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Users.Models; namespace BotSharp.Abstraction.Repositories; @@ -34,6 +35,16 @@ public interface IBotSharpRepository string GetAgentTemplate(string agentId, string templateName); #endregion + #region Agent Task + PagedItems GetAgentTasks(AgentTaskFilter filter); + AgentTask? GetAgentTask(string agentId, string taskId); + void InsertAgentTask(AgentTask task); + void BulkInsertAgentTasks(List tasks); + void UpdateAgentTask(AgentTask task, AgentTaskField field); + bool DeleteAgentTask(string agentId, string taskId); + bool DeleteAgentTasks(); + #endregion + #region Conversation void CreateNewConversation(Conversation conversation); bool DeleteConversation(string conversationId); @@ -49,9 +60,7 @@ public interface IBotSharpRepository List GetLastConversations(); bool TruncateConversation(string conversationId, string messageId); #endregion - #region Statistics - void IncrementConversationCount(); - #endregion + #region Execution Log void AddExecutionLogs(string conversationId, List logs); List GetExecutionLogs(string conversationId); @@ -60,4 +69,8 @@ public interface IBotSharpRepository #region LLM Completion Log void SaveLlmCompletionLog(LlmCompletionLog log); #endregion + + #region Statistics + void IncrementConversationCount(); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs index a5369af93..b20da8de3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/IRoutingService.cs @@ -40,4 +40,6 @@ public interface IRoutingService /// /// Task InstructDirect(Agent agent, RoleDialogModel message); + + Task GetConversationContent(List dialogs, int maxDialogCount = 50); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs new file mode 100644 index 000000000..7f148b8c2 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Abstraction.Routing.Models; + +public class DecomposedStep +{ + public string Description { get; set; } + + [JsonPropertyName("total_remaining_steps")] + public int TotalRemainingSteps { get; set; } + + [JsonPropertyName("should_stop")] + public bool ShouldStop { get; set; } + + [JsonPropertyName("stop_reason")] + public string? StopReason { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs index 4053f81c6..0e8bf803c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Routing/Planning/IPlaner.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.Routing.Models; namespace BotSharp.Abstraction.Routing.Planning; @@ -8,7 +9,8 @@ namespace BotSharp.Abstraction.Routing.Planning; /// public interface IPlaner { - Task GetNextInstruction(Agent router, string messageId); + Task GetNextInstruction(Agent router, string messageId, List dialogs); Task AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); Task AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message); + int MaxLoopCount => 5; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/Enums/TaskExecutionStatus.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/Enums/TaskExecutionStatus.cs new file mode 100644 index 000000000..eb546682a --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/Enums/TaskExecutionStatus.cs @@ -0,0 +1,7 @@ +public class TaskExecutionStatus +{ + public const string New = "new"; + public const string Running = "running"; + public const string Success = "success"; + public const string Failed = "failed"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/IAgentTaskService.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/IAgentTaskService.cs new file mode 100644 index 000000000..34199eccd --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/IAgentTaskService.cs @@ -0,0 +1,17 @@ +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Tasks.Models; + +namespace BotSharp.Abstraction.Tasks; + +public interface IAgentTaskService +{ + Task> GetTasks(AgentTaskFilter filter); + + Task GetTask(string agentId, string taskId); + + Task CreateTask(AgentTask task); + + Task UpdateTask(AgentTask task, AgentTaskField field); + + Task DeleteTask(string agentId, string taskId); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs new file mode 100644 index 000000000..b3a6aaf29 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Tasks/Models/AgentTask.cs @@ -0,0 +1,35 @@ +namespace BotSharp.Abstraction.Tasks.Models; + +public class AgentTask : AgentTaskMetaData +{ + public string Id { get; set; } + public string Content { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public string AgentId { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.Always)] + public Agent Agent { get; set; } + + public AgentTask() + { + + } + + public AgentTask(string id, string name, string? description = null) + { + Id = id; + Name = name; + Description = description; + } +} + +public class AgentTaskMetaData +{ + public string Name { get; set; } + public string? Description { get; set; } + public bool Enabled { get; set; } + public string? DirectAgentId { get; set; } + public DateTime CreatedDateTime { get; set; } + public DateTime UpdatedDateTime { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs index 267a997c0..33b8086f5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Users/IAuthenticationHook.cs @@ -7,4 +7,5 @@ public interface IAuthenticationHook { Task Authenticate(string id, string password); void AddClaims(List claims); + void BeforeSending(Token token); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs b/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs index e2dafa7f1..d11752f64 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Utilities/Pagination.cs @@ -2,16 +2,38 @@ namespace BotSharp.Abstraction.Utilities; public class Pagination { - public int Page { get; set; } = 1; - /// - /// Use -1 for all records - /// - public int Size { get; set; } = 20; - public int Offset => (Page - 1) * Size; + private int _page; + private int _size; + + public int Page + { + get { return _page > 0 ? _page : 1; } + set { _page = value; } + } + + public int Size + { + get + { + if (_size <= 0) return 20; + if (_size > 100) return 100; + + return _size; + } + set + { + _size = value; + } + } + + public int Offset + { + get { return (Page - 1) * Size; } + } } public class PagedItems { public int Count { get; set; } - public IEnumerable Items { get; set; } + public IEnumerable Items { get; set; } = new List(); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs index 01ca0c03d..ce6512af6 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.CreateAgent.cs @@ -1,7 +1,10 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Abstraction.Users.Models; using System.IO; +using System.Text.RegularExpressions; namespace BotSharp.Core.Agents.Services; @@ -156,4 +159,71 @@ private List FetchSamplesFromFile(string fileDir) var samples = File.ReadAllLines(file); return samples?.ToList() ?? new List(); } + + private List FetchTasksFromFile(string fileDir) + { + var tasks = new List(); + var taskDir = Path.Combine(fileDir, "tasks"); + if (!Directory.Exists(taskDir)) return tasks; + + var agentId = fileDir.Split(Path.DirectorySeparatorChar).Last(); + foreach (var file in Directory.GetFiles(taskDir)) + { + var parsedTask = ParseAgentTask(file); + if (parsedTask == null) continue; + + var task = new AgentTask + { + Id = parsedTask.Id, + Name = parsedTask.Name, + Description = parsedTask.Description, + Enabled = parsedTask.Enabled, + DirectAgentId = parsedTask.DirectAgentId, + Content = parsedTask.Content, + AgentId = agentId, + CreatedDateTime = parsedTask.CreatedDateTime, + UpdatedDateTime = parsedTask.UpdatedDateTime + }; + tasks.Add(task); + } + + return tasks; + } + + private AgentTask? ParseAgentTask(string taskFile) + { + if (string.IsNullOrWhiteSpace(taskFile)) return null; + + var prefix = @"#metadata"; + var suffix = @"/metadata"; + var fileName = taskFile.Split(Path.DirectorySeparatorChar).Last(); + var id = fileName.Split('.').First(); + var data = File.ReadAllText(taskFile); + var pattern = $@"{prefix}.+{suffix}"; + var metaData = Regex.Match(data, pattern, RegexOptions.Singleline); + + if (!metaData.Success) return null; + + var task = metaData.Value.JsonContent(); + if (task == null) return null; + + task.Id = id; + pattern = $@"{suffix}.+"; + var content = Regex.Match(data, pattern, RegexOptions.Singleline).Value; + task.Content = content.Substring(suffix.Length).Trim(); + return task; + } + + private UserAgent BuildUserAgent(string agentId, string userId, bool editable = false) + { + return new UserAgent + { + Id = Guid.NewGuid().ToString(), + UserId = userId, + AgentId = agentId, + Editable = editable, + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow + }; + } } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs index 43757571a..2831a669e 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.RefreshAgents.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Tasks.Models; using Microsoft.Extensions.Caching.Memory; +using System.Collections.Generic; using System.IO; namespace BotSharp.Core.Agents.Services; @@ -9,8 +11,9 @@ public partial class AgentService { public async Task RefreshAgents() { - var isDeleted = _db.DeleteAgents(); - if (!isDeleted) return; + var isAgentDeleted = _db.DeleteAgents(); + var isTaskDeleted = _db.DeleteAgentTasks(); + if (!isAgentDeleted) return; var dbSettings = _services.GetRequiredService(); var agentDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, @@ -20,6 +23,7 @@ public async Task RefreshAgents() var user = _db.GetUserById(_user.Id); var agents = new List(); var userAgents = new List(); + var agentTasks = new List(); foreach (var dir in Directory.GetDirectories(agentDir)) { @@ -37,23 +41,18 @@ public async Task RefreshAgents() .SetFunctions(functions) .SetResponses(responses) .SetSamples(samples); - - var userAgent = new UserAgent - { - Id = Guid.NewGuid().ToString(), - UserId = user.Id, - AgentId = agent.Id, - Editable = false, - CreatedTime = DateTime.UtcNow, - UpdatedTime = DateTime.UtcNow - }; - agents.Add(agent); + + var userAgent = BuildUserAgent(agent.Id, user.Id); userAgents.Add(userAgent); + + var tasks = FetchTasksFromFile(dir); + agentTasks.AddRange(tasks); } _db.BulkInsertAgents(agents); _db.BulkInsertUserAgents(userAgents); + _db.BulkInsertAgentTasks(agentTasks); Utilities.ClearCache(); } diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index e6df922d9..75287927c 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -53,6 +53,7 @@ + @@ -75,6 +76,9 @@ PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs index 1fffa03b3..cf092b510 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStorage.cs @@ -67,6 +67,7 @@ public List GetDialogs(string conversationId) { var db = _services.GetRequiredService(); var dialogs = db.GetConversationDialogs(conversationId); + var hooks = _services.GetServices(); var results = new List(); foreach (var dialog in dialogs) @@ -79,16 +80,28 @@ public List GetDialogs(string conversationId) var function = role == AgentRole.Function ? meta.FunctionName : null; var senderId = role == AgentRole.Function ? currentAgentId : meta.SenderId; var createdAt = meta.CreateTime; - - results.Add(new RoleDialogModel(role, content) + + var record = new RoleDialogModel(role, content) { CurrentAgentId = currentAgentId, MessageId = messageId, CreatedAt = createdAt, SenderId = senderId, FunctionName = function - }); + }; + results.Add(record); + + foreach(var hook in hooks) + { + hook.OnDialogRecordLoaded(record).Wait(); + } } + + foreach (var hook in hooks) + { + hook.OnDialogsLoaded(results).Wait(); + } + return results; } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index b60f6662e..7cce46b0c 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -38,6 +38,18 @@ public List GetProviderModels(string provider) ?.Models ?? new List(); } + public LlmModelSetting GetProviderModel(string provider, string id) + { + var models = GetProviderModels(provider) + .Where(x => x.Id == id) + .ToList(); + + var random = new Random(); + var index = random.Next(0, models.Count()); + var modelSetting = models.ElementAt(index); + return modelSetting; + } + public LlmModelSetting? GetSetting(string provider, string model) { var settings = _services.GetRequiredService>(); diff --git a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs index fef3c6dca..4223aedc4 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/BotSharpDbContext.cs @@ -3,6 +3,7 @@ using BotSharp.Abstraction.Repositories; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Repositories.Models; +using BotSharp.Abstraction.Tasks.Models; using BotSharp.Abstraction.Users.Models; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -119,6 +120,42 @@ public bool DeleteAgents() } #endregion + #region Agent Task + public PagedItems GetAgentTasks(AgentTaskFilter filter) + { + throw new NotImplementedException(); + } + + public AgentTask? GetAgentTask(string agentId, string taskId) + { + throw new NotImplementedException(); + } + + public void InsertAgentTask(AgentTask task) + { + throw new NotImplementedException(); + } + + public void BulkInsertAgentTasks(List tasks) + { + throw new NotImplementedException(); + } + + public void UpdateAgentTask(AgentTask task, AgentTaskField field) + { + throw new NotImplementedException(); + } + + public bool DeleteAgentTask(string agentId, string taskId) + { + throw new NotImplementedException(); + } + + public bool DeleteAgentTasks() + { + throw new NotImplementedException(); + } + #endregion #region Conversation public void CreateNewConversation(Conversation conversation) @@ -184,13 +221,7 @@ public bool TruncateConversation(string conversationId, string messageId) throw new NotImplementedException(); } #endregion - #region Stats - public void IncrementConversationCount() - { - throw new NotImplementedException(); - } - #endregion - + #region User public User? GetUserByEmail(string email) => throw new NotImplementedException(); @@ -205,7 +236,6 @@ public void CreateUser(User user) => throw new NotImplementedException(); #endregion - #region Execution Log public void AddExecutionLogs(string conversationId, List logs) { @@ -224,4 +254,11 @@ public void SaveLlmCompletionLog(LlmCompletionLog log) throw new NotImplementedException(); } #endregion + + #region Stats + public void IncrementConversationCount() + { + throw new NotImplementedException(); + } + #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index c9aebd1e0..4bdb3edf9 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -1,9 +1,9 @@ using BotSharp.Abstraction.Agents.Models; -using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Routing.Models; -using BotSharp.Abstraction.Routing.Settings; +using BotSharp.Abstraction.Tasks.Models; +using Microsoft.Extensions.Logging; using System.IO; namespace BotSharp.Core.Repository @@ -329,8 +329,8 @@ public List GetAgentResponses(string agentId, string prefix, string inte var responses = FetchResponses(dir); return record.SetInstruction(instruction) .SetFunctions(functions) - .SetSamples(samples) .SetTemplates(templates) + .SetSamples(samples) .SetResponses(responses); } @@ -405,6 +405,7 @@ public string GetAgentTemplate(string agentId, string templateName) return string.Empty; } + public void BulkInsertAgents(List agents) { } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs new file mode 100644 index 000000000..73a8e3ed4 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.AgentTask.cs @@ -0,0 +1,234 @@ +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Tasks.Models; +using System.IO; +using System.Threading.Tasks; + +namespace BotSharp.Core.Repository; + +public partial class FileRepository +{ + #region Task + public PagedItems GetAgentTasks(AgentTaskFilter filter) + { + var tasks = new List(); + var pager = filter.Pager ?? new Pagination(); + var skipCount = 0; + var takeCount = 0; + var totalCount = 0; + var matched = true; + + var dir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir); + if (!Directory.Exists(dir)) return new PagedItems(); + + foreach (var agentDir in Directory.GetDirectories(dir)) + { + var taskDir = Path.Combine(agentDir, "tasks"); + if (!Directory.Exists(taskDir)) continue; + + var agentId = agentDir.Split(Path.DirectorySeparatorChar).Last(); + + matched = true; + if (filter?.AgentId != null) + { + matched = agentId == filter.AgentId; + } + + if (!matched) continue; + + var curTasks = new List(); + foreach (var taskFile in Directory.GetFiles(taskDir)) + { + var task = ParseAgentTask(taskFile); + if (task == null) continue; + + matched = true; + if (filter?.Enabled != null) + { + matched = matched && task.Enabled == filter.Enabled; + } + + if (!matched) continue; + + totalCount++; + if (takeCount >= pager.Size) continue; + + if (skipCount < pager.Offset) + { + skipCount++; + } + else + { + curTasks.Add(task); + takeCount++; + } + } + + if (curTasks.IsNullOrEmpty()) continue; + + var agent = ParseAgent(agentDir); + curTasks.ForEach(t => + { + t.AgentId = agentId; + t.Agent = agent; + }); + tasks.AddRange(curTasks); + } + + return new PagedItems + { + Items = tasks, + Count = totalCount + }; + } + + public AgentTask? GetAgentTask(string agentId, string taskId) + { + var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); + if (!Directory.Exists(agentDir)) return null; + + var taskDir = Path.Combine(agentDir, "tasks"); + if (!Directory.Exists(taskDir)) return null; + + var taskFile = FindTaskFileById(taskDir, taskId); + if (taskFile == null) return null; + + var task = ParseAgentTask(taskFile); + if (task == null) return null; + + var agent = ParseAgent(agentDir); + task.AgentId = agentId; + task.Agent = agent; + return task; + } + + public void InsertAgentTask(AgentTask task) + { + if (task == null || string.IsNullOrEmpty(task.AgentId)) return; + + var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, task.AgentId); + if (!Directory.Exists(agentDir)) return; + + var taskDir = Path.Combine(agentDir, "tasks"); + if (!Directory.Exists(taskDir)) + { + Directory.CreateDirectory(taskDir); + } + + var fileName = $"{Guid.NewGuid()}.liquid"; + var taskFile = Path.Combine(taskDir, fileName); + var metaData = new AgentTaskMetaData + { + Name = task.Name, + Description = task.Description, + Enabled = task.Enabled, + DirectAgentId = task.DirectAgentId, + CreatedDateTime = DateTime.UtcNow, + UpdatedDateTime = DateTime.UtcNow + }; + + var fileContent = BuildAgentTaskFileContent(metaData, task.Content); + File.WriteAllText(taskFile, fileContent); + } + + public void BulkInsertAgentTasks(List tasks) + { + + } + + public void UpdateAgentTask(AgentTask task, AgentTaskField field) + { + if (task == null || string.IsNullOrEmpty(task.Id)) return; + + var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, task.AgentId); + if (!Directory.Exists(agentDir)) return; + + var taskDir = Path.Combine(agentDir, "tasks"); + if (!Directory.Exists(taskDir)) return; + + var taskFile = FindTaskFileById(taskDir, task.Id); + if (string.IsNullOrEmpty(taskFile)) return; + + var parsedTask = ParseAgentTask(taskFile); + if (parsedTask == null) return; + + var metaData = new AgentTaskMetaData + { + Name = parsedTask.Name, + Description = parsedTask.Description, + Enabled = parsedTask.Enabled, + DirectAgentId = parsedTask.DirectAgentId, + CreatedDateTime = parsedTask.CreatedDateTime, + UpdatedDateTime = DateTime.UtcNow + }; + var content = parsedTask.Content; + + switch (field) + { + case AgentTaskField.Name: + metaData.Name = task.Name; + break; + case AgentTaskField.Description: + metaData.Description = task.Description; + break; + case AgentTaskField.Enabled: + metaData.Enabled = task.Enabled; + break; + case AgentTaskField.DirectAgentId: + metaData.DirectAgentId = task.DirectAgentId; + break; + case AgentTaskField.Content: + content = task.Content; + break; + case AgentTaskField.All: + metaData.Name = task.Name; + metaData.Description = task.Description; + metaData.Enabled = task.Enabled; + metaData.DirectAgentId = task.DirectAgentId; + content = task.Content; + break; + } + + var fileContent = BuildAgentTaskFileContent(metaData, content); + File.WriteAllText(taskFile, fileContent); + } + + public bool DeleteAgentTask(string agentId, string taskId) + { + var agentDir = Path.Combine(_dbSettings.FileRepository, _agentSettings.DataDir, agentId); + if (!Directory.Exists(agentDir)) return false; + + var taskDir = Path.Combine(agentDir, "tasks"); + if (!Directory.Exists(taskDir)) return false; + + var taskFile = FindTaskFileById(taskDir, taskId); + if (string.IsNullOrWhiteSpace(taskFile)) return false; + + File.Delete(taskFile); + return true; + } + + public bool DeleteAgentTasks() + { + return false; + } + + private string? FindTaskFileById(string taskDir, string taskId) + { + if (!Directory.Exists(taskDir) || string.IsNullOrEmpty(taskId)) return null; + + var taskFile = Directory.GetFiles(taskDir).FirstOrDefault(file => + { + var fileName = file.Split(Path.DirectorySeparatorChar).Last(); + var id = fileName.Split('.').First(); + return id.IsEqualTo(taskId); + }); + + return taskFile; + } + + private string BuildAgentTaskFileContent(AgentTaskMetaData metaData, string taskContent) + { + return $"{AGENT_TASK_PREFIX}\n{JsonSerializer.Serialize(metaData, _options)}\n{AGENT_TASK_SUFFIX}\n\n{taskContent}"; + } + #endregion +} diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs index 8131bdd2f..36eaa3edf 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Conversation.cs @@ -35,9 +35,9 @@ public void CreateNewConversation(Conversation conversation) { Key = x.Key, Values = new List - { - new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } - } + { + new StateValue { Data = x.Value, UpdateTime = DateTime.UtcNow } + } }).ToList(); File.WriteAllText(stateFile, JsonSerializer.Serialize(initialStates, _options)); } diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs index b5d4683b6..13e19e54c 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.cs @@ -7,6 +7,8 @@ using System.Text.Encodings.Web; using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Statistics.Settings; +using BotSharp.Abstraction.Tasks.Models; +using System.Text.RegularExpressions; namespace BotSharp.Core.Repository; @@ -31,6 +33,8 @@ public partial class FileRepository : IBotSharpRepository private const string STATE_FILE = "state.json"; private const string EXECUTION_LOG_FILE = "execution.log"; private const string PLUGIN_CONFIG_FILE = "config.json"; + private const string AGENT_TASK_PREFIX = "#metadata"; + private const string AGENT_TASK_SUFFIX = "/metadata"; public FileRepository( IServiceProvider services, @@ -114,8 +118,8 @@ private IQueryable Agents if (agent != null) { agent = agent.SetInstruction(FetchInstruction(d)) - .SetTemplates(FetchTemplates(d)) .SetFunctions(FetchFunctions(d)) + .SetTemplates(FetchTemplates(d)) .SetResponses(FetchResponses(d)) .SetSamples(FetchSamples(d)); _agents.Add(agent); @@ -225,6 +229,23 @@ private List FetchTemplates(string fileDir) return templates; } + private List FetchTasks(string fileDir) + { + var tasks = new List(); + var taskDir = Path.Combine(fileDir, "tasks"); + if (!Directory.Exists(taskDir)) return tasks; + + foreach (var file in Directory.GetFiles(taskDir)) + { + var task = ParseAgentTask(file); + if (task == null) continue; + + tasks.Add(task); + } + + return tasks; + } + private List FetchResponses(string fileDir) { var responses = new List(); @@ -244,6 +265,51 @@ private List FetchResponses(string fileDir) return responses; } + private Agent? ParseAgent(string agentDir) + { + if (string.IsNullOrEmpty(agentDir)) return null; + + var agentJson = File.ReadAllText(Path.Combine(agentDir, AGENT_FILE)); + if (string.IsNullOrEmpty(agentJson)) return null; + + var agent = JsonSerializer.Deserialize(agentJson, _options); + if (agent == null) return null; + + var instruction = FetchInstruction(agentDir); + var functions = FetchFunctions(agentDir); + var samples = FetchSamples(agentDir); + var templates = FetchTemplates(agentDir); + var responses = FetchResponses(agentDir); + + return agent.SetInstruction(instruction) + .SetFunctions(functions) + .SetTemplates(templates) + .SetSamples(samples) + .SetResponses(responses); + } + + private AgentTask? ParseAgentTask(string taskFile) + { + if (string.IsNullOrWhiteSpace(taskFile)) return null; + + var fileName = taskFile.Split(Path.DirectorySeparatorChar).Last(); + var id = fileName.Split('.').First(); + var data = File.ReadAllText(taskFile); + var pattern = $@"{AGENT_TASK_PREFIX}.+{AGENT_TASK_SUFFIX}"; + var metaData = Regex.Match(data, pattern, RegexOptions.Singleline); + + if (!metaData.Success) return null; + + var task = metaData.Value.JsonContent(); + if (task == null) return null; + + task.Id = id; + pattern = $@"{AGENT_TASK_SUFFIX}.+"; + var content = Regex.Match(data, pattern, RegexOptions.Singleline).Value; + task.Content = content.Substring(AGENT_TASK_SUFFIX.Length).Trim(); + return task; + } + private string[] ParseFileNameByPath(string path, string separator = ".") { var name = path.Split(Path.DirectorySeparatorChar).Last(); diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs index 9cff003a5..2a1b5e9b9 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/HFPlanner.cs @@ -22,7 +22,7 @@ public HFPlanner(IServiceProvider services, ILogger logger) _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -38,11 +38,11 @@ public async Task GetNextInstruction(Agent router, string m { try { - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { - FunctionName = nameof(NaivePlanner), + FunctionName = nameof(HFPlanner), MessageId = messageId } }; diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs index fa4b20a0a..fc445a067 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/NaivePlanner.cs @@ -18,7 +18,7 @@ public NaivePlanner(IServiceProvider services, ILogger logger) _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { var next = GetNextStepPrompt(router); @@ -44,7 +44,7 @@ public async Task GetNextInstruction(Agent router, string m { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { diff --git a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs index 1b24152a5..7ecd27b63 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs @@ -1,9 +1,11 @@ using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Functions.Models; +using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Abstraction.Templating; +using System.Drawing; namespace BotSharp.Core.Routing.Planning; @@ -12,14 +14,40 @@ public class SequentialPlanner : IPlaner private readonly IServiceProvider _services; private readonly ILogger _logger; + public bool HideDialogContext => true; + public int MaxLoopCount => 100; + private FunctionCallFromLlm _lastInst; + public SequentialPlanner(IServiceProvider services, ILogger logger) { _services = services; _logger = logger; } - public async Task GetNextInstruction(Agent router, string messageId) + public async Task GetNextInstruction(Agent router, string messageId, List dialogs) { + var decomposation = await GetDecomposedStepAsync(router, messageId, dialogs); + if (decomposation.TotalRemainingSteps > 0 && _lastInst != null) + { + _lastInst.Response = decomposation.Description; + _lastInst.Reason = $"Having {decomposation.TotalRemainingSteps} steps left."; + return _lastInst; + } + else if (decomposation.TotalRemainingSteps == 0 || decomposation.ShouldStop) + { + if (!string.IsNullOrEmpty(decomposation.StopReason)) + { + // Tell router all steps are done + dialogs.Add(new RoleDialogModel(AgentRole.Assistant, decomposation.StopReason) + { + CurrentAgentId = router.Id, + MessageId = messageId + }); + router.TemplateDict["conversation"] = router.TemplateDict["conversation"].ToString().TrimEnd() + + $"\r\n{router.Name}: {decomposation.StopReason}"; + } + } + var next = GetNextStepPrompt(router); var inst = new FunctionCallFromLlm(); @@ -44,11 +72,11 @@ public async Task GetNextInstruction(Agent router, string m { // text completion // text = await completion.GetCompletion(content, router.Id, messageId); - var dialogs = new List + dialogs = new List { new RoleDialogModel(AgentRole.User, next) { - FunctionName = nameof(NaivePlanner), + FunctionName = nameof(SequentialPlanner), MessageId = messageId } }; @@ -70,6 +98,14 @@ public async Task GetNextInstruction(Agent router, string m } } + if (decomposation.TotalRemainingSteps > 0) + { + inst.Response = decomposation.Description; + inst.Reason = $"{decomposation.TotalRemainingSteps} steps left."; + inst.HideDialogContext = true; + } + + _lastInst = inst; return inst; } @@ -110,4 +146,63 @@ private string GetNextStepPrompt(Agent router) { }); } + + public async Task GetDecomposedStepAsync(Agent router, string messageId, List dialogs) + { + var systemPrompt = GetDecomposeTaskPrompt(router); + + var inst = new DecomposedStep(); + + var llmProviderService = _services.GetRequiredService(); + var model = llmProviderService.GetProviderModel("azure-openai", "gpt-4"); + + // chat completion + var completion = CompletionProvider.GetChatCompletion(_services, + provider: "azure-openai", + model: model.Name); + + int retryCount = 0; + while (retryCount < 2) + { + string text = string.Empty; + try + { + var response = await completion.GetChatCompletions(new Agent + { + Id = router.Id, + Name = nameof(SequentialPlanner), + Instruction = systemPrompt + }, dialogs); + + text = response.Content; + inst = response.Content.JsonContent(); + break; + } + catch (Exception ex) + { + _logger.LogError($"{ex.Message}: {text}"); + } + finally + { + retryCount++; + } + } + + return inst; + } + + private string GetDecomposeTaskPrompt(Agent router) + { + var template = router.Templates.First(x => x.Name == "planner_prompt.sequential.get_remaining_task").Content; + + var render = _services.GetRequiredService(); + return render.Render(template, new Dictionary + { + }); + } + + public Task GetNextInstruction(Agent router, string messageId) + { + throw new NotImplementedException(); + } } diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs index e144a688f..9ea133fa8 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.InvokeAgent.cs @@ -29,6 +29,10 @@ public async Task InvokeAgent(string agentId, List dialog { message = RoleDialogModel.From(message, role: AgentRole.Function); + if (response.FunctionName != null && response.FunctionName.Contains("/")) + { + response.FunctionName = response.FunctionName.Split("/").Last(); + } message.FunctionName = response.FunctionName; message.FunctionArgs = response.FunctionArgs; message.CurrentAgentId = agent.Id; diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs index 8a34d6843..b71a0a296 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs @@ -39,7 +39,15 @@ public async Task InstructDirect(Agent agent, RoleDialogModel m var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent"); var conv = _services.GetRequiredService(); - var dialogs = conv.GetDialogHistory(); + var dialogs = new List(); + if (conv.States.GetState("hide_context", "false") == "true") + { + dialogs.Add(message); + } + else + { + dialogs = conv.GetDialogHistory(); + } handler.SetDialogs(dialogs); var inst = new FunctionCallFromLlm @@ -80,7 +88,7 @@ public async Task InstructLoop(RoleDialogModel message) context.Push(_router.Id); int loopCount = 0; - while (loopCount < 5 && !context.IsEmpty) + while (loopCount < planner.MaxLoopCount && !context.IsEmpty) { loopCount++; @@ -88,7 +96,7 @@ public async Task InstructLoop(RoleDialogModel message) _router.TemplateDict["conversation"] = conversation; // Get instruction from Planner - var inst = await planner.GetNextInstruction(_router, message.MessageId); + var inst = await planner.GetNextInstruction(_router, message.MessageId, dialogs); // Save states states.SaveStateByArgs(inst.Arguments); @@ -100,8 +108,20 @@ public async Task InstructLoop(RoleDialogModel message) #endif await planner.AgentExecuting(_router, inst, message); - // Handle instruction by Executor - response = await executor.Execute(this, inst, message, dialogs); + // Handover to Task Agent + if (inst.HideDialogContext) + { + var dialogWithoutContext = new List + { + new RoleDialogModel(AgentRole.User, inst.Response) + }; + response = await executor.Execute(this, inst, message, dialogWithoutContext); + dialogs.AddRange(dialogWithoutContext.Skip(1)); + } + else + { + response = await executor.Execute(this, inst, message, dialogs); + } await planner.AgentExecuted(_router, inst, response); } diff --git a/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs new file mode 100644 index 000000000..1e7cdfdf9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Tasks/Services/AgentTaskService.cs @@ -0,0 +1,78 @@ +using BotSharp.Abstraction.Repositories; +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Tasks; +using BotSharp.Abstraction.Tasks.Models; + +namespace BotSharp.Core.Tasks.Services; + +public class AgentTaskService : IAgentTaskService +{ + private readonly IServiceProvider _services; + public AgentTaskService(IServiceProvider services) + { + _services = services; + } + + /// + /// Get agent tasks using pagination + /// + /// + /// + public async Task> GetTasks(AgentTaskFilter filter) + { + var db = _services.GetRequiredService(); + var pagedTasks = db.GetAgentTasks(filter); + return await Task.FromResult(pagedTasks); + } + + /// + /// Get an agent task + /// + /// + /// + /// + public async Task GetTask(string agentId, string taskId) + { + var db = _services.GetRequiredService(); + var task = db.GetAgentTask(agentId, taskId); + return await Task.FromResult(task); + } + + /// + /// Create a new agent task + /// + /// + /// + public async Task CreateTask(AgentTask task) + { + var db = _services.GetRequiredService(); + db.InsertAgentTask(task); + await Task.CompletedTask; + } + + /// + /// Update an agent task by a single field or all fields + /// + /// + /// + /// + public async Task UpdateTask(AgentTask task, AgentTaskField field) + { + var db = _services.GetRequiredService(); + db.UpdateAgentTask(task, field); + await Task.CompletedTask; + } + + /// + /// Delete an agent task + /// + /// + /// + /// + public async Task DeleteTask(string agentId, string taskId) + { + var db = _services.GetRequiredService(); + var isDeleted = db.DeleteAgentTask(agentId, taskId); + return await Task.FromResult(isDeleted); + } +} diff --git a/src/Infrastructure/BotSharp.Core/Tasks/TaskPlugin.cs b/src/Infrastructure/BotSharp.Core/Tasks/TaskPlugin.cs new file mode 100644 index 000000000..ddf709ec5 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Tasks/TaskPlugin.cs @@ -0,0 +1,26 @@ +using BotSharp.Abstraction.Plugins.Models; +using BotSharp.Abstraction.Tasks; +using BotSharp.Core.Tasks.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Tasks; + +public class TaskPlugin : IBotSharpPlugin +{ + public string Id => "e1fb196a-8be9-4c3b-91ba-adfab5a359ef"; + public string Name => "Agent Task"; + public string Description => "Define some specific task templates and execute them. It can been used for the fixed business scenarios."; + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } + + public bool AttachMenu(List menu) + { + var section = menu.First(x => x.Label == "Apps"); + menu.Add(new PluginMenuDef("Task", link: "page/task", icon: "bx bx-task", weight: section.Weight + 8)); + + return true; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs index d0060671a..ce9a9d602 100644 --- a/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs +++ b/src/Infrastructure/BotSharp.Core/Users/Services/UserService.cs @@ -71,13 +71,13 @@ public async Task GetToken(string authorization) record = db.GetUserByUserName(id); } + var hooks = _services.GetServices(); if (record == null || record.Source != "internal") { // check 3rd party user - var validators = _services.GetServices(); - foreach (var validator in validators) + foreach (var hook in hooks) { - var user = await validator.Authenticate(id, password); + var user = await hook.Authenticate(id, password); if (user == null) { continue; @@ -120,13 +120,20 @@ record = db.GetUserByUserName(id); var accessToken = GenerateJwtToken(record); var jwt = new JwtSecurityTokenHandler().ReadJwtToken(accessToken); - return new Token + var token = new Token { AccessToken = accessToken, ExpireTime = jwt.Payload.Exp.Value, TokenType = "Bearer", Scope = "api" }; + + foreach (var hook in hooks) + { + hook.BeforeSending(token); + } + + return token; } private string GenerateJwtToken(User user) @@ -139,7 +146,7 @@ private string GenerateJwtToken(User user) new Claim(JwtRegisteredClaimNames.GivenName, user.FirstName), new Claim(JwtRegisteredClaimNames.FamilyName, user.LastName), new Claim("source", user.Source), - new Claim("external_id", user.ExternalId), + new Claim("external_id", user.ExternalId??string.Empty), new Claim(JwtRegisteredClaimNames.Jti, Guid.NewGuid().ToString()) }; @@ -180,16 +187,6 @@ public async Task GetUser(string id) { var db = _services.GetRequiredService(); var user = db.GetUserById(id); - if (user == null) - { - user = new User - { - Id = id, - FirstName = "Unknown", - LastName = "Anonymous", - Role = AgentRole.User - }; - } return user; } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid new file mode 100644 index 000000000..0c93d4e8d --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid @@ -0,0 +1,12 @@ +User will give you a task list with steps, which is going to be executed. +If a specific step has been exectued, you will get something indicates the result. +If there is no any result provided, it means all the steps have not been executed yet. +You need to figure out which steps have not been completed. +Tell me what is the first remaining step from user steps that have not been completed based on the context. +Output in JSON +{ + "description": "step detail with arguments", + "total_remaining_steps": 0, + "should_stop": false, + "stop_reason": "the reason why it should stop" +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid index 77faafc2b..d0d3a4add 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid @@ -1,2 +1,2 @@ -In order to sequentially execute user needs, +In order to sequentially execute user tasks, What is the next step based on the CONVERSATION? \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/plugins/config.json b/src/Infrastructure/BotSharp.Core/data/plugins/config.json index 900c72253..31643fdb0 100644 --- a/src/Infrastructure/BotSharp.Core/data/plugins/config.json +++ b/src/Infrastructure/BotSharp.Core/data/plugins/config.json @@ -1,6 +1,7 @@ { "Comment": "User can change this configuration dynamically and reflect updates to UI.", - "EnabledPlugins": [ - "6e52d42d-1e23-406b-8599-36af36c83209" - ] + "EnabledPlugins": [ + "6e52d42d-1e23-406b-8599-36af36c83209", + "d42a0c21-b461-44f6-ada2-499510d260af" + ] } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index e1bf2e4ca..2f67f7c71 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) enable enable $(LangVersion) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentTaskController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentTaskController.cs new file mode 100644 index 000000000..3d585378a --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentTaskController.cs @@ -0,0 +1,107 @@ +using BotSharp.Abstraction.Tasks; + +namespace BotSharp.OpenAPI.Controllers; + +[Authorize] +[ApiController] +public class AgentTaskController : ControllerBase +{ + private readonly IAgentTaskService _agentTaskService; + private readonly IServiceProvider _services; + + public AgentTaskController(IAgentTaskService agentTaskService, IServiceProvider services) + { + _agentTaskService = agentTaskService; + _services = services; + } + + /// + /// Get an agent task + /// + /// + /// + /// + [HttpGet("/agent/{agentId}/task/{taskId}")] + public async Task GetAgentTask([FromRoute] string agentId, [FromRoute] string taskId) + { + var task = await _agentTaskService.GetTask(agentId, taskId); + if (task == null) return null; + + return AgentTaskViewModel.From(task); + } + + /// + /// Get agent tasks by pagination + /// + /// + /// + [HttpGet("/agent/tasks")] + public async Task> GetAgentTasks([FromQuery] AgentTaskFilter filter) + { + var tasks = await _agentTaskService.GetTasks(filter); + return new PagedItems + { + Items = tasks.Items.Select(x => AgentTaskViewModel.From(x)), + Count = tasks.Count + }; + } + + /// + /// Create a new agent task + /// + /// + /// + /// + [HttpPost("/agent/{agentId}/task")] + public async Task CreateAgentTask([FromRoute] string agentId, [FromBody] AgentTaskCreateModel task) + { + var agentTask = task.ToAgentTask(); + agentTask.AgentId = agentId; + await _agentTaskService.CreateTask(agentTask); + } + + /// + /// Update an agent task + /// + /// + /// + /// + /// + [HttpPut("/agent/{agentId}/task/{taskId}")] + public async Task UpdateAgentTask([FromRoute] string agentId, [FromRoute] string taskId, [FromBody] AgentTaskUpdateModel task) + { + var agentTask = task.ToAgentTask(); + agentTask.AgentId = agentId; + agentTask.Id = taskId; + await _agentTaskService.UpdateTask(agentTask, AgentTaskField.All); + } + + /// + /// Update an agent task by a single field + /// + /// + /// + /// + /// + /// + [HttpPatch("/agent/{agentId}/task/{taskId}/{field}")] + public async Task PatchAgentTaskByField([FromRoute] string agentId, [FromRoute] string taskId, [FromRoute] AgentTaskField field, [FromBody] AgentTaskUpdateModel task) + { + var agentTask = task.ToAgentTask(); + agentTask.AgentId = agentId; + agentTask.Id = taskId; + await _agentTaskService.UpdateTask(agentTask, field); + } + + /// + /// Delete an agent task + /// + /// + /// + /// + [HttpDelete("/agent/{agentId}/task/{taskId}")] + public async Task DeleteAgentTask([FromRoute] string agentId, [FromRoute] string taskId) + { + return await _agentTaskService.DeleteTask(agentId, taskId); + } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index ec1d75e66..474111675 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -83,6 +83,7 @@ public async Task> GetDialogs([FromRoute] string MessageId = message.MessageId, CreatedAt = message.CreatedAt, Text = message.Content, + Data = message.Data, Sender = UserViewModel.FromUser(user) }); } @@ -95,6 +96,7 @@ public async Task> GetDialogs([FromRoute] string MessageId = message.MessageId, CreatedAt = message.CreatedAt, Text = message.Content, + Data = message.Data, Sender = new UserViewModel { FirstName = agent.Name, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs new file mode 100644 index 000000000..03410f187 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskCreateModel.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.Tasks.Models; + +namespace BotSharp.OpenAPI.ViewModels.Agents; + +public class AgentTaskCreateModel +{ + public string Name { get; set; } + public string? Description { get; set; } + public string Content { get; set; } + public bool Enabled { get; set; } + public string? DirectAgentId { get; set; } + + public AgentTask ToAgentTask() + { + return new AgentTask + { + Name = Name, + Description = Description, + Content = Content, + Enabled = Enabled, + DirectAgentId = DirectAgentId + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs new file mode 100644 index 000000000..5dcea21cf --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskUpdateModel.cs @@ -0,0 +1,24 @@ +using BotSharp.Abstraction.Tasks.Models; + +namespace BotSharp.OpenAPI.ViewModels.Agents; + +public class AgentTaskUpdateModel +{ + public string? Name { get; set; } + public string? Description { get; set; } + public string? Content { get; set; } + public bool Enabled { get; set; } + public string? DirectAgentId { get; set; } + + public AgentTask ToAgentTask() + { + return new AgentTask + { + Name = Name, + Description = Description, + Content = Content, + Enabled = Enabled, + DirectAgentId = DirectAgentId + }; + } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs new file mode 100644 index 000000000..1e77f2509 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentTaskViewModel.cs @@ -0,0 +1,40 @@ +using BotSharp.Abstraction.Tasks.Models; +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Agents; + +public class AgentTaskViewModel +{ + public string Id { get; set; } + public string Name { get; set; } + public string? Description { get; set; } + public string Content { get; set; } + public bool Enabled { get; set; } + [JsonPropertyName("created_datetime")] + public DateTime CreatedDateTime { get; set; } + [JsonPropertyName("updated_datetime")] + public DateTime UpdatedDateTime { get; set; } + [JsonPropertyName("agent_id")] + public string AgentId { get; set; } + [JsonPropertyName("agent_name")] + public string AgentName { get; set; } + [JsonPropertyName("direct_agent_id")] + public string? DirectAgentId { get; set; } + + public static AgentTaskViewModel From(AgentTask task) + { + return new AgentTaskViewModel + { + Id = task.Id, + Name = task.Name, + Description = task.Description, + Content = task.Content, + Enabled = task.Enabled, + AgentId = task.AgentId, + AgentName = task.Agent?.Name, + DirectAgentId = task?.DirectAgentId, + CreatedDateTime = task.CreatedDateTime, + UpdatedDateTime = task.UpdatedDateTime + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj index 7d5709f61..d6a41a19e 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/BotSharp.Plugin.AzureOpenAI.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj index 52358e1d9..4f076a9e3 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj +++ b/src/Plugins/BotSharp.Plugin.ChatHub/BotSharp.Plugin.ChatHub.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) enable enable $(LangVersion) diff --git a/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj b/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj index 4d5294201..63adc0b8c 100644 --- a/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj +++ b/src/Plugins/BotSharp.Plugin.ChatbotUI/BotSharp.Plugin.ChatbotUI.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) enable $(LangVersion) $(BotSharpVersion) diff --git a/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs b/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs index e136a9614..72375cf4b 100644 --- a/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Dashboard/DashboardPlugin.cs @@ -25,8 +25,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) public bool AttachMenu(List menu) { - var section = menu.First(x => x.Label == "Apps"); - menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: section.Weight - 1)); + menu.Add(new PluginMenuDef("Dashboard", link: "page/dashboard", icon: "bx bx-home-circle", weight: 1)); return true; } } diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj index 49f445dba..15ceb9d3b 100644 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/BotSharp.Plugin.HttpHandler.csproj @@ -10,6 +10,24 @@ $(SolutionDir)packages + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json b/src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json deleted file mode 100644 index fc879e14c..000000000 --- a/src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "HTTP Handler", - "description": "Use Web API to interact with other systems to obtain or update data.", - "createdDateTime": "2024-01-06T00:00:00Z", - "updatedDateTime": "2024-01-06T00:00:00Z", - "id": "87c458fc-ec5f-40ae-8ed6-05dda8a07523", - "allowRouting": true, - "isPublic": true -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json new file mode 100644 index 000000000..a2b969277 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/agent.json @@ -0,0 +1,14 @@ +{ + "id": "87c458fc-ec5f-40ae-8ed6-05dda8a07523", + "name": "HTTP Handler", + "description": "Use Web Open API to interact with 3rd system.", + "type": "task", + "createdDateTime": "2024-01-06T00:00:00Z", + "updatedDateTime": "2024-01-06T00:00:00Z", + "disabled": false, + "isPublic": true, + "profiles": [ "tool", "webapi" ], + "llmConfig": { + "max_recursion_depth": 1 + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/functions.json b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/functions.json similarity index 100% rename from src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/functions.json rename to src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/functions.json diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid b/src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid similarity index 100% rename from src/Plugins/BotSharp.Plugin.HttpHandler/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid rename to src/Plugins/BotSharp.Plugin.HttpHandler/data/agents/87c458fc-ec5f-40ae-8ed6-05dda8a07523/instruction.liquid diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs new file mode 100644 index 000000000..6acc4f5bf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentTaskDocument.cs @@ -0,0 +1,13 @@ +namespace BotSharp.Plugin.MongoStorage.Collections; + +public class AgentTaskDocument : MongoBase +{ + public string Name { get; set; } + public string? Description { get; set; } + public string Content { get; set; } + public bool Enabled { get; set; } + public string AgentId { get; set; } + public string? DirectAgentId { get; set; } + public DateTime CreatedTime { get; set; } + public DateTime UpdatedTime { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs index a3e994b49..0067f885c 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/MongoDbContext.cs @@ -31,6 +31,9 @@ private string GetDatabaseName(string mongoDbConnectionString) public IMongoCollection Agents => Database.GetCollection($"{_collectionPrefix}_Agents"); + public IMongoCollection AgentTasks + => Database.GetCollection($"{_collectionPrefix}_AgentTasks"); + public IMongoCollection Conversations => Database.GetCollection($"{_collectionPrefix}_Conversations"); diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 58fe44ff4..b974f6811 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -1,9 +1,11 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Evaluations.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories.Filters; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Routing.Settings; +using BotSharp.Abstraction.Tasks.Models; using BotSharp.Plugin.MongoStorage.Collections; using BotSharp.Plugin.MongoStorage.Models; @@ -261,33 +263,7 @@ private void UpdateAgentAllFields(Agent agent) var agent = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == agentId); if (agent == null) return null; - return new Agent - { - Id = agent.Id, - Name = agent.Name, - IconUrl = agent.IconUrl, - Description = agent.Description, - Instruction = agent.Instruction, - Templates = !agent.Templates.IsNullOrEmpty() ? agent.Templates - .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) - .ToList() : new List(), - Functions = !agent.Functions.IsNullOrEmpty() ? agent.Functions - .Select(f => FunctionDefMongoElement.ToDomainElement(f)) - .ToList() : new List(), - Responses = !agent.Responses.IsNullOrEmpty() ? agent.Responses - .Select(r => AgentResponseMongoElement.ToDomainElement(r)) - .ToList() : new List(), - Samples = agent.Samples ?? new List(), - IsPublic = agent.IsPublic, - Disabled = agent.Disabled, - Type = agent.Type, - InheritAgentId = agent.InheritAgentId, - Profiles = agent.Profiles, - RoutingRules = !agent.RoutingRules.IsNullOrEmpty() ? agent.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(agent.Id, agent.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agent.LlmConfig) - }; + return TransformAgentDocument(agent); } public List GetAgents(AgentFilter filter) @@ -323,33 +299,7 @@ public List GetAgents(AgentFilter filter) var agentDocs = _dc.Agents.Find(builder.And(filters)).ToList(); - return agentDocs.Select(x => new Agent - { - Id = x.Id, - Name = x.Name, - IconUrl = x.IconUrl, - Description = x.Description, - Instruction = x.Instruction, - Templates = !x.Templates.IsNullOrEmpty() ? x.Templates - .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) - .ToList() : new List(), - Functions = !x.Functions.IsNullOrEmpty() ? x.Functions - .Select(f => FunctionDefMongoElement.ToDomainElement(f)) - .ToList() : new List(), - Responses = !x.Responses.IsNullOrEmpty() ? x.Responses - .Select(r => AgentResponseMongoElement.ToDomainElement(r)) - .ToList() : new List(), - Samples = x.Samples ?? new List(), - IsPublic = x.IsPublic, - Disabled = x.Disabled, - Type = x.Type, - InheritAgentId = x.InheritAgentId, - Profiles = x.Profiles, - RoutingRules = !x.RoutingRules.IsNullOrEmpty() ? x.RoutingRules - .Select(r => RoutingRuleMongoElement.ToDomainElement(x.Id, x.Name, r)) - .ToList() : new List(), - LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(x.LlmConfig) - }).ToList(); + return agentDocs.Select(x => TransformAgentDocument(x)).ToList(); } public List GetAgentsByUser(string userId) @@ -453,4 +403,37 @@ public bool DeleteAgents() } } + + private Agent TransformAgentDocument(AgentDocument? agentDoc) + { + if (agentDoc == null) return new Agent(); + + return new Agent + { + Id = agentDoc.Id, + Name = agentDoc.Name, + IconUrl = agentDoc.IconUrl, + Description = agentDoc.Description, + Instruction = agentDoc.Instruction, + Templates = !agentDoc.Templates.IsNullOrEmpty() ? agentDoc.Templates + .Select(t => AgentTemplateMongoElement.ToDomainElement(t)) + .ToList() : new List(), + Functions = !agentDoc.Functions.IsNullOrEmpty() ? agentDoc.Functions + .Select(f => FunctionDefMongoElement.ToDomainElement(f)) + .ToList() : new List(), + Responses = !agentDoc.Responses.IsNullOrEmpty() ? agentDoc.Responses + .Select(r => AgentResponseMongoElement.ToDomainElement(r)) + .ToList() : new List(), + Samples = agentDoc.Samples ?? new List(), + IsPublic = agentDoc.IsPublic, + Disabled = agentDoc.Disabled, + Type = agentDoc.Type, + InheritAgentId = agentDoc.InheritAgentId, + Profiles = agentDoc.Profiles, + RoutingRules = !agentDoc.RoutingRules.IsNullOrEmpty() ? agentDoc.RoutingRules + .Select(r => RoutingRuleMongoElement.ToDomainElement(agentDoc.Id, agentDoc.Name, r)) + .ToList() : new List(), + LlmConfig = AgentLlmConfigMongoElement.ToDomainElement(agentDoc.LlmConfig) + }; + } } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs new file mode 100644 index 000000000..125f3cd89 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.AgentTask.cs @@ -0,0 +1,180 @@ +using BotSharp.Abstraction.Repositories.Filters; +using BotSharp.Abstraction.Tasks.Models; +using BotSharp.Plugin.MongoStorage.Collections; + +namespace BotSharp.Plugin.MongoStorage.Repository; + +public partial class MongoRepository +{ + #region Task + public PagedItems GetAgentTasks(AgentTaskFilter filter) + { + var pager = filter.Pager ?? new Pagination(); + var builder = Builders.Filter; + var filters = new List>() { builder.Empty }; + + if (!string.IsNullOrEmpty(filter.AgentId)) + { + filters.Add(builder.Eq(x => x.AgentId, filter.AgentId)); + } + + if (filter.Enabled.HasValue) + { + filters.Add(builder.Eq(x => x.Enabled, filter.Enabled.Value)); + } + + var filterDef = builder.And(filters); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); + var totalTasks = _dc.AgentTasks.CountDocuments(filterDef); + var taskDocs = _dc.AgentTasks.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); + + var agentIds = taskDocs.Select(x => x.AgentId).Distinct().ToList(); + var agents = GetAgents(new AgentFilter { AgentIds = agentIds }); + + var tasks = taskDocs.Select(x => new AgentTask + { + Id = x.Id, + Name = x.Name, + Description = x.Description, + Enabled = x.Enabled, + AgentId = x.AgentId, + DirectAgentId = x.DirectAgentId, + Content = x.Content, + CreatedDateTime = x.CreatedTime, + UpdatedDateTime = x.UpdatedTime, + Agent = agents.FirstOrDefault(a => a.Id == x.AgentId) + }).ToList(); + + return new PagedItems + { + Items = tasks, + Count = (int)totalTasks + }; + } + + public AgentTask? GetAgentTask(string agentId, string taskId) + { + if (string.IsNullOrEmpty(taskId)) return null; + + var taskDoc = _dc.AgentTasks.AsQueryable().FirstOrDefault(x => x.Id == taskId); + if (taskDoc == null) return null; + + var agentDoc = _dc.Agents.AsQueryable().FirstOrDefault(x => x.Id == taskDoc.AgentId); + var agent = TransformAgentDocument(agentDoc); + + var task = new AgentTask + { + Id = taskDoc.Id, + Name = taskDoc.Name, + Description = taskDoc.Description, + Enabled = taskDoc.Enabled, + AgentId = taskDoc.AgentId, + DirectAgentId = taskDoc.DirectAgentId, + Content = taskDoc.Content, + CreatedDateTime = taskDoc.CreatedTime, + UpdatedDateTime = taskDoc.UpdatedTime, + Agent = agent + }; + + return task; + } + + public void InsertAgentTask(AgentTask task) + { + var taskDoc = new AgentTaskDocument + { + Id = Guid.NewGuid().ToString(), + Name = task.Name, + Description = task.Description, + Enabled = task.Enabled, + AgentId = task.AgentId, + DirectAgentId = task.DirectAgentId, + Content = task.Content, + CreatedTime = DateTime.UtcNow, + UpdatedTime = DateTime.UtcNow + }; + + _dc.AgentTasks.InsertOne(taskDoc); + } + + public void BulkInsertAgentTasks(List tasks) + { + if (tasks.IsNullOrEmpty()) return; + + var taskDocs = tasks.Select(x => new AgentTaskDocument + { + Id = string.IsNullOrEmpty(x.Id) ? Guid.NewGuid().ToString() : x.Id, + Name = x.Name, + Description = x.Description, + Enabled = x.Enabled, + AgentId = x.AgentId, + DirectAgentId = x.DirectAgentId, + Content = x.Content, + CreatedTime = x.CreatedDateTime, + UpdatedTime = x.UpdatedDateTime + }).ToList(); + + _dc.AgentTasks.InsertMany(taskDocs); + } + + public void UpdateAgentTask(AgentTask task, AgentTaskField field) + { + if (task == null || string.IsNullOrEmpty(task.Id)) return; + + var filter = Builders.Filter.Eq(x => x.Id, task.Id); + var taskDoc = _dc.AgentTasks.Find(filter).FirstOrDefault(); + if (taskDoc == null) return; + + switch (field) + { + case AgentTaskField.Name: + taskDoc.Name = task.Name; + break; + case AgentTaskField.Description: + taskDoc.Description = task.Description; + break; + case AgentTaskField.Enabled: + taskDoc.Enabled = task.Enabled; + break; + case AgentTaskField.Content: + taskDoc.Content = task.Content; + break; + case AgentTaskField.DirectAgentId: + taskDoc.DirectAgentId = task.DirectAgentId; + break; + case AgentTaskField.All: + taskDoc.Name = task.Name; + taskDoc.Description = task.Description; + taskDoc.Enabled = task.Enabled; + taskDoc.Content = task.Content; + taskDoc.DirectAgentId = task.DirectAgentId; + break; + } + + taskDoc.UpdatedTime = DateTime.UtcNow; + _dc.AgentTasks.ReplaceOne(filter, taskDoc); + } + + public bool DeleteAgentTask(string agentId, string taskId) + { + if (string.IsNullOrEmpty(taskId)) return false; + + var filter = Builders.Filter.Eq(x => x.Id, taskId); + var taskDeleted = _dc.AgentTasks.DeleteOne(filter); + return taskDeleted.DeletedCount > 0; + } + + public bool DeleteAgentTasks() + { + try + { + _dc.AgentTasks.DeleteMany(Builders.Filter.Empty); + return true; + } + catch + { + return false; + } + } + #endregion +} diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs index 0a1c2530e..9f1b2a204 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Conversation.cs @@ -217,9 +217,9 @@ public PagedItems GetConversations(ConversationFilter filter) if (!string.IsNullOrEmpty(filter.UserId)) filters.Add(builder.Eq(x => x.UserId, filter.UserId)); var filterDef = builder.And(filters); - var sortDefinition = Builders.Sort.Descending(x => x.CreatedTime); + var sortDef = Builders.Sort.Descending(x => x.CreatedTime); var pager = filter?.Pager ?? new Pagination(); - var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDefinition).Skip(pager.Offset).Limit(pager.Size).ToList(); + var conversationDocs = _dc.Conversations.Find(filterDef).Sort(sortDef).Skip(pager.Offset).Limit(pager.Size).ToList(); var count = _dc.Conversations.CountDocuments(filterDef); foreach (var conv in conversationDocs) diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj index aa5124f80..b7b43e177 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/BotSharp.Plugin.SqlDriver.csproj @@ -10,6 +10,24 @@ $(SolutionDir)packages + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json index 5d24c701c..73b5151f6 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/agent.json @@ -2,8 +2,13 @@ "id": "beda4c12-e1ec-4b4b-b328-3df4a6687c4f", "name": "SQL Driver", "description": "Convert the requirements into corresponding SQL statements according to the table structure and execute them if needed.", + "type": "task", "createdDateTime": "2023-11-15T13:49:00Z", "updatedDateTime": "2023-11-15T13:49:00Z", + "disabled": false, "isPublic": false, - "allowRouting": true + "profiles": [ "tool", "sql" ], + "llmConfig": { + "max_recursion_depth": 1 + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid index f9aa8a1f9..fca8fa076 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/data/agents/beda4c12-e1ec-4b4b-b328-3df4a6687c4f/instruction.liquid @@ -1,8 +1 @@ -You are a SQL Driver who knows how to convert business requirements to SQL expressions. - -Follow these steps: -1: Look at the table DDL defintions especially for the CONSTRAINT and FOREIGN KEY REFERENCES. -2: Translate user requirements into SQL statements step by step. -3: Double check, don't miss any requirements, all the parameters must have values. -4: Confirm with the user whether to execute the sql statement. - If user confirms to run the query, call function execute_sql to execute it. \ No newline at end of file +You are a SQL Driver who knows how to convert human language to SQL statements. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj index de9ea7998..c8f1a6f59 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj +++ b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) $(LangVersion) $(BotSharpVersion) $(GeneratePackageOnBuild) diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs new file mode 100644 index 000000000..dba3a491e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Plugin.WebDriver.Drivers; + +public interface IWebBrowser +{ + Task LaunchBrowser(string? url); + Task ScreenshotAsync(string path); + Task InputUserText(BrowserActionParams actionParams); + Task InputUserPassword(BrowserActionParams actionParams); + Task ClickButton(BrowserActionParams actionParams); + Task ClickElement(BrowserActionParams actionParams); + Task ChangeListValue(BrowserActionParams actionParams); + Task CheckRadioButton(BrowserActionParams actionParams); + Task ChangeCheckbox(BrowserActionParams actionParams); + Task GoToPage(BrowserActionParams actionParams); + Task ExtractData(BrowserActionParams actionParams); + Task EvaluateScript(string script); + Task CloseBrowser(); +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs index b8e2e5b20..a1939921c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs @@ -1,40 +1,68 @@ +using System.IO; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public class PlaywrightInstance : IDisposable { IPlaywright _playwright; - IBrowser _browser; - IPage _page; + IBrowserContext _context; - // public IPlaywright Playwright => _playwright; - public IBrowser Browser => _browser; - public IPage Page => _page; + public IBrowserContext Context => _context; + public IPage Page + { + get + { + if (_context == null) + { + InitInstance().Wait(); + } + return _context.Pages.LastOrDefault(); + } + } public async Task InitInstance() { if (_playwright == null) { _playwright = await Playwright.CreateAsync(); + } - /*_browser = await _playwright.Chromium.LaunchPersistentContextAsync(@"C:\Users\haipi\AppData\Local\Google\Chrome\User Data", new BrowserTypeLaunchPersistentContextOptions - { - Headless = false, - Channel = "chrome", - });*/ - _browser = await _playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions + if (_context == null) + { + string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}"; + _context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions { Headless = false, Channel = "chrome", - Args = new[] - { - "--start-maximized" + IgnoreDefaultArgs = new[] + { + "--disable-infobars" + }, + Args = new[] + { + "--disable-infobars", + // "--start-maximized" } }); + + _context.Page += async (sender, e) => + { + e.Close += async (sender, e) => + { + Serilog.Log.Information($"Page is closed: {e.Url}"); + }; + Serilog.Log.Information($"New page is created: {e.Url}"); + await e.SetViewportSizeAsync(1280, 800); + }; + + _context.Close += async (sender, e) => + { + Serilog.Log.Warning($"Playwright browser context is closed"); + _context = null; + }; } } - public void SetPage(IPage page) { _page = page; } - public void Dispose() { _playwright.Dispose(); diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs new file mode 100644 index 000000000..8893c424d --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs @@ -0,0 +1,89 @@ +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ChangeCheckbox(BrowserActionParams actionParams) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Retrieve the page raw html and infer the element path + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + var elements = _instance.Page.GetByText(regex); + var count = await elements.CountAsync(); + + if (count == 0) + { + return false; + } + else if (count > 1) + { + _logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}"); + var allElements = await elements.AllAsync(); + foreach (var element in allElements) + { + + } + return false; + } + + var parentElement = elements.Locator(".."); + count = await parentElement.CountAsync(); + if (count == 0) + { + return false; + } + + var id = await elements.GetAttributeAsync("for"); + if (id == null) + { + elements = parentElement.Locator("input"); + } + else + { + elements = _instance.Page.Locator($"#{id}"); + } + count = await elements.CountAsync(); + + if (count == 0) + { + return false; + } + else if (count > 1) + { + _logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}"); + return false; + } + + try + { + var isChecked = await elements.IsCheckedAsync(); + if (actionParams.Context.UpdateValue == "check" && !isChecked) + { + await elements.ClickAsync(); + } + else if (actionParams.Context.UpdateValue == "uncheck" && isChecked) + { + await elements.ClickAsync(); + } + + return true; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + + return false; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs index 476e926e9..c52767261 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs @@ -1,10 +1,14 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string messageId) + public async Task ChangeListValue(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); @@ -55,10 +59,10 @@ public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string } var driverService = _services.GetRequiredService(); - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); + var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, + string.Join("", str), + actionParams.Context.ElementName, + actionParams.MessageId); ILocator element = Locator(htmlElementContextOut); try @@ -80,7 +84,7 @@ await _instance.Page.EvaluateAsync(@"(element) => { await element.FocusAsync(); await element.SelectOptionAsync(new SelectOptionValue { - Label = context.UpdateValue + Label = actionParams.Context.UpdateValue }); // Click on the blank area to activate posting @@ -96,10 +100,13 @@ await _instance.Page.EvaluateAsync(@"(element) => { element.style.visibility = 'hidden'; }", control); } + return true; } catch (Exception ex) { - throw new Exception(ex.Message); + _logger.LogError(ex.Message); } + + return false; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs new file mode 100644 index 000000000..fc69b82b7 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs @@ -0,0 +1,60 @@ +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task CheckRadioButton(BrowserActionParams actionParams) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Retrieve the page raw html and infer the element path + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + var elements = _instance.Page.GetByText(regex); + var count = await elements.CountAsync(); + + if (count == 0) + { + return false; + } + + var parentElement = elements.Locator(".."); + count = await parentElement.CountAsync(); + if (count == 0) + { + return false; + } + + elements = parentElement.GetByText(new Regex($"{actionParams.Context.UpdateValue}", RegexOptions.IgnoreCase)); + + count = await elements.CountAsync(); + + if (count == 0) + { + return false; + } + + try + { + // var label = await elements.GetAttributeAsync("for"); + await elements.SetCheckedAsync(true); + + return true; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + + return false; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs index 294e6894c..b4a9ce9ac 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickButton.cs @@ -1,10 +1,72 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ClickElement(Agent agent, BrowsingContextIn context, string messageId) + public async Task ClickButton(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + await Task.Delay(100); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Button, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); + var count = await elements.CountAsync(); + + if (count == 0) + { + elements = _instance.Page.GetByRole(AriaRole.Link, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementName + }); + count = await elements.CountAsync(); + } + + if (count == 0) + { + elements = _instance.Page.GetByText(actionParams.Context.ElementName); + count = await elements.CountAsync(); + } + + if (count == 0) + { + // Infer element if not found + var driverService = _services.GetRequiredService(); + var html = await FilteredButtonHtml(); + var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, + html, + actionParams.Context.ElementName, + actionParams.MessageId); + elements = Locator(htmlElementContextOut); + + if (elements == null) + { + return false; + } + } + + try + { + await elements.HoverAsync(); + await elements.ClickAsync(); + + await Task.Delay(300); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + return false; + } + + private async Task FilteredButtonHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -32,12 +94,6 @@ public async Task ClickElement(Agent agent, BrowsingContextIn context, string me })); } - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - await element.ClickAsync(); - await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs new file mode 100644 index 000000000..1c13c6589 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ClickElement.cs @@ -0,0 +1,61 @@ +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ClickElement(BrowserActionParams actionParams) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Retrieve the page raw html and infer the element path + var regexExpression = actionParams.Context.MatchRule.ToLower() switch + { + "startwith" => $"^{actionParams.Context.ElementText}", + "endwith" => $"{actionParams.Context.ElementText}$", + "contains" => $"{actionParams.Context.ElementText}", + _ => $"^{actionParams.Context.ElementText}$" + }; + var regex = new Regex(regexExpression, RegexOptions.IgnoreCase); + var elements = _instance.Page.GetByText(regex); + var count = await elements.CountAsync(); + + // try placeholder + if (count == 0) + { + elements = _instance.Page.GetByPlaceholder(regex); + count = await elements.CountAsync(); + } + + if (count == 0) + { + _logger.LogError($"Can't locate element by keyword {actionParams.Context.ElementText}"); + } + else if (count == 1) + { + // var tagName = await elements.EvaluateAsync("el => el.tagName"); + + await elements.HoverAsync(); + await elements.ClickAsync(); + + // Triggered ajax + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + return true; + } + else if (count > 1) + { + _logger.LogWarning($"Multiple elements are found by keyword {actionParams.Context.ElementText}"); + var all = await elements.AllAsync(); + foreach (var element in all) + { + var content = await element.TextContentAsync(); + _logger.LogWarning(content); + } + } + + return false; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs index 7538af779..a098ac010 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CloseBrowser.cs @@ -1,12 +1,13 @@ -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task CloseBrowser(Agent agent, BrowsingContextIn context, string messageId) + public async Task CloseBrowser() { - // await _instance.Browser.CloseAsync(); - _logger.LogInformation($"{agent.Name} closed browser with page {_instance.Page.Url}"); + if (_instance.Context != null) + { + await _instance.Context.CloseAsync(); + } } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs new file mode 100644 index 000000000..554eb39da --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.EvaluateScript.cs @@ -0,0 +1,12 @@ +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task EvaluateScript(string script) + { + await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + return await _instance.Page.EvaluateAsync(script); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs index 2134e6310..8ad05cfd0 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ExtractData.cs @@ -2,16 +2,19 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task ExtractData(Agent agent, BrowsingContextIn context, string messageId) + public async Task ExtractData(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + await Task.Delay(3000); // Retrieve the page raw html and infer the element path var body = await _instance.Page.QuerySelectorAsync("body"); var content = await body.InnerTextAsync(); var driverService = _services.GetRequiredService(); - var answer = await driverService.ExtraData(agent, content, context.Question, messageId); + var answer = await driverService.ExtraData(actionParams.Agent, content, actionParams.Context.Question, actionParams.MessageId); return answer; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs index 09b9deb7b..bd8304fb6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.GoToPage.cs @@ -2,9 +2,10 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task GoToPage(Agent agent, BrowsingContextIn context, string messageId) + public async Task GoToPage(BrowserActionParams actionParams) { - await _instance.Page.GotoAsync(context.Url); + await _instance.Page.GotoAsync(actionParams.Context.Url); await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs index 6bbeef6df..34ab49835 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserPassword.cs @@ -1,8 +1,10 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId) + public async Task InputUserPassword(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); @@ -14,19 +16,21 @@ public async Task InputUserPassword(Agent agent, BrowsingContextIn context, stri if (password == null) { - throw new Exception($"Can't locate the web element {context.ElementName}."); + throw new Exception($"Can't locate the web element {actionParams.Context.ElementName}."); } var config = _services.GetRequiredService(); try { - var key = context.Password.Replace("@", "").Replace(".", ":"); + var key = actionParams.Context.Password.Replace("@", "").Replace(".", ":"); var value = config.GetValue(key); await password.FillAsync(value); + return true; } catch (Exception ex) { - throw new Exception(ex.Message); + _logger.LogError(ex.Message); } + return false; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs index 28a652b7c..b0bccb97d 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.InputUserText.cs @@ -1,11 +1,67 @@ +using Microsoft.Extensions.Logging; + namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task InputUserText(Agent agent, BrowsingContextIn context, string messageId) + public async Task InputUserText(BrowserActionParams actionParams) { await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); - // await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + + // Find by text exactly match + var elements = _instance.Page.GetByRole(AriaRole.Textbox, new PageGetByRoleOptions + { + Name = actionParams.Context.ElementText + }); + var count = await elements.CountAsync(); + if (count == 0) + { + elements = _instance.Page.GetByPlaceholder(actionParams.Context.ElementText); + count = await elements.CountAsync(); + } + + if (count == 0) + { + var driverService = _services.GetRequiredService(); + var html = await FilteredInputHtml(); + var htmlElementContextOut = await driverService.InferElement(actionParams.Agent, + html, + actionParams.Context.ElementText, + actionParams.MessageId); + elements = Locator(htmlElementContextOut); + count = await elements.CountAsync(); + } + + if (count == 0) + { + + } + else if (count == 1) + { + try + { + await elements.FillAsync(actionParams.Context.InputText); + if (actionParams.Context.PressEnter.HasValue && actionParams.Context.PressEnter.Value) + { + await elements.PressAsync("Enter"); + } + + // Triggered ajax + await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle); + return true; + } + catch (Exception ex) + { + _logger.LogError(ex.Message); + } + } + + return false; + } + + private async Task FilteredInputHtml() + { var driverService = _services.GetRequiredService(); // Retrieve the page raw html and infer the element path @@ -48,24 +104,7 @@ public async Task InputUserText(Agent agent, BrowsingContextIn context, string m Placeholder = placeholder })); } - - var htmlElementContextOut = await driverService.LocateElement(agent, - string.Join("", str), - context.ElementName, - messageId); - ILocator element = Locator(htmlElementContextOut); - try - { - await element.FillAsync(context.InputText); - if (context.PressEnter) - { - await element.PressAsync("Enter"); - } - } - catch (Exception ex) - { - throw new Exception(ex.Message); - } + return string.Join("", str); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs index e64017953..3e92618d6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LaunchBrowser.cs @@ -2,21 +2,26 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; public partial class PlaywrightWebDriver { - public async Task LaunchBrowser(string? url) + public async Task LaunchBrowser(string? url) { await _instance.InitInstance(); if (!string.IsNullOrEmpty(url)) { - var page = await _instance.Browser.NewPageAsync(new BrowserNewPageOptions + var page = _instance.Context.Pages.LastOrDefault(); + if (page == null) { - ViewportSize = ViewportSize.NoViewport - }); - _instance.SetPage(page); - var response = await page.GotoAsync(url); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + page = await _instance.Context.NewPageAsync(); + } + + if (!string.IsNullOrEmpty(url)) + { + var response = await page.GotoAsync(url, new PageGotoOptions + { + Timeout = 15 * 1000 + }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); + } } - - return _instance.Browser; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs new file mode 100644 index 000000000..d1fbea55b --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs @@ -0,0 +1,15 @@ + +namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; + +public partial class PlaywrightWebDriver +{ + public async Task ScreenshotAsync(string path) + { + var bytes = await _instance.Page.ScreenshotAsync(new PageScreenshotOptions + { + Path = path, + }); + + return "data:image/png;base64," + Convert.ToBase64String(bytes); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs index 8adfdc39c..ef3a18475 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs @@ -2,13 +2,16 @@ namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; -public partial class PlaywrightWebDriver +public partial class PlaywrightWebDriver : IWebBrowser { private readonly IServiceProvider _services; private readonly PlaywrightInstance _instance; private readonly ILogger _logger; public PlaywrightInstance Instance => _instance; + public Agent Agent => _agent; + private Agent _agent; + public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance, ILogger logger) { _services = services; @@ -16,12 +19,16 @@ public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instanc _logger = logger; } - private ILocator Locator(HtmlElementContextOut context) + public void SetAgent(Agent agent) + { + _agent = agent; + } + + private ILocator? Locator(HtmlElementContextOut context) { ILocator element = default; if (!string.IsNullOrEmpty(context.ElementId)) { - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"#{context.ElementId}"); } else if (!string.IsNullOrEmpty(context.ElementName)) @@ -33,19 +40,25 @@ private ILocator Locator(HtmlElementContextOut context) "button" => AriaRole.Button, _ => AriaRole.Generic }; - // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 }); element = _instance.Page.Locator($"[name='{context.ElementName}']"); - - if (element.CountAsync().Result == 0) + var count = element.CountAsync().Result; + if (count == 0) { _logger.LogError($"Can't locate element {role} {context.ElementName}"); + return null; + } + else if (count > 1) + { + _logger.LogError($"Located multiple elements {role} {context.ElementName}"); + return null; } } else { if (context.Index < 0) { - throw new Exception($"Can't locate the web element {context.Index}."); + _logger.LogError($"Can't locate the web element {context.Index}."); + return null; } element = _instance.Page.Locator(context.TagName).Nth(context.Index); } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs new file mode 100644 index 000000000..6100158c6 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeCheckboxFn.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ChangeCheckboxFn : IFunctionCallback +{ + public string Name => "change_checkbox"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ChangeCheckboxFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + var result = await _browser.ChangeCheckbox(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"{(args.UpdateValue == "check" ? "Check" : "Uncheck")} checkbox of '{args.ElementText}'"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs index ac4c9116e..94eb4522f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class ChangeListValueFn : IFunctionCallback @@ -8,13 +5,13 @@ public class ChangeListValueFn : IFunctionCallback public string Name => "change_list_value"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public ChangeListValueFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -23,9 +20,18 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.ChangeListValue(agent, args, message.MessageId); + var result = await _browser.ChangeListValue(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"Change value to '{args.UpdateValue}' for {args.ElementName}"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); - message.Content = $"Updat the value of \"{args.ElementName}\" to \"{args.UpdateValue}\" successfully."; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs new file mode 100644 index 000000000..422d90aff --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CheckRadioButtonFn.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class CheckRadioButtonFn : IFunctionCallback +{ + public string Name => "check_radio_button"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public CheckRadioButtonFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + var result = await _browser.CheckRadioButton(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"Check value of '{args.UpdateValue}' for radio button '{args.ElementName}'"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs index 3ecbf6b16..f9b5ba98f 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class ClickButtonFn : IFunctionCallback @@ -8,13 +5,13 @@ public class ClickButtonFn : IFunctionCallback public string Name => "click_button"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public ClickButtonFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -23,9 +20,17 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.ClickElement(agent, args, message.MessageId); + var result = await _browser.ClickButton(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"Click button of '{args.ElementName}'"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Content = $"Click button {args.ElementName} successfully."; + message.Data = await _browser.ScreenshotAsync(path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs new file mode 100644 index 000000000..6bc954031 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickElementFn.cs @@ -0,0 +1,37 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class ClickElementFn : IFunctionCallback +{ + public string Name => "click_element"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public ClickElementFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + var args = JsonSerializer.Deserialize(message.FunctionArgs); + + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(message.CurrentAgentId); + var result = await _browser.ClickElement(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"Click element {args.MatchRule} text '{args.ElementText}'"; + message.Content = result ? + $"{content} successfully" : + $"{content} failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs index 1e10e0844..579b514bb 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/CloseBrowserFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class CloseBrowserFn : IFunctionCallback @@ -7,13 +5,13 @@ public class CloseBrowserFn : IFunctionCallback public string Name => "close_browser"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public CloseBrowserFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -21,7 +19,7 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.CloseBrowser(agent, args, message.MessageId); + await _browser.CloseBrowser(); message.Content = $"Browser is closed"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs new file mode 100644 index 000000000..974473427 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/EvaluateScriptFn.cs @@ -0,0 +1,22 @@ +namespace BotSharp.Plugin.WebDriver.Functions; + +public class EvaluateScriptFn : IFunctionCallback +{ + public string Name => "evaluate_script"; + + private readonly IServiceProvider _services; + private readonly IWebBrowser _browser; + + public EvaluateScriptFn(IServiceProvider services, + IWebBrowser browser) + { + _services = services; + _browser = browser; + } + + public async Task Execute(RoleDialogModel message) + { + message.Data = await _browser.EvaluateScript(message.Content); + return true; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs index 271476f08..bf8aeda3c 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class ExtractDataFn : IFunctionCallback @@ -7,13 +5,13 @@ public class ExtractDataFn : IFunctionCallback public string Name => "extract_data_from_page"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public ExtractDataFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -21,7 +19,13 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - message.Content = await _driver.ExtractData(agent, args, message.MessageId); + message.Content = await _browser.ExtractData(new BrowserActionParams(agent, args, message.MessageId)); + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs index b25763598..42bdecd98 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/GoToPageFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class GoToPageFn : IFunctionCallback @@ -7,13 +5,13 @@ public class GoToPageFn : IFunctionCallback public string Name => "go_to_page"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public GoToPageFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -21,8 +19,14 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs); var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.GoToPage(agent, args, message.MessageId); + await _browser.GoToPage(new BrowserActionParams(agent, args, message.MessageId)); message.Content = $"Page {args.Url} is open."; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs index 30aae3327..0a9f67792 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class InputUserPasswordFn : IFunctionCallback @@ -8,13 +5,13 @@ public class InputUserPasswordFn : IFunctionCallback public string Name => "input_user_password"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public InputUserPasswordFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -23,9 +20,15 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.InputUserPassword(agent, args, message.MessageId); + var result = await _browser.InputUserPassword(new BrowserActionParams(agent, args, message.MessageId)); + + message.Content = result ? "Input password successfully" : "Input password failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); - message.Content = "Input password successfully"; return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs index 9248d8783..0ab145055 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs @@ -1,6 +1,3 @@ -using BotSharp.Abstraction.Agents; -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class InputUserTextFn : IFunctionCallback @@ -8,13 +5,13 @@ public class InputUserTextFn : IFunctionCallback public string Name => "input_user_text"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public InputUserTextFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) @@ -23,9 +20,22 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var agent = await agentService.LoadAgent(message.CurrentAgentId); - await _driver.InputUserText(agent, args, message.MessageId); + var result = await _browser.InputUserText(new BrowserActionParams(agent, args, message.MessageId)); + + var content = $"Input '{args.InputText}' in element '{args.ElementText}'"; + if (args.PressEnter != null && args.PressEnter == true) + { + content += " and pressed Enter"; + } + + message.Content = result ? + content + " successfully" : + content + " failed"; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); - message.Content = $"Input text \"{args.InputText}\" successfully."; + message.Data = await _browser.ScreenshotAsync(path); return true; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs index 8a7323da6..945e1fffe 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; - namespace BotSharp.Plugin.WebDriver.Functions; public class OpenBrowserFn : IFunctionCallback @@ -7,20 +5,26 @@ public class OpenBrowserFn : IFunctionCallback public string Name => "open_browser"; private readonly IServiceProvider _services; - private readonly PlaywrightWebDriver _driver; + private readonly IWebBrowser _browser; public OpenBrowserFn(IServiceProvider services, - PlaywrightWebDriver driver) + IWebBrowser browser) { _services = services; - _driver = driver; + _browser = browser; } public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs); - var browser = await _driver.LaunchBrowser(args.Url); + await _browser.LaunchBrowser(args.Url); message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully."; + + var webDriverService = _services.GetRequiredService(); + var path = webDriverService.GetScreenshotFilePath(message.MessageId); + + message.Data = await _browser.ScreenshotAsync(path); + return true; } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs new file mode 100644 index 000000000..0215c0f91 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebDriverConversationHook.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.WebDriver.Hooks; + +public class WebDriverConversationHook : ConversationHookBase +{ + private readonly IServiceProvider _services; + public WebDriverConversationHook(IServiceProvider services) + { + _services = services; + } + + public override async Task OnDialogRecordLoaded(RoleDialogModel dialog) + { + var webDriverService = _services.GetRequiredService(); + + // load screenshot + dialog.Data = "data:image/png;base64," + webDriverService.GetScreenshotBase64(dialog.MessageId); + await base.OnDialogRecordLoaded(dialog); + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs index efd327eb4..6cd74e955 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/BrowsingContextIn.cs @@ -10,11 +10,20 @@ public class BrowsingContextIn [JsonPropertyName("element_name")] public string? ElementName { get; set; } + [JsonPropertyName("element_type")] + public string? ElementType { get; set; } + [JsonPropertyName("input_text")] public string? InputText { get; set; } + [JsonPropertyName("element_text")] + public string? ElementText { get; set; } + [JsonPropertyName("press_enter")] - public bool PressEnter { get; set; } + public bool? PressEnter { get; set; } + + [JsonPropertyName("match_rule")] + public string MatchRule { get; set; } = string.Empty; [JsonPropertyName("update_value")] public string? UpdateValue { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs new file mode 100644 index 000000000..3a18e2602 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Models/BrowserActionParams.cs @@ -0,0 +1,15 @@ +namespace BotSharp.Plugin.WebDriver.Models; + +public class BrowserActionParams +{ + public Agent Agent { get; set; } + public BrowsingContextIn Context { get; set; } + public string MessageId { get; set; } + + public BrowserActionParams(Agent agent, BrowsingContextIn context, string messageId) + { + Agent = agent; + Context = context; + MessageId = messageId; + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs rename to src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs index 690e55cb4..0f6a242dc 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.LocateElement.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.InferElement.cs @@ -6,7 +6,7 @@ namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService { - public async Task LocateElement(Agent agent, string html, string elementName, string messageId) + public async Task InferElement(Agent agent, string html, string elementName, string messageId) { var parserInstruction = agent.Templates.First(x => x.Name == "html_parser").Content; diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs new file mode 100644 index 000000000..431df6a42 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.Screenshot.cs @@ -0,0 +1,38 @@ +using System.IO; + +namespace BotSharp.Plugin.WebDriver.Services; + +public partial class WebDriverService +{ + public string GetScreenshotDir() + { + var conversation = _services.GetRequiredService(); + var agentService = _services.GetRequiredService(); + var dir = $"{agentService.GetDataDir()}/conversations/{conversation.ConversationId}/screenshots"; + if (!Directory.Exists(dir)) + { + Directory.CreateDirectory(dir); + } + return dir; + } + + public string GetScreenshotFilePath(string messageId) + { + var dir = GetScreenshotDir(); + return $"{dir}/{messageId}.png"; + } + + public string? GetScreenshotBase64(string messageId) + { + var filePath = GetScreenshotFilePath(messageId); + if (File.Exists(filePath)) + { + var bytes = File.ReadAllBytes(filePath); + return Convert.ToBase64String(bytes); + } + else + { + return null; + } + } +} diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs index 4b7ac0d96..860db602b 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/Services/WebDriverService.cs @@ -1,7 +1,3 @@ -using BotSharp.Abstraction.Agents.Enums; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Core.Infrastructures; - namespace BotSharp.Plugin.WebDriver.Services; public partial class WebDriverService diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs index 6990980af..2c7a4133e 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.WebDriver/WebDriverPlugin.cs @@ -1,6 +1,5 @@ using BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver; -using BotSharp.Plugin.WebDriver.Services; -using Microsoft.Extensions.Configuration; +using BotSharp.Plugin.WebDriver.Hooks; namespace BotSharp.Plugin.Playwrights; @@ -14,8 +13,9 @@ public class WebDriverPlugin : IBotSharpPlugin public void RegisterDI(IServiceCollection services, IConfiguration config) { - services.AddScoped(); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json index c87e2a3ba..75ab58384 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json @@ -5,9 +5,10 @@ "type": "task", "createdDateTime": "2024-01-02T00:00:00Z", "updatedDateTime": "2024-01-02T00:00:00Z", + "disabled": false, "isPublic": true, - "profiles": [ "web-driver" ], + "profiles": [ "tool", "browser" ], "llmConfig": { - "max_recursion_depth": 10 + "max_recursion_depth": 1 } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json index 59a678aa9..de0d201c6 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json @@ -71,9 +71,9 @@ "parameters": { "type": "object", "properties": { - "element_name": { + "element_text": { "type": "string", - "description": "the html input box element name." + "description": "text or placeholder shown in the element." }, "input_text": { "type": "string", @@ -81,10 +81,10 @@ }, "press_enter": { "type": "boolean", - "description": "whether to press ENTER" + "description": "whether to press Enter key" } }, - "required": [ "element_name", "input_text" ] + "required": [ "element_text", "input_text" ] } }, { @@ -118,5 +118,71 @@ }, "required": [ "password" ] } + }, + { + "name": "change_checkbox", + "description": "Check or uncheck checkbox", + "parameters": { + "type": "object", + "properties": { + "element_text": { + "type": "string", + "description": "the element title" + }, + "update_value": { + "type": "string", + "description": "check or uncheck" + }, + "match_rule": { + "type": "string", + "description": "text matching rule: EndWith, StartWith, Contains, Match" + } + }, + "required": [ "element_text", "update_value", "match_rule" ] + } + }, + { + "name": "click_element", + "description": "Click or check an element contains some text", + "parameters": { + "type": "object", + "properties": { + "element_type": { + "type": "string", + "description": "the element tag name" + }, + "element_text": { + "type": "string", + "description": "text or placeholder shown in the element." + }, + "match_rule": { + "type": "string", + "description": "text matching rule: EndWith, StartWith, Contains, Match" + } + }, + "required": [ "element_type", "element_text", "match_rule" ] + } + }, + { + "name": "check_radio_button", + "description": "Check value in a radio button", + "parameters": { + "type": "object", + "properties": { + "element_text": { + "type": "string", + "description": "the element title" + }, + "update_value": { + "type": "string", + "description": "the value in the radio button." + }, + "match_rule": { + "type": "string", + "description": "text matching rule: EndWith, StartWith, Contains, Match" + } + }, + "required": [ "update_value", "element_text", "match_rule" ] + } } ] diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid index 82c9303dd..1960b0fce 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid @@ -1,11 +1,9 @@ You are a Web Driver that can manipulate web elements through automation tools. Follow below steps to response: -1. Analyze user's latest request in the conversation. +1. Analyze user's request. 2. Call appropriate function to execute the instruction. 3. If user requests execute multiple steps, execute them sequentially. Additional response requirements: -* Call function input_user_password if user wants to input password. -* Don't do extra steps if user didn't ask. -* Don't miss any steps. \ No newline at end of file +* Call function input_user_password if user wants to input password. \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid index d037018b3..cd4db60c1 100644 --- a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid +++ b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid @@ -2,5 +2,5 @@ === According to above HTML === Find the html element in the similar meaning of "{{ element_name }}". -Output in JSON format {"tag_name": "", "element_id": "populated if element has id", "element_name": "populated if element has name", "index": -1}. +Output in JSON format {"tag_name": "", "element_id": "the id attribute", "element_name", "the name attribute", "index": -1}. The index is the position of the element which starts with 0. \ No newline at end of file diff --git a/src/WebStarter/WebStarter.csproj b/src/WebStarter/WebStarter.csproj index 42f4ef2a0..88d2fe6ca 100644 --- a/src/WebStarter/WebStarter.csproj +++ b/src/WebStarter/WebStarter.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) enable enable OutOfProcess diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 9ff26d274..33cffc3c3 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -82,9 +82,11 @@ "EnableLlmCompletionLog": false, "EnableExecutionLog": true }, + "Statistics": { "DataDir": "stats" }, + "LlamaSharp": { "Interactive": true, "ModelDir": "C:/Users/haipi/Downloads", diff --git a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj index 27e9bec93..516b56abf 100644 --- a/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj +++ b/tests/BotSharp.Plugin.SemanticKernel.UnitTests/BotSharp.Plugin.SemanticKernel.UnitTests.csproj @@ -1,7 +1,7 @@ - net8.0 + $(TargetFramework) enable enable diff --git a/tests/UnitTest/UnitTest.csproj b/tests/UnitTest/UnitTest.csproj index 79441c623..be9245737 100644 --- a/tests/UnitTest/UnitTest.csproj +++ b/tests/UnitTest/UnitTest.csproj @@ -1,7 +1,7 @@ - net6.0 + $(TargetFramework) enable enable