Merge pull request #264 from hchen2020/master

Oceania2018 · web-flow · commit 4083e7651817 · 2024-01-23T20:10:25.000-06:00
SequentialPlanner draft.
diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj
@@ -50,8 +50,9 @@
     <None Remove="data\agents\01e2fc5c-2c89-4ec7-8470-7688608b496c\instruction.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\agent.json" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instruction.liquid" />
-    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.hf_planner.liquid" />
-    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.hf.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid" />
+    <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid" />
     <None Remove="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\welcome.liquid" />
     <None Remove="data\plugins\config.json" />
@@ -70,10 +71,13 @@
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\instruction.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.hf_planner.liquid">
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.sequential.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\next_step_prompt.liquid">
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.hf.liquid">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\planner_prompt.naive.liquid">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
     <Content Include="data\agents\01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a\templates\response_with_function.liquid">
diff --git a/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/HFPlanner.cs
@@ -91,7 +91,7 @@ public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, Ro
 
     private string GetNextStepPrompt(Agent router)
     {
-        var template = router.Templates.First(x => x.Name == "next_step_prompt.hf_planner").Content;
+        var template = router.Templates.First(x => x.Name == "planner_prompt.hf").Content;
         var render = _services.GetRequiredService<ITemplateRender>();
         var prompt = render.Render(template, router.TemplateDict);
         return prompt.Trim();
diff --git a/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/NaivePlanner.cs
@@ -108,7 +108,7 @@ public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, Ro
 
     private string GetNextStepPrompt(Agent router)
     {
-        var template = router.Templates.First(x => x.Name == "next_step_prompt").Content;
+        var template = router.Templates.First(x => x.Name == "planner_prompt.naive").Content;
 
         var render = _services.GetRequiredService<ITemplateRender>();
         return render.Render(template, new Dictionary<string, object>
diff --git a/src/Infrastructure/BotSharp.Core/Planning/SequentialPlanner.cs b/src/Infrastructure/BotSharp.Core/Planning/SequentialPlanner.cs
@@ -0,0 +1,112 @@
+using BotSharp.Abstraction.Agents.Models;
+using BotSharp.Abstraction.Functions.Models;
+using BotSharp.Abstraction.Planning;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Abstraction.Routing.Models;
+using BotSharp.Abstraction.Templating;
+
+namespace BotSharp.Core.Planning;
+
+public class SequentialPlanner : IPlaner
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public SequentialPlanner(IServiceProvider services, ILogger<NaivePlanner> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string messageId)
+    {
+        var next = GetNextStepPrompt(router);
+
+        var inst = new FunctionCallFromLlm();
+
+        // text completion
+        /*var agentService = _services.GetRequiredService<IAgentService>();
+        var instruction = agentService.RenderedInstruction(router);
+        var content = $"{instruction}\r\n###\r\n{next}";
+        content =  content + "\r\nResponse: ";
+        var completion = CompletionProvider.GetTextCompletion(_services);*/
+
+        // chat completion
+        var completion = CompletionProvider.GetChatCompletion(_services,
+            provider: router?.LlmConfig?.Provider,
+            model: router?.LlmConfig?.Model);
+
+        int retryCount = 0;
+        while (retryCount < 3)
+        {
+            string text = string.Empty;
+            try
+            {
+                // text completion
+                // text = await completion.GetCompletion(content, router.Id, messageId);
+                var dialogs = new List<RoleDialogModel>
+                {
+                    new RoleDialogModel(AgentRole.User, next)
+                    {
+                        MessageId = messageId
+                    }
+                };
+                var response = await completion.GetChatCompletions(router, dialogs);
+
+                inst = response.Content.JsonContent<FunctionCallFromLlm>();
+                break;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"{ex.Message}: {text}");
+                inst.Function = "response_to_user";
+                inst.Response = ex.Message;
+                inst.AgentName = "Router";
+            }
+            finally
+            {
+                retryCount++;
+            }
+        }
+
+        return inst;
+    }
+
+    public async Task<bool> AgentExecuting(Agent router, FunctionCallFromLlm inst, RoleDialogModel message)
+    {
+        // Set user content as Planner's question
+        message.FunctionName = inst.Function;
+        message.FunctionArgs = inst.Arguments == null ? "{}" : JsonSerializer.Serialize(inst.Arguments);
+
+        return true;
+    }
+
+    public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, RoleDialogModel message)
+    {
+        var context = _services.GetRequiredService<RoutingContext>();
+
+        if (message.StopCompletion)
+        {
+            context.Empty();
+            return false;
+        }
+        
+        // Handover to Router;
+        context.Pop();
+
+        var routing = _services.GetRequiredService<IRoutingService>();
+        routing.ResetRecursiveCounter();
+
+        return true;
+    }
+
+    private string GetNextStepPrompt(Agent router)
+    {
+        var template = router.Templates.First(x => x.Name == "planner_prompt.sequential").Content;
+
+        var render = _services.GetRequiredService<ITemplateRender>();
+        return render.Render(template, new Dictionary<string, object>
+        {
+        });
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingPlugin.cs
@@ -37,12 +37,16 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
 
         services.AddScoped<NaivePlanner>();
         services.AddScoped<HFPlanner>();
+        services.AddScoped<SequentialPlanner>();
+
         services.AddScoped<IPlaner>(provider =>
         {
             var settingService = provider.GetRequiredService<ISettingService>();
             var routingSettings = settingService.Bind<RoutingSettings>("Router");
             if (routingSettings.Planner == nameof(HFPlanner))
                 return provider.GetRequiredService<HFPlanner>();
+            else if (routingSettings.Planner == nameof(SequentialPlanner))
+                return provider.GetRequiredService<SequentialPlanner>();
             else
                 return provider.GetRequiredService<NaivePlanner>();
         });
diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.hf.liquid
diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.naive.liquid
diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.liquid
@@ -0,0 +1,3 @@
+In order to execute the instructions listed by the user in the order specified by the user.
+What is the next step based on the CONVERSATION? 
+Response must be in required JSON format.
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.Selenium.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.Selenium.csproj
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj b/src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Playwright" Version="1.39.0" />
+    <PackageReference Include="Microsoft.Playwright" Version="1.41.1" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs
@@ -1,4 +1,5 @@
 using BotSharp.Plugin.WebDriver.Services;
+using System.Threading;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
@@ -10,22 +11,49 @@ public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string
         var body = await _instance.Page.QuerySelectorAsync("body");
 
         var str = new List<string>();
-        var inputs = await body.QuerySelectorAllAsync("input");
+        var inputs = await body.QuerySelectorAllAsync("select");
         foreach (var input in inputs)
         {
-            var text = await input.TextContentAsync();
+            var html = "<select";
+            var id = await input.GetAttributeAsync("id");
+            if (!string.IsNullOrEmpty(id))
+            {
+                html += $" id='{id}'";
+            }
             var name = await input.GetAttributeAsync("name");
-            var type = await input.GetAttributeAsync("type");
-            str.Add($"<input name='{name}' type='{type}'>{text}</input>");
-        }
+            if (!string.IsNullOrEmpty(name))
+            {
+                html += $" name='{id}'";
+            }
+            html += ">";
 
-        inputs = await body.QuerySelectorAllAsync("textarea");
-        foreach (var input in inputs)
-        {
-            var text = await input.TextContentAsync();
-            var name = await input.GetAttributeAsync("name");
-            var type = await input.GetAttributeAsync("type");
-            str.Add($"<textarea name='{name}' type='{type}'>{text}</textarea>");
+            var options = await input.QuerySelectorAllAsync("option");
+            if (options != null)
+            {
+                foreach (var option in options)
+                {
+                    html += "<option";
+                    var value = await option.GetAttributeAsync("value");
+                    if (!string.IsNullOrEmpty(value))
+                    {
+                        html += $" value='{value}'";
+                    }
+                    html += ">";
+                    var text = await option.TextContentAsync();
+                    if (!string.IsNullOrEmpty(text))
+                    {
+                        html += text;
+                    }
+                    else
+                    {
+                        html += "'<NULL>'";
+                    }
+                    html += "</option>";
+                }
+            }
+
+            html += "</select>";
+            str.Add(html);
         }
 
         var driverService = _services.GetRequiredService<WebDriverService>();
@@ -36,10 +64,41 @@ public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string
             throw new Exception($"Can't locate the web element {context.ElementName}.");
         }
 
-        var element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
+        ILocator element = default;
+        if (!string.IsNullOrEmpty(htmlElementContextOut.ElementId))
+        {
+            // await _instance.Page.WaitForSelectorAsync($"#{htmlElementContextOut.ElementId}", new PageWaitForSelectorOptions { Timeout = 3 });
+            element = _instance.Page.Locator($"#{htmlElementContextOut.ElementId}");
+        }
+        else
+        {
+            element = _instance.Page.Locator(htmlElementContextOut.TagName).Nth(htmlElementContextOut.Index);
+        }
+        
         try
         {
-            await element.FillAsync(context.InputText);
+            var isVisible = await element.IsVisibleAsync();
+
+            if (!isVisible)
+            {
+                // Select the element you want to make visible (replace with your own selector)
+                var control = await _instance.Page.QuerySelectorAsync($"#{htmlElementContextOut.ElementId}");
+
+                // Show the element by modifying its CSS styles
+                await _instance.Page.EvaluateAsync(@"(element) => {
+                    element.style.display = 'block';
+                    element.style.visibility = 'visible';
+                }", control);
+            }
+
+            await element.FocusAsync();
+            await element.SelectOptionAsync(new SelectOptionValue
+            {
+                Label = context.UpdateValue
+            });
+            
+            // Click on the blank area to activate posting
+            await body.ClickAsync();
         }
         catch (Exception ex)
         {
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs
@@ -4,6 +4,7 @@ public partial class PlaywrightWebDriver
 {
     private readonly IServiceProvider _services;
     private readonly PlaywrightInstance _instance;
+    public PlaywrightInstance Instance => _instance;
 
     public PlaywrightWebDriver(IServiceProvider services, PlaywrightInstance instance)
     {
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ChangeListValueFn.cs
@@ -23,9 +23,10 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ChangeListValue(agent, args, message.MessageId);
 
-        message.Content = "Update successfully.";
+        message.Content = $"Updat the value of \"${args.ElementName}\" to \"{args.UpdateValue}\" successfully.";
         return true;
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ClickButtonFn.cs
@@ -23,9 +23,10 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.ClickElement(agent, args, message.MessageId);
 
-        message.Content = "Executed successfully.";
+        message.Content = $"Click button {args.ElementName} successfully.";
 
         return true;
     }
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/ExtractDataFn.cs
@@ -23,6 +23,7 @@ public async Task<bool> Execute(RoleDialogModel message)
         var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         message.Content = await _driver.ExtractData(agent, args, message.MessageId);
         return true;
     }
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserPasswordFn.cs
@@ -23,6 +23,7 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserPassword(agent, args, message.MessageId);
 
         message.Content = "Input password successfully";
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/InputUserTextFn.cs
@@ -23,9 +23,10 @@ public async Task<bool> Execute(RoleDialogModel message)
 
         var agentService = _services.GetRequiredService<IAgentService>();
         var agent = await agentService.LoadAgent(message.CurrentAgentId);
+        await _driver.Instance.Page.WaitForLoadStateAsync(LoadState.Load);
         await _driver.InputUserText(agent, args, message.MessageId);
 
-        message.Content = "Input text successfully.";
+        message.Content = $"Input text \"{args.InputText}\" successfully.";
         return true;
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs b/src/Plugins/BotSharp.Plugin.WebDriver/Functions/OpenBrowserFn.cs
@@ -20,9 +20,7 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var args = JsonSerializer.Deserialize<BrowsingContextIn>(message.FunctionArgs);
         var browser = await _driver.LaunchBrowser(args.Url);
-        message.Content = string.IsNullOrEmpty(args.Url) ? "Launch browser successfully." : $"Open website successfully.";
-        message.Content += "\r\nWhat would you like to do next?";
-        message.StopCompletion = true;
+        message.Content = string.IsNullOrEmpty(args.Url) ? $"Launch browser with blank page successfully." : $"Open website {args.Url} successfully.";
         return true;
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs b/src/Plugins/BotSharp.Plugin.WebDriver/LlmContexts/HtmlElementContextOut.cs
@@ -4,6 +4,9 @@ namespace BotSharp.Plugin.WebDriver.LlmContexts;
 
 public class HtmlElementContextOut
 {
+    [JsonPropertyName("element_id")]
+    public string ElementId { get; set; }
+
     [JsonPropertyName("tag_name")]
     public string TagName { get; set; }
 
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/agent.json
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/functions.json
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/instruction.liquid
diff --git a/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid b/src/Plugins/BotSharp.Plugin.WebDriver/data/agents/f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b/templates/html_parser.liquid

Original file line number	Diff line number	Diff line change
`@@ -91,7 +91,7 @@ public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, Ro`
`91`	`91`
`92`	`92`	`private string GetNextStepPrompt(Agent router)`
`93`	`93`	`{`
`94`		`- var template = router.Templates.First(x => x.Name == "next_step_prompt.hf_planner").Content;`
	`94`	`+ var template = router.Templates.First(x => x.Name == "planner_prompt.hf").Content;`
`95`	`95`	`var render = _services.GetRequiredService<ITemplateRender>();`
`96`	`96`	`var prompt = render.Render(template, router.TemplateDict);`
`97`	`97`	`return prompt.Trim();`
Original file line number	Diff line number	Diff line change
`@@ -108,7 +108,7 @@ public async Task<bool> AgentExecuted(Agent router, FunctionCallFromLlm inst, Ro`
`108`	`108`
`109`	`109`	`private string GetNextStepPrompt(Agent router)`
`110`	`110`	`{`
`111`		`- var template = router.Templates.First(x => x.Name == "next_step_prompt").Content;`
	`111`	`+ var template = router.Templates.First(x => x.Name == "planner_prompt.naive").Content;`
`112`	`112`
`113`	`113`	`var render = _services.GetRequiredService<ITemplateRender>();`
`114`	`114`	`return render.Render(template, new Dictionary<string, object>`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+In order to execute the instructions listed by the user in the order specified by the user.`
	`2`	`+What is the next step based on the CONVERSATION?`
	`3`	`+Response must be in required JSON format.`