Skip to content

Commit 1d69fe1

Browse files
Oceania2018Oceania2018
authored
Merge pull request #291 from hchen2020/master
Abstract IWebBrowser.
2 parents 08620ab + c84da67 commit 1d69fe1

36 files changed

+591
-150
lines changed

‎src/Infrastructure/BotSharp.Abstraction/Routing/Models/DecomposedStep.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ public class DecomposedStep
77
[JsonPropertyName("total_remaining_steps")]
88
public int TotalRemainingSteps { get; set; }
99

10+
[JsonPropertyName("should_stop")]
11+
public bool ShouldStop { get; set; }
12+
1013
[JsonPropertyName("stop_reason")]
1114
public string? StopReason { get; set; }
1215
}

‎src/Infrastructure/BotSharp.Core/Routing/Planning/SequentialPlanner.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,19 @@ public async Task<FunctionCallFromLlm> GetNextInstruction(Agent router, string m
3131
{
3232
_lastInst.Response = decomposation.Description;
3333
_lastInst.Reason = $"{decomposation.TotalRemainingSteps} left.";
34+
dialogs.Add(new RoleDialogModel(AgentRole.User, decomposation.Description)
35+
{
36+
CurrentAgentId = router.Id,
37+
MessageId = messageId
38+
});
3439
return _lastInst;
3540
}
36-
else if (decomposation.TotalRemainingSteps == 0)
41+
else if (decomposation.TotalRemainingSteps == 0 || decomposation.ShouldStop)
3742
{
3843
// Tell router all steps are done
3944
dialogs.Add(new RoleDialogModel(AgentRole.Assistant, decomposation.StopReason)
4045
{
4146
CurrentAgentId = router.Id,
42-
FunctionName = nameof(SequentialPlanner),
4347
MessageId = messageId
4448
});
4549
router.TemplateDict["conversation"] = router.TemplateDict["conversation"].ToString().TrimEnd() +

‎src/Infrastructure/BotSharp.Core/Routing/RoutingService.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,15 @@ public async Task<RoleDialogModel> InstructDirect(Agent agent, RoleDialogModel m
3939
var handler = handlers.FirstOrDefault(x => x.Name == "route_to_agent");
4040

4141
var conv = _services.GetRequiredService<IConversationService>();
42-
var dialogs = conv.GetDialogHistory();
42+
var dialogs = new List<RoleDialogModel>();
43+
if (conv.States.GetState("hide_context", "false") == "true")
44+
{
45+
dialogs.Add(message);
46+
}
47+
else
48+
{
49+
dialogs = conv.GetDialogHistory();
50+
}
4351
handler.SetDialogs(dialogs);
4452

4553
var inst = new FunctionCallFromLlm

‎src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/planner_prompt.sequential.get_remaining_task.liquid

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,10 @@ If a specific step has been exectued, you will get something indicates the resul
33
If there is no any result provided, it means all the steps have not been executed yet.
44
You need to figure out which steps have not been completed.
55
Tell me what is the first remaining step from user steps that have not been completed based on the context.
6-
Output in JSON { "description": "step detail with arguments", "total_remaining_steps": 0, "stop_reason": "the reason why it should total remaining steps is zero"}
6+
Output in JSON
7+
{
8+
"description": "step detail with arguments",
9+
"total_remaining_steps": 0,
10+
"should_stop": false,
11+
"stop_reason": "the reason why it should total remaining steps is zero"
12+
}

‎src/Infrastructure/BotSharp.Core/data/plugins/config.json

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"Comment": "User can change this configuration dynamically and reflect updates to UI.",
3-
"EnabledPlugins": [
4-
"6e52d42d-1e23-406b-8599-36af36c83209"
5-
]
3+
"EnabledPlugins": [
4+
"6e52d42d-1e23-406b-8599-36af36c83209",
5+
"d42a0c21-b461-44f6-ada2-499510d260af"
6+
]
67
}

‎src/Plugins/BotSharp.Plugin.WebDriver/Drivers/IWebBrowser.cs

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,16 @@ namespace BotSharp.Plugin.WebDriver.Drivers;
33
public interface IWebBrowser
44
{
55
Task LaunchBrowser(string? url);
6-
Task InputUserText(Agent agent, BrowsingContextIn context, string messageId);
7-
Task InputUserPassword(Agent agent, BrowsingContextIn context, string messageId);
8-
Task ClickElement(Agent agent, BrowsingContextIn context, string messageId);
9-
Task GoToPage(Agent agent, BrowsingContextIn context, string messageId);
6+
Task<string> ScreenshotAsync(string path);
7+
Task<bool> InputUserText(BrowserActionParams actionParams);
8+
Task<bool> InputUserPassword(BrowserActionParams actionParams);
9+
Task<bool> ClickButton(BrowserActionParams actionParams);
10+
Task<bool> ClickElement(BrowserActionParams actionParams);
11+
Task<bool> ChangeListValue(BrowserActionParams actionParams);
12+
Task<bool> CheckRadioButton(BrowserActionParams actionParams);
13+
Task<bool> ChangeCheckbox(BrowserActionParams actionParams);
14+
Task<bool> GoToPage(BrowserActionParams actionParams);
15+
Task<string> ExtractData(BrowserActionParams actionParams);
16+
Task<T> EvaluateScript<T>(string script);
17+
Task CloseBrowser();
1018
}

‎src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightInstance.cs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,17 @@ public class PlaywrightInstance : IDisposable
88
IBrowserContext _context;
99

1010
public IBrowserContext Context => _context;
11-
public IPage Page => _context.Pages.LastOrDefault();
11+
public IPage Page
12+
{
13+
get
14+
{
15+
if (_context == null)
16+
{
17+
InitInstance().Wait();
18+
}
19+
return _context.Pages.LastOrDefault();
20+
}
21+
}
1222

1323
public async Task InitInstance()
1424
{
@@ -19,10 +29,10 @@ public async Task InitInstance()
1929

2030
if (_context == null)
2131
{
22-
string tempFolderPath = $"{Path.GetTempPath()}\\playwright";
32+
string tempFolderPath = $"{Path.GetTempPath()}\\playwright\\{Guid.NewGuid()}";
2333
_context = await _playwright.Chromium.LaunchPersistentContextAsync(tempFolderPath, new BrowserTypeLaunchPersistentContextOptions
2434
{
25-
Headless = false,
35+
Headless = true,
2636
Channel = "chrome",
2737
IgnoreDefaultArgs = new[]
2838
{

‎src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeCheckbox.cs

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
using Microsoft.Extensions.Logging;
2+
using System.Text.RegularExpressions;
3+
4+
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
5+
6+
public partial class PlaywrightWebDriver
7+
{
8+
public async Task<bool> ChangeCheckbox(BrowserActionParams actionParams)
9+
{
10+
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
11+
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
12+
13+
// Retrieve the page raw html and infer the element path
14+
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
15+
{
16+
"startwith" => $"^{actionParams.Context.ElementText}",
17+
"endwith" => $"{actionParams.Context.ElementText}$",
18+
"contains" => $"{actionParams.Context.ElementText}",
19+
_ => $"^{actionParams.Context.ElementText}$"
20+
};
21+
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
22+
var elements = _instance.Page.GetByText(regex);
23+
var count = await elements.CountAsync();
24+
25+
if (count == 0)
26+
{
27+
return false;
28+
}
29+
else if (count > 1)
30+
{
31+
_logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}");
32+
var allElements = await elements.AllAsync();
33+
foreach (var element in allElements)
34+
{
35+
36+
}
37+
return false;
38+
}
39+
40+
var parentElement = elements.Locator("..");
41+
count = await parentElement.CountAsync();
42+
if (count == 0)
43+
{
44+
return false;
45+
}
46+
47+
var id = await elements.GetAttributeAsync("for");
48+
if (id == null)
49+
{
50+
elements = parentElement.Locator("input");
51+
}
52+
else
53+
{
54+
elements = _instance.Page.Locator($"#{id}");
55+
}
56+
count = await elements.CountAsync();
57+
58+
if (count == 0)
59+
{
60+
return false;
61+
}
62+
else if (count > 1)
63+
{
64+
_logger.LogError($"Located multiple elements by {actionParams.Context.ElementText}");
65+
return false;
66+
}
67+
68+
try
69+
{
70+
var isChecked = await elements.IsCheckedAsync();
71+
if (actionParams.Context.UpdateValue == "checked" && !isChecked)
72+
{
73+
await elements.ClickAsync();
74+
}
75+
else if (actionParams.Context.UpdateValue == "unchecked" && isChecked)
76+
{
77+
await elements.ClickAsync();
78+
}
79+
80+
return true;
81+
}
82+
catch (Exception ex)
83+
{
84+
_logger.LogError(ex.Message);
85+
}
86+
87+
return false;
88+
}
89+
}

‎src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ChangeListValue.cs

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
using Microsoft.Extensions.Logging;
2+
13
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
24

35
public partial class PlaywrightWebDriver
46
{
5-
public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string messageId)
7+
public async Task<bool> ChangeListValue(BrowserActionParams actionParams)
68
{
79
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
10+
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
11+
812
// Retrieve the page raw html and infer the element path
913
var body = await _instance.Page.QuerySelectorAsync("body");
1014

@@ -55,10 +59,10 @@ public async Task ChangeListValue(Agent agent, BrowsingContextIn context, string
5559
}
5660

5761
var driverService = _services.GetRequiredService<WebDriverService>();
58-
var htmlElementContextOut = await driverService.InferElement(agent,
59-
string.Join("", str),
60-
context.ElementName,
61-
messageId);
62+
var htmlElementContextOut = await driverService.InferElement(actionParams.Agent,
63+
string.Join("", str),
64+
actionParams.Context.ElementName,
65+
actionParams.MessageId);
6266
ILocator element = Locator(htmlElementContextOut);
6367

6468
try
@@ -80,7 +84,7 @@ await _instance.Page.EvaluateAsync(@"(element) => {
8084
await element.FocusAsync();
8185
await element.SelectOptionAsync(new SelectOptionValue
8286
{
83-
Label = context.UpdateValue
87+
Label = actionParams.Context.UpdateValue
8488
});
8589

8690
// Click on the blank area to activate posting
@@ -96,10 +100,13 @@ await _instance.Page.EvaluateAsync(@"(element) => {
96100
element.style.visibility = 'hidden';
97101
}", control);
98102
}
103+
return true;
99104
}
100105
catch (Exception ex)
101106
{
102-
throw new Exception(ex.Message);
107+
_logger.LogError(ex.Message);
103108
}
109+
110+
return false;
104111
}
105112
}

‎src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.CheckRadioButton.cs

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
using Microsoft.Extensions.Logging;
2+
using System.Text.RegularExpressions;
3+
4+
namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
5+
6+
public partial class PlaywrightWebDriver
7+
{
8+
public async Task<bool> CheckRadioButton(BrowserActionParams actionParams)
9+
{
10+
await _instance.Page.WaitForLoadStateAsync(LoadState.DOMContentLoaded);
11+
await _instance.Page.WaitForLoadStateAsync(LoadState.NetworkIdle);
12+
13+
// Retrieve the page raw html and infer the element path
14+
var regexExpression = actionParams.Context.MatchRule.ToLower() switch
15+
{
16+
"startwith" => $"^{actionParams.Context.ElementText}",
17+
"endwith" => $"{actionParams.Context.ElementText}$",
18+
"contains" => $"{actionParams.Context.ElementText}",
19+
_ => $"^{actionParams.Context.ElementText}$"
20+
};
21+
var regex = new Regex(regexExpression, RegexOptions.IgnoreCase);
22+
var elements = _instance.Page.GetByText(regex);
23+
var count = await elements.CountAsync();
24+
25+
if (count == 0)
26+
{
27+
return false;
28+
}
29+
30+
var parentElement = elements.Locator("..");
31+
count = await parentElement.CountAsync();
32+
if (count == 0)
33+
{
34+
return false;
35+
}
36+
37+
elements = parentElement.GetByText(new Regex($"{actionParams.Context.UpdateValue}", RegexOptions.IgnoreCase));
38+
39+
count = await elements.CountAsync();
40+
41+
if (count == 0)
42+
{
43+
return false;
44+
}
45+
46+
try
47+
{
48+
// var label = await elements.GetAttributeAsync("for");
49+
await elements.SetCheckedAsync(true);
50+
51+
return true;
52+
}
53+
catch (Exception ex)
54+
{
55+
_logger.LogError(ex.Message);
56+
}
57+
58+
return false;
59+
}
60+
}

0 commit comments

Comments
 (0)