Skip to content

Commit bb3e293

Browse files
authored
Merge pull request #735 from iceljc/features/add-evaluation
init conv simulation
2 parents f06a080 + e215dec commit bb3e293

File tree

12 files changed

+183
-14
lines changed

12 files changed

+183
-14
lines changed

src/Infrastructure/BotSharp.Abstraction/Agents/Enums/BuiltInAgentId.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,9 @@ public class BuiltInAgentId
4646
/// Programming source code generation
4747
/// </summary>
4848
public const string CodeDriver = "c0ded7d9-3f9d-4ef6-b7ce-56a892dcef62";
49+
50+
/// <summary>
51+
/// Evaluate prompt and conversation
52+
/// </summary>
53+
public const string Evaluator = "dfd9b46d-d00c-40af-8a75-3fbdc2b89869";
4954
}
Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,18 @@
1+
using BotSharp.Abstraction.Processors.Models;
2+
13
namespace BotSharp.Abstraction.Evaluations.Models;
24

3-
public class EvaluationRequest
5+
public class EvaluationRequest : LlmBaseRequest
46
{
5-
public string AgentId { get; set; }
7+
[JsonPropertyName("agent_id")]
8+
public new string AgentId { get; set; }
9+
10+
[JsonPropertyName("states")]
11+
public IEnumerable<MessageState> States { get; set; } = [];
12+
13+
[JsonPropertyName("max_rounds")]
14+
public int MaxRounds { get; set; } = 20;
15+
16+
[JsonPropertyName("ref_conversation_id")]
17+
public string RefConversationId { get; set; } = null!;
618
}

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ public class EvaluationResult
55
public List<RoleDialogModel> Dialogs { get; set; }
66
public string TaskInstruction { get; set; }
77
public string SystemPrompt { get; set; }
8+
public string GeneratedConversationId { get; set; }
89
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
namespace BotSharp.Abstraction.Evaluations.Models;
2+
3+
public class SimulationResult
4+
{
5+
[JsonPropertyName("generated_message")]
6+
public string GeneratedMessage { get; set; }
7+
8+
[JsonPropertyName("stop")]
9+
public bool Stop { get; set; }
10+
11+
[JsonPropertyName("reason")]
12+
public string? Reason { get; set; }
13+
}

src/Infrastructure/BotSharp.Abstraction/Plugins/Models/PluginDef.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ public class PluginDef
77
public string Description { get; set; }
88
public string Assembly { get; set; }
99
[JsonPropertyName("is_core")]
10-
public bool IsCore => Assembly == "BotSharp.Core";
10+
public bool IsCore => Assembly == "BotSharp.Core" || Assembly == "BotSharp.Core.SideCar";
1111

1212
[JsonPropertyName("icon_url")]
1313
public string? IconUrl { get; set; }

src/Infrastructure/BotSharp.Abstraction/Processors/Models/LlmBaseRequest.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,15 @@ namespace BotSharp.Abstraction.Processors.Models;
22

33
public class LlmBaseRequest
44
{
5+
[JsonPropertyName("provider")]
56
public string Provider { get; set; }
7+
8+
[JsonPropertyName("model")]
69
public string Model { get; set; }
10+
11+
[JsonPropertyName("agent_id")]
712
public string? AgentId { get; set; }
13+
14+
[JsonPropertyName("template_name")]
815
public string? TemplateName { get; set; }
916
}

src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,9 @@
8383
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\agent.json" />
8484
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\instructions\instruction.liquid" />
8585
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.executor.liquid" />
86+
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid" />
8687
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.reviewer.liquid" />
88+
<None Remove="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid" />
8789
<None Remove="data\plugins\config.json" />
8890
</ItemGroup>
8991

@@ -163,6 +165,12 @@
163165
<Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\select_file_prompt.liquid">
164166
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
165167
</Content>
168+
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.simulator.liquid">
169+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
170+
</Content>
171+
<Content Include="data\agents\dfd9b46d-d00c-40af-8a75-3fbdc2b89869\templates\instruction.metrics.liquid">
172+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
173+
</Content>
166174
<Content Include="data\plugins\config.json">
167175
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
168176
</Content>

src/Infrastructure/BotSharp.Core/Evaluations/EvaluationPlugin.cs

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
using BotSharp.Abstraction.Evaluations.Settings;
22
using BotSharp.Abstraction.Evaluations;
33
using BotSharp.Abstraction.Settings;
4-
using BotSharp.Core.Evaluatings;
54
using Microsoft.Extensions.Configuration;
5+
using BotSharp.Core.Evaluations.Services;
66

77
namespace BotSharp.Core.Evaluations;
88

@@ -21,7 +21,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
2121
return settingService.Bind<EvaluatorSetting>("Evaluator");
2222
});
2323

24-
services.AddScoped<IConversationHook, EvaluationConversationHook>();
2524
services.AddScoped<IEvaluatingService, EvaluatingService>();
2625
services.AddScoped<IExecutionLogger, ExecutionLogger>();
2726
}
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
using BotSharp.Abstraction.Evaluations.Models;
2+
using BotSharp.Abstraction.Instructs;
3+
using BotSharp.Abstraction.Instructs.Models;
4+
5+
namespace BotSharp.Core.Evaluations.Services;
6+
7+
public partial class EvaluatingService
8+
{
9+
public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
10+
{
11+
var storage = _services.GetRequiredService<IConversationStorage>();
12+
var refDialogs = storage.GetDialogs(request.RefConversationId);
13+
var refDialogContents = GetConversationContent(refDialogs);
14+
15+
var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
16+
var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello";
17+
18+
var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
19+
20+
return new EvaluationResult
21+
{
22+
GeneratedConversationId = generatedConvId
23+
};
24+
}
25+
26+
private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refConversation, EvaluationRequest request)
27+
{
28+
var count = 0;
29+
var curConvId = Guid.NewGuid().ToString();
30+
var curConversation = new List<string>();
31+
var curMessage = initMessage;
32+
33+
var storage = _services.GetRequiredService<IConversationStorage>();
34+
var agentService = _services.GetRequiredService<IAgentService>();
35+
var instructService = _services.GetRequiredService<IInstructService>();
36+
37+
var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone.";
38+
var targetAgentId = request.AgentId;
39+
var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
40+
var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
41+
42+
while (true)
43+
{
44+
curConversation.Add($"{AgentRole.User}: {curMessage}");
45+
var dialog = await SendMessage(targetAgentId, curConvId, curMessage);
46+
var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
47+
curConversation.Add($"{AgentRole.Assistant}: {botMessage}");
48+
count++;
49+
50+
var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
51+
new InstructOptions
52+
{
53+
Provider = request.Provider,
54+
Model = request.Model,
55+
Message = query,
56+
Data = new Dictionary<string, object>
57+
{
58+
{ "ref_conversation", refConversation },
59+
{ "cur_conversation", curConversation },
60+
}
61+
});
62+
63+
if (count > request.MaxRounds || (result != null && result.Stop))
64+
{
65+
break;
66+
}
67+
68+
curMessage = result?.GeneratedMessage ?? string.Empty;
69+
}
70+
71+
return curConvId;
72+
}
73+
74+
private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
75+
{
76+
var contents = new List<string>();
77+
78+
foreach (var dialog in dialogs)
79+
{
80+
var role = dialog.Role;
81+
if (role == AgentRole.Function) continue;
82+
83+
if (role != AgentRole.User)
84+
{
85+
role = AgentRole.Assistant;
86+
}
87+
88+
contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}");
89+
}
90+
91+
return contents;
92+
}
93+
}

src/Infrastructure/BotSharp.Core/Evaluations/EvaluatingService.cs renamed to src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,22 @@
44
using BotSharp.Abstraction.Evaluations.Settings;
55
using BotSharp.Abstraction.Models;
66
using BotSharp.Abstraction.Templating;
7-
using System.Drawing;
87

9-
namespace BotSharp.Core.Evaluatings;
8+
namespace BotSharp.Core.Evaluations.Services;
109

11-
public class EvaluatingService : IEvaluatingService
10+
public partial class EvaluatingService : IEvaluatingService
1211
{
1312
private readonly IServiceProvider _services;
13+
private readonly ILogger<EvaluatingService> _logger;
1414
private readonly EvaluatorSetting _settings;
15-
public EvaluatingService(IServiceProvider services, EvaluatorSetting settings)
15+
16+
public EvaluatingService(
17+
IServiceProvider services,
18+
ILogger<EvaluatingService> logger,
19+
EvaluatorSetting settings)
1620
{
1721
_services = services;
22+
_logger = logger;
1823
_settings = settings;
1924
}
2025

@@ -81,11 +86,6 @@ public async Task<Conversation> Execute(string task, EvaluationRequest request)
8186
return conv;
8287
}
8388

84-
public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
85-
{
86-
throw new NotImplementedException();
87-
}
88-
8989
private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
9090
{
9191
var conv = _services.GetRequiredService<IConversationService>();

0 commit comments

Comments
 (0)