Merge pull request #743 from iceljc/features/add-evaluation

iceljc · web-flow · commit aaf8c598cc16 · 2024-11-12T17:09:36.000-06:00
Features/add evaluation
diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs
@@ -10,6 +10,16 @@ public class EvaluationRequest : LlmBaseRequest
     [JsonPropertyName("states")]
     public IEnumerable<MessageState> States { get; set; } = [];
 
+    [JsonPropertyName("chat")]
+    public ChatEvaluationRequest Chat { get; set; } = new ChatEvaluationRequest();
+
+    [JsonPropertyName("metric")]
+    public MetricEvaluationRequest Metric { get; set; } = new MetricEvaluationRequest();
+}
+
+
+public class ChatEvaluationRequest
+{
     [JsonPropertyName("duplicate_limit")]
     public int DuplicateLimit { get; set; } = 2;
 
@@ -24,4 +34,26 @@ public class EvaluationRequest : LlmBaseRequest
     [JsonPropertyName("stop_criteria")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public string? StopCriteria { get; set; }
+
+    public ChatEvaluationRequest()
+    {
+        
+    }
 }
+
+
+public class MetricEvaluationRequest
+{
+    [JsonPropertyName("additional_instruction")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? AdditionalInstruction { get; set; }
+
+    [JsonPropertyName("metrics")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public IEnumerable<NameDesc>? Metrics { get; set; } = [];
+
+    public MetricEvaluationRequest()
+    {
+        
+    }
+}
diff --git a/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs b/src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationResult.cs
@@ -6,4 +6,5 @@ public class EvaluationResult
     public string TaskInstruction { get; set; }
     public string SystemPrompt { get; set; }
     public string GeneratedConversationId { get; set; }
+    public string? MetricResult { get; set; }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs
@@ -1,6 +1,7 @@
 using BotSharp.Abstraction.Evaluations.Models;
 using BotSharp.Abstraction.Instructs;
 using BotSharp.Abstraction.Instructs.Models;
+using BotSharp.Abstraction.Models;
 
 namespace BotSharp.Core.Evaluations.Services;
 
@@ -31,15 +32,19 @@ public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRe
             return result;
         }
 
-        var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
+        var initialStates = GetInitialStates(conversationId);
+        var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request, initialStates);
+        var metricResult = await EvaluateMetrics(generatedConvId, refDialogContents, request);
 
         return new EvaluationResult
         {
-            GeneratedConversationId = generatedConvId
+            GeneratedConversationId = generatedConvId,
+            MetricResult = metricResult
         };
     }
 
-    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs, EvaluationRequest request)
+    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs,
+        EvaluationRequest request, IEnumerable<MessageState>? states = null)
     {
         var count = 0;
         var duplicateCount = 0;
@@ -49,20 +54,22 @@ private async Task<string> SimulateConversation(string initMessage, IEnumerable<
         var prevUserMsg = string.Empty;
         var curBotMsg = string.Empty;
         var prevBotMsg = string.Empty;
+        var initialStates = states?.ToList() ?? [];
 
         var storage = _services.GetRequiredService<IConversationStorage>();
         var agentService = _services.GetRequiredService<IAgentService>();
         var instructService = _services.GetRequiredService<IInstructService>();
 
         var query = "Please see yourself as a user and follow the instruction to generate a message.";
         var targetAgentId = request.AgentId;
-        var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
-        var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
+        var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
+        var simulatorPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
 
         while (true)
         {
             curDialogs.Add($"{AgentRole.User}: {curUserMsg}");
-            var dialog = await SendMessage(targetAgentId, convId, curUserMsg);
+            var dialog = await SendMessage(targetAgentId, convId, curUserMsg, states: initialStates);
+            initialStates = [];
 
             prevBotMsg = curBotMsg;
             curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
@@ -80,30 +87,20 @@ private async Task<string> SimulateConversation(string initMessage, IEnumerable<
                                 {
                                     { "ref_conversation", refDialogs },
                                     { "cur_conversation", curDialogs },
-                                    { "additional_instruction", request.AdditionalInstruction },
-                                    { "stop_criteria", request.StopCriteria }
+                                    { "additional_instruction", request.Chat.AdditionalInstruction },
+                                    { "stop_criteria", request.Chat.StopCriteria }
                                 }
                             });
 
             _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
 
-            if (count > request.MaxRounds || (result != null && result.Stop))
+            if (count > request.Chat.MaxRounds || (result != null && result.Stop))
             {
                 break;
             }
 
-
-            if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg))
-            {
-                duplicateCount++;
-            }
-            else
-            {
-                duplicateCount = 0;
-            }
-
-
-            if (duplicateCount >= request.DuplicateLimit)
+            duplicateCount = curBotMsg.IsEqualTo(prevBotMsg) ? duplicateCount + 1 : 0;
+            if (duplicateCount >= request.Chat.DuplicateLimit)
             {
                 break;
             }
@@ -115,6 +112,38 @@ private async Task<string> SimulateConversation(string initMessage, IEnumerable<
         return convId;
     }
 
+
+    private async Task<string?> EvaluateMetrics(string curConversationId, IEnumerable<string> refDialogs, EvaluationRequest request)
+    {
+        var storage = _services.GetRequiredService<IConversationStorage>();
+        var agentService = _services.GetRequiredService<IAgentService>();
+        var instructService = _services.GetRequiredService<IInstructService>();
+
+        var curDialogs = storage.GetDialogs(curConversationId);
+        var curDialogContents = GetConversationContent(curDialogs);
+
+        var evaluator = await agentService.GetAgent(BuiltInAgentId.Evaluator);
+        var metricPrompt = evaluator.Templates.FirstOrDefault(x => x.Name == "instruction.metrics")?.Content ?? string.Empty;
+        var query = "Please follow the instruction for evaluation.";
+
+        var result = await instructService.Instruct<JsonDocument>(metricPrompt, BuiltInAgentId.Evaluator,
+                            new InstructOptions
+                            {
+                                Provider = request.Provider,
+                                Model = request.Model,
+                                Message = query,
+                                Data = new Dictionary<string, object>
+                                {
+                                    { "ref_conversation", refDialogs },
+                                    { "cur_conversation", curDialogs },
+                                    { "additional_instruction", request.Metric.AdditionalInstruction },
+                                    { "metrics", request.Metric.Metrics }
+                                }
+                            });
+
+        return result != null ? result.RootElement.GetRawText() : null;
+    }
+
     private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
     {
         var contents = new List<string>();
@@ -134,4 +163,30 @@ private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel>
 
         return contents;
     }
+
+    private IEnumerable<MessageState> GetInitialStates(string conversationId)
+    {
+        if (string.IsNullOrWhiteSpace(conversationId))
+        {
+            return [];
+        }
+
+        var db = _services.GetRequiredService<IBotSharpRepository>();
+        var states = db.GetConversationStates(conversationId);
+        var initialStates = new List<MessageState>();
+
+        foreach (var state in states)
+        {
+            var value = state.Value?.Values?.FirstOrDefault(x => string.IsNullOrEmpty(x.MessageId));
+
+            if (string.IsNullOrEmpty(value?.Data))
+            {
+                continue;
+            }
+
+            initialStates.Add(new MessageState(state.Key, value.Data, value.ActiveRounds));
+        }
+
+        return initialStates;
+    }
 }
diff --git a/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs b/src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.cs
@@ -61,7 +61,10 @@ public async Task<Conversation> Execute(string task, EvaluationRequest request)
             dialogs.Add(new RoleDialogModel(AgentRole.User, question));
             prompt += question.Trim();
 
-            response = await SendMessage(request.AgentId, conv.Id, question);
+            response = await SendMessage(request.AgentId, conv.Id, question, states: new List<MessageState>
+            {
+                new MessageState("channel", ConversationChannel.OpenAPI)
+            });
             dialogs.Add(new RoleDialogModel(AgentRole.Assistant, response.Content));
             prompt += $"\r\n{AgentRole.Assistant}: {response.Content.Trim()}";
             prompt += $"\r\n{AgentRole.User}: ";
@@ -86,17 +89,16 @@ public async Task<Conversation> Execute(string task, EvaluationRequest request)
         return conv;
     }
 
-    private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text)
+    private async Task<RoleDialogModel> SendMessage(string agentId, string conversationId, string text,
+        PostbackMessageModel? postback = null,
+        List<MessageState>? states = null)
     {
         var conv = _services.GetRequiredService<IConversationService>();
         var routing = _services.GetRequiredService<IRoutingService>();
 
         var inputMsg = new RoleDialogModel(AgentRole.User, text);
         routing.Context.SetMessageId(conversationId, inputMsg.MessageId);
-        conv.SetConversationId(conversationId, new List<MessageState>
-        {
-            new MessageState("channel", ConversationChannel.OpenAPI)
-        });
+        conv.SetConversationId(conversationId, states ?? []);
 
         RoleDialogModel response = default;
 
diff --git a/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid b/src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.metrics.liquid
@@ -1 +1,45 @@
-You are a conversation evaluator.
+You are a conversaton evaluator.
+Please take the content in the [REFERENCE CONVERSATION] section and [ONGOING CONVERSATION] section, and evaluate the metrics defined in [OUTPUT JSON FORMAT].
+
+** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and evaluate the metrics listed in [OUTPUT JSON FORMAT].
+
+
+=================
+[ADDITIONAL INSTRUCTION]
+{{ "\r\n" }}
+{%- if additional_instruction != empty -%}
+{{ additional_instruction }}
+{%- endif -%}
+{{ "\r\n" }}
+
+
+=================
+[OUTPUT JSON FORMAT]
+
+** The output must be in JSON format:
+{
+    {%- if metrics != empty -%}
+        {{ "\r\n" }}
+        {% for metric in metrics -%}
+        {{ metric.name }}: {{ metric.description }},{{ "\r\n" }}
+        {%- endfor %}
+    {%- else -%}
+        "summary": a short summary that summarizes the [ONGOING CONVERSATION] content compared to the [REFERENCE CONVERSATION]
+    {%- endif -%}
+}
+
+
+=================
+[REFERENCE CONVERSATION]
+
+{% for text in ref_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}
+
+
+=================
+[ONGOING CONVERSATION]
+
+{% for text in cur_conversation -%}
+{{ text }}{{ "\r\n" }}
+{%- endfor %}
diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Hooks/SqlDriverPlanningHook.cs
@@ -53,6 +53,7 @@ await HookEmitter.Emit<ISqlDriverHook>(_services, async (hook) =>
         // Invoke "execute_sql"
         var routing = _services.GetRequiredService<IRoutingService>();
         await routing.InvokeFunction(response.FunctionName, response);
+
         msg.CurrentAgentId = agent.Id;
         msg.FunctionName = response.FunctionName;
         msg.FunctionArgs = response.FunctionArgs;
@@ -64,13 +65,10 @@ public async Task<string> GetSummaryAdditionalRequirements(string planner, RoleD
     {
         var settings = _services.GetRequiredService<SqlDriverSetting>();
         var sqlHooks = _services.GetServices<ISqlDriverHook>();
-        
-        var dbType = sqlHooks.Any() ? 
-            sqlHooks.First().GetDatabaseType(message) :
-            settings.DatabaseType;
+        var agentService = _services.GetRequiredService<IAgentService>();
 
-        var agent = await _services.GetRequiredService<IAgentService>()
-            .LoadAgent(BuiltInAgentId.SqlDriver);
+        var dbType = !sqlHooks.IsNullOrEmpty() ? sqlHooks.First().GetDatabaseType(message) : settings.DatabaseType;
+        var agent = await agentService.LoadAgent(BuiltInAgentId.SqlDriver);
 
         return agent.Templates.FirstOrDefault(x => x.Name == $"database.summarize.{dbType}")?.Content ?? string.Empty;
     }
@@ -101,7 +99,6 @@ private RichContent<IRichMessage> BuildRunQueryButton(string conversationId, str
                         Type = "text",
                         Title = "Execute the SQL Statement",
                         Payload = sql,
-
                         IsPrimary = true
                     },
                     new ElementButton

Original file line number	Diff line number	Diff line change
`@@ -6,4 +6,5 @@ public class EvaluationResult`
`6`	`6`	`public string TaskInstruction { get; set; }`
`7`	`7`	`public string SystemPrompt { get; set; }`
`8`	`8`	`public string GeneratedConversationId { get; set; }`
	`9`	`+ public string? MetricResult { get; set; }`
`9`	`10`	`}`