Merge pull request #736 from iceljc/features/add-evaluation

refine stop condition

Author: iceljc

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/EvaluationRequest.cs

@@ -12,7 +12,4 @@ public class EvaluationRequest : LlmBaseRequest
 
     [JsonPropertyName("max_rounds")]
     public int MaxRounds { get; set; } = 20;
-
-    [JsonPropertyName("ref_conversation_id")]
-    public string RefConversationId { get; set; } = null!;
 }

src/Infrastructure/BotSharp.Abstraction/Evaluations/Models/SimulationResult.cs

@@ -5,7 +5,7 @@ public class SimulationResult
     [JsonPropertyName("generated_message")]
     public string GeneratedMessage { get; set; }
 
-    [JsonPropertyName("stop")]
+    [JsonPropertyName("stop_conversation")]
     public bool Stop { get; set; }
 
     [JsonPropertyName("reason")]

src/Infrastructure/BotSharp.Core/Evaluations/Services/EvaluatingService.Evaluate.cs

@@ -8,12 +8,28 @@ public partial class EvaluatingService
 {
     public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRequest request)
     {
+        var result = new EvaluationResult();
+        if (string.IsNullOrEmpty(conversationId))
+        {
+            return result;
+        }
+
         var storage = _services.GetRequiredService<IConversationStorage>();
-        var refDialogs = storage.GetDialogs(request.RefConversationId);
-        var refDialogContents = GetConversationContent(refDialogs);
+        var refDialogs = storage.GetDialogs(conversationId);
+
+        if (refDialogs.IsNullOrEmpty())
+        {
+            return result;
+        }
 
+        var refDialogContents = GetConversationContent(refDialogs);
         var initDialog = refDialogs.FirstOrDefault(x => x.Role == AgentRole.User);
-        var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content ?? "Hello";
+        var initMessage = initDialog?.RichContent?.Message?.Text ?? initDialog?.Content;
+
+        if (string.IsNullOrWhiteSpace(initMessage))
+        {
+            return result;
+        }
 
         var generatedConvId = await SimulateConversation(initMessage, refDialogContents, request);
 
@@ -23,28 +39,34 @@ public async Task<EvaluationResult> Evaluate(string conversationId, EvaluationRe
         };
     }
 
-    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refConversation, EvaluationRequest request)
+    private async Task<string> SimulateConversation(string initMessage, IEnumerable<string> refDialogs, EvaluationRequest request)
     {
         var count = 0;
-        var curConvId = Guid.NewGuid().ToString();
-        var curConversation = new List<string>();
-        var curMessage = initMessage;
+        var convId = Guid.NewGuid().ToString();
+        var curDialogs = new List<string>();
+        var curUserMsg = initMessage;
+        var prevUserMsg = string.Empty;
+        var curBotMsg = string.Empty;
+        var prevBotMsg = string.Empty;
 
         var storage = _services.GetRequiredService<IConversationStorage>();
         var agentService = _services.GetRequiredService<IAgentService>();
         var instructService = _services.GetRequiredService<IInstructService>();
 
-        var query = "Please take yourself as a user and follow the instruction to generate a message in the user tone.";
+        var query = "Please see yourself as a user and follow the instruction to generate a message.";
         var targetAgentId = request.AgentId;
         var evaluatorAgent = await agentService.GetAgent(BuiltInAgentId.Evaluator);
         var simulatorPrompt = evaluatorAgent.Templates.FirstOrDefault(x => x.Name == "instruction.simulator")?.Content ?? string.Empty;
 
         while (true)
         {
-            curConversation.Add($"{AgentRole.User}: {curMessage}");
-            var dialog = await SendMessage(targetAgentId, curConvId, curMessage);
-            var botMessage = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
-            curConversation.Add($"{AgentRole.Assistant}: {botMessage}");
+            curDialogs.Add($"{AgentRole.User}: {curUserMsg}");
+            var dialog = await SendMessage(targetAgentId, convId, curUserMsg);
+
+            prevBotMsg = curBotMsg;
+            curBotMsg = dialog?.RichContent?.Message?.Text ?? dialog?.Content ?? string.Empty;
+            curDialogs.Add($"{AgentRole.Assistant}: {curBotMsg}");
+
             count++;
 
             var result = await instructService.Instruct<SimulationResult>(simulatorPrompt, BuiltInAgentId.Evaluator,
@@ -55,20 +77,24 @@ private async Task<string> SimulateConversation(string initMessage, IEnumerable<
                                 Message = query,
                                 Data = new Dictionary<string, object>
                                 {
-                                    { "ref_conversation", refConversation },
-                                    { "cur_conversation", curConversation },
+                                    { "ref_conversation", refDialogs },
+                                    { "cur_conversation", curDialogs },
                                 }
                             });
 
-            if (count > request.MaxRounds || (result != null && result.Stop))
+            _logger.LogInformation($"Generated message: {result?.GeneratedMessage}, stop: {result?.Stop}, reason: {result?.Reason}");
+
+            if (curUserMsg.IsEqualTo(prevUserMsg) || curBotMsg.IsEqualTo(prevBotMsg)
+                || count > request.MaxRounds || (result != null && result.Stop))
             {
                 break;
             }
 
-            curMessage = result?.GeneratedMessage ?? string.Empty;
+            prevUserMsg = curUserMsg;
+            curUserMsg = result?.GeneratedMessage ?? string.Empty;
         }
 
-        return curConvId;
+        return convId;
     }
 
     private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel> dialogs)
@@ -85,7 +111,7 @@ private IEnumerable<string> GetConversationContent(IEnumerable<RoleDialogModel>
                 role = AgentRole.Assistant;
             }
 
-            contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content}");
+            contents.Add($"{role}: {dialog.RichContent?.Message?.Text ?? dialog.Content ?? string.Empty}");
         }
 
         return contents;

src/Infrastructure/BotSharp.Core/data/agents/dfd9b46d-d00c-40af-8a75-3fbdc2b89869/templates/instruction.simulator.liquid

@@ -2,14 +2,14 @@ You are a conversaton simulator.
 Please take the content in the [REFERENCE CONVERSATION] section as a reference, and focus on the [ONGOING CONVERSATION] section to generate a message based on the context.
 
 ** You need to take a close look at the content in both [REFERENCE CONVERSATION] and [ONGOING CONVERSATION], and determine whether to generate a text message or stop the ongoing conversation.
-** When you generate a message, please assume you as the user and reply as the user perceptive.
-** Please do not generate or append messages with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
-** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the [ONGOING CONVERSATION] immediately and turn the "stop" to true in the output json.
+** When you generate a message, please assume you are the user and reply in the user perceptive.
+** Please do not generate or append a message with similar meaning that you have already mentioned in the [ONGOING CONVERSATION].
+** If you see the assistant replies two or more than two similar messages in the [ONGOING CONVERSATION], please stop the conversation immediately.
 ** The output must be in JSON format:
 {
-    "generated_message": the generated text message as the user tone,
-    "stop": a boolean to indicate whether to stop the [ONGOING CONVERSATION],
-    "reason": the reason why you generate the message or stop the [ONGOING CONVERSATION]
+    "generated_message": the generated text message using the user tone,
+    "stop_conversation": the boolean value to indicate whether to stop the conversation,
+    "reason": the reason why you generate the message or stop the conversation
 }