prevent initial response interruption

Author: Haiping Chen

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -24,7 +24,7 @@ Task Connect(RealtimeHubConnection conn,
     Task Disconnect();
 
     Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogModel> conversations);
-    Task UpdateSession(RealtimeHubConnection conn);
+    Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true);
     Task InsertConversationItem(RoleDialogModel message);
     Task RemoveConversationItem(string itemId);
     Task TriggerModelInference(string? instructions = null);

src/Infrastructure/BotSharp.Core/Realtime/RealtimeHub.cs

@@ -99,8 +99,8 @@ private async Task ConnectToModel(IRealTimeCompletion completer, WebSocket userW
         await completer.Connect(conn, 
             onModelReady: async () => 
             {
-                // Control initial session
-                await completer.UpdateSession(conn);
+                // Control initial session, prevent initial response interruption
+                await completer.UpdateSession(conn, turnDetection: false);
 
                 // Add dialog history
                 foreach (var item in dialogs)
@@ -110,12 +110,16 @@ await completer.Connect(conn,
 
                 if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant)
                 {
-                    // await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
+                    await completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}");
                 }
                 else
                 {
                     await completer.TriggerModelInference("Reply based on the conversation context.");
                 }
+
+                // Start turn detection
+                await Task.Delay(1000 * 8);
+                await completer.UpdateSession(conn, turnDetection: true);
             },
             onModelAudioDeltaReceived: async audioDeltaData =>
             {

src/Plugins/BotSharp.Plugin.OpenAI/Models/Realtime/RealtimeSessionBody.cs

@@ -47,7 +47,7 @@ public class RealtimeSessionBody
     public FunctionDef[] Tools { get; set; } = [];
 
     [JsonPropertyName("turn_detection")]
-    public RealtimeSessionTurnDetection TurnDetection { get; set; } = new();
+    public RealtimeSessionTurnDetection? TurnDetection { get; set; } = new();
 }
 
 public class RealtimeSessionTurnDetection

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -59,10 +59,9 @@ public async Task Connect(RealtimeHubConnection conn,
 
         if (_webSocket.State == WebSocketState.Open)
         {
-            onModelReady();
-
             // Receive a message
             _ = ReceiveMessage(conn,
+                onModelReady,
                 onModelAudioDeltaReceived,
                 onModelAudioResponseDone,
                 onAudioTranscriptDone,
@@ -122,7 +121,8 @@ await SendEventToModel(new
         });
     }
 
-    private async Task ReceiveMessage(RealtimeHubConnection conn, 
+    private async Task ReceiveMessage(RealtimeHubConnection conn,
+        Action onModelReady,
         Action<string> onModelAudioDeltaReceived,
         Action onModelAudioResponseDone,
         Action<string> onAudioTranscriptDone,
@@ -156,6 +156,7 @@ private async Task ReceiveMessage(RealtimeHubConnection conn,
             else if (response.Type == "session.created")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
+                onModelReady();
             }
             else if (response.Type == "session.updated")
             {
@@ -295,7 +296,7 @@ public async Task<RealtimeSession> CreateSession(Agent agent, List<RoleDialogMod
         return session;
     }
 
-    public async Task UpdateSession(RealtimeHubConnection conn)
+    public async Task UpdateSession(RealtimeHubConnection conn, bool turnDetection = true)
     {
         var convService = _services.GetRequiredService<IConversationService>();
         var conv = await convService.GetConversation(conn.ConversationId);
@@ -346,6 +347,11 @@ public async Task UpdateSession(RealtimeHubConnection conn)
             }
         };
 
+        if (!turnDetection)
+        {
+            sessionUpdate.session.TurnDetection = null;
+        }
+
         await HookEmitter.Emit<IContentGeneratingHook>(_services, async hook =>
         {
             await hook.OnSessionUpdated(agent, instruction, functions);

src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs

@@ -1,13 +1,9 @@
 using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Realtime.Models;
-using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.Interfaces;
 using BotSharp.Plugin.Twilio.Models.Stream;
 using Microsoft.AspNetCore.Http;
 using System.Net.WebSockets;
-using System.Text.Json;
-using System.Collections.Concurrent;
-using System.Text;
 using Task = System.Threading.Tasks.Task;
 
 namespace BotSharp.Plugin.Twilio.Services.Stream;