Merge pull request #948 from hchen2020/master

Oceania2018 · web-flow · commit fd0e89bf637a · 2025-03-17T17:28:04.000-05:00
util-twilio-transfer_phone_call
diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs
@@ -92,6 +92,7 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         }
 
         routing.Context.SetDialogs(dialogs);
+        routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);
 
         var states = _services.GetRequiredService<IConversationStateService>();
 
@@ -188,6 +189,7 @@ await _completer.Connect(_conn,
                 // append input audio transcript to conversation
                 dialogs.Add(message);
                 storage.Append(_conn.ConversationId, message);
+                routing.Context.SetMessageId(_conn.ConversationId, message.MessageId);
 
                 foreach (var hook in hookProvider.HooksOrderByPriority)
                 {
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj b/src/Plugins/BotSharp.Plugin.Twilio/BotSharp.Plugin.Twilio.csproj
@@ -10,6 +10,9 @@
   </PropertyGroup>
   
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-transfer_phone_call.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-twilio-hangup_phone_call.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs
@@ -35,14 +35,16 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq
             throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
         }
 
+        var twilio = _services.GetRequiredService<TwilioService>();
         VoiceResponse response = default!;
 
         if (request.AnsweredBy == "machine_start" &&
             request.Direction == "outbound-api" &&
             request.InitAudioFile != null)
         {
             response = new VoiceResponse();
-            response.Play(new Uri(request.InitAudioFile));
+            var url = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);
+            response.Play(new Uri(url));
             return TwiML(response);
         }
 
@@ -68,9 +70,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
 
         request.ConversationId = await InitConversation(request);
 
-        var twilio = _services.GetRequiredService<TwilioService>();
-
-        response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);
+        response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);
 
         await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>
         {
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
@@ -438,6 +438,26 @@ public async Task<TwiMLResult> Hangup(ConversationalVoiceRequest request)
         return TwiML(response);
     }
 
+    [ValidateRequest]
+    [HttpPost("twilio/voice/transfer-call")]
+    public async Task<TwiMLResult> TransferCall(ConversationalVoiceRequest request)
+    {
+        var instruction = new ConversationalVoiceResponse
+        {
+            ConversationId = request.ConversationId,
+            TransferTo = request.TransferTo
+        };
+
+        if (request.InitAudioFile != null)
+        {
+            instruction.SpeechPaths.Add(request.InitAudioFile);
+        }
+
+        var twilio = _services.GetRequiredService<TwilioService>();
+        var response = twilio.TransferCall(instruction);
+        return TwiML(response);
+    }
+
     [ValidateRequest]
     [HttpPost("twilio/voice/status")]
     public async Task<ActionResult> PhoneCallStatus(ConversationalVoiceRequest request)
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs
@@ -27,6 +27,9 @@ public class ConversationalVoiceRequest : VoiceRequest
     [FromForm]
     public string? CallbackSource { get; set; }
 
+    [FromQuery(Name = "transfer-to")]
+    public string? TransferTo { get; set; }
+
     /// <summary>
     /// machine_start
     /// </summary>
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceResponse.cs
@@ -13,4 +13,9 @@ public class ConversationalVoiceResponse
     public int Timeout { get; set; } = 3;
 
     public string Hints { get; set; }
+
+    /// <summary>
+    /// The Phone Number to transfer to
+    /// </summary>
+    public string? TransferTo { get; set; }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs
@@ -108,6 +108,7 @@ public async Task<bool> Execute(RoleDialogModel message)
             statusCallback: new Uri(statusUrl),
             // https://www.twilio.com/docs/voice/answering-machine-detection
             machineDetection: _twilioSetting.MachineDetection,
+            machineDetectionSilenceTimeout: _twilioSetting.MachineDetectionSilenceTimeout,
             record: _twilioSetting.RecordingEnabled,
             recordingStatusCallback: $"{_twilioSetting.CallbackHost}/twilio/record/status?conversation-id={newConversationId}");
 
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/TransferPhoneCallFn.cs
@@ -0,0 +1,78 @@
+using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Options;
+using BotSharp.Abstraction.Routing;
+using BotSharp.Core.Infrastructures;
+using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+using Twilio.Rest.Api.V2010.Account;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Functions;
+
+public class TransferPhoneCallFn : IFunctionCallback
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+    private readonly BotSharpOptions _options;
+    private readonly TwilioSetting _twilioSetting;
+
+    public string Name => "util-twilio-transfer_phone_call";
+    public string Indication => "Transferring the active line";
+
+    public TransferPhoneCallFn(
+        IServiceProvider services,
+        ILogger<TransferPhoneCallFn> logger,
+        BotSharpOptions options,
+        TwilioSetting twilioSetting)
+    {
+        _services = services;
+        _logger = logger;
+        _options = options;
+        _twilioSetting = twilioSetting;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var args = JsonSerializer.Deserialize<ForwardPhoneCallArgs>(message.FunctionArgs, _options.JsonSerializerOptions);
+
+        var fileStorage = _services.GetRequiredService<IFileStorageService>();
+        var states = _services.GetRequiredService<IConversationStateService>();
+        var routing = _services.GetRequiredService<IRoutingService>();
+        var conversationId = routing.Context.ConversationId;
+        var processUrl = $"{_twilioSetting.CallbackHost}/twilio/voice/transfer-call?conversation-id={conversationId}&transfer-to={args.PhoneNumber}";
+
+        // Generate initial assistant audio
+        string initAudioFile = null;
+        if (!string.IsNullOrEmpty(args.TransitionMessage))
+        {
+            var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1");
+            var data = await completion.GenerateAudioFromTextAsync(args.TransitionMessage);
+            initAudioFile = "transfer.mp3";
+            fileStorage.SaveSpeechFile(conversationId, initAudioFile, data);
+
+            processUrl += $"&init-audio-file={initAudioFile}";
+        }
+
+        if (!string.IsNullOrEmpty(initAudioFile))
+        {
+            processUrl += $"&init-audio-file={initAudioFile}";
+        }
+
+        // Forward call
+        var sid = states.GetState("twilio_call_sid");
+
+        if (string.IsNullOrEmpty(sid))
+        {
+            _logger.LogError("Twilio call sid is empty.");
+            message.Content = "There is an error when transferring the phone call.";
+            return false;
+        }
+        else
+        {
+            var call = CallResource.Update(
+                pathSid: sid,
+                url: new Uri(processUrl));
+
+            message.Content = args.TransitionMessage;
+            return true;
+        }
+    }
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Hooks/OutboundPhoneCallHandlerUtilityHook.cs
@@ -7,8 +7,9 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook
 {
     private static string PREFIX = "util-twilio-";
     private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";
+    private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call";
     private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";
-    public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
+    private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";
 
     public void AddUtilities(List<AgentUtility> utilities)
     {
@@ -18,6 +19,7 @@ public void AddUtilities(List<AgentUtility> utilities)
             Functions = 
             [
                 new($"{OUTBOUND_PHONE_CALL_FN}"),
+                new($"{TRANSFER_PHONE_CALL_FN}"),
                 new($"{HANGUP_PHONE_CALL_FN}"),
                 new($"{TEXT_MESSAGE_FN}")
             ],
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/LlmContexts/ForwardPhoneCallArgs.cs
@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.LlmContexts;
+
+public class ForwardPhoneCallArgs
+{
+    [JsonPropertyName("phone_number")]
+    public string PhoneNumber { get; set; } = null!;
+
+    [JsonPropertyName("transition_message")]
+    public string TransitionMessage { get; set; } = null!;
+}
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
@@ -160,6 +160,23 @@ public VoiceResponse DialCsrAgent(string speechPath)
         return response;
     }
 
+    public VoiceResponse TransferCall(ConversationalVoiceResponse conversationalVoiceResponse)
+    {
+        var response = new VoiceResponse();
+        var conversationId = conversationalVoiceResponse.ConversationId;
+        if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
+        {
+            foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
+            {
+                var uri = GetSpeechPath(conversationId, speechPath);
+                response.Play(new Uri(uri));
+            }
+        }
+        response.Dial(conversationalVoiceResponse.TransferTo, answerOnBridge: true);
+
+        return response;
+    }
+
     public VoiceResponse HoldOn(int interval, string message = null)
     {
         var twilioSetting = _services.GetRequiredService<TwilioSetting>();
@@ -190,26 +207,17 @@ public VoiceResponse HoldOn(int interval, string message = null)
     /// </summary>
     /// <param name="conversationalVoiceResponse"></param>
     /// <returns></returns>
-    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversationId, ConversationalVoiceResponse conversationalVoiceResponse)
+    public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse)
     {
         var response = new VoiceResponse();
+        var conversationId = conversationalVoiceResponse.ConversationId;
 
         if (conversationalVoiceResponse.SpeechPaths != null && conversationalVoiceResponse.SpeechPaths.Any())
         {
             foreach (var speechPath in conversationalVoiceResponse.SpeechPaths)
             {
-                if (speechPath.StartsWith("twilio/"))
-                {
-                    response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
-                }
-                else if (speechPath.StartsWith(_settings.CallbackHost))
-                {
-                    response.Play(new Uri(speechPath));
-                }
-                else
-                {
-                    response.Play(new Uri($"{_settings.CallbackHost}/twilio/voice/speeches/{conversationalVoiceResponse.ConversationId}/{speechPath}"));
-                }
+                var uri = GetSpeechPath(conversationId, speechPath);
+                response.Play(new Uri(uri));
             }
         }
 
@@ -220,4 +228,20 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(string conversa
 
         return response;
     }
+
+    public string GetSpeechPath(string conversationId, string speechPath)
+    {
+        if (speechPath.StartsWith("twilio/"))
+        {
+            return $"{_settings.CallbackHost}/{speechPath}";
+        }
+        else if (speechPath.StartsWith(_settings.CallbackHost))
+        {
+            return speechPath;
+        }
+        else
+        {
+            return $"{_settings.CallbackHost}/twilio/voice/speeches/{conversationId}/{speechPath}";
+        }
+    }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs
@@ -33,6 +33,7 @@ public class TwilioSetting
     public int MaxGatherAttempts { get; set; } = 4;
 
     public string? MachineDetection { get; set; }
+    public int MachineDetectionSilenceTimeout { get; set; } = 2500;
 
     public bool RecordingEnabled { get; set; } = false;
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json b/src/Plugins/BotSharp.Plugin.Twilio/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-twilio-transfer_phone_call.json
@@ -0,0 +1,19 @@
+{
+  "name": "util-twilio-transfer_phone_call",
+  "description": "When user wants to transfer the phone call",
+  "visibility_expression": "{% if states.channel == 'phone' %}visible{% endif %}",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "transition_message": {
+        "type": "string",
+        "description": "Transition message when forwarding."
+      },
+      "phone_number": {
+        "type": "string",
+        "description": "Phone number transfer to."
+      }
+    },
+    "required": [ "transition_message", "phone_number" ]
+  }
+}

Original file line number	Diff line number	Diff line change
`@@ -92,6 +92,7 @@ private async Task ConnectToModel(WebSocket userWebSocket)`
`92`	`92`	`}`
`93`	`93`
`94`	`94`	`routing.Context.SetDialogs(dialogs);`
	`95`	`+ routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId);`
`95`	`96`
`96`	`97`	`var states = _services.GetRequiredService<IConversationStateService>();`
`97`	`98`
`@@ -188,6 +189,7 @@ await _completer.Connect(_conn,`
`188`	`189`	`// append input audio transcript to conversation`
`189`	`190`	`dialogs.Add(message);`
`190`	`191`	`storage.Append(_conn.ConversationId, message);`
	`192`	`+ routing.Context.SetMessageId(_conn.ConversationId, message.MessageId);`
`191`	`193`
`192`	`194`	`foreach (var hook in hookProvider.HooksOrderByPriority)`
`193`	`195`	`{`
Original file line number	Diff line number	Diff line change
`@@ -35,14 +35,16 @@ public async Task<TwiMLResult> InitiateStreamConversation(ConversationalVoiceReq`
`35`	`35`	`throw new ArgumentNullException(nameof(VoiceRequest.CallSid));`
`36`	`36`	`}`
`37`	`37`
	`38`	`+ var twilio = _services.GetRequiredService<TwilioService>();`
`38`	`39`	`VoiceResponse response = default!;`
`39`	`40`
`40`	`41`	`if (request.AnsweredBy == "machine_start" &&`
`41`	`42`	`request.Direction == "outbound-api" &&`
`42`	`43`	`request.InitAudioFile != null)`
`43`	`44`	`{`
`44`	`45`	`response = new VoiceResponse();`
`45`		`- response.Play(new Uri(request.InitAudioFile));`
	`46`	`+ var url = twilio.GetSpeechPath(request.ConversationId, request.InitAudioFile);`
	`47`	`+ response.Play(new Uri(url));`
`46`	`48`	`return TwiML(response);`
`47`	`49`	`}`
`48`	`50`
`@@ -68,9 +70,7 @@ await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>`
`68`	`70`
`69`	`71`	`request.ConversationId = await InitConversation(request);`
`70`	`72`
`71`		`- var twilio = _services.GetRequiredService<TwilioService>();`
`72`		`-`
`73`		`- response = twilio.ReturnBidirectionalMediaStreamsInstructions(request.ConversationId, instruction);`
	`73`	`+ response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction);`
`74`	`74`
`75`	`75`	`await HookEmitter.Emit<ITwilioSessionHook>(_services, async hook =>`
`76`	`76`	`{`
Original file line number	Diff line number	Diff line change
`@@ -13,4 +13,9 @@ public class ConversationalVoiceResponse`
`13`	`13`	`public int Timeout { get; set; } = 3;`
`14`	`14`
`15`	`15`	`public string Hints { get; set; }`
	`16`	`+`
	`17`	`+ /// <summary>`
	`18`	`+ /// The Phone Number to transfer to`
	`19`	`+ /// </summary>`
	`20`	`+ public string? TransferTo { get; set; }`
`16`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,8 +7,9 @@ public class OutboundPhoneCallHandlerUtilityHook : IAgentUtilityHook`
`7`	`7`	`{`
`8`	`8`	`private static string PREFIX = "util-twilio-";`
`9`	`9`	`private static string OUTBOUND_PHONE_CALL_FN = $"{PREFIX}outbound_phone_call";`
	`10`	`+ private static string TRANSFER_PHONE_CALL_FN = $"{PREFIX}transfer_phone_call";`
`10`	`11`	`private static string HANGUP_PHONE_CALL_FN = $"{PREFIX}hangup_phone_call";`
`11`		`- public static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";`
	`12`	`+ private static string TEXT_MESSAGE_FN = $"{PREFIX}text_message";`
`12`	`13`
`13`	`14`	`public void AddUtilities(List<AgentUtility> utilities)`
`14`	`15`	`{`
`@@ -18,6 +19,7 @@ public void AddUtilities(List<AgentUtility> utilities)`
`18`	`19`	`Functions =`
`19`	`20`	`[`
`20`	`21`	`new($"{OUTBOUND_PHONE_CALL_FN}"),`
	`22`	`+ new($"{TRANSFER_PHONE_CALL_FN}"),`
`21`	`23`	`new($"{HANGUP_PHONE_CALL_FN}"),`
`22`	`24`	`new($"{TEXT_MESSAGE_FN}")`
`23`	`25`	`],`