draft

Bo Yin · Bo Yin · commit dc4269d9fa57 · 2024-08-08T22:28:39.000-05:00
diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs b/src/Infrastructure/BotSharp.Core/Files/Services/BotSharpFileService.Speech.cs
@@ -2,7 +2,7 @@
 
 namespace BotSharp.Core.Files.Services
 {
-    public partial class BotSharpFileService
+    public partial class LocalFileStorageService
     {
         public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)
         {
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs
@@ -86,24 +86,24 @@ public async Task<TwiMLResult> ReceivedVoiceMessage([FromRoute] string agentId,
     }
 
 
-    [HttpPost("anonymous/start")]
+    [HttpPost("start")]
     public TwiMLResult InitiateConversation(VoiceRequest request)
     {
         if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid));
         string sessionId = $"TwilioVoice_{request.CallSid}";
         var twilio = _services.GetRequiredService<TwilioService>();
-        var url = $"twiliovoice/anonymous/{sessionId}/send/0";
-        var response = twilio.DummyInstructions("Hello, how may I help you?", url, false);
+        var url = $"twiliovoice/{sessionId}/send/0";
+        var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, false);
         return TwiML(response);
     }
 
-    [HttpPost("anonymous/{sessionId}/send/{seqNum}")]
+    [HttpPost("{sessionId}/send/{seqNum}")]
     public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
     {
         var twilio = _services.GetRequiredService<TwilioService>();
         var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();
         var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();
-        var url = $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}";
+        var url = $"twiliovoice/{sessionId}/reply/{seqNum}";
         var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum);
         if (!string.IsNullOrWhiteSpace(request.SpeechResult))
         {
@@ -121,16 +121,16 @@ public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [
                 From = request.From
             };
             await messageQueue.EnqueueAsync(callerMessage);
-            response = twilio.DummyInstructions("Please hold on and wait a moment.", url, true);
+            response = twilio.ReturnInstructions("twilio/holdon.mp3", url, true);
         }
         else
         {
-            response = twilio.HangUp("Thanks for calling. Good bye.");
+            response = twilio.HangUp("twilio/holdon.mp3");
         }
         return TwiML(response);
     }
 
-    [HttpPost("anonymous/{sessionId}/reply/{seqNum}")]
+    [HttpPost("{sessionId}/reply/{seqNum}")]
     public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)
     {
         var nextSeqNum = seqNum + 1;
@@ -144,38 +144,28 @@ public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId,
         VoiceResponse response;
         if (string.IsNullOrEmpty(reply))
         {
-            response = twilio.ReturnInstructions(null, $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}", true);
+            response = twilio.ReturnInstructions(null, $"twiliovoice/{sessionId}/reply/{seqNum}", true);
         }
         else
         {
 
             var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
-            var fileService = _services.GetRequiredService<IBotSharpFileService>();
+            var fileService = _services.GetRequiredService<IFileStorageService>();
             var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply);
             var fileName = $"{seqNum}.mp3";
             await fileService.SaveSpeechFileAsync(sessionId, fileName, data);
-            response = twilio.ReturnInstructions($"twiliovoice/anonymous/speeches/{sessionId}/{fileName}", $"twiliovoice/anonymous/{sessionId}/send/{nextSeqNum}", true);
+            response = twilio.ReturnInstructions($"twiliovoice/speeches/{sessionId}/{fileName}", $"twiliovoice/{sessionId}/send/{nextSeqNum}", true);
         }
         return TwiML(response);
     }
 
-    [HttpGet("anonymous/speeches/{conversationId}/{fileName}")]
+    [HttpGet("speeches/{conversationId}/{fileName}")]
     public async Task<FileContentResult> RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)
     {
-        var fileService = _services.GetRequiredService<IBotSharpFileService>();
+        var fileService = _services.GetRequiredService<IFileStorageService>();
         var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName);
-        var result = new FileContentResult(data.ToArray(), "application/octet-stream");
+        var result = new FileContentResult(data.ToArray(), "audio/mpeg");
         result.FileDownloadName = fileName;
         return result;
     }
-
-    [HttpGet("anonymous/text-to-speech")]
-    public async Task<IActionResult> TextToSpeech([FromQuery] string text)
-    {
-        var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");
-        var data = await textToSpeechService.GenerateSpeechFromTextAsync(text);
-        var fileService = _services.GetRequiredService<IBotSharpFileService>();
-        await fileService.SaveSpeechFileAsync("123", "sample.mp3", data);
-        return Ok();
-    }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/CallerMessage.cs
@@ -9,7 +9,7 @@ public class CallerMessage
 
         public override string ToString()
         {
-            return $"({SessionId}-{SeqNumber}) {Content}";
+            return $"{SessionId}-{SeqNumber}";
         }
     }
 }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs
@@ -25,6 +25,7 @@ public async ValueTask EnqueueAsync(CallerMessage request)
 
         internal void Stop()
         {
+            Console.WriteLine($"[{DateTime.UtcNow}] Complete queue");
             _queue.Writer.TryComplete();
         }
     }
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs
@@ -1,7 +1,6 @@
 using BotSharp.Abstraction.Routing;
 using BotSharp.Plugin.Twilio.Models;
 using Microsoft.Extensions.Hosting;
-using System.Security.Cryptography;
 using System.Threading;
 using Task = System.Threading.Tasks.Task;
 
@@ -31,12 +30,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 {
                     try
                     {
-                        Console.WriteLine("Processing {message}.", message);
+                        Console.WriteLine($"Start processing {message}.");
                         await ProcessUserMessageAsync(message);
                     }
                     catch (Exception ex)
                     {
-                        Console.WriteLine("Processing {message} failed due to {ex}.", message, ex.Message);
+                        Console.WriteLine($"Processing {message} failed due to {ex.Message}.");
                     }
                     finally
                     {
@@ -57,18 +56,17 @@ private async Task ProcessUserMessageAsync(CallerMessage message)
             using var scope = _serviceProvider.CreateScope();
             var sp = scope.ServiceProvider;
             string reply = null;
-            //await Task.Delay(2000);
-            //reply = $"response for sequence number {message.SeqNumber}";
             var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);
             var conv = sp.GetRequiredService<IConversationService>();
             var routing = sp.GetRequiredService<IRoutingService>();
+            var config = sp.GetRequiredService<TwilioSetting>();
             routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId);
             conv.SetConversationId(message.SessionId, new List<MessageState>
             {
                 new MessageState("channel", ConversationChannel.Phone),
                 new MessageState("calling_phone", message.From)
             });
-            var result = await conv.SendMessage("2cd4b805-7078-4405-87e9-2ec9aadf8a11",
+            var result = await conv.SendMessage(config.AgentId,
                 inputMsg,
                 replyMessage: null,
                 async msg =>
@@ -82,7 +80,7 @@ private async Task ProcessUserMessageAsync(CallerMessage message)
             );
             if (string.IsNullOrWhiteSpace(reply))
             {
-                reply = "Bye.";
+                reply = "Sorry, something was wrong.";
             }
             var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();
             await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply);
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs
@@ -1,4 +1,3 @@
-using BotSharp.Plugin.Twilio.Settings;
 using Twilio.Jwt.AccessToken;
 using Token = Twilio.Jwt.AccessToken.Token;
 
@@ -74,6 +73,7 @@ public VoiceResponse ReturnInstructions(string speechPath, string callbackPath,
                 Gather.InputEnum.Speech
             },
             Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"),
+            SpeechTimeout = "3",
             ActionOnEmptyResult = actionOnEmptyResult
         };
         if (!string.IsNullOrEmpty(speechPath))
@@ -84,32 +84,12 @@ public VoiceResponse ReturnInstructions(string speechPath, string callbackPath,
         return response;
     }
 
-    public VoiceResponse DummyInstructions(string message, string callbackPath, bool actionOnEmptyResult)
+    public VoiceResponse HangUp(string speechPath)
     {
         var response = new VoiceResponse();
-        var gather = new Gather()
-        {
-            Input = new List<Gather.InputEnum>()
-            {
-                Gather.InputEnum.Speech
-            },
-            Action = new Uri($"{_settings.CallbackHost}/{callbackPath}"),
-            ActionOnEmptyResult = actionOnEmptyResult
-        };
-        if (!string.IsNullOrEmpty(message))
-        {
-            gather.Say(message);
-        }
-        response.Append(gather);
-        return response;
-    }
-
-    public VoiceResponse HangUp(string message)
-    {
-        var response = new VoiceResponse();
-        if (!string.IsNullOrEmpty(message))
+        if (!string.IsNullOrEmpty(speechPath))
         {
-            response.Say(message);
+            response.Play(new Uri($"{_settings.CallbackHost}/{speechPath}"));
         }
         response.Hangup();
         return response;
diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs
@@ -19,7 +19,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         });
 
         services.AddScoped<TwilioService>();
-        var conn = ConnectionMultiplexer.Connect("10.2.3.227");
+        var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]);
         var sessionManager = new TwilioSessionManager(conn);
         services.AddSingleton<ITwilioSessionManager>(sessionManager);
         services.AddSingleton<TwilioMessageQueue>();

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,7 @@`
`2`	`2`
`3`	`3`	`namespace BotSharp.Core.Files.Services`
`4`	`4`	`{`
`5`		`- public partial class BotSharpFileService`
	`5`	`+ public partial class LocalFileStorageService`
`6`	`6`	`{`
`7`	`7`	`public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data)`
`8`	`8`	`{`
Original file line number	Diff line number	Diff line change
`@@ -86,24 +86,24 @@ public async Task<TwiMLResult> ReceivedVoiceMessage([FromRoute] string agentId,`
`86`	`86`	`}`
`87`	`87`
`88`	`88`
`89`		`- [HttpPost("anonymous/start")]`
	`89`	`+ [HttpPost("start")]`
`90`	`90`	`public TwiMLResult InitiateConversation(VoiceRequest request)`
`91`	`91`	`{`
`92`	`92`	`if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid));`
`93`	`93`	`string sessionId = $"TwilioVoice_{request.CallSid}";`
`94`	`94`	`var twilio = _services.GetRequiredService<TwilioService>();`
`95`		`- var url = $"twiliovoice/anonymous/{sessionId}/send/0";`
`96`		`- var response = twilio.DummyInstructions("Hello, how may I help you?", url, false);`
	`95`	`+ var url = $"twiliovoice/{sessionId}/send/0";`
	`96`	`+ var response = twilio.ReturnInstructions("twilio/welcome.mp3", url, false);`
`97`	`97`	`return TwiML(response);`
`98`	`98`	`}`
`99`	`99`
`100`		`- [HttpPost("anonymous/{sessionId}/send/{seqNum}")]`
	`100`	`+ [HttpPost("{sessionId}/send/{seqNum}")]`
`101`	`101`	`public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)`
`102`	`102`	`{`
`103`	`103`	`var twilio = _services.GetRequiredService<TwilioService>();`
`104`	`104`	`var messageQueue = _services.GetRequiredService<TwilioMessageQueue>();`
`105`	`105`	`var sessionManager = _services.GetRequiredService<ITwilioSessionManager>();`
`106`		`- var url = $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}";`
	`106`	`+ var url = $"twiliovoice/{sessionId}/reply/{seqNum}";`
`107`	`107`	`var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(sessionId, seqNum);`
`108`	`108`	`if (!string.IsNullOrWhiteSpace(request.SpeechResult))`
`109`	`109`	`{`
`@@ -121,16 +121,16 @@ public async Task<TwiMLResult> SendCallerMessage([FromRoute] string sessionId, [`
`121`	`121`	`From = request.From`
`122`	`122`	`};`
`123`	`123`	`await messageQueue.EnqueueAsync(callerMessage);`
`124`		`- response = twilio.DummyInstructions("Please hold on and wait a moment.", url, true);`
	`124`	`+ response = twilio.ReturnInstructions("twilio/holdon.mp3", url, true);`
`125`	`125`	`}`
`126`	`126`	`else`
`127`	`127`	`{`
`128`		`- response = twilio.HangUp("Thanks for calling. Good bye.");`
	`128`	`+ response = twilio.HangUp("twilio/holdon.mp3");`
`129`	`129`	`}`
`130`	`130`	`return TwiML(response);`
`131`	`131`	`}`
`132`	`132`
`133`		`- [HttpPost("anonymous/{sessionId}/reply/{seqNum}")]`
	`133`	`+ [HttpPost("{sessionId}/reply/{seqNum}")]`
`134`	`134`	`public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId, [FromRoute] int seqNum, VoiceRequest request)`
`135`	`135`	`{`
`136`	`136`	`var nextSeqNum = seqNum + 1;`
`@@ -144,38 +144,28 @@ public async Task<TwiMLResult> ReplyCallerMessage([FromRoute] string sessionId,`
`144`	`144`	`VoiceResponse response;`
`145`	`145`	`if (string.IsNullOrEmpty(reply))`
`146`	`146`	`{`
`147`		`- response = twilio.ReturnInstructions(null, $"twiliovoice/anonymous/{sessionId}/reply/{seqNum}", true);`
	`147`	`+ response = twilio.ReturnInstructions(null, $"twiliovoice/{sessionId}/reply/{seqNum}", true);`
`148`	`148`	`}`
`149`	`149`	`else`
`150`	`150`	`{`
`151`	`151`
`152`	`152`	`var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");`
`153`		`- var fileService = _services.GetRequiredService<IBotSharpFileService>();`
	`153`	`+ var fileService = _services.GetRequiredService<IFileStorageService>();`
`154`	`154`	`var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply);`
`155`	`155`	`var fileName = $"{seqNum}.mp3";`
`156`	`156`	`await fileService.SaveSpeechFileAsync(sessionId, fileName, data);`
`157`		`- response = twilio.ReturnInstructions($"twiliovoice/anonymous/speeches/{sessionId}/{fileName}", $"twiliovoice/anonymous/{sessionId}/send/{nextSeqNum}", true);`
	`157`	`+ response = twilio.ReturnInstructions($"twiliovoice/speeches/{sessionId}/{fileName}", $"twiliovoice/{sessionId}/send/{nextSeqNum}", true);`
`158`	`158`	`}`
`159`	`159`	`return TwiML(response);`
`160`	`160`	`}`
`161`	`161`
`162`		`- [HttpGet("anonymous/speeches/{conversationId}/{fileName}")]`
	`162`	`+ [HttpGet("speeches/{conversationId}/{fileName}")]`
`163`	`163`	`public async Task<FileContentResult> RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName)`
`164`	`164`	`{`
`165`		`- var fileService = _services.GetRequiredService<IBotSharpFileService>();`
	`165`	`+ var fileService = _services.GetRequiredService<IFileStorageService>();`
`166`	`166`	`var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName);`
`167`		`- var result = new FileContentResult(data.ToArray(), "application/octet-stream");`
	`167`	`+ var result = new FileContentResult(data.ToArray(), "audio/mpeg");`
`168`	`168`	`result.FileDownloadName = fileName;`
`169`	`169`	`return result;`
`170`	`170`	`}`
`171`		`-`
`172`		`- [HttpGet("anonymous/text-to-speech")]`
`173`		`- public async Task<IActionResult> TextToSpeech([FromQuery] string text)`
`174`		`- {`
`175`		`- var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1");`
`176`		`- var data = await textToSpeechService.GenerateSpeechFromTextAsync(text);`
`177`		`- var fileService = _services.GetRequiredService<IBotSharpFileService>();`
`178`		`- await fileService.SaveSpeechFileAsync("123", "sample.mp3", data);`
`179`		`- return Ok();`
`180`		`- }`
`181`	`171`	`}`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ public class CallerMessage`
`9`	`9`
`10`	`10`	`public override string ToString()`
`11`	`11`	`{`
`12`		`- return $"({SessionId}-{SeqNumber}) {Content}";`
	`12`	`+ return $"{SessionId}-{SeqNumber}";`
`13`	`13`	`}`
`14`	`14`	`}`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@ public async ValueTask EnqueueAsync(CallerMessage request)`
`25`	`25`
`26`	`26`	`internal void Stop()`
`27`	`27`	`{`
	`28`	`+ Console.WriteLine($"[{DateTime.UtcNow}] Complete queue");`
`28`	`29`	`_queue.Writer.TryComplete();`
`29`	`30`	`}`
`30`	`31`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`using BotSharp.Abstraction.Routing;`
`2`	`2`	`using BotSharp.Plugin.Twilio.Models;`
`3`	`3`	`using Microsoft.Extensions.Hosting;`
`4`		`-using System.Security.Cryptography;`
`5`	`4`	`using System.Threading;`
`6`	`5`	`using Task = System.Threading.Tasks.Task;`
`7`	`6`
`@@ -31,12 +30,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)`
`31`	`30`	`{`
`32`	`31`	`try`
`33`	`32`	`{`
`34`		`- Console.WriteLine("Processing {message}.", message);`
	`33`	`+ Console.WriteLine($"Start processing {message}.");`
`35`	`34`	`await ProcessUserMessageAsync(message);`
`36`	`35`	`}`
`37`	`36`	`catch (Exception ex)`
`38`	`37`	`{`
`39`		`- Console.WriteLine("Processing {message} failed due to {ex}.", message, ex.Message);`
	`38`	`+ Console.WriteLine($"Processing {message} failed due to {ex.Message}.");`
`40`	`39`	`}`
`41`	`40`	`finally`
`42`	`41`	`{`
`@@ -57,18 +56,17 @@ private async Task ProcessUserMessageAsync(CallerMessage message)`
`57`	`56`	`using var scope = _serviceProvider.CreateScope();`
`58`	`57`	`var sp = scope.ServiceProvider;`
`59`	`58`	`string reply = null;`
`60`		`- //await Task.Delay(2000);`
`61`		`- //reply = $"response for sequence number {message.SeqNumber}";`
`62`	`59`	`var inputMsg = new RoleDialogModel(AgentRole.User, message.Content);`
`63`	`60`	`var conv = sp.GetRequiredService<IConversationService>();`
`64`	`61`	`var routing = sp.GetRequiredService<IRoutingService>();`
	`62`	`+ var config = sp.GetRequiredService<TwilioSetting>();`
`65`	`63`	`routing.Context.SetMessageId(message.SessionId, inputMsg.MessageId);`
`66`	`64`	`conv.SetConversationId(message.SessionId, new List<MessageState>`
`67`	`65`	`{`
`68`	`66`	`new MessageState("channel", ConversationChannel.Phone),`
`69`	`67`	`new MessageState("calling_phone", message.From)`
`70`	`68`	`});`
`71`		`- var result = await conv.SendMessage("2cd4b805-7078-4405-87e9-2ec9aadf8a11",`
	`69`	`+ var result = await conv.SendMessage(config.AgentId,`
`72`	`70`	`inputMsg,`
`73`	`71`	`replyMessage: null,`
`74`	`72`	`async msg =>`
`@@ -82,7 +80,7 @@ private async Task ProcessUserMessageAsync(CallerMessage message)`
`82`	`80`	`);`
`83`	`81`	`if (string.IsNullOrWhiteSpace(reply))`
`84`	`82`	`{`
`85`		`- reply = "Bye.";`
	`83`	`+ reply = "Sorry, something was wrong.";`
`86`	`84`	`}`
`87`	`85`	`var sessionManager = sp.GetRequiredService<ITwilioSessionManager>();`
`88`	`86`	`await sessionManager.SetAssistantReplyAsync(message.SessionId, message.SeqNumber, reply);`