diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs index 2ad8f1dfb..c7409f595 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHook.cs @@ -1,6 +1,10 @@ +using BotSharp.Abstraction.MLTasks; + namespace BotSharp.Abstraction.Realtime; public interface IRealtimeHook { + Task OnModeReady(Agent agent, IRealTimeCompletion completer); string[] OnModelTranscriptPrompt(Agent agent); + Task OnTranscribeCompleted(RoleDialogModel message, TranscriptionData data); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs new file mode 100644 index 000000000..c76be9cd7 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Realtime/Models/TranscriptionData.cs @@ -0,0 +1,6 @@ +public class TranscriptionData +{ + public string Transcript { get; set; } = null!; + public float Confidence { get; set; } + public string Language { get; set; } = null!; +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs index b334b8734..4879eb541 100644 --- a/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs +++ b/src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Infrastructures; + namespace BotSharp.Core.Realtime.Services; public class RealtimeHub : IRealtimeHub @@ -70,8 +72,8 @@ private async Task ConnectToModel(WebSocket userWebSocket) _conn.CurrentAgentId = agent.Id; // Set model - var model = agent.LlmConfig.Model; - if (!model.Contains("-realtime-")) + var model = "gpt-4o-mini-realtime"; + if (agent.Profiles.Contains("realtime")) { var llmProviderService = _services.GetRequiredService(); model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name; @@ -85,14 +87,14 @@ private async Task ConnectToModel(WebSocket userWebSocket) var storage = _services.GetRequiredService(); var dialogs = convService.GetDialogHistory(); - if (dialogs.Count == 0) + /*if (dialogs.Count == 0) { dialogs.Add(new RoleDialogModel(AgentRole.User, "Hi")); storage.Append(_conn.ConversationId, dialogs.First()); - } + }*/ routing.Context.SetDialogs(dialogs); - routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); + // routing.Context.SetMessageId(_conn.ConversationId, dialogs.Last().MessageId); var states = _services.GetRequiredService(); @@ -102,29 +104,7 @@ await _completer.Connect(_conn, // Not TriggerModelInference, waiting for user utter. var instruction = await _completer.UpdateSession(_conn); - // Trigger model inference if there is no audio file in the conversation - if (!states.ContainsState("init_audio_file")) - { - if (dialogs.LastOrDefault()?.Role == AgentRole.Assistant) - { - await _completer.TriggerModelInference($"Rephase your last response:\r\n{dialogs.LastOrDefault()?.Content}"); - } - else - { - await _completer.TriggerModelInference("Reply based on the conversation context."); - } - } - else - { - // Append dialogs into model context - var history = "[CONVERSATION HISTORY]\r\n"; - foreach (var message in dialogs) - { - history += $"{message.Role}: {message.Content}\r\n"; - } - - await _completer.TriggerModelInference($"{instruction}\r\n\r\n{history}\r\n\r\nAssist user without repeating your previous statement."); - } + await HookEmitter.Emit(_services, async hook => await hook.OnModeReady(agent, _completer)); }, onModelAudioDeltaReceived: async (audioDeltaData, itemId) => { diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs index 092273fa0..4cac4c868 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.GetAgents.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Agents.Services; public partial class AgentService { #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task> GetAgents(AgentFilter filter) { @@ -28,7 +28,7 @@ public async Task> GetAgents(AgentFilter filter) } #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task> GetAgentOptions(List? agentIds) { @@ -40,7 +40,7 @@ public async Task> GetAgentOptions(List? agentIds) } #if !DEBUG - [SharpCache(10, perInstanceCache: true)] + [SharpCache(10)] #endif public async Task GetAgent(string id) { diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs new file mode 100644 index 000000000..a3a11c8ce --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioInboundController.cs @@ -0,0 +1,186 @@ +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Infrastructures.Enums; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using BotSharp.Plugin.Twilio.Services; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Twilio.Http; +using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; +using Task = System.Threading.Tasks.Task; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioInboundController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly IHttpContextAccessor _context; + private readonly ILogger _logger; + + public TwilioInboundController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _context = context; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/inbound")] + public async Task InitiateStreamConversation(ConversationalVoiceRequest request) + { + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + + var twilio = _services.GetRequiredService(); + VoiceResponse response = default!; + + var instruction = new ConversationalVoiceResponse + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SpeechPaths = [], + ActionOnEmptyResult = true, + }; + + if (request.InitAudioFile != null) + { + instruction.SpeechPaths.Add(request.InitAudioFile); + } + + // Load agent profile + var agentService = _services.GetRequiredService(); + var agent = await agentService.LoadAgent(request.AgentId); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreating(request, instruction); + }); + + request.ConversationId = await InitConversation(request, agent); + instruction.AgentId = request.AgentId; + instruction.ConversationId = request.ConversationId; + + if (request.AnsweredBy == "machine_start" && + request.Direction == "outbound-api") + { + response = new VoiceResponse(); + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnVoicemailStarting(request); + }); + + var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); + response.Play(new Uri(url)); + } + else + { + if (agent.Profiles.Contains("realtime")) + { + response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction, agent); + } + else + { + if (string.IsNullOrWhiteSpace(request.Intent)) + { + instruction.CallbackPath = $"twilio/voice/receive/0?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"; + response = twilio.ReturnNoninterruptedInstructions(instruction); + } + else + { + int seqNum = 0; + var messageQueue = _services.GetRequiredService(); + var sessionManager = _services.GetRequiredService(); + await sessionManager.StageCallerMessageAsync(request.ConversationId, seqNum, request.Intent); + var callerMessage = new CallerMessage() + { + AgentId = request.AgentId, + ConversationId = request.ConversationId, + SeqNumber = seqNum, + Content = request.Intent, + From = request.From, + States = ParseStates(request.States) + }; + await messageQueue.EnqueueAsync(callerMessage); + response = new VoiceResponse(); + // delay 3 seconds to wait for the first message reply and caller is listening dudu sound + await Task.Delay(1000 * 3); + response.Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/reply/{seqNum}?agent-id={request.AgentId}&conversation-id={request.ConversationId}&{twilio.GenerateStatesParameter(request.States)}"), HttpMethod.Post); + } + } + } + + await HookEmitter.Emit(_services, async hook => + { + await hook.OnSessionCreated(request); + }); + + return TwiML(response); + } + + protected Dictionary ParseStates(List states) + { + var result = new Dictionary(); + if (states is null || !states.Any()) + { + return result; + } + foreach (var kvp in states) + { + var parts = kvp.Split(':', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 2) + { + result.Add(parts[0], parts[1]); + } + } + return result; + } + + private async Task InitConversation(ConversationalVoiceRequest request, Agent agent) + { + var convService = _services.GetRequiredService(); + var conversation = await convService.GetConversation(request.ConversationId); + if (conversation == null) + { + var conv = new Conversation + { + AgentId = request.AgentId, + Channel = ConversationChannel.Phone, + ChannelId = request.CallSid, + Title = $"Incoming phone call from {request.From}", + Tags = [], + }; + + conversation = await convService.NewConversation(conv); + } + + var states = new List + { + new("channel", ConversationChannel.Phone), + new("calling_phone", request.From), + new("phone_direction", request.Direction), + new("twilio_call_sid", request.CallSid), + }; + + // Enable lazy routing mode to optimize realtime experience + if (agent.Profiles.Contains("realtime") && agent.Type == AgentType.Routing) + { + states.Add(new(StateConst.ROUTING_MODE, "lazy")); + } + + if (request.InitAudioFile != null) + { + states.Add(new("init_audio_file", request.InitAudioFile)); + } + + convService.SetConversationId(conversation.Id, states); + convService.SaveStates(); + + return conversation.Id; + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs index e8f055fec..65b2a5adf 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioRecordController.cs @@ -1,12 +1,8 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.Models; -using Microsoft.AspNetCore.Cors.Infrastructure; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Newtonsoft.Json; -using System.IO; namespace BotSharp.Plugin.Twilio.Controllers; @@ -45,31 +41,4 @@ public async Task PhoneRecordingStatus(ConversationalVoiceRequest return Ok(); } - - [ValidateRequest] - [HttpPost("twilio/record/transcribe")] - public async Task PhoneRecordingTranscribe(ConversationalVoiceRequest request) - { - if (request.Final == "true") - { - _logger.LogError($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}"); - - // transcription completed - await HookEmitter.Emit(_services, x => x.OnTranscribeCompleted(request)); - - // Append the transcription to the dialog history - var transcript = JsonConvert.DeserializeObject(request.TranscriptionData); - if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript)) - { - var storage = _services.GetRequiredService(); - var message = new RoleDialogModel(AgentRole.User, transcript.Transcript) - { - CurrentAgentId = request.AgentId - }; - storage.Append(request.ConversationId, message); - } - } - - return Ok(); - } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs deleted file mode 100644 index 717f4fa66..000000000 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioStreamController.cs +++ /dev/null @@ -1,125 +0,0 @@ -using BotSharp.Abstraction.Infrastructures; -using BotSharp.Abstraction.Infrastructures.Enums; -using BotSharp.Core.Infrastructures; -using BotSharp.Plugin.Twilio.Interfaces; -using BotSharp.Plugin.Twilio.Models; -using BotSharp.Plugin.Twilio.Services; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Conversation = BotSharp.Abstraction.Conversations.Models.Conversation; - -namespace BotSharp.Plugin.Twilio.Controllers; - -public class TwilioStreamController : TwilioController -{ - private readonly TwilioSetting _settings; - private readonly IServiceProvider _services; - private readonly IHttpContextAccessor _context; - private readonly ILogger _logger; - - public TwilioStreamController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) - { - _settings = settings; - _services = services; - _context = context; - _logger = logger; - } - - [ValidateRequest] - [HttpPost("twilio/stream")] - public async Task InitiateStreamConversation(ConversationalVoiceRequest request) - { - var text = JsonSerializer.Serialize(request); - if (request?.CallSid == null) - { - throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); - } - - var twilio = _services.GetRequiredService(); - VoiceResponse response = default!; - - var instruction = new ConversationalVoiceResponse - { - ConversationId = request.ConversationId, - SpeechPaths = [], - ActionOnEmptyResult = true - }; - - if (request.InitAudioFile != null) - { - instruction.SpeechPaths.Add(request.InitAudioFile); - } - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnSessionCreating(request, instruction); - }); - - request.ConversationId = await InitConversation(request); - instruction.AgentId = request.AgentId; - instruction.ConversationId = request.ConversationId; - - if (request.AnsweredBy == "machine_start" && - request.Direction == "outbound-api") - { - response = new VoiceResponse(); - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnVoicemailStarting(request); - }); - - var url = twilio.GetSpeechPath(request.ConversationId, "voicemail.mp3"); - response.Play(new Uri(url)); - } - else - { - response = twilio.ReturnBidirectionalMediaStreamsInstructions(instruction); - } - - await HookEmitter.Emit(_services, async hook => - { - await hook.OnSessionCreated(request); - }); - - return TwiML(response); - } - - private async Task InitConversation(ConversationalVoiceRequest request) - { - var convService = _services.GetRequiredService(); - var conversation = await convService.GetConversation(request.ConversationId); - if (conversation == null) - { - var conv = new Conversation - { - AgentId = request.AgentId, - Channel = ConversationChannel.Phone, - ChannelId = request.CallSid, - Title = $"Incoming phone call from {request.From}", - Tags = [], - }; - - conversation = await convService.NewConversation(conv); - } - - var states = new List - { - new("channel", ConversationChannel.Phone), - new("calling_phone", request.From), - new("twilio_call_sid", request.CallSid), - // Enable lazy routing mode to optimize realtime experience - new(StateConst.ROUTING_MODE, "lazy"), - }; - - if (request.InitAudioFile != null) - { - states.Add(new("init_audio_file", request.InitAudioFile)); - } - - convService.SetConversationId(conversation.Id, states); - convService.SaveStates(); - - return conversation.Id; - } -} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs new file mode 100644 index 000000000..1fa020c00 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioTranscribeController.cs @@ -0,0 +1,58 @@ +using BotSharp.Abstraction.Realtime; +using BotSharp.Abstraction.Routing; +using BotSharp.Core.Infrastructures; +using BotSharp.Plugin.Twilio.Interfaces; +using BotSharp.Plugin.Twilio.Models; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Newtonsoft.Json; + +namespace BotSharp.Plugin.Twilio.Controllers; + +public class TwilioTranscribeController : TwilioController +{ + private readonly TwilioSetting _settings; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public TwilioTranscribeController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) + { + _settings = settings; + _services = services; + _logger = logger; + } + + [ValidateRequest] + [HttpPost("twilio/transcribe")] + public async Task PhoneRecordingTranscribe(ConversationalVoiceRequest request) + { + if (request.Final == "true") + { + _logger.LogInformation($"Transcription completed for {request.CallSid}, the transcription is: {request.TranscriptionData}"); + + // Append the transcription to the dialog history + var transcript = JsonConvert.DeserializeObject(request.TranscriptionData); + if (transcript != null && !string.IsNullOrEmpty(transcript.Transcript)) + { + var storage = _services.GetRequiredService(); + var message = new RoleDialogModel(AgentRole.User, transcript.Transcript) + { + CurrentAgentId = request.AgentId + }; + storage.Append(request.ConversationId, message); + + var routing = _services.GetRequiredService(); + routing.Context.SetMessageId(request.ConversationId, message.MessageId); + + var convService = _services.GetRequiredService(); + convService.SetConversationId(request.ConversationId, []); + + // transcription completed + transcript.Language = request.LanguageCode; + await HookEmitter.Emit(_services, async x => await x.OnTranscribeCompleted(message, transcript)); + } + } + + return Ok(); + } +} diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 6078dd7bc..8f042ca44 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -12,10 +12,10 @@ namespace BotSharp.Plugin.Twilio.Controllers; public class TwilioVoiceController : TwilioController { - private readonly TwilioSetting _settings; - private readonly IServiceProvider _services; - private readonly IHttpContextAccessor _context; - private readonly ILogger _logger; + protected readonly TwilioSetting _settings; + protected readonly IServiceProvider _services; + protected readonly IHttpContextAccessor _context; + protected readonly ILogger _logger; public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, IHttpContextAccessor context, ILogger logger) { @@ -32,6 +32,7 @@ public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, /// /// /// + [Obsolete("Use twilio/inbound for streaming and non-streaming instead of this endpoint")] [ValidateRequest] [HttpPost("twilio/voice/welcome")] public async Task InitiateConversation(ConversationalVoiceRequest request) @@ -352,11 +353,15 @@ await HookEmitter.Emit(_services, async hook => { await HookEmitter.Emit(_services, x => x.OnCallBusyStatus(request)); } + else if (request.CallStatus == "no-answer") + { + await HookEmitter.Emit(_services, x => x.OnCallNoAnswerStatus(request)); + } return Ok(); } - private Dictionary ParseStates(List states) + protected Dictionary ParseStates(List states) { var result = new Dictionary(); if (states is null || !states.Any()) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs index 899cc5260..747679cc2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Interfaces/ITwilioCallStatusHook.cs @@ -8,7 +8,6 @@ public interface ITwilioCallStatusHook Task OnVoicemailLeft(ConversationalVoiceRequest request); Task OnUserDisconnected(ConversationalVoiceRequest request); Task OnRecordingCompleted(ConversationalVoiceRequest request); - Task OnTranscribeCompleted(ConversationalVoiceRequest request); Task OnVoicemailStarting(ConversationalVoiceRequest request); /// @@ -19,4 +18,6 @@ public interface ITwilioCallStatusHook /// /// Task OnCallBusyStatus(ConversationalVoiceRequest request); + + Task OnCallNoAnswerStatus(ConversationalVoiceRequest request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs index d9dde562a..df08df0a0 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Models/ConversationalVoiceRequest.cs @@ -64,9 +64,4 @@ public class ConversationalVoiceRequest : VoiceRequest [FromForm] public string? TranscriptionEvent { get; set; } #endregion -} - -public class TranscriptionData -{ - public string Transcript { get; set; } = null!; } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs index afe6b89c6..6d1e2d119 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/OutboundPhoneCallHandler/Functions/OutboundPhoneCallFn.cs @@ -77,8 +77,12 @@ public async Task Execute(RoleDialogModel message) statusUrl += $"&init-audio-file={initAudioFile}"; } + // load agent profile + var agentService = _services.GetRequiredService(); + var agent = await agentService.GetAgent(message.CurrentAgentId); + // Set up process URL streaming or synchronous - if (_twilioSetting.StreamingEnabled) + if (agent.Profiles.Contains("realtime")) { processUrl += "/stream"; } @@ -119,7 +123,7 @@ public async Task Execute(RoleDialogModel message) await ForkConversation(args, entryAgentId, originConversationId, newConversationId, call); - message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, STREAMING: {_twilioSetting.StreamingEnabled}, RECORDING: {_twilioSetting.RecordingEnabled}]"; + message.Content = $"The generated phone initial message: \"{args.InitialMessage}.\" [NEW CONVERSATION ID: {newConversationId}, TWILIO CALL SID: {call.Sid}, RECORDING: {_twilioSetting.RecordingEnabled}]"; message.StopCompletion = true; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index fda279547..fb637677f 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,4 +1,6 @@ +using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Files; +using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Utilities; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Interfaces; @@ -199,7 +201,7 @@ public VoiceResponse HoldOn(int interval, string message = null) /// /// /// - public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse) + public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalVoiceResponse conversationalVoiceResponse, Agent agent) { var response = new VoiceResponse(); @@ -207,11 +209,16 @@ public VoiceResponse ReturnBidirectionalMediaStreamsInstructions(ConversationalV if (_settings.TranscribeEnabled) { + var words = new List(); + HookEmitter.Emit(_services, hook => words.AddRange(hook.OnModelTranscriptPrompt(agent))); + var hints = string.Join(", ", words); var start = new Start(); start.Transcription( track: "inbound_track", partialResults: false, - statusCallbackUrl: $"{_settings.CallbackHost}/twilio/record/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", name: conversationId); + statusCallbackUrl: $"{_settings.CallbackHost}/twilio/transcribe?agent-id={conversationalVoiceResponse.AgentId}&conversation-id={conversationId}", + name: conversationId, + hints: hints); response.Append(start); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs index f0d845dc0..a7cf6f9ee 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Settings/TwilioSetting.cs @@ -7,10 +7,6 @@ public class TwilioSetting /// public string? PhoneNumber { get; set; } - /// - /// Enable streaming for outbound phone call - /// - public bool StreamingEnabled { get; set; } = false; public string AccountSID { get; set; } public string AuthToken { get; set; } public string AppSID { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index a31dbb027..d78489ad7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -1,9 +1,7 @@ -using BotSharp.Abstraction.Realtime; using BotSharp.Abstraction.Settings; using BotSharp.Plugin.Twilio.Interfaces; using BotSharp.Plugin.Twilio.OutboundPhoneCallHandler.Hooks; using BotSharp.Plugin.Twilio.Services; -using BotSharp.Plugin.Twilio.Services.Stream; using StackExchange.Redis; using Twilio; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs similarity index 99% rename from src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs rename to src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs index 35118ab30..7ff5ab98d 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/Stream/TwilioStreamMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioStreamMiddleware.cs @@ -6,7 +6,7 @@ using System.Net.WebSockets; using Task = System.Threading.Tasks.Task; -namespace BotSharp.Plugin.Twilio.Services.Stream; +namespace BotSharp.Plugin.Twilio; /// /// Reference to https://github.com/twilio-samples/speech-assistant-openai-realtime-api-node/blob/main/index.js