Test/realtime test

Directory.Packages.props

@@ -111,10 +111,10 @@
     <PackageVersion Include="MSTest.TestAdapter" Version="3.1.1" />
     <PackageVersion Include="MSTest.TestFramework" Version="3.1.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
-    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />  
+    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
     <PackageVersion Include="Shouldly" Version="4.3.0" />
-    <PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.5" /> 
-    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.5" />
+    <PackageVersion Include="ModelContextProtocol" Version="0.1.0-preview.8" />
+    <PackageVersion Include="ModelContextProtocol.AspNetCore" Version="0.1.0-preview.8" />
   </ItemGroup>
   <ItemGroup>
     <PackageVersion Include="BotSharp.Core" Version="$(BotSharpVersion)" />

src/Infrastructure/BotSharp.Abstraction/Functions/IFunctionCallback.cs

@@ -2,6 +2,7 @@ namespace BotSharp.Abstraction.Functions;
 
 public interface IFunctionCallback
 {
+    string Provider => "Botsharp";
     string Name { get; }
 
     /// <summary>

src/Infrastructure/BotSharp.Abstraction/MCP/Models/McpServerConfigModel.cs

@@ -12,22 +12,21 @@ public class McpServerConfigModel
     /// </summary>
     public string Name { get; set; } = null!;
 
-    /// <summary>
-    /// The type of transport to use.
-    /// </summary>
-    [JsonPropertyName("transport_type")]
-    public string TransportType { get; set; } = null!;
-
-    /// <summary>
-    /// For stdio transport: path to the executable
-    /// For HTTP transport: base URL of the server
-    /// </summary>
-    public string? Location { get; set; }
+    public McpSseServerConfig? SseConfig { get; set; }
+    public McpStdioServerConfig? StdioConfig { get; set; }
+}
 
-    /// <summary>
-    /// Additional transport-specific configuration.
-    /// </summary>
-    [JsonPropertyName("transport_options")]
-    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-    public Dictionary<string, string>? TransportOptions { get; set; }
+public class McpSseServerConfig
+{
+    public string EndPoint { get; set; } = null!;
+    public TimeSpan ConnectionTimeout { get; init; } = TimeSpan.FromSeconds(30);
+    public Dictionary<string, string>? AdditionalHeaders { get; set; }
 }
+
+public class McpStdioServerConfig
+{
+    public string Command { get; set; } = null!;
+    public IList<string>? Arguments { get; set; }
+    public Dictionary<string, string>? EnvironmentVariables { get; set; }
+    public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5);
+}

src/Infrastructure/BotSharp.Abstraction/MCP/Models/McpServerOptionModel.cs

@@ -0,0 +1,19 @@
+namespace BotSharp.Abstraction.MCP.Models;
+
+public class McpServerOptionModel : IdName
+{
+    public IEnumerable<string> Tools { get; set; } = [];
+
+    public McpServerOptionModel() : base()
+    {
+
+    }
+
+    public McpServerOptionModel(
+        string id,
+        string name,
+        IEnumerable<string> tools) : base(id, name)
+    {
+        Tools = tools ?? [];
+    }
+}

src/Infrastructure/BotSharp.Abstraction/MCP/Services/IMcpService.cs

@@ -2,5 +2,5 @@ namespace BotSharp.Abstraction.MCP.Services;
 
 public interface IMcpService
 {
-    IEnumerable<McpServerConfigModel> GetServerConfigs() => [];
+    IEnumerable<McpServerOptionModel> GetServerConfigs() => [];
 }

src/Infrastructure/BotSharp.Abstraction/Models/IdName.cs

@@ -8,6 +8,11 @@ public class IdName
     [JsonPropertyName("name")]
     public string Name { get; set; } = default!;
 
+    public IdName()
+    {
+
+    }
+
     public IdName(string id, string name)
     {
         Id = id;

src/Infrastructure/BotSharp.Abstraction/Models/MessageState.cs

@@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Models;
 public class MessageState
 {
     public string Key { get; set; }
-    public string Value { get; set; }
+    public object Value { get; set; }
 
     [JsonPropertyName("active_rounds")]
     public int ActiveRounds { get; set; } = -1;
@@ -13,7 +13,7 @@ public MessageState()
 
     }
 
-    public MessageState(string key, string value, int activeRounds = -1)
+    public MessageState(string key, object value, int activeRounds = -1)
     {
         Key = key;
         Value = value;

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -13,6 +13,7 @@ public class RealtimeHubConnection
     public Func<string, string> OnModelMessageReceived { get; set; } = null!;
     public Func<string> OnModelAudioResponseDone { get; set; } = null!;
     public Func<string> OnModelUserInterrupted { get; set; } = null!;
+    public Func<string> OnUserSpeechDetected { get; set; } = () => string.Empty;
 
     public void ResetResponseState()
     {

src/Infrastructure/BotSharp.Abstraction/Utilities/StringExtensions.cs

@@ -108,4 +108,27 @@ public static bool IsPrimitiveValue(this string value)
                uint.TryParse(value, out _) ||
                ulong.TryParse(value, out _);
     }
+
+
+    public static string ConvertToString<T>(this T? value, JsonSerializerOptions? jsonOptions = null)
+    {
+        if (value == null)
+        {
+            return string.Empty;
+        }
+
+        if (value is string s)
+        {
+            return s;
+        }
+
+        if (value is JsonElement elem
+            && elem.ValueKind == JsonValueKind.String)
+        {
+            return elem.ToString();
+        }
+
+        var str = JsonSerializer.Serialize(value, jsonOptions);
+        return str;
+    }
 }

src/Infrastructure/BotSharp.Core.MCP/BotSharpMCPExtensions.cs

@@ -4,7 +4,6 @@
 using BotSharp.Core.MCP.Services;
 using BotSharp.Core.MCP.Settings;
 using Microsoft.Extensions.Configuration;
-using ModelContextProtocol;
 using ModelContextProtocol.Client;
 
 namespace BotSharp.Core.MCP;
@@ -21,7 +20,7 @@ public static IServiceCollection AddBotSharpMCP(this IServiceCollection services
     {
         services.AddScoped<IMcpService, McpService>();
         var settings = config.GetSection("MCP").Get<McpSettings>();
-        services.AddScoped(provider => { return settings; });
+        services.AddScoped(provider => settings);
 
         if (settings != null && settings.Enabled && !settings.McpServerConfigs.IsNullOrEmpty())
         {
@@ -42,18 +41,18 @@ public static IServiceCollection AddBotSharpMCP(this IServiceCollection services
         return services;
     }
 
-    private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfig server, McpClientManager clientManager)
+    private static async Task RegisterFunctionCall(IServiceCollection services, McpServerConfigModel server, McpClientManager clientManager)
     {
         var client = await clientManager.GetMcpClientAsync(server.Id);
         var tools = await client.ListToolsAsync();
 
         foreach (var tool in tools)
         {
-            services.AddScoped(provider => { return tool; });
+            services.AddScoped(provider => tool);
 
             services.AddScoped<IFunctionCallback>(provider =>
             {
-                var funcTool = new McpToolAdapter(provider, tool, clientManager);
+                var funcTool = new McpToolAdapter(provider, server.Name, tool, clientManager);
                 return funcTool;
             });
         }

src/Infrastructure/BotSharp.Core.MCP/Functions/McpToolAdapter.cs

@@ -6,17 +6,24 @@ namespace BotSharp.Core.MCP.Functions;
 
 public class McpToolAdapter : IFunctionCallback
 {
+    private readonly string _provider;
     private readonly McpClientTool _tool;
     private readonly McpClientManager _clientManager;
     private readonly IServiceProvider _services;
 
-    public McpToolAdapter(IServiceProvider services, McpClientTool tool, McpClientManager client)
+    public McpToolAdapter(
+        IServiceProvider services,
+        string serverName,
+        McpClientTool tool,
+        McpClientManager client)
     {
         _services = services ?? throw new ArgumentNullException(nameof(services));
         _tool = tool ?? throw new ArgumentNullException(nameof(tool));
         _clientManager = client ?? throw new ArgumentNullException(nameof(client));
+        _provider = serverName;
     }
 
+    public string Provider => _provider;
     public string Name => _tool.Name;
 
     public async Task<bool> Execute(RoleDialogModel message)

src/Infrastructure/BotSharp.Core.MCP/Managers/McpClientManager.cs

@@ -1,5 +1,6 @@
 using BotSharp.Core.MCP.Settings;
 using ModelContextProtocol.Client;
+using ModelContextProtocol.Protocol.Transport;
 
 namespace BotSharp.Core.MCP.Managers;
 
@@ -14,9 +15,33 @@ public McpClientManager(McpSettings mcpSettings)
 
     public async Task<IMcpClient> GetMcpClientAsync(string serverId)
     {
-        return await McpClientFactory.CreateAsync(
-            _mcpSettings.McpServerConfigs.Where(x=> x.Name == serverId).First(),
-            _mcpSettings.McpClientOptions);
+        var config = _mcpSettings.McpServerConfigs.Where(x => x.Id == serverId).FirstOrDefault();
+
+        IClientTransport transport;
+        if (config.SseConfig != null)
+        {
+            transport = new SseClientTransport(new SseClientTransportOptions
+            {
+                Name = config.Name,
+                Endpoint = new Uri(config.SseConfig.EndPoint)
+            });
+        }
+        else if (config.StdioConfig != null)
+        {
+            transport = new StdioClientTransport(new StdioClientTransportOptions
+            {
+                Name = config.Name,
+                Command = config.StdioConfig.Command,
+                Arguments = config.StdioConfig.Arguments,
+                EnvironmentVariables = config.StdioConfig.EnvironmentVariables
+            });
+        }
+        else
+        {
+            throw new ArgumentNullException("Invalid MCP server configuration!");
+        }
+
+        return await McpClientFactory.CreateAsync(transport, _mcpSettings.McpClientOptions);
     }
 
     public void Dispose()

src/Infrastructure/BotSharp.Core.MCP/Services/McpService.cs

@@ -16,17 +16,26 @@ public McpService(
         _logger = logger;
     }
 
-    public IEnumerable<McpServerConfigModel> GetServerConfigs()
+    public IEnumerable<McpServerOptionModel> GetServerConfigs()
     {
+        var options = new List<McpServerOptionModel>();
         var settings = _services.GetRequiredService<McpSettings>();
         var configs = settings?.McpServerConfigs ?? [];
-        return configs.Select(x => new McpServerConfigModel
+
+        foreach (var config in configs)
         {
-            Id = x.Id,
-            Name = x.Name,
-            TransportType = x.TransportType,
-            TransportOptions = x.TransportOptions,
-            Location = x.Location
-        });
+            var tools = _services.GetServices<IFunctionCallback>()
+                                 .Where(x => x.Provider == config.Name)
+                                 .Select(x => x.Name);
+
+            options.Add(new McpServerOptionModel
+            {
+                Id = config.Id,
+                Name = config.Name,
+                Tools = tools
+            });
+        }
+
+        return options;
     }
 }

src/Infrastructure/BotSharp.Core.MCP/Settings/MCPSettings.cs

@@ -1,12 +1,11 @@
 using ModelContextProtocol.Client;
-using ModelContextProtocol;
 
 namespace BotSharp.Core.MCP.Settings;
 
 public class McpSettings
 {
     public bool Enabled { get; set; } = true;
     public McpClientOptions McpClientOptions { get; set; }
-    public List<McpServerConfig> McpServerConfigs { get; set; } = new();
+    public List<McpServerConfigModel> McpServerConfigs { get; set; } = [];
 
 }

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -150,6 +150,9 @@ await _completer.Connect(_conn,
                     var data = _conn.OnModelUserInterrupted();
                     await (responseToUser?.Invoke(data) ?? Task.CompletedTask);
                 }
+
+                var res = _conn.OnUserSpeechDetected();
+                await (responseToUser?.Invoke(res) ?? Task.CompletedTask);
             });
     }
 

src/Infrastructure/BotSharp.Core.Realtime/Services/WaveStreamChannel.cs

@@ -38,10 +38,13 @@ public async Task ConnectAsync(string conversationId)
         // Initialize audio output for streaming
         var waveFormat = new WaveFormat(24000, 16, 1); // 24000 Hz, 16-bit PCM, Mono
         _bufferedWaveProvider = new BufferedWaveProvider(waveFormat);
-        _bufferedWaveProvider.BufferLength = 1024 * 1024; // Buffer length
+        _bufferedWaveProvider.BufferDuration = TimeSpan.FromMinutes(10);
         _bufferedWaveProvider.DiscardOnBufferOverflow = true;
-
-        _waveOut = new WaveOutEvent();
+
+        _waveOut = new WaveOutEvent()
+        {
+            DeviceNumber = 0
+        };
         _waveOut.Init(_bufferedWaveProvider);
         _waveOut.Play();
     }

src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs

@@ -67,7 +67,7 @@ public async Task<Agent> LoadAgent(string id, bool loadUtility = true)
                 hook.OnSamplesLoaded(agent.Samples);
             }
 
-            if (loadUtility)
+            if (loadUtility && !agent.Utilities.IsNullOrEmpty())
             {
                 hook.OnAgentUtilityLoaded(agent);
             }

src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationStateService.cs

@@ -15,6 +15,7 @@ limitations under the License.
 ******************************************************************************/
 
 using BotSharp.Abstraction.Conversations.Enums;
+using BotSharp.Abstraction.Options;
 using BotSharp.Abstraction.SideCar;
 
 namespace BotSharp.Core.Conversations.Services;
@@ -69,9 +70,11 @@ public IConversationStateService SetState<T>(string name, T value, bool isNeedVe
             return this;
         }
 
+        var options = _services.GetRequiredService<BotSharpOptions>();
+
         var defaultRound = -1;
         var preValue = string.Empty;
-        var currentValue = value.ToString();
+        var currentValue = value.ConvertToString(options.JsonSerializerOptions);
         var curActive = true;
         StateKeyValue? pair = null;
         StateValue? prevLeafNode = null;