From fb86b42aa2e26e582581593f5ce1b4345bfec910 Mon Sep 17 00:00:00 2001 From: Bruno Capuano Date: Fri, 21 Feb 2025 11:58:13 -0500 Subject: [PATCH] Enhance MemoryContext and UI components DETAILS - Updated `MemoryContext.cs` with a new `SystemPrompt` constant, improved initialization logic, and enhanced logging for better clarity. - Refined search functionality for more detailed responses and better error handling. - Added logging in `Program.cs` to track the initialization of the vector database. - Modified `SearchResponse.cs` to include a constructor that initializes the `Products` list. - Improved UI in `Home.razor` with conditional rendering for the microphone component, a new button for live chat, and a "Realtime Audio Log" section. - Enhanced chat message handling by adding timestamps and improving microphone state management. - Updated `chat.css` to include styles for a new `.btn-purple` class, improving the visual design. --- src/Products/Memory/MemoryContext.cs | 118 +++++++++--------- src/Products/Program.cs | 5 + src/SearchEntities/SearchResponse.cs | 5 + src/StoreRealtime/Components/Pages/Home.razor | 44 +++++-- src/StoreRealtime/wwwroot/css/chat.css | 19 +++ 5 files changed, 121 insertions(+), 70 deletions(-) diff --git a/src/Products/Memory/MemoryContext.cs b/src/Products/Memory/MemoryContext.cs index 8dc6b71..a05b83c 100644 --- a/src/Products/Memory/MemoryContext.cs +++ b/src/Products/Memory/MemoryContext.cs @@ -1,24 +1,26 @@ -using Microsoft.EntityFrameworkCore; -using SearchEntities; -using DataEntities; -using OpenAI.Chat; -using OpenAI.Embeddings; -using VectorEntities; -using Microsoft.SemanticKernel.Connectors.InMemory; +using DataEntities; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.InMemory; using Newtonsoft.Json; +using OpenAI.Chat; +using OpenAI.Embeddings; using Products.Models; +using SearchEntities; +using System.Text; +using VectorEntities; namespace Products.Memory; public class MemoryContext { - private ILogger _logger; - public ChatClient? _chatClient; - public EmbeddingClient? _embeddingClient; - public IVectorStoreRecordCollection _productsCollection; - private string _systemPrompt = ""; - private bool _isMemoryCollectionInitialized = false; + private const string SystemPrompt = "You are a useful assistant. You always reply with a short and funny message. If you do not know an answer, you say 'I don't know that.' You only answer questions related to outdoor camping products. For any other type of questions, explain to the user that you only answer outdoor camping products questions. Do not store memory of the chat conversation."; + + private readonly ILogger _logger; + private readonly ChatClient? _chatClient; + private readonly EmbeddingClient? _embeddingClient; + private IVectorStoreRecordCollection? _productsCollection; + private bool _isMemoryCollectionInitialized; public MemoryContext(ILogger logger, ChatClient? chatClient, EmbeddingClient? embeddingClient) { @@ -33,16 +35,18 @@ public MemoryContext(ILogger logger, ChatClient? chatClient, EmbeddingClient? em public async Task InitMemoryContextAsync(Context db) { + if (_isMemoryCollectionInitialized) + { + _logger.LogInformation("Memory context already initialized"); + return true; + } + _logger.LogInformation("Initializing memory context"); var vectorProductStore = new InMemoryVectorStore(); _productsCollection = vectorProductStore.GetCollection("products"); await _productsCollection.CreateCollectionIfNotExistsAsync(); - // define system prompt - _systemPrompt = "You are a useful assistant. You always reply with a short and funny message. If you do not know an answer, you say 'I don't know that.' You only answer questions related to outdoor camping products. For any other type of questions, explain to the user that you only answer outdoor camping products questions. Do not store memory of the chat conversation."; - _logger.LogInformation("Get a copy of the list of products"); - // get a copy of the list of products var products = await db.Product.ToListAsync(); _logger.LogInformation("Filling products in memory"); @@ -64,7 +68,7 @@ public async Task InitMemoryContextAsync(Context db) Price = product.Price, ImageUrl = product.ImageUrl }; - var result = await _embeddingClient.GenerateEmbeddingAsync(productInfo); + var result = await _embeddingClient!.GenerateEmbeddingAsync(productInfo); productVector.Vector = result.Value.ToFloats(); var recordId = await _productsCollection.UpsertAsync(productVector); @@ -72,10 +76,12 @@ public async Task InitMemoryContextAsync(Context db) } catch (Exception exc) { - _logger.LogError(exc, "Error adding product to memory"); + _logger.LogError(exc, $"Error adding product {product.Name} to memory"); + return false; } } + _isMemoryCollectionInitialized = true; _logger.LogInformation("DONE! Filling products in memory"); return true; } @@ -85,73 +91,63 @@ public async Task Search(string search, Context db) if (!_isMemoryCollectionInitialized) { await InitMemoryContextAsync(db); - _isMemoryCollectionInitialized = true; } - var response = new SearchResponse(); - response.Response = $"I don't know the answer for your question. Your question is: [{search}]"; - Product? firstProduct = new Product(); - var responseText = ""; + var response = new SearchResponse + { + Response = $"I don't know the answer for your question. Your question is: [{search}]" + }; try { - var result = await _embeddingClient.GenerateEmbeddingAsync(search); + var result = await _embeddingClient!.GenerateEmbeddingAsync(search); var vectorSearchQuery = result.Value.ToFloats(); - var searchOptions = new VectorSearchOptions() + var searchOptions = new VectorSearchOptions { - Top = 1, + Top = 2, VectorPropertyName = "Vector" }; // search the vector database for the most similar product - var searchResults = await _productsCollection.VectorizedSearchAsync(vectorSearchQuery, searchOptions); - double searchScore = 0.0; + var searchResults = await _productsCollection!.VectorizedSearchAsync(vectorSearchQuery, searchOptions); + var sbFoundProducts = new StringBuilder(); + int productPosition = 1; await foreach (var searchItem in searchResults.Results) { if (searchItem.Score > 0.5) { - // product found, search the db for the product details - firstProduct = new Product + var foundProduct = await db.FindAsync(searchItem.Record.Id); + if (foundProduct != null) { - Id = searchItem.Record.Id, - Name = searchItem.Record.Name, - Description = searchItem.Record.Description, - Price = searchItem.Record.Price, - ImageUrl = searchItem.Record.ImageUrl - }; - - searchScore = searchItem.Score.Value; - responseText = $"The product [{firstProduct.Name}] fits with the search criteria [{search}][{searchItem.Score.Value.ToString("0.00")}]"; - _logger.LogInformation($"Search Response: {responseText}"); + response.Products.Add(foundProduct); + sbFoundProducts.AppendLine($"- Product {productPosition}:"); + sbFoundProducts.AppendLine($" - Name: {foundProduct.Name}"); + sbFoundProducts.AppendLine($" - Description: {foundProduct.Description}"); + sbFoundProducts.AppendLine($" - Price: {foundProduct.Price}"); + productPosition++; + } } } // let's improve the response message - var prompt = @$"You are an intelligent assistant helping clients with their search about outdoor products. Generate a catchy and friendly message using the following information: + var prompt = @$"You are an intelligent assistant helping clients with their search about outdoor products. +Generate a catchy and friendly message using the information below. +Add a comparison between the products found and the search criteria. +Include products details. - User Question: {search} - - Found Product Name: {firstProduct.Name} - - Found Product Description: {firstProduct.Description} - - Found Product Price: {firstProduct.Price} -Include the found product information in the response to the user question."; + - Found Products: +{sbFoundProducts}"; var messages = new List - { - new SystemChatMessage(_systemPrompt), - new UserChatMessage(prompt) - }; - - _logger.LogInformation("{ChatHistory}", JsonConvert.SerializeObject(messages)); - - var resultPrompt = await _chatClient.CompleteChatAsync(messages); - responseText = resultPrompt.Value.Content[0].Text!; - - // create a response object - response = new SearchResponse { - Products = firstProduct == null ? [new Product()] : [firstProduct], - Response = responseText + new SystemChatMessage(SystemPrompt), + new UserChatMessage(prompt) }; + _logger.LogInformation("{ChatHistory}", JsonConvert.SerializeObject(messages)); + + var resultPrompt = await _chatClient!.CompleteChatAsync(messages); + response.Response = resultPrompt.Value.Content[0].Text!; } catch (Exception ex) { @@ -160,4 +156,4 @@ public async Task Search(string search, Context db) } return response; } -} \ No newline at end of file +} diff --git a/src/Products/Program.cs b/src/Products/Program.cs index 332287c..cb51dda 100644 --- a/src/Products/Program.cs +++ b/src/Products/Program.cs @@ -107,6 +107,11 @@ app.Logger.LogError(exc, "Error creating database"); } DbInitializer.Initialize(context); + + app.Logger.LogInformation("Start fill products in vector db"); + var memoryContext = app.Services.GetRequiredService(); + await memoryContext.InitMemoryContextAsync(context); + app.Logger.LogInformation("Done fill products in vector db"); } app.Run(); \ No newline at end of file diff --git a/src/SearchEntities/SearchResponse.cs b/src/SearchEntities/SearchResponse.cs index 0fd36c7..69adb29 100644 --- a/src/SearchEntities/SearchResponse.cs +++ b/src/SearchEntities/SearchResponse.cs @@ -4,6 +4,11 @@ namespace SearchEntities; public class SearchResponse { + public SearchResponse() + { + Products = new List(); + } + [JsonPropertyName("id")] public string? Response { get; set; } diff --git a/src/StoreRealtime/Components/Pages/Home.razor b/src/StoreRealtime/Components/Pages/Home.razor index b57687c..e5c9388 100644 --- a/src/StoreRealtime/Components/Pages/Home.razor +++ b/src/StoreRealtime/Components/Pages/Home.razor @@ -13,8 +13,11 @@ eShopLite - Realtime chat support -@* AUDIO DEVICES *@ - +@if (isMicActive) +{ + +} +
@@ -38,11 +41,16 @@
} + @* REALTIME AUDIO LOG SECTION *@
+

Realtime Audio Log

+ 🔊
    @foreach (var message in messages) { @@ -55,17 +63,34 @@ @code { private ConversationManager? conversationManager; - private readonly CancellationTokenSource disposalCts = new(); + private CancellationTokenSource disposalCts = new(); + private bool isMicActive = false; private Speaker? speaker; private readonly List messages = new(); private readonly List chatMessages = new(); private ElementReference mainContentSection; + private void ToggleMic() + { + if (isMicActive) + { + // Stop interaction + conversationManager?.Dispose(); + isMicActive = false; + } + else + { + // Start interaction + disposalCts = new CancellationTokenSource(); + isMicActive = true; + } + } + private void OnMicAvailable(PipeReader micReader) { conversationManager = new(RealtimeConversationClient, contosoProductContext, logger); _ = RunAsBackgroundTask(() => conversationManager.RunAsync(micReader.AsStream(), speaker!, - AddMessageAsync, AddChatMessageAsync, disposalCts.Token)); + AddMessageAsync, AddChatMessageAsync, disposalCts.Token)); } public void Dispose() @@ -77,10 +102,11 @@ private async Task AddChatMessageAsync(string message, bool isUserMessage) { var cm = new RealtimeChatMessage - { - Message = message, - IsUser = isUserMessage - }; + { + Message = message, + IsUser = isUserMessage, + Timestamp = DateTime.Now + }; chatMessages.Add(cm); await InvokeAsync(StateHasChanged); ScrollToBottom(mainContentSection); @@ -113,4 +139,4 @@ } } - \ No newline at end of file + diff --git a/src/StoreRealtime/wwwroot/css/chat.css b/src/StoreRealtime/wwwroot/css/chat.css index ae109a3..6ec5c9e 100644 --- a/src/StoreRealtime/wwwroot/css/chat.css +++ b/src/StoreRealtime/wwwroot/css/chat.css @@ -251,3 +251,22 @@ height: 100%; } +.btn-purple { + background-color: #512bd4; /* .NET purple */ + color: white; + border: none; + padding: 10px 20px; + cursor: pointer; + border-radius: 5px; +} + + .btn-purple:hover { + background-color: #3e1a9e; /* Darker .NET purple */ + } + +.icon-top-right { + position: absolute; + top: 10px; + right: 10px; + font-size: 24px; +}