diff --git a/include/ryzenai/inference_engine.h b/include/ryzenai/inference_engine.h index ca55da9..a537e0b 100644 --- a/include/ryzenai/inference_engine.h +++ b/include/ryzenai/inference_engine.h @@ -5,6 +5,7 @@ #include #include #include +#include // Forward declarations for ONNX Runtime GenAI struct OgaModel; @@ -16,6 +17,9 @@ struct OgaMultiModalProcessor; namespace ryzenai { +// Single global multi-turn session until clients opt in via conversation_id. +inline constexpr const char* kDefaultConversationId = "default"; + // Timing data returned from completion struct CompletionTimingData { int token_count = 0; // Number of generated tokens @@ -41,7 +45,41 @@ class InferenceEngine { StreamCallback callback); // Apply chat template to messages - std::string applyChatTemplate(const std::string& messages_json, const std::string& tools_json = ""); + std::string applyChatTemplate(const std::string& messages_json, + const std::string& tools_json = "", + bool add_generation_prompt = true); + + // Multi-turn chat (non-tool): reuse one OgaGenerator per conversation_id and + // AppendTokens only the delta prompt on subsequent turns. + std::string completeMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + CompletionTimingData* out_timing = nullptr); + + void streamMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + StreamCallback callback); + + void resetMultiTurnSession(const std::string& conversation_id); + + // Multi-turn VLM: reuse ChatSession KV cache; turn 1 may SetInputs (vision), + // later text-only turns append delta from last <|im_start|>user via jinja template. + std::string completeMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + CompletionTimingData* out_timing = nullptr); + + void streamMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + StreamCallback callback); // Apply the model's chat template strictly via OGA (jinja), without the // text-only manual fallbacks. Required for multimodal models whose template @@ -85,6 +123,25 @@ class InferenceEngine { std::string resolveModelPath(const std::string& path); std::vector truncatePrompt(const std::vector& input_ids); bool validateModelDirectory(const std::string& path); + + struct ChatSession { + std::unique_ptr generator; + std::unique_ptr gen_params; + size_t turn_count = 0; + }; + + ChatSession& getOrCreateChatSession(const std::string& conversation_id); + void resetMultiTurnSessionLocked(const std::string& conversation_id); + // Extract the latest user turn from full prompt: suffix starting at last <|im_start|>user. + std::string extractDeltaPromptFromLastUser(const std::string& full_prompt) const; + void appendPromptText(OgaGenerator& generator, const std::string& text); + void configureGeneratorParams(OgaGeneratorParams& gen_params, + const GenerationParams& params, + int total_max_length) const; + std::string applyStopSequences(const std::string& text, const GenerationParams& params) const; + std::vector encodeText(const std::string& text) const; + + std::unordered_map chat_sessions_; std::unique_ptr model_; std::unique_ptr tokenizer_; diff --git a/include/ryzenai/server.h b/include/ryzenai/server.h index 9f27a7a..a2dc0e6 100644 --- a/include/ryzenai/server.h +++ b/include/ryzenai/server.h @@ -33,9 +33,13 @@ class RyzenAIServer { void handleCompletions(const httplib::Request& req, httplib::Response& res); void handleChatCompletions(const httplib::Request& req, httplib::Response& res); void handleMultimodalChat(const json& request_json, const ChatCompletionRequest& chat_req, - const std::vector& images, httplib::Response& res); + const json& messages_array, + const std::vector& new_turn_images, + const std::vector& all_images, + httplib::Response& res); void handleResponses(const httplib::Request& req, httplib::Response& res); - + void handleSessionReset(const httplib::Request& req, httplib::Response& res); + // Helper methods json createErrorResponse(const std::string& message, const std::string& type); void setupCORS(httplib::Response& res); diff --git a/include/ryzenai/types.h b/include/ryzenai/types.h index 55b840c..b93b0d2 100644 --- a/include/ryzenai/types.h +++ b/include/ryzenai/types.h @@ -43,6 +43,7 @@ struct CompletionRequest { // Chat completion request (OpenAI format) struct ChatCompletionRequest { std::vector messages; + std::string conversation_id; // reserved: parsed but not required; server uses default session int max_tokens = 1500; float temperature = 0.7f; float top_p = 0.9f; diff --git a/src/inference_engine.cpp b/src/inference_engine.cpp index b76ac35..7f7e512 100644 --- a/src/inference_engine.cpp +++ b/src/inference_engine.cpp @@ -8,11 +8,138 @@ #include #include #include +#include +#include namespace ryzenai { namespace fs = std::filesystem; +namespace { + +void writeAppendDebugFile(const std::string& label, + const std::string& session_id, + size_t turn_number, + const std::string& text) { + const char* debug_dir_env = std::getenv("AMDGPU_DEBUG_DIR"); + if (!debug_dir_env || debug_dir_env[0] == '\0') { + return; + } + + try { + fs::path dir(debug_dir_env); + fs::create_directories(dir); + + auto now = std::chrono::system_clock::now(); + auto ms = std::chrono::duration_cast(now.time_since_epoch()) % 1000; + std::time_t tt = std::chrono::system_clock::to_time_t(now); + std::tm local_tm{}; + localtime_s(&local_tm, &tt); + + std::ostringstream name; + name << "server_append_" << std::put_time(&local_tm, "%Y%m%d_%H%M%S") + << '_' << std::setw(3) << std::setfill('0') << ms.count() + << ".txt"; + + fs::path out = dir / name.str(); + std::ofstream file(out); + file << "label=" << label << "\n"; + file << "session=" << session_id << "\n"; + file << "turn=" << turn_number << "\n"; + file << "chars=" << text.length() << "\n"; + file << "----text----\n"; + file << text; + std::cout << "[InferenceEngine] Wrote append debug: " << out.string() << std::endl; + } catch (const std::exception& e) { + std::cerr << "[WARNING] Failed to write append debug file: " << e.what() << std::endl; + } +} + +size_t countSubstring(const std::string& haystack, const std::string& needle) { + size_t count = 0; + size_t pos = 0; + while ((pos = haystack.find(needle, pos)) != std::string::npos) { + ++count; + pos += needle.size(); + } + return count; +} + +void logMultiTurnAppendPlan(const char* path_label, + const std::string& conversation_id, + size_t turn_number, + bool is_first_turn, + const std::string& full_prompt, + const std::string& text_to_append, + int full_tokens, + int append_tokens, + const char* input_mode) { + static constexpr const char* kUserTurnMarker = "<|im_start|>user"; + const size_t user_markers = countSubstring(full_prompt, kUserTurnMarker); + const size_t full_chars = full_prompt.size(); + const size_t append_chars = text_to_append.size(); + const int saved_tokens = full_tokens - append_tokens; + const int saved_pct = (full_tokens > 0) ? static_cast((saved_tokens * 100LL) / full_tokens) : 0; + + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " session=" << conversation_id + << " turn=" << turn_number + << " mode=" << (is_first_turn ? "APPEND_FULL" : "APPEND_DELTA") + << " input=" << input_mode + << " user_markers=" << user_markers + << " full_chars=" << full_chars + << " full_tokens=" << full_tokens + << " append_chars=" << append_chars + << " append_tokens=" << append_tokens; + if (!is_first_turn) { + std::cout << " saved_tokens=" << saved_tokens << " saved_pct=" << saved_pct << "%"; + } + std::cout << std::endl; + + if (!is_first_turn) { + std::cout << "[InferenceEngine][MultiTurnDebug] delta_head: " + << text_to_append.substr(0, std::min(size_t(240), append_chars)) << std::endl; + } +} + +void logMultiTurnAfterAppend(const char* path_label, + size_t seq_before, + size_t seq_after, + bool is_done) { + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " seq_before=" << seq_before + << " seq_after=" << seq_after + << " seq_delta=" << (seq_after > seq_before ? seq_after - seq_before : 0) + << " is_done=" << (is_done ? "true" : "false") << std::endl; + if (is_done && seq_after <= seq_before) { + std::cerr << "[InferenceEngine][MultiTurnDebug] WARNING: generator IsDone after append " + << "with no new sequence tokens — generation loop will produce 0 output" + << std::endl; + } +} + +void logMultiTurnCompleted(const char* path_label, + size_t turn_number, + size_t seq_before, + size_t seq_after, + int generated_tokens, + int streamed_tokens) { + std::cout << "[InferenceEngine][MultiTurnDebug] path=" << path_label + << " turn=" << turn_number + << " completed generated_tokens=" << generated_tokens + << " seq_before=" << seq_before + << " seq_after=" << seq_after; + if (streamed_tokens >= 0) { + std::cout << " streamed_tokens=" << streamed_tokens; + } + if (generated_tokens == 0) { + std::cout << " WARNING=zero_output"; + } + std::cout << std::endl; +} + +} // namespace + InferenceEngine::InferenceEngine(const std::string& model_path, int ctx_size) : ctx_size_(ctx_size) { @@ -125,7 +252,9 @@ GenerationParams InferenceEngine::getDefaultParams() const { return default_params_; } -std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, const std::string& tools_json) { +std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, + const std::string& tools_json, + bool add_generation_prompt) { // Parse messages json messages = json::parse(messages_json); std::ostringstream prompt; @@ -145,7 +274,7 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, template_str, messages_json.c_str(), tools_str, - true + add_generation_prompt ); std::cout << "[InferenceEngine] Applied chat template with tools" << std::endl; @@ -165,8 +294,9 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, << content << "<|im_end|>\n"; } - // Add generation prompt for assistant - prompt << "<|im_start|>assistant\n"; + if (add_generation_prompt) { + prompt << "<|im_start|>assistant\n"; + } std::cout << "[InferenceEngine] Applied Qwen/ChatML template" << std::endl; } else { @@ -178,7 +308,7 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, template_str, messages_json.c_str(), nullptr, - true + add_generation_prompt ); return std::string(result); @@ -202,7 +332,9 @@ std::string InferenceEngine::applyChatTemplate(const std::string& messages_json, } } - prompt << "Assistant: "; + if (add_generation_prompt) { + prompt << "Assistant: "; + } } } @@ -702,6 +834,299 @@ int InferenceEngine::countTokens(const std::string& text) { } } +std::vector InferenceEngine::encodeText(const std::string& text) const { + auto sequences = OgaSequences::Create(); + tokenizer_->Encode(text.c_str(), *sequences); + const int32_t* input_ids_ptr = sequences->SequenceData(0); + size_t input_ids_count = sequences->SequenceCount(0); + return std::vector(input_ids_ptr, input_ids_ptr + input_ids_count); +} + +void InferenceEngine::appendPromptText(OgaGenerator& generator, const std::string& text) { + if (text.empty()) { + return; + } + std::vector input_ids = encodeText(text); + generator.AppendTokens(input_ids.data(), input_ids.size()); +} + +void InferenceEngine::configureGeneratorParams(OgaGeneratorParams& gen_params, + const GenerationParams& params, + int total_max_length) const { + gen_params.SetSearchOption("max_length", total_max_length); + gen_params.SetSearchOption("temperature", params.temperature); + gen_params.SetSearchOption("top_p", params.top_p); + gen_params.SetSearchOption("top_k", static_cast(params.top_k)); + gen_params.SetSearchOption("repetition_penalty", params.repetition_penalty); + gen_params.SetSearchOptionBool("do_sample", params.do_sample); + gen_params.SetSearchOption("random_seed", 1.0); +} + +std::string InferenceEngine::applyStopSequences(const std::string& text, + const GenerationParams& params) const { + std::string result = text; + for (const auto& stop_seq : params.stop_sequences) { + size_t pos = result.find(stop_seq); + if (pos != std::string::npos) { + result = result.substr(0, pos); + break; + } + } + return result; +} + +std::string InferenceEngine::extractDeltaPromptFromLastUser(const std::string& full_prompt) const { + static constexpr const char* kUserTurnMarker = "<|im_start|>user"; + const size_t pos = full_prompt.rfind(kUserTurnMarker); + if (pos == std::string::npos) { + throw std::runtime_error( + std::string("Multi-turn: could not find latest user turn marker '") + kUserTurnMarker + + "' in full prompt"); + } + return full_prompt.substr(pos); +} + +InferenceEngine::ChatSession& InferenceEngine::getOrCreateChatSession(const std::string& conversation_id) { + auto it = chat_sessions_.find(conversation_id); + if (it != chat_sessions_.end()) { + return it->second; + } + + ChatSession session; + session.gen_params = OgaGeneratorParams::Create(*model_); + session.generator = OgaGenerator::Create(*model_, *session.gen_params); + session.turn_count = 0; + + auto [inserted_it, inserted] = chat_sessions_.emplace(conversation_id, std::move(session)); + if (!inserted) { + throw std::runtime_error("Failed to create chat session: " + conversation_id); + } + + std::cout << "[InferenceEngine] Created multi-turn session: " << conversation_id << std::endl; + return inserted_it->second; +} + +void InferenceEngine::resetMultiTurnSession(const std::string& conversation_id) { + std::lock_guard lock(inference_mutex_); + resetMultiTurnSessionLocked(conversation_id); +} + +void InferenceEngine::resetMultiTurnSessionLocked(const std::string& conversation_id) { + auto it = chat_sessions_.find(conversation_id); + if (it == chat_sessions_.end()) { + std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id + << " (no existing session, no-op)" << std::endl; + return; + } + + ChatSession& session = it->second; + try { + session.generator->RewindTo(0); + session.turn_count = 0; + std::cout << "[InferenceEngine] Reset multi-turn session: " << conversation_id + << " (RewindTo(0), turn_count=0)" << std::endl; + } catch (const std::exception& e) { + std::cerr << "[InferenceEngine] RewindTo(0) failed for session " << conversation_id + << ", recreating generator: " << e.what() << std::endl; + chat_sessions_.erase(it); + } +} + +std::string InferenceEngine::completeMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + CompletionTimingData* out_timing) { + std::lock_guard lock(inference_mutex_); + + try { + const std::string full_prompt = applyChatTemplate(messages_json, "", true); + ChatSession& session = getOrCreateChatSession(conversation_id); + const bool is_first_turn = (session.turn_count == 0); + const std::string delta_prompt = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + const std::string text_to_append = delta_prompt; + + if (!is_first_turn && text_to_append.empty()) { + throw std::runtime_error("Multi-turn request produced an empty delta prompt"); + } + + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + + logMultiTurnAppendPlan("text", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + "AppendTokens"); + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + const size_t seq_before = session.generator->GetSequenceCount(0); + appendPromptText(*session.generator, text_to_append); + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("text", seq_before, seq_after_append, session.generator->IsDone()); + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto start_time = std::chrono::high_resolution_clock::now(); + auto first_token_time = start_time; + bool first_token_received = false; + + while (!session.generator->IsDone()) { + session.generator->GenerateNextToken(); + if (!first_token_received) { + first_token_time = std::chrono::high_resolution_clock::now(); + first_token_received = true; + } + } + + auto end_time = std::chrono::high_resolution_clock::now(); + const int32_t* output_ptr = session.generator->GetSequenceData(0); + const size_t output_count = session.generator->GetSequenceCount(0); + const int generated_token_count = (output_count > seq_before) + ? static_cast(output_count - seq_before) + : 0; + + std::string result; + if (output_count > seq_before) { + auto decoded = tokenizer_->Decode(output_ptr + seq_before, output_count - seq_before); + result = applyStopSequences(std::string(decoded), params); + } + + session.turn_count++; + + logMultiTurnCompleted("text", session.turn_count, seq_before, output_count, + generated_token_count, -1); + + if (out_timing != nullptr) { + auto total_duration = std::chrono::duration_cast(end_time - start_time); + auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); + const double ttft_seconds = ttft_duration.count() / 1000.0; + const double total_time_ms = static_cast(total_duration.count()); + const double decode_time_seconds = (total_duration.count() - ttft_duration.count()) / 1000.0; + double tps = 0.0; + if (generated_token_count > 1 && decode_time_seconds > 0) { + tps = (generated_token_count - 1) / decode_time_seconds; + } else if (generated_token_count == 1 && total_time_ms > 0) { + tps = 1.0 / (total_time_ms / 1000.0); + } + + out_timing->token_count = generated_token_count; + out_timing->ttft_seconds = ttft_seconds; + out_timing->tps = tps; + out_timing->total_time_ms = total_time_ms; + } + + std::cout << "[InferenceEngine] Multi-turn completed turn=" << session.turn_count + << " generated=" << generated_token_count + << " token_count=" << session.generator->TokenCount() << std::endl; + + return result; + } catch (const std::exception& e) { + throw std::runtime_error("Multi-turn inference failed: " + std::string(e.what())); + } +} + +void InferenceEngine::streamMultiTurn(const std::string& conversation_id, + const std::string& messages_json, + const GenerationParams& params, + StreamCallback callback) { + std::lock_guard lock(inference_mutex_); + + try { + const std::string full_prompt = applyChatTemplate(messages_json, "", true); + ChatSession& session = getOrCreateChatSession(conversation_id); + const bool is_first_turn = (session.turn_count == 0); + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + + if (!is_first_turn && text_to_append.empty()) { + throw std::runtime_error("Multi-turn request produced an empty delta prompt"); + } + + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + + logMultiTurnAppendPlan("text-stream", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + "AppendTokens"); + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + const size_t seq_before = session.generator->GetSequenceCount(0); + appendPromptText(*session.generator, text_to_append); + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("text-stream", seq_before, seq_after_append, session.generator->IsDone()); + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto tokenizer_stream = OgaTokenizerStream::Create(*tokenizer_); + std::string accumulated_output; + bool client_disconnected = false; + + while (!session.generator->IsDone() && !client_disconnected) { + session.generator->GenerateNextToken(); + + const int32_t* all_tokens = session.generator->GetSequenceData(0); + const size_t num_tokens = session.generator->GetSequenceCount(0); + const int32_t new_token = all_tokens[num_tokens - 1]; + + const char* decoded = tokenizer_stream->Decode(new_token); + if (decoded && decoded[0] != '\0') { + std::string token_str(decoded); + + bool should_stop = false; + for (const auto& stop_seq : params.stop_sequences) { + std::string temp_output = accumulated_output + token_str; + if (temp_output.find(stop_seq) != std::string::npos) { + should_stop = true; + break; + } + } + if (should_stop) { + break; + } + + accumulated_output += token_str; + const bool is_final = session.generator->IsDone(); + if (!callback(token_str, is_final)) { + client_disconnected = true; + break; + } + } + } + + accumulated_output = applyStopSequences(accumulated_output, params); + session.turn_count++; + + const size_t seq_after = session.generator->GetSequenceCount(0); + const int generated_token_count = (seq_after > seq_before) + ? static_cast(seq_after - seq_before) + : 0; + logMultiTurnCompleted("text-stream", session.turn_count, seq_before, seq_after, + generated_token_count, -1); + + std::cout << "[InferenceEngine] Multi-turn stream completed turn=" << session.turn_count + << " token_count=" << session.generator->TokenCount() << std::endl; + } catch (const std::exception& e) { + throw std::runtime_error("Multi-turn streaming failed: " + std::string(e.what())); + } +} + void InferenceEngine::detectMultimodal() { // A model is multimodal if genai_config.json declares a vision (or speech) // pipeline, or a processor_config.json is present alongside the model. @@ -761,6 +1186,281 @@ void apply_search_options(OgaGeneratorParams& gen_params, const GenerationParams } } // namespace +namespace { + +std::unique_ptr loadOgaImages(const std::vector& images) { + if (images.empty()) { + return nullptr; + } + std::vector data_ptrs; + std::vector data_sizes; + data_ptrs.reserve(images.size()); + data_sizes.reserve(images.size()); + for (const auto& img : images) { + data_ptrs.push_back(img.data()); + data_sizes.push_back(img.size()); + } + return OgaImages::Load(data_ptrs.data(), data_sizes.data(), data_ptrs.size()); +} + +} // namespace + +std::string InferenceEngine::completeMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + CompletionTimingData* out_timing) { + std::lock_guard lock(inference_mutex_); + if (!processor_) { + throw std::runtime_error("Multimodal multi-turn requested but model is not multimodal"); + } + + try { + const std::string full_prompt = applyChatTemplateRaw(messages_json, tools_json); + bool is_first_turn = true; + if (auto it = chat_sessions_.find(conversation_id); it != chat_sessions_.end()) { + is_first_turn = (it->second.turn_count == 0); + } + + if (!is_first_turn && !new_turn_images.empty()) { + std::cout << "[InferenceEngine] Multimodal multi-turn: new images on follow-up turn, " + << "resetting session and reprocessing full prompt" << std::endl; + chat_sessions_.erase(conversation_id); + is_first_turn = true; + } + + ChatSession& active_session = getOrCreateChatSession(conversation_id); + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + + if (text_to_append.empty()) { + throw std::runtime_error("Multimodal multi-turn request produced an empty prompt segment"); + } + + const std::vector& images_for_process = + is_first_turn ? all_images : new_turn_images; + + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + const char* input_mode = + (is_first_turn && !images_for_process.empty()) ? "SetInputs" : "AppendTokens"; + + logMultiTurnAppendPlan("vlm", conversation_id, active_session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + input_mode); + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + active_session.turn_count + 1, + text_to_append); + + const size_t seq_before = active_session.generator->GetSequenceCount(0); + + if (is_first_turn && !images_for_process.empty()) { + auto oga_images = loadOgaImages(images_for_process); + auto inputs = processor_->ProcessImages(text_to_append.c_str(), oga_images.get()); + active_session.generator->SetInputs(*inputs); + } else { + appendPromptText(*active_session.generator, text_to_append); + } + + const size_t seq_after_append = active_session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("vlm", seq_before, seq_after_append, active_session.generator->IsDone()); + + int total_max_length = static_cast(active_session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*active_session.gen_params, params, total_max_length); + + auto start_time = std::chrono::high_resolution_clock::now(); + auto first_token_time = start_time; + bool first_token_received = false; + + while (!active_session.generator->IsDone()) { + active_session.generator->GenerateNextToken(); + if (!first_token_received) { + first_token_time = std::chrono::high_resolution_clock::now(); + first_token_received = true; + } + } + + auto end_time = std::chrono::high_resolution_clock::now(); + const int32_t* output_ptr = active_session.generator->GetSequenceData(0); + const size_t output_count = active_session.generator->GetSequenceCount(0); + const int generated_token_count = (output_count > seq_before) + ? static_cast(output_count - seq_before) + : 0; + + std::string result; + if (output_count > seq_before) { + auto decoded = processor_->Decode(output_ptr + seq_before, output_count - seq_before); + result = applyStopSequences(std::string(decoded), params); + } + + active_session.turn_count++; + + logMultiTurnCompleted("vlm", active_session.turn_count, seq_before, output_count, + generated_token_count, -1); + + if (out_timing != nullptr) { + auto total_duration = std::chrono::duration_cast(end_time - start_time); + auto ttft_duration = std::chrono::duration_cast(first_token_time - start_time); + const double ttft_seconds = ttft_duration.count() / 1000.0; + const double total_time_ms = static_cast(total_duration.count()); + const double decode_time_seconds = (total_duration.count() - ttft_duration.count()) / 1000.0; + double tps = 0.0; + if (generated_token_count > 1 && decode_time_seconds > 0) { + tps = (generated_token_count - 1) / decode_time_seconds; + } else if (generated_token_count == 1 && total_time_ms > 0) { + tps = 1.0 / (total_time_ms / 1000.0); + } + + out_timing->token_count = generated_token_count; + out_timing->ttft_seconds = ttft_seconds; + out_timing->tps = tps; + out_timing->total_time_ms = total_time_ms; + } + + std::cout << "[InferenceEngine] Multimodal multi-turn completed turn=" << active_session.turn_count + << " generated=" << generated_token_count << std::endl; + return result; + } catch (const std::exception& e) { + throw std::runtime_error("Multimodal multi-turn inference failed: " + std::string(e.what())); + } +} + +void InferenceEngine::streamMultiTurnMultimodal(const std::string& conversation_id, + const std::string& messages_json, + const std::vector& new_turn_images, + const std::vector& all_images, + const std::string& tools_json, + const GenerationParams& params, + StreamCallback callback) { + std::lock_guard lock(inference_mutex_); + if (!processor_) { + throw std::runtime_error("Multimodal multi-turn requested but model is not multimodal"); + } + + try { + const std::string full_prompt = applyChatTemplateRaw(messages_json, tools_json); + bool is_first_turn = true; + if (auto it = chat_sessions_.find(conversation_id); it != chat_sessions_.end()) { + is_first_turn = (it->second.turn_count == 0); + } + + if (!is_first_turn && !new_turn_images.empty()) { + std::cout << "[InferenceEngine] Multimodal multi-turn stream: new images on follow-up turn, " + << "resetting session and reprocessing full prompt" << std::endl; + chat_sessions_.erase(conversation_id); + is_first_turn = true; + } + + ChatSession& session = getOrCreateChatSession(conversation_id); + + const std::string text_to_append = is_first_turn + ? full_prompt + : extractDeltaPromptFromLastUser(full_prompt); + + if (text_to_append.empty()) { + throw std::runtime_error("Multimodal multi-turn request produced an empty prompt segment"); + } + + const std::vector& images_for_process = + is_first_turn ? all_images : new_turn_images; + + const int append_token_count = countTokens(text_to_append); + const int full_token_count = countTokens(full_prompt); + const char* input_mode = + (is_first_turn && !images_for_process.empty()) ? "SetInputs" : "AppendTokens"; + + logMultiTurnAppendPlan("vlm-stream", conversation_id, session.turn_count + 1, is_first_turn, + full_prompt, text_to_append, full_token_count, append_token_count, + input_mode); + std::cout << "[InferenceEngine][MultiTurnDebug] vlm-stream new_turn_images=" + << new_turn_images.size() << " process_images=" << images_for_process.size() + << std::endl; + + writeAppendDebugFile(is_first_turn ? "APPEND_FULL" : "APPEND_DELTA", + conversation_id, + session.turn_count + 1, + text_to_append); + + const size_t seq_before = session.generator->GetSequenceCount(0); + + if (is_first_turn && !images_for_process.empty()) { + auto oga_images = loadOgaImages(images_for_process); + auto inputs = processor_->ProcessImages(text_to_append.c_str(), oga_images.get()); + session.generator->SetInputs(*inputs); + } else { + appendPromptText(*session.generator, text_to_append); + } + + const size_t seq_after_append = session.generator->GetSequenceCount(0); + logMultiTurnAfterAppend("vlm-stream", seq_before, seq_after_append, session.generator->IsDone()); + + int total_max_length = static_cast(session.generator->GetSequenceCount(0)) + params.max_length; + if (model_context_length_ > 0 && total_max_length > model_context_length_) { + total_max_length = model_context_length_; + } + configureGeneratorParams(*session.gen_params, params, total_max_length); + + auto tokenizer_stream = OgaTokenizerStream::Create(*processor_); + std::string accumulated_output; + bool client_disconnected = false; + + while (!session.generator->IsDone() && !client_disconnected) { + session.generator->GenerateNextToken(); + + const int32_t* all_tokens = session.generator->GetSequenceData(0); + const size_t num_tokens = session.generator->GetSequenceCount(0); + const int32_t new_token = all_tokens[num_tokens - 1]; + + const char* decoded = tokenizer_stream->Decode(new_token); + if (decoded && decoded[0] != '\0') { + std::string token_str(decoded); + + bool should_stop = false; + for (const auto& stop_seq : params.stop_sequences) { + std::string temp_output = accumulated_output + token_str; + if (temp_output.find(stop_seq) != std::string::npos) { + should_stop = true; + break; + } + } + if (should_stop) { + break; + } + + accumulated_output += token_str; + const bool is_final = session.generator->IsDone(); + if (!callback(token_str, is_final)) { + client_disconnected = true; + break; + } + } + } + + accumulated_output = applyStopSequences(accumulated_output, params); + session.turn_count++; + + const size_t seq_after = session.generator->GetSequenceCount(0); + const int generated_token_count = (seq_after > seq_before) + ? static_cast(seq_after - seq_before) + : 0; + logMultiTurnCompleted("vlm-stream", session.turn_count, seq_before, seq_after, + generated_token_count, -1); + + std::cout << "[InferenceEngine] Multimodal multi-turn stream completed turn=" + << session.turn_count << std::endl; + } catch (const std::exception& e) { + throw std::runtime_error("Multimodal multi-turn streaming failed: " + std::string(e.what())); + } +} + std::string InferenceEngine::completeMultimodal(const std::string& prompt, const std::vector& images, const GenerationParams& params, diff --git a/src/server.cpp b/src/server.cpp index 4564a9d..99901e3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -12,6 +12,33 @@ namespace ryzenai { namespace { +std::string escapeJsonString(const std::string& s) { + static constexpr char hex[] = "0123456789abcdef"; + std::string escaped; + escaped.reserve(s.size() + 8); + for (unsigned char c : s) { + switch (c) { + case '\\': escaped += "\\\\"; break; + case '"': escaped += "\\\""; break; + case '\b': escaped += "\\b"; break; + case '\f': escaped += "\\f"; break; + case '\n': escaped += "\\n"; break; + case '\r': escaped += "\\r"; break; + case '\t': escaped += "\\t"; break; + default: + if (c < 0x20) { + escaped += "\\u00"; + escaped += hex[(c >> 4) & 0x0f]; + escaped += hex[c & 0x0f]; + } else { + escaped.push_back(static_cast(c)); + } + break; + } + } + return escaped; +} + // Minimal base64 decoder for image data URLs. std::string base64_decode(const std::string& input) { static const std::string chars = @@ -91,6 +118,70 @@ std::vector extract_request_images(const json& request_json) { return images; } +// Only images attached to the latest user turn. Follow-up text turns must not +// re-ingest images from earlier history (that forced full prompt reprocessing). +std::vector extract_images_from_last_user_message(const json& request_json) { + std::vector images; + if (!request_json.contains("messages") || !request_json["messages"].is_array()) { + return images; + } + + const json* last_user = nullptr; + for (const auto& msg : request_json["messages"]) { + if (msg.value("role", "") == "user") { + last_user = &msg; + } + } + if (last_user == nullptr || !last_user->contains("content") || !(*last_user)["content"].is_array()) { + return images; + } + + for (const auto& item : (*last_user)["content"]) { + if (!item.is_object()) continue; + std::string url; + if (item.contains("image_url")) { + const auto& iu = item["image_url"]; + if (iu.is_object() && iu.contains("url") && iu["url"].is_string()) { + url = iu["url"].get(); + } else if (iu.is_string()) { + url = iu.get(); + } + } else if (item.value("type", "") == "image" && item.contains("url") && item["url"].is_string()) { + url = item["url"].get(); + } + if (url.empty()) continue; + std::string bytes = resolve_image_bytes(url); + if (!bytes.empty()) { + images.push_back(std::move(bytes)); + } else { + std::cerr << "[WARNING] Could not resolve image reference in latest user turn (skipped)" + << std::endl; + } + } + return images; +} + +void logOpenAIChatRequestSummary(const ChatCompletionRequest& chat_req, bool multimodal) { + std::ostringstream role_seq; + for (size_t i = 0; i < chat_req.messages.size(); ++i) { + if (i > 0) { + role_seq << " -> "; + } + role_seq << chat_req.messages[i].role; + if (chat_req.messages[i].content.find("") != std::string::npos) { + role_seq << "(tool_response)"; + } + } + + const size_t tool_count = chat_req.tools.is_array() ? chat_req.tools.size() : 0; + std::cout << "[Server][ChatDebug] client=openai-compat multimodal=" << (multimodal ? 1 : 0) + << " messages=" << chat_req.messages.size() + << " tools=" << tool_count + << " stream=" << chat_req.stream + << " session=" << kDefaultConversationId + << " roles=" << role_seq.str() << std::endl; +} + } // namespace RyzenAIServer::RyzenAIServer(const CommandLineArgs& args) @@ -221,6 +312,10 @@ void RyzenAIServer::setupRoutes() { http_server_->Post("/v1/responses", [this](const httplib::Request& req, httplib::Response& res) { handleResponses(req, res); }); + + http_server_->Post("/v1/sessions/reset", [this](const httplib::Request& req, httplib::Response& res) { + handleSessionReset(req, res); + }); // Root redirect http_server_->Get("/", [this](const httplib::Request&, httplib::Response& res) { @@ -232,7 +327,8 @@ void RyzenAIServer::setupRoutes() { "/health", "/v1/completions", "/v1/chat/completions", - "/v1/responses" + "/v1/responses", + "/v1/sessions/reset" }} }; res.set_content(response.dump(2), "application/json"); @@ -263,6 +359,32 @@ void RyzenAIServer::handleHealth(const httplib::Request& req, httplib::Response& res.set_content(response.dump(2), "application/json"); } +void RyzenAIServer::handleSessionReset(const httplib::Request& req, httplib::Response& res) { + try { + std::string conversation_id = kDefaultConversationId; + if (!req.body.empty()) { + json request_json = json::parse(req.body); + if (request_json.contains("conversation_id") && + request_json["conversation_id"].is_string()) { + conversation_id = request_json["conversation_id"].get(); + } + } + + inference_engine_->resetMultiTurnSession(conversation_id); + + json response = { + {"status", "success"}, + {"conversation_id", conversation_id}, + {"message", "Chat session KV cache reset"} + }; + res.set_content(response.dump(), "application/json"); + } catch (const std::exception& e) { + res.status = 500; + res.set_content(createErrorResponse(e.what(), "session_reset_error").dump(), + "application/json"); + } +} + void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Response& res) { try { // Parse request @@ -330,30 +452,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -434,30 +534,8 @@ void RyzenAIServer::handleCompletions(const httplib::Request& req, httplib::Resp // After generation completes, do a final flush to catch any remaining buffered content auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -614,14 +692,16 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: return; } - // Multimodal branch: a VLM request carrying images is handled separately - // (OpenAI image_url content -> OGA vision pipeline). Returns early. + logOpenAIChatRequestSummary(chat_req, inference_engine_->isMultimodal()); + + // VLM models always use the multimodal multi-turn path (KV cache reuse on + // text-only follow-ups). Only images in the latest user turn are processed. if (inference_engine_->isMultimodal()) { - std::vector images = extract_request_images(request_json); - if (!images.empty()) { - handleMultimodalChat(request_json, chat_req, images, res); - return; - } + json messages_array = request_json.contains("messages") ? request_json["messages"] : json::array(); + std::vector new_turn_images = extract_images_from_last_user_message(request_json); + std::vector all_images = extract_request_images(request_json); + handleMultimodalChat(request_json, chat_req, messages_array, new_turn_images, all_images, res); + return; } // Convert messages to JSON array for chat template. Use the raw request @@ -643,6 +723,10 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: std::string prompt = inference_engine_->applyChatTemplate(messages_array.dump(), tools_json); std::cout << "[Server DEBUG] Generated prompt length: " << prompt.length() << " chars" << std::endl; std::cout << "[Server DEBUG] Prompt (first 500 chars): " << prompt.substr(0, std::min(size_t(500), prompt.length())) << std::endl; + + // Text chat always uses the single default multi-turn AppendTokens session. + const bool use_multi_turn = true; + const std::string session_id = kDefaultConversationId; if (chat_req.stream) { // REAL-TIME STREAMING: Send chunks as tokens are generated @@ -662,9 +746,12 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Count prompt tokens before streaming int prompt_tokens = inference_engine_->countTokens(prompt); + std::string messages_json = messages_array.dump(); + res.set_chunked_content_provider( "text/event-stream", - [this, prompt, params, model_id, has_tools, prompt_tokens](size_t offset, httplib::DataSink& sink) { + [this, prompt, params, model_id, has_tools, prompt_tokens, use_multi_turn, + session_id, messages_json](size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; // Only run once try { @@ -680,8 +767,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Create reasoning parser for streaming ReasoningStreamParser reasoning_parser; - // Generate and send tokens in real-time - inference_engine_->streamComplete(prompt, params, + StreamCallback stream_callback = [&sink, model_id, &token_count, &full_response, &reasoning_parser, &first_token_received, &first_token_time](const std::string& token, bool is_final) -> bool { // Track time to first token if (!first_token_received && !token.empty()) { @@ -695,30 +781,8 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: // Process token through reasoning parser auto [reasoning_part, content_part] = reasoning_parser.processToken(token); - // Helper function to escape JSON strings auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send reasoning content chunk if present @@ -793,37 +857,20 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: token_count++; return true; // Continue generation - } - ); + }; + + if (use_multi_turn) { + inference_engine_->streamMultiTurn(session_id, messages_json, params, stream_callback); + } else { + inference_engine_->streamComplete(prompt, params, stream_callback); + } // After generation completes, do a final flush to catch any remaining buffered content // This handles the case where the last few tokens didn't trigger processing due to buffer size auto [final_reasoning, final_content] = reasoning_parser.flush(); - // Helper function to escape JSON strings (reused from above) auto escapeJson = [](const std::string& str) -> std::string { - std::string escaped = str; - size_t pos = 0; - while ((pos = escaped.find('\\', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('"', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\n', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped.find('\r', pos)) != std::string::npos) { - escaped.replace(pos, 1, "\\r"); - pos += 2; - } - return escaped; + return escapeJsonString(str); }; // Send any remaining reasoning content @@ -864,17 +911,7 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: for (const auto& tool_call : extracted_tool_calls) { // Escape arguments for JSON std::string tool_call_args = tool_call.arguments.dump(); - std::string escaped_args = tool_call_args; - size_t pos = 0; - while ((pos = escaped_args.find('\\', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_args.find('"', pos)) != std::string::npos) { - escaped_args.replace(pos, 1, "\\\""); - pos += 2; - } + std::string escaped_args = escapeJsonString(tool_call_args); std::string tool_call_chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + @@ -938,7 +975,16 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: ); CompletionTimingData timing; - std::string output = inference_engine_->complete(prompt, params, &timing); + std::string output; + if (use_multi_turn) { + output = inference_engine_->completeMultiTurn( + session_id, + messages_array.dump(), + params, + &timing); + } else { + output = inference_engine_->complete(prompt, params, &timing); + } // Parse reasoning content from output auto reasoning_result = parseReasoningContent(output); @@ -1029,37 +1075,24 @@ void RyzenAIServer::handleChatCompletions(const httplib::Request& req, httplib:: void RyzenAIServer::handleMultimodalChat(const json& request_json, const ChatCompletionRequest& chat_req, - const std::vector& images, + const json& messages_array, + const std::vector& new_turn_images, + const std::vector& all_images, httplib::Response& res) { - // Build the prompt strictly via the OGA jinja template so image placeholders - // (<|vision_start|><|image_pad|><|vision_end|>) are emitted for each image. - json messages_array = request_json.contains("messages") ? request_json["messages"] : json::array(); std::string tools_json = chat_req.tools.empty() ? "" : chat_req.tools.dump(); - std::string prompt = inference_engine_->applyChatTemplateRaw(messages_array.dump(), tools_json); + const std::string session_id = kDefaultConversationId; + const std::string messages_json = messages_array.dump(); - std::cout << "[Server] Multimodal chat request (images=" << images.size() - << ", stream=" << chat_req.stream << ")" << std::endl; - std::cout << "[Server DEBUG] MM prompt (first 300): " - << prompt.substr(0, std::min(size_t(300), prompt.length())) << std::endl; + std::cout << "[Server] Multimodal chat request (new_turn_images=" << new_turn_images.size() + << ", all_images=" << all_images.size() + << ", stream=" << chat_req.stream << ", multi_turn=1)" << std::endl; GenerationParams params = createGenerationParams( chat_req.max_tokens, chat_req.temperature, chat_req.top_p, chat_req.top_k, chat_req.repeat_penalty, chat_req.stop); auto escapeJson = [](const std::string& str) -> std::string { - std::string e; - e.reserve(str.size() + 8); - for (char c : str) { - switch (c) { - case '\\': e += "\\\\"; break; - case '"': e += "\\\""; break; - case '\n': e += "\\n"; break; - case '\r': e += "\\r"; break; - case '\t': e += "\\t"; break; - default: e += c; break; - } - } - return e; + return escapeJsonString(str); }; const std::string model_id = model_id_; @@ -1070,15 +1103,18 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, res.set_header("Connection", "keep-alive"); res.set_header("X-Accel-Buffering", "no"); - int prompt_tokens = inference_engine_->countTokens(prompt); + std::string preview_prompt = inference_engine_->applyChatTemplateRaw(messages_json, tools_json); + int prompt_tokens = inference_engine_->countTokens(preview_prompt); res.set_chunked_content_provider( "text/event-stream", - [this, prompt, images, params, model_id, prompt_tokens, escapeJson](size_t offset, httplib::DataSink& sink) { + [this, messages_json, new_turn_images, all_images, tools_json, params, model_id, prompt_tokens, session_id, escapeJson]( + size_t offset, httplib::DataSink& sink) { if (offset > 0) return false; try { int token_count = 0; - inference_engine_->streamCompleteMultimodal(prompt, images, params, + inference_engine_->streamMultiTurnMultimodal( + session_id, messages_json, new_turn_images, all_images, tools_json, params, [&sink, model_id, &token_count, &escapeJson](const std::string& token, bool /*is_final*/) -> bool { std::string chunk = "data: {\"id\":\"chatcmpl-" + std::to_string(std::time(nullptr)) + @@ -1102,9 +1138,10 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, const char* done = "data: [DONE]\n\n"; sink.write(done, strlen(done)); sink.done(); - std::cout << "[Server] [OK] Streamed " << token_count << " multimodal tokens" << std::endl; + std::cout << "[Server] [OK] Streamed " << token_count << " multimodal multi-turn tokens" + << std::endl; } catch (const std::exception& e) { - std::cerr << "[ERROR] Multimodal streaming failed: " << e.what() << std::endl; + std::cerr << "[ERROR] Multimodal multi-turn streaming failed: " << e.what() << std::endl; json err = createErrorResponse(e.what(), "inference_error"); std::string es = "data: " + err.dump() + "\n\n"; sink.write(es.c_str(), es.size()); @@ -1115,10 +1152,11 @@ void RyzenAIServer::handleMultimodalChat(const json& request_json, return; } - // Non-streaming CompletionTimingData timing; - int prompt_tokens = inference_engine_->countTokens(prompt); - std::string content = inference_engine_->completeMultimodal(prompt, images, params, &timing); + std::string preview_prompt = inference_engine_->applyChatTemplateRaw(messages_json, tools_json); + int prompt_tokens = inference_engine_->countTokens(preview_prompt); + std::string content = inference_engine_->completeMultiTurnMultimodal( + session_id, messages_json, new_turn_images, all_images, tools_json, params, &timing); json response = { {"id", "chatcmpl-" + std::to_string(std::time(nullptr))}, @@ -1155,6 +1193,7 @@ void RyzenAIServer::run() { std::cout << " GET http://" << args_.host << ":" << args_.port << "/health\n"; std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/completions\n"; std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/chat/completions\n"; + std::cout << " POST http://" << args_.host << ":" << args_.port << "/v1/sessions/reset\n"; std::cout << "\n"; std::cout << "Press Ctrl+C to stop the server\n"; std::cout << "===============================================================\n\n"; @@ -1235,28 +1274,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon // Generate and send tokens in real-time inference_engine_->streamComplete(prompt, params, [&sink, &full_response](const std::string& token, bool is_final) -> bool { - // Escape special characters for JSON - std::string escaped_token = token; - size_t pos = 0; - while ((pos = escaped_token.find('\\', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('"', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\n', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_token.find('\r', pos)) != std::string::npos) { - escaped_token.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_token = escapeJsonString(token); // Accumulate unescaped token for final response full_response += token; @@ -1277,28 +1295,7 @@ void RyzenAIServer::handleResponses(const httplib::Request& req, httplib::Respon std::cout << "[Server] Token generation completed, sending final events" << std::endl; - // Escape full_response for JSON - std::string escaped_full_response = full_response; - size_t pos = 0; - while ((pos = escaped_full_response.find('\\', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\\"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('"', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\\""); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\n', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\n"); - pos += 2; - } - pos = 0; - while ((pos = escaped_full_response.find('\r', pos)) != std::string::npos) { - escaped_full_response.replace(pos, 1, "\\r"); - pos += 2; - } + std::string escaped_full_response = escapeJsonString(full_response); // Send response.completed event std::string completed_time = std::to_string(std::time(nullptr)); diff --git a/src/types.cpp b/src/types.cpp index 0e57953..e096360 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -140,6 +140,10 @@ ChatCompletionRequest ChatCompletionRequest::fromJSON(const json& j) { if (j.contains("tools")) { req.tools = j["tools"]; } + + if (j.contains("conversation_id") && j["conversation_id"].is_string()) { + req.conversation_id = j["conversation_id"]; + } return req; }