From 95bd5d300481e896788e4b35da96150bfe925b11 Mon Sep 17 00:00:00 2001 From: Baiju Meswani Date: Fri, 7 Aug 2026 15:27:29 -0700 Subject: [PATCH] Honor default execution provider --- .../cpp/src/inferencing/model_load_manager.cc | 21 +++---------- .../cpp/src/inferencing/model_load_manager.h | 4 +-- .../internal_api/model_load_manager_test.cc | 31 ++++++------------- 3 files changed, 16 insertions(+), 40 deletions(-) diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.cc b/sdk_v2/cpp/src/inferencing/model_load_manager.cc index 0bc321b9b..a64f478e5 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.cc +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.cc @@ -114,28 +114,16 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ } auto genai_config = GenAIConfig::LoadFromFile(config_path); - - // Determine execution provider auto resolved_ep = ep_override; - - if (resolved_ep == ExecutionProvider::kDefault) { - // Auto-select EP for generic-gpu models: DML models are compatible with - // CUDA and WebGPU, so try those in order when available. - if (id_str.find("generic-gpu") != std::string::npos) { - if (HasEP("CUDAExecutionProvider")) { - resolved_ep = ExecutionProvider::kCUDA; - logger_.Log(LogLevel::Information, fmt::format("using CUDA EP for model: {}", id_str)); - } else if (HasEP("WebGpuExecutionProvider")) { - resolved_ep = ExecutionProvider::kWebGPU; - logger_.Log(LogLevel::Information, fmt::format("using WebGPU EP for model: {}", id_str)); - } - } + if (resolved_ep == ExecutionProvider::kDefault && + id_str.find("generic-gpu") != std::string::npos && + genai_config.DefaultProvider() == "dml") { + resolved_ep = ExecutionProvider::kWebGPU; } // EP guard: verify the required EP is registered before attempting to load. // OGA will crash or hang if we try to load a model with an unregistered EP. if (resolved_ep != ExecutionProvider::kDefault && resolved_ep != ExecutionProvider::kCPU) { - // Explicit EP resolved — check it directly auto required = EPUtils::EPtoRegistrationName(resolved_ep); if (!required.empty() && !HasEP(std::string(required))) { FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, @@ -143,7 +131,6 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_ " which is not registered. Call DownloadAndRegisterEps() first."); } } else { - // No explicit EP — check model_id for device hints auto required = RequiredEpForModelId(id_str); if (!required.empty() && !HasEP(std::string(required))) { FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE, diff --git a/sdk_v2/cpp/src/inferencing/model_load_manager.h b/sdk_v2/cpp/src/inferencing/model_load_manager.h index f582217ad..1d051481d 100644 --- a/sdk_v2/cpp/src/inferencing/model_load_manager.h +++ b/sdk_v2/cpp/src/inferencing/model_load_manager.h @@ -48,8 +48,8 @@ class ModelLoadManager { /// Load a model from the given path using ORT GenAI. /// @param model_path Path to the model directory (must contain genai_config.json). /// @param model_id Unique identifier for the model. - /// @param ep_override Execution provider override (kDefault = use genai_config.json default, - /// or auto-select CUDA for generic-gpu models if available). + /// @param ep_override Execution provider override. kDefault preserves genai_config.json, except unsupported DML + /// generic-gpu models use WebGPU. /// @returns LoadResult with status and non-owning pointer to the loaded model. LoadResult LoadModel(std::string_view model_path, std::string_view model_id, diff --git a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc index 695f0e5d7..1e2f4b6b0 100644 --- a/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc +++ b/sdk_v2/cpp/test/internal_api/model_load_manager_test.cc @@ -17,17 +17,6 @@ namespace { -/// EP detector that reports GPU EPs as available. -class GpuEpDetector : public fl::IEpDetector { - public: - std::map> GetAvailableDevicesToEPs() const override { - return { - {"CPU", {"CPUExecutionProvider"}}, - {"GPU", {"CUDAExecutionProvider"}}, - }; - } -}; - /// EP detector that reports CPU only. class CpuOnlyDetector : public fl::IEpDetector { public: @@ -40,13 +29,14 @@ class CpuOnlyDetector : public fl::IEpDetector { /// Cleans up on destruction. class TempModelDir { public: - TempModelDir(const std::string& model_name) { + TempModelDir(const std::string& model_name, + const std::string& config_json = R"({"model": {"type": "phi3"}})") { path_ = (std::filesystem::temp_directory_path() / ("fl_test_" + model_name)).string(); std::filesystem::create_directories(path_); // Write a minimal genai_config.json std::ofstream config(std::filesystem::path(path_) / "genai_config.json"); - config << R"({"model": {"type": "phi3"}})"; + config << config_json; } ~TempModelDir() { @@ -153,20 +143,19 @@ TEST(ModelLoadManagerTest, LoadCpuModel_AlwaysSucceeds_NoEpGuard) { } } -TEST(ModelLoadManagerTest, LoadGenericGpuModel_CudaAvailable_AutoSelectsCuda) { - GpuEpDetector ep; +TEST(ModelLoadManagerTest, LoadGenericGpuDmlModel_DefaultRoutesToWebGpu) { + CpuOnlyDetector ep; fl::StderrLogger logger; fl::ModelLoadManager mgr(ep, logger); + TempModelDir dir("phi-4-mini-generic-gpu", + R"({"model":{"type":"phi3","decoder":{"session_options":{"provider_options":[{"dml":{}}]}}}})"); - TempModelDir dir("phi-4-mini-generic-gpu"); - - // Will fail at GenAIModelInstance construction, but should NOT fail at EP guard. - // The auto-select logic should pick CUDA, and CUDA IS available. try { mgr.LoadModel(dir.path(), "phi-4-mini-generic-gpu"); + FAIL() << "Expected WebGPU availability error"; } catch (const fl::Exception& e) { - // Should NOT be an EP guard error - EXPECT_NE(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE); + EXPECT_NE(std::string(e.what()).find("WebGpuExecutionProvider"), std::string::npos); } }