Skip to content

Honor default ep selection based on genai_config.json #966

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Baiju Meswani (baijumeswani) wants to merge 1 commit into from
Closed

Honor default ep selection based on genai_config.json #966

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 4 additions & 17 deletions sdk_v2/cpp/src/inferencing/model_load_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,36 +114,23 @@ ModelLoadManager::LoadResult ModelLoadManager::LoadModel(std::string_view model_
}

auto genai_config = GenAIConfig::LoadFromFile(config_path);

// Determine execution provider
auto resolved_ep = ep_override;

if (resolved_ep == ExecutionProvider::kDefault) {
// Auto-select EP for generic-gpu models: DML models are compatible with
// CUDA and WebGPU, so try those in order when available.
if (id_str.find("generic-gpu") != std::string::npos) {
if (HasEP("CUDAExecutionProvider")) {
resolved_ep = ExecutionProvider::kCUDA;
logger_.Log(LogLevel::Information, fmt::format("using CUDA EP for model: {}", id_str));
} else if (HasEP("WebGpuExecutionProvider")) {
resolved_ep = ExecutionProvider::kWebGPU;
logger_.Log(LogLevel::Information, fmt::format("using WebGPU EP for model: {}", id_str));
}
}
if (resolved_ep == ExecutionProvider::kDefault &&
id_str.find("generic-gpu") != std::string::npos &&
genai_config.DefaultProvider() == "dml") {
resolved_ep = ExecutionProvider::kWebGPU;
Comment on lines +118 to +121
}

// EP guard: verify the required EP is registered before attempting to load.
// OGA will crash or hang if we try to load a model with an unregistered EP.
if (resolved_ep != ExecutionProvider::kDefault && resolved_ep != ExecutionProvider::kCPU) {
// Explicit EP resolved — check it directly
auto required = EPUtils::EPtoRegistrationName(resolved_ep);
if (!required.empty() && !HasEP(std::string(required))) {
FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
"model '", id_str, "' requires ", required,
" which is not registered. Call DownloadAndRegisterEps() first.");
}
} else {
// No explicit EP — check model_id for device hints
auto required = RequiredEpForModelId(id_str);
if (!required.empty() && !HasEP(std::string(required))) {
FL_LOG_AND_THROW(logger_, FOUNDRY_LOCAL_ERROR_INVALID_USAGE,
Expand Down
4 changes: 2 additions & 2 deletions sdk_v2/cpp/src/inferencing/model_load_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ class ModelLoadManager {
/// Load a model from the given path using ORT GenAI.
/// @param model_path Path to the model directory (must contain genai_config.json).
/// @param model_id Unique identifier for the model.
/// @param ep_override Execution provider override (kDefault = use genai_config.json default,
/// or auto-select CUDA for generic-gpu models if available).
/// @param ep_override Execution provider override. kDefault preserves genai_config.json, except unsupported DML
/// generic-gpu models use WebGPU.
/// @returns LoadResult with status and non-owning pointer to the loaded model.
LoadResult LoadModel(std::string_view model_path,
std::string_view model_id,
Expand Down
31 changes: 10 additions & 21 deletions sdk_v2/cpp/test/internal_api/model_load_manager_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,6 @@

namespace {

/// EP detector that reports GPU EPs as available.
class GpuEpDetector : public fl::IEpDetector {
public:
std::map<std::string, std::vector<std::string>> GetAvailableDevicesToEPs() const override {
return {
{"CPU", {"CPUExecutionProvider"}},
{"GPU", {"CUDAExecutionProvider"}},
};
}
};

/// EP detector that reports CPU only.
class CpuOnlyDetector : public fl::IEpDetector {
public:
Expand All @@ -40,13 +29,14 @@ class CpuOnlyDetector : public fl::IEpDetector {
/// Cleans up on destruction.
class TempModelDir {
public:
TempModelDir(const std::string& model_name) {
TempModelDir(const std::string& model_name,
const std::string& config_json = R"({"model": {"type": "phi3"}})") {
path_ = (std::filesystem::temp_directory_path() / ("fl_test_" + model_name)).string();
std::filesystem::create_directories(path_);

// Write a minimal genai_config.json
std::ofstream config(std::filesystem::path(path_) / "genai_config.json");
config << R"({"model": {"type": "phi3"}})";
config << config_json;
}

~TempModelDir() {
Expand Down Expand Up @@ -153,20 +143,19 @@ TEST(ModelLoadManagerTest, LoadCpuModel_AlwaysSucceeds_NoEpGuard) {
}
}

TEST(ModelLoadManagerTest, LoadGenericGpuModel_CudaAvailable_AutoSelectsCuda) {
GpuEpDetector ep;
TEST(ModelLoadManagerTest, LoadGenericGpuDmlModel_DefaultRoutesToWebGpu) {
CpuOnlyDetector ep;
fl::StderrLogger logger;
fl::ModelLoadManager mgr(ep, logger);
TempModelDir dir("phi-4-mini-generic-gpu",
R"({"model":{"type":"phi3","decoder":{"session_options":{"provider_options":[{"dml":{}}]}}}})");

TempModelDir dir("phi-4-mini-generic-gpu");

// Will fail at GenAIModelInstance construction, but should NOT fail at EP guard.
// The auto-select logic should pick CUDA, and CUDA IS available.
try {
mgr.LoadModel(dir.path(), "phi-4-mini-generic-gpu");
FAIL() << "Expected WebGPU availability error";
} catch (const fl::Exception& e) {
// Should NOT be an EP guard error
EXPECT_NE(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE);
EXPECT_EQ(e.code(), FOUNDRY_LOCAL_ERROR_INVALID_USAGE);
EXPECT_NE(std::string(e.what()).find("WebGpuExecutionProvider"), std::string::npos);
}
}

Expand Down
Loading