Read the tokenizer config using path.open to support special characte…

…rs (#891)
microsoft · Feb 20, 2025 · 2893ff3 · 2893ff3
1 parent 30609b7
commit 2893ff3
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/operators/tokenizer/tokenizer_jsconfig.hpp b/operators/tokenizer/tokenizer_jsconfig.hpp
@@ -116,7 +116,7 @@ class TokenJsonConfig final {
         vocab_stream = std::make_unique<std::istringstream>(vocab_str);
       }
     } else {
-      auto ifs = std::make_unique<std::ifstream>(vocab_path_);
+      auto ifs = std::make_unique<std::ifstream>(path(vocab_path_.data()).open());
       if (!ifs->is_open()) {
         return OrtxStatus(extError_t::kOrtxErrorInvalidArgument, vocab_path_ + ": does not exist.");
       }