Gate regex lookahead in cmake

Author: jackzhxng

CMakeLists.txt

@@ -18,6 +18,7 @@ project(Tokenizers)
 
 option(TOKENIZERS_BUILD_TEST "Build tests" OFF)
 option(TOKENIZERS_BUILD_TOOLS "Build tools" OFF)
+option(SUPPORT_REGEX_LOOKAHEAD "Support regex lookahead patterns (requires PCRE2)" OFF)
 
 # Ignore weak attribute warning
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
@@ -31,20 +32,25 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/re2)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece)
 
 # Configure PCRE2
-set(PCRE2_BUILD_PCRE2_8 ON)
-set(PCRE2_BUILD_PCRE2_16 OFF)
-set(PCRE2_BUILD_PCRE2_32 OFF)
-set(PCRE2_BUILD_TESTS OFF)
-set(PCRE2_BUILD_PCRE2GREP OFF)
-set(PCRE2_BUILD_PCRE2TEST OFF)
-set(PCRE2_BUILD_PCRE2GPERF OFF)
-set(PCRE2_BUILD_DOCS OFF)
-set(PCRE2_BUILD_LIBPCRE2_PDB OFF)
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2)
+if(SUPPORT_REGEX_LOOKAHEAD)
+    set(PCRE2_BUILD_PCRE2_8 ON)
+    set(PCRE2_BUILD_PCRE2_16 OFF)
+    set(PCRE2_BUILD_PCRE2_32 OFF)
+    set(PCRE2_BUILD_TESTS OFF)
+    set(PCRE2_BUILD_PCRE2GREP OFF)
+    set(PCRE2_BUILD_PCRE2TEST OFF)
+    set(PCRE2_BUILD_PCRE2GPERF OFF)
+    set(PCRE2_BUILD_DOCS OFF)
+    set(PCRE2_BUILD_LIBPCRE2_PDB OFF)
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2)
+endif()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
 
 file(GLOB tokenizers_source_files ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
+if(NOT SUPPORT_REGEX_LOOKAHEAD)
+    list(REMOVE_ITEM tokenizers_source_files ${CMAKE_CURRENT_SOURCE_DIR}/src/pcre2_regex.cpp)
+endif()
 file(GLOB unicode_source_files
      ${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/src/*.cpp)
 add_library(tokenizers STATIC ${tokenizers_source_files}
@@ -58,10 +64,14 @@ target_include_directories(
          ${CMAKE_CURRENT_SOURCE_DIR}/third-party/sentencepiece/src
          ${CMAKE_CURRENT_SOURCE_DIR}/third-party/re2
          ${CMAKE_CURRENT_SOURCE_DIR}/third-party/json/single_include
-         ${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/include
-         ${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src)
+         ${CMAKE_CURRENT_SOURCE_DIR}/third-party/llama.cpp-unicode/include)
+target_link_libraries(tokenizers PUBLIC sentencepiece-static re2::re2)
 
-target_link_libraries(tokenizers PUBLIC sentencepiece-static re2::re2 pcre2-8)
+if(SUPPORT_REGEX_LOOKAHEAD)
+    target_include_directories(tokenizers PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src)
+    target_link_libraries(tokenizers PUBLIC pcre2-8)
+    target_compile_definitions(tokenizers PUBLIC SUPPORT_REGEX_LOOKAHEAD)
+endif()
 
 # Build test
 if(TOKENIZERS_BUILD_TEST)

src/regex.cpp

@@ -6,7 +6,9 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+#ifdef SUPPORT_REGEX_LOOKAHEAD
 #include <pytorch/tokenizers/pcre2_regex.h>
+#endif
 #include <pytorch/tokenizers/re2_regex.h>
 #include <pytorch/tokenizers/regex.h>
 #include <pytorch/tokenizers/std_regex.h>
@@ -19,8 +21,8 @@ namespace tokenizers {
 
 /**
  * @brief Factory function that creates a regex object using RE2 if possible.
- *        Falls back to PCRE2 if RE2 rejects the pattern, then to std::regex if
- * PCRE2 fails.
+ *        Falls back to PCRE2 if RE2 rejects the pattern and SUPPORT_REGEX_LOOKAHEAD is enabled.
+ *        Otherwise, returns an error.
  */
 Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern) {
   // Try RE2 first
@@ -30,6 +32,12 @@ Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern) {
     return static_cast<std::unique_ptr<IRegex>>(std::move(re2));
   }
 
+#ifndef SUPPORT_REGEX_LOOKAHEAD
+  std::cerr << "RE2 failed to compile pattern with lookahead: " << pattern << "\n";
+  std::cerr << "Error: " << (re2->regex_->error()) << std::endl;
+  std::cerr << "Compile with SUPPORT_REGEX_LOOKAHEAD=ON to enable support for lookahead patterns." << std::endl;
+  return tokenizers::Error::LoadFailure;
+#else
   if (re2->regex_->error_code() == re2::RE2::ErrorBadPerlOp) {
     // RE2 doesn't support some Perl features, try PCRE2
     auto pcre2 = std::make_unique<Pcre2Regex>("(" + pattern + ")");
@@ -56,6 +64,7 @@ Result<std::unique_ptr<IRegex>> create_regex(const std::string& pattern) {
     std::cerr << "Error: " << (re2->regex_->error()) << std::endl;
     return tokenizers::Error::LoadFailure;
   }
+#endif
 }
 
 } // namespace tokenizers