[RELEASE ONLY CHANGES] Leave the tokenizer dependency unpinned

Author: JacobSzwejbka

pyproject.toml

@@ -63,8 +63,7 @@ requires-python = ">=3.11,<3.16"
 # computed in setup.py, so the EXECUTORCH_BUILD_MINIMAL wheel can ship a slimmer
 # set than the full wheel. See `_base_dependencies()` / `_minimal_dependencies()`
 # in setup.py.
-# The 1.6 full wheel pins torch==2.15.0 and requires pytorch-tokenizers>=1.6.0
-# and torchao>=0.18.0.
+# The 1.6 full wheel pins torch==2.15.0 and requires torchao>=0.18.0.
 
 [project.optional-dependencies]
 vgf = [

setup.py

@@ -1157,7 +1157,7 @@ def _base_dependencies() -> List[str]:
         # scope. Neither is needed merely to import the backend.
         "py-cpuinfo",
         "requests",
-        "pytorch-tokenizers>=1.6.0",
+        "pytorch-tokenizers",
         "torch==2.15.0",
         # Shipped code imports torchao at module scope in many places, so a plain install cannot
         # lower a model without it. Among others: the XNNPACK utilities the partitioner uses