From b80c5662e2aa7aa21bdb9b2ab1a7e6ae6486db57 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 28 Aug 2025 15:37:30 -0600 Subject: [PATCH 1/3] Support the free-threaded build --- .github/workflows/build_wheels.yml | 6 +++--- src/lib.rs | 2 +- src/py.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9f14b28..9f5b572 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,12 +17,12 @@ jobs: # cibuildwheel builds linux wheels inside a manylinux container # it also takes care of procuring the correct python version for us os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [39, 310, 311, 312, 313] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.22.0 + - uses: pypa/cibuildwheel@v3.1.4 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" @@ -39,7 +39,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04-arm] - python-version: [39, 310, 311, 312, 313] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 diff --git a/src/lib.rs b/src/lib.rs index 80f4acc..5e89bfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -187,7 +187,7 @@ impl std::error::Error for EncodeError {} const MAX_NUM_THREADS: usize = 128; -#[cfg_attr(feature = "python", pyclass)] +#[cfg_attr(feature = "python", pyclass(frozen))] #[derive(Clone)] pub struct CoreBPE { encoder: HashMap, Rank>, diff --git a/src/py.rs b/src/py.rs index 253e5d0..ee8c52d 100644 --- a/src/py.rs +++ b/src/py.rs @@ -183,7 +183,7 @@ impl CoreBPE { } } -#[pyclass] +#[pyclass(frozen)] struct TiktokenBuffer { tokens: Vec, } @@ -248,7 +248,7 @@ impl TiktokenBuffer { } } -#[pymodule] +#[pymodule(gil_used = false)] fn _tiktoken(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; Ok(()) From e61c88de995543acf83ff1a1f089e5a0ef8042b2 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 28 Aug 2025 15:43:49 -0600 Subject: [PATCH 2/3] adjust cibuildwheel config --- .github/workflows/build_wheels.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9f5b572..34fe41f 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,14 +17,15 @@ jobs: # cibuildwheel builds linux wheels inside a manylinux container # it also takes care of procuring the correct python version for us os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] + python-version: [39, 310, 311, 312, 313, 313t] steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v3.1.4 + - uses: pypa/cibuildwheel@v2.23.3 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" + CIBW_ENABLE: cpython-freethreading - uses: actions/upload-artifact@v4 with: @@ -39,19 +40,20 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04-arm] - python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] + python-version: [39, 310, 311, 312, 313, 313t] steps: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" CIBW_ARCHS: aarch64 CIBW_BUILD_VERBOSITY: 3 # https://github.com/rust-lang/cargo/issues/10583 CIBW_ENVIRONMENT_LINUX: PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI=true + CIBW_ENABLE: cpython-freethreading - uses: actions/upload-artifact@v4 with: From 1ecd7571d4c59b32ab7efca4e424c6a36c87a5cc Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 28 Aug 2025 16:32:21 -0600 Subject: [PATCH 3/3] add python 3.14 support --- .github/workflows/build_wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 34fe41f..31a74cd 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,12 +17,12 @@ jobs: # cibuildwheel builds linux wheels inside a manylinux container # it also takes care of procuring the correct python version for us os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [39, 310, 311, 312, 313, 313t] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.23.3 + - uses: pypa/cibuildwheel@v3.1.4 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" CIBW_ENABLE: cpython-freethreading @@ -40,13 +40,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04-arm] - python-version: [39, 310, 311, 312, 313, 313t] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.1.4 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" CIBW_ARCHS: aarch64