diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 9f14b28..31a74cd 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -17,14 +17,15 @@ jobs: # cibuildwheel builds linux wheels inside a manylinux container # it also takes care of procuring the correct python version for us os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [39, 310, 311, 312, 313] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.22.0 + - uses: pypa/cibuildwheel@v3.1.4 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" + CIBW_ENABLE: cpython-freethreading - uses: actions/upload-artifact@v4 with: @@ -39,19 +40,20 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04-arm] - python-version: [39, 310, 311, 312, 313] + python-version: [39, 310, 311, 312, 313, 313t, 314, 314t] steps: - uses: actions/checkout@v4 - name: Build wheels - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.1.4 env: CIBW_BUILD: "cp${{ matrix.python-version}}-*" CIBW_ARCHS: aarch64 CIBW_BUILD_VERBOSITY: 3 # https://github.com/rust-lang/cargo/issues/10583 CIBW_ENVIRONMENT_LINUX: PATH="$PATH:$HOME/.cargo/bin" CARGO_NET_GIT_FETCH_WITH_CLI=true + CIBW_ENABLE: cpython-freethreading - uses: actions/upload-artifact@v4 with: diff --git a/src/lib.rs b/src/lib.rs index 80f4acc..5e89bfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -187,7 +187,7 @@ impl std::error::Error for EncodeError {} const MAX_NUM_THREADS: usize = 128; -#[cfg_attr(feature = "python", pyclass)] +#[cfg_attr(feature = "python", pyclass(frozen))] #[derive(Clone)] pub struct CoreBPE { encoder: HashMap, Rank>, diff --git a/src/py.rs b/src/py.rs index 253e5d0..ee8c52d 100644 --- a/src/py.rs +++ b/src/py.rs @@ -183,7 +183,7 @@ impl CoreBPE { } } -#[pyclass] +#[pyclass(frozen)] struct TiktokenBuffer { tokens: Vec, } @@ -248,7 +248,7 @@ impl TiktokenBuffer { } } -#[pymodule] +#[pymodule(gil_used = false)] fn _tiktoken(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; Ok(())