From a5b392513528103ab0334987d85d4a388f80c7c5 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 25 Oct 2025 21:45:50 +0100 Subject: [PATCH 1/3] replace some extras functions with functions from the stable abi --- src/C/extras.jl | 59 +++++++++++++++++++--------------------- src/C/pointers.jl | 6 ++++ src/Convert/pyconvert.jl | 2 +- src/Wrap/PyArray.jl | 2 +- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/C/extras.jl b/src/C/extras.jl index cf46b6b8..520535a7 100644 --- a/src/C/extras.jl +++ b/src/C/extras.jl @@ -2,48 +2,45 @@ asptr(x) = Base.unsafe_convert(PyPtr, x) Py_Type(x) = Base.GC.@preserve x PyPtr(UnsafePtr(asptr(x)).type[!]) -PyObject_Type(x) = Base.GC.@preserve x (t = Py_Type(asptr(x)); Py_IncRef(t); t) - Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), asptr(t)) Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f) -PyType_IsSubtypeFast(t, f::Integer) = - Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f)) +PyType_IsSubtypeFast(t, f::Integer) = Cint(!iszero(PyType_GetFlags(t) & f)) PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) PyType_CheckBuffer(t) = Base.GC.@preserve t begin - p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[] - return p != C_NULL && p.get[!] != C_NULL + o = Ref{PyObject}(PyObject(0, asptr(t))) + PyObject_CheckBuffer(o) end -PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o))) +# PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o))) -PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin - o = asptr(_o) - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if p == C_NULL || p.get[!] == C_NULL - PyErr_SetString( - POINTERS.PyExc_TypeError, - "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'", - ) - return Cint(-1) - end - return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) -end +# PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin +# o = asptr(_o) +# p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] +# if p == C_NULL || p.get[!] == C_NULL +# PyErr_SetString( +# POINTERS.PyExc_TypeError, +# "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'", +# ) +# return Cint(-1) +# end +# return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags) +# end -PyBuffer_Release(_b) = begin - b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) - o = b.obj[] - o == C_NULL && return - p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] - if (p != C_NULL && p.release[!] != C_NULL) - ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) - end - b.obj[] = C_NULL - Py_DecRef(o) - return -end +# PyBuffer_Release(_b) = begin +# b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b)) +# o = b.obj[] +# o == C_NULL && return +# p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[] +# if (p != C_NULL && p.release[!] != C_NULL) +# ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b) +# end +# b.obj[] = C_NULL +# Py_DecRef(o) +# return +# end function PyOS_SetInputHook(hook::Ptr{Cvoid}) Base.unsafe_store!(POINTERS.PyOS_InputHookPtr, hook) diff --git a/src/C/pointers.jl b/src/C/pointers.jl index d7441960..c7cc2701 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -72,11 +72,15 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyObject_GetIter => (PyPtr,) => PyPtr, :PyObject_Call => (PyPtr, PyPtr, PyPtr) => PyPtr, :PyObject_CallObject => (PyPtr, PyPtr) => PyPtr, + :PyObject_Type => (PyPtr,) => PyPtr, + :PyObject_CheckBuffer => (PyPtr,) => Cint, + :PyObject_GetBuffer => (PyPtr, Ptr{Py_buffer}, Cint) => Cint, # TYPE :PyType_IsSubtype => (PyPtr, PyPtr) => Cint, :PyType_Ready => (PyPtr,) => Cint, :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr, :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr, + :PyType_GetFlags => (PyPtr,) => Culong, # MAPPING :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint, :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint, @@ -175,6 +179,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyCapsule_SetName => (PyPtr, Ptr{Cchar}) => Cint, :PyCapsule_GetPointer => (PyPtr, Ptr{Cchar}) => Ptr{Cvoid}, :PyCapsule_SetDestructor => (PyPtr, Ptr{Cvoid}) => Cint, + # BUFFER + :PyBuffer_Release => (Ptr{Py_buffer},) => Cvoid, ) const CAPI_EXCEPTIONS = Set([ diff --git a/src/Convert/pyconvert.jl b/src/Convert/pyconvert.jl index 2b03eb72..13581108 100644 --- a/src/Convert/pyconvert.jl +++ b/src/Convert/pyconvert.jl @@ -230,7 +230,7 @@ function _pyconvert_get_rules(pytype::Py) end end for (t, x) in reverse(collect(zip(mro, xmro))) - if C.PyType_CheckBuffer(t) + if C.PyType_CheckBuffer(t) == 1 push!(x, "") break end diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index 7b1dc8d3..43a8bad3 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -83,7 +83,7 @@ function pyarray_make( @debug "failed to make PyArray from __array_interface__" exc = exc end end - if buffer && C.PyObject_CheckBuffer(x) + if buffer && C.PyObject_CheckBuffer(x) == 1 try return pyarray_make(A, x, PyArraySource_Buffer(x)) catch exc From 6fac15c790206bc18492378a57b61b5bd33f7017 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 25 Oct 2025 22:27:21 +0100 Subject: [PATCH 2/3] remove reliance on PyMemoryView layout --- src/API/types.jl | 4 ++-- src/C/consts.jl | 10 --------- src/C/extras.jl | 2 -- src/JlWrap/io.jl | 46 ++++++++++++++++++-------------------- src/Wrap/PyArray.jl | 54 ++++++++++++++++++++++++--------------------- 5 files changed, 52 insertions(+), 64 deletions(-) diff --git a/src/API/types.jl b/src/API/types.jl index 55a4211e..3ee0fced 100644 --- a/src/API/types.jl +++ b/src/API/types.jl @@ -72,14 +72,14 @@ struct PyArray{T,N,M,L,R} <: AbstractArray{T,N} size::NTuple{N,Int} # size of the array strides::NTuple{N,Int} # strides (in bytes) between elements py::Py # underlying python object - handle::Py # the data in this array is valid as long as this handle is alive + handle::Any # the data in this array is valid as long as this handle is alive function PyArray{T,N,M,L,R}( ::Val{:new}, ptr::Ptr{R}, size::NTuple{N,Int}, strides::NTuple{N,Int}, py::Py, - handle::Py, + handle::Any, ) where {T,N,M,L,R} T isa Type || error("T must be a Type") N isa Int || error("N must be an Int") diff --git a/src/C/consts.jl b/src/C/consts.jl index 4560e928..aaebb6fb 100644 --- a/src/C/consts.jl +++ b/src/C/consts.jl @@ -239,16 +239,6 @@ end internal::Ptr{Cvoid} = C_NULL end -@kwdef struct PyMemoryViewObject - ob_base::PyVarObject = PyVarObject() - mbuf::PyPtr = PyNULL - hash::Py_hash_t = 0 - flags::Cint = 0 - exports::Py_ssize_t = 0 - view::Py_buffer = Py_buffer() - weakreflist::PyPtr = PyNULL -end - @kwdef struct PyTypeObject ob_base::PyVarObject = PyVarObject() name::Cstring = C_NULL diff --git a/src/C/extras.jl b/src/C/extras.jl index 520535a7..b18b6813 100644 --- a/src/C/extras.jl +++ b/src/C/extras.jl @@ -7,8 +7,6 @@ Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Ty PyType_IsSubtypeFast(t, f::Integer) = Cint(!iszero(PyType_GetFlags(t) & f)) -PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view) - PyType_CheckBuffer(t) = Base.GC.@preserve t begin o = Ref{PyObject}(PyObject(0, asptr(t))) PyObject_CheckBuffer(o) diff --git a/src/JlWrap/io.jl b/src/JlWrap/io.jl index 5fa54a57..74c6c155 100644 --- a/src/JlWrap/io.jl +++ b/src/JlWrap/io.jl @@ -94,42 +94,38 @@ pyjl_handle_error_type(::typeof(pyjlbinaryio_readline), io, exc) = exc isa MethodError && exc.f === read ? pybuiltins.ValueError : PyNULL function pyjlbinaryio_readinto(io::IO, b::Py) - m = pybuiltins.memoryview(b) - c = m.c_contiguous - if !pytruth(c) - pydel!(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + buf = Ref{Py_buffer}() + if PyObject_GetBuffer(b, buf, PyBUF_SIMPLE | PyBUF_WRITABLE) < 0 return PyNULL end - pydel!(c) - buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) - if buf.readonly != 0 - pydel!(m) - errset(pybuiltins.ValueError, "output buffer is read-only") - return PyNULL + local nb + ptr = buf[].buf + len = buf[].len + try + data = unsafe_wrap(Array, Ptr{UInt8}(ptr), len) + nb = readbytes!(io, data) + finally + PyBuffer_Release(buf) end - data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) - nb = readbytes!(io, data) - pydel!(m) return Py(nb) end pyjl_handle_error_type(::typeof(pyjlbinaryio_readinto), io, exc) = exc isa MethodError && exc.f === readbytes! ? pybuiltins.ValueError : PyNULL function pyjlbinaryio_write(io::IO, b::Py) - m = pybuiltins.memoryview(b) - c = m.c_contiguous - if !pytruth(c) - pydel!(c) - errset(pybuiltins.ValueError, "input buffer is not contiguous") + buf = Ref{Py_buffer}() + if PyObject_GetBuffer(b, buf, PyBUF_SIMPLE) < 0 return PyNULL end - pydel!(c) - buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m)) - data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len) - write(io, data) - pydel!(m) - return Py(buf.len) + ptr = buf[].buf + len = buf[].len + try + data = unsafe_wrap(Array, Ptr{UInt8}(ptr), len) + write(io, data) + finally + PyBuffer_Release(buf) + end + return Py(len) end pyjl_handle_error_type(::typeof(pyjlbinaryio_write), io, exc) = exc isa MethodError && exc.f === write ? pybuiltins.ValueError : PyNULL diff --git a/src/Wrap/PyArray.jl b/src/Wrap/PyArray.jl index 43a8bad3..fc5b788f 100644 --- a/src/Wrap/PyArray.jl +++ b/src/Wrap/PyArray.jl @@ -187,7 +187,7 @@ struct PyArraySource_ArrayInterface <: PyArraySource dict::Py ptr::Ptr{Cvoid} readonly::Bool - handle::Py + handle::Any end function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) # offset @@ -202,14 +202,16 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__) ptr = Ptr{Cvoid}(pyconvert(UInt, data[0])) readonly = pyconvert(Bool, data[1]) pydel!(data) - handle = Py((x, d)) + handle = (x, d) else - memview = @py memoryview(data === None ? x : data) - pydel!(data) - buf = UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) - ptr = buf.buf[!] - readonly = buf.readonly[] != 0 - handle = Py((x, memview)) + buf = Ref{C.Py_buffer}() + if C.PyObject_GetBuffer(data === None ? x : data, buf, C.PyBUF_RECORDS) < 0 + pythrow() + end + finalizer(C.PyBuffer_Release, buf) + ptr = buf[].buf + readonly = buf[].readonly != 0 + handle = buf end PyArraySource_ArrayInterface(x, d, ptr, readonly, handle) end @@ -525,13 +527,15 @@ end struct PyArraySource_Buffer <: PyArraySource obj::Py - memview::Py - buf::C.UnsafePtr{C.Py_buffer} + buf::Base.RefValue{C.Py_buffer} end function PyArraySource_Buffer(x::Py) - memview = pybuiltins.memoryview(x) - buf = C.UnsafePtr(C.PyMemoryView_GET_BUFFER(memview)) - PyArraySource_Buffer(x, memview, buf) + buf = Ref{C.Py_buffer}() + if C.PyObject_GetBuffer(x, buf, C.PyBUF_RECORDS) < 0 + pythrow() + end + finalizer(C.PyBuffer_Release, buf) + PyArraySource_Buffer(x, buf) end const PYARRAY_BUFFERFORMAT_TO_TYPE = let c = Utils.islittleendian() ? '<' : '>' @@ -578,20 +582,20 @@ pyarray_bufferformat_to_type(fmt::String) = get( ) function pyarray_get_R(src::PyArraySource_Buffer) - ptr = src.buf.format[] - return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(String(ptr)) + ptr = src.buf[].format + return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(unsafe_string(ptr)) end -pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf.buf[!]) +pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf[].buf) -pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf.ndim[]) +pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf[].ndim) function pyarray_get_size(src::PyArraySource_Buffer, ::Val{N}) where {N} - size = src.buf.shape[] + size = src.buf[].shape if size == C_NULL - N == 0 ? () : N == 1 ? (Int(src.buf.len[]),) : @assert false + N == 0 ? () : N == 1 ? (Int(src.buf[].len),) : @assert false else - ntuple(i -> Int(size[i]), N) + ntuple(i -> Int(unsafe_load(size, i)), N) end end @@ -601,18 +605,18 @@ function pyarray_get_strides( ::Type{R}, size::NTuple{N,Int}, ) where {N,R} - strides = src.buf.strides[] + strides = src.buf[].strides if strides == C_NULL - itemsize = src.buf.shape[] == C_NULL ? 1 : src.buf.itemsize[] + itemsize = src.buf[].shape == C_NULL ? 1 : src.buf[].itemsize Utils.size_to_cstrides(itemsize, size) else - ntuple(i -> Int(strides[i]), N) + ntuple(i -> Int(unsafe_load(strides, i)), N) end end -pyarray_get_M(src::PyArraySource_Buffer) = src.buf.readonly[] == 0 +pyarray_get_M(src::PyArraySource_Buffer) = src.buf[].readonly == 0 -pyarray_get_handle(src::PyArraySource_Buffer) = src.memview +pyarray_get_handle(src::PyArraySource_Buffer) = src.buf # AbstractArray methods From e7e0fbcbf5b63b48f71c2892d478dbf501aa2f78 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Sat, 25 Oct 2025 22:34:14 +0100 Subject: [PATCH 3/3] test more python versions --- .github/workflows/tests.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index db580f50..3b8d0bc0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: julia: - name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonexe }}) + name: Test Julia (${{ matrix.jlversion }}, ${{ matrix.os }}, ${{ matrix.pythonstr }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -21,11 +21,39 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] jlversion: ['1','1.10'] pythonexe: ['@CondaPkg'] + pythonver: ['@CondaPkg'] + pythonstr: ['@CondaPkg'] include: - arch: x64 os: ubuntu-latest jlversion: '1' pythonexe: python + pythonver: '3.14' + pythonstr: python 3.14 + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python + pythonver: '3.13' + pythonstr: python 3.13 + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python + pythonver: '3.12' + pythonstr: python 3.12 + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python + pythonver: '3.11' + pythonstr: python 3.11 + - arch: x64 + os: ubuntu-latest + jlversion: '1' + pythonexe: python + pythonver: '3.10' + pythonstr: python 3.10 steps: - uses: actions/checkout@v5 @@ -36,6 +64,12 @@ jobs: version: ${{ matrix.jlversion }} arch: ${{ matrix.arch }} + - name: Set up Python ${{ matrix.pythonver }} + if: ${{ matrix.pythonexe == 'python' }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.pythonver }} + - uses: julia-actions/cache@v2 - name: Build package