From 5e2afe41f9d984b8adb27b9c63179443176673a4 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sun, 13 Apr 2025 17:05:07 +0200 Subject: [PATCH] Expose cereggii's C-APIs --- examples/c-api/CMakeLists.txt | 5 + examples/c-api/README.md | 11 + examples/c-api/pyproject.toml | 34 + examples/c-api/src/CMakeLists.txt | 41 + examples/c-api/src/c_api/__init__.py | 10 + examples/c-api/src/c_api/c_api.c | 92 ++ pyproject.toml | 8 +- src/CMakeLists.txt | 31 +- src/cereggii/atomic_dict/accessor_storage.c | 4 +- src/cereggii/atomic_dict/atomic_dict.c | 6 +- src/cereggii/atomic_dict/blocks.c | 6 +- src/cereggii/atomic_dict/delete.c | 4 +- src/cereggii/atomic_dict/insert.c | 10 +- src/cereggii/atomic_dict/iter.c | 4 +- src/cereggii/atomic_dict/lookup.c | 6 +- src/cereggii/atomic_dict/meta.c | 4 +- src/cereggii/atomic_dict/migrate.c | 12 +- src/cereggii/atomic_dict/node_ops.c | 4 +- src/cereggii/atomic_event.c | 2 +- src/cereggii/atomic_int/atomic_int.c | 6 +- src/cereggii/atomic_int/handle.c | 4 +- src/cereggii/atomic_ops.c | 404 ------ src/cereggii/atomic_ref.c | 6 +- src/cereggii/cereggii.c | 12 +- src/cereggii/constants.c | 2 +- .../include/cereggii}/_internal_py_core.h | 0 .../include/cereggii}/atomic_dict.h | 0 .../include/cereggii}/atomic_dict_internal.h | 0 .../include/cereggii}/atomic_event.h | 0 .../include/cereggii}/atomic_int.h | 0 .../include/cereggii}/atomic_int_internal.h | 0 .../include/cereggii}/atomic_ref.h | 0 src/cereggii/include/cereggii/atomics.h | 65 + .../include/cereggii}/constants.h | 0 .../include/cereggii/internal/.gitignore | 1 + .../cereggii/internal/atomics_declarations.h | 445 ++++++ .../include/cereggii/internal/atomics_gcc.h | 619 +++++++++ .../include/cereggii/internal/atomics_msc.h | 1172 ++++++++++++++++ .../cereggii/internal/atomics_stdatomic.h | 624 +++++++++ .../include/cereggii/internal/export.h | 18 + .../include/cereggii/internal/pyatomic_msc.h | 1187 +++++++++++++++++ .../include/cereggii}/thread_id.h | 0 src/include/atomic_ops.h | 320 ----- 43 files changed, 4401 insertions(+), 778 deletions(-) create mode 100644 examples/c-api/CMakeLists.txt create mode 100644 examples/c-api/README.md create mode 100644 examples/c-api/pyproject.toml create mode 100644 examples/c-api/src/CMakeLists.txt create mode 100644 examples/c-api/src/c_api/__init__.py create mode 100644 examples/c-api/src/c_api/c_api.c delete mode 100644 src/cereggii/atomic_ops.c rename src/{include => cereggii/include/cereggii}/_internal_py_core.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_dict.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_dict_internal.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_event.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_int.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_int_internal.h (100%) rename src/{include => cereggii/include/cereggii}/atomic_ref.h (100%) create mode 100644 src/cereggii/include/cereggii/atomics.h rename src/{include => cereggii/include/cereggii}/constants.h (100%) create mode 100644 src/cereggii/include/cereggii/internal/.gitignore create mode 100644 src/cereggii/include/cereggii/internal/atomics_declarations.h create mode 100644 src/cereggii/include/cereggii/internal/atomics_gcc.h create mode 100644 src/cereggii/include/cereggii/internal/atomics_msc.h create mode 100644 src/cereggii/include/cereggii/internal/atomics_stdatomic.h create mode 100644 src/cereggii/include/cereggii/internal/export.h create mode 100644 src/cereggii/include/cereggii/internal/pyatomic_msc.h rename src/{include => cereggii/include/cereggii}/thread_id.h (100%) delete mode 100644 src/include/atomic_ops.h diff --git a/examples/c-api/CMakeLists.txt b/examples/c-api/CMakeLists.txt new file mode 100644 index 00000000..a627c6fc --- /dev/null +++ b/examples/c-api/CMakeLists.txt @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.18) +project(c_api) + + +add_subdirectory(src) diff --git a/examples/c-api/README.md b/examples/c-api/README.md new file mode 100644 index 00000000..3c333b36 --- /dev/null +++ b/examples/c-api/README.md @@ -0,0 +1,11 @@ +# Using cereggii's C-APIs + +This examples showcases how to use cereggii's C-level APIs, highlighting how to +correctly specify it as a build-time dependency. + +The problems: + +1. we need to be able to compile the program using cereggii's headers. +2. we may additionally need to link the pre-built shared library to use the C-APIs +at runtime. + - you don't need to link against the shared library, if you only need the header-only atomics. diff --git a/examples/c-api/pyproject.toml b/examples/c-api/pyproject.toml new file mode 100644 index 00000000..8c813be5 --- /dev/null +++ b/examples/c-api/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = [ + "cereggii", + "py-build-cmake", +] +build-backend = "py_build_cmake.build" + +[project] +name = "c_api" +readme = "README.md" +requires-python = ">=3.13" +authors = [ + { name = "dpdani", email = "git@danieleparmeggiani.me" }, +] +dependencies = [ + "cereggii" +] + +[tool.py-build-cmake.module] +directory = "src" + +[tool.py-build-cmake.sdist] +include = [ + "CMakeLists.txt", + "src/CMakeLists.txt", +] + +[tool.py-build-cmake.cmake] +minimum_version = "3.18" +build_type = "Debug" +source_path = "src" +args = ["-L"] +build_args = ["-j"] +install_components = ["python_modules"] diff --git a/examples/c-api/src/CMakeLists.txt b/examples/c-api/src/CMakeLists.txt new file mode 100644 index 00000000..0828723f --- /dev/null +++ b/examples/c-api/src/CMakeLists.txt @@ -0,0 +1,41 @@ +cmake_minimum_required(VERSION 3.18) +project(c_api) + +set(PY_BUILD_CMAKE_MODULE_NAME "c_api") + +# find Python.h et al. +execute_process(COMMAND python -c "import sysconfig; print(sysconfig.get_path('include'), end='')" OUTPUT_VARIABLE Python3_INCLUDE_DIR) +find_package(Python3 REQUIRED COMPONENTS Development.Module) + +# find cereggii/*.h +execute_process(COMMAND python -c "import cereggii; from pathlib import Path; print(Path(cereggii.__file__).parent, end='')" OUTPUT_VARIABLE CEREGGII_INSTALLATION_DIR) +message("CEREGGII_INSTALLATION_DIR=${CEREGGII_INSTALLATION_DIR}") + +execute_process(COMMAND python -c "import cereggii; print(cereggii.__version__, end='')" OUTPUT_VARIABLE CEREGGII_VERSION) +message(${CEREGGII_VERSION}) + +set(CEREGGII_INCLUDE_DIR "${CEREGGII_INSTALLATION_DIR}/include") +if (NOT IS_DIRECTORY ${CEREGGII_INCLUDE_DIR}) + message(FATAL_ERROR "Module cereggii not found in ${CEREGGII_INCLUDE_DIR}. Please add it to the build-system.requires section of your pyproject.toml file.") +endif () +set(CEREGGII_LIB "${CEREGGII_INSTALLATION_DIR}/libcereggii.${CEREGGII_VERSION}.dylib") +if (NOT EXISTS ${CEREGGII_LIB}) + message(FATAL_ERROR "cereggii shared object not found in ${CEREGGII_LIB}.") +endif () + +# add cereggii to the search path for #include +include_directories(${CEREGGII_INCLUDE_DIR}) + +link_libraries(${CEREGGII_LIB}) + + +# Add the module to compile +Python3_add_library(_c_api MODULE + "c_api/c_api.c" +) + +# Install the module +install(TARGETS _c_api + EXCLUDE_FROM_ALL + COMPONENT python_modules + DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) diff --git a/examples/c-api/src/c_api/__init__.py b/examples/c-api/src/c_api/__init__.py new file mode 100644 index 00000000..a9fb56fd --- /dev/null +++ b/examples/c-api/src/c_api/__init__.py @@ -0,0 +1,10 @@ +import sys +from pathlib import Path + +import cereggii + +sys.path.append(str(Path(cereggii.__file__).parent)) + +from c_api._c_api import * + +__version__ = "1.0" diff --git a/examples/c-api/src/c_api/c_api.c b/examples/c-api/src/c_api/c_api.c new file mode 100644 index 00000000..e14e888b --- /dev/null +++ b/examples/c-api/src/c_api/c_api.c @@ -0,0 +1,92 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "cereggii/atomics.h" + +typedef struct { + PyObject_HEAD + Py_ssize_t ref; +} PyAtomic; + +// Constructor: Atomic(ref) +static int PyAtomic_init(PyAtomic *self, PyObject *args, PyObject *kwargs) { + Py_ssize_t initial; + if (!PyArg_ParseTuple(args, "n", &initial)) { + return -1; + } + self->ref = initial; + return 0; +} + +// Destructor +static void PyAtomic_dealloc(PyAtomic* self) { + Py_TYPE(self)->tp_free((PyObject*)self); +} + +// Atomic compare-and-exchange: Atomic.compare_exchange(expected, desired) +static PyObject* PyAtomic_compare_exchange(PyAtomic *self, PyObject *args) { + Py_ssize_t expected; + Py_ssize_t desired; + + if (!PyArg_ParseTuple(args, "nn", &expected, &desired)) { + return NULL; + } + + int success = CereggiiAtomic_CompareExchangeSsize(&self->ref, expected, desired); + return PyLong_FromLong(success); +} + +static PyObject* PyAtomic_get(PyAtomic *self, PyObject *args) { + return PyLong_FromSsize_t(self->ref); +} + +// Atomic methods +static PyMethodDef PyAtomic_methods[] = { + {"compare_exchange", (PyCFunction)PyAtomic_compare_exchange, METH_VARARGS, ""}, + {"get", (PyCFunction)PyAtomic_get, METH_NOARGS, ""}, + {NULL} // Sentinel +}; + +// Atomic type definition +static PyTypeObject PyAtomicType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "atomic_module.Atomic", + .tp_basicsize = sizeof(PyAtomic), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + .tp_init = (initproc) PyAtomic_init, + .tp_dealloc = (destructor) PyAtomic_dealloc, + .tp_methods = PyAtomic_methods, +}; + +// Module methods +static PyMethodDef AtomicMethods[] = { + {NULL, NULL, 0, NULL} +}; + +// Module definition +static struct PyModuleDef atomicmodule = { + PyModuleDef_HEAD_INIT, + "atomic_module", + "Python wrapper for atomic operations", + -1, + AtomicMethods +}; + +// Module initialization +PyMODINIT_FUNC PyInit__c_api(void) { + PyObject *m; + if (PyType_Ready(&PyAtomicType) < 0) return NULL; + + m = PyModule_Create(&atomicmodule); + if (!m) return NULL; + + Py_INCREF(&PyAtomicType); + if (PyModule_AddObject(m, "Atomic", (PyObject *) &PyAtomicType) < 0) { + Py_DECREF(&PyAtomicType); + Py_DECREF(m); + return NULL; + } + + return m; +} diff --git a/pyproject.toml b/pyproject.toml index 9c8d88bf..95df7c10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ include = [ "CMakeLists.txt", "src/CMakeLists.txt", "src/cereggiiconfig.h.in", - "src/include/*", + "src/cereggii/include/*", ] [tool.py-build-cmake.cmake] @@ -70,6 +70,12 @@ build_args = ["-j"] install_components = ["python_modules"] +[tool.cibuildwheel] +enable = [ + "cpython-freethreading", +] +build = "cp313*-manylinux_aarch64" + [tool.black] target-version = ["py313"] line-length = 120 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 76ebeb99..173ae0fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,7 +6,6 @@ set(PY_BUILD_CMAKE_MODULE_NAME "cereggii") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if (CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") - add_compile_options("-mcx16") add_compile_options("-msse4.2") execute_process(COMMAND getconf LEVEL1_DCACHE_LINESIZE OUTPUT_VARIABLE LEVEL1_DCACHE_LINESIZE) elseif (CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") @@ -15,16 +14,15 @@ else () message(FATAL_ERROR "Unsupported platform: ${CMAKE_HOST_SYSTEM_NAME}") endif () -configure_file(cereggiiconfig.h.in "${CMAKE_CURRENT_SOURCE_DIR}/include/internal/cereggiiconfig.h") +configure_file(cereggiiconfig.h.in "${CMAKE_CURRENT_SOURCE_DIR}/include/cereggii/internal/cereggiiconfig.h") execute_process(COMMAND python -c "import sysconfig; print(sysconfig.get_path('include'), end='')" OUTPUT_VARIABLE Python3_INCLUDE_DIR) -find_package(Python3 REQUIRED COMPONENTS Development.Module) +find_package(Python3 REQUIRED COMPONENTS Development) -include_directories("include/") +include_directories("cereggii/include/" PUBLIC) -# Add the module to compile -Python3_add_library(_cereggii MODULE +set(sources "cereggii/atomic_dict/accessor_storage.c" "cereggii/atomic_dict/atomic_dict.c" "cereggii/atomic_dict/blocks.c" @@ -38,18 +36,37 @@ Python3_add_library(_cereggii MODULE "cereggii/atomic_int/atomic_int.c" "cereggii/atomic_int/handle.c" "cereggii/atomic_event.c" - "cereggii/atomic_ops.c" "cereggii/atomic_ref.c" "cereggii/cereggii.c" "cereggii/constants.c" ) +# Add the module to compile +Python3_add_library(_cereggii MODULE ${sources}) + +# Add library... todo +set(CEREGGII_LIB "cereggii.${PY_BUILD_CMAKE_PROJECT_VERSION}") +Python3_add_library(${CEREGGII_LIB} SHARED ${sources}) + +# Set visibility rules +set(CMAKE_CXX_VISIBILITY_PRESET hidden) +set(CMAKE_VISIBILITY_INLINES_HIDDEN 0) + +# Define export macro +target_compile_definitions(${CEREGGII_LIB} PRIVATE BUILDING_CEREGGII) + # Install the module install(TARGETS _cereggii EXCLUDE_FROM_ALL COMPONENT python_modules DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) +# Install the library +install(TARGETS ${CEREGGII_LIB} + EXCLUDE_FROM_ALL + COMPONENT python_modules + DESTINATION ${PY_BUILD_CMAKE_MODULE_NAME}) + # Install stubs to get autocomplete and type hints install(FILES cereggii/_cereggii.pyi EXCLUDE_FROM_ALL diff --git a/src/cereggii/atomic_dict/accessor_storage.c b/src/cereggii/atomic_dict/accessor_storage.c index 54967973..33ae9600 100644 --- a/src/cereggii/atomic_dict/accessor_storage.c +++ b/src/cereggii/atomic_dict/accessor_storage.c @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "atomic_dict.h" -#include "atomic_dict_internal.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" AtomicDict_AccessorStorage * diff --git a/src/cereggii/atomic_dict/atomic_dict.c b/src/cereggii/atomic_dict/atomic_dict.c index 704281d9..7e9a1b9c 100644 --- a/src/cereggii/atomic_dict/atomic_dict.c +++ b/src/cereggii/atomic_dict/atomic_dict.c @@ -4,9 +4,9 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict.h" -#include "atomic_dict_internal.h" -#include "atomic_ref.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomic_ref.h" #include "pythread.h" diff --git a/src/cereggii/atomic_dict/blocks.c b/src/cereggii/atomic_dict/blocks.c index e240423a..63178dc5 100644 --- a/src/cereggii/atomic_dict/blocks.c +++ b/src/cereggii/atomic_dict/blocks.c @@ -4,9 +4,9 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict.h" -#include "atomic_dict_internal.h" -#include "atomic_ops.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomics.h" AtomicDict_Block * diff --git a/src/cereggii/atomic_dict/delete.c b/src/cereggii/atomic_dict/delete.c index 8a50a261..b7ac7eb3 100644 --- a/src/cereggii/atomic_dict/delete.c +++ b/src/cereggii/atomic_dict/delete.c @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "atomic_dict_internal.h" -#include "atomic_ops.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomics.h" int diff --git a/src/cereggii/atomic_dict/insert.c b/src/cereggii/atomic_dict/insert.c index ac321a6f..f4a105b5 100644 --- a/src/cereggii/atomic_dict/insert.c +++ b/src/cereggii/atomic_dict/insert.c @@ -4,12 +4,12 @@ #define PY_SSIZE_T_CLEAN -#include "constants.h" -#include "atomic_dict_internal.h" -#include "atomic_ref.h" -#include "atomic_ops.h" +#include "cereggii/constants.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomic_ref.h" +#include "cereggii/atomics.h" #include "pythread.h" -#include "_internal_py_core.h" +#include "cereggii/_internal_py_core.h" int diff --git a/src/cereggii/atomic_dict/iter.c b/src/cereggii/atomic_dict/iter.c index 576773f9..4f59714b 100644 --- a/src/cereggii/atomic_dict/iter.c +++ b/src/cereggii/atomic_dict/iter.c @@ -4,8 +4,8 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict.h" -#include "atomic_dict_internal.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" PyObject * diff --git a/src/cereggii/atomic_dict/lookup.c b/src/cereggii/atomic_dict/lookup.c index 48dd8aa1..a1c79d11 100644 --- a/src/cereggii/atomic_dict/lookup.c +++ b/src/cereggii/atomic_dict/lookup.c @@ -4,9 +4,9 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict.h" -#include "atomic_dict_internal.h" -#include "constants.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/constants.h" void diff --git a/src/cereggii/atomic_dict/meta.c b/src/cereggii/atomic_dict/meta.c index 12ca8fc3..25fb7acc 100644 --- a/src/cereggii/atomic_dict/meta.c +++ b/src/cereggii/atomic_dict/meta.c @@ -4,8 +4,8 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict_internal.h" -#include "atomic_ref.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomic_ref.h" #include "pythread.h" diff --git a/src/cereggii/atomic_dict/migrate.c b/src/cereggii/atomic_dict/migrate.c index 46677cee..c7659c3d 100644 --- a/src/cereggii/atomic_dict/migrate.c +++ b/src/cereggii/atomic_dict/migrate.c @@ -4,12 +4,12 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_dict.h" -#include "atomic_dict_internal.h" -#include "atomic_ref.h" -#include "atomic_ops.h" -#include "constants.h" -#include "thread_id.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomic_ref.h" +#include "cereggii/atomics.h" +#include "cereggii/constants.h" +#include "cereggii/thread_id.h" int diff --git a/src/cereggii/atomic_dict/node_ops.c b/src/cereggii/atomic_dict/node_ops.c index 5a4d9bf0..a1dcaa35 100644 --- a/src/cereggii/atomic_dict/node_ops.c +++ b/src/cereggii/atomic_dict/node_ops.c @@ -3,8 +3,8 @@ // SPDX-License-Identifier: Apache-2.0 #include -#include "atomic_dict_internal.h" -#include "atomic_ops.h" +#include "cereggii/atomic_dict_internal.h" +#include "cereggii/atomics.h" // these functions take a pointer to meta, but to avoid multiple reads // you should dereference dk->meta (i.e. make a thread-local copy) and diff --git a/src/cereggii/atomic_event.c b/src/cereggii/atomic_event.c index a5a26bf9..05b7c85c 100644 --- a/src/cereggii/atomic_event.c +++ b/src/cereggii/atomic_event.c @@ -4,7 +4,7 @@ #define PY_SSIZE_T_CLEAN -#include "atomic_event.h" +#include "cereggii/atomic_event.h" PyObject * diff --git a/src/cereggii/atomic_int/atomic_int.c b/src/cereggii/atomic_int/atomic_int.c index c51b8e46..38f7e0d0 100644 --- a/src/cereggii/atomic_int/atomic_int.c +++ b/src/cereggii/atomic_int/atomic_int.c @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "atomic_int.h" -#include "atomic_int_internal.h" -#include "atomic_ops.h" +#include "cereggii/atomic_int.h" +#include "cereggii/atomic_int_internal.h" +#include "cereggii/atomics.h" #include "pyhash.h" diff --git a/src/cereggii/atomic_int/handle.c b/src/cereggii/atomic_int/handle.c index f5e1753b..0af04024 100644 --- a/src/cereggii/atomic_int/handle.c +++ b/src/cereggii/atomic_int/handle.c @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "atomic_int.h" -#include "atomic_int_internal.h" +#include "cereggii/atomic_int.h" +#include "cereggii/atomic_int_internal.h" int AtomicInt64Handle_init(AtomicInt64Handle *self, PyObject *args, PyObject *Py_UNUSED(kwargs)) diff --git a/src/cereggii/atomic_ops.c b/src/cereggii/atomic_ops.c deleted file mode 100644 index b5d2b3c1..00000000 --- a/src/cereggii/atomic_ops.c +++ /dev/null @@ -1,404 +0,0 @@ -// SPDX-FileCopyrightText: 2023-present dpdani -// -// SPDX-License-Identifier: Apache-2.0 - -#include "atomic_ops.h" - -#include - - -#pragma clang diagnostic push -#pragma ide diagnostic ignored "readability-non-const-parameter" - - -inline int -CereggiiAtomic_AddInt(int *obj, int value) -{ return atomic_fetch_add_explicit((_Atomic(int) *)obj, value, memory_order_acq_rel); } - -inline int8_t -CereggiiAtomic_AddInt8(int8_t *obj, int8_t value) -{ return atomic_fetch_add_explicit((_Atomic(int8_t) *)obj, value, memory_order_acq_rel); } - -inline int16_t -CereggiiAtomic_AddInt16(int16_t *obj, int16_t value) -{ return atomic_fetch_add_explicit((_Atomic(int16_t) *)obj, value, memory_order_acq_rel); } - -inline int32_t -CereggiiAtomic_AddInt32(int32_t *obj, int32_t value) -{ return atomic_fetch_add_explicit((_Atomic(int32_t) *)obj, value, memory_order_acq_rel); } - -inline int64_t -CereggiiAtomic_AddInt64(int64_t *obj, int64_t value) -{ return atomic_fetch_add_explicit((_Atomic(int64_t) *)obj, value, memory_order_acq_rel); } - -inline intptr_t -CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value) -{ return atomic_fetch_add_explicit((_Atomic(intptr_t) *)obj, value, memory_order_acq_rel); } - -inline unsigned int -CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value) -{ return atomic_fetch_add_explicit((_Atomic(unsigned int) *) obj, value, memory_order_acq_rel); } - -inline uint8_t -CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value) -{ return atomic_fetch_add_explicit((_Atomic(uint8_t) *)obj, value, memory_order_acq_rel); } - -inline uint16_t -CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value) -{ return atomic_fetch_add_explicit((_Atomic(uint16_t) *)obj, value, memory_order_acq_rel); } - -inline uint32_t -CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value) -{ return atomic_fetch_add_explicit((_Atomic(uint32_t) *)obj, value, memory_order_acq_rel); } - -inline uint64_t -CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value) -{ return atomic_fetch_add_explicit((_Atomic(uint64_t) *)obj, value, memory_order_acq_rel); } - -inline uintptr_t -CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value) -{ return atomic_fetch_add_explicit((_Atomic(uintptr_t) *)obj, value, memory_order_acq_rel); } - -inline Py_ssize_t -CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value) -{ return atomic_fetch_add_explicit((_Atomic(Py_ssize_t) *)obj, value, memory_order_acq_rel); } - - -inline int -CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(int) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(int8_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(int16_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(int32_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(int64_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeInt128(__int128_t *obj, __int128_t expected, __int128_t desired) -{ - return __sync_bool_compare_and_swap_16(obj, expected, desired); -} - -inline int -CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(intptr_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(unsigned) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(uint8_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(uint16_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(uint32_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(uint64_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeUInt128(__uint128_t *obj, __uint128_t expected, __uint128_t desired) -{ - return __sync_bool_compare_and_swap_16(obj, expected, desired); -} - -inline int -CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(uintptr_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(Py_ssize_t) *) obj, &expected, desired, memory_order_acq_rel, memory_order_acquire); -} - -inline int -CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *desired) -{ - return atomic_compare_exchange_strong_explicit((_Atomic(void *) *) obj, &expected, desired, memory_order_acq_rel, - memory_order_acquire); -} - - -inline int -CereggiiAtomic_ExchangeInt(int *obj, int value) -{ return atomic_exchange_explicit((_Atomic(int) *) obj, value, memory_order_acq_rel); } - -inline int8_t -CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value) -{ return atomic_exchange_explicit((_Atomic(int8_t) *) obj, value, memory_order_acq_rel); } - -inline int16_t -CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value) -{ return atomic_exchange_explicit((_Atomic(int16_t) *) obj, value, memory_order_acq_rel); } - -inline int32_t -CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value) -{ return atomic_exchange_explicit((_Atomic(int32_t) *) obj, value, memory_order_acq_rel); } - -inline int64_t -CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value) -{ return atomic_exchange_explicit((_Atomic(int64_t) *) obj, value, memory_order_acq_rel); } - -inline intptr_t -CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value) -{ return atomic_exchange_explicit((_Atomic(intptr_t) *) obj, value, memory_order_acq_rel); } - -inline unsigned int -CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value) -{ return atomic_exchange_explicit((_Atomic(unsigned int) *) obj, value, memory_order_acq_rel); } - -inline uint8_t -CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value) -{ return atomic_exchange_explicit((_Atomic(uint8_t) *) obj, value, memory_order_acq_rel); } - -inline uint16_t -CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value) -{ return atomic_exchange_explicit((_Atomic(uint16_t) *) obj, value, memory_order_acq_rel); } - -inline uint32_t -CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value) -{ return atomic_exchange_explicit((_Atomic(uint32_t) *) obj, value, memory_order_acq_rel); } - -inline uint64_t -CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value) -{ return atomic_exchange_explicit((_Atomic(uint64_t) *) obj, value, memory_order_acq_rel); } - -inline uintptr_t -CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value) -{ return atomic_exchange_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_acq_rel); } - -inline Py_ssize_t -CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value) -{ return atomic_exchange_explicit((_Atomic(Py_ssize_t) *) obj, value, memory_order_acq_rel); } - -inline void * -CereggiiAtomic_ExchangePtr(void **obj, void *value) -{ return atomic_exchange_explicit((_Atomic(void *) *)obj, value, memory_order_acq_rel); } - - -inline uint8_t -CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value) -{ return atomic_fetch_and_explicit((_Atomic(uint8_t) *) obj, value, memory_order_acq_rel); } - -inline uint16_t -CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value) -{ return atomic_fetch_and_explicit((_Atomic(uint16_t) *) obj, value, memory_order_acq_rel); } - -inline uint32_t -CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value) -{ return atomic_fetch_and_explicit((_Atomic(uint32_t) *) obj, value, memory_order_acq_rel); } - -inline uint64_t -CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value) -{ return atomic_fetch_and_explicit((_Atomic(uint64_t) *) obj, value, memory_order_acq_rel); } - -inline uintptr_t -CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value) -{ return atomic_fetch_and_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_acq_rel); } - - -inline uint8_t -CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value) -{ return atomic_fetch_or_explicit((_Atomic(uint8_t) *) obj, value, memory_order_acq_rel); } - -inline uint16_t -CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value) -{ return atomic_fetch_or_explicit((_Atomic(uint16_t) *) obj, value, memory_order_acq_rel); } - -inline uint32_t -CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value) -{ return atomic_fetch_or_explicit((_Atomic(uint32_t) *) obj, value, memory_order_acq_rel); } - -inline uint64_t -CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value) -{ return atomic_fetch_or_explicit((_Atomic(uint64_t) *) obj, value, memory_order_acq_rel); } - -inline uintptr_t -CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value) -{ return atomic_fetch_or_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_acq_rel); } - - -inline int -CereggiiAtomic_LoadInt(const int *obj) -{ return atomic_load_explicit((_Atomic(int) *) obj, memory_order_acquire); } - -inline int8_t -CereggiiAtomic_LoadInt8(const int8_t *obj) -{ return atomic_load_explicit((_Atomic(int8_t) *) obj, memory_order_acquire); } - -inline int16_t -CereggiiAtomic_LoadInt16(const int16_t *obj) -{ return atomic_load_explicit((_Atomic(int16_t) *) obj, memory_order_acquire); } - -inline int32_t -CereggiiAtomic_LoadInt32(const int32_t *obj) -{ return atomic_load_explicit((_Atomic(int32_t) *) obj, memory_order_acquire); } - -inline int64_t -CereggiiAtomic_LoadInt64(const int64_t *obj) -{ return atomic_load_explicit((_Atomic(int64_t) *) obj, memory_order_acquire); } - -inline intptr_t -CereggiiAtomic_LoadIntPtr(const intptr_t *obj) -{ return atomic_load_explicit((_Atomic(intptr_t) *) obj, memory_order_acquire); } - -inline uint8_t -CereggiiAtomic_LoadUInt8(const uint8_t *obj) -{ return atomic_load_explicit((_Atomic(uint8_t) *) obj, memory_order_acquire); } - -inline uint16_t -CereggiiAtomic_LoadUInt16(const uint16_t *obj) -{ return atomic_load_explicit((_Atomic(uint16_t) *) obj, memory_order_acquire); } - -inline uint32_t -CereggiiAtomic_LoadUInt32(const uint32_t *obj) -{ return atomic_load_explicit((_Atomic(uint32_t) *) obj, memory_order_acquire); } - -inline uint64_t -CereggiiAtomic_LoadUInt64(const uint64_t *obj) -{ return atomic_load_explicit((_Atomic(uint64_t) *) obj, memory_order_acquire); } - -inline uintptr_t -CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj) -{ return atomic_load_explicit((_Atomic(uintptr_t) *) obj, memory_order_acquire); } - -inline unsigned int -CereggiiAtomic_LoadUInt(const unsigned int *obj) -{ return atomic_load_explicit((_Atomic(unsigned int) *) obj, memory_order_acquire); } - -inline Py_ssize_t -CereggiiAtomic_LoadSsize(const Py_ssize_t *obj) -{ return atomic_load_explicit((_Atomic(Py_ssize_t) *) obj, memory_order_acquire); } - -inline void * -CereggiiAtomic_LoadPtr(const void **obj) -{ return (void *) atomic_load_explicit((_Atomic(void *) *) obj, memory_order_acquire); } - - -inline void -CereggiiAtomic_StoreInt(int *obj, int value) -{ atomic_store_explicit((_Atomic(int) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value) -{ atomic_store_explicit((_Atomic(int8_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value) -{ atomic_store_explicit((_Atomic(int16_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value) -{ atomic_store_explicit((_Atomic(int32_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value) -{ atomic_store_explicit((_Atomic(int64_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreIntPtr(intptr_t *obj, intptr_t value) -{ atomic_store_explicit((_Atomic(intptr_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value) -{ atomic_store_explicit((_Atomic(uint8_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value) -{ atomic_store_explicit((_Atomic(uint16_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value) -{ atomic_store_explicit((_Atomic(uint32_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value) -{ atomic_store_explicit((_Atomic(uint64_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUIntPtr(uintptr_t *obj, uintptr_t value) -{ atomic_store_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value) -{ atomic_store_explicit((_Atomic(unsigned int) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StorePtr(void **obj, void *value) -{ atomic_store_explicit((_Atomic(void *) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value) -{ atomic_store_explicit((_Atomic(Py_ssize_t) *) obj, value, memory_order_release); } - - -inline void * -CereggiiAtomic_LoadPtrAcquire(const void **obj) -{ return (void *) atomic_load_explicit((_Atomic(void *) *) obj, memory_order_acquire); } - -inline void -CereggiiAtomic_StorePtrRelease(void **obj, void *value) -{ atomic_store_explicit((_Atomic(void *) *) obj, value, memory_order_release); } - -inline void -CereggiiAtomic_StoreIntRelease(int *obj, int value) -{ atomic_store_explicit((_Atomic(int) *) obj, value, memory_order_release); } - -inline int -CereggiiAtomic_LoadIntAcquire(const int *obj) -{ return atomic_load_explicit((_Atomic(int) *) obj, memory_order_acquire); } - - -inline void -CereggiiAtomic_FenceSeqCst(void) -{ atomic_thread_fence(memory_order_seq_cst); } - -inline void -CereggiiAtomic_FenceRelease(void) -{ atomic_thread_fence(memory_order_release); } - - -#pragma clang diagnostic pop diff --git a/src/cereggii/atomic_ref.c b/src/cereggii/atomic_ref.c index 7b1a0b34..648e86d5 100644 --- a/src/cereggii/atomic_ref.c +++ b/src/cereggii/atomic_ref.c @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "atomic_ref.h" -#include "atomic_ops.h" -#include "_internal_py_core.h" +#include "cereggii/atomic_ref.h" +#include "cereggii/atomics.h" +#include "cereggii/_internal_py_core.h" PyObject * diff --git a/src/cereggii/cereggii.c b/src/cereggii/cereggii.c index 6825b05f..04352986 100644 --- a/src/cereggii/cereggii.c +++ b/src/cereggii/cereggii.c @@ -4,12 +4,12 @@ #define PY_SSIZE_T_CLEAN -#include "constants.h" -#include "atomic_event.h" -#include "atomic_int.h" -#include "atomic_ref.h" -#include "atomic_dict.h" -#include "atomic_dict_internal.h" +#include "cereggii/constants.h" +#include "cereggii/atomic_event.h" +#include "cereggii/atomic_int.h" +#include "cereggii/atomic_ref.h" +#include "cereggii/atomic_dict.h" +#include "cereggii/atomic_dict_internal.h" static PyMethodDef AtomicInt64_methods[] = { diff --git a/src/cereggii/constants.c b/src/cereggii/constants.c index 26cad5f9..ebacf0fb 100644 --- a/src/cereggii/constants.c +++ b/src/cereggii/constants.c @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -#include "constants.h" +#include "cereggii/constants.h" PyObject * diff --git a/src/include/_internal_py_core.h b/src/cereggii/include/cereggii/_internal_py_core.h similarity index 100% rename from src/include/_internal_py_core.h rename to src/cereggii/include/cereggii/_internal_py_core.h diff --git a/src/include/atomic_dict.h b/src/cereggii/include/cereggii/atomic_dict.h similarity index 100% rename from src/include/atomic_dict.h rename to src/cereggii/include/cereggii/atomic_dict.h diff --git a/src/include/atomic_dict_internal.h b/src/cereggii/include/cereggii/atomic_dict_internal.h similarity index 100% rename from src/include/atomic_dict_internal.h rename to src/cereggii/include/cereggii/atomic_dict_internal.h diff --git a/src/include/atomic_event.h b/src/cereggii/include/cereggii/atomic_event.h similarity index 100% rename from src/include/atomic_event.h rename to src/cereggii/include/cereggii/atomic_event.h diff --git a/src/include/atomic_int.h b/src/cereggii/include/cereggii/atomic_int.h similarity index 100% rename from src/include/atomic_int.h rename to src/cereggii/include/cereggii/atomic_int.h diff --git a/src/include/atomic_int_internal.h b/src/cereggii/include/cereggii/atomic_int_internal.h similarity index 100% rename from src/include/atomic_int_internal.h rename to src/cereggii/include/cereggii/atomic_int_internal.h diff --git a/src/include/atomic_ref.h b/src/cereggii/include/cereggii/atomic_ref.h similarity index 100% rename from src/include/atomic_ref.h rename to src/cereggii/include/cereggii/atomic_ref.h diff --git a/src/cereggii/include/cereggii/atomics.h b/src/cereggii/include/cereggii/atomics.h new file mode 100644 index 00000000..ae498a87 --- /dev/null +++ b/src/cereggii/include/cereggii/atomics.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +/* +Cross-platform atomic primitives. + +Largely adapted from CPython. +*/ + +#ifndef CEREGGII_ATOMICS_H +#define CEREGGII_ATOMICS_H + + +#include + + +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)) +# define USE_GCC_ATOMICS 1 +#elif defined(__clang__) +# if __has_builtin(__atomic_load) +# define USE_GCC_ATOMICS 1 +# endif +#endif + +#if USE_GCC_ATOMICS +# include "cereggii/internal/atomics_gcc.h" +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +# include "cereggii/internal/atomics_stdatomic.h" +#elif defined(_MSC_VER) +# include "cereggii/internal/atomics_msc.h" +#else +# error "no available atomics implementation for this platform/compiler" +#endif + +#ifdef USE_GCC_ATOMICS +#undef USE_GCC_ATOMICS +#endif + + +// --- aliases --------------------------------------------------------------- + +#if SIZEOF_LONG == 8 +# define CereggiiAtomic_LoadULong(p) \ + CereggiiAtomic_LoadUInt64((uint64_t *)p) +# define CereggiiAtomic_LoadULongRelaxed(p) \ + CereggiiAtomic_LoadUInt64Relaxed((uint64_t *)p) +# define CereggiiAtomic_StoreULong(p, v) \ + CereggiiAtomic_StoreUInt64((uint64_t *)p, v) +# define CereggiiAtomic_StoreULongRelaxed(p, v) \ + CereggiiAtomic_StoreUInt64Relaxed((uint64_t *)p, v) +#elif SIZEOF_LONG == 4 +# define CereggiiAtomic_LoadULong(p) \ + CereggiiAtomic_LoadUInt32((uint32_t *)p) +# define CereggiiAtomic_LoadULongRelaxed(p) \ + CereggiiAtomic_LoadUInt32Relaxed((uint32_t *)p) +# define CereggiiAtomic_StoreULong(p, v) \ + CereggiiAtomic_StoreUInt32((uint32_t *)p, v) +# define CereggiiAtomic_StoreULongRelaxed(p, v) \ + CereggiiAtomic_StoreUInt32Relaxed((uint32_t *)p, v) +#else +# error "long must be 4 or 8 bytes in size" +#endif // SIZEOF_LONG + +#endif //CEREGGII_ATOMICS_H diff --git a/src/include/constants.h b/src/cereggii/include/cereggii/constants.h similarity index 100% rename from src/include/constants.h rename to src/cereggii/include/cereggii/constants.h diff --git a/src/cereggii/include/cereggii/internal/.gitignore b/src/cereggii/include/cereggii/internal/.gitignore new file mode 100644 index 00000000..4503c216 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/.gitignore @@ -0,0 +1 @@ +cereggiiconfig.h diff --git a/src/cereggii/include/cereggii/internal/atomics_declarations.h b/src/cereggii/include/cereggii/internal/atomics_declarations.h new file mode 100644 index 00000000..f7aaae29 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/atomics_declarations.h @@ -0,0 +1,445 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef CEREGGII_ATOMICS_DECL_H +#define CEREGGII_ATOMICS_DECL_H + +#include + + +inline int +CereggiiAtomic_AddInt(int *obj, int value); + +inline int8_t +CereggiiAtomic_AddInt8(int8_t *obj, int8_t value); + +inline int16_t +CereggiiAtomic_AddInt16(int16_t *obj, int16_t value); + +inline int32_t +CereggiiAtomic_AddInt32(int32_t *obj, int32_t value); + +inline int64_t +CereggiiAtomic_AddInt64(int64_t *obj, int64_t value); + +inline intptr_t +CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value); + +inline unsigned int +CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value); + +inline uint8_t +CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value); + +inline uint16_t +CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value); + +inline uint32_t +CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value); + +inline uint64_t +CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value); + +inline uintptr_t +CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value); + +inline Py_ssize_t +CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value); + + +// ---------------------------------------------------------------------------- +// compare exchange: +// 1. atomically read the value of obj +// 2. if it is expected, then update it with desired and return 1; +// 3. otherwise, don't change it and return 0 + +inline int +CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int desired); + +inline int +CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t desired); + +inline int +CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t desired); + +inline int +CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t desired); + +inline int +CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desired); + +inline int +CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t desired); + +inline int +CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int desired); + +inline int +CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t desired); + +inline int +CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t desired); + +inline int +CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t desired); + +inline int +CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t desired); + +inline int +CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t desired); + +inline int +CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t desired); + +inline int +CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *desired); + + +// ---------------------------------------------------------------------------- +// exchange: + +inline int +CereggiiAtomic_ExchangeInt(int *obj, int value); + +inline int8_t +CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value); + +inline int16_t +CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value); + +inline int32_t +CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value); + +inline int64_t +CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value); + +inline intptr_t +CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value); + +inline unsigned int +CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value); + +inline uint8_t +CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value); + +inline uint16_t +CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value); + +inline uint32_t +CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value); + +inline uint64_t +CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value); + +inline uintptr_t +CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value); + +inline Py_ssize_t +CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value); + +inline void * +CereggiiAtomic_ExchangePtr(void **obj, void *value); + +inline uint8_t +CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value); + +inline uint16_t +CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value); + +inline uint32_t +CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value); + +inline uint64_t +CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value); + +inline uintptr_t +CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value); + +inline uint8_t +CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value); + +inline uint16_t +CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value); + +inline uint32_t +CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value); + +inline uint64_t +CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value); + +inline uintptr_t +CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value); + +inline int +CereggiiAtomic_LoadInt(const int *obj); + +inline int8_t +CereggiiAtomic_LoadInt8(const int8_t *obj); + +inline int16_t +CereggiiAtomic_LoadInt16(const int16_t *obj); + +inline int32_t +CereggiiAtomic_LoadInt32(const int32_t *obj); + +inline int64_t +CereggiiAtomic_LoadInt64(const int64_t *obj); + +inline intptr_t +CereggiiAtomic_LoadIntPtr(const intptr_t *obj); + +inline uint8_t +CereggiiAtomic_LoadUInt8(const uint8_t *obj); + +inline uint16_t +CereggiiAtomic_LoadUInt16(const uint16_t *obj); + +inline uint32_t +CereggiiAtomic_LoadUInt32(const uint32_t *obj); + +inline uint64_t +CereggiiAtomic_LoadUInt64(const uint64_t *obj); + +inline uintptr_t +CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj); + +inline unsigned int +CereggiiAtomic_LoadUInt(const unsigned int *obj); + +inline Py_ssize_t +CereggiiAtomic_LoadSsize(const Py_ssize_t *obj); + +inline void * +CereggiiAtomic_LoadPtr(const void **obj); + +inline int +CereggiiAtomic_LoadIntRelaxed(const int *obj); + +inline char +CereggiiAtomic_LoadCharRelaxed(const char *obj); + +inline unsigned char +CereggiiAtomic_LoadUCharRelaxed(const unsigned char *obj); + +inline short +CereggiiAtomic_LoadShortRelaxed(const short *obj); + +inline unsigned short +CereggiiAtomic_LoadUShortRelaxed(const unsigned short *obj); + +inline long +CereggiiAtomic_LoadLongRelaxed(const long *obj); + +inline float +CereggiiAtomic_LoadFloatRelaxed(const float *obj); + +inline double +CereggiiAtomic_LoadDoubleRelaxed(const double *obj); + +inline int8_t +CereggiiAtomic_LoadInt8Relaxed(const int8_t *obj); + +inline int16_t +CereggiiAtomic_LoadInt16Relaxed(const int16_t *obj); + +inline int32_t +CereggiiAtomic_LoadInt32Relaxed(const int32_t *obj); + +inline int64_t +CereggiiAtomic_LoadInt64Relaxed(const int64_t *obj); + +inline intptr_t +CereggiiAtomic_LoadIntPtrRelaxed(const intptr_t *obj); + +inline uint8_t +CereggiiAtomic_LoadUInt8Relaxed(const uint8_t *obj); + +inline uint16_t +CereggiiAtomic_LoadUInt16Relaxed(const uint16_t *obj); + +inline uint32_t +CereggiiAtomic_LoadUInt32Relaxed(const uint32_t *obj); + +inline uint64_t +CereggiiAtomic_LoadUInt64Relaxed(const uint64_t *obj); + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrRelaxed(const uintptr_t *obj); + +inline unsigned int +CereggiiAtomic_LoadUIntRelaxed(const unsigned int *obj); + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeRelaxed(const Py_ssize_t *obj); + +inline void * +CereggiiAtomic_LoadPtrRelaxed(const void **obj); + +inline unsigned long long +CereggiiAtomic_LoadULLongRelaxed(const unsigned long long *obj); + +inline long long +CereggiiAtomic_LoadLLongRelaxed(const long long *obj); + +inline void +CereggiiAtomic_StoreInt(int *obj, int value); + +inline void +CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value); + +inline void +CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value); + +inline void +CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value); + +inline void +CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value); + +inline void +CereggiiAtomic_StoreIntPtr(intptr_t *obj, intptr_t value); + +inline void +CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value); + +inline void +CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value); + +inline void +CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value); + +inline void +CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value); + +inline void +CereggiiAtomic_StoreUIntPtr(uintptr_t *obj, uintptr_t value); + +inline void +CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value); + +inline void +CereggiiAtomic_StorePtr(void **obj, void *value); + +inline void +CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value); + +inline void +CereggiiAtomic_StoreIntRelaxed(int *obj, int value); + +inline void +CereggiiAtomic_StoreInt8Relaxed(int8_t *obj, int8_t value); + +inline void +CereggiiAtomic_StoreInt16Relaxed(int16_t *obj, int16_t value); + +inline void +CereggiiAtomic_StoreInt32Relaxed(int32_t *obj, int32_t value); + +inline void +CereggiiAtomic_StoreInt64Relaxed(int64_t *obj, int64_t value); + +inline void +CereggiiAtomic_StoreIntPtrRelaxed(intptr_t *obj, intptr_t value); + +inline void +CereggiiAtomic_StoreUInt8Relaxed(uint8_t *obj, uint8_t value); + +inline void +CereggiiAtomic_StoreUInt16Relaxed(uint16_t *obj, uint16_t value); + +inline void +CereggiiAtomic_StoreUInt32Relaxed(uint32_t *obj, uint32_t value); + +inline void +CereggiiAtomic_StoreUInt64Relaxed(uint64_t *obj, uint64_t value); + +inline void +CereggiiAtomic_StoreUIntPtrRelaxed(uintptr_t *obj, uintptr_t value); + +inline void +CereggiiAtomic_StoreUIntRelaxed(unsigned int *obj, unsigned int value); + +inline void +CereggiiAtomic_StorePtrRelaxed(void **obj, void *value); + +inline void +CereggiiAtomic_StoreSsizeRelaxed(Py_ssize_t *obj, Py_ssize_t value); + +inline void +CereggiiAtomic_StoreULLongRelaxed(unsigned long long *obj, unsigned long long value); + +inline void +CereggiiAtomic_StoreCharRelaxed(char *obj, char value); + +inline void +CereggiiAtomic_StoreUCharRelaxed(unsigned char *obj, unsigned char value); + +inline void +CereggiiAtomic_StoreShortRelaxed(short *obj, short value); + +inline void +CereggiiAtomic_StoreUShortRelaxed(unsigned short *obj, unsigned short value); + +inline void +CereggiiAtomic_StoreLongRelaxed(long *obj, long value); + +inline void +CereggiiAtomic_StoreFloatRelaxed(float *obj, float value); + +inline void +CereggiiAtomic_StoreDoubleRelaxed(double *obj, double value); + +inline void +CereggiiAtomic_StoreLLongRelaxed(long long *obj, long long value); + +inline void * +CereggiiAtomic_LoadPtrAcquire(const void **obj); + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrAcquire(const uintptr_t *obj); + +inline void +CereggiiAtomic_StorePtrRelease(void **obj, void *value); + +inline void +CereggiiAtomic_StoreUIntPtrRelease(uintptr_t *obj, uintptr_t value); + +inline void +CereggiiAtomic_StoreIntRelease(int *obj, int value); + +inline void +CereggiiAtomic_StoreUIntRelease(unsigned int *obj, unsigned int value); + +inline void +CereggiiAtomic_StoreSsizeRelease(Py_ssize_t *obj, Py_ssize_t value); + +inline int +CereggiiAtomic_LoadIntAcquire(const int *obj); + +inline void +CereggiiAtomic_StoreUInt32Release(uint32_t *obj, uint32_t value); + +inline void +CereggiiAtomic_StoreUInt64Release(uint64_t *obj, uint64_t value); + +inline uint64_t +CereggiiAtomic_LoadUInt64Acquire(const uint64_t *obj); + +inline uint32_t +CereggiiAtomic_LoadUInt32Acquire(const uint32_t *obj); + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeAcquire(const Py_ssize_t *obj); + +inline void +CereggiiAtomic_FenceSeqCst(void); + +inline void +CereggiiAtomic_FenceAcquire(void); + +inline void +CereggiiAtomic_FenceRelease(void); + +#endif // CEREGGII_ATOMICS_DECL_H diff --git a/src/cereggii/include/cereggii/internal/atomics_gcc.h b/src/cereggii/include/cereggii/internal/atomics_gcc.h new file mode 100644 index 00000000..6a1cd265 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/atomics_gcc.h @@ -0,0 +1,619 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#include + +#pragma clang diagnostic push +#pragma ide diagnostic ignored "readability-non-const-parameter" + + +// ---------------------------------------------------------------------------- +// add: atomically fetch and add value to obj + +inline int +CereggiiAtomic_AddInt(int *obj, int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline int8_t +CereggiiAtomic_AddInt8(int8_t *obj, int8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline int16_t +CereggiiAtomic_AddInt16(int16_t *obj, int16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline int32_t +CereggiiAtomic_AddInt32(int32_t *obj, int32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline int64_t +CereggiiAtomic_AddInt64(int64_t *obj, int64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline intptr_t +CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline unsigned int +CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline uint8_t +CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline uint16_t +CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline uint32_t +CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline uint64_t +CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline uintptr_t +CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + +inline Py_ssize_t +CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// compare exchange: +// 1. atomically read the value of obj +// 2. if it is expected, then update it with desired and return 1; +// 3. otherwise, don't change it and return 0 + +inline int +CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +__attribute__((visibility("default"))) inline int +CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + +inline int +CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *desired) +{ return __atomic_compare_exchange_n(obj, &expected, desired, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// exchange: atomically swap the current value V of obj with value and return V + +inline int +CereggiiAtomic_ExchangeInt(int *obj, int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline int8_t +CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline int16_t +CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline int32_t +CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline int64_t +CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline intptr_t +CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline unsigned int +CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline uint8_t +CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline uint16_t +CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_ACQ_REL); } + +inline uint32_t +CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline uint64_t +CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline uintptr_t +CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline Py_ssize_t +CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void * +CereggiiAtomic_ExchangePtr(void **obj, void *value) +{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// and: atomically fetch and bitwise-and obj with value + +inline uint8_t +CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +inline uint16_t +CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +inline uint32_t +CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +inline uint64_t +CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + +inline uintptr_t +CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// or: atomically fetch and bitwise-or obj with value + +inline uint8_t +CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +inline uint16_t +CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +inline uint32_t +CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +inline uint64_t +CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + +inline uintptr_t +CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value) +{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// load: atomically read the value of obj + +inline int +CereggiiAtomic_LoadInt(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline int8_t +CereggiiAtomic_LoadInt8(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline int16_t +CereggiiAtomic_LoadInt16(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline int32_t +CereggiiAtomic_LoadInt32(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline int64_t +CereggiiAtomic_LoadInt64(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline intptr_t +CereggiiAtomic_LoadIntPtr(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline uint8_t +CereggiiAtomic_LoadUInt8(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline uint16_t +CereggiiAtomic_LoadUInt16(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline uint32_t +CereggiiAtomic_LoadUInt32(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline uint64_t +CereggiiAtomic_LoadUInt64(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline unsigned int +CereggiiAtomic_LoadUInt(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsize(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); } + +inline void * +CereggiiAtomic_LoadPtr(const void **obj) +{ return __atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// relaxed load: plainly read the value of obj + +inline int +CereggiiAtomic_LoadIntRelaxed(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline char +CereggiiAtomic_LoadCharRelaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline unsigned char +CereggiiAtomic_LoadUCharRelaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline short +CereggiiAtomic_LoadShortRelaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline unsigned short +CereggiiAtomic_LoadUShortRelaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline long +CereggiiAtomic_LoadLongRelaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline float +CereggiiAtomic_LoadFloatRelaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + +inline double +CereggiiAtomic_LoadDoubleRelaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + +inline int8_t +CereggiiAtomic_LoadInt8Relaxed(const int8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline int16_t +CereggiiAtomic_LoadInt16Relaxed(const int16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline int32_t +CereggiiAtomic_LoadInt32Relaxed(const int32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline int64_t +CereggiiAtomic_LoadInt64Relaxed(const int64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline intptr_t +CereggiiAtomic_LoadIntPtrRelaxed(const intptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline uint8_t +CereggiiAtomic_LoadUInt8Relaxed(const uint8_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline uint16_t +CereggiiAtomic_LoadUInt16Relaxed(const uint16_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline uint32_t +CereggiiAtomic_LoadUInt32Relaxed(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline uint64_t +CereggiiAtomic_LoadUInt64Relaxed(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrRelaxed(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline unsigned int +CereggiiAtomic_LoadUIntRelaxed(const unsigned int *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeRelaxed(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline void * +CereggiiAtomic_LoadPtrRelaxed(const void **obj) +{ return __atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); } + +inline unsigned long long +CereggiiAtomic_LoadULLongRelaxed(const unsigned long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +inline long long +CereggiiAtomic_LoadLLongRelaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + + +// ---------------------------------------------------------------------------- +// store: atomically write value into obj + +inline void +CereggiiAtomic_StoreInt(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreIntPtr(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUIntPtr(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StorePtr(void **obj, void *value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); } + + +// ---------------------------------------------------------------------------- +// relaxed store: plainly write value into obj + +inline void +CereggiiAtomic_StoreIntRelaxed(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreInt8Relaxed(int8_t *obj, int8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreInt16Relaxed(int16_t *obj, int16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreInt32Relaxed(int32_t *obj, int32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreInt64Relaxed(int64_t *obj, int64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreIntPtrRelaxed(intptr_t *obj, intptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUInt8Relaxed(uint8_t *obj, uint8_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUInt16Relaxed(uint16_t *obj, uint16_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUInt32Relaxed(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUInt64Relaxed(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUIntPtrRelaxed(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUIntRelaxed(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StorePtrRelaxed(void **obj, void *value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreSsizeRelaxed(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreULLongRelaxed(unsigned long long *obj, unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreCharRelaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreUCharRelaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreShortRelaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreUShortRelaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreLongRelaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreFloatRelaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreDoubleRelaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +inline void +CereggiiAtomic_StoreLLongRelaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + + +// ---------------------------------------------------------------------------- +// _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release + +inline void * +CereggiiAtomic_LoadPtrAcquire(const void **obj) +{ return __atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrAcquire(const uintptr_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +inline void +CereggiiAtomic_StorePtrRelease(void **obj, void *value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreUIntPtrRelease(uintptr_t *obj, uintptr_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreIntRelease(int *obj, int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreUIntRelease(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreSsizeRelease(Py_ssize_t *obj, Py_ssize_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline int +CereggiiAtomic_LoadIntAcquire(const int *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +inline void +CereggiiAtomic_StoreUInt32Release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreUInt64Release(uint64_t *obj, uint64_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +inline uint64_t +CereggiiAtomic_LoadUInt64Acquire(const uint64_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +inline uint32_t +CereggiiAtomic_LoadUInt32Acquire(const uint32_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeAcquire(const Py_ssize_t *obj) +{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } + + +// ---------------------------------------------------------------------------- +// fence: separate regions of code + +inline void +CereggiiAtomic_FenceSeqCst(void) +{ __atomic_thread_fence(__ATOMIC_SEQ_CST); } + +inline void +CereggiiAtomic_FenceAcquire(void) +{ __atomic_thread_fence(__ATOMIC_ACQUIRE); } + +inline void +CereggiiAtomic_FenceRelease(void) +{ __atomic_thread_fence(__ATOMIC_RELEASE); } diff --git a/src/cereggii/include/cereggii/internal/atomics_msc.h b/src/cereggii/include/cereggii/internal/atomics_msc.h new file mode 100644 index 00000000..9d5ea444 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/atomics_msc.h @@ -0,0 +1,1172 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include "cereggii/internal/atomics_declarations.h" + + +#define ASSERT_CAN_CAST_OBJ_TO(TYPE) \ + static_assert(sizeof(*obj) == sizeof(TYPE), "no suitable atomics implementation") + + +// ---------------------------------------------------------------------------- +// add: atomically fetch and add value to obj + +inline int8_t +CereggiiAtomic_AddInt8(int8_t *obj, int8_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(char); + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +} + +inline int16_t +CereggiiAtomic_AddInt16(int16_t *obj, int16_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(short); + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +} + +inline int32_t +CereggiiAtomic_AddInt32(int32_t *obj, int32_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(long); + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +} + +inline int64_t +CereggiiAtomic_AddInt64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = CereggiiAtomic_LoadInt64Relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (CereggiiAtomic_CompareExchangeInt64(obj, old_value, new_value)) { + return old_value; + } + } +#endif +} + + +inline uint8_t +CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)CereggiiAtomic_AddInt8((int8_t *)obj, (int8_t)value); +} + +inline uint16_t +CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)CereggiiAtomic_AddInt16((int16_t *)obj, (int16_t)value); +} + +inline uint32_t +CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)CereggiiAtomic_AddInt32((int32_t *)obj, (int32_t)value); +} + +inline int +CereggiiAtomic_AddInt(int *obj, int value) +{ + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return (int)CereggiiAtomic_AddInt32((int32_t *)obj, (int32_t)value); +} + +inline unsigned int +CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value) +{ + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return (unsigned int)CereggiiAtomic_AddInt32((int32_t *)obj, (int32_t)value); +} + +inline uint64_t +CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)CereggiiAtomic_AddInt64((int64_t *)obj, (int64_t)value); +} + +inline intptr_t +CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + ASSERT_CAN_CAST_OBJ_TO(int64_t); + return (intptr_t)CereggiiAtomic_AddInt64((int64_t *)obj, (int64_t)value); +#else + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return (intptr_t)CereggiiAtomic_AddInt32((int32_t *)obj, (int32_t)value); +#endif +} + +inline uintptr_t +CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(intptr_t); + return (uintptr_t)CereggiiAtomic_AddIntPtr((intptr_t *)obj, (intptr_t)value); +} + +inline Py_ssize_t +CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(intptr_t); + return (Py_ssize_t)CereggiiAtomic_AddIntPtr((intptr_t *)obj, (intptr_t)value); +} + + +// ---------------------------------------------------------------------------- +// compare exchange: +// 1. atomically read the value of obj +// 2. if it is expected, then update it with desired and return 1; +// 3. otherwise, don't change it and return 0 + +inline int +CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(char); + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)expected); + return initial == expected; +} + +inline int +CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(short); + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)expected); + return initial == expected; +} + +inline int +CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(long); + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)expected); + return initial == expected; +} + +inline int +CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(__int64); + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)expected); + return initial == expected; +} + +inline int +CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *value) +{ + void *initial = _InterlockedCompareExchangePointer( + obj, + value, + expected); + return initial == expected; +} + + +inline int +CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t value) +{ + return CereggiiAtomic_CompareExchangeInt8((int8_t *)obj, + (int8_t)expected, + (int8_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t value) +{ + return CereggiiAtomic_CompareExchangeInt16((int16_t *)obj, + (int16_t)expected, + (int16_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t value) +{ + return CereggiiAtomic_CompareExchangeInt32((int32_t *)obj, + (int32_t)expected, + (int32_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int value) +{ + return CereggiiAtomic_CompareExchangeInt32((int32_t *)obj, + (int32_t)expected, + (int32_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int value) +{ + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return CereggiiAtomic_CompareExchangeInt32((int32_t *)obj, + (int32_t)expected, + (int32_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t value) +{ + return CereggiiAtomic_CompareExchangeInt64((int64_t *)obj, + (int64_t)expected, + (int64_t)value); +} + +inline int +CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return CereggiiAtomic_CompareExchangePtr((void**)obj, + (void*)expected, + (void*)value); +} + +inline int +CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return CereggiiAtomic_CompareExchangePtr((void**)obj, + (void*)expected, + (void*)value); +} + +inline int +CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return CereggiiAtomic_CompareExchangePtr((void**)obj, + (void*)expected, + (void*)value); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +inline int8_t +CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(char); + return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); +} + +inline int16_t +CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(short); + return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); +} + +inline int32_t +CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(long); + return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); +} + +inline int64_t +CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(__int64); + return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = CereggiiAtomic_LoadInt64Relaxed(obj); + for (;;) { + if (CereggiiAtomic_CompareExchangeInt64(obj, old_value, value)) { + return old_value; + } + } +#endif +} + +inline void* +CereggiiAtomic_ExchangePtr(void **obj, void *value) +{ + return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); +} + + +inline uint8_t +CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)CereggiiAtomic_ExchangeInt8((int8_t *)obj, + (int8_t)value); +} + +inline uint16_t +CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)CereggiiAtomic_ExchangeInt16((int16_t *)obj, + (int16_t)value); +} + +inline uint32_t +CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)CereggiiAtomic_ExchangeInt32((int32_t *)obj, + (int32_t)value); +} + +inline int +CereggiiAtomic_ExchangeInt(int *obj, int value) +{ + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return (int)CereggiiAtomic_ExchangeInt32((int32_t *)obj, + (int32_t)value); +} + +inline unsigned int +CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value) +{ + ASSERT_CAN_CAST_OBJ_TO(int32_t); + return (unsigned int)CereggiiAtomic_ExchangeInt32((int32_t *)obj, + (int32_t)value); +} + +inline uint64_t +CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)CereggiiAtomic_ExchangeInt64((int64_t *)obj, + (int64_t)value); +} + +inline intptr_t +CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (intptr_t)CereggiiAtomic_ExchangePtr((void**)obj, + (void*)value); +} + +inline uintptr_t +CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (uintptr_t)CereggiiAtomic_ExchangePtr((void**)obj, + (void*)value); +} + +inline Py_ssize_t +CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (Py_ssize_t)CereggiiAtomic_ExchangePtr((void**)obj, + (void*)value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +inline uint8_t +CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(char); + return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); +} + +inline uint16_t +CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(short); + return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); +} + +inline uint32_t +CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(long); + return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); +} + +inline uint64_t +CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(__int64); + return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = CereggiiAtomic_LoadUInt64Relaxed(obj); + for (;;) { + uint64_t new_value = old_value & value; + if (CereggiiAtomic_CompareExchangeUInt64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +inline uintptr_t +CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + ASSERT_CAN_CAST_OBJ_TO(uint64_t); + return (uintptr_t)CereggiiAtomic_AndUInt64((uint64_t *)obj, + (uint64_t)value); +#else + ASSERT_CAN_CAST_OBJ_TO(uint32_t); + return (uintptr_t)CereggiiAtomic_AndUInt32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +inline uint8_t +CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(char); + return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); +} + +inline uint16_t +CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(short); + return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); +} + +inline uint32_t +CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value) +{ + ASSERT_CAN_CAST_OBJ_TO(long); + return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); +} + +inline uint64_t +CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(__int64); + return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = CereggiiAtomic_LoadUInt64Relaxed(obj); + for (;;) { + uint64_t new_value = old_value | value; + if (CereggiiAtomic_CompareExchangeUInt64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +inline uintptr_t +CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + ASSERT_CAN_CAST_OBJ_TO(uint64_t); + return (uintptr_t)CereggiiAtomic_OrUInt64((uint64_t *)obj, + (uint64_t)value); +#else + ASSERT_CAN_CAST_OBJ_TO(uint32_t); + return (uintptr_t)CereggiiAtomic_OrUInt32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +inline uint8_t +CereggiiAtomic_LoadUInt8(const uint8_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)obj; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt8" +#endif +} + +inline uint16_t +CereggiiAtomic_LoadUInt16(const uint16_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint16_t *)obj; +#elif defined(_M_ARM64) + return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt16" +#endif +} + +inline uint32_t +CereggiiAtomic_LoadUInt32(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint32_t *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt32" +#endif +} + +inline uint64_t +CereggiiAtomic_LoadUInt64(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint64_t *)obj; +#elif defined(_M_ARM64) + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt64" +#endif +} + +inline int8_t +CereggiiAtomic_LoadInt8(const int8_t *obj) +{ + return (int8_t)CereggiiAtomic_LoadUInt8((const uint8_t *)obj); +} + +inline int16_t +CereggiiAtomic_LoadInt16(const int16_t *obj) +{ + return (int16_t)CereggiiAtomic_LoadUInt16((const uint16_t *)obj); +} + +inline int32_t +CereggiiAtomic_LoadInt32(const int32_t *obj) +{ + return (int32_t)CereggiiAtomic_LoadUInt32((const uint32_t *)obj); +} + +inline int +CereggiiAtomic_LoadInt(const int *obj) +{ + ASSERT_CAN_CAST_OBJ_TO(uint32_t); + return (int)CereggiiAtomic_LoadUInt32((uint32_t *)obj); +} + +inline unsigned int +CereggiiAtomic_LoadUInt(const unsigned int *obj) +{ + ASSERT_CAN_CAST_OBJ_TO(uint32_t); + return (unsigned int)CereggiiAtomic_LoadUInt32((uint32_t *)obj); +} + +inline int64_t +CereggiiAtomic_LoadInt64(const int64_t *obj) +{ + return (int64_t)CereggiiAtomic_LoadUInt64((const uint64_t *)obj); +} + +inline void* +CereggiiAtomic_LoadPtr(const void **obj) +{ +#if SIZEOF_VOID_P == 8 + return (void*)CereggiiAtomic_LoadUInt64((const uint64_t *)obj); +#else + return (void*)CereggiiAtomic_LoadUInt32((const uint32_t *)obj); +#endif +} + +inline intptr_t +CereggiiAtomic_LoadIntPtr(const intptr_t *obj) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (intptr_t)CereggiiAtomic_LoadPtr((void*)obj); +} + +inline uintptr_t +CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (uintptr_t)CereggiiAtomic_LoadPtr((void*)obj); +} + +inline Py_ssize_t +CereggiiAtomic_LoadSsize(const Py_ssize_t *obj) +{ + ASSERT_CAN_CAST_OBJ_TO(void*); + return (Py_ssize_t)CereggiiAtomic_LoadPtr((void*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +inline int +CereggiiAtomic_LoadIntRelaxed(const int *obj) +{ + return *(volatile int *)obj; +} + +inline char +CereggiiAtomic_LoadCharRelaxed(const char *obj) +{ + return *(volatile char *)obj; +} + +inline unsigned char +CereggiiAtomic_LoadUCharRelaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + +inline short +CereggiiAtomic_LoadShortRelaxed(const short *obj) +{ + return *(volatile short *)obj; +} + +inline unsigned short +CereggiiAtomic_LoadUShortRelaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + +inline long +CereggiiAtomic_LoadLongRelaxed(const long *obj) +{ + return *(volatile long *)obj; +} + +inline float +CereggiiAtomic_LoadFloatRelaxed(const float *obj) +{ + return *(volatile float *)obj; +} + +inline double +CereggiiAtomic_LoadDoubleRelaxed(const double *obj) +{ + return *(volatile double *)obj; +} + +inline int8_t +CereggiiAtomic_LoadInt8Relaxed(const int8_t *obj) +{ + return *(volatile int8_t *)obj; +} + +inline int16_t +CereggiiAtomic_LoadInt16Relaxed(const int16_t *obj) +{ + return *(volatile int16_t *)obj; +} + +inline int32_t +CereggiiAtomic_LoadInt32Relaxed(const int32_t *obj) +{ + return *(volatile int32_t *)obj; +} + +inline int64_t +CereggiiAtomic_LoadInt64Relaxed(const int64_t *obj) +{ + return *(volatile int64_t *)obj; +} + +inline intptr_t +CereggiiAtomic_LoadIntPtrRelaxed(const intptr_t *obj) +{ + return *(volatile intptr_t *)obj; +} + +inline uint8_t +CereggiiAtomic_LoadUInt8Relaxed(const uint8_t *obj) +{ + return *(volatile uint8_t *)obj; +} + +inline uint16_t +CereggiiAtomic_LoadUInt16Relaxed(const uint16_t *obj) +{ + return *(volatile uint16_t *)obj; +} + +inline uint32_t +CereggiiAtomic_LoadUInt32Relaxed(const uint32_t *obj) +{ + return *(volatile uint32_t *)obj; +} + +inline uint64_t +CereggiiAtomic_LoadUInt64Relaxed(const uint64_t *obj) +{ + return *(volatile uint64_t *)obj; +} + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrRelaxed(const uintptr_t *obj) +{ + return *(volatile uintptr_t *)obj; +} + +inline unsigned int +CereggiiAtomic_LoadUIntRelaxed(const unsigned int *obj) +{ + return *(volatile unsigned int *)obj; +} + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeRelaxed(const Py_ssize_t *obj) +{ + return *(volatile Py_ssize_t *)obj; +} + +inline void* +CereggiiAtomic_LoadPtrRelaxed(const void **obj) +{ + return *(void * volatile *)obj; +} + +inline unsigned long long +CereggiiAtomic_LoadULLongRelaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + +inline long long +CereggiiAtomic_LoadLLongRelaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +inline void +CereggiiAtomic_StoreInt(int *obj, int value) +{ + (void)CereggiiAtomic_ExchangeInt(obj, value); +} + +inline void +CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value) +{ + (void)CereggiiAtomic_ExchangeInt8(obj, value); +} + +inline void +CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value) +{ + (void)CereggiiAtomic_ExchangeInt16(obj, value); +} + +inline void +CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value) +{ + (void)CereggiiAtomic_ExchangeInt32(obj, value); +} + +inline void +CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value) +{ + (void)CereggiiAtomic_ExchangeInt64(obj, value); +} + +inline void +CereggiiAtomic_StoreIntptr(intptr_t *obj, intptr_t value) +{ + (void)CereggiiAtomic_ExchangeIntPtr(obj, value); +} + +inline void +CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value) +{ + (void)CereggiiAtomic_ExchangeUInt8(obj, value); +} + +inline void +CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value) +{ + (void)CereggiiAtomic_ExchangeUInt16(obj, value); +} + +inline void +CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value) +{ + (void)CereggiiAtomic_ExchangeUInt32(obj, value); +} + +inline void +CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value) +{ + (void)CereggiiAtomic_ExchangeUInt64(obj, value); +} + +inline void +CereggiiAtomic_StoreUIntptr(uintptr_t *obj, uintptr_t value) +{ + (void)CereggiiAtomic_ExchangeUIntPtr(obj, value); +} + +inline void +CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value) +{ + (void)CereggiiAtomic_ExchangeUInt(obj, value); +} + +inline void +CereggiiAtomic_StorePtr(void **obj, void *value) +{ + (void)CereggiiAtomic_ExchangePtr(obj, value); +} + +inline void +CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value) +{ + (void)CereggiiAtomic_ExchangeSsize(obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +inline void +CereggiiAtomic_StoreIntRelaxed(int *obj, int value) +{ + *(volatile int *)obj = value; +} + +inline void +CereggiiAtomic_StoreInt8Relaxed(int8_t *obj, int8_t value) +{ + *(volatile int8_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreInt16Relaxed(int16_t *obj, int16_t value) +{ + *(volatile int16_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreInt32Relaxed(int32_t *obj, int32_t value) +{ + *(volatile int32_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreInt64Relaxed(int64_t *obj, int64_t value) +{ + *(volatile int64_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreIntptrRelaxed(intptr_t *obj, intptr_t value) +{ + *(volatile intptr_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUInt8Relaxed(uint8_t *obj, uint8_t value) +{ + *(volatile uint8_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUInt16Relaxed(uint16_t *obj, uint16_t value) +{ + *(volatile uint16_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUInt32Relaxed(uint32_t *obj, uint32_t value) +{ + *(volatile uint32_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUInt64Relaxed(uint64_t *obj, uint64_t value) +{ + *(volatile uint64_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUIntptrRelaxed(uintptr_t *obj, uintptr_t value) +{ + *(volatile uintptr_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUIntRelaxed(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +inline void +CereggiiAtomic_StorePtrRelaxed(void **obj, void* value) +{ + *(void * volatile *)obj = value; +} + +inline void +CereggiiAtomic_StoreSsizeRelaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + *(volatile Py_ssize_t *)obj = value; +} + +inline void +CereggiiAtomic_StoreUllongRelaxed(unsigned long long *obj, unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + +inline void +CereggiiAtomic_StoreCharRelaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + +inline void +CereggiiAtomic_StoreUcharRelaxed(unsigned char *obj, unsigned char value) +{ + *(volatile unsigned char *)obj = value; +} + +inline void +CereggiiAtomic_StoreShortRelaxed(short *obj, short value) +{ + *(volatile short *)obj = value; +} + +inline void +CereggiiAtomic_StoreUshortRelaxed(unsigned short *obj, unsigned short value) +{ + *(volatile unsigned short *)obj = value; +} + +inline void +CereggiiAtomic_StoreUIntRelease(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +inline void +CereggiiAtomic_StoreLongRelaxed(long *obj, long value) +{ + *(volatile long *)obj = value; +} + +inline void +CereggiiAtomic_StoreFloatRelaxed(float *obj, float value) +{ + *(volatile float *)obj = value; +} + +inline void +CereggiiAtomic_StoreDoubleRelaxed(double *obj, double value) +{ + *(volatile double *)obj = value; +} + +inline void +CereggiiAtomic_StoreLlongRelaxed(long long *obj, long long value) +{ + *(volatile long long *)obj = value; +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +inline void * +CereggiiAtomic_LoadPtrAcquire(const void **obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(void * volatile *)obj; +#elif defined(_M_ARM64) + return (void *)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadPtrAcquire" +#endif +} + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrAcquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUIntPtrAcquire" +#endif +} + +inline void +CereggiiAtomic_StorePtrRelease(void **obj, void *value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(void * volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); +#else +# error "no implementation of CereggiiAtomic_StorePtrRelease" +#endif +} + +inline void +CereggiiAtomic_StoreUIntPtrRelease(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of CereggiiAtomic_StoreUIntPtrRelease" +#endif +} + +inline void +CereggiiAtomic_StoreIntRelease(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of CereggiiAtomic_StoreIntRelease" +#endif +} + +inline void +CereggiiAtomic_StoreSsizeRelease(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of CereggiiAtomic_StoreSsizeRelease" +#endif +} + +inline int +CereggiiAtomic_LoadIntAcquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadIntAcquire" +#endif +} + +inline void +CereggiiAtomic_StoreUInt32Release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of CereggiiAtomic_StoreUInt32Release" +#endif +} + +inline void +CereggiiAtomic_StoreUInt64Release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of CereggiiAtomic_StoreUInt64Release" +#endif +} + +inline uint64_t +CereggiiAtomic_LoadUInt64Acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + ASSERT_CAN_CAST_OBJ_TO(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt64Acquire" +#endif +} + +inline uint32_t +CereggiiAtomic_LoadUInt32Acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadUInt32Acquire" +#endif +} + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeAcquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of CereggiiAtomic_LoadSsizeAcquire" +#endif +} + +// --- _Py_atomic_fence ------------------------------------------------------ + +inline void +CereggiiAtomic_FenceSeqCst(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) + __faststorefence(); +#elif defined(_M_IX86) + _mm_mfence(); +#else +# error "no implementation of CereggiiAtomic_FenceSeqCst" +#endif +} + +inline void +CereggiiAtomic_FenceAcquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of CereggiiAtomic_FenceAcquire" +#endif +} + +inline void +CereggiiAtomic_FenceRelease(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadWriteBarrier(); +#else +# error "no implementation of CereggiiAtomic_FenceRelease" +#endif +} + +#undef ASSERT_CAN_CAST_OBJ_TO diff --git a/src/cereggii/include/cereggii/internal/atomics_stdatomic.h b/src/cereggii/include/cereggii/internal/atomics_stdatomic.h new file mode 100644 index 00000000..f671ef94 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/atomics_stdatomic.h @@ -0,0 +1,624 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#pragma clang diagnostic push +#pragma ide diagnostic ignored "readability-non-const-parameter" + + +// ---------------------------------------------------------------------------- +// add: atomically fetch and add value to obj + +inline int +CereggiiAtomic_AddInt(int *obj, int value) +{ return atomic_fetch_add_explicit((_Atomic(int) *)obj, value, memory_order_seq_cst); } + +inline int8_t +CereggiiAtomic_AddInt8(int8_t *obj, int8_t value) +{ return atomic_fetch_add_explicit((_Atomic(int8_t) *)obj, value, memory_order_seq_cst); } + +inline int16_t +CereggiiAtomic_AddInt16(int16_t *obj, int16_t value) +{ return atomic_fetch_add_explicit((_Atomic(int16_t) *)obj, value, memory_order_seq_cst); } + +inline int32_t +CereggiiAtomic_AddInt32(int32_t *obj, int32_t value) +{ return atomic_fetch_add_explicit((_Atomic(int32_t) *)obj, value, memory_order_seq_cst); } + +inline int64_t +CereggiiAtomic_AddInt64(int64_t *obj, int64_t value) +{ return atomic_fetch_add_explicit((_Atomic(int64_t) *)obj, value, memory_order_seq_cst); } + +inline intptr_t +CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value) +{ return atomic_fetch_add_explicit((_Atomic(intptr_t) *)obj, value, memory_order_seq_cst); } + +inline unsigned int +CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value) +{ return atomic_fetch_add_explicit((_Atomic(unsigned int) *) obj, value, memory_order_seq_cst); } + +inline uint8_t +CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value) +{ return atomic_fetch_add_explicit((_Atomic(uint8_t) *)obj, value, memory_order_seq_cst); } + +inline uint16_t +CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value) +{ return atomic_fetch_add_explicit((_Atomic(uint16_t) *)obj, value, memory_order_seq_cst); } + +inline uint32_t +CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value) +{ return atomic_fetch_add_explicit((_Atomic(uint32_t) *)obj, value, memory_order_seq_cst); } + +inline uint64_t +CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value) +{ return atomic_fetch_add_explicit((_Atomic(uint64_t) *)obj, value, memory_order_seq_cst); } + +inline uintptr_t +CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value) +{ return atomic_fetch_add_explicit((_Atomic(uintptr_t) *)obj, value, memory_order_seq_cst); } + +inline Py_ssize_t +CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value) +{ return atomic_fetch_add_explicit((_Atomic(Py_ssize_t) *)obj, value, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// compare exchange: +// 1. atomically read the value of obj +// 2. if it is expected, then update it with desired and return 1; +// 3. otherwise, don't change it and return 0 + +inline int +CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(int) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(int8_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(int16_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(int32_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(int64_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(intptr_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(unsigned int) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(uint8_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(uint16_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(uint32_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(uint64_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(uintptr_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(Py_ssize_t) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + +inline int +CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *desired) +{ return atomic_compare_exchange_strong_explicit((_Atomic(void *) *) obj, &expected, desired, memory_order_seq_cst, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// exchange: atomically swap the current value V of obj with value and return V + +inline int +CereggiiAtomic_ExchangeInt(int *obj, int value) +{ return atomic_exchange_explicit((_Atomic(int) *) obj, value, memory_order_seq_cst); } + +inline int8_t +CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value) +{ return atomic_exchange_explicit((_Atomic(int8_t) *) obj, value, memory_order_seq_cst); } + +inline int16_t +CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value) +{ return atomic_exchange_explicit((_Atomic(int16_t) *) obj, value, memory_order_seq_cst); } + +inline int32_t +CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value) +{ return atomic_exchange_explicit((_Atomic(int32_t) *) obj, value, memory_order_seq_cst); } + +inline int64_t +CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value) +{ return atomic_exchange_explicit((_Atomic(int64_t) *) obj, value, memory_order_seq_cst); } + +inline intptr_t +CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value) +{ return atomic_exchange_explicit((_Atomic(intptr_t) *) obj, value, memory_order_seq_cst); } + +inline unsigned int +CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value) +{ return atomic_exchange_explicit((_Atomic(unsigned int) *) obj, value, memory_order_seq_cst); } + +inline uint8_t +CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value) +{ return atomic_exchange_explicit((_Atomic(uint8_t) *) obj, value, memory_order_seq_cst); } + +inline uint16_t +CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value) +{ return atomic_exchange_explicit((_Atomic(uint16_t) *) obj, value, memory_order_seq_cst); } + +inline uint32_t +CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value) +{ return atomic_exchange_explicit((_Atomic(uint32_t) *) obj, value, memory_order_seq_cst); } + +inline uint64_t +CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value) +{ return atomic_exchange_explicit((_Atomic(uint64_t) *) obj, value, memory_order_seq_cst); } + +inline uintptr_t +CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value) +{ return atomic_exchange_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_seq_cst); } + +inline Py_ssize_t +CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value) +{ return atomic_exchange_explicit((_Atomic(Py_ssize_t) *) obj, value, memory_order_seq_cst); } + +inline void * +CereggiiAtomic_ExchangePtr(void **obj, void *value) +{ return atomic_exchange_explicit((_Atomic(void *) *)obj, value, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// and: atomically fetch and bitwise-and obj with value + +inline uint8_t +CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value) +{ return atomic_fetch_and_explicit((_Atomic(uint8_t) *) obj, value, memory_order_seq_cst); } + +inline uint16_t +CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value) +{ return atomic_fetch_and_explicit((_Atomic(uint16_t) *) obj, value, memory_order_seq_cst); } + +inline uint32_t +CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value) +{ return atomic_fetch_and_explicit((_Atomic(uint32_t) *) obj, value, memory_order_seq_cst); } + +inline uint64_t +CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value) +{ return atomic_fetch_and_explicit((_Atomic(uint64_t) *) obj, value, memory_order_seq_cst); } + +inline uintptr_t +CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value) +{ return atomic_fetch_and_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// or: atomically fetch and bitwise-or obj with value + +inline uint8_t +CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value) +{ return atomic_fetch_or_explicit((_Atomic(uint8_t) *) obj, value, memory_order_seq_cst); } + +inline uint16_t +CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value) +{ return atomic_fetch_or_explicit((_Atomic(uint16_t) *) obj, value, memory_order_seq_cst); } + +inline uint32_t +CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value) +{ return atomic_fetch_or_explicit((_Atomic(uint32_t) *) obj, value, memory_order_seq_cst); } + +inline uint64_t +CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value) +{ return atomic_fetch_or_explicit((_Atomic(uint64_t) *) obj, value, memory_order_seq_cst); } + +inline uintptr_t +CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value) +{ return atomic_fetch_or_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// load: atomically read the value of obj + +inline int +CereggiiAtomic_LoadInt(const int *obj) +{ return atomic_load_explicit((_Atomic(int) *) obj, memory_order_seq_cst); } + +inline int8_t +CereggiiAtomic_LoadInt8(const int8_t *obj) +{ return atomic_load_explicit((_Atomic(int8_t) *) obj, memory_order_seq_cst); } + +inline int16_t +CereggiiAtomic_LoadInt16(const int16_t *obj) +{ return atomic_load_explicit((_Atomic(int16_t) *) obj, memory_order_seq_cst); } + +inline int32_t +CereggiiAtomic_LoadInt32(const int32_t *obj) +{ return atomic_load_explicit((_Atomic(int32_t) *) obj, memory_order_seq_cst); } + +inline int64_t +CereggiiAtomic_LoadInt64(const int64_t *obj) +{ return atomic_load_explicit((_Atomic(int64_t) *) obj, memory_order_seq_cst); } + +inline intptr_t +CereggiiAtomic_LoadIntPtr(const intptr_t *obj) +{ return atomic_load_explicit((_Atomic(intptr_t) *) obj, memory_order_seq_cst); } + +inline uint8_t +CereggiiAtomic_LoadUInt8(const uint8_t *obj) +{ return atomic_load_explicit((_Atomic(uint8_t) *) obj, memory_order_seq_cst); } + +inline uint16_t +CereggiiAtomic_LoadUInt16(const uint16_t *obj) +{ return atomic_load_explicit((_Atomic(uint16_t) *) obj, memory_order_seq_cst); } + +inline uint32_t +CereggiiAtomic_LoadUInt32(const uint32_t *obj) +{ return atomic_load_explicit((_Atomic(uint32_t) *) obj, memory_order_seq_cst); } + +inline uint64_t +CereggiiAtomic_LoadUInt64(const uint64_t *obj) +{ return atomic_load_explicit((_Atomic(uint64_t) *) obj, memory_order_seq_cst); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj) +{ return atomic_load_explicit((_Atomic(uintptr_t) *) obj, memory_order_seq_cst); } + +inline unsigned int +CereggiiAtomic_LoadUInt(const unsigned int *obj) +{ return atomic_load_explicit((_Atomic(unsigned int) *) obj, memory_order_seq_cst); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsize(const Py_ssize_t *obj) +{ return atomic_load_explicit((_Atomic(Py_ssize_t) *) obj, memory_order_seq_cst); } + +inline void * +CereggiiAtomic_LoadPtr(const void **obj) +{ return (void *) atomic_load_explicit((_Atomic(void *) *) obj, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// relaxed load: plainly read the value of obj + +inline int +CereggiiAtomic_LoadIntRelaxed(const int *obj) +{ return atomic_load_explicit((_Atomic(int) *)obj, memory_order_relaxed); } + +inline char +CereggiiAtomic_LoadCharRelaxed(const char *obj) +{ return atomic_load_explicit((_Atomic(char) *) obj, memory_order_relaxed); } + +inline unsigned char +CereggiiAtomic_LoadUCharRelaxed(const unsigned char *obj) +{ return atomic_load_explicit((_Atomic(unsigned char) *) obj, memory_order_relaxed); } + +inline short +CereggiiAtomic_LoadShortRelaxed(const short *obj) +{ return atomic_load_explicit((_Atomic(short) *) obj, memory_order_relaxed); } + +inline unsigned short +CereggiiAtomic_LoadUShortRelaxed(const unsigned short *obj) +{ return atomic_load_explicit((_Atomic(unsigned short) *) obj, memory_order_relaxed); } + +inline long +CereggiiAtomic_LoadLongRelaxed(const long *obj) +{ return atomic_load_explicit((_Atomic(long) *) obj, memory_order_relaxed); } + +inline float +CereggiiAtomic_LoadFloatRelaxed(const float *obj) +{ return atomic_load_explicit((_Atomic(float) *) obj, memory_order_relaxed); } + +inline double +CereggiiAtomic_LoadDoubleRelaxed(const double *obj) +{ return atomic_load_explicit((_Atomic(double) *) obj, memory_order_relaxed); } + +inline int8_t +CereggiiAtomic_LoadInt8Relaxed(const int8_t *obj) +{ return atomic_load_explicit((_Atomic(int8_t) *) obj, memory_order_relaxed); } + +inline int16_t +CereggiiAtomic_LoadInt16Relaxed(const int16_t *obj) +{ return atomic_load_explicit((_Atomic(int16_t) *) obj, memory_order_relaxed); } + +inline int32_t +CereggiiAtomic_LoadInt32Relaxed(const int32_t *obj) +{ return atomic_load_explicit((_Atomic(int32_t) *) obj, memory_order_relaxed); } + +inline int64_t +CereggiiAtomic_LoadInt64Relaxed(const int64_t *obj) +{ return atomic_load_explicit((_Atomic(int64_t) *) obj, memory_order_relaxed); } + +inline intptr_t +CereggiiAtomic_LoadIntPtrRelaxed(const intptr_t *obj) +{ return atomic_load_explicit((_Atomic(intptr_t) *) obj, memory_order_relaxed); } + +inline uint8_t +CereggiiAtomic_LoadUInt8Relaxed(const uint8_t *obj) +{ return atomic_load_explicit((_Atomic(uint8_t) *) obj, memory_order_relaxed); } + +inline uint16_t +CereggiiAtomic_LoadUInt16Relaxed(const uint16_t *obj) +{ return atomic_load_explicit((_Atomic(uint16_t) *) obj, memory_order_relaxed); } + +inline uint32_t +CereggiiAtomic_LoadUInt32Relaxed(const uint32_t *obj) +{ return atomic_load_explicit((_Atomic(uint32_t) *) obj, memory_order_relaxed); } + +inline uint64_t +CereggiiAtomic_LoadUInt64Relaxed(const uint64_t *obj) +{ return atomic_load_explicit((_Atomic(uint64_t) *) obj, memory_order_relaxed); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrRelaxed(const uintptr_t *obj) +{ return atomic_load_explicit((_Atomic(uintptr_t) *) obj, memory_order_relaxed); } + +inline unsigned int +CereggiiAtomic_LoadUIntRelaxed(const unsigned int *obj) +{ return atomic_load_explicit((_Atomic(unsigned int) *) obj, memory_order_relaxed); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeRelaxed(const Py_ssize_t *obj) +{ return atomic_load_explicit((_Atomic(Py_ssize_t) *) obj, memory_order_relaxed); } + +inline void * +CereggiiAtomic_LoadPtrRelaxed(const void **obj) +{ return atomic_load_explicit((_Atomic(void*)*) obj, memory_order_relaxed); } + +inline unsigned long long +CereggiiAtomic_LoadULLongRelaxed(const unsigned long long *obj) +{ return atomic_load_explicit((_Atomic(unsigned long long) *) obj, memory_order_relaxed); } + +inline long long +CereggiiAtomic_LoadLLongRelaxed(const long long *obj) +{ return atomic_load_explicit((_Atomic(long long) *) obj, memory_order_relaxed); } + + +// ---------------------------------------------------------------------------- +// store: atomically write value into obj + +inline void +CereggiiAtomic_StoreInt(int *obj, int value) +{ atomic_store_explicit((_Atomic(int) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value) +{ atomic_store_explicit((_Atomic(int8_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value) +{ atomic_store_explicit((_Atomic(int16_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value) +{ atomic_store_explicit((_Atomic(int32_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value) +{ atomic_store_explicit((_Atomic(int64_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreIntPtr(intptr_t *obj, intptr_t value) +{ atomic_store_explicit((_Atomic(intptr_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value) +{ atomic_store_explicit((_Atomic(uint8_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value) +{ atomic_store_explicit((_Atomic(uint16_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value) +{ atomic_store_explicit((_Atomic(uint32_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value) +{ atomic_store_explicit((_Atomic(uint64_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUIntPtr(uintptr_t *obj, uintptr_t value) +{ atomic_store_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value) +{ atomic_store_explicit((_Atomic(unsigned int) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StorePtr(void **obj, void *value) +{ atomic_store_explicit((_Atomic(void *) *) obj, value, memory_order_seq_cst); } + +inline void +CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value) +{ atomic_store_explicit((_Atomic(Py_ssize_t) *) obj, value, memory_order_seq_cst); } + + +// ---------------------------------------------------------------------------- +// relaxed store: plainly write value into obj + +inline void +CereggiiAtomic_StoreIntRelaxed(int *obj, int value) +{ atomic_store_explicit((_Atomic(int) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreInt8Relaxed(int8_t *obj, int8_t value) +{ atomic_store_explicit((_Atomic(int8_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreInt16Relaxed(int16_t *obj, int16_t value) +{ atomic_store_explicit((_Atomic(int16_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreInt32Relaxed(int32_t *obj, int32_t value) +{ atomic_store_explicit((_Atomic(int32_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreInt64Relaxed(int64_t *obj, int64_t value) +{ atomic_store_explicit((_Atomic(int64_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreIntPtrRelaxed(intptr_t *obj, intptr_t value) +{ atomic_store_explicit((_Atomic(intptr_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUInt8Relaxed(uint8_t *obj, uint8_t value) +{ atomic_store_explicit((_Atomic(uint8_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUInt16Relaxed(uint16_t *obj, uint16_t value) +{ atomic_store_explicit((_Atomic(uint16_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUInt32Relaxed(uint32_t *obj, uint32_t value) +{ atomic_store_explicit((_Atomic(uint32_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUInt64Relaxed(uint64_t *obj, uint64_t value) +{ atomic_store_explicit((_Atomic(uint64_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUIntPtrRelaxed(uintptr_t *obj, uintptr_t value) +{ atomic_store_explicit((_Atomic(uintptr_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUIntRelaxed(unsigned int *obj, unsigned int value) +{ atomic_store_explicit((_Atomic(unsigned int) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StorePtrRelaxed(void **obj, void *value) +{ atomic_store_explicit((_Atomic(void *) *)(void **)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreSsizeRelaxed(Py_ssize_t *obj, Py_ssize_t value) +{ atomic_store_explicit((_Atomic(Py_ssize_t) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreULLongRelaxed(unsigned long long *obj, + unsigned long long value) +{ atomic_store_explicit((_Atomic(unsigned long long) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreCharRelaxed(char *obj, char value) +{ atomic_store_explicit((_Atomic(char) *)obj, value, __ATOMIC_RELEASE); } + +inline void +CereggiiAtomic_StoreUCharRelaxed(unsigned char *obj, unsigned char value) +{ atomic_store_explicit((_Atomic(unsigned char) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreShortRelaxed(short *obj, short value) +{ atomic_store_explicit((_Atomic(short) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreUShortRelaxed(unsigned short *obj, unsigned short value) +{ atomic_store_explicit((_Atomic(unsigned short) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreLongRelaxed(long *obj, long value) +{ atomic_store_explicit((_Atomic(long) *)obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreFloatRelaxed(float *obj, float value) +{ atomic_store_explicit((_Atomic(float) *) obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreDoubleRelaxed(double *obj, double value) +{ atomic_store_explicit((_Atomic(double) *) obj, value, memory_order_relaxed); } + +inline void +CereggiiAtomic_StoreLLongRelaxed(long long *obj, long long value) +{ atomic_store_explicit((_Atomic(long long) *)obj, value, memory_order_relaxed); } + + +// ---------------------------------------------------------------------------- +// _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release + +inline void * +CereggiiAtomic_LoadPtrAcquire(const void **obj) +{ return (void *) atomic_load_explicit((_Atomic(void *) *) obj, memory_order_acquire); } + +inline uintptr_t +CereggiiAtomic_LoadUIntPtrAcquire(const uintptr_t *obj) +{ return (uintptr_t)atomic_load_explicit((_Atomic(uintptr_t) *) obj, memory_order_acquire); } + +inline void +CereggiiAtomic_StorePtrRelease(void **obj, void *value) +{ atomic_store_explicit((_Atomic(void *) *) obj, value, memory_order_release); } + +inline void +CereggiiAtomic_StoreUIntPtrRelease(uintptr_t *obj, uintptr_t value) +{ atomic_store_explicit((_Atomic(uintptr_t) *) obj, value, memory_order_release); } + +inline void +CereggiiAtomic_StoreIntRelease(int *obj, int value) +{ atomic_store_explicit((_Atomic(int) *) obj, value, memory_order_release); } + +inline void +CereggiiAtomic_StoreUIntRelease(unsigned int *obj, unsigned int value) +{ atomic_store_explicit((_Atomic(unsigned int) *) obj, value, memory_order_release); } + +inline void +CereggiiAtomic_StoreSsizeRelease(Py_ssize_t *obj, Py_ssize_t value) +{ atomic_store_explicit((_Atomic(Py_ssize_t) *) obj, value, memory_order_release); } + +inline int +CereggiiAtomic_LoadIntAcquire(const int *obj) +{ return atomic_load_explicit((_Atomic(int) *) obj, memory_order_acquire); } + +inline void +CereggiiAtomic_StoreUInt32Release(uint32_t *obj, uint32_t value) +{ atomic_store_explicit((_Atomic(uint32_t) *) obj, value, memory_order_release); } + +inline void +CereggiiAtomic_StoreUInt64Release(uint64_t *obj, uint64_t value) +{ atomic_store_explicit((_Atomic(uint64_t) *) obj, value, memory_order_release); } + +inline uint64_t +CereggiiAtomic_LoadUInt64Acquire(const uint64_t *obj) +{ return atomic_load_explicit((_Atomic(uint64_t) *) obj, memory_order_acquire); } + +inline uint32_t +CereggiiAtomic_LoadUInt32Acquire(const uint32_t *obj) +{ return atomic_load_explicit((_Atomic(uint32_t) *) obj, memory_order_acquire); } + +inline Py_ssize_t +CereggiiAtomic_LoadSsizeAcquire(const Py_ssize_t *obj) +{ return atomic_load_explicit((_Atomic(Py_ssize_t) *) obj, memory_order_acquire); } + + +// ---------------------------------------------------------------------------- +// fence: separate regions of code + +inline void +CereggiiAtomic_FenceSeqCst(void) +{ atomic_thread_fence(memory_order_seq_cst); } + +inline void +CereggiiAtomic_FenceAcquire(void) +{ atomic_thread_fence(memory_order_acquire); } + +inline void +CereggiiAtomic_FenceRelease(void) +{ atomic_thread_fence(memory_order_release); } + + +#pragma clang diagnostic pop diff --git a/src/cereggii/include/cereggii/internal/export.h b/src/cereggii/include/cereggii/internal/export.h new file mode 100644 index 00000000..554070a0 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/export.h @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#ifndef CEREGGII_PUBLIC_H +#define CEREGGII_PUBLIC_H + +#if defined(_WIN32) || defined(__CYGWIN__) + #ifdef BUILDING_CEREGGII + #define CEREGGII_PUBLIC __declspec(dllexport) + #else + #define CEREGGII_PUBLIC __declspec(dllimport) + #endif +#else + #define CEREGGII_PUBLIC __attribute__((visibility("default"))) +#endif + +#endif // CEREGGII_PUBLIC_H diff --git a/src/cereggii/include/cereggii/internal/pyatomic_msc.h b/src/cereggii/include/cereggii/internal/pyatomic_msc.h new file mode 100644 index 00000000..f3900658 --- /dev/null +++ b/src/cereggii/include/cereggii/internal/pyatomic_msc.h @@ -0,0 +1,1187 @@ +// SPDX-FileCopyrightText: 2023-present dpdani +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \ + Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE)) + + +// --- _Py_atomic_add -------------------------------------------------------- + +static inline int8_t +_Py_atomic_add_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_add_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_add_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_add_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + int64_t new_value = old_value + value; + if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uint8_t +_Py_atomic_add_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value); +} + +static inline uint16_t +_Py_atomic_add_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value); +} + +static inline uint32_t +_Py_atomic_add_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline int +_Py_atomic_add_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline unsigned int +_Py_atomic_add_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +} + +static inline uint64_t +_Py_atomic_add_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +} + +static inline intptr_t +_Py_atomic_add_intptr(intptr_t *obj, intptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(int64_t); + return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value); +#endif +} + +static inline uintptr_t +_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + +static inline Py_ssize_t +_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(intptr_t); + return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value); +} + + +// --- _Py_atomic_compare_exchange ------------------------------------------- + +static inline int +_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + int8_t initial = (int8_t)_InterlockedCompareExchange8( + (volatile char *)obj, + (char)value, + (char)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + int16_t initial = (int16_t)_InterlockedCompareExchange16( + (volatile short *)obj, + (short)value, + (short)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + int32_t initial = (int32_t)_InterlockedCompareExchange( + (volatile long *)obj, + (long)value, + (long)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(__int64); + int64_t initial = (int64_t)_InterlockedCompareExchange64( + (volatile __int64 *)obj, + (__int64)value, + (__int64)*expected); + if (initial == *expected) { + return 1; + } + *expected = initial; + return 0; +} + +static inline int +_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value) +{ + void *initial = _InterlockedCompareExchangePointer( + (void**)obj, + value, + *(void**)expected); + if (initial == *(void**)expected) { + return 1; + } + *(void**)expected = initial; + return 0; +} + + +static inline int +_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value) +{ + return _Py_atomic_compare_exchange_int8((int8_t *)obj, + (int8_t *)expected, + (int8_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value) +{ + return _Py_atomic_compare_exchange_int16((int16_t *)obj, + (int16_t *)expected, + (int16_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value) +{ + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_int(int *obj, int *expected, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return _Py_atomic_compare_exchange_int32((int32_t *)obj, + (int32_t *)expected, + (int32_t)value); +} + +static inline int +_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value) +{ + return _Py_atomic_compare_exchange_int64((int64_t *)obj, + (int64_t *)expected, + (int64_t)value); +} + +static inline int +_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + +static inline int +_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return _Py_atomic_compare_exchange_ptr((void**)obj, + (void**)expected, + (void*)value); +} + + +// --- _Py_atomic_exchange --------------------------------------------------- + +static inline int8_t +_Py_atomic_exchange_int8(int8_t *obj, int8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value); +} + +static inline int16_t +_Py_atomic_exchange_int16(int16_t *obj, int16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value); +} + +static inline int32_t +_Py_atomic_exchange_int32(int32_t *obj, int32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value); +} + +static inline int64_t +_Py_atomic_exchange_int64(int64_t *obj, int64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value); +#else + int64_t old_value = _Py_atomic_load_int64_relaxed(obj); + for (;;) { + if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) { + return old_value; + } + } +#endif +} + +static inline void* +_Py_atomic_exchange_ptr(void *obj, void *value) +{ + return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value); +} + + +static inline uint8_t +_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value) +{ + return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj, + (int8_t)value); +} + +static inline uint16_t +_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value) +{ + return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj, + (int16_t)value); +} + +static inline uint32_t +_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value) +{ + return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline int +_Py_atomic_exchange_int(int *obj, int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline unsigned int +_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value) +{ + _Py_atomic_ASSERT_ARG_TYPE(int32_t); + return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj, + (int32_t)value); +} + +static inline uint64_t +_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value) +{ + return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj, + (int64_t)value); +} + +static inline intptr_t +_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline uintptr_t +_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + +static inline Py_ssize_t +_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj, + (void*)value); +} + + +// --- _Py_atomic_and -------------------------------------------------------- + +static inline uint8_t +_Py_atomic_and_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_and_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_and_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_and_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value & value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + +static inline uintptr_t +_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_or --------------------------------------------------------- + +static inline uint8_t +_Py_atomic_or_uint8(uint8_t *obj, uint8_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(char); + return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value); +} + +static inline uint16_t +_Py_atomic_or_uint16(uint16_t *obj, uint16_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(short); + return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value); +} + +static inline uint32_t +_Py_atomic_or_uint32(uint32_t *obj, uint32_t value) +{ + _Py_atomic_ASSERT_ARG_TYPE(long); + return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value); +} + +static inline uint64_t +_Py_atomic_or_uint64(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value); +#else + uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj); + for (;;) { + uint64_t new_value = old_value | value; + if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) { + return old_value; + } + } +#endif +} + + +static inline uintptr_t +_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value) +{ +#if SIZEOF_VOID_P == 8 + _Py_atomic_ASSERT_ARG_TYPE(uint64_t); + return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj, + (uint64_t)value); +#else + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj, + (uint32_t)value); +#endif +} + + +// --- _Py_atomic_load ------------------------------------------------------- + +static inline uint8_t +_Py_atomic_load_uint8(const uint8_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint8_t *)obj; +#elif defined(_M_ARM64) + return (uint8_t)__ldar8((unsigned __int8 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint8" +#endif +} + +static inline uint16_t +_Py_atomic_load_uint16(const uint16_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint16_t *)obj; +#elif defined(_M_ARM64) + return (uint16_t)__ldar16((unsigned __int16 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint16" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint32_t *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(volatile uint64_t *)obj; +#elif defined(_M_ARM64) + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64" +#endif +} + +static inline int8_t +_Py_atomic_load_int8(const int8_t *obj) +{ + return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj); +} + +static inline int16_t +_Py_atomic_load_int16(const int16_t *obj) +{ + return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj); +} + +static inline int32_t +_Py_atomic_load_int32(const int32_t *obj) +{ + return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj); +} + +static inline int +_Py_atomic_load_int(const int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline unsigned int +_Py_atomic_load_uint(const unsigned int *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(uint32_t); + return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj); +} + +static inline int64_t +_Py_atomic_load_int64(const int64_t *obj) +{ + return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj); +} + +static inline void* +_Py_atomic_load_ptr(const void *obj) +{ +#if SIZEOF_VOID_P == 8 + return (void*)_Py_atomic_load_uint64((const uint64_t *)obj); +#else + return (void*)_Py_atomic_load_uint32((const uint32_t *)obj); +#endif +} + +static inline intptr_t +_Py_atomic_load_intptr(const intptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (intptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline uintptr_t +_Py_atomic_load_uintptr(const uintptr_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (uintptr_t)_Py_atomic_load_ptr((void*)obj); +} + +static inline Py_ssize_t +_Py_atomic_load_ssize(const Py_ssize_t *obj) +{ + _Py_atomic_ASSERT_ARG_TYPE(void*); + return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj); +} + + +// --- _Py_atomic_load_relaxed ----------------------------------------------- + +static inline int +_Py_atomic_load_int_relaxed(const int *obj) +{ + return *(volatile int *)obj; +} + +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + +static inline int8_t +_Py_atomic_load_int8_relaxed(const int8_t *obj) +{ + return *(volatile int8_t *)obj; +} + +static inline int16_t +_Py_atomic_load_int16_relaxed(const int16_t *obj) +{ + return *(volatile int16_t *)obj; +} + +static inline int32_t +_Py_atomic_load_int32_relaxed(const int32_t *obj) +{ + return *(volatile int32_t *)obj; +} + +static inline int64_t +_Py_atomic_load_int64_relaxed(const int64_t *obj) +{ + return *(volatile int64_t *)obj; +} + +static inline intptr_t +_Py_atomic_load_intptr_relaxed(const intptr_t *obj) +{ + return *(volatile intptr_t *)obj; +} + +static inline uint8_t +_Py_atomic_load_uint8_relaxed(const uint8_t *obj) +{ + return *(volatile uint8_t *)obj; +} + +static inline uint16_t +_Py_atomic_load_uint16_relaxed(const uint16_t *obj) +{ + return *(volatile uint16_t *)obj; +} + +static inline uint32_t +_Py_atomic_load_uint32_relaxed(const uint32_t *obj) +{ + return *(volatile uint32_t *)obj; +} + +static inline uint64_t +_Py_atomic_load_uint64_relaxed(const uint64_t *obj) +{ + return *(volatile uint64_t *)obj; +} + +static inline uintptr_t +_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj) +{ + return *(volatile uintptr_t *)obj; +} + +static inline unsigned int +_Py_atomic_load_uint_relaxed(const unsigned int *obj) +{ + return *(volatile unsigned int *)obj; +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj) +{ + return *(volatile Py_ssize_t *)obj; +} + +static inline void* +_Py_atomic_load_ptr_relaxed(const void *obj) +{ + return *(void * volatile *)obj; +} + +static inline unsigned long long +_Py_atomic_load_ullong_relaxed(const unsigned long long *obj) +{ + return *(volatile unsigned long long *)obj; +} + +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + + +// --- _Py_atomic_store ------------------------------------------------------ + +static inline void +_Py_atomic_store_int(int *obj, int value) +{ + (void)_Py_atomic_exchange_int(obj, value); +} + +static inline void +_Py_atomic_store_int8(int8_t *obj, int8_t value) +{ + (void)_Py_atomic_exchange_int8(obj, value); +} + +static inline void +_Py_atomic_store_int16(int16_t *obj, int16_t value) +{ + (void)_Py_atomic_exchange_int16(obj, value); +} + +static inline void +_Py_atomic_store_int32(int32_t *obj, int32_t value) +{ + (void)_Py_atomic_exchange_int32(obj, value); +} + +static inline void +_Py_atomic_store_int64(int64_t *obj, int64_t value) +{ + (void)_Py_atomic_exchange_int64(obj, value); +} + +static inline void +_Py_atomic_store_intptr(intptr_t *obj, intptr_t value) +{ + (void)_Py_atomic_exchange_intptr(obj, value); +} + +static inline void +_Py_atomic_store_uint8(uint8_t *obj, uint8_t value) +{ + (void)_Py_atomic_exchange_uint8(obj, value); +} + +static inline void +_Py_atomic_store_uint16(uint16_t *obj, uint16_t value) +{ + (void)_Py_atomic_exchange_uint16(obj, value); +} + +static inline void +_Py_atomic_store_uint32(uint32_t *obj, uint32_t value) +{ + (void)_Py_atomic_exchange_uint32(obj, value); +} + +static inline void +_Py_atomic_store_uint64(uint64_t *obj, uint64_t value) +{ + (void)_Py_atomic_exchange_uint64(obj, value); +} + +static inline void +_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value) +{ + (void)_Py_atomic_exchange_uintptr(obj, value); +} + +static inline void +_Py_atomic_store_uint(unsigned int *obj, unsigned int value) +{ + (void)_Py_atomic_exchange_uint(obj, value); +} + +static inline void +_Py_atomic_store_ptr(void *obj, void *value) +{ + (void)_Py_atomic_exchange_ptr(obj, value); +} + +static inline void +_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value) +{ + (void)_Py_atomic_exchange_ssize(obj, value); +} + + +// --- _Py_atomic_store_relaxed ---------------------------------------------- + +static inline void +_Py_atomic_store_int_relaxed(int *obj, int value) +{ + *(volatile int *)obj = value; +} + +static inline void +_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value) +{ + *(volatile int8_t *)obj = value; +} + +static inline void +_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value) +{ + *(volatile int16_t *)obj = value; +} + +static inline void +_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value) +{ + *(volatile int32_t *)obj = value; +} + +static inline void +_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value) +{ + *(volatile int64_t *)obj = value; +} + +static inline void +_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value) +{ + *(volatile intptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value) +{ + *(volatile uint8_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value) +{ + *(volatile uint16_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value) +{ + *(volatile uint32_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value) +{ + *(volatile uint64_t *)obj = value; +} + +static inline void +_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value) +{ + *(volatile uintptr_t *)obj = value; +} + +static inline void +_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_ptr_relaxed(void *obj, void* value) +{ + *(void * volatile *)obj = value; +} + +static inline void +_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value) +{ + *(volatile Py_ssize_t *)obj = value; +} + +static inline void +_Py_atomic_store_ullong_relaxed(unsigned long long *obj, + unsigned long long value) +{ + *(volatile unsigned long long *)obj = value; +} + +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + *(volatile unsigned char *)obj = value; +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + *(volatile short *)obj = value; +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + *(volatile unsigned short *)obj = value; +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + *(volatile long *)obj = value; +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + *(volatile float *)obj = value; +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + *(volatile double *)obj = value; +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + *(volatile long long *)obj = value; +} + + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(void * volatile *)obj; +#elif defined(_M_ARM64) + return (void *)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ptr_acquire" +#endif +} + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uintptr_t volatile *)obj; +#elif defined(_M_ARM64) + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uintptr_acquire" +#endif +} + +static inline void +_Py_atomic_store_ptr_release(void *obj, void *value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(void * volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); +#else +# error "no implementation of _Py_atomic_store_ptr_release" +#endif +} + +static inline void +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uintptr_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uintptr_release" +#endif +} + +static inline void +_Py_atomic_store_int_release(int *obj, int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(int volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_int_release" +#endif +} + +static inline void +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(Py_ssize_t volatile *)obj = value; +#elif defined(_M_ARM64) + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_ssize_release" +#endif +} + +static inline int +_Py_atomic_load_int_acquire(const int *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(int volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + return (int)__ldar32((unsigned __int32 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_int_acquire" +#endif +} + +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + +static inline void +_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint64_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_uint64_release" +#endif +} + +static inline uint64_t +_Py_atomic_load_uint64_acquire(const uint64_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint64_t volatile *)obj; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(__int64); + return (uint64_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint64_acquire" +#endif +} + +static inline uint32_t +_Py_atomic_load_uint32_acquire(const uint32_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(uint32_t volatile *)obj; +#elif defined(_M_ARM64) + return (uint32_t)__ldar32((uint32_t volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_uint32_acquire" +#endif +} + +static inline Py_ssize_t +_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj) +{ +#if defined(_M_X64) || defined(_M_IX86) + return *(Py_ssize_t volatile *)obj; +#elif defined(_M_ARM64) + return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj); +#else +# error "no implementation of _Py_atomic_load_ssize_acquire" +#endif +} + +// --- _Py_atomic_fence ------------------------------------------------------ + + static inline void +_Py_atomic_fence_seq_cst(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) + __faststorefence(); +#elif defined(_M_IX86) + _mm_mfence(); +#else +# error "no implementation of _Py_atomic_fence_seq_cst" +#endif +} + + static inline void +_Py_atomic_fence_acquire(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISHLD); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadBarrier(); +#else +# error "no implementation of _Py_atomic_fence_acquire" +#endif +} + + static inline void +_Py_atomic_fence_release(void) +{ +#if defined(_M_ARM64) + __dmb(_ARM64_BARRIER_ISH); +#elif defined(_M_X64) || defined(_M_IX86) + _ReadWriteBarrier(); +#else +# error "no implementation of _Py_atomic_fence_release" +#endif +} + +#undef _Py_atomic_ASSERT_ARG_TYPE diff --git a/src/include/thread_id.h b/src/cereggii/include/cereggii/thread_id.h similarity index 100% rename from src/include/thread_id.h rename to src/cereggii/include/cereggii/thread_id.h diff --git a/src/include/atomic_ops.h b/src/include/atomic_ops.h deleted file mode 100644 index 6847f7b8..00000000 --- a/src/include/atomic_ops.h +++ /dev/null @@ -1,320 +0,0 @@ -// SPDX-FileCopyrightText: 2023-present dpdani -// -// SPDX-License-Identifier: Apache-2.0 - -#ifndef CEREGGII_ATOMIC_OPS_H -#define CEREGGII_ATOMIC_OPS_H - -#include "Python.h" - -// --- CereggiiAtomic_Add -------------------------------------------------------- -// Atomically adds `value` to `obj` and returns the previous value - -int CereggiiAtomic_AddInt(int *obj, int value); - -int8_t CereggiiAtomic_AddInt8(int8_t *obj, int8_t value); - -int16_t CereggiiAtomic_AddInt16(int16_t *obj, int16_t value); - -int32_t CereggiiAtomic_AddInt32(int32_t *obj, int32_t value); - -int64_t CereggiiAtomic_AddInt64(int64_t *obj, int64_t value); - -intptr_t CereggiiAtomic_AddIntPtr(intptr_t *obj, intptr_t value); - -unsigned int CereggiiAtomic_AddUInt(unsigned int *obj, unsigned int value); - -uint8_t CereggiiAtomic_AddUInt8(uint8_t *obj, uint8_t value); - -uint16_t CereggiiAtomic_AddUInt16(uint16_t *obj, uint16_t value); - -uint32_t CereggiiAtomic_AddUInt32(uint32_t *obj, uint32_t value); - -uint64_t CereggiiAtomic_AddUInt64(uint64_t *obj, uint64_t value); - -uintptr_t CereggiiAtomic_AddUIntPtr(uintptr_t *obj, uintptr_t value); - -Py_ssize_t CereggiiAtomic_AddSsize(Py_ssize_t *obj, Py_ssize_t value); - - -// --- CereggiiAtomic_CompareExchange ------------------------------------------- -// Performs an atomic compare-and-exchange. -// -// - If `*obj` and `*expected` are equal, store `desired` into `*obj` -// and return 1 (success). -// - Otherwise, store the `*obj` current value into `*expected` -// and return 0 (failure). -// -// These correspond to the C11 atomic_compare_exchange_strong() function. - -int CereggiiAtomic_CompareExchangeInt(int *obj, int expected, int desired); - -int CereggiiAtomic_CompareExchangeInt8(int8_t *obj, int8_t expected, int8_t desired); - -int CereggiiAtomic_CompareExchangeInt16(int16_t *obj, int16_t expected, int16_t desired); - -int CereggiiAtomic_CompareExchangeInt32(int32_t *obj, int32_t expected, int32_t desired); - -int CereggiiAtomic_CompareExchangeInt64(int64_t *obj, int64_t expected, int64_t desired); - -int CereggiiAtomic_CompareExchangeInt128(__int128_t *obj, __int128_t expected, __int128_t desired); - -int CereggiiAtomic_CompareExchangeIntPtr(intptr_t *obj, intptr_t expected, intptr_t desired); - -int CereggiiAtomic_CompareExchangeUInt(unsigned int *obj, unsigned int expected, unsigned int desired); - -int CereggiiAtomic_CompareExchangeUInt8(uint8_t *obj, uint8_t expected, uint8_t desired); - -int CereggiiAtomic_CompareExchangeUInt16(uint16_t *obj, uint16_t expected, uint16_t desired); - -int CereggiiAtomic_CompareExchangeUInt32(uint32_t *obj, uint32_t expected, uint32_t desired); - -int CereggiiAtomic_CompareExchangeUInt64(uint64_t *obj, uint64_t expected, uint64_t desired); - -int CereggiiAtomic_CompareExchangeUInt128(__uint128_t *obj, __uint128_t expected, __uint128_t desired); - -int CereggiiAtomic_CompareExchangeUIntPtr(uintptr_t *obj, uintptr_t expected, uintptr_t desired); - -int CereggiiAtomic_CompareExchangeSsize(Py_ssize_t *obj, Py_ssize_t expected, Py_ssize_t desired); - -int CereggiiAtomic_CompareExchangePtr(void **obj, void *expected, void *desired); - - -// --- CereggiiAtomic_Exchange --------------------------------------------------- -// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`. - -int CereggiiAtomic_ExchangeInt(int *obj, int value); - -int8_t CereggiiAtomic_ExchangeInt8(int8_t *obj, int8_t value); - -int16_t CereggiiAtomic_ExchangeInt16(int16_t *obj, int16_t value); - -int32_t CereggiiAtomic_ExchangeInt32(int32_t *obj, int32_t value); - -int64_t CereggiiAtomic_ExchangeInt64(int64_t *obj, int64_t value); - -intptr_t CereggiiAtomic_ExchangeIntPtr(intptr_t *obj, intptr_t value); - -unsigned int CereggiiAtomic_ExchangeUInt(unsigned int *obj, unsigned int value); - -uint8_t CereggiiAtomic_ExchangeUInt8(uint8_t *obj, uint8_t value); - -uint16_t CereggiiAtomic_ExchangeUInt16(uint16_t *obj, uint16_t value); - -uint32_t CereggiiAtomic_ExchangeUInt32(uint32_t *obj, uint32_t value); - -uint64_t CereggiiAtomic_ExchangeUInt64(uint64_t *obj, uint64_t value); - -uintptr_t CereggiiAtomic_ExchangeUIntPtr(uintptr_t *obj, uintptr_t value); - -Py_ssize_t CereggiiAtomic_ExchangeSsize(Py_ssize_t *obj, Py_ssize_t value); - -void *CereggiiAtomic_ExchangePtr(void **obj, void *value); - - -// --- CereggiiAtomic_and -------------------------------------------------------- -// Performs `*obj &= value` atomically and returns the previous value of `*obj`. - -uint8_t CereggiiAtomic_AndUInt8(uint8_t *obj, uint8_t value); - -uint16_t CereggiiAtomic_AndUInt16(uint16_t *obj, uint16_t value); - -uint32_t CereggiiAtomic_AndUInt32(uint32_t *obj, uint32_t value); - -uint64_t CereggiiAtomic_AndUInt64(uint64_t *obj, uint64_t value); - -uintptr_t CereggiiAtomic_AndUIntPtr(uintptr_t *obj, uintptr_t value); - - -// --- CereggiiAtomic_or --------------------------------------------------------- -// Performs `*obj |= value` atomically and returns the previous value of `*obj`. - -uint8_t CereggiiAtomic_OrUInt8(uint8_t *obj, uint8_t value); - -uint16_t CereggiiAtomic_OrUInt16(uint16_t *obj, uint16_t value); - -uint32_t CereggiiAtomic_OrUInt32(uint32_t *obj, uint32_t value); - -uint64_t CereggiiAtomic_OrUInt64(uint64_t *obj, uint64_t value); - -uintptr_t CereggiiAtomic_OrUIntPtr(uintptr_t *obj, uintptr_t value); - - -// --- CereggiiAtomic_load ------------------------------------------------------- -// Atomically loads `*obj` (sequential consistency) - -int CereggiiAtomic_LoadInt(const int *obj); - -int8_t CereggiiAtomic_LoadInt8(const int8_t *obj); - -int16_t CereggiiAtomic_LoadInt16(const int16_t *obj); - -int32_t CereggiiAtomic_LoadInt32(const int32_t *obj); - -int64_t CereggiiAtomic_LoadInt64(const int64_t *obj); - -intptr_t CereggiiAtomic_LoadIntPtr(const intptr_t *obj); - -uint8_t CereggiiAtomic_LoadUInt8(const uint8_t *obj); - -uint16_t CereggiiAtomic_LoadUInt16(const uint16_t *obj); - -uint32_t CereggiiAtomic_LoadUInt32(const uint32_t *obj); - -uint64_t CereggiiAtomic_LoadUInt64(const uint64_t *obj); - -uintptr_t CereggiiAtomic_LoadUIntPtr(const uintptr_t *obj); - -unsigned int CereggiiAtomic_LoadUInt(const unsigned int *obj); - -Py_ssize_t CereggiiAtomic_LoadSsize(const Py_ssize_t *obj); - -void *CereggiiAtomic_LoadPtr(const void **obj); - - -// --- CereggiiAtomic_LoadRelaxed ----------------------------------------------- -// Loads `*obj` (relaxed consistency, i.e., no ordering) - -int CereggiiAtomic_LoadIntRelaxed(const int *obj); - -int8_t CereggiiAtomic_LoadInt8Relaxed(const int8_t *obj); - -int16_t CereggiiAtomic_LoadInt16Relaxed(const int16_t *obj); - -int32_t CereggiiAtomic_LoadInt32Relaxed(const int32_t *obj); - -int64_t CereggiiAtomic_LoadInt64Relaxed(const int64_t *obj); - -intptr_t CereggiiAtomic_LoadIntptrRelaxed(const intptr_t *obj); - -uint8_t CereggiiAtomic_LoadUInt8Relaxed(const uint8_t *obj); - -uint16_t CereggiiAtomic_LoadUInt16Relaxed(const uint16_t *obj); - -uint32_t CereggiiAtomic_LoadUInt32Relaxed(const uint32_t *obj); - -uint64_t CereggiiAtomic_LoadUInt64Relaxed(const uint64_t *obj); - -uintptr_t CereggiiAtomic_LoadUIntptrRelaxed(const uintptr_t *obj); - -unsigned int CereggiiAtomic_LoadUIntRelaxed(const unsigned int *obj); - -Py_ssize_t CereggiiAtomic_LoadSsizeRelaxed(const Py_ssize_t *obj); - -void *CereggiiAtomic_LoadPtrRelaxed(const void *obj); - - -// --- CereggiiAtomic_store ------------------------------------------------------ -// Atomically performs `*obj = value` (sequential consistency) - -void CereggiiAtomic_StoreInt(int *obj, int value); - -void CereggiiAtomic_StoreInt8(int8_t *obj, int8_t value); - -void CereggiiAtomic_StoreInt16(int16_t *obj, int16_t value); - -void CereggiiAtomic_StoreInt32(int32_t *obj, int32_t value); - -void CereggiiAtomic_StoreInt64(int64_t *obj, int64_t value); - -void CereggiiAtomic_StoreIntPtr(intptr_t *obj, intptr_t value); - -void CereggiiAtomic_StoreUInt8(uint8_t *obj, uint8_t value); - -void CereggiiAtomic_StoreUInt16(uint16_t *obj, uint16_t value); - -void CereggiiAtomic_StoreUInt32(uint32_t *obj, uint32_t value); - -void CereggiiAtomic_StoreUInt64(uint64_t *obj, uint64_t value); - -void CereggiiAtomic_StoreUIntPtr(uintptr_t *obj, uintptr_t value); - -void CereggiiAtomic_StoreUInt(unsigned int *obj, unsigned int value); - -void CereggiiAtomic_StorePtr(void **obj, void *value); - -void CereggiiAtomic_StoreSsize(Py_ssize_t *obj, Py_ssize_t value); - - -// --- CereggiiAtomic_storeRelaxed ---------------------------------------------- -// Stores `*obj = value` (relaxed consistency, i.e., no ordering) - -void CereggiiAtomic_StoreIntRelaxed(int *obj, int value); - -void CereggiiAtomic_StoreInt8Relaxed(int8_t *obj, int8_t value); - -void CereggiiAtomic_StoreInt16Relaxed(int16_t *obj, int16_t value); - -void CereggiiAtomic_StoreInt32Relaxed(int32_t *obj, int32_t value); - -void CereggiiAtomic_StoreInt64Relaxed(int64_t *obj, int64_t value); - -void CereggiiAtomic_StoreIntptrRelaxed(intptr_t *obj, intptr_t value); - -void CereggiiAtomic_StoreUInt8Relaxed(uint8_t *obj, uint8_t value); - -void CereggiiAtomic_StoreUInt16Relaxed(uint16_t *obj, uint16_t value); - -void CereggiiAtomic_StoreUInt32Relaxed(uint32_t *obj, uint32_t value); - -void CereggiiAtomic_StoreUInt64Relaxed(uint64_t *obj, uint64_t value); - -void CereggiiAtomic_StoreUIntptrRelaxed(uintptr_t *obj, uintptr_t value); - -void CereggiiAtomic_StoreUIntRelaxed(unsigned int *obj, unsigned int value); - -void CereggiiAtomic_StorePtrRelaxed(void **obj, void *value); - -void CereggiiAtomic_StoreSsizeRelaxed(Py_ssize_t *obj, Py_ssize_t value); - - -// --- CereggiiAtomic_load_ptr_acquire / CereggiiAtomic_store_ptr_release ------------ - -// Loads `*obj` (acquire operation) -void *CereggiiAtomic_LoadPtrAcquire(const void **obj); - -// Stores `*obj = value` (release operation) -void CereggiiAtomic_StorePtrRelease(void **obj, void *value); - -void CereggiiAtomic_StoreIntRelease(int *obj, int value); - -int CereggiiAtomic_LoadIntAcquire(const int *obj); - - -// --- CereggiiAtomic_fence ------------------------------------------------------ - -// Sequential consistency fence. C11 fences have complex semantics. When -// possible, use the atomic operations on variables defined above, which -// generally do not require explicit use of a fence. -// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence -void CereggiiAtomic_FenceSeqCst(void); - -// Release fence -void CereggiiAtomic_FenceRelease(void); - - -// --- aliases --------------------------------------------------------------- - -#if SIZEOF_LONG == 8 -# define CereggiiAtomic_LoadULong(p) \ - CereggiiAtomic_LoadUInt64((uint64_t *)p) -# define CereggiiAtomic_LoadULongRelaxed(p) \ - CereggiiAtomic_LoadUInt64Relaxed((uint64_t *)p) -# define CereggiiAtomic_StoreULong(p, v) \ - CereggiiAtomic_StoreUInt64((uint64_t *)p, v) -# define CereggiiAtomic_StoreULongRelaxed(p, v) \ - CereggiiAtomic_StoreUInt64Relaxed((uint64_t *)p, v) -#elif SIZEOF_LONG == 4 -# define CereggiiAtomic_LoadULong(p) \ - CereggiiAtomic_LoadUInt32((uint32_t *)p) -# define CereggiiAtomic_LoadULongRelaxed(p) \ - CereggiiAtomic_LoadUInt32Relaxed((uint32_t *)p) -# define CereggiiAtomic_StoreULong(p, v) \ - CereggiiAtomic_StoreUInt32((uint32_t *)p, v) -# define CereggiiAtomic_StoreULongRelaxed(p, v) \ - CereggiiAtomic_StoreUInt32Relaxed((uint32_t *)p, v) -#else -# error "long must be 4 or 8 bytes in size" -#endif // SIZEOF_LONG - -#endif //CEREGGII_ATOMIC_OPS_H