diff --git a/.gitmodules b/.gitmodules index 7d8938e..e5ea87d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,9 +2,14 @@ path = deps/cpython url = https://github.com/nanvix/cpython.git branch = nanvix/v3.12.3 - shallow = true + shallow = true [submodule "deps/cymem"] - path = deps/cymem - url = https://github.com/nanvix/cymem.git - branch = nanvix/v2.0.11 - shallow = true + path = deps/cymem + url = https://github.com/nanvix/cymem.git + branch = nanvix/v2.0.11 + shallow = true +[submodule "deps/numpy"] + path = deps/numpy + url = https://github.com/nanvix/numpy.git + branch = nanvix/v1.26.4 + shallow = true diff --git a/deps/numpy b/deps/numpy new file mode 160000 index 0000000..c128afb --- /dev/null +++ b/deps/numpy @@ -0,0 +1 @@ +Subproject commit c128afb0626c703cb6cec53e25736223e76b84ab diff --git a/patches/nanvix_numpy_bootstrap.py b/patches/nanvix_numpy_bootstrap.py new file mode 100644 index 0000000..44d4195 --- /dev/null +++ b/patches/nanvix_numpy_bootstrap.py @@ -0,0 +1,89 @@ +""" +Lazy import redirector for NumPy built-in modules on Nanvix. +""" +import importlib +import importlib.abc +import importlib.machinery +import sys +import types + +_ALIASES = { + "numpy.core._multiarray_umath": "_np_multiarray_umath", + "numpy._core._multiarray_umath": "_np_multiarray_umath", + "_multiarray_umath": "_np_multiarray_umath", +} + +# C extension modules that are not statically linked but numpy tries +# to import them during init. Provide permissive stubs so init succeeds. +_STUBS = { + "numpy.core._multiarray_tests", + "numpy.core._simd", + "numpy.linalg._umath_linalg", + "numpy.linalg.lapack_lite", + "numpy.fft._pocketfft_internal", + "numpy.random._common", + "numpy.random._bounded_integers", + "numpy.random._generator", + "numpy.random._mt19937", + "numpy.random._pcg64", + "numpy.random._philox", + "numpy.random._sfc64", + "numpy.random.bit_generator", + "numpy.random.mtrand", +} + +_loading = set() + + +class _StubModule(types.ModuleType): + __all__ = [] + """Module stub that returns None for any missing attribute.""" + def __getattr__(self, name): + return None + + +class _NanvixLoader(importlib.abc.Loader): + def __init__(self, fullname): + self._fullname = fullname + + def create_module(self, spec): + fullname = self._fullname + + if fullname in _STUBS: + mod = _StubModule(fullname) + mod.__file__ = "" + return mod + + builtin_name = _ALIASES.get(fullname) + if not builtin_name: + return None + + # Try sys.modules first (C-level pre-registration) + mod = sys.modules.get(builtin_name) + if mod is not None: + return mod + + _loading.add(fullname) + try: + mod = importlib.import_module(builtin_name) + finally: + _loading.discard(fullname) + return mod + + def exec_module(self, module): + fullname = self._fullname + sys.modules[fullname] = module + if fullname != "_multiarray_umath" and fullname in _ALIASES: + sys.modules["_multiarray_umath"] = module + + +class _NanvixNumpyFinder(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path, target=None): + if fullname in _STUBS: + return importlib.machinery.ModuleSpec(fullname, _NanvixLoader(fullname)) + if fullname in _ALIASES and fullname not in _loading: + return importlib.machinery.ModuleSpec(fullname, _NanvixLoader(fullname)) + return None + + +sys.meta_path.insert(0, _NanvixNumpyFinder()) diff --git a/patches/numpy_libm_compat.c b/patches/numpy_libm_compat.c new file mode 100644 index 0000000..ac2e757 --- /dev/null +++ b/patches/numpy_libm_compat.c @@ -0,0 +1,8 @@ +#include + +long double __kernel_tanl(long double x, long double y, int iy) +{ + (void)y; + (void)iy; + return (long double)tan((double)x); +} diff --git a/patches/numpy_makefile_nanvix.patch b/patches/numpy_makefile_nanvix.patch new file mode 100644 index 0000000..7c7c706 --- /dev/null +++ b/patches/numpy_makefile_nanvix.patch @@ -0,0 +1,25 @@ +diff --git a/Makefile.nanvix b/Makefile.nanvix +index 0f9703e9a2..66c67c2814 100644 +--- a/Makefile.nanvix ++++ b/Makefile.nanvix +@@ -76,9 +76,9 @@ $(CROSS_FILE): + @echo '' >> $@ + @echo '[built-in options]' >> $@ + @echo "c_args = ['-I$(SYSROOT_PATH)/include', '-I$(SYSROOT_PATH)/include/python3.12']" >> $@ +- @echo "c_link_args = ['-static', '-L$(SYSROOT_PATH)/lib', '-T$(SYSROOT_PATH)/lib/user.ld', '-Wl,--start-group', '$(SYSROOT_PATH)/lib/libposix.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libc.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libm.a', '-Wl,--end-group']" >> $@ ++ @echo "c_link_args = ['-static', '-L$(SYSROOT_PATH)/lib', '-T$(SYSROOT_PATH)/lib/user.ld', '-Wl,--allow-multiple-definition', '-Wl,--start-group', '$(SYSROOT_PATH)/lib/libposix.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libc.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libm.a', '-Wl,--end-group']" >> $@ + @echo "cpp_args = ['-I$(SYSROOT_PATH)/include', '-I$(SYSROOT_PATH)/include/python3.12']" >> $@ +- @echo "cpp_link_args = ['-static', '-L$(SYSROOT_PATH)/lib', '-T$(SYSROOT_PATH)/lib/user.ld', '-Wl,--start-group', '$(SYSROOT_PATH)/lib/libposix.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libc.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libm.a', '-lstdc++', '-Wl,--end-group']" >> $@ ++ @echo "cpp_link_args = ['-static', '-L$(SYSROOT_PATH)/lib', '-T$(SYSROOT_PATH)/lib/user.ld', '-Wl,--allow-multiple-definition', '-Wl,--start-group', '$(SYSROOT_PATH)/lib/libposix.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libc.a', '$(TOOLCHAIN_PREFIX)/i686-nanvix/lib/libm.a', '-lstdc++', '-Wl,--end-group']" >> $@ + @echo '' >> $@ + @echo '[host_machine]' >> $@ + @echo "system = 'nanvix'" >> $@ +@@ -95,7 +95,7 @@ configure: $(CONFIGURED_MARKER) + + $(CONFIGURED_MARKER): $(CROSS_FILE) + PKG_CONFIG_LIBDIR="$(SYSROOT_PATH)/lib/pkgconfig" \ +- /usr/bin/python3 vendored-meson/meson/meson.py setup $(BUILDDIR) \ ++ python3 vendored-meson/meson/meson.py setup $(BUILDDIR) \ + --cross-file $(CROSS_FILE) \ + --pkg-config-path="$(SYSROOT_PATH)/lib/pkgconfig" \ + -Dallow-noblas=true \ diff --git a/patches/numpy_static_builtin.md b/patches/numpy_static_builtin.md new file mode 100644 index 0000000..ecaf9a4 --- /dev/null +++ b/patches/numpy_static_builtin.md @@ -0,0 +1,131 @@ +# NumPy Static Built-in Patch for Nanvix + +This patch modifies NumPy 1.26.4 so that its core C extension +(`_multiarray_umath`) can be **statically linked into the CPython binary** +as a built-in module on the Nanvix microkernel. + +Two files are touched: + +| File | Purpose | +|------|---------| +| `numpy/core/src/common/npy_cpu_features.c` | CPU feature detection | +| `numpy/core/src/multiarray/multiarraymodule.c` | Module initialisation | + +--- + +## 1. CPU feature detection (`npy_cpu_features.c`) + +### Problem + +NumPy probes the host CPU at import time with the privileged `CPUID` and +`XGETBV` instructions (inline assembly). On Nanvix these instructions +are not emulated in user mode and cause the process to **hang +indefinitely**. + +### Fix + +Guard both `npy__cpu_cpuid()` and `npy__cpu_getxcr0()` with +`#ifdef __nanvix__` so that on Nanvix they return zeroed-out results +instead of executing the instructions. NumPy then treats the CPU as +having no optional SIMD extensions, which is correct for the Nanvix +i686 target. + +--- + +## 2. Module initialisation (`multiarraymodule.c`) + +Three separate issues are addressed in this file. + +### 2a. Import-lock deadlock during `npy_cache_import` + +#### Problem + +`initialize_static_globals()` calls +`npy_cache_import("numpy.exceptions", ...)` and +`npy_cache_import("numpy.core._exceptions", ...)`. These trigger +`import numpy`, which in turn imports `numpy.core._multiarray_umath` — +the very module whose `PyInit` is currently executing. Because CPython +holds a **per-module import lock** for the duration of `PyInit`, the +re-entrant import attempt deadlocks. + +#### Fix + +Wrap both `npy_cache_import` calls in `#ifndef __nanvix__`. The two +global pointers (`npy_DTypePromotionError`, `npy_UFuncNoLoopError`) +remain `NULL` after init; they are populated lazily by +`npy_cache_import`'s own `*cache == NULL` guard on first actual use. + +An early-return guard (`if (npy_DTypePromotionError != NULL && …)`) is +also added so that a re-entrant call into `initialize_static_globals()` +is a harmless no-op instead of hitting the original `assert()` (which +would abort the process). + +### 2b. Module renamed to `_np_multiarray_umath` + +#### Problem + +If the built-in is registered under the canonical name +`_multiarray_umath`, any Python code that does +`import _multiarray_umath` during `PyInit` will re-enter the same +`PyInit` function (import-lock deadlock, same as 2a but from a different +call site). + +#### Fix + +On Nanvix the module name in `PyModuleDef` is changed to +`_np_multiarray_umath` and the init function is renamed to +`PyInit__np_multiarray_umath`. A meta-path finder installed in +`sitecustomize.py` at runtime redirects all imports of the dotted name +(`numpy.core._multiarray_umath`) to the renamed flat built-in, breaking +the circular chain. + +### 2c. Pre-registration in `sys.modules` + +#### Problem + +CPython only adds a built-in module to `sys.modules` **after** `PyInit` +returns. Any Python-level import triggered during `PyInit` (e.g. +`import datetime` via `PyDateTime_IMPORT`) that transitively tries to +resolve `numpy.core._multiarray_umath` will not find the module in +`sys.modules` yet, causing an `ImportError` or a second call to +`PyInit`. + +#### Fix + +Immediately after `PyModule_Create`, the partially-initialised module +object is inserted into `sys.modules` under three keys: + +| Key | Why | +|-----|-----| +| `_np_multiarray_umath` | The built-in's own name | +| `_multiarray_umath` | The name NumPy Python code expects | +| `numpy.core._multiarray_umath` | The fully-qualified dotted path | + +This lets the import machinery return the already-created (but not yet +fully populated) module object instead of calling `PyInit` again. + +--- + +## Runtime companion: `nanvix_numpy_bootstrap.py` + +The patch alone is not sufficient — a Python-side `sitecustomize.py` +module (`patches/nanvix_numpy_bootstrap.py`) is installed alongside +NumPy. It provides: + +* A **`find_spec`-based meta-path finder** that intercepts + `import numpy.core._multiarray_umath` (and similar dotted paths) and + redirects them to the renamed `_np_multiarray_umath` built-in. +* **Permissive stub modules** (`_StubModule`) for C extensions that are + not statically linked (e.g. `_multiarray_tests`, `_simd`, linalg, + fft, random sub-modules). These stubs have `__all__ = []` and a + `__getattr__` that returns `None`, allowing NumPy's init to complete + without those extensions. + +--- + +## Companion files + +| File | Role | +|------|------| +| `patches/numpy_libm_compat.c` | Provides `__kernel_tanl` stub missing from Nanvix newlib | +| `patches/nanvix_numpy_bootstrap.py` | Runtime meta-path finder and stub modules | diff --git a/patches/numpy_static_builtin.patch b/patches/numpy_static_builtin.patch new file mode 100644 index 0000000..9bcb6ef --- /dev/null +++ b/patches/numpy_static_builtin.patch @@ -0,0 +1,135 @@ +diff --git a/numpy/core/src/common/npy_cpu_features.c b/numpy/core/src/common/npy_cpu_features.c +index bd149f8b43..8ad7e17955 100644 +--- a/numpy/core/src/common/npy_cpu_features.c ++++ b/numpy/core/src/common/npy_cpu_features.c +@@ -376,7 +376,9 @@ npy__cpu_check_env(int disable, const char *env) { + static int + npy__cpu_getxcr0(void) + { +-#if defined(_MSC_VER) || defined (__INTEL_COMPILER) ++#ifdef __nanvix__ ++ return 0; /* XGETBV not available on Nanvix */ ++#elif defined(_MSC_VER) || defined (__INTEL_COMPILER) + return _xgetbv(0); + #elif defined(__GNUC__) || defined(__clang__) + /* named form of xgetbv not supported on OSX, so must use byte form, see: +@@ -393,7 +395,10 @@ npy__cpu_getxcr0(void) + static void + npy__cpu_cpuid(int reg[4], int func_id) + { +-#if defined(_MSC_VER) ++#ifdef __nanvix__ ++ /* CPUID instruction may hang on the Nanvix microkernel – skip */ ++ reg[0] = reg[1] = reg[2] = reg[3] = 0; ++#elif defined(_MSC_VER) + __cpuidex(reg, func_id, 0); + #elif defined(__INTEL_COMPILER) + __cpuid(reg, func_id); +diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c +index 66305722d1..8006fba2b3 100644 +--- a/numpy/core/src/multiarray/multiarraymodule.c ++++ b/numpy/core/src/multiarray/multiarraymodule.c +@@ -5024,11 +5024,31 @@ intern_strings(void) + * + * These globals should not need the C-layer at all and will be imported + * before anything on the C-side is initialized. ++ * ++ * Nanvix: When _multiarray_umath is a CPython built-in (statically linked), ++ * PyInit__multiarray_umath may be called more than once due to circular ++ * imports during numpy.core bootstrap. The original assertions are replaced ++ * with an early-return guard so that re-entrant initialisation succeeds. ++ * npy_cache_import is already idempotent (*cache == NULL guard). + */ + static int + initialize_static_globals(void) + { +- assert(npy_DTypePromotionError == NULL); ++ /* If both are already set we are in a re-entrant call – nothing to do. */ ++ if (npy_DTypePromotionError != NULL && npy_UFuncNoLoopError != NULL) { ++ return 0; ++ } ++ ++ /* ++ * Nanvix: When _np_multiarray_umath is a CPython built-in, calling ++ * npy_cache_import("numpy.exceptions", ...) here triggers ++ * `import numpy` which re-imports numpy.core._multiarray_umath, ++ * deadlocking on CPython's per-module import lock. ++ * ++ * Skip these imports on Nanvix; the globals remain NULL and callers ++ * use npy_cache_import lazily on first use. ++ */ ++#ifndef __nanvix__ + npy_cache_import( + "numpy.exceptions", "DTypePromotionError", + &npy_DTypePromotionError); +@@ -5036,13 +5056,13 @@ initialize_static_globals(void) + return -1; + } + +- assert(npy_UFuncNoLoopError == NULL); + npy_cache_import( + "numpy.core._exceptions", "_UFuncNoLoopError", + &npy_UFuncNoLoopError); + if (npy_UFuncNoLoopError == NULL) { + return -1; + } ++#endif + + /* Initialize from certain environment variabels: */ + char *env = getenv("NPY_NUMPY_2_BEHAVIOR"); +@@ -5064,7 +5084,11 @@ initialize_static_globals(void) + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, ++#ifdef __nanvix__ ++ "_np_multiarray_umath", ++#else + "_multiarray_umath", ++#endif + NULL, + -1, + array_module_methods, +@@ -5075,16 +5099,43 @@ static struct PyModuleDef moduledef = { + }; + + /* Initialization function for the module */ ++#ifdef __nanvix__ ++PyMODINIT_FUNC PyInit__np_multiarray_umath(void) { ++#else + PyMODINIT_FUNC PyInit__multiarray_umath(void) { ++#endif + PyObject *m, *d, *s; + PyObject *c_api; + ++ /* ++ * Nanvix: guard against re-entrant init when statically linked as a ++ * CPython built-in. If sys.modules already contains a partially- ++ * initialised module object (from an earlier call still on the stack), ++ * return it instead of creating a second one. ++ */ + /* Create the module and add the functions */ + m = PyModule_Create(&moduledef); + if (!m) { + return NULL; + } + ++#ifdef __nanvix__ ++ /* ++ * Nanvix: pre-register the (still empty) module in sys.modules under ++ * both the renamed flat name and the original name so that circular ++ * imports during our init (datetime, etc.) can find the partially- ++ * initialised module without re-entering PyInit. ++ */ ++ { ++ PyObject *modules = PyImport_GetModuleDict(); ++ if (modules != NULL) { ++ PyDict_SetItemString(modules, "_np_multiarray_umath", m); ++ PyDict_SetItemString(modules, "_multiarray_umath", m); ++ PyDict_SetItemString(modules, "numpy.core._multiarray_umath", m); ++ } ++ } ++#endif ++ + /* Initialize CPU features */ + if (npy_cpu_init() < 0) { + goto err; diff --git a/tests/func/test_102_numpy.py b/tests/func/test_102_numpy.py new file mode 100644 index 0000000..0c0ee12 --- /dev/null +++ b/tests/func/test_102_numpy.py @@ -0,0 +1,15 @@ +"""Test: numpy""" +import sys + +sys.stdout.reconfigure(line_buffering=True) + +try: + import numpy as np + + arr = np.array([1, 2, 3], dtype=np.int32) + assert int(arr.sum()) == 6 + assert arr.dtype == np.int32 + print("numpy: PASS") +except Exception as e: + print(f"numpy: FAIL: {e}") + sys.exit(1) diff --git a/z b/z index 22571b8..e9b6bf6 100755 --- a/z +++ b/z @@ -31,7 +31,7 @@ Usage: ./z Commands: setup Download Nanvix runtime, init submodules, fetch CPython build deps. - build Build cymem and CPython from source with cymem as a built-in module. + build Build cymem, numpy, and CPython from source with built-in modules. test Run smoke and functional tests on nanvixd.elf. release Package runtime artifacts and standalone bundle for release. @@ -86,20 +86,38 @@ cmd_setup() { git config --global --add safe.directory "$ROOT_DIR" git config --global --add safe.directory "$ROOT_DIR/deps/cpython" git config --global --add safe.directory "$ROOT_DIR/deps/cymem" + git config --global --add safe.directory "$ROOT_DIR/deps/numpy" # Init/update git submodules log "initializing submodules" - git submodule update --init --depth 1 + git submodule update --init --recursive --depth 1 # Exclude build artifacts from submodule dirty tracking log "configuring submodule excludes" - local cpython_gitdir cymem_gitdir + local cpython_gitdir cymem_gitdir numpy_gitdir cpython_gitdir="$(git -C "$ROOT_DIR/deps/cpython" rev-parse --git-dir)" cymem_gitdir="$(git -C "$ROOT_DIR/deps/cymem" rev-parse --git-dir)" + numpy_gitdir="$(git -C "$ROOT_DIR/deps/numpy" rev-parse --git-dir)" grep -qxF 'Modules/cymem_builtin.c' "$cpython_gitdir/info/exclude" 2>/dev/null \ || echo 'Modules/cymem_builtin.c' >> "$cpython_gitdir/info/exclude" grep -qxF 'libcymem.a' "$cymem_gitdir/info/exclude" 2>/dev/null \ || echo 'libcymem.a' >> "$cymem_gitdir/info/exclude" + grep -qxF 'builddir/' "$numpy_gitdir/info/exclude" 2>/dev/null \ + || echo 'builddir/' >> "$numpy_gitdir/info/exclude" + grep -qxF 'nanvix-cross.ini' "$numpy_gitdir/info/exclude" 2>/dev/null \ + || echo 'nanvix-cross.ini' >> "$numpy_gitdir/info/exclude" + grep -qxF '.nanvix-configured' "$numpy_gitdir/info/exclude" 2>/dev/null \ + || echo '.nanvix-configured' >> "$numpy_gitdir/info/exclude" + grep -qxF '.nanvix-cython-bin/' "$numpy_gitdir/info/exclude" 2>/dev/null \ + || echo '.nanvix-cython-bin/' >> "$numpy_gitdir/info/exclude" + + # Install Cython in a virtual environment (required by the NumPy Meson build + # to transpile .pyx files). The venv is created inside the work directory so + # it is disposable and does not pollute the system Python. + log "installing Cython" + python3 -m venv "$WORK_DIR/venv" + "$WORK_DIR/venv/bin/pip" install --quiet cython + export PATH="$WORK_DIR/venv/bin:$PATH" # Download CPython build dependencies (sqlite, zlib, openssl, bzip2, libffi) # via CPython's own setup script, reusing our NANVIX_HOME @@ -136,13 +154,69 @@ nanvix_make() { make -C "$dir" "${make_args[@]}" "$@" } +# Helper: build numpy archives with Docker fallback because numpy's +# Makefile.nanvix expects a native toolchain path even when Docker is preferred. +nanvix_make_numpy() { + local numpy_dir="$1" + shift + + if [[ -n "$NANVIX_TOOLCHAIN" && -x "$NANVIX_TOOLCHAIN/bin/i686-nanvix-gcc" ]]; then + nanvix_make "$numpy_dir" "$@" + return + fi + + command -v docker >/dev/null 2>&1 || die "docker not found and native Nanvix toolchain is unavailable" + docker image inspect nanvix/toolchain:latest-minimal >/dev/null 2>&1 \ + || die "docker image nanvix/toolchain:latest-minimal not found" + + local host_cython_dir="/usr/lib/python3/dist-packages/Cython" + [[ -d "$host_cython_dir" ]] || die "host Cython package not found at $host_cython_dir" + + local cython_shim_dir="$numpy_dir/.nanvix-cython-bin" + mkdir -p "$cython_shim_dir" + cat > "$cython_shim_dir/cython" <<'EOF' +#!/usr/bin/env sh +if [ "$#" -gt 0 ] && { [ "$1" = "-V" ] || [ "$1" = "--version" ]; }; then + python3 - <<'PY' +import Cython +print(f"Cython version {Cython.__version__}") +PY + exit 0 +fi +exec python3 -m Cython.Compiler.Main "$@" +EOF + chmod +x "$cython_shim_dir/cython" + cp "$cython_shim_dir/cython" "$cython_shim_dir/cython3" + + docker run --rm \ + --user "$(id -u):$(id -g)" \ + -v "$numpy_dir:/mnt/workspace" \ + -v "$SYSROOT:/mnt/sysroot" \ + -v "$SYSROOT:/sysroot" \ + -v "$cython_shim_dir:/mnt/cython-bin:ro" \ + -v "$host_cython_dir:/mnt/host-cython/Cython:ro" \ + -w /mnt/workspace \ + -e HOME=/tmp \ + -e PYTHONPATH=/mnt/host-cython \ + -e PATH=/mnt/cython-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + nanvix/toolchain:latest-minimal \ + make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/mnt/sysroot NANVIX_TOOLCHAIN=/opt/nanvix "$@" +} + cmd_build() { [[ -d "$SYSROOT" ]] || die "sysroot not found at $SYSROOT. Run './z setup' first." [[ -d "$ROOT_DIR/deps/cpython/Include" ]] || die "cpython submodule not initialized. Run './z setup' first." [[ -d "$ROOT_DIR/deps/cymem/cymem" ]] || die "cymem submodule not initialized. Run './z setup' first." + [[ -d "$ROOT_DIR/deps/numpy/numpy" ]] || die "numpy submodule not initialized. Run './z setup' first." + + # Activate the build virtual environment created by setup (provides Cython) + if [[ -f "$WORK_DIR/venv/bin/activate" ]]; then + source "$WORK_DIR/venv/bin/activate" + fi local cpython_dir="$ROOT_DIR/deps/cpython" local cymem_dir="$ROOT_DIR/deps/cymem" + local numpy_dir="$ROOT_DIR/deps/numpy" # Step 1: Configure CPython (generates pyconfig.h and Makefile) log "configuring CPython" @@ -162,24 +236,55 @@ cmd_build() { cp "$cymem_dir/libcymem.a" "$SYSROOT/lib/" log "installed libcymem.a into sysroot" - # Step 4: Prepare CPython to include cymem as a built-in static module. + # Step 4: Patch numpy for static built-in module support (re-entrant init) + log "patching numpy for static built-in support" + ( + cd "$numpy_dir" + if ! grep -q 'Nanvix.*re-entrant' numpy/core/src/multiarray/multiarraymodule.c 2>/dev/null; then + git apply "$ROOT_DIR/patches/numpy_static_builtin.patch" 2>/dev/null \ + || log "numpy patch already applied or not needed" + fi + git apply "$ROOT_DIR/patches/numpy_makefile_nanvix.patch" 2>/dev/null \ + || log "numpy makefile patch already applied or not needed" + ) + + # Step 5: Build numpy static archives for CPython built-in modules + log "building numpy archives" + rm -rf "$numpy_dir/builddir" "$numpy_dir/.nanvix-configured" "$numpy_dir/nanvix-cross.ini" + nanvix_make_numpy "$numpy_dir" all + log "installed numpy archives into sysroot/lib/numpy" + + # Step 5: Prepare CPython to include cymem and numpy as built-in static modules. # makesetup rejects dotted names, so we register under the flat name "_cymem". # A Python-side shim (cymem/cymem.py) re-exports via `from _cymem import *`. - log "patching CPython for cymem built-in module" + log "patching CPython for cymem and numpy built-in modules" cp "$ROOT_DIR/patches/cymem_builtin.c" "$cpython_dir/Modules/" - cat > "$cpython_dir/Modules/Setup.local" << 'SETUP' + cp "$ROOT_DIR/patches/numpy_libm_compat.c" "$cpython_dir/Modules/numpy_builtin/" + local cpython_numpy_sysroot="$SYSROOT" + if [[ -z "$NANVIX_TOOLCHAIN" || ! -x "$NANVIX_TOOLCHAIN/bin/i686-nanvix-gcc" ]]; then + cpython_numpy_sysroot="/mnt/sysroot" + fi + local numpy_lib_dir="${cpython_numpy_sysroot}/lib/numpy" + cat > "$cpython_dir/Modules/Setup.local" <> "$cpython_dir/Modules/Setup.local" + fi + rm -f "$cpython_dir/Modules/config.c" - # Step 5: Build CPython with cymem linked in + # Step 6: Build CPython with cymem/numpy linked in # makesetup detects the new Setup.local and regenerates config.c log "building CPython" nanvix_make "$cpython_dir" build - # Step 6: Install CPython into sysroot + # Step 7: Install CPython into sysroot log "installing CPython into sysroot" local staging="$cpython_dir/.nanvix/_install_staging" rm -rf "$staging" @@ -187,12 +292,46 @@ SETUP cp -a "$staging/sysroot/." "$SYSROOT/" rm -rf "$staging" - # Step 7: Install cymem Python-side shim so "from cymem.cymem import Pool" works + # Step 8: Install cymem Python-side shim so "from cymem.cymem import Pool" works log "installing cymem Python shim" local cymem_pkg="$SYSROOT/lib/python3.12/site-packages/cymem" mkdir -p "$cymem_pkg" cp "$ROOT_DIR/patches/cymem_shim.py" "$cymem_pkg/cymem.py" + # Step 9: Install numpy Python package plus Nanvix import bootstrap + log "installing numpy Python package" + local numpy_pkg="$SYSROOT/lib/python3.12/site-packages/numpy" + rm -rf "$numpy_pkg" + cp -a "$numpy_dir/numpy" "$numpy_pkg" + if [[ -f "$numpy_dir/builddir/numpy/version.py" ]]; then + cp "$numpy_dir/builddir/numpy/version.py" "$numpy_pkg/version.py" + else + cat > "$numpy_pkg/version.py" << 'EOF' +version = "1.26.4" +__version__ = version +full_version = version +short_version = version +git_revision = "nanvix" +release = True +EOF + fi + + cat > "$numpy_pkg/__config__.py" << 'EOF' +def show(): + return None + +def get_info(*args, **kwargs): + return {} +EOF + + rm -f "$numpy_pkg/__config__.py.in" + + # Remove __config__.py.in template, replaced by our stub above. + rm -f "$numpy_pkg/__config__.py.in" + + cp "$ROOT_DIR/patches/nanvix_numpy_bootstrap.py" "$SYSROOT/lib/python3.12/sitecustomize.py" + rm -f "$SYSROOT/lib/python3.12/site-packages/nanvix_numpy_bootstrap.pth" + log "build complete" log "python3.12 at: $SYSROOT/bin/python3.12" }