Feat/add cymem integration

.github/workflows/ci.yml

@@ -4,11 +4,6 @@ on:
   repository_dispatch:
     types: [cpython-release]
   workflow_dispatch:
-    inputs:
-      cpython_tag:
-        description: "CPython release tag to test (e.g., abc1234-nanvix-def5678)"
-        required: false
-        default: ""
   pull_request:
     branches:
       - main
@@ -21,7 +16,7 @@ permissions:
   issues: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.client_payload.cpython_tag || github.ref_name || 'default' }}
+  group: ${{ github.workflow }}-${{ github.ref_name || 'default' }}
   cancel-in-progress: true
 
 jobs:
@@ -80,7 +75,6 @@ jobs:
           echo "Short SHA: $SHORT_SHA"
 
   test:
-    needs: get-release-info
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -105,10 +99,12 @@ jobs:
       NANVIX_TOOLCHAIN: /opt/nanvix
       NANVIX_PLATFORM: ${{ matrix.platform }}
       NANVIX_PROCESS_MODE: ${{ matrix.process-mode }}
-      CPYTHON_TAG: ${{ needs.get-release-info.outputs.cpython_tag }}
       USER: runner
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 1
 
       - name: Setup
         env:
@@ -117,9 +113,35 @@ jobs:
           curl -fsSL https://bootstrap.pypa.io/get-pip.py | python3 - --break-system-packages
           ./z setup
 
+      - name: Build
+        run: ./z build
+
       - name: Test
         run: ./z test
 
+      - name: Collect failure logs
+        if: ${{ failure() }}
+        run: |
+          mkdir -p ci-logs
+          cp -f /tmp/*.log ci-logs/ 2>/dev/null || true
+          if [[ -d .nanvix ]]; then
+            find .nanvix -type f \( -name "*.log" -o -name "config.log" \) -print0 \
+              | xargs -0 -I{} cp -f "{}" ci-logs/ 2>/dev/null || true
+          fi
+          if [[ -d deps/cpython ]]; then
+            find deps/cpython -type f -name "config.log" -print0 \
+              | xargs -0 -I{} cp -f "{}" ci-logs/ 2>/dev/null || true
+          fi
+          ls -lah ci-logs || true
+
+      - name: Upload failure logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: failure-logs-${{ matrix.platform }}-${{ matrix.process-mode }}
+          path: ci-logs/*
+          if-no-files-found: warn
+
       - name: Collect runtime artifacts
         if: ${{ success() }}
         run: ./z release
@@ -133,7 +155,7 @@ jobs:
           if-no-files-found: error
 
   release-runtime-artifacts:
-    needs: [get-release-info, test]
+    needs: [test, get-release-info]
     if: >-
       ${{ success() &&
           (github.event_name == 'repository_dispatch' || github.event_name == 'workflow_dispatch' || github.event_name == 'push') }}
@@ -154,15 +176,14 @@ jobs:
           body: |
             Runtime artifacts generated by CI after successful tests.
 
-            - CPython tag: ${{ needs.get-release-info.outputs.cpython_tag }}
-            - Nanvix SHA: ${{ needs.get-release-info.outputs.nanvix_sha }}
+            - Commit: ${{ github.sha }}
             - Workflow run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
           files: |
             release-assets/*.tar.bz2
           overwrite_files: true
 
   report-failure:
-    needs: [get-release-info, test]
+    needs: [test]
     if: >-
       ${{ always() &&
           github.event_name == 'repository_dispatch' &&
@@ -172,15 +193,14 @@ jobs:
       - name: Create failure issue
         uses: actions/github-script@v7
         env:
-          CPYTHON_TAG: ${{ needs.get-release-info.outputs.cpython_tag }}
           TEST_RESULT: ${{ needs.test.result }}
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
-            const title = `Functional test failure for CPython ${process.env.CPYTHON_TAG}`;
+            const title = `Functional test failure for ${context.sha.substring(0, 7)}`;
             const body = [
-              `Functional tests failed for CPython release \`${process.env.CPYTHON_TAG}\`.`,
+              `Functional tests failed for commit \`${context.sha}\`.`,
               `- Trigger: ${context.eventName}`,
               `- Run: [#${context.runNumber}](${runUrl})`,
               `- test result: ${process.env.TEST_RESULT}`,

.gitmodules

@@ -0,0 +1,10 @@
+[submodule "deps/cpython"]
+	path = deps/cpython
+	url = https://github.com/nanvix/cpython.git
+	branch = nanvix/v3.12.3
+        shallow = true
+[submodule "deps/cymem"]
+        path = deps/cymem
+        url = https://github.com/nanvix/cymem.git
+        branch = nanvix/v2.0.11
+        shallow = true

patches/cymem_builtin.c

@@ -0,0 +1,22 @@
+/*
+ * cymem_builtin.c - Shim to register cymem as a CPython built-in module.
+ *
+ * makesetup does not support dotted module names, so the Cython extension
+ * is registered under the flat name "_cymem".  A pure-Python shim at
+ * cymem/cymem.py re-exports everything via `from _cymem import *`,
+ * making `from cymem.cymem import Pool` work transparently.
+ *
+ * The Cython-generated code in libcymem.a exports PyInit_cymem.
+ * This wrapper provides PyInit__cymem so the name matches the
+ * entry in Modules/Setup.local.
+ */
+
+#include "Python.h"
+
+extern PyObject* PyInit_cymem(void);
+
+PyMODINIT_FUNC
+PyInit__cymem(void)
+{
+    return PyInit_cymem();
+}

patches/cymem_shim.py

@@ -0,0 +1,7 @@
+# cymem/cymem.py - Pure-Python bridge to the _cymem built-in module.
+#
+# The Cython extension is statically linked into the CPython binary as the
+# built-in module "_cymem".  This shim re-exports its contents so that the
+# standard import path `from cymem.cymem import Pool` keeps working.
+
+from _cymem import *  # noqa: F401,F403

requirements/site-packages-extra.txt

@@ -63,4 +63,5 @@ wrapt==2.0.1
 
 # System and tooling
 cffi==1.17.1
+cymem
 plumbum==1.9.0

tests/func/test_101_cymem.py

@@ -0,0 +1,11 @@
+"""Test: cymem"""
+import sys
+sys.stdout.reconfigure(line_buffering=True)
+try:
+    from cymem.cymem import Pool
+    mem = Pool()
+    assert mem.size == 0
+    print("cymem: PASS")
+except Exception as e:
+    print(f"cymem: FAIL: {e}")
+    sys.exit(1)

z

@@ -9,7 +9,6 @@ NANVIX_HOME="${NANVIX_HOME:-}"
 NANVIX_TOOLCHAIN="${NANVIX_TOOLCHAIN:-/opt/nanvix}"
 NANVIX_PLATFORM="${NANVIX_PLATFORM:-hyperlight}"
 NANVIX_PROCESS_MODE="${NANVIX_PROCESS_MODE:-multi-process}"
-CPYTHON_TAG="${CPYTHON_TAG:-}"
 WORK_DIR="${WORK_DIR:-$ROOT_DIR/.nanvix}"
 DOWNLOAD_DIR="$WORK_DIR/downloads"
 SYSROOT="$WORK_DIR/sysroot"
@@ -31,15 +30,15 @@ usage() {
 Usage: ./z <command>
 
 Commands:
-  setup    Download Nanvix runtime and CPython release artifacts.
+  setup    Download Nanvix runtime, init submodules, fetch CPython build deps.
+  build    Build cymem and CPython from source with cymem as a built-in module.
   test     Run smoke and functional tests on nanvixd.elf.
   release  Package runtime artifacts and standalone bundle for release.
 
 Environment:
   NANVIX_PLATFORM     Target platform (default: hyperlight).
   NANVIX_PROCESS_MODE Process mode (default: multi-process).
   NANVIX_TOOLCHAIN    Toolchain root (default: /opt/nanvix).
-  CPYTHON_TAG         CPython release tag to download (default: latest).
   RELEASE_DIR         Output directory for release artifacts (default: ./release-assets).
   TEST_START           First test number to run (default: 1).
   TEST_END             Last test number to run (default: 999).
@@ -81,32 +80,121 @@ cmd_setup() {
 
   NANVIX_HOME="$SYSROOT"
 
-  # Download CPython release artifacts
-  log "downloading CPython release artifacts"
-  local cpython_artifact="cpython-${NANVIX_PLATFORM}-${NANVIX_PROCESS_MODE}.tar.bz2"
-  local cpython_archive="$DOWNLOAD_DIR/$cpython_artifact"
+  # Allow git operations in containerized CI where repository ownership
+  # may differ from the executing user (e.g., root in job containers).
+  log "configuring git safe directories"
+  git config --global --add safe.directory "$ROOT_DIR"
+  git config --global --add safe.directory "$ROOT_DIR/deps/cpython"
+  git config --global --add safe.directory "$ROOT_DIR/deps/cymem"
+
+  # Init/update git submodules
+  log "initializing submodules"
+  git submodule update --init --depth 1
+
+  # Exclude build artifacts from submodule dirty tracking
+  log "configuring submodule excludes"
+  local cpython_gitdir cymem_gitdir
+  cpython_gitdir="$(git -C "$ROOT_DIR/deps/cpython" rev-parse --git-dir)"
+  cymem_gitdir="$(git -C "$ROOT_DIR/deps/cymem" rev-parse --git-dir)"
+  grep -qxF 'Modules/cymem_builtin.c' "$cpython_gitdir/info/exclude" 2>/dev/null \
+    || echo 'Modules/cymem_builtin.c' >> "$cpython_gitdir/info/exclude"
+  grep -qxF 'libcymem.a' "$cymem_gitdir/info/exclude" 2>/dev/null \
+    || echo 'libcymem.a' >> "$cymem_gitdir/info/exclude"
+
+  # Download CPython build dependencies (sqlite, zlib, openssl, bzip2, libffi)
+  # via CPython's own setup script, reusing our NANVIX_HOME
+  log "downloading CPython build dependencies"
+  (
+    cd "$ROOT_DIR/deps/cpython"
+    NANVIX_HOME="$NANVIX_HOME" \
+    NANVIX_TOOLCHAIN="$NANVIX_TOOLCHAIN" \
+    NANVIX_PLATFORM="$NANVIX_PLATFORM" \
+    NANVIX_PROCESS_MODE="$NANVIX_PROCESS_MODE" \
+    GH_TOKEN="${GH_TOKEN:-}" \
+    ./z setup
+  )
 
-  if [[ -n "$CPYTHON_TAG" ]]; then
-    local url="https://github.com/nanvix/cpython/releases/download/${CPYTHON_TAG}/${cpython_artifact}"
-  else
-    local url="https://github.com/nanvix/cpython/releases/latest/download/${cpython_artifact}"
-  fi
+  log "setup complete"
+  log "SYSROOT=$SYSROOT"
+}
 
-  log "downloading: $url"
-  curl -fsSL "${curl_headers[@]}" -o "$cpython_archive" "$url"
+# Helper: invoke make with Nanvix cross-compilation settings
+nanvix_make() {
+  local dir="$1"
+  shift
 
-  # Overlay CPython onto sysroot
-  log "extracting CPython release: $cpython_artifact"
-  tar -xjf "$cpython_archive" -C "$WORK_DIR"
+  local make_args=(
+    -f Makefile.nanvix
+    CONFIG_NANVIX=y
+    "NANVIX_HOME=$SYSROOT"
+  )
 
-  # Merge cpython sysroot into main sysroot
-  if [[ -d "$WORK_DIR/sysroot" && "$WORK_DIR/sysroot" != "$SYSROOT" ]]; then
-    cp -a "$WORK_DIR/sysroot/." "$SYSROOT/"
-    rm -rf "$WORK_DIR/sysroot"
+  if [[ -n "$NANVIX_TOOLCHAIN" ]]; then
+    make_args+=("NANVIX_TOOLCHAIN=$NANVIX_TOOLCHAIN")
   fi
 
-  log "setup complete"
-  log "SYSROOT=$SYSROOT"
+  make -C "$dir" "${make_args[@]}" "$@"
+}
+
+cmd_build() {
+  [[ -d "$SYSROOT" ]] || die "sysroot not found at $SYSROOT. Run './z setup' first."
+  [[ -d "$ROOT_DIR/deps/cpython/Include" ]] || die "cpython submodule not initialized. Run './z setup' first."
+  [[ -d "$ROOT_DIR/deps/cymem/cymem" ]] || die "cymem submodule not initialized. Run './z setup' first."
+
+  local cpython_dir="$ROOT_DIR/deps/cpython"
+  local cymem_dir="$ROOT_DIR/deps/cymem"
+
+  # Step 1: Configure CPython (generates pyconfig.h and Makefile)
+  log "configuring CPython"
+  nanvix_make "$cpython_dir" configure
+
+  # Step 2: Install CPython headers into sysroot so cymem can compile against them
+  log "installing CPython headers into sysroot"
+  mkdir -p "$SYSROOT/include/python3.12"
+  cp -a "$cpython_dir"/Include/*.h "$SYSROOT/include/python3.12/"
+  cp -a "$cpython_dir"/Include/cpython "$SYSROOT/include/python3.12/" 2>/dev/null || true
+  cp -a "$cpython_dir"/Include/internal "$SYSROOT/include/python3.12/" 2>/dev/null || true
+  cp "$cpython_dir/pyconfig.h" "$SYSROOT/include/python3.12/"
+
+  # Step 3: Build cymem from source
+  log "building cymem"
+  nanvix_make "$cymem_dir" all
+  cp "$cymem_dir/libcymem.a" "$SYSROOT/lib/"
+  log "installed libcymem.a into sysroot"
+
+  # Step 4: Prepare CPython to include cymem as a built-in static module.
+  # makesetup rejects dotted names, so we register under the flat name "_cymem".
+  # A Python-side shim (cymem/cymem.py) re-exports via `from _cymem import *`.
+  log "patching CPython for cymem built-in module"
+  cp "$ROOT_DIR/patches/cymem_builtin.c" "$cpython_dir/Modules/"
+  cat > "$cpython_dir/Modules/Setup.local" << 'SETUP'
+# Built-in extensions for Nanvix (auto-generated by nanvix-python ./z build)
+# cymem: Cython memory pool helper - linked statically as _cymem
+*static*
+_cymem cymem_builtin.c -lcymem
+SETUP
+
+  # Step 5: Build CPython with cymem linked in
+  # makesetup detects the new Setup.local and regenerates config.c
+  log "building CPython"
+  nanvix_make "$cpython_dir" build
+
+  # Step 6: Install CPython into sysroot
+  log "installing CPython into sysroot"
+  local staging="$cpython_dir/.nanvix/_install_staging"
+  rm -rf "$staging"
+  nanvix_make "$cpython_dir" install DESTDIR="$staging"
+  cp -a "$staging/sysroot/." "$SYSROOT/"
+  rm -rf "$staging"
+
+  # Step 7: Install cymem Python-side shim so "from cymem.cymem import Pool" works
+  log "installing cymem Python shim"
+  local cymem_pkg="$SYSROOT/lib/python3.12/site-packages/cymem"
+  mkdir -p "$cymem_pkg"
+  cp "$ROOT_DIR/patches/cymem_shim.py" "$cymem_pkg/cymem.py"
+
+  log "build complete"
+  log "python3.12 at: $SYSROOT/bin/python3.12"
 }
 
 install_site_packages() {
@@ -239,7 +327,7 @@ run_functional_tests() {
 cmd_test() {
   [[ -d "$SYSROOT" ]] || die "sysroot not found at $SYSROOT. Run './z setup' first."
   [[ -x "$SYSROOT/bin/nanvixd.elf" ]] || die "missing executable: $SYSROOT/bin/nanvixd.elf"
-  [[ -x "$SYSROOT/bin/python3.12" ]] || die "missing executable: $SYSROOT/bin/python3.12"
+  [[ -x "$SYSROOT/bin/python3.12" ]] || die "missing executable: $SYSROOT/bin/python3.12. Run './z build' first."
 
   local site_pkg="$SYSROOT/lib/python3.12/site-packages"
   mkdir -p "$site_pkg"
@@ -252,7 +340,7 @@ cmd_test() {
 cmd_release() {
   [[ -d "$SYSROOT" ]] || die "sysroot not found at $SYSROOT. Run './z setup' first."
   [[ -x "$SYSROOT/bin/nanvixd.elf" ]] || die "missing executable: $SYSROOT/bin/nanvixd.elf"
-  [[ -x "$SYSROOT/bin/python3.12" ]] || die "missing executable: $SYSROOT/bin/python3.12"
+  [[ -x "$SYSROOT/bin/python3.12" ]] || die "missing executable: $SYSROOT/bin/python3.12. Run './z build' first."
 
   local release_dir="${RELEASE_DIR:-$ROOT_DIR/release-assets}"
   local asset_prefix="${NANVIX_PLATFORM}-${NANVIX_PROCESS_MODE}"
@@ -272,13 +360,6 @@ cmd_release() {
     log "release: added $(basename "$archive")"
   done
 
-  log "release: copying CPython runtime archives"
-  for archive in "$DOWNLOAD_DIR"/*.tar.bz2; do
-    [[ -f "$archive" ]] || continue
-    cp -a "$archive" "$release_dir/${asset_prefix}-$(basename "$archive")"
-    log "release: added $(basename "$archive")"
-  done
-
   log "release: building standalone bundle layout"
   cp -a "$SYSROOT" "$bundle_dir/sysroot"
 
@@ -331,6 +412,9 @@ case "$COMMAND" in
   setup)
     cmd_setup
     ;;
+  build)
+    cmd_build
+    ;;
   test)
     cmd_test
     ;;