Skip to content

feat(ovpack): recursively enqueue directory vectorization #1

feat(ovpack): recursively enqueue directory vectorization

feat(ovpack): recursively enqueue directory vectorization #1

Workflow file for this run

name: 15. _Build Distribution
on:
workflow_call:
inputs:
os_json:
description: 'JSON string of runner labels to build on (ubuntu-24.04=x86_64, ubuntu-24.04-arm=aarch64, macos-14=arm64, macos-15-intel=x86_64, windows-latest=x86_64)'
required: false
type: string
default: '["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]'
python_json:
description: 'JSON string of Python versions'
required: false
type: string
default: '["3.10"]'
build_sdist:
description: 'Whether to build source distribution'
required: false
type: boolean
default: true
build_wheels:
description: 'Whether to build wheel distribution'
required: false
type: boolean
default: true
workflow_dispatch:
inputs:
build_sdist:
description: 'Whether to build source distribution'
required: false
type: boolean
default: true
build_wheels:
description: 'Whether to build wheel distribution'
required: false
type: boolean
default: true
os_json:
description: 'JSON string of runner labels to build on (ubuntu-24.04=x86_64, ubuntu-24.04-arm=aarch64, macos-14=arm64, macos-15-intel=x86_64, windows-latest=x86_64)'
required: false
default: '["ubuntu-24.04", "ubuntu-24.04-arm", "macos-14", "macos-15-intel", "windows-latest"]'
python_json:
description: 'JSON string of Python versions'
required: false
default: '["3.10"]'
jobs:
build-sdist:
name: Build source distribution py3.12
if: inputs.build_sdist
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
fetch-depth: 0 # Required for setuptools_scm to detect version from git tags
- name: Fetch all tags
run: git fetch --force --tags
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
- name: Create venv
run: uv venv
- name: Install build dependencies
run: uv pip install build setuptools_scm
- name: Clean workspace (force ignore dirty)
shell: bash
run: |
git reset --hard HEAD
git clean -fd
# For sdist, ensure local runtime binaries are not packaged even if present
rm -rf openviking/bin openviking/lib third_party/agfs/bin || true
rm -f openviking/storage/vectordb/*.so openviking/storage/vectordb/*.dylib openviking/storage/vectordb/*.dll openviking/storage/vectordb/*.exe || true
rm -rf openviking/_version.py openviking.egg-info
# Ignore uv.lock changes to avoid dirty state in setuptools_scm
git update-index --assume-unchanged uv.lock || true
- name: Debug Git and SCM
shell: bash
run: |
echo "=== Git Describe ==="
git describe --tags --long --dirty --always
echo "=== Setuptools SCM Version ==="
uv run --frozen python -m setuptools_scm
echo "=== Git Status (Ignored included) ==="
git status --ignored
echo "=== Check openviking/_version.py ==="
if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi
- name: Build sdist
run: uv build --sdist
- name: Store the distribution packages
uses: actions/upload-artifact@v7
with:
name: python-package-distributions-sdist
path: dist/*.tar.gz
- name: Display built sdist version
continue-on-error: true
run: |
VERSION=$(ls dist/*.tar.gz | head -n 1 | xargs basename | sed -E 's/^[^-]+-(.+)\.tar\.gz$/\1/')
echo "Build Version: $VERSION"
echo "::notice::Build sdist Version: $VERSION"
build-linux:
name: Build distribution on Linux ${{ matrix.arch }} (glibc 2.31) py${{ matrix.python-version }}
# Run if Linux runners are requested (explicit labels or generic 'linux')
if: >-
inputs.build_wheels &&
(
contains(inputs.os_json, 'linux') ||
contains(inputs.os_json, '"ubuntu-24.04"') ||
contains(inputs.os_json, 'ubuntu-24.04-arm')
)
runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
container: ubuntu:20.04
env:
DEBIAN_FRONTEND: noninteractive
TZ: Etc/UTC
strategy:
fail-fast: false
matrix:
python-version: ${{ fromJson(inputs.python_json) }}
arch: ${{ contains(inputs.os_json, 'linux') && fromJson('["x86_64","aarch64"]') || (contains(inputs.os_json, '"ubuntu-24.04"') && contains(inputs.os_json, 'ubuntu-24.04-arm')) && fromJson('["x86_64","aarch64"]') || contains(inputs.os_json, 'ubuntu-24.04-arm') && fromJson('["aarch64"]') || fromJson('["x86_64"]') }}
steps:
- name: Install system dependencies (Linux)
run: |
# Replace archive.ubuntu.com with azure.archive.ubuntu.com for better stability in GH Actions
sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//http:\/\/azure.archive.ubuntu.com\/ubuntu\//g' /etc/apt/sources.list
# Retry apt-get update
for i in 1 2 3 4 5; do apt-get update && break || sleep 5; done
apt-get install -y \
git ca-certificates cmake build-essential tzdata curl \
libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
libffi-dev liblzma-dev libgdbm-dev libnss3-dev libncurses5-dev \
libncursesw5-dev tk-dev uuid-dev libexpat1-dev
ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime
dpkg-reconfigure -f noninteractive tzdata
- uses: actions/checkout@v6
with:
submodules: recursive
fetch-depth: 0 # Required for setuptools_scm to detect version from git tags
- name: Fetch all tags
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
git fetch --force --tags
- name: Build CPython (Dynamic Selection)
run: |
# Map short version to full version for our specific build environment
PYTHON_VERSION="${{ matrix.python-version }}"
case "$PYTHON_VERSION" in
"3.9") PYTHON_FULL="3.9.18" ;;
"3.10") PYTHON_FULL="3.10.13" ;;
"3.11") PYTHON_FULL="3.11.8" ;;
"3.12") PYTHON_FULL="3.12.2" ;;
"3.13") PYTHON_FULL="3.13.2" ;;
"3.14") PYTHON_FULL="3.14.3" ;;
*)
echo "Error: Unknown python version $PYTHON_VERSION"
exit 1
;;
esac
PYTHON_PREFIX="/opt/python/${PYTHON_FULL}"
PYTHON_BIN="${PYTHON_PREFIX}/bin/python${{ matrix.python-version }}"
if [ ! -x "$PYTHON_BIN" ]; then
curl -fsSL -o /tmp/Python-${PYTHON_FULL}.tgz \
https://www.python.org/ftp/python/${PYTHON_FULL}/Python-${PYTHON_FULL}.tgz
tar -xzf /tmp/Python-${PYTHON_FULL}.tgz -C /tmp
cd /tmp/Python-${PYTHON_FULL}
CFLAGS="-fPIC" \
./configure --prefix="${PYTHON_PREFIX}" --with-ensurepip=install --enable-shared
make -j"$(nproc)"
make install
fi
echo "PYTHON_BIN=${PYTHON_BIN}" >> "$GITHUB_ENV"
echo "LD_LIBRARY_PATH=${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
export LD_LIBRARY_PATH="${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}"
"$PYTHON_BIN" -V
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'
- name: Set up Rust
uses: dtolnay/rust-toolchain@v1
with:
toolchain: stable
targets: ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }}
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
- name: Create venv (Linux)
run: uv venv --python "$PYTHON_BIN"
- name: Seed pip (Linux)
run: uv run python -m ensurepip --upgrade
- name: Install dependencies
run: uv sync --frozen
- name: Install build dependencies
run: uv pip install setuptools setuptools_scm cmake wheel build
- name: Resolve OpenViking version for Rust CLI (Linux)
shell: bash
run: |
OPENVIKING_VERSION=$(
uv run --frozen python -c "from build_support.versioning import resolve_openviking_version; print(resolve_openviking_version())"
)
echo "OPENVIKING_VERSION=$OPENVIKING_VERSION" >> "$GITHUB_ENV"
echo "Resolved OpenViking version: $OPENVIKING_VERSION"
- name: Build Rust CLI (Linux)
run: cargo build --release --target ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }} -p ov_cli
- name: Copy Rust CLI binary (Linux)
run: |
mkdir -p openviking/bin
cp target/${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }}/release/ov openviking/bin/
chmod +x openviking/bin/ov
- name: Build ragfs-python and extract into openviking/lib/ (Linux)
shell: bash
run: |
uv pip install maturin
TMPDIR=$(mktemp -d)
cd crates/ragfs-python
maturin build --release \
--target ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-linux-gnu' || 'x86_64-unknown-linux-gnu' }} \
--out "$TMPDIR"
cd ../..
mkdir -p openviking/lib
python3 -c "
import zipfile, glob, os, sys

Check failure on line 253 in .github/workflows/_build.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/_build.yml

Invalid workflow file

You have an error in your yaml syntax on line 253
whls = glob.glob(os.path.join('$TMPDIR', 'ragfs_python-*.whl'))
assert whls, 'maturin produced no wheel'
with zipfile.ZipFile(whls[0]) as zf:
for name in zf.namelist():
bn = os.path.basename(name)
if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd')):
dst = os.path.join('openviking', 'lib', bn)
with zf.open(name) as src, open(dst, 'wb') as f:
f.write(src.read())
os.chmod(dst, 0o755)
print(f'Extracted {bn} -> {dst}')
sys.exit(0)
print('ERROR: No ragfs_python .so/.pyd found in wheel')
sys.exit(1)
"
rm -rf "$TMPDIR"
echo "Contents of openviking/lib/:"
ls -la openviking/lib/
- name: Clean workspace (force ignore dirty)
shell: bash
run: |
# Back up pre-built artifacts before cleaning
cp -a openviking/bin /tmp/_ov_bin || true
cp -a openviking/lib /tmp/_ov_lib || true
git reset --hard HEAD
git clean -fd
rm -rf openviking/_version.py openviking.egg-info
# Restore pre-built artifacts
cp -a /tmp/_ov_bin openviking/bin || true
cp -a /tmp/_ov_lib openviking/lib || true
# Ignore uv.lock changes to avoid dirty state in setuptools_scm
git update-index --assume-unchanged uv.lock || true
- name: Debug Git and SCM
shell: bash
run: |
echo "=== Git Describe ==="
git describe --tags --long --dirty --always
echo "=== Setuptools SCM Version ==="
uv run --frozen python -m setuptools_scm
echo "=== Git Status (Ignored included) ==="
git status --ignored
echo "=== Check openviking/_version.py ==="
if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi
echo "=== Verify pre-built artifacts survived clean ==="
ls -la openviking/bin/ openviking/lib/ || true
- name: Build package (Wheel Only)
run: uv build --wheel
- name: Install patchelf (Linux)
run: |
PATCHELF_VERSION=0.18.0
curl -fsSL -o /tmp/patchelf-${PATCHELF_VERSION}.tar.gz \
https://github.com/NixOS/patchelf/releases/download/${PATCHELF_VERSION}/patchelf-${PATCHELF_VERSION}.tar.gz
tar -xzf /tmp/patchelf-${PATCHELF_VERSION}.tar.gz -C /tmp
cd /tmp/patchelf-${PATCHELF_VERSION}
./configure
make -j"$(nproc)"
make install
patchelf --version
- name: Repair wheels (Linux)
run: |
uv pip install auditwheel
uv run auditwheel repair dist/*.whl -w dist_fixed
rm dist/*.whl
mv dist_fixed/*.whl dist/
rmdir dist_fixed
- name: Store the distribution packages
uses: actions/upload-artifact@v7
with:
name: python-package-distributions-linux-${{ matrix.arch }}-${{ matrix.python-version }}
path: dist/
- name: Display built wheel version
continue-on-error: true
run: |
VERSION=$(ls dist/*.whl | head -n 1 | xargs basename | cut -d- -f2)
echo "Build Version: $VERSION"
echo "::notice::Build Wheel Version (Linux ${{ matrix.arch }} glibc 2.31 py${{ matrix.python-version }}): $VERSION"
build-other:
name: Build non-Linux distributions
# Run only when non-Linux runners are explicitly requested
if: inputs.build_wheels && (contains(inputs.os_json, 'macos') || contains(inputs.os_json, 'windows'))
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: ${{ fromJson(inputs.os_json) }}
python-version: ${{ fromJson(inputs.python_json) }}
# Exclude ubuntu-24.04 from this matrix if it was passed in inputs
exclude:
- os: linux
- os: ubuntu-24.04
- os: ubuntu-24.04-arm
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
fetch-depth: 0 # Required for setuptools_scm to detect version from git tags
- name: Fetch all tags
run: git fetch --force --tags
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Configure macOS wheel architecture tag
if: runner.os == 'macOS'
shell: bash
run: |
if [[ "${{ matrix.os }}" == "macos-14" ]]; then
TARGET_ARCH="arm64"
MACOS_VERSION="14.0"
elif [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
TARGET_ARCH="x86_64"
MACOS_VERSION="15.0"
else
echo "Unsupported macOS runner for release wheels: ${{ matrix.os }}"
exit 1
fi
echo "ARCHFLAGS=-arch ${TARGET_ARCH}" >> "$GITHUB_ENV"
echo "CMAKE_OSX_ARCHITECTURES=${TARGET_ARCH}" >> "$GITHUB_ENV"
echo "_PYTHON_HOST_PLATFORM=macosx-${MACOS_VERSION}-${TARGET_ARCH}" >> "$GITHUB_ENV"
echo "Configured macOS wheel platform: macosx-${MACOS_VERSION}-${TARGET_ARCH}"
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'
- name: Set up Rust
uses: dtolnay/rust-toolchain@v1
with:
toolchain: stable
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: brew install cmake
- name: Install system dependencies (Windows)
if: runner.os == 'Windows'
run: |
choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System'
choco install mingw
- name: Install dependencies
run: uv sync --frozen
- name: Install build dependencies
run: uv pip install setuptools setuptools_scm cmake wheel build
- name: Resolve OpenViking version for Rust CLI (macOS/Windows)
shell: bash
run: |
OPENVIKING_VERSION=$(
uv run --frozen python -c "from build_support.versioning import resolve_openviking_version; print(resolve_openviking_version())"
)
echo "OPENVIKING_VERSION=$OPENVIKING_VERSION" >> "$GITHUB_ENV"
echo "Resolved OpenViking version: $OPENVIKING_VERSION"
- name: Build Rust CLI (macOS/Windows)
shell: bash
run: |
if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
cargo build --release --target x86_64-pc-windows-msvc -p ov_cli
else
cargo build --release -p ov_cli
fi
- name: Copy Rust CLI binary (macOS/Windows)
shell: bash
run: |
mkdir -p openviking/bin
if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
cp target/x86_64-pc-windows-msvc/release/ov.exe openviking/bin/
else
cp target/release/ov openviking/bin/
chmod +x openviking/bin/ov
fi
- name: Build ragfs-python and extract into openviking/lib/ (macOS/Windows)
shell: bash
run: |
uv pip install maturin
TMPDIR=$(mktemp -d)
cd crates/ragfs-python
if [[ "${{ matrix.os }}" == "windows-latest" ]]; then
maturin build --release --target x86_64-pc-windows-msvc --out "$TMPDIR"
else
maturin build --release --out "$TMPDIR"
fi
cd ../..
mkdir -p openviking/lib
python3 -c "
import zipfile, glob, os, sys
whls = glob.glob(os.path.join('$TMPDIR', 'ragfs_python-*.whl'))
assert whls, 'maturin produced no wheel'
with zipfile.ZipFile(whls[0]) as zf:
for name in zf.namelist():
bn = os.path.basename(name)
if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd')):
dst = os.path.join('openviking', 'lib', bn)
with zf.open(name) as src, open(dst, 'wb') as f:
f.write(src.read())
os.chmod(dst, 0o755)
print(f'Extracted {bn} -> {dst}')
sys.exit(0)
print('ERROR: No ragfs_python .so/.pyd found in wheel')
sys.exit(1)
"
rm -rf "$TMPDIR"
echo "Contents of openviking/lib/:"
ls -la openviking/lib/
- name: Clean workspace (force ignore dirty)
shell: bash
run: |
# Back up pre-built artifacts before cleaning
cp -a openviking/bin /tmp/_ov_bin || true
cp -a openviking/lib /tmp/_ov_lib || true
git reset --hard HEAD
git clean -fd
rm -rf openviking/_version.py openviking.egg-info
# Restore pre-built artifacts
cp -a /tmp/_ov_bin openviking/bin || true
cp -a /tmp/_ov_lib openviking/lib || true
# Ignore uv.lock changes to avoid dirty state in setuptools_scm
git update-index --assume-unchanged uv.lock || true
- name: Debug Git and SCM
shell: bash
run: |
echo "=== Git Describe ==="
git describe --tags --long --dirty --always
echo "=== Setuptools SCM Version ==="
uv run --frozen python -m setuptools_scm
echo "=== Git Status (Ignored included) ==="
git status --ignored
echo "=== Check openviking/_version.py ==="
if [ -f openviking/_version.py ]; then cat openviking/_version.py; else echo "Not found"; fi
echo "=== Verify pre-built artifacts survived clean ==="
ls -la openviking/bin/ openviking/lib/ || true
- name: Build package (Wheel Only)
run: uv build --wheel
- name: Smoke test built wheel (Windows)
if: runner.os == 'Windows'
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --force-reinstall dist/*.whl
cd "$RUNNER_TEMP"
python - <<'PY'
import importlib.util
import openviking.storage.vectordb.engine as engine
print(f"Loaded runtime engine variant {engine.ENGINE_VARIANT}")
print(f"Available engine variants {engine.AVAILABLE_ENGINE_VARIANTS}")
module_name = f"openviking.storage.vectordb.engine._{engine.ENGINE_VARIANT}"
backend_spec = importlib.util.find_spec(module_name)
if backend_spec is None or backend_spec.origin is None:
raise SystemExit(f"backend module {module_name} was not installed")
print(f"Imported backend module {module_name}")
print(f"Backend module origin {backend_spec.origin}")
PY
- name: Store the distribution packages
uses: actions/upload-artifact@v7
with:
name: python-package-distributions-${{ matrix.os == 'macos-14' && 'macos-arm64' || matrix.os == 'macos-15-intel' && 'macos-x86_64' || matrix.os == 'windows-latest' && 'windows-x86_64' || matrix.os }}-${{ matrix.python-version }}
path: dist/
- name: Display built wheel version
shell: bash
continue-on-error: true
run: |
VERSION=$(ls dist/*.whl | head -n 1 | xargs basename | cut -d- -f2)
echo "Build Version: $VERSION"
echo "::notice::Build Wheel Version (${{ matrix.os == 'macos-14' && 'macOS arm64 (macos-14)' || matrix.os == 'macos-15-intel' && 'macOS x86_64 (macos-15-intel)' || matrix.os == 'windows-latest' && 'Windows x86_64 (windows-latest)' || matrix.os }} py${{ matrix.python-version }}): $VERSION"