Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 50 additions & 78 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,56 +44,79 @@ jobs:
echo "DuckDB mirror failed; downstream builds would 404"
exit 1

lint-toml:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6

validate-workflow-yaml:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Validate YAML file
run: |
# Lint the workflows and yamllint's configuration file.
yamllint \
--strict \
-c .yamllint.yaml \
.github/

python-lint:
name: "Python (lint)"
lightweight-checks:
name: "Formatting, Python lint and script tests"
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=amd64-ci-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=python-lint', github.run_id)
&& format('runs-on={0}/runner=amd64-ci-small/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=lightweight-checks', github.run_id)
|| 'ubuntu-latest' }}
timeout-minutes: 10
steps:
- parallel:
- uses: runs-on/action@v2
if: github.repository == 'vortex-data/vortex'
with:
sccache: s3
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: ./.github/actions/setup-prebuild
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
enable-sccache: "true"
sync: false
- name: Install missing formatting tools
run: |
packages=()
command -v yamllint >/dev/null || packages+=(yamllint)
command -v clang-format >/dev/null || packages+=(clang-format)
if [ ${#packages[@]} -gt 0 ]; then
sudo apt-get update
sudo apt-get install -y "${packages[@]}"
fi
- name: TOML Lint
if: "!cancelled()"
uses: spiraldb/actions/.github/actions/lint-toml@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
- name: Validate YAML file
if: "!cancelled()"
run: |
# Lint the workflows and yamllint's configuration file.
yamllint \
--strict \
-c .yamllint.yaml \
.github/
- name: C/C++ Lint - clang-format
if: "!cancelled()"
run: |
git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \
| grep -E '\.(cpp|hpp|cu|cuh|h)$' \
| grep -v 'arrow/reference/arrow_c_device\.h$' \
| grep -v 'kernels/src/bit_unpack_.*\.cu$' \
| grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \
| xargs clang-format --dry-run --Werror --style=file

# Use uvx for ruff to avoid building the Rust extension (saves ~4.5 min)
- name: Python Lint - Format
if: "!cancelled()"
run: uvx ruff format --check .
- name: Python Lint - Ruff
if: "!cancelled()"
run: uvx ruff check .
# ty needs the Python dependencies for type information, but resolves the
# vortex package itself from the in-tree sources and stubs, so skip the
# Rust extension build entirely.
- name: Python Lint - ty
if: "!cancelled()"
run: |
uv sync --all-packages --no-install-workspace
uvx ty check vortex-python vortex-python-cuda vortex-ffi/cmake/tests scripts/tests

# Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
# would break ON CONFLICT idempotency against the rows already in production.
- name: Pytest - measurement_id, benchmark reporting and ingestion
if: "!cancelled()"
run: |
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
--with tabulate --with xxhash \
pytest scripts/tests/test_measurement_id.py \
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py

python-test:
name: "Python (test)"
runs-on: >-
Expand Down Expand Up @@ -207,42 +230,6 @@ jobs:
print("vortex", vx.__version__, "round-tripped a file on Windows")
PY

# Pins the measurement_id hash that scripts/post-ingest.py uses as the primary key of every
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
# would break ON CONFLICT idempotency against the rows already in production.
measurement-id-golden:
name: "measurement_id golden vectors"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
# sync: false — the test runs via `uv run --no-project` and needs no workspace
# packages; the default `uv sync` builds the vortex-data Rust extension (~6 min).
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
- name: Pytest - measurement_id golden vectors
run: |
uv run --no-project --with pytest --with xxhash \
pytest scripts/tests/test_measurement_id.py

benchmark-reporting:
name: "Benchmark reporting"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
# sync: false -- the tests run via `uv run --no-project` and need no workspace packages.
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
- name: Pytest - benchmark reporting and ingestion
run: |
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
--with tabulate --with xxhash \
pytest scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py

python-cuda-test:
name: "Python CUDA (test)"
if: github.repository == 'vortex-data/vortex'
Expand Down Expand Up @@ -490,21 +477,6 @@ jobs:
run: |
cargo hack --no-dev-deps --ignore-private clippy --profile ci --no-default-features -- -D warnings

cpp-lint:
name: "C/C++ (lint)"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: C/C++ Lint - clang-format
run: |
git ls-files lang/cpp vortex-cuda vortex-duckdb vortex-ffi \
| grep -E '\.(cpp|hpp|cu|cuh|h)$' \
| grep -v 'arrow/reference/arrow_c_device\.h$' \
| grep -v 'kernels/src/bit_unpack_.*\.cu$' \
| grep -v 'kernels/src/bit_unpack_.*_lanes\.cuh$' \
| xargs clang-format --dry-run --Werror --style=file

rust-test-windows:
name: "Rust tests (windows-x64)"
env:
Expand Down
Loading