Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions .github/workflows/_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,6 @@ jobs:
git reset --hard HEAD
git clean -fd
# For sdist, ensure local runtime binaries are not packaged even if present
rm -rf openviking/bin openviking/lib third_party/agfs/bin || true
rm -f openviking/storage/vectordb/*.so openviking/storage/vectordb/*.dylib openviking/storage/vectordb/*.dll openviking/storage/vectordb/*.exe || true
rm -rf openviking/_version.py openviking.egg-info
# Ignore uv.lock changes to avoid dirty state in setuptools_scm
git update-index --assume-unchanged uv.lock || true

Expand Down Expand Up @@ -193,11 +190,6 @@ jobs:
echo "LD_LIBRARY_PATH=${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
export LD_LIBRARY_PATH="${PYTHON_PREFIX}/lib:${LD_LIBRARY_PATH}"
"$PYTHON_BIN" -V
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'

- name: Set up Rust
uses: dtolnay/rust-toolchain@v1
with:
Expand Down Expand Up @@ -351,11 +343,6 @@ jobs:
echo "_PYTHON_HOST_PLATFORM=macosx-${MACOS_VERSION}-${TARGET_ARCH}" >> "$GITHUB_ENV"
echo "Configured macOS wheel platform: macosx-${MACOS_VERSION}-${TARGET_ARCH}"

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'

- name: Set up Rust
uses: dtolnay/rust-toolchain@v1
with:
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/_codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,6 @@ jobs:
with:
python-version: '3.11'

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 'stable'

- name: Install uv
uses: astral-sh/setup-uv@v7
with:
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/_lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,6 @@ jobs:
with:
python-version: '3.11'

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 'stable'

- name: Install uv
uses: astral-sh/setup-uv@v7
with:
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/_test_full.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'

- name: Install uv
uses: astral-sh/setup-uv@v7
with:
Expand Down
5 changes: 0 additions & 5 deletions .github/workflows/_test_lite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,6 @@ jobs:
with:
python-version: ${{ matrix.python-version }}

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'

- name: Install uv
uses: astral-sh/setup-uv@v7
with:
Expand Down
14 changes: 0 additions & 14 deletions .github/workflows/api_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,6 @@ jobs:
with:
python-version: '3.10'

- name: Cache Go modules
uses: actions/cache@v5
continue-on-error: true
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('third_party/agfs/**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-

- name: Cache Python dependencies (Unix)
if: runner.os != 'Windows'
uses: actions/cache@v5
Expand All @@ -86,11 +77,6 @@ jobs:
restore-keys: |
${{ runner.os }}-pip-

- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.25.1'

- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
Expand Down
21 changes: 7 additions & 14 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,22 +1,17 @@
# syntax=docker/dockerfile:1.9

# Stage 1: provide Go toolchain (required by setup.py -> build_agfs_artifacts -> make build)
FROM golang:1.26-trixie AS go-toolchain

# Stage 2: provide Rust toolchain (required by setup.py -> build_ov_cli_artifact -> cargo build)
# Stage 1: provide Rust toolchain (required by setup.py -> build_ov_cli_artifact -> cargo build)
FROM rust:1.88-trixie AS rust-toolchain

# Stage 3: build Python environment with uv (builds AGFS + Rust CLI + C++ extension from source)
# Stage 2: build Python environment with uv (builds Rust CLI + C++ extension from source)
FROM ghcr.io/astral-sh/uv:python3.13-trixie-slim AS py-builder

# Reuse Go toolchain from stage 1 so setup.py can compile agfs-server in-place.
COPY --from=go-toolchain /usr/local/go /usr/local/go
# Reuse Rust toolchain from stage 2 so setup.py can compile ov CLI in-place.
# Reuse Rust toolchain from stage 1 so setup.py can compile ov CLI in-place.
COPY --from=rust-toolchain /usr/local/cargo /usr/local/cargo
COPY --from=rust-toolchain /usr/local/rustup /usr/local/rustup
ENV CARGO_HOME=/usr/local/cargo
ENV RUSTUP_HOME=/usr/local/rustup
ENV PATH="/app/.venv/bin:/usr/local/cargo/bin:/usr/local/go/bin:${PATH}"
ENV PATH="/app/.venv/bin:/usr/local/cargo/bin:${PATH}"
ARG OPENVIKING_VERSION=0.0.0
ARG TARGETPLATFORM
ARG UV_LOCK_STRATEGY=auto
Expand All @@ -42,7 +37,6 @@ COPY crates/ crates/
COPY openviking/ openviking/
COPY openviking_cli/ openviking_cli/
COPY src/ src/
COPY third_party/ third_party/

# Install project and dependencies (triggers setup.py artifact builds + build_extension).
# Default to auto-refreshing uv.lock inside the ephemeral build context when it is
Expand All @@ -65,9 +59,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,id=uv-${TARGETPLATFORM} \
;; \
esac

# Build ragfs-python (Rust AGFS binding) and extract the native extension
# into the installed openviking package so it ships alongside the Go binding.
# Selection at runtime via RAGFS_IMPL env var (auto/rust/go).
# Build ragfs-python (Rust RAGFS binding) and extract the native extension
# into the installed openviking package.
RUN --mount=type=cache,target=/root/.cache/uv,id=uv-${TARGETPLATFORM} \
uv pip install maturin && \
export _TMPDIR=$(mktemp -d) && \
Expand Down Expand Up @@ -103,7 +96,7 @@ print("WARNING: No ragfs_python .so/.pyd in wheel")
sys.exit(1)
PY

# Stage 4: runtime
# Stage 3: runtime
FROM python:3.13-slim-trixie

RUN apt-get update && apt-get install -y --no-install-recommends \
Expand Down
3 changes: 0 additions & 3 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ graft third_party/leveldb-1.23
graft third_party/spdlog-1.14.1
graft third_party/croaring
graft third_party/rapidjson
recursive-include third_party/agfs/agfs-server *.go go.mod go.sum Makefile
recursive-include third_party/agfs/agfs-sdk/go *.go go.mod
include third_party/agfs/bin/agfs-server
include LICENSE
include README.md
include pyproject.toml
Expand Down
17 changes: 4 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@
# Variables
PYTHON ?= python3
SETUP_PY := setup.py
AGFS_SERVER_DIR := third_party/agfs/agfs-server
OV_CLI_DIR := crates/ov_cli

# Dependency Versions
MIN_PYTHON_VERSION := 3.10
MIN_GO_VERSION := 1.22
MIN_CMAKE_VERSION := 3.12
MIN_RUST_VERSION := 1.88
MIN_GCC_VERSION := 9
Expand All @@ -21,7 +19,6 @@ CLEAN_DIRS := \
*.egg-info/ \
openviking/bin/ \
openviking/lib/ \
$(AGFS_SERVER_DIR)/build/ \
$(OV_CLI_DIR)/target/ \
src/cmake_build/ \
.pytest_cache/ \
Expand All @@ -35,9 +32,9 @@ all: build

help:
@echo "Available targets:"
@echo " build - Build AGFS, ov CLI, and C++ extensions using setup.py"
@echo " build - Build ragfs-python and C++ extensions using setup.py"
@echo " clean - Remove build artifacts and temporary files"
@echo " check-deps - Check if required dependencies (Go, Rust, CMake, etc.) are installed"
@echo " check-deps - Check if required dependencies (Rust, CMake, etc.) are installed"
@echo " help - Show this help message"

check-pip:
Expand All @@ -59,11 +56,6 @@ check-deps:
@# Python check
@$(PYTHON) -c "import sys; v=sys.version_info; exit(0 if v.major > 3 or (v.major == 3 and v.minor >= 10) else 1)" || (echo "Error: Python >= $(MIN_PYTHON_VERSION) is required."; exit 1)
@echo " [OK] Python $$( $(PYTHON) -V | cut -d' ' -f2 )"
@# Go check
@command -v go > /dev/null 2>&1 || (echo "Error: Go is not installed."; exit 1)
@GO_VER=$$(go version | awk '{print $$3}' | sed 's/go//'); \
$(PYTHON) -c "v='$$GO_VER'.split('.'); exit(0 if int(v[0]) > 1 or (int(v[0]) == 1 and int(v[1]) >= 22) else 1)" || (echo "Error: Go >= $(MIN_GO_VERSION) is required. Found $$GO_VER"; exit 1); \
echo " [OK] Go $$GO_VER"
@# CMake check
@command -v cmake > /dev/null 2>&1 || (echo "Error: CMake is not installed."; exit 1)
@CMAKE_VER=$$(cmake --version | head -n1 | awk '{print $$3}'); \
Expand Down Expand Up @@ -99,7 +91,7 @@ build: check-deps check-pip
echo " [OK] pip found, use pip to install..."; \
$(PYTHON) -m pip install -e .; \
fi
@echo "Building ragfs-python (Rust AGFS binding) into openviking/lib/..."
@echo "Building ragfs-python (Rust RAGFS binding) into openviking/lib/..."
@MATURIN_CMD=""; \
if command -v maturin > /dev/null 2>&1; then \
MATURIN_CMD=maturin; \
Expand Down Expand Up @@ -131,7 +123,6 @@ build: check-deps check-pip
else \
echo " [SKIP] maturin not found, ragfs-python (Rust binding) will not be built."; \
echo " Install maturin to enable: uv pip install maturin"; \
echo " The Go binding will be used as fallback."; \
fi
@echo "Build completed successfully."

Expand All @@ -145,4 +136,4 @@ clean:
done
@find . -name "*.pyc" -delete
@find . -name "__pycache__" -type d -exec rm -rf {} +
@echo "Cleanup completed."
@echo "Cleanup completed."
71 changes: 47 additions & 24 deletions crates/ragfs/src/plugins/s3fs/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ pub struct S3Client {
bucket: String,
prefix: String,
marker_mode: DirectoryMarkerMode,
disable_batch_delete: bool,
}

impl S3Client {
Expand Down Expand Up @@ -125,6 +126,11 @@ impl S3Client {
.map(|s| DirectoryMarkerMode::from_str(s))
.unwrap_or(DirectoryMarkerMode::Empty);

let disable_batch_delete = config
.get("disable_batch_delete")
.and_then(|v| v.as_bool())
.unwrap_or(false);

// Build S3 config
let mut s3_config_builder = aws_sdk_s3::Config::builder()
.behavior_version(BehaviorVersion::latest())
Expand All @@ -150,6 +156,7 @@ impl S3Client {
bucket,
prefix,
marker_mode,
disable_batch_delete,
})
}

Expand Down Expand Up @@ -258,35 +265,51 @@ impl S3Client {
}

/// Batch delete objects (up to 1000 per call)
/// If disable_batch_delete is true, use sequential single-object deletes
/// for S3-compatible services (e.g., Alibaba Cloud OSS) that require
/// Content-MD5 for DeleteObjects but AWS SDK v2 does not send it by default.
pub async fn delete_objects(&self, keys: &[String]) -> Result<()> {
if keys.is_empty() {
return Ok(());
}

// S3 batch delete limit is 1000
for chunk in keys.chunks(1000) {
let objects: Vec<_> = chunk
.iter()
.map(|k| {
aws_sdk_s3::types::ObjectIdentifier::builder()
.key(k.as_str())
.build()
.unwrap()
})
.collect();

let delete = aws_sdk_s3::types::Delete::builder()
.set_objects(Some(objects))
.build()
.map_err(|e| Error::internal(format!("S3 build delete: {}", e)))?;

self.client
.delete_objects()
.bucket(&self.bucket)
.delete(delete)
.send()
.await
.map_err(|e| Error::internal(format!("S3 DeleteObjects error: {}", e)))?;
if self.disable_batch_delete {
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Suggestion] (non-blocking) Please add a regression test that exercises this branch end-to-end. #1333 existed because the old stack silently missed forwarding this flag once already, and this PR rewrites the whole Go path away. Without a test that proves disable_batch_delete=true reaches the sequential-delete path, this is easy to regress again in a future refactor.

// Sequential single-object delete
for key in keys {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key.as_str())
.send()
.await
.map_err(|e| Error::internal(format!("S3 DeleteObject error: {}", e)))?;
}
} else {
// S3 batch delete limit is 1000
for chunk in keys.chunks(1000) {
let objects: Vec<_> = chunk
.iter()
.map(|k| {
aws_sdk_s3::types::ObjectIdentifier::builder()
.key(k.as_str())
.build()
.unwrap()
})
.collect();

let delete = aws_sdk_s3::types::Delete::builder()
.set_objects(Some(objects))
.build()
.map_err(|e| Error::internal(format!("S3 build delete: {}", e)))?;

self.client
.delete_objects()
.bucket(&self.bucket)
.delete(delete)
.send()
.await
.map_err(|e| Error::internal(format!("S3 DeleteObjects error: {}", e)))?;
}
}

Ok(())
Expand Down
19 changes: 19 additions & 0 deletions crates/ragfs/src/plugins/s3fs/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,12 @@ impl S3FSPlugin {
"empty",
"Directory marker mode: none, empty, nonempty",
),
ConfigParameter::optional(
"disable_batch_delete",
"bool",
"false",
"Disable batch delete (DeleteObjects) for S3-compatible services like OSS",
),
ConfigParameter::optional(
"cache_enabled",
"bool",
Expand Down Expand Up @@ -636,6 +642,19 @@ plugins:
directory_marker_mode: nonempty
```

### Alibaba Cloud OSS
```yaml
plugins:
s3fs:
enabled: true
path: /s3
config:
bucket: my-oss-bucket
region: cn-beijing
endpoint: http://s3.oss-cn-beijing.aliyuncs.com
disable_batch_delete: true
```

## Directory Marker Modes

- `empty` (default): Zero-byte marker objects for directories
Expand Down
3 changes: 2 additions & 1 deletion examples/ov.conf.example
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@
"secret_key": null,
"endpoint": null,
"prefix": "",
"use_ssl": true
"use_ssl": true,
"disable_batch_delete": false
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Bug] (blocking) This example now advertises disable_batch_delete, but the surrounding agfs block still contains port, log_level, and retry_times above. After the AGFSConfig schema change in this PR those fields are invalid, so copying this example into ov.conf will fail before the new S3 option can even be used.

}
}
},
Expand Down
Loading
Loading