feat(filter,cli,server): sandbox CLI Lua execution and inline scripts on publish (#194)

## Summary

- **Extract `tokf-filter` crate** with the filter engine and sandboxed
Lua execution, shared between CLI and server
- **Sandbox all Lua execution** — remove unsandboxed `run_lua_script()`;
all paths now use `run_lua_script_sandboxed()` with instruction-count
(1M) and memory (16MB) limits
- **Unify `apply()`/`apply_sandboxed()`** into shared
`apply_internal()`, eliminating ~96% code duplication
- **Auto-inline external Lua scripts on publish** — `tokf publish` reads
`lua_script.file`, embeds as inline `source`, with path traversal
protection via `canonicalize()` + `starts_with()`
- **Server-side test verification** — filters are verified against their
test suites before persisting to storage (both publish and test update
endpoints)
- **Fail-fast server validation** — rejects `lua_script.file` before
hashing/upload with a helpful hint
- **Documentation updates** — new `publishing-filters.md`, updated Lua
escape hatch docs with sandbox limits, updated writing-filters with
`[lua_script]` reference
- **File size compliance** — split `publish.rs` and `update_tests.rs`
into directory modules (tests extracted to sibling files)

## Test plan

- [x] All 1152 workspace tests pass (`cargo test --workspace`)
- [x] Clippy clean (`cargo clippy --workspace --all-targets -- -D
warnings`)
- [x] File size limits pass (all files under 700 lines)
- [ ] DB integration tests (`just test-db` — requires CockroachDB)
- [ ] End-to-end tests (`just test-e2e` — requires CockroachDB)
- [ ] Manual test: `tokf publish` with `lua_script.file` auto-inlines
correctly
- [ ] Manual test: path traversal (`lua_script.file =
"../../etc/passwd"`) is rejected

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: mpecan

.dupes-ignore.toml

@@ -117,3 +117,11 @@ reason = "TestHarness::blocking_update_tests/blocking_search_filters - spawn_blo
 [[ignore]]
 fingerprint = "9fc57c07628e198e"
 reason = "TestHarness::blocking_gain/blocking_list_machines - spawn_blocking test boilerplate; test clarity > DRY"
+
+[[ignore]]
+fingerprint = "ccbacfce8575cd81"
+reason = "publish_filter_rejects_missing_* tests - same post+assert+check-error pattern with different payloads and error messages; test clarity > DRY"
+
+[[ignore]]
+fingerprint = "91e815e8a2c7146b"
+reason = "publish_filter_rejects_oversized/failing tests - same post+assert+check-error pattern with different inputs and assertions; test clarity > DRY"

.github/workflows/deploy-server.yml

@@ -6,6 +6,7 @@ on:
     paths:
       - "crates/tokf-server/**"
       - "crates/tokf-common/**"
+      - "crates/tokf-filter/**"
       - "Cargo.toml"
       - "Cargo.lock"
       - "Dockerfile"

CONTRIBUTING.md

@@ -108,9 +108,16 @@ Every filter in the stdlib **must** have a `_test/` suite — CI enforces this w
 
 ## Lua filters
 
-For filters that need logic beyond what TOML can express, use the `[lua_script]` section with [Luau](https://luau.org/). The sandbox blocks `io`, `os`, and `package` — scripts cannot access the filesystem or network.
+For filters that need logic beyond what TOML can express, use the `[lua_script]` section with [Luau](https://luau.org/).
 
-See the [README](README.md#lua-escape-hatch) for the full API and the built-in filter library for examples.
+All Lua execution is sandboxed:
+
+- **Blocked libraries:** `io`, `os`, `package` — no filesystem or network access.
+- **Resource limits:** 1 million VM instructions, 16 MB memory (prevents infinite loops and memory exhaustion).
+
+For local development, you can reference external scripts with `lua_script.file = "script.luau"`. For published filters, use inline `source` — `tokf publish` automatically inlines file references before uploading.
+
+See `docs/lua-escape-hatch.md` for the full API, globals, and examples.
 
 ---
 

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace]
-members = ["crates/tokf-cli", "crates/tokf-common", "crates/tokf-server", "crates/crdb-test-macro", "crates/e2e-tests"]
+members = ["crates/tokf-cli", "crates/tokf-common", "crates/tokf-filter", "crates/tokf-server", "crates/crdb-test-macro", "crates/e2e-tests"]
 resolver = "2"
 
 [workspace.package]

Cargo.toml

@@ -1,6 +1,8 @@
 # syntax=docker/dockerfile:1
 # ── Stage 1: build ──────────────────────────────────────────────────────────
 FROM rust:slim AS builder
+# g++ is required for mlua's vendored Luau build (C++ source compiled via cc crate)
+RUN apt-get update && apt-get install -y --no-install-recommends g++ && rm -rf /var/lib/apt/lists/*
 WORKDIR /app
 COPY Cargo.toml Cargo.lock ./
 COPY crates/ crates/

Dockerfile

@@ -298,6 +298,12 @@ collapse_empty_lines = true   # collapse consecutive blank lines into one
 
 show_history_hint = true      # append a hint line pointing to the full output in history
 
+# Lua escape hatch — for logic TOML can't express (see Lua Escape Hatch section)
+[lua_script]
+lang = "luau"
+source = 'return output:upper()'    # inline script
+# file = "transform.luau"           # or reference a local file (auto-inlined on publish)
+
 match_output = [              # whole-output substring checks, short-circuit the pipeline
   { contains = "rejected", output = "push rejected" },
 ]
@@ -453,7 +459,28 @@ end
 
 Available globals: `output` (string), `exit_code` (integer — the underlying command's real exit code, unaffected by `--no-mask-exit-code`), `args` (table).
 Return a string to replace output, or `nil` to fall through to the rest of the TOML pipeline.
-The sandbox blocks `io`, `os`, and `package` — no filesystem or network access from scripts.
+
+### Sandbox
+
+All Lua execution is sandboxed — both in the CLI and on the server:
+
+- **Blocked libraries:** `io`, `os`, `package` — no filesystem or network access.
+- **Instruction limit:** 1 million VM instructions (prevents infinite loops).
+- **Memory limit:** 16 MB (prevents memory exhaustion).
+
+Scripts that exceed these limits are terminated and treated as a passthrough (the TOML pipeline continues as if no Lua script was configured).
+
+### External script files
+
+For local development you can keep the script in a separate `.luau` file:
+
+```toml
+[lua_script]
+lang = "luau"
+file = "transform.luau"
+```
+
+Only one of `file` or `source` may be set — not both. When you run `tokf publish`, file references are automatically inlined (the file content is embedded as `source`) so the published filter is self-contained. The script file must reside within the filter's directory — path traversal (e.g. `../secret.txt`) is rejected.
 
 ---
 
@@ -887,9 +914,56 @@ tokf publish --update-tests git/push --dry-run  # preview only
 
 ---
 
+---
+
+
 ## Publishing a Filter
 
-See [Publishing Filters](./publishing-filters.md) for how to share your own filters.
+```sh
+tokf publish <filter-name>
+```
+
+Publishes a local filter to the community registry under the MIT license. Authentication is required — run `tokf auth login` first.
+
+### Requirements
+
+- The filter must be a **user-level or project-local** filter (not a built-in). Use `tokf eject` first if needed.
+- At least one **test file** must exist in the adjacent `_test/` directory. The server runs these tests against your filter before accepting the upload.
+- You must accept the **MIT license** (prompted on first publish, remembered afterwards).
+
+### What happens on publish
+
+1. The filter TOML is read and validated.
+2. If the filter uses `lua_script.file`, the referenced script is **automatically inlined** — its content is embedded as `lua_script.source` so the published filter is self-contained. The script file must reside within the filter's directory (path traversal is rejected).
+3. A content hash is computed from the parsed config. This hash is the filter's permanent identity.
+4. The filter and test files are uploaded. The server verifies tests pass before accepting.
+5. On success, the registry URL is printed.
+
+### Options
+
+| Flag | Description |
+|------|-------------|
+| `--dry-run` | Preview what would be published without uploading |
+| `--update-tests` | Replace the test suite for an already-published filter |
+
+### Examples
+
+```sh
+tokf publish git/push                  # publish a filter
+tokf publish git/push --dry-run        # preview only
+tokf publish --update-tests git/push   # replace test suite
+```
+
+### Size limits
+
+- Filter TOML: 64 KB max
+- Total upload (filter + tests): 1 MB max
+
+### Lua scripts in published filters
+
+Published filters must use **inline `source`** for Lua scripts — `lua_script.file` is not supported on the server. The `tokf publish` command handles this automatically by reading the file and embedding its content. You don't need to change your filter.
+
+All Lua scripts in published filters are executed in a sandbox with resource limits (1 million instructions, 16 MB memory) during server-side test verification.
 
 ---
 

README.md

@@ -11,16 +11,26 @@ mod harness;
 const FILTER_TOML: &[u8] = b"command = \"git push\"\n";
 
 fn valid_test(name: &str) -> (String, Vec<u8>) {
-    let content = format!("name = \"{name}\"\n\n[[expect]]\ncontains = \"ok\"\n");
+    let content =
+        format!("name = \"{name}\"\ninline = \"ok output\"\n\n[[expect]]\ncontains = \"ok\"\n");
     (format!("{name}.toml"), content.into_bytes())
 }
 
+fn default_test() -> (String, Vec<u8>) {
+    (
+        "default.toml".to_string(),
+        b"name = \"default\"\ninline = \"\"\n\n[[expect]]\nequals = \"\"\n".to_vec(),
+    )
+}
+
 /// Publish a filter → verify is_new=true, hash returned.
 #[crdb_test_macro::crdb_test(migrations = "../tokf-server/migrations")]
 async fn publish_filter_returns_hash(pool: PgPool) {
     let h = harness::TestHarness::with_storage(pool).await;
 
-    let (is_new, resp) = h.blocking_publish(FILTER_TOML.to_vec(), vec![]).await;
+    let (is_new, resp) = h
+        .blocking_publish(FILTER_TOML.to_vec(), vec![default_test()])
+        .await;
 
     assert!(is_new, "expected is_new=true for first publish");
     assert!(!resp.content_hash.is_empty());
@@ -44,10 +54,14 @@ async fn publish_with_invalid_test_fails(pool: PgPool) {
 async fn duplicate_publish_returns_existing(pool: PgPool) {
     let h = harness::TestHarness::with_storage(pool).await;
 
-    let (is_new1, resp1) = h.blocking_publish(FILTER_TOML.to_vec(), vec![]).await;
+    let (is_new1, resp1) = h
+        .blocking_publish(FILTER_TOML.to_vec(), vec![default_test()])
+        .await;
     assert!(is_new1);
 
-    let (is_new2, resp2) = h.blocking_publish(FILTER_TOML.to_vec(), vec![]).await;
+    let (is_new2, resp2) = h
+        .blocking_publish(FILTER_TOML.to_vec(), vec![default_test()])
+        .await;
     assert!(!is_new2, "expected is_new=false for duplicate publish");
     assert_eq!(resp1.content_hash, resp2.content_hash);
 }

crates/e2e-tests/tests/publish_flow.rs

@@ -15,6 +15,7 @@ path = "src/main.rs"
 
 [dependencies]
 tokf-common = { path = "../tokf-common", version = "0.2.12", features = ["validation"] }
+tokf-filter = { path = "../tokf-filter", version = "0.2.12" }
 clap = { version = "4", features = ["derive", "env"] }
 toml = "1.0"
 serde = { version = "1", features = ["derive"] }
@@ -28,7 +29,6 @@ include_dir = { version = "0.7", features = ["glob"] }
 # with sqlx-sqlite's links = "sqlite3" constraint in the workspace resolver.
 rusqlite = { version = "0.32", features = ["bundled"] }
 rkyv = { version = "0.8", features = ["bytecheck", "unaligned"] }
-mlua = { version = "0.11.6", features = ["luau", "vendored", "error-send"] }
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "blocking", "multipart"] }
 # linux-native uses kernel keyutils (no libdbus-sys dependency).
 # sync-secret-service would need libdbus-1-dev on Linux CI/build hosts.