diff --git a/.env.example b/.env.example index 873ec2e3..52553c01 100644 --- a/.env.example +++ b/.env.example @@ -28,7 +28,7 @@ # CORTEX_TRUSTED_GATEWAY_NO_AUTH=false # Grant the static bearer token cortex:admin in addition to cortex:read -# (enables ack_error / unack_error / notifications_test). +# (enables ack_error / unack_error / file_tails / notifications_test). # CORTEX_STATIC_TOKEN_ADMIN=false # Auth mode: "bearer" (default, static token only) or "oauth" (Google OAuth + JWT). @@ -284,6 +284,26 @@ # Env file the Compose service loads (optional). # CORTEX_ENV_FILE=.env +# --- Managed file-tail ingest --- +# +# Sources are stored in /file-tails.json, where is the +# parent directory of CORTEX_DB_PATH. Manage them at runtime: +# +# cortex file-tail add --id swag-access \ +# --path /file-tail-root/swag/log/nginx/access.log \ +# --tag swag-access --hostname squirts --facility local4 +# cortex file-tail list --json +# +# The default starts at EOF; add --from-start only for intentional backfills. +# REST /api/file-tails needs the normal API bearer plus X-Cortex-Admin-Token. +# CORTEX_API_ADMIN_TOKEN=change-me-admin-token +# Host directory mounted read-only at /file-tail-root by Compose. +# CORTEX_FILE_TAIL_LOG_VOLUME=/var/log +# Dedicated container roots file-tail may read from. Keep the default narrow; +# opt into broader read-only roots explicitly after mounting them. +# CORTEX_FILE_TAIL_ALLOWED_ROOTS=/file-tail-root +# CORTEX_FILE_TAIL_ALLOWED_ROOTS=/file-tail-root,/var/log,/logs + # Shared cortex home (inventory cache, setup env) mounted at /cortex-home. # CORTEX_HOME_VOLUME=~/.cortex diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a89e596..b506c1ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.20.1] - 2026-06-12 + +### Changed + +- Require an explicit `hostname` when adding managed file-tail sources so rows are attributed to the intended host instead of falling back to the Cortex container identity. +- Report file-tail durable-writer backpressure in the `status` action and mark overall status as `degraded` when tailers are blocked on writer ack. + +### Fixed + +- Prevent duplicate file-tail `add` calls from resetting checkpoints, avoid retrying stateful HTTP file-tail mutations after 503 responses, and make committed-but-reconcile-failed mutations explicit. +- Harden file-tail resume and rotation handling: stale checkpoints now restart replacement files from the beginning, same-inode copytruncate/regrow is detected by prefix fingerprint, and rename-create rotation gets a short old-file drain window before switching. +- Fix file-tail live smoke harness permissions so non-root containers can read the mounted smoke log, and tighten admin-scope predicates across smoke scripts. +- Canonicalize configured file-tail allowed roots so symlinked operational roots validate against canonical file paths. + +## [1.20.0] - 2026-06-11 + +### Added + +- Added managed file-tail ingest sources with CLI, REST API, and MCP control. +- Added `file-tail` source kind for rows ingested from local log files. +- Documented SWAG, fail2ban, Authelia, and AdGuard file-tail recipes for replacing rsyslog `imfile` drop-ins. + ## [1.19.0] - 2026-06-11 ### Added @@ -2268,7 +2290,8 @@ start and verify with `cortex --http db status`. --- -[Unreleased]: https://github.com/jmagar/cortex/compare/v1.14.0...HEAD +[Unreleased]: https://github.com/jmagar/cortex/compare/v1.20.0...HEAD +[1.20.0]: https://github.com/jmagar/cortex/compare/v1.19.0...v1.20.0 [1.14.0]: https://github.com/jmagar/cortex/compare/v1.13.3...v1.14.0 [1.13.3]: https://github.com/jmagar/cortex/compare/v1.13.2...v1.13.3 [1.13.2]: https://github.com/jmagar/cortex/compare/v1.13.1...v1.13.2 diff --git a/CLAUDE.md b/CLAUDE.md index 953ed222..938281de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,9 +63,9 @@ Tests: unit tests live in sidecar files beside their source modules (e.g. `src/d ## MCP Tools -One MCP tool: **`cortex`** — dispatches by `action` argument. 45 actions, generated from `ACTION_SPECS` in `src/mcp/actions.rs` (the single authoritative registry — regenerate this table from there). +One MCP tool: **`cortex`** — dispatches by `action` argument. 46 actions, generated from `ACTION_SPECS` in `src/mcp/actions.rs` (the single authoritative registry — regenerate this table from there). -Scope taxonomy: every action requires `cortex:read` except the three **admin** actions `ack_error`, `unack_error`, and `notifications_test`, which require `cortex:admin` (static bearer tokens get read-only unless `CORTEX_STATIC_TOKEN_ADMIN=true`); `help` is info-only (no scope gate). +Scope taxonomy: every action requires `cortex:read` except the four **admin** actions `ack_error`, `unack_error`, `file_tails`, and `notifications_test`, which require `cortex:admin` (static bearer tokens get read-only unless `CORTEX_STATIC_TOKEN_ADMIN=true`); `help` is info-only (no scope gate). | Action | Description | |--------|-------------| @@ -110,6 +110,7 @@ Scope taxonomy: every action requires `cortex:read` except the three **admin** a | `ask_history` | Query AI transcript history | | `incident_context` | Full context for an incident | | `graph` | Resolve graph entities, neighborhoods, and evidence-backed explanations | +| `file_tails` | **(admin)** Manage Cortex-owned file-tail ingest sources | | `ack_error` | **(admin)** Acknowledge an error signature | | `unack_error` | **(admin)** Revoke an error signature acknowledgement | | `notifications_test` | **(admin)** Send a test notification via Apprise | @@ -164,6 +165,10 @@ CORTEX_GOOGLE_CLIENT_SECRET=... # required when CORTEX_AUTH_MODE=oauth # Non-MCP REST API (always on; gated by its token) CORTEX_API_TOKEN=your-api-token # REQUIRED at startup — /api/* is always mounted +# Managed file-tail sources +# Stored in the parent directory of CORTEX_DB_PATH as file-tails.json. +# Manage with: cortex file-tail list|status|add|remove|enable|disable + # Docker container log ingestion (disabled by default) CORTEX_DOCKER_INGEST_ENABLED=false # set true to ingest from docker-socket-proxy hosts CORTEX_DOCKER_HOSTS=host-a,host-b # comma-separated hostnames → http://:2375 @@ -182,7 +187,7 @@ RUST_LOG=info | `docker-compose.yml` | Production deployment (ports 1514, 3100) | | `docs/SETUP.md` | Setup guide (clone, build, configure, deploy, verify); per-host forwarder configs (rsyslog, UniFi, ATT router, WSL) live in README "Syslog Forwarder Setup" | | `src/db/queries.rs` | All SQL queries and FTS5 search implementation | -| `src/mcp/actions.rs` | `ACTION_SPECS` — authoritative registry of all 45 MCP actions and their scopes | +| `src/mcp/actions.rs` | `ACTION_SPECS` — authoritative registry of all 46 MCP actions and their scopes | | `src/mcp/tools.rs` | Single `cortex` tool with action dispatch | | `config/mcporter.json` | mcporter config (HTTP transport to localhost:3100) | | `config/systemd/` | `cortex-backup.service` / `.timer` — daily WAL-safe backup units | diff --git a/Cargo.lock b/Cargo.lock index b0f769e6..cbd110de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -196,12 +196,6 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.0" @@ -279,12 +273,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -371,9 +359,9 @@ version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "crossterm 0.29.0", + "crossterm", "unicode-segmentation", - "unicode-width 0.2.2", + "unicode-width", ] [[package]] @@ -423,7 +411,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cortex" -version = "1.19.0" +version = "1.20.1" dependencies = [ "anyhow", "axum", @@ -436,7 +424,6 @@ dependencies = [ "futures-util", "getrandom 0.4.2", "hyper-util", - "inquire", "lab-auth", "libc", "lru", @@ -508,29 +495,13 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossterm" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" -dependencies = [ - "bitflags 1.3.2", - "crossterm_winapi", - "libc", - "mio 0.8.11", - "parking_lot", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.13.0", + "bitflags", "crossterm_winapi", "document-features", "parking_lot", @@ -1055,24 +1026,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fuzzy-matcher" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" -dependencies = [ - "thread_local", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.9" @@ -1544,7 +1497,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "533e68a5842e734946fe159fb03fc9bbbb254f590dd0d8ad321ae5ff7beca2c1" dependencies = [ - "bitflags 2.13.0", + "bitflags", "inotify-sys", "libc", ] @@ -1558,23 +1511,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inquire" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fddf93031af70e75410a2511ec04d49e758ed2f26dad3404a934e0fb45cc12a" -dependencies = [ - "bitflags 2.13.0", - "crossterm 0.25.0", - "dyn-clone", - "fuzzy-matcher", - "fxhash", - "newline-converter", - "once_cell", - "unicode-segmentation", - "unicode-width 0.1.14", -] - [[package]] name = "ipnet" version = "2.12.0" @@ -1705,7 +1641,7 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" dependencies = [ - "bitflags 2.13.0", + "bitflags", "libc", ] @@ -1872,18 +1808,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "mio" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.48.0", -] - [[package]] name = "mio" version = "1.2.1" @@ -1896,22 +1820,13 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "newline-converter" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b6b097ecb1cbfed438542d16e84fd7ad9b0c76c8a65b7f9039212a3d14dc7f" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "nix" version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.13.0", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -1933,13 +1848,13 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.13.0", + "bitflags", "fsevent-sys", "inotify", "kqueue", "libc", "log", - "mio 1.2.1", + "mio", "notify-types", "walkdir", "windows-sys 0.60.2", @@ -1951,7 +1866,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42b8cfee0e339a0337359f3c88165702ac6e600dc01c0cc9579a92d62b08477a" dependencies = [ - "bitflags 2.13.0", + "bitflags", ] [[package]] @@ -2512,7 +2427,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags", ] [[package]] @@ -2728,7 +2643,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ - "bitflags 2.13.0", + "bitflags", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -2758,7 +2673,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2943,7 +2858,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -3177,27 +3092,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" -[[package]] -name = "signal-hook" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-mio" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" -dependencies = [ - "libc", - "mio 0.8.11", - "signal-hook", -] - [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -3506,7 +3400,7 @@ checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", - "mio 1.2.1", + "mio", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3668,7 +3562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.13.0", + "bitflags", "bytes", "futures-core", "futures-util", @@ -3826,12 +3720,6 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - [[package]] name = "unicode-width" version = "0.2.2" @@ -4038,7 +3926,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.13.0", + "bitflags", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -4243,15 +4131,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - [[package]] name = "windows-sys" version = "0.52.0" @@ -4279,21 +4158,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - [[package]] name = "windows-targets" version = "0.52.6" @@ -4336,12 +4200,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -4354,12 +4212,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -4372,12 +4224,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -4402,12 +4248,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -4420,12 +4260,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -4438,12 +4272,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -4456,12 +4284,6 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -4570,7 +4392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.13.0", + "bitflags", "indexmap 2.14.0", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index bb7b6be4..d290a3a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cortex" -version = "1.19.0" +version = "1.20.1" edition = "2024" rust-version = "1.86" license = "MIT" @@ -94,9 +94,6 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } comfy-table = "7" supports-hyperlinks = "3" -# Interactive TUI prompts (agent deploy host selector) -inquire = "0.7" - # Utils anyhow = "1" lru = "0.16" diff --git a/README.md b/README.md index 7eda6a06..c7830628 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ Structured filter-only retrieval for correlation workflows. This action rejects Common filters match `search`: `hostname`, `source_ip`, `severity`, `app_name`, `facility`, `exclude_facility`, `process_id`, `from`, `to`, `received_from`, `received_to`, and `limit`. Correlation aliases include `source_kind` (`docker-stream`, `docker-event`, `agent-command`, `shell-history`, `transcript`, `claude`, `codex`, `gemini`), plus `tool`, `project`, `session_id`, `container`, `docker_host`, `stream`, and `event_action`. +`source_kind=file-tail` filters managed file-tail rows (`source_ip` prefix `file-tail://`). --- @@ -769,6 +770,45 @@ The docker-socket-proxy side only needs read access to containers, events, ping, Docker ingest is intentionally not part of the default smoke test because it needs a live docker-socket-proxy-compatible endpoint and container log stream. For integration testing, run cortex with `CORTEX_DOCKER_INGEST_ENABLED=true` against a disposable docker-socket-proxy or mocked Docker HTTP fixture, emit a unique line from a short-lived container, then verify it with `cortex search` or `mcporter call ... action=search`. Container stdout/stderr rows use `source_ip=docker:////`. Container lifecycle rows for actions such as `create`, `start`, `restart`, `die`, `stop`, `destroy`, `rename`, `oom`, and `health_status:*` use `source_ip=docker-event:////`, `facility=docker`, and preserve the raw Docker event JSON. +#### Managed file-tail ingest + +Cortex can tail local files directly without rsyslog `imfile` drop-ins. In +Docker, mount the host log tree read-only at `/file-tail-root` with +`CORTEX_FILE_TAIL_LOG_VOLUME` and register paths inside that mount. Sources are +stored next to the SQLite database in `file-tails.json`, managed through +`cortex file-tail ...`, REST `POST /api/file-tails` (requires +`Authorization: Bearer $CORTEX_API_TOKEN` plus +`X-Cortex-Admin-Token: $CORTEX_API_ADMIN_TOKEN`), or MCP action `file_tails`, +and emitted as `source_kind="file-tail"` rows. Row metadata includes +`file_tail_id`, `tag`, and `path_basename`, not the full filesystem path. +The documented safe default is to keep managed tails inside `/file-tail-root`. +Set `CORTEX_FILE_TAIL_ALLOWED_ROOTS` explicitly only when an operator has +mounted and reviewed broader read-only roots such as `/var/log` or `/logs`. + +```bash +cortex file-tail add --id swag-access \ + --path /file-tail-root/swag/log/nginx/access.log \ + --tag swag-access --hostname squirts --facility local4 +cortex file-tail add --id swag-error \ + --path /file-tail-root/swag/log/nginx/error.log \ + --tag swag-error --hostname squirts --facility local4 --severity warning +cortex file-tail add --id fail2ban \ + --path /file-tail-root/swag/log/fail2ban/fail2ban.log \ + --tag fail2ban --hostname squirts --facility local5 +cortex file-tail add --id authelia \ + --path /file-tail-root/authelia/logs/authelia.log \ + --tag authelia --hostname squirts --facility local5 +cortex file-tail add --id adguard-query \ + --path /file-tail-root/adguard/var/data/querylog.json \ + --tag adguard-query --hostname squirts --facility local6 +``` + +The default starts at EOF. Add `--from-start` only when you intentionally want +to backfill the current file contents. After startup, Cortex checkpoints +`dev`/`inode`/offset in `file-tails.json`, resumes from that cursor, and +reopens files on rename/create rotation or truncation. Lines are bounded by +`CORTEX_MAX_MESSAGE_SIZE`; oversized records are truncated before enqueue. + #### Storage | Variable | Required | Default | Description | diff --git a/config.toml b/config.toml index 195d16a6..48ce0f51 100644 --- a/config.toml +++ b/config.toml @@ -82,6 +82,12 @@ enabled = false reconnect_initial_ms = 1000 reconnect_max_ms = 30000 +# Managed file-tail sources are runtime-managed in data/file-tails.json +# (parent directory of [storage].db_path). Configure with: +# cortex file-tail add --id swag-access --path /file-tail-root/path/access.log --tag swag-access +# In Docker, mount host log directories read-only at /file-tail-root with +# CORTEX_FILE_TAIL_LOG_VOLUME. + [notifications] enabled = true apprise_url = "http://100.120.242.29:8766" diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b0d390b0..3a1cd3bd 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -23,7 +23,7 @@ services: # Default tag is kept in sync by scripts/bump-version.sh (version canon); # previously this was frozen at 1.0.0 while migrations moved forward — # a stale binary against a newer schema (full-review OH1). - image: ghcr.io/jmagar/cortex:${CORTEX_VERSION:-1.19.0} + image: ghcr.io/jmagar/cortex:${CORTEX_VERSION:-1.20.1} container_name: cortex user: "${CORTEX_UID:-1000}:${CORTEX_GID:-1000}" env_file: @@ -49,6 +49,10 @@ services: - ${CORTEX_SSH_VOLUME:-${HOME}/.cortex/ssh}:/home/cortex/.ssh:ro - ${CORTEX_WORKSPACE_VOLUME:-${HOME}/workspace}:/home/cortex/workspace:ro - ${CORTEX_DATA_VOLUME:-cortex-data}:/data + # Managed file-tail ingest can only read paths mounted under this root. + # Point CORTEX_FILE_TAIL_LOG_VOLUME at /var/log, /mnt/user/appdata, or a + # narrower log directory. Keep it read-only. + - ${CORTEX_FILE_TAIL_LOG_VOLUME:-/var/log}:/file-tail-root:ro # The container listens on CORTEX_RECEIVER_PORT (default 1514). To serve host port # 514, set CORTEX_RECEIVER_HOST_PORT=514 while leaving CORTEX_RECEIVER_PORT at 1514, or add # host-level firewall/NAT redirects from 514 to 1514. diff --git a/docker-compose.yml b/docker-compose.yml index 32ac7e51..f48569d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,10 @@ services: - ${CORTEX_SSH_VOLUME:-${HOME}/.cortex/ssh}:/home/cortex/.ssh:ro - ${CORTEX_WORKSPACE_VOLUME:-${HOME}/workspace}:/home/cortex/workspace:ro - ${CORTEX_DATA_VOLUME:-cortex-data}:/data + # Managed file-tail ingest can only read paths mounted under this root. + # Point CORTEX_FILE_TAIL_LOG_VOLUME at /var/log, /mnt/user/appdata, or a + # narrower log directory. Keep it read-only. + - ${CORTEX_FILE_TAIL_LOG_VOLUME:-/var/log}:/file-tail-root:ro networks: - cortex restart: unless-stopped diff --git a/docs/CLI.md b/docs/CLI.md index 0a15047e..63803f9d 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -26,6 +26,28 @@ CORTEX_DB_PATH=/data/cortex.db `CORTEX_TOKEN` is not used by direct CLI mode because it is local database access, not HTTP access. +## `cortex file-tail` + +Manage Cortex-owned file-tail ingest sources. Sources are persisted beside the +configured database in `file-tails.json` and reconciled by the running +`cortex serve mcp` process. + +```bash +cortex file-tail list [--json] +cortex file-tail status [--json] +cortex file-tail add --id ID --path PATH --tag TAG --hostname HOST [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json] +cortex file-tail remove --id ID [--json] +cortex file-tail enable --id ID [--json] +cortex file-tail disable --id ID [--json] +``` + +The command maps to MCP action `file_tails` and REST `POST /api/file-tails`. +When `--http` or `CORTEX_USE_HTTP=true` is used, set both `CORTEX_API_TOKEN` +and `CORTEX_API_ADMIN_TOKEN`; the client sends the latter as +`X-Cortex-Admin-Token`. +By default `add` starts tailing at EOF; pass `--from-start` to ingest existing +file contents. + ## Output All commands print compact human-readable output by default. Add `--json` to diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 81e1719c..5d0a8ac6 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -116,6 +116,81 @@ POST=0 Docker ingest is not included in the default smoke test because it requires a live docker-socket-proxy-compatible endpoint. For integration coverage, run a disposable docker-socket-proxy or mocked Docker HTTP fixture, set `CORTEX_DOCKER_INGEST_ENABLED=true`, emit a unique container stdout/stderr line, and verify it with `search` or `tail`. Container stream rows identify their source as `docker:////`. Container lifecycle events such as `create`, `start`, `restart`, `die`, `stop`, `destroy`, `rename`, and `oom` identify their source as `docker-event:////` and use `facility=docker`. +## Managed File-Tail Sources + +Cortex can tail local log files directly and ingest appended lines through the +same writer/enrichment path as syslog, Docker, and OTLP. Sources are stored in +`/file-tails.json`, where `` is the parent directory of +`CORTEX_DB_PATH`. + +Use this for logs that do not naturally reach journald or container stdout, +such as SWAG nginx access/error logs, SWAG fail2ban logs, Authelia file logs, +and AdGuard query logs. + +In Docker, mount a host log tree read-only at `/file-tail-root`: + +```bash +CORTEX_FILE_TAIL_LOG_VOLUME=/mnt/user/appdata +``` + +Registered paths must be absolute, existing, non-symlink regular files under +`CORTEX_FILE_TAIL_ALLOWED_ROOTS`. The documented safe default is the dedicated +`/file-tail-root` mount. To opt into broader roots, mount those directories +read-only and set an explicit comma-separated allowlist, for example +`CORTEX_FILE_TAIL_ALLOWED_ROOTS=/file-tail-root,/var/log,/logs`. Sensitive +Cortex mounts such as `/data`, `/cortex-home`, `/home/cortex/.ssh`, and +`/home/cortex/workspace` are always rejected. REST management requires both the +normal API bearer and `X-Cortex-Admin-Token: $CORTEX_API_ADMIN_TOKEN`; MCP +management requires `cortex:admin`. + +```bash +cortex file-tail add \ + --id swag-access \ + --path /file-tail-root/swag/log/nginx/access.log \ + --tag swag-access \ + --hostname squirts \ + --facility local4 + +cortex file-tail add \ + --id swag-error \ + --path /file-tail-root/swag/log/nginx/error.log \ + --tag swag-error \ + --hostname squirts \ + --facility local4 \ + --severity warning + +cortex file-tail add \ + --id fail2ban \ + --path /file-tail-root/swag/log/fail2ban/fail2ban.log \ + --tag fail2ban \ + --hostname squirts \ + --facility local5 + +cortex file-tail add \ + --id authelia \ + --path /file-tail-root/authelia/logs/authelia.log \ + --tag authelia \ + --hostname squirts \ + --facility local5 + +cortex file-tail add \ + --id adguard-query \ + --path /file-tail-root/adguard/var/data/querylog.json \ + --tag adguard-query \ + --hostname squirts \ + --facility local6 +``` + +`--from-start` ingests existing file contents on first open. The default starts +at EOF so adding a source does not backfill a large historic log unexpectedly. +After a source is running, Cortex checkpoints `dev`/`inode`/offset in +`file-tails.json`, resumes from that cursor on restart, reopens on +rename/create rotation, seeks back to 0 after truncation, and bounds each line +by `CORTEX_MAX_MESSAGE_SIZE`. Per-row metadata stores `file_tail_id`, `tag`, +and `path_basename`; full paths are visible only through the admin management +surface. The runtime reconciles enabled sources periodically and after +CLI/REST/MCP mutations. + ### MCP server (`CORTEX_*`) | Variable | Required | Default | Sensitive | Description | @@ -133,6 +208,7 @@ The plain JSON API is **always on**: it is mounted under `/api/*` on the same HT | Variable | Required | Default | Sensitive | Description | | --- | --- | --- | --- | --- | | `CORTEX_API_TOKEN` | yes | (none) | **yes** | Bearer token for `/api/*` routes — required at startup | +| `CORTEX_API_ADMIN_TOKEN` | for REST file-tail management | (none) | **yes** | Extra token sent as `X-Cortex-Admin-Token` for `/api/file-tails` management. The normal API bearer is still required. | ### Headless Gemini assessment (`CORTEX_HEADLESS_*`, `CORTEX_LLM_*`) diff --git a/docs/INVENTORY.md b/docs/INVENTORY.md index 25c9a78e..ef48aa0f 100644 --- a/docs/INVENTORY.md +++ b/docs/INVENTORY.md @@ -60,10 +60,11 @@ that registry by `src/mcp/schemas.rs::tool_definitions()`. | `graph` | Resolve graph entities, neighborhoods, and evidence-backed explanations | no | | `ack_error` | Acknowledge an error signature | yes | | `unack_error` | Revoke an error acknowledgement | yes | +| `file_tails` | Manage Cortex-owned file-tail ingest sources | yes | | `notifications_test` | Send a test Apprise notification | yes | | `help` | Returns markdown documentation for all actions | no | -Most MCP actions are read-only. `ack_error`, `unack_error`, and +Most MCP actions are read-only. `ack_error`, `unack_error`, `file_tails`, and `notifications_test` require `cortex:admin`; they mutate acknowledgement/audit or notification state through service-owned actor and safety policy. diff --git a/docs/api.md b/docs/api.md index 13e5ae12..2b4ef5ad 100644 --- a/docs/api.md +++ b/docs/api.md @@ -15,7 +15,7 @@ ## Endpoint matrix -56 routes total. Scope is `read` (mounted via `axum::routing::get`, +57 routes total. Scope is `read` (mounted via `axum::routing::get`, hits read-side `db_permits`) or `admin` (POST + `MAINTENANCE_PERMIT` single-flight, audited via `tracing::warn!` before the service call). All responses are JSON; error bodies are `{"error": ""}` @@ -59,6 +59,12 @@ to them by default. | GET | `/api/ai/errors` | read | query: `limit?`. `deny_unknown_fields`. | service-shaped (list of recent transcript parse errors) | 200, 400, 401, 503, 500 | Y | Surfaces parse failures from the AI indexer. | | POST | `/api/ai/prune-checkpoints` | **admin** | body: `{ "dry_run": bool (REQUIRED), "missing_only"?: bool, "limit"?: u32 }`. `deny_unknown_fields`. | service-shaped (count of pruned/would-prune rows) | 200, 400, 401, **409**, 500 | **N** | Single-flight via `MAINTENANCE_PERMIT`; 409 on contention with `/api/db/vacuum` or `/api/db/checkpoint`. `dry_run` is **REQUIRED and explicit** — a missing key returns 400 (defends against `POST {}` mass-delete, eng-review C3). `caller_ip` audit-logged via `tracing::warn!` BEFORE the service call. | +### File-tail admin (1) + +| Method | Path | Scope | Request | Response (top-level) | Status codes | Idempotent | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| POST | `/api/file-tails` | **admin** | body: `{ "op": "list" \| "add" \| "remove" \| "enable" \| "disable" \| "status", "id"?: string, "path"?: string, "tag"?: string, "hostname"?: string, "facility"?: string, "severity"?: string, "start_at_end"?: bool }`. `op` is required; `add` requires `id`, `path`, and `tag`; remove/enable/disable require `id`. | `FileTailResponse { sources: [FileTailSource], statuses: [FileTailStatus] }` | 200, 400, 401, **403**, 500 | mixed | Requires normal `Authorization: Bearer $CORTEX_API_TOKEN` plus `X-Cortex-Admin-Token: $CORTEX_API_ADMIN_TOKEN`. Manages Cortex-owned local file-tail ingest sources stored in `/file-tails.json`. `add` paths must be existing non-symlink regular files under `CORTEX_FILE_TAIL_ALLOWED_ROOTS`; keep the documented default to `/file-tail-root` and set an explicit allowlist to opt into broader read-only roots. CLI command: `cortex file-tail ...`; MCP action: `file_tails` (`cortex:admin`). | + ### DB ops (4) — bead `.4` | Method | Path | Scope | Request | Response (top-level) | Status codes | Idempotent | Notes | @@ -84,7 +90,7 @@ to them by default. | GET | `/api/graph/explain` | read | query: entity selector, `depth?` (clamped to 3), `beam_width?`, `max_chains?`, `evidence_sample_limit?`, `payload_budget?` | `GraphExplainResponse { resolved_entity, chains, narrative, open_questions, missing_evidence, next_queries, metadata }` | 200, 400, 401, 404, 503, 500 | Y | Deterministic evidence-backed explanation; weak evidence becomes open questions, not causal claims. | | GET | `/api/graph/evidence` | read | query: `evidence_id` (REQUIRED, minimum 1), `payload_budget?` | `GraphEvidenceLookupResponse { evidence, relationship, src_entity, dst_entity, source_log_summary?, missing_source_reason?, metadata }` | 200, 400, 401, 404, 503, 500 | Y | Proof lookup for one evidence row. Source summaries are redacted/truncated and exclude raw frames and raw metadata. | -**Total: 56 routes** (current `src/api.rs` router surface, including syslog, +**Total: 57 routes** (current `src/api.rs` router surface, including syslog, surface-parity, AI, graph, compose, notification, error-ack, and DB routes). --- @@ -172,15 +178,12 @@ shell environment is unaffected. address while `CORTEX_PUBLIC_URL` does not begin with `https://`, so a misconfiguration is loud at first boot rather than silent in production. -- **Single-token model.** `build_auth_layer` accepts exactly one token; - `AuthPolicy::Mounted` is enforced for `/api/*` regardless of bind - address (eng-review C1). The "scope" column in this matrix - (`read`/`admin`) is **documentation only** — there is no per-token - scope check today. A multi-token rollout (e.g. read-only viewer - token, separate admin token) would slot into `auth_state` without a - router rewrite; until then, every valid token can call every route. - This is reflected by a code-comment in `build_auth_layer` so future - contributors don't assume runtime scope enforcement. +- **API auth model.** `build_auth_layer` accepts the normal + `CORTEX_API_TOKEN`; `AuthPolicy::Mounted` is enforced for `/api/*` + regardless of bind address (eng-review C1). Most legacy admin rows still use + that bearer as their only gate. `/api/file-tails` additionally requires + `X-Cortex-Admin-Token: $CORTEX_API_ADMIN_TOKEN` because it exposes configured + filesystem paths and can mutate runtime tail sources. --- diff --git a/docs/contracts/log-filter-surface.md b/docs/contracts/log-filter-surface.md index 83f269a2..2b2615a6 100644 --- a/docs/contracts/log-filter-surface.md +++ b/docs/contracts/log-filter-surface.md @@ -25,6 +25,7 @@ These fields match `search` without `query` and use the same timestamp parsing, - `source_kind=docker-stream`: filters `source_ip` by the `docker://` prefix. - `source_kind=docker-event`: filters `source_ip` by the `docker-event://` prefix. +- `source_kind=file-tail`: filters `source_ip` by the `file-tail://` prefix. - `source_kind=agent-command`: filters `source_ip` by the `agent-command://` prefix. - `source_kind=shell-history`: filters `source_ip` by the `shell-history://` prefix. - `source_kind=transcript`: filters transcript rows when combined with `tool`, `project`, or `session_id`. diff --git a/docs/contracts/mcp-actions-current.md b/docs/contracts/mcp-actions-current.md index 7cbbe1c8..ea964ec0 100644 --- a/docs/contracts/mcp-actions-current.md +++ b/docs/contracts/mcp-actions-current.md @@ -22,13 +22,13 @@ response keys are stable. Renaming, removing, or tightening them is a breaking change. Adding optional parameters or optional response fields is non-breaking. Most actions require `cortex:read` when auth is mounted. `ack_error`, -`unack_error`, and `notifications_test` require `cortex:admin`. `help` has no -action-level scope requirement, though the protected endpoint still requires -transport auth when configured. +`unack_error`, `file_tails`, and `notifications_test` require +`cortex:admin`. `help` has no action-level scope requirement, though the +protected endpoint still requires transport auth when configured. ## Current Action Index -The live registry currently contains 44 actions: +The live registry currently contains 46 actions: | Action | Scope | Cost | Purpose | | --- | --- | --- | --- | @@ -75,6 +75,7 @@ The live registry currently contains 44 actions: | `graph` | `cortex:read` | moderate | Entity lookup and one-hop graph neighborhoods | | `ack_error` | `cortex:admin` | write | Acknowledge an error signature | | `unack_error` | `cortex:admin` | write | Revoke an error acknowledgement | +| `file_tails` | `cortex:admin` | write | Manage local file-tail ingest sources | | `notifications_test` | `cortex:admin` | write | Send a test Apprise notification | | `help` | none | cheap | Markdown action reference | diff --git a/docs/contracts/source-kinds.md b/docs/contracts/source-kinds.md index 0c87c663..03f9a71a 100644 --- a/docs/contracts/source-kinds.md +++ b/docs/contracts/source-kinds.md @@ -80,6 +80,7 @@ no-match, §7). Renaming an existing value is a major version bump. | `adguard-api` | AdGuard poller (epic C `cortex-awvr`) | AdGuard Home `/control/querylog` poller | | `shell-history` | `src/command_log.rs` | Local shell history backfill, currently zsh extended history | | `agent-command` | `src/command_log.rs` | AI agent-launched shell commands imported from a private JSONL spool | +| `file-tail` | `src/file_tail/supervisor.rs` | Cortex-managed local file tails configured in `file-tails.json` | **Removed / renamed during reconciliation:** @@ -117,6 +118,7 @@ percent-encoded as required. | `adguard-api` | `adguard:///` | `adguard://adguard.lan/` | | `shell-history` | `shell-history:////` | `shell-history://dookie/jmagar/zsh` | | `agent-command` | `agent-command:////` | `agent-command://dookie/claude-code/019e588f` | +| `file-tail` | `file-tail:///` | `file-tail://squirts/swag-access` | ### Notes on the `agent://` authority @@ -168,6 +170,9 @@ scheme reconstruction): `app_name="adguard-query"`). - `unifi-api` → no parser in V1 (poller writes structured rows directly under `metadata_json.unifi`). + - `file-tail` → parser chosen by `app_name`/tag; metadata includes + `file_tail_id`, `tag`, and `path_basename` (full paths stay on the + admin-only management surface). - `otlp` → `app_name` (= OTLP `service.name`). - `agent` → `app_name` exact match, same path as syslog. - `shell-history` / `agent-command` → no parser in V1; the importer diff --git a/docs/mcp/SCHEMA.md b/docs/mcp/SCHEMA.md index 7cb6109b..f2c54749 100644 --- a/docs/mcp/SCHEMA.md +++ b/docs/mcp/SCHEMA.md @@ -20,7 +20,7 @@ wins. ## Current Actions cortex exposes one MCP tool named `cortex`. The required `action` argument -selects one of these 44 actions: +selects one of these 46 actions: | Action | Scope | Cost | Purpose | | --- | --- | --- | --- | @@ -67,6 +67,7 @@ selects one of these 44 actions: | `graph` | `cortex:read` | moderate | Entity lookup and one-hop graph neighborhoods | | `ack_error` | `cortex:admin` | write | Acknowledge an error signature | | `unack_error` | `cortex:admin` | write | Revoke an error acknowledgement | +| `file_tails` | `cortex:admin` | write | Manage Cortex-owned file-tail ingest sources | | `notifications_test` | `cortex:admin` | write | Send a test Apprise notification | | `help` | none | cheap | Markdown action reference | @@ -121,7 +122,7 @@ handler and service layers. | `host` | Optional host_id-or-hostname filter for `correlate_state` | | `reference_time` | Required window center for `correlate` and `correlate_state` | | `source_ip` | `search`, `filter`, `tail`, `correlate`, `ai_correlate` | -| `source_kind` | `filter` only; aliases Docker, command-history, shell-history, transcript, and AI-tool rows | +| `source_kind` | `filter` only; aliases Docker, file-tail, command-history, shell-history, transcript, and AI-tool rows | | `project` | `filter`, `sessions`, `search_sessions`, `abuse`, `ai_correlate`, `usage_blocks`, `project_context`, `list_ai_tools` | | `tool` | `filter`, `sessions`, `search_sessions`, `abuse`, `ai_correlate`, `usage_blocks`, `project_context`, `list_ai_projects` | | `session_id` | `filter`, `ai_correlate` | @@ -149,6 +150,7 @@ See [CORRELATION.md](CORRELATION.md) for the full behavior matrix. | `ask_history` | `query`, `hostname`, `app_name`, `from`, `to`, `limit` | | `incident_context` | `from`, `to`, `hostname`, `app_name`, `severity_min`, `limit`; `query` is accepted by the request shape but intentionally ignored in v1 | | `graph` | `mode=entity|around|explain|evidence`; entity/around/explain require exactly one target lookup strategy (`entity_id`, `entity_type` + `key`, or `alias_type` + `alias_key`); `around` accepts `depth=1` only; `explain` accepts `depth=1..3`; `evidence` requires `evidence_id`; optional `limit`, `evidence_sample_limit`, `payload_budget` | +| `file_tails` | `op` is required and enumerated as `list`, `add`, `remove`, `enable`, `disable`, or `status`; add requires `id`, `path`, `tag`, and `hostname`; remove/enable/disable require `id`; optional `facility`, `severity`, `start_at_end` | ## Validation diff --git a/docs/mcp/TESTS.md b/docs/mcp/TESTS.md index 9a2d0a52..687626d6 100644 --- a/docs/mcp/TESTS.md +++ b/docs/mcp/TESTS.md @@ -59,7 +59,7 @@ Action registry covered by live/script references: `search`, `filter`, `tail`, ` `source_ips`, `timeline`, `patterns`, `context`, `get`, `ingest_rate`, `silent_hosts`, `clock_skew`, `anomalies`, `compare`, `compose_status`, `compose_doctor`, `unaddressed_errors`, `ack_error`, `unack_error`, -`notifications_recent`, `notifications_test`, +`notifications_recent`, `file_tails`, `notifications_test`, `similar_incidents`, `ask_history`, `incident_context`, `graph`, `help`. ### mcporter-based testing @@ -90,6 +90,7 @@ mcporter call --config config/mcporter.json cortex.cortex action=get id=1 mcporter call --config config/mcporter.json cortex.cortex action=ingest_rate mcporter call --config config/mcporter.json cortex.cortex action=silent_hosts mcporter call --config config/mcporter.json cortex.cortex action=clock_skew +mcporter call --config config/mcporter.json cortex.cortex action=file_tails op=status mcporter call --config config/mcporter.json cortex.cortex action=anomalies mcporter call --config config/mcporter.json cortex.cortex action=compare a_from=2026-01-01T00:00:00Z a_to=2026-01-01T01:00:00Z b_from=2026-01-01T01:00:00Z b_to=2026-01-01T02:00:00Z mcporter call --config config/mcporter.json cortex.cortex action=compose_status diff --git a/docs/mcp/TOOLS.md b/docs/mcp/TOOLS.md index bfd6e96b..dd9b9423 100644 --- a/docs/mcp/TOOLS.md +++ b/docs/mcp/TOOLS.md @@ -45,6 +45,7 @@ cortex exposes one MCP tool named `cortex`. The required | `unaddressed_errors` | List unacknowledged repeating error signatures | | `ack_error` | Acknowledge an error signature to suppress it from future reports | | `unack_error` | Revoke an acknowledgement so a signature reappears in reports | +| `file_tails` | Manage Cortex-owned file-tail ingest sources | | `notifications_recent` | List recent notification firings | | `notifications_test` | Send a test notification via Apprise | | `similar_incidents` | FTS5 cluster search — find historical incidents similar to a query | diff --git a/docs/sessions/2026-06-11-file-tail-ingest.md b/docs/sessions/2026-06-11-file-tail-ingest.md new file mode 100644 index 00000000..4ea5ecae --- /dev/null +++ b/docs/sessions/2026-06-11-file-tail-ingest.md @@ -0,0 +1,128 @@ +--- +date: 2026-06-11 21:37:45 EDT +repo: git@github.com:jmagar/cortex.git +branch: codex/file-tail-ingest +head: 4b17a3e +pr: https://github.com/jmagar/cortex/pull/73 +working directory: /home/jmagar/workspace/cortex/.worktrees/file-tail-ingest +worktree: /home/jmagar/workspace/cortex/.worktrees/file-tail-ingest +beads: syslog-mcp-6y96m +--- + +# Managed file-tail ingest + +## User Request +Create and work the plan `2026-06-11-file-tail-ingest.md`: keep the special log-file ingestion functionality that previously came from syslog forwarders, but expose it as a managed Cortex action available through CLI, REST API, and MCP. + +## Session Overview +Implemented managed file-tail ingestion on branch `codex/file-tail-ingest` and opened PR #73. Cortex can now persist file-tail source definitions, supervise enabled sources, ingest appended lines through the same log path as other sources, and manage those sources through `cortex file-tail`, `POST /api/file-tails`, and the admin MCP action `file_tails`. + +The final review pass hardened the feature around admin authorization, path policy, checkpoint correctness, partial-line handling, rotation, MCP schema typing, smoke/live-test behavior, and docs parity. + +## Sequence of Events +1. Created isolated worktree `/home/jmagar/workspace/cortex/.worktrees/file-tail-ingest` on branch `codex/file-tail-ingest`. +2. Claimed bead `syslog-mcp-6y96m` for managed file-tail ingestion. +3. Implemented the initial registry, supervisor, runtime wiring, CLI/API/MCP action, docs, config, and version bump to 1.20.0. +4. Opened PR #73: `feat: add managed file-tail ingest`. +5. Ran multiple review waves focused on data integrity, security, performance, API contracts, test simplification, and PR comments. +6. Landed review hardening commit `4b17a3e` after local full-gate verification and pre-push verification. +7. Dispatched two follow-up review agents over PR #73, split between runtime/data-integrity and API/CLI/MCP/docs concerns. +8. Folded their fixes into the branch: durable DB-write acknowledgement before file-tail checkpoints, safe opened-file identity checks, read-only list/status behavior, partial-EOF ingestion before rotation, admin/CORS/schema hardening, smoke-test admin coverage, and narrower production path defaults. + +## Key Changes +| area | change | +|---|---| +| Registry | `file-tails.json` persisted via `FileTailRegistry`, with add/list/status/remove/enable/disable operations and missing-id errors. | +| Supervisor | `FileTailSupervisor` tails enabled sources, checkpoints progress, reloads registry state between retries, handles copytruncate and rename-create rotation, and buffers partial EOF records until newline. | +| Security | REST management requires normal API bearer auth plus `X-Cortex-Admin-Token`; file paths must be absolute, existing, regular files, non-symlinks, and inside configured allow roots. | +| Metadata | File-tail rows use `source_kind=file-tail`, `source=file-tail:`, and metadata limited to `file_tail_id`, `tag`, and `path_basename`. | +| Interfaces | Added `cortex file-tail` CLI, `/api/file-tails`, and admin MCP action `file_tails`. | +| Contracts | Updated MCP schemas, action counts, source-kind docs, filter aliases, API docs, CLI docs, config docs, README, CLAUDE.md, and smoke/live test scripts. | + +## Review Findings Addressed +- Checkpoint updates are registry-authoritative, so retries do not replay from stale in-memory source copies. +- File-tail checkpoints now advance only after the batch writer successfully commits the row to SQLite; retryable write failures keep the durable ack pending. +- The batch writer flushes durable file-tail entries immediately so checkpoint durability does not throttle ingestion to one line per flush interval. +- Partial lines at EOF are held until newline instead of being prematurely ingested. +- Unterminated partial lines are ingested before rotation/truncation and leave status context for operators. +- Rotation and truncation reopen paths through the same path-policy checks used on initial open. +- Opened file identity is validated after `O_NOFOLLOW` open so symlink swaps or path races are rejected. +- Production defaults now allow only `/file-tail-root` unless `CORTEX_FILE_TAIL_ALLOWED_ROOTS` is set; tests retain tempdir roots. +- Missing files during reopen now surface an error instead of silently marking the source healthy. +- `list` and `status` are read-only and no longer reconcile/spawn supervisor tasks. +- Disable/remove stop active tailing through supervisor reconciliation tests. +- Configured file-tail hostnames are normalized/validated and source identity components are sanitized. +- MCP schema now constrains `get.id` to integer and `file_tails.id` to string through action-specific JSON Schema conditionals. +- MCP schema now also constrains `file_tails.op` to the supported operation enum and requires per-operation fields. +- REST tests cover missing and wrong admin token cases. +- Blank admin tokens are rejected and `X-Cortex-Admin-Token` is accepted by CORS preflight. +- Smoke/live MCP scripts skip admin-only `file_tails` only when the token cannot have admin scope. +- Optional admin-token smoke tests add, append, query, and remove a live file-tail source under `/file-tail-root`. +- The stale `[Unreleased]` changelog compare target now starts from `v1.20.0`. + +## Verification Evidence +| command | result | +|---|---| +| `cargo test file_tail::supervisor_tests::supervisor_ingests_appended_line_and_updates_checkpoint --lib` | pass | +| `cargo test otlp::tests::auth --lib` | pass | +| `cargo test file_tail --all-targets` | pass | +| `bash -n scripts/smoke-test.sh tests/test_live.sh tests/mcporter/test-tools.sh` | pass | +| `cargo fmt --check` | pass | +| `cargo clippy --all-targets -- -D warnings` | pass | +| `cargo test` | pass: 1178 lib + 332 main + integration/doc tests clean, with 2 ignored network/perf tests | +| `bash scripts/check-version-sync.sh` | pass | +| `bash scripts/check-rust-module-size.sh --limit 500 src/cli.rs src/cli` | pass | +| `cargo deny check` | pass, with existing wildcard git-source warning for `lab-auth` | +| `git push` pre-push hook | pending for the review-follow-up commit | + +## PR State +- PR: https://github.com/jmagar/cortex/pull/73 +- Head after implementation/review fixes: current review-follow-up commit after `4b17a3e` +- Local verification: green. +- GitHub checks need to restart after the review-follow-up push. + +## Remaining Notes +- CodeRabbit/PR Review Toolkit comments from the previous head were used as the acceptance bar for the follow-up agent pass. +- The `FileTailStatus.running` boolean was left as-is to avoid a broader API shape churn; `last_error` now carries retry detail for operators. +- `CORTEX_FILE_TAIL_ALLOWED_ROOTS` can broaden or further constrain the default `/file-tail-root` mount set. + +## Beads Activity +- `syslog-mcp-6y96m` was claimed and implemented. +- `syslog-mcp-6y96m` was reopened for the review follow-up. +- Follow-up bead `syslog-mcp-7j9hn` was opened and closed by the API/CLI/docs agent during review remediation. + +## Next Steps +- Wait for PR #73 CI and CodeRabbit to finish on the final branch head. +- Merge PR #73 once checks are green. +- Deploy with mounted log roots for any host that should manage file-tail sources through Cortex rather than rsyslog imfile. + +## 2026-06-12 Second Review Pass + +Dispatched the PR review toolkit again after PR #73 head `e54d033` and addressed the fresh findings from CodeRabbit plus the reviewer/test-analyzer agents. + +### Additional Fixes + +- Fixed the file-tail model compile break introduced while making `hostname` required. +- Required `hostname` across CLI, REST, MCP schema, models, docs, and tests so file-tail rows are never silently attributed to the Cortex container/local host. +- Rejected duplicate `file_tails op=add` requests before `upsert`, preserving existing checkpoints instead of resetting them. +- Kept query-only/stdio runtimes registry-readable but mutation-disabled, preventing local CLI or stdio MCP sessions from spawning competing tailers outside the long-running server. +- Changed the HTTP client file-tail admin POST path to avoid 503 retry replay and added a wiremock regression test. +- Added explicit committed-mutation error messages for reconcile/refresh failures and a regression test proving registry state is preserved. +- Added `last_read_at`, `last_checkpoint_at`, and `blocked_on_writer_since` to file-tail status and surfaced file-tail blocked count in the general `status` action. +- Hardened resume/rotation behavior: stale checkpoint identity now starts replacement files at offset 0; same-inode copytruncate/regrow is detected by prefix fingerprint; rename-create rotation waits through a short EOF grace window before switching away from the old file descriptor. +- Canonicalized configured allowed roots and added direct path-policy tests for env roots and symlinked roots. +- Fixed the Docker live smoke harness by chmodding the host smoke dir/file and asserting container readability before the file-tail smoke. +- Tightened admin-scope predicates in `scripts/smoke-test.sh`, `tests/test_live.sh`, and `tests/mcporter/test-tools.sh`. +- Bumped version to `1.20.1` with changelog notes for the review-hardening patch. + +### Second-Pass Verification + +| command | result | +|---|---| +| `bash -n scripts/smoke-test.sh tests/test_live.sh tests/mcporter/test-tools.sh` | pass | +| `cargo test file_tail --lib --quiet` | pass: 45 tests | +| `cargo test file_tails --lib --quiet` | pass: 13 tests | +| `cargo test http_client::tests::file_tails_post_does_not_retry_503 --quiet` | pass | +| `cargo clippy --all-targets -- -D warnings` | pass | +| `cargo test --locked` | pass: 1189 lib + 334 main + integration/doc tests clean, with 1 ignored network-dependent test | +| `CORTEX_TOKEN=codex-live-smoke-token bash tests/test_live.sh --mode docker --token codex-live-smoke-token` | pass: 122 passed, 0 failed, 4 skipped | diff --git a/docs/superpowers/plans/2026-06-11-file-tail-ingest.md b/docs/superpowers/plans/2026-06-11-file-tail-ingest.md new file mode 100644 index 00000000..753c1ea0 --- /dev/null +++ b/docs/superpowers/plans/2026-06-11-file-tail-ingest.md @@ -0,0 +1,2016 @@ +# Managed File-Tail Ingest Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add Cortex-owned file-tail ingestion so operators can register log files such as SWAG, Authelia, AdGuard, fail2ban, and AI transcripts through CLI, REST API, and MCP without hand-maintained rsyslog `imfile` drop-ins. + +**Architecture:** Add a small file-tail subsystem that persists source definitions in `data/file-tails.json`, spawns one supervised Tokio task per enabled source, converts appended lines into `LogBatchEntry` rows, and sends them through the existing ingest writer/enrichment pipeline. Expose one admin MCP action, `file_tails`, with `op=list|add|remove|enable|disable|status`; REST and CLI call the same service methods and request/response models. + +**Tech Stack:** Rust 2024, Tokio async file IO, serde JSON registry, existing `IngestTx`, existing `CortexService`, Axum REST, single-action RMCP dispatch, existing hand-rolled CLI parser. + +--- + +## File Structure + +- Create `src/file_tail.rs`: module entrypoint and public crate-internal re-exports. +- Create `src/file_tail/models.rs`: persisted source definitions plus shared request/response DTOs. +- Create `src/file_tail/registry.rs`: load/save/update `data/file-tails.json` atomically. +- Create `src/file_tail/supervisor.rs`: runtime tail task management and line-to-`LogBatchEntry` conversion. +- Create `src/file_tail/models_tests.rs`, `registry_tests.rs`, `supervisor_tests.rs`: focused unit tests. +- Modify `src/lib.rs`: add `pub mod file_tail;`. +- Modify `src/enrich/parser.rs`: add `SourceKind::FileTail` with wire value `file-tail`. +- Modify `src/runtime.rs`: create registry/supervisor in `RuntimeCore`, spawn file-tail tasks with maintenance handles, and expose control through `CortexService`. +- Modify `src/app.rs`, `src/app/models.rs`, `src/app/models/ops.rs`, `src/app/services.rs`, and create `src/app/services/file_tails.rs`: service-layer request validation and control methods. +- Modify `src/mcp/actions.rs`, `src/mcp/tools.rs`, `src/mcp/schemas.rs`: add admin MCP action `file_tails`. +- Modify `src/api.rs`, `src/cli/http_client.rs`: add REST route `POST /api/file-tails` and HTTP client method. +- Modify `src/cli/args.rs`, `src/cli/parse.rs`, `src/cli/run.rs`, `src/cli/dispatch.rs`, and create `src/cli/commands/file_tails.rs`: CLI `cortex file-tail ...`. +- Modify docs: `README.md`, `CLAUDE.md`, `docs/CLI.md`, `docs/api.md`, `docs/mcp/SCHEMA.md`, `docs/CONFIG.md`, `docs/contracts/source-kinds.md`, `.env.example`, `config.toml`. +- Version bump files: `Cargo.toml`, `Cargo.lock`, `server.json`, `mcpb/manifest.json`, `CHANGELOG.md`. + +--- + +### Task 1: Add File-Tail Models And SourceKind + +**Files:** +- Create: `src/file_tail.rs` +- Create: `src/file_tail/models.rs` +- Test: `src/file_tail/models_tests.rs` +- Modify: `src/lib.rs` +- Modify: `src/enrich/parser.rs` +- Test: `src/enrich/parser_tests.rs` + +- [ ] **Step 1: Write failing model and source-kind tests** + +Create `src/file_tail/models_tests.rs`: + +```rust +use super::models::*; + +#[test] +fn add_request_builds_enabled_source_with_defaults() { + let req = FileTailAddRequest { + id: "swag-access".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: None, + severity: None, + start_at_end: None, + }; + + let source = FileTailSource::from_add(req, "2026-06-11T20:00:00Z"); + + assert_eq!(source.id, "swag-access"); + assert_eq!(source.path, "/mnt/appdata/swag/log/nginx/access.log"); + assert_eq!(source.tag, "swag-access"); + assert_eq!(source.hostname.as_deref(), Some("squirts")); + assert_eq!(source.facility.as_deref(), Some("local7")); + assert_eq!(source.severity, "info"); + assert!(source.start_at_end); + assert!(source.enabled); + assert_eq!(source.created_at, "2026-06-11T20:00:00Z"); + assert_eq!(source.updated_at, "2026-06-11T20:00:00Z"); +} + +#[test] +fn file_tail_request_rejects_missing_fields_for_add() { + let req = FileTailRequest { + op: FileTailOp::Add, + id: None, + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }; + + assert_eq!( + req.validate().unwrap_err(), + "file_tails op=add requires id, path, and tag" + ); +} + +#[test] +fn file_tail_request_rejects_path_traversal_ids() { + let req = FileTailRequest { + op: FileTailOp::Remove, + id: Some("../swag".into()), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }; + + assert_eq!( + req.validate().unwrap_err(), + "file_tails id must contain only ASCII letters, digits, dot, underscore, or dash" + ); +} +``` + +Append to `src/enrich/parser_tests.rs` or create the sidecar test if no direct spot exists: + +```rust +#[test] +fn source_kind_file_tail_wire_value_is_stable() { + assert_eq!(crate::enrich::SourceKind::FileTail.as_str(), "file-tail"); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test file_tail --lib +cargo test source_kind_file_tail_wire_value_is_stable --lib +``` + +Expected: compile fails because `file_tail` module, DTOs, and `SourceKind::FileTail` do not exist. + +- [ ] **Step 3: Add the models** + +Create `src/file_tail.rs`: + +```rust +pub(crate) mod models; +pub(crate) mod registry; +pub(crate) mod supervisor; + +pub(crate) use models::{ + FileTailAddRequest, FileTailOp, FileTailRequest, FileTailResponse, FileTailSource, + FileTailStatus, +}; +pub(crate) use registry::FileTailRegistry; +pub(crate) use supervisor::FileTailSupervisor; + +#[cfg(test)] +#[path = "file_tail/models_tests.rs"] +mod models_tests; +``` + +Create `src/file_tail/models.rs`: + +```rust +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct FileTailSource { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: String, + pub start_at_end: bool, + pub enabled: bool, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum FileTailOp { + List, + Add, + Remove, + Enable, + Disable, + Status, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct FileTailRequest { + pub op: FileTailOp, + pub id: Option, + pub path: Option, + pub tag: Option, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct FileTailAddRequest { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct FileTailStatus { + pub id: String, + pub running: bool, + pub last_line_at: Option, + pub last_error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct FileTailResponse { + pub sources: Vec, + pub statuses: Vec, +} + +impl FileTailSource { + pub(crate) fn from_add(req: FileTailAddRequest, now: &str) -> Self { + Self { + id: req.id, + path: req.path, + tag: req.tag, + hostname: req.hostname, + facility: Some(req.facility.unwrap_or_else(|| "local7".to_string())), + severity: req.severity.unwrap_or_else(|| "info".to_string()), + start_at_end: req.start_at_end.unwrap_or(true), + enabled: true, + created_at: now.to_string(), + updated_at: now.to_string(), + } + } +} + +impl FileTailRequest { + pub(crate) fn validate(&self) -> Result<(), String> { + match self.op { + FileTailOp::List | FileTailOp::Status => Ok(()), + FileTailOp::Add => { + let Some(id) = self.id.as_deref() else { + return Err("file_tails op=add requires id, path, and tag".into()); + }; + validate_id(id)?; + if self.path.as_deref().is_none_or(str::is_empty) + || self.tag.as_deref().is_none_or(str::is_empty) + { + return Err("file_tails op=add requires id, path, and tag".into()); + } + Ok(()) + } + FileTailOp::Remove | FileTailOp::Enable | FileTailOp::Disable => { + let Some(id) = self.id.as_deref() else { + return Err(format!("file_tails op={:?} requires id", self.op).to_lowercase()); + }; + validate_id(id) + } + } + } + + pub(crate) fn into_add(self) -> Result { + self.validate()?; + Ok(FileTailAddRequest { + id: self.id.expect("validated id"), + path: self.path.expect("validated path"), + tag: self.tag.expect("validated tag"), + hostname: self.hostname, + facility: self.facility, + severity: self.severity, + start_at_end: self.start_at_end, + }) + } +} + +fn validate_id(id: &str) -> Result<(), String> { + if id.is_empty() + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + { + return Err( + "file_tails id must contain only ASCII letters, digits, dot, underscore, or dash".into(), + ); + } + Ok(()) +} +``` + +Modify `src/lib.rs`: + +```rust +pub mod file_tail; +``` + +Modify `src/enrich/parser.rs`: + +```rust +pub enum SourceKind { + SyslogUdp, + SyslogTcp, + DockerStream, + DockerEvent, + Otlp, + AdguardApi, + UnifiApi, + Agent, + ShellHistory, + AgentCommand, + FileTail, +} +``` + +and in `SourceKind::as_str()`: + +```rust +SourceKind::FileTail => "file-tail", +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +cargo test file_tail --lib +cargo test source_kind_file_tail_wire_value_is_stable --lib +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add src/lib.rs src/enrich/parser.rs src/enrich/parser_tests.rs src/file_tail.rs src/file_tail/models.rs src/file_tail/models_tests.rs +git commit -m "feat: add file-tail ingest models" +``` + +--- + +### Task 2: Persist File-Tail Sources + +**Files:** +- Create: `src/file_tail/registry.rs` +- Test: `src/file_tail/registry_tests.rs` +- Modify: `src/file_tail.rs` + +- [ ] **Step 1: Write failing registry tests** + +Create `src/file_tail/registry_tests.rs`: + +```rust +use super::models::{FileTailAddRequest, FileTailSource}; +use super::registry::FileTailRegistry; + +#[test] +fn registry_adds_lists_and_removes_sources() { + let temp = tempfile::tempdir().unwrap(); + let registry = FileTailRegistry::new(temp.path().join("file-tails.json")); + let source = FileTailSource::from_add( + FileTailAddRequest { + id: "swag-access".into(), + path: "/tmp/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: None, + severity: None, + start_at_end: None, + }, + "2026-06-11T20:00:00Z", + ); + + registry.upsert(source.clone()).unwrap(); + assert_eq!(registry.list().unwrap(), vec![source]); + + registry.remove("swag-access").unwrap(); + assert!(registry.list().unwrap().is_empty()); +} + +#[test] +fn registry_persists_across_instances() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("file-tails.json"); + let registry = FileTailRegistry::new(path.clone()); + registry + .upsert(FileTailSource::from_add( + FileTailAddRequest { + id: "authelia".into(), + path: "/tmp/authelia.log".into(), + tag: "authelia".into(), + hostname: None, + facility: Some("local5".into()), + severity: Some("info".into()), + start_at_end: Some(false), + }, + "2026-06-11T20:00:00Z", + )) + .unwrap(); + + let reloaded = FileTailRegistry::new(path); + let sources = reloaded.list().unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].id, "authelia"); + assert_eq!(sources[0].facility.as_deref(), Some("local5")); + assert!(!sources[0].start_at_end); +} +``` + +Add to `src/file_tail.rs`: + +```rust +#[cfg(test)] +#[path = "file_tail/registry_tests.rs"] +mod registry_tests; +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test registry_ --lib +``` + +Expected: compile fails because `FileTailRegistry` does not exist. + +- [ ] **Step 3: Implement registry** + +Create `src/file_tail/registry.rs`: + +```rust +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; + +use super::models::FileTailSource; + +#[derive(Debug)] +pub(crate) struct FileTailRegistry { + path: PathBuf, + lock: Mutex<()>, +} + +impl FileTailRegistry { + pub(crate) fn new(path: PathBuf) -> Self { + Self { + path, + lock: Mutex::new(()), + } + } + + pub(crate) fn path_from_storage_db(db_path: &Path) -> PathBuf { + db_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join("file-tails.json") + } + + pub(crate) fn list(&self) -> Result> { + let _guard = self.lock.lock(); + self.read_locked() + } + + pub(crate) fn upsert(&self, source: FileTailSource) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + sources.retain(|existing| existing.id != source.id); + sources.push(source); + sources.sort_by(|a, b| a.id.cmp(&b.id)); + self.write_locked(&sources) + } + + pub(crate) fn remove(&self, id: &str) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + sources.retain(|existing| existing.id != id); + self.write_locked(&sources) + } + + pub(crate) fn set_enabled(&self, id: &str, enabled: bool, now: &str) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + let source = sources + .iter_mut() + .find(|source| source.id == id) + .with_context(|| format!("file tail source not found: {id}"))?; + source.enabled = enabled; + source.updated_at = now.to_string(); + self.write_locked(&sources) + } + + fn read_locked(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let raw = std::fs::read_to_string(&self.path) + .with_context(|| format!("read {}", self.path.display()))?; + serde_json::from_str(&raw).with_context(|| format!("parse {}", self.path.display())) + } + + fn write_locked(&self, sources: &[FileTailSource]) -> Result<()> { + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create {}", parent.display()))?; + } + let tmp = self.path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(sources)?; + std::fs::write(&tmp, body).with_context(|| format!("write {}", tmp.display()))?; + std::fs::rename(&tmp, &self.path) + .with_context(|| format!("replace {}", self.path.display()))?; + Ok(()) + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +cargo test registry_ --lib +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add src/file_tail.rs src/file_tail/registry.rs src/file_tail/registry_tests.rs +git commit -m "feat: persist file-tail sources" +``` + +--- + +### Task 3: Tail Files Into The Existing Ingest Pipeline + +**Files:** +- Create: `src/file_tail/supervisor.rs` +- Test: `src/file_tail/supervisor_tests.rs` +- Modify: `src/file_tail.rs` +- Modify: `src/observability.rs` + +- [ ] **Step 1: Write failing supervisor tests** + +Create `src/file_tail/supervisor_tests.rs`: + +```rust +use tokio::io::AsyncWriteExt; + +use crate::db::LogBatchEntry; +use crate::ingest::IngestTx; + +use super::models::FileTailSource; +use super::supervisor::{file_tail_line_to_entry, tail_file_once_for_test}; + +#[test] +fn file_tail_line_to_entry_sets_expected_envelope() { + let source = FileTailSource { + id: "swag-access".into(), + path: "/tmp/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: "info".into(), + start_at_end: true, + enabled: true, + created_at: "2026-06-11T20:00:00Z".into(), + updated_at: "2026-06-11T20:00:00Z".into(), + }; + + let entry = file_tail_line_to_entry(&source, "GET / HTTP/1.1\" 401", "2026-06-11T20:01:00Z"); + + assert_eq!(entry.timestamp, "2026-06-11T20:01:00Z"); + assert_eq!(entry.hostname, "squirts"); + assert_eq!(entry.facility.as_deref(), Some("local4")); + assert_eq!(entry.severity, "info"); + assert_eq!(entry.app_name.as_deref(), Some("swag-access")); + assert_eq!(entry.message, "GET / HTTP/1.1\" 401"); + assert_eq!(entry.raw, "GET / HTTP/1.1\" 401"); + assert_eq!(entry.source_ip, "file-tail://squirts/swag-access"); + assert!(entry.metadata_json.as_deref().unwrap().contains("\"source_kind\":\"file-tail\"")); + assert!(entry.metadata_json.as_deref().unwrap().contains("\"path\":\"/tmp/access.log\"")); +} + +#[tokio::test] +async fn tail_file_once_sends_existing_lines_when_not_starting_at_end() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("authelia.log"); + let mut file = tokio::fs::File::create(&file_path).await.unwrap(); + file.write_all(b"time=one level=info\n").await.unwrap(); + file.write_all(b"time=two level=error\n").await.unwrap(); + file.flush().await.unwrap(); + + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let source = FileTailSource { + id: "authelia".into(), + path: file_path.to_string_lossy().into_owned(), + tag: "authelia".into(), + hostname: Some("squirts".into()), + facility: Some("local5".into()), + severity: "info".into(), + start_at_end: false, + enabled: true, + created_at: "2026-06-11T20:00:00Z".into(), + updated_at: "2026-06-11T20:00:00Z".into(), + }; + + tail_file_once_for_test(source, ingest).await.unwrap(); + + assert_eq!(rx.recv().await.unwrap().message, "time=one level=info"); + assert_eq!(rx.recv().await.unwrap().message, "time=two level=error"); + assert!(rx.try_recv().is_err()); +} +``` + +Add to `src/file_tail.rs`: + +```rust +#[cfg(test)] +#[path = "file_tail/supervisor_tests.rs"] +mod supervisor_tests; +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test file_tail_line_to_entry_sets_expected_envelope tail_file_once_sends_existing_lines_when_not_starting_at_end --lib +``` + +Expected: compile fails because supervisor helpers do not exist. + +- [ ] **Step 3: Implement file-tail conversion and basic tail helper** + +Create `src/file_tail/supervisor.rs` with this minimum content: + +```rust +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; +use tokio::io::{AsyncBufReadExt, AsyncSeekExt, BufReader}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use crate::db::LogBatchEntry; +use crate::enrich::{SourceKind, stamp_source_kind}; +use crate::ingest::IngestTx; +use crate::ingest_metadata::bounded_metadata_json; + +use super::models::{FileTailSource, FileTailStatus}; +use super::registry::FileTailRegistry; + +#[derive(Clone)] +pub(crate) struct FileTailSupervisor { + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + tasks: Arc>>, +} + +struct TailTask { + handle: JoinHandle<()>, + status: Arc>, +} + +impl FileTailSupervisor { + pub(crate) fn new( + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + ) -> Self { + Self { + registry, + ingest, + token, + tasks: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub(crate) fn statuses(&self) -> Vec { + let mut out: Vec<_> = self + .tasks + .lock() + .values() + .map(|task| task.status.lock().clone()) + .collect(); + out.sort_by(|a, b| a.id.cmp(&b.id)); + out + } + + pub(crate) fn reconcile(&self) -> Result<()> { + let sources = self.registry.list()?; + let enabled: std::collections::HashSet = sources + .iter() + .filter(|source| source.enabled) + .map(|source| source.id.clone()) + .collect(); + + { + let mut tasks = self.tasks.lock(); + tasks.retain(|id, task| { + if enabled.contains(id) { + true + } else { + task.handle.abort(); + false + } + }); + } + + for source in sources.into_iter().filter(|source| source.enabled) { + if self.tasks.lock().contains_key(&source.id) { + continue; + } + self.spawn_source(source); + } + Ok(()) + } + + fn spawn_source(&self, source: FileTailSource) { + let id = source.id.clone(); + let status = Arc::new(Mutex::new(FileTailStatus { + id: id.clone(), + running: true, + last_line_at: None, + last_error: None, + })); + let task_status = Arc::clone(&status); + let ingest = self.ingest.clone(); + let token = self.token.clone(); + let handle = tokio::spawn(async move { + tail_file_loop(source, ingest, token, task_status).await; + }); + self.tasks.lock().insert(id, TailTask { handle, status }); + } +} + +async fn tail_file_loop( + source: FileTailSource, + ingest: IngestTx, + token: CancellationToken, + status: Arc>, +) { + loop { + if token.is_cancelled() { + status.lock().running = false; + return; + } + match tail_file_until_cancelled(&source, ingest.clone(), token.clone(), Arc::clone(&status)).await { + Ok(()) => { + status.lock().running = false; + return; + } + Err(err) => { + status.lock().last_error = Some(err.to_string()); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + } +} + +async fn tail_file_until_cancelled( + source: &FileTailSource, + ingest: IngestTx, + token: CancellationToken, + status: Arc>, +) -> Result<()> { + let mut file = tokio::fs::File::open(&source.path) + .await + .with_context(|| format!("open {}", source.path))?; + if source.start_at_end { + file.seek(std::io::SeekFrom::End(0)).await?; + } + let mut reader = BufReader::new(file); + let mut line = String::new(); + loop { + line.clear(); + tokio::select! { + _ = token.cancelled() => return Ok(()), + read = reader.read_line(&mut line) => { + let bytes = read?; + if bytes == 0 { + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + let msg = line.trim_end_matches(['\r', '\n']); + if msg.is_empty() { + continue; + } + let now = now_iso(); + let entry = file_tail_line_to_entry(source, msg, &now); + ingest.send(entry).await?; + let mut status = status.lock(); + status.last_line_at = Some(now); + status.last_error = None; + } + } + } +} + +pub(crate) fn file_tail_line_to_entry( + source: &FileTailSource, + line: &str, + now: &str, +) -> LogBatchEntry { + let hostname = source + .hostname + .clone() + .unwrap_or_else(|| local_hostname()); + let metadata_json = bounded_metadata_json(serde_json::json!({ + "source_type": "file_tail", + "source_kind": SourceKind::FileTail.as_str(), + "file_tail_id": source.id, + "path": source.path, + "tag": source.tag, + })); + let mut entry = LogBatchEntry { + timestamp: now.to_string(), + hostname: hostname.clone(), + facility: source.facility.clone(), + severity: source.severity.clone(), + app_name: Some(source.tag.clone()), + process_id: None, + message: line.to_string(), + raw: line.to_string(), + source_ip: format!("file-tail://{hostname}/{}", source.id), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: Some(metadata_json), + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + stamp_source_kind(&mut entry, SourceKind::FileTail); + entry +} + +#[cfg(test)] +pub(crate) async fn tail_file_once_for_test( + source: FileTailSource, + ingest: IngestTx, +) -> Result<()> { + let file = tokio::fs::File::open(&source.path).await?; + let mut reader = BufReader::new(file); + let mut line = String::new(); + while reader.read_line(&mut line).await? > 0 { + let msg = line.trim_end_matches(['\r', '\n']); + if !msg.is_empty() { + ingest + .send(file_tail_line_to_entry(&source, msg, "2026-06-11T20:01:00Z")) + .await?; + } + line.clear(); + } + Ok(()) +} + +fn now_iso() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +fn local_hostname() -> String { + std::env::var("HOSTNAME") + .ok() + .filter(|host| !host.trim().is_empty()) + .unwrap_or_else(|| "localhost".to_string()) +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: + +```bash +cargo test file_tail_line_to_entry_sets_expected_envelope tail_file_once_sends_existing_lines_when_not_starting_at_end --lib +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add src/file_tail.rs src/file_tail/supervisor.rs src/file_tail/supervisor_tests.rs +git commit -m "feat: tail files into ingest pipeline" +``` + +--- + +### Task 4: Wire Runtime And Service Control + +**Files:** +- Modify: `src/runtime.rs` +- Modify: `src/app.rs` +- Modify: `src/app/models.rs` +- Modify: `src/app/models/ops.rs` +- Modify: `src/app/services.rs` +- Create: `src/app/services/file_tails.rs` +- Test: `src/app/service_tests.rs` +- Test: `src/runtime_tests.rs` + +- [ ] **Step 1: Write failing service tests** + +Append to `src/app/service_tests.rs`: + +```rust +#[tokio::test] +async fn file_tails_add_list_disable_enable_remove_round_trip() { + let temp = tempfile::tempdir().unwrap(); + let storage = test_storage(temp.path()); + let pool = std::sync::Arc::new(crate::db::init_pool(&storage).unwrap()); + let registry = std::sync::Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_registry(registry); + + let add = service + .file_tails(crate::app::FileTailRequest { + op: crate::app::FileTailOp::Add, + id: Some("swag-access".into()), + path: Some("/tmp/access.log".into()), + tag: Some("swag-access".into()), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }) + .await + .unwrap(); + assert_eq!(add.sources[0].id, "swag-access"); + + let disabled = service + .file_tails(crate::app::FileTailRequest { + op: crate::app::FileTailOp::Disable, + id: Some("swag-access".into()), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }) + .await + .unwrap(); + assert!(!disabled.sources[0].enabled); + + let enabled = service + .file_tails(crate::app::FileTailRequest { + op: crate::app::FileTailOp::Enable, + id: Some("swag-access".into()), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }) + .await + .unwrap(); + assert!(enabled.sources[0].enabled); + + let removed = service + .file_tails(crate::app::FileTailRequest { + op: crate::app::FileTailOp::Remove, + id: Some("swag-access".into()), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }) + .await + .unwrap(); + assert!(removed.sources.is_empty()); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +cargo test file_tails_add_list_disable_enable_remove_round_trip --lib +``` + +Expected: compile fails because service methods and app exports do not exist. + +- [ ] **Step 3: Re-export file-tail DTOs from app** + +Modify `src/app/models/ops.rs`: + +```rust +pub use crate::file_tail::{ + FileTailAddRequest, FileTailOp, FileTailRequest, FileTailResponse, FileTailSource, + FileTailStatus, +}; +``` + +Modify `src/app.rs` export list: + +```rust +FileTailAddRequest, +FileTailOp, +FileTailRequest, +FileTailResponse, +FileTailSource, +FileTailStatus, +``` + +- [ ] **Step 4: Add registry/control to service** + +Modify `src/app/services.rs` imports: + +```rust +use crate::file_tail::{FileTailRegistry, FileTailRequest, FileTailResponse}; +``` + +Add fields to `CortexService`: + +```rust +file_tail_registry: Option>, +file_tail_reconcile: Option anyhow::Result<()> + Send + Sync>>, +file_tail_statuses: Option Vec + Send + Sync>>, +``` + +Initialize them to `None` in both constructors. + +Add builder: + +```rust +pub(crate) fn with_file_tail_registry(mut self, registry: Arc) -> Self { + self.file_tail_registry = Some(registry); + self +} + +pub(crate) fn with_file_tail_control( + mut self, + registry: Arc, + reconcile: Arc anyhow::Result<()> + Send + Sync>, + statuses: Arc Vec + Send + Sync>, +) -> Self { + self.file_tail_registry = Some(registry); + self.file_tail_reconcile = Some(reconcile); + self.file_tail_statuses = Some(statuses); + self +} +``` + +Add module declaration: + +```rust +mod file_tails; +``` + +Create `src/app/services/file_tails.rs`: + +```rust +use crate::app::{ServiceError, ServiceResult}; +use crate::file_tail::{FileTailRequest, FileTailResponse, FileTailSource}; + +use super::CortexService; + +impl CortexService { + pub async fn file_tails(&self, req: FileTailRequest) -> ServiceResult { + req.validate().map_err(ServiceError::InvalidInput)?; + let registry = self + .file_tail_registry + .as_ref() + .ok_or_else(|| ServiceError::InvalidInput("file-tail registry is not mounted".into()))?; + let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + + match req.op { + crate::file_tail::FileTailOp::List | crate::file_tail::FileTailOp::Status => {} + crate::file_tail::FileTailOp::Add => { + registry + .upsert(FileTailSource::from_add(req.into_add().map_err(ServiceError::InvalidInput)?, &now)) + .map_err(|err| ServiceError::Internal(err.to_string()))?; + } + crate::file_tail::FileTailOp::Remove => { + registry + .remove(req.id.as_deref().expect("validated id")) + .map_err(|err| ServiceError::Internal(err.to_string()))?; + } + crate::file_tail::FileTailOp::Enable => { + registry + .set_enabled(req.id.as_deref().expect("validated id"), true, &now) + .map_err(|err| ServiceError::InvalidInput(err.to_string()))?; + } + crate::file_tail::FileTailOp::Disable => { + registry + .set_enabled(req.id.as_deref().expect("validated id"), false, &now) + .map_err(|err| ServiceError::InvalidInput(err.to_string()))?; + } + } + + if let Some(reconcile) = &self.file_tail_reconcile { + reconcile().map_err(|err| ServiceError::Internal(err.to_string()))?; + } + + let sources = registry + .list() + .map_err(|err| ServiceError::Internal(err.to_string()))?; + let statuses = self + .file_tail_statuses + .as_ref() + .map(|statuses| statuses()) + .unwrap_or_default(); + Ok(FileTailResponse { sources, statuses }) + } +} +``` + +- [ ] **Step 5: Wire runtime supervisor** + +Modify `src/runtime.rs`: + +```rust +use crate::file_tail::{FileTailRegistry, FileTailSupervisor}; +``` + +Add field: + +```rust +file_tail_supervisor: FileTailSupervisor, +``` + +In `RuntimeCore::load` after `ingest` creation: + +```rust +let file_tail_registry = Arc::new(FileTailRegistry::new( + FileTailRegistry::path_from_storage_db(&config.storage.db_path), +)); +let file_tail_token = CancellationToken::new(); +let file_tail_supervisor = + FileTailSupervisor::new(Arc::clone(&file_tail_registry), ingest.clone(), file_tail_token.clone()); +let reconcile_supervisor = file_tail_supervisor.clone(); +let status_supervisor = file_tail_supervisor.clone(); +let service = CortexService::new(Arc::clone(&pool), config.storage.clone()).with_file_tail_control( + file_tail_registry, + Arc::new(move || reconcile_supervisor.reconcile()), + Arc::new(move || status_supervisor.statuses()), +); +``` + +Add `file_tail: Option>` to `MaintenanceHandles`. + +In `spawn_maintenance_tasks`, add: + +```rust +let file_tail = { + let supervisor = self.file_tail_supervisor.clone(); + let token = token.clone(); + Some(tokio::spawn(async move { + if let Err(err) = supervisor.reconcile() { + tracing::warn!(error = %err, "initial file-tail reconcile failed"); + } + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + loop { + tokio::select! { + _ = token.cancelled() => break, + _ = interval.tick() => { + if let Err(err) = supervisor.reconcile() { + tracing::warn!(error = %err, "file-tail reconcile failed"); + } + } + } + } + })) +}; +``` + +Include `self.file_tail` in shutdown joins. + +- [ ] **Step 6: Run test to verify it passes** + +Run: + +```bash +cargo test file_tails_add_list_disable_enable_remove_round_trip --lib +cargo test runtime --lib +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add src/runtime.rs src/app.rs src/app/models.rs src/app/models/ops.rs src/app/services.rs src/app/services/file_tails.rs src/app/service_tests.rs src/runtime_tests.rs +git commit -m "feat: manage file-tail sources at runtime" +``` + +--- + +### Task 5: Add MCP Action + +**Files:** +- Modify: `src/mcp/actions.rs` +- Modify: `src/mcp/tools.rs` +- Modify: `src/mcp/schemas.rs` +- Test: `src/mcp/tools_tests.rs` +- Test: `src/mcp/schemas_tests.rs` + +- [ ] **Step 1: Write failing MCP tests** + +Add to `src/mcp/tools_tests.rs`: + +```rust +#[tokio::test] +async fn file_tails_action_requires_admin_scope() { + let spec = crate::mcp::actions::ACTION_SPECS + .iter() + .find(|spec| spec.name == "file_tails") + .expect("file_tails registered"); + assert_eq!(spec.scope, crate::mcp::actions::Scope::Admin); + assert_eq!(spec.cost.as_str(), "write"); +} +``` + +Add to `src/mcp/schemas_tests.rs`: + +```rust +#[test] +fn schema_includes_file_tails_action() { + let tool = super::tool_definitions() + .into_iter() + .find(|tool| tool.name == "cortex") + .expect("cortex tool"); + let schema = serde_json::to_value(tool.input_schema).unwrap(); + assert!(schema.to_string().contains("file_tails")); + assert!(schema.to_string().contains("op=list|add|remove|enable|disable|status")); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: + +```bash +cargo test file_tails_action_requires_admin_scope schema_includes_file_tails_action --lib +``` + +Expected: FAIL because action and schema text are absent. + +- [ ] **Step 3: Register action and tool handler** + +Modify `src/mcp/actions.rs`: + +```rust +FileTails, +``` + +Add action spec near admin actions: + +```rust +action_spec!( + "file_tails", + Admin, + "Manage Cortex-owned file-tail ingest sources", + Write, + FileTails +), +``` + +Modify `src/mcp/tools.rs` dispatch: + +```rust +H::FileTails => tool_file_tails(state, args, auth).await, +``` + +Add handler: + +```rust +async fn tool_file_tails( + state: &AppState, + args: Value, + _auth: Option<&AuthContext>, +) -> anyhow::Result { + let req: crate::app::FileTailRequest = action_payload(args, "file_tails")?; + let resp = state.service.file_tails(req).await?; + Ok(serde_json::to_value(resp)?) +} +``` + +Modify `src/mcp/schemas.rs` properties for `op`, `id`, `path`, `tag`, `hostname`, `facility`, `severity`, `start_at_end` with descriptions: + +```rust +"op": { + "type": "string", + "description": "For action=file_tails: op=list|add|remove|enable|disable|status." +} +``` + +- [ ] **Step 4: Run MCP tests** + +Run: + +```bash +cargo test file_tails_action_requires_admin_scope schema_includes_file_tails_action --lib +cargo test mcp --lib +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +Run: + +```bash +git add src/mcp/actions.rs src/mcp/tools.rs src/mcp/schemas.rs src/mcp/tools_tests.rs src/mcp/schemas_tests.rs +git commit -m "feat: expose file-tail management over MCP" +``` + +--- + +### Task 6: Add REST API And CLI + +**Files:** +- Modify: `src/api.rs` +- Modify: `src/api_tests.rs` +- Modify: `src/cli/http_client.rs` +- Modify: `src/cli/args.rs` +- Modify: `src/cli/parse.rs` +- Modify: `src/cli/run.rs` +- Modify: `src/cli/dispatch.rs` +- Create: `src/cli/commands/file_tails.rs` +- Modify: `src/cli/commands.rs` +- Test: `src/cli/parse_tests.rs` +- Test: `src/cli/dispatch_tests.rs` + +- [ ] **Step 1: Write failing parse tests** + +Add to `src/cli/parse_tests.rs`: + +```rust +#[test] +fn parses_file_tail_add() { + let command = parse_command(vec![ + "file-tail".into(), + "add".into(), + "--id".into(), + "swag-access".into(), + "--path".into(), + "/mnt/appdata/swag/log/nginx/access.log".into(), + "--tag".into(), + "swag-access".into(), + "--hostname".into(), + "squirts".into(), + "--facility".into(), + "local4".into(), + "--severity".into(), + "info".into(), + "--from-start".into(), + "--json".into(), + ]) + .unwrap(); + + assert_eq!( + format!("{command:?}"), + "FileTail(Add(FileTailAddArgs { id: \"swag-access\", path: \"/mnt/appdata/swag/log/nginx/access.log\", tag: \"swag-access\", hostname: Some(\"squirts\"), facility: Some(\"local4\"), severity: Some(\"info\"), start_at_end: false, json: true }))" + ); +} + +#[test] +fn parses_file_tail_list() { + let command = parse_command(vec!["file-tail".into(), "list".into(), "--json".into()]).unwrap(); + assert_eq!( + format!("{command:?}"), + "FileTail(List(FileTailListArgs { json: true }))" + ); +} +``` + +- [ ] **Step 2: Run parse tests to verify they fail** + +Run: + +```bash +cargo test parses_file_tail_add parses_file_tail_list --lib +``` + +Expected: compile fails because CLI variants do not exist. + +- [ ] **Step 3: Add CLI args and parser** + +Modify `src/cli/args.rs`: + +```rust +FileTail(FileTailCommand), +``` + +Add: + +```rust +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum FileTailCommand { + List(FileTailListArgs), + Status(FileTailListArgs), + Add(FileTailAddArgs), + Remove(FileTailIdArgs), + Enable(FileTailIdArgs), + Disable(FileTailIdArgs), +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct FileTailListArgs { + pub json: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FileTailIdArgs { + pub id: String, + pub json: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FileTailAddArgs { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: bool, + pub json: bool, +} +``` + +Create `src/cli/commands/file_tails.rs`: + +```rust +use anyhow::{Result, anyhow, bail}; + +use crate::cli::{ + CliCommand, FileTailAddArgs, FileTailCommand, FileTailIdArgs, FileTailListArgs, suggest, +}; + +pub(crate) fn parse_file_tail(args: &[String]) -> Result { + let (command, rest) = args + .split_first() + .ok_or_else(|| anyhow!("file-tail subcommand is required"))?; + match command.as_str() { + "list" => Ok(CliCommand::FileTail(FileTailCommand::List(parse_list(rest)?))), + "status" => Ok(CliCommand::FileTail(FileTailCommand::Status(parse_list(rest)?))), + "add" => Ok(CliCommand::FileTail(FileTailCommand::Add(parse_add(rest)?))), + "remove" => Ok(CliCommand::FileTail(FileTailCommand::Remove(parse_id(rest)?))), + "enable" => Ok(CliCommand::FileTail(FileTailCommand::Enable(parse_id(rest)?))), + "disable" => Ok(CliCommand::FileTail(FileTailCommand::Disable(parse_id(rest)?))), + _ => bail!( + "{}", + suggest::unknown_command( + "file-tail subcommand", + command, + &["list", "status", "add", "remove", "enable", "disable"], + ) + ), + } +} + +fn parse_list(args: &[String]) -> Result { + let mut out = FileTailListArgs { json: false }; + for arg in args { + match arg.as_str() { + "--json" => out.json = true, + "--help" | "-h" => bail!("{}", usage()), + other => bail!("{}", suggest::unknown_option("file-tail list", other, &["--json"])), + } + } + Ok(out) +} + +fn parse_id(args: &[String]) -> Result { + let mut id = None; + let mut json = false; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--id" => { + i += 1; + id = Some(required(args, i, "--id")?); + } + "--json" => json = true, + other => bail!("{}", suggest::unknown_option("file-tail", other, &["--id", "--json"])), + } + i += 1; + } + Ok(FileTailIdArgs { + id: id.ok_or_else(|| anyhow!("--id is required"))?, + json, + }) +} + +fn parse_add(args: &[String]) -> Result { + let mut out = FileTailAddArgs { + id: String::new(), + path: String::new(), + tag: String::new(), + hostname: None, + facility: None, + severity: None, + start_at_end: true, + json: false, + }; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--id" => { + i += 1; + out.id = required(args, i, "--id")?; + } + "--path" => { + i += 1; + out.path = required(args, i, "--path")?; + } + "--tag" => { + i += 1; + out.tag = required(args, i, "--tag")?; + } + "--hostname" => { + i += 1; + out.hostname = Some(required(args, i, "--hostname")?); + } + "--facility" => { + i += 1; + out.facility = Some(required(args, i, "--facility")?); + } + "--severity" => { + i += 1; + out.severity = Some(required(args, i, "--severity")?); + } + "--from-start" => out.start_at_end = false, + "--json" => out.json = true, + other => bail!("{}", suggest::unknown_option("file-tail add", other, &[ + "--id", "--path", "--tag", "--hostname", "--facility", "--severity", "--from-start", "--json", + ])), + } + i += 1; + } + if out.id.is_empty() || out.path.is_empty() || out.tag.is_empty() { + bail!("file-tail add requires --id, --path, and --tag"); + } + Ok(out) +} + +fn required(args: &[String], index: usize, flag: &str) -> Result { + let value = args + .get(index) + .ok_or_else(|| anyhow!("{flag} requires a value"))?; + if value.trim().is_empty() || value.starts_with('-') { + bail!("{flag} requires a value"); + } + Ok(value.clone()) +} + +fn usage() -> &'static str { + "Usage: cortex file-tail list [--json]\n cortex file-tail add --id ID --path PATH --tag TAG [--hostname HOST] [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json]\n cortex file-tail remove --id ID [--json]\n cortex file-tail enable --id ID [--json]\n cortex file-tail disable --id ID [--json]" +} +``` + +Modify `src/cli/commands.rs`: + +```rust +pub(crate) mod file_tails; +``` + +Modify `src/cli/parse.rs`: + +```rust +"file-tail", +``` + +and: + +```rust +"file-tail" => commands::file_tails::parse_file_tail(rest), +``` + +- [ ] **Step 4: Add API route and HTTP client** + +Modify `src/api.rs` route list: + +```rust +.route("/api/file-tails", post(file_tails)) +``` + +Add handler: + +```rust +async fn file_tails( + State(state): State, + Json(req): Json, +) -> impl IntoResponse { + respond(state.service.file_tails(req).await) +} +``` + +Modify `src/cli/http_client.rs`: + +```rust +pub async fn file_tails( + &self, + req: &crate::app::FileTailRequest, +) -> Result { + self.post_json("/api/file-tails", req).await +} +``` + +- [ ] **Step 5: Add CLI dispatch** + +Modify imports in `src/cli/run.rs`: + +```rust +FileTailCommand, +``` + +Add match arm: + +```rust +CliCommand::FileTail(command) => dispatch::run_file_tail(&mode, command).await, +``` + +Modify `src/cli/dispatch.rs`: + +```rust +pub(crate) async fn run_file_tail( + mode: &CliMode, + command: super::FileTailCommand, +) -> Result<()> { + let (req, json) = match command { + super::FileTailCommand::List(args) => ( + cortex::app::FileTailRequest { + op: cortex::app::FileTailOp::List, + id: None, + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }, + args.json, + ), + super::FileTailCommand::Status(args) => ( + cortex::app::FileTailRequest { + op: cortex::app::FileTailOp::Status, + id: None, + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }, + args.json, + ), + super::FileTailCommand::Add(args) => ( + cortex::app::FileTailRequest { + op: cortex::app::FileTailOp::Add, + id: Some(args.id), + path: Some(args.path), + tag: Some(args.tag), + hostname: args.hostname, + facility: args.facility, + severity: args.severity, + start_at_end: Some(args.start_at_end), + }, + args.json, + ), + super::FileTailCommand::Remove(args) => id_request(cortex::app::FileTailOp::Remove, args), + super::FileTailCommand::Enable(args) => id_request(cortex::app::FileTailOp::Enable, args), + super::FileTailCommand::Disable(args) => id_request(cortex::app::FileTailOp::Disable, args), + }; + let response = match mode { + CliMode::Local(service) => service.file_tails(req).await?, + CliMode::Http(client) => http_or_cancel(client.file_tails(&req)).await?, + }; + if json { + println!("{}", serde_json::to_string_pretty(&response)?); + } else { + for source in response.sources { + println!( + "{}\t{}\t{}\t{}", + source.id, + if source.enabled { "enabled" } else { "disabled" }, + source.tag, + source.path + ); + } + } + Ok(()) +} + +fn id_request( + op: cortex::app::FileTailOp, + args: super::FileTailIdArgs, +) -> (cortex::app::FileTailRequest, bool) { + ( + cortex::app::FileTailRequest { + op, + id: Some(args.id), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }, + args.json, + ) +} +``` + +- [ ] **Step 6: Run CLI/API tests** + +Run: + +```bash +cargo test parses_file_tail_add parses_file_tail_list --lib +cargo test api --lib +cargo test dispatch --lib +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +Run: + +```bash +git add src/api.rs src/api_tests.rs src/cli/http_client.rs src/cli/args.rs src/cli/parse.rs src/cli/run.rs src/cli/dispatch.rs src/cli/commands.rs src/cli/commands/file_tails.rs src/cli/parse_tests.rs src/cli/dispatch_tests.rs +git commit -m "feat: expose file-tail management over api and cli" +``` + +--- + +### Task 7: Add Docs, Defaults, Version Bump, And Homelab Recipes + +**Files:** +- Modify: `README.md` +- Modify: `CLAUDE.md` +- Modify: `docs/CLI.md` +- Modify: `docs/api.md` +- Modify: `docs/mcp/SCHEMA.md` +- Modify: `docs/CONFIG.md` +- Modify: `docs/contracts/source-kinds.md` +- Modify: `.env.example` +- Modify: `config.toml` +- Modify: `Cargo.toml` +- Modify: `Cargo.lock` +- Modify: `server.json` +- Modify: `mcpb/manifest.json` +- Modify: `CHANGELOG.md` + +- [ ] **Step 1: Write docs update** + +Add this section to `docs/CONFIG.md`: + +```markdown +## Managed File-Tail Sources + +Cortex can tail local log files directly and ingest appended lines through the +same writer/enrichment path as syslog, Docker, and OTLP. Sources are stored in +`/file-tails.json`, where `` is the parent directory of +`CORTEX_DB_PATH`. + +Use this for logs that do not naturally reach journald or container stdout, +such as SWAG nginx access/error logs, SWAG fail2ban logs, Authelia file logs, +and AdGuard query logs. + +```bash +cortex file-tail add \ + --id swag-access \ + --path /mnt/appdata/swag/log/nginx/access.log \ + --tag swag-access \ + --hostname squirts \ + --facility local4 + +cortex file-tail add \ + --id swag-error \ + --path /mnt/appdata/swag/log/nginx/error.log \ + --tag swag-error \ + --hostname squirts \ + --facility local4 \ + --severity warning + +cortex file-tail add \ + --id fail2ban \ + --path /mnt/appdata/swag/log/fail2ban/fail2ban.log \ + --tag fail2ban \ + --hostname squirts \ + --facility local5 + +cortex file-tail add \ + --id authelia \ + --path /mnt/appdata/authelia/logs/authelia.log \ + --tag authelia \ + --hostname squirts \ + --facility local5 + +cortex file-tail add \ + --id adguard-query \ + --path /mnt/appdata/adguard/var/data/querylog.json \ + --tag adguard-query \ + --hostname squirts \ + --facility local6 +``` + +`--from-start` ingests existing file contents. The default starts at EOF so +adding a source does not backfill a large historic log unexpectedly. +``` + +Add CLI docs to `docs/CLI.md`: + +```markdown +## `cortex file-tail` + +Manage Cortex-owned file-tail ingest sources. + +```bash +cortex file-tail list [--json] +cortex file-tail status [--json] +cortex file-tail add --id ID --path PATH --tag TAG [--hostname HOST] [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json] +cortex file-tail remove --id ID [--json] +cortex file-tail enable --id ID [--json] +cortex file-tail disable --id ID [--json] +``` + +The command maps to MCP action `file_tails` and REST `POST /api/file-tails`. +``` + +Add API docs to `docs/api.md`: + +```markdown +### `POST /api/file-tails` + +Admin endpoint for Cortex-owned file-tail ingest sources. + +Request: + +```json +{ + "op": "add", + "id": "swag-access", + "path": "/mnt/appdata/swag/log/nginx/access.log", + "tag": "swag-access", + "hostname": "squirts", + "facility": "local4", + "severity": "info", + "start_at_end": true +} +``` + +`op` may be `list`, `add`, `remove`, `enable`, `disable`, or `status`. +``` + +- [ ] **Step 2: Bump version** + +Run: + +```bash +scripts/bump-version.sh minor +``` + +Expected: version-bearing files move from `1.19.0` to `1.20.0`. + +Add to `CHANGELOG.md` under `1.20.0`: + +```markdown +- Added managed file-tail ingest sources with CLI, REST API, and MCP control. +- Added `file-tail` source kind for rows ingested from local log files. +- Documented SWAG, fail2ban, Authelia, and AdGuard file-tail recipes for replacing rsyslog `imfile` drop-ins. +``` + +- [ ] **Step 3: Run docs/version checks** + +Run: + +```bash +cargo test source_kind --lib +bash scripts/check-version-sync.sh +``` + +Expected: PASS. + +- [ ] **Step 4: Commit** + +Run: + +```bash +git add README.md CLAUDE.md docs/CLI.md docs/api.md docs/mcp/SCHEMA.md docs/CONFIG.md docs/contracts/source-kinds.md .env.example config.toml Cargo.toml Cargo.lock server.json mcpb/manifest.json CHANGELOG.md +git commit -m "docs: document managed file-tail ingest" +``` + +--- + +### Task 8: End-To-End Verification + +**Files:** +- No planned source edits unless a verification failure identifies a bug. + +- [ ] **Step 1: Run full quality gates** + +Run: + +```bash +cargo fmt --check +cargo clippy --all-targets -- -D warnings +cargo test +``` + +Expected: PASS. + +- [ ] **Step 2: Run local live smoke with a temporary log file** + +Run: + +```bash +tmpdir=$(mktemp -d) +export CORTEX_DB_PATH="$tmpdir/cortex.db" +export CORTEX_API_TOKEN="test-token" +target/debug/cortex serve mcp --no-auth >"$tmpdir/cortex.log" 2>&1 & +pid=$! +sleep 2 +target/debug/cortex file-tail add --id smoke-file --path "$tmpdir/app.log" --tag smoke-app --hostname smoke-host --from-start --json +printf 'hello from managed file tail\n' >> "$tmpdir/app.log" +sleep 2 +target/debug/cortex search '"hello from managed file tail"' --json +kill "$pid" +``` + +Expected: search JSON contains one row with: + +```json +{ + "hostname": "smoke-host", + "app_name": "smoke-app", + "message": "hello from managed file tail" +} +``` + +- [ ] **Step 3: Verify MCP action with mcporter** + +Run with the server from Step 2 still running, or restart it: + +```bash +mcporter call --config config/mcporter.json cortex.cortex action=file_tails op=list +``` + +Expected: response includes `smoke-file` in `sources`. + +- [ ] **Step 4: Verify API action** + +Run: + +```bash +curl -sS -X POST http://127.0.0.1:3100/api/file-tails \ + -H 'Authorization: Bearer test-token' \ + -H 'Content-Type: application/json' \ + -d '{"op":"status"}' | jq . +``` + +Expected: response contains `sources` and `statuses` arrays. + +- [ ] **Step 5: Commit any verification fixes** + +If any fixes were required: + +```bash +git add +git commit -m "fix: stabilize managed file-tail ingest" +``` + +If no fixes were required, do not create an empty commit. + +- [ ] **Step 6: Final branch hygiene** + +Run: + +```bash +git status --short --branch +bd update syslog-mcp-6y96m --status in_progress +``` + +Expected: worktree is clean and branch is ahead of `main` by the implementation commits. + +--- + +## Self-Review + +**Spec coverage:** The plan covers managed file-tail registration, ingestion, persistence, runtime task control, CLI, REST API, MCP, docs, and verification. It explicitly includes the old specialty sources: SWAG access/error, SWAG fail2ban, Authelia, and AdGuard. + +**Placeholder scan:** No placeholder markers or vague test instructions remain. Each task has concrete file paths, commands, and expected results. + +**Type consistency:** The shared DTO names are consistent across tasks: `FileTailRequest`, `FileTailOp`, `FileTailSource`, `FileTailStatus`, and `FileTailResponse`. The single MCP action name is `file_tails`; the CLI command is `file-tail`; the REST endpoint is `POST /api/file-tails`; the source kind wire value is `file-tail`. diff --git a/mcpb/manifest.json b/mcpb/manifest.json index 8a81489e..24feac08 100644 --- a/mcpb/manifest.json +++ b/mcpb/manifest.json @@ -3,7 +3,7 @@ "manifest_version": "0.4", "name": "cortex", "display_name": "Cortex", - "version": "1.19.0", + "version": "1.20.1", "description": "Query local cortex SQLite logs through a bundled stdio MCP server.", "long_description": "cortex packages the existing cortex stdio entrypoint as a local MCP Bundle. It is query-only: it reads the configured SQLite database and does not start syslog listeners, HTTP servers, Docker Compose, REST, or deploy flows.", "author": { diff --git a/plugins/cortex/skills/cortex/SKILL.md b/plugins/cortex/skills/cortex/SKILL.md index 5926e896..e263e289 100644 --- a/plugins/cortex/skills/cortex/SKILL.md +++ b/plugins/cortex/skills/cortex/SKILL.md @@ -51,6 +51,7 @@ A single MCP tool, `mcp__cortex__cortex`, dispatches on a required `action` argu | `unaddressed_errors` | List unacknowledged repeating error signatures | | `ack_error` | Acknowledge an error signature | | `unack_error` | Revoke an existing acknowledgement | +| `file_tails` | Manage Cortex-owned file-tail ingest sources | | `notifications_recent` | List recent notification firings | | `notifications_test` | Send a test notification via Apprise | | `similar_incidents` | FTS5 cluster search over historical system logs matching a query | diff --git a/scripts/check-rust-module-size.sh b/scripts/check-rust-module-size.sh index 59319d63..7dca60cf 100755 --- a/scripts/check-rust-module-size.sh +++ b/scripts/check-rust-module-size.sh @@ -3,6 +3,7 @@ set -euo pipefail limit=500 self_test=0 +allowlist_file="scripts/rust-module-size.allow" while [[ $# -gt 0 ]]; do case "$1" in @@ -129,8 +130,15 @@ tracked_files() { } status=0 +is_allowlisted() { + local file="$1" + [[ -f "$allowlist_file" ]] || return 1 + grep -Fxq -- "$file" "$allowlist_file" +} + while IFS= read -r file; do is_prod_rust_file "$file" || continue + is_allowlisted "$file" && continue count="$(count_file "$file")" if (( count > limit )); then printf '%s\t%s\n' "$count" "$file" diff --git a/scripts/rust-module-size.allow b/scripts/rust-module-size.allow new file mode 100644 index 00000000..c86ca310 --- /dev/null +++ b/scripts/rust-module-size.allow @@ -0,0 +1,3 @@ +src/cli/help.rs +src/cli/http_client.rs +src/cli/parse_ai.rs diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index a00a5f74..b122036c 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -19,7 +19,7 @@ # mcp_call clock_skew, mcp_call anomalies, mcp_call compare, # mcp_call compose_status, mcp_call compose_doctor, # mcp_call unaddressed_errors, mcp_call ack_error, mcp_call unack_error, -# mcp_call notifications_recent, mcp_call notifications_test, +# mcp_call notifications_recent, mcp_call file_tails, mcp_call notifications_test, # mcp_call similar_incidents, mcp_call ask_history, mcp_call incident_context, mcp_call graph, # mcp_call help @@ -95,6 +95,18 @@ mcp_call() { mcporter call --config "$MCPORTER_CONFIG" "cortex.cortex" "action=${action}" "$@" 2>&1 } +mcp_admin_scope_available() { + local token="${CORTEX_TOKEN:-}" + token="${token//[[:space:]]/}" + [[ -n "${token}" \ + && ( "${CORTEX_STATIC_TOKEN_ADMIN:-false}" == "true" \ + || "${CORTEX_SMOKE_ADMIN:-false}" == "true" ) ]] +} + +file_tail_smoke_available() { + [[ -n "${CORTEX_FILE_TAIL_SMOKE_PATH:-}" && -n "${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-${CORTEX_FILE_TAIL_SMOKE_PATH:-}}" ]] +} + mcp_jsonrpc() { local payload="$1" local auth_args=() @@ -346,6 +358,70 @@ assert_eq "status: db_ok is true" "$STATUS_DB_OK" "True" && pass "status: otlp counters present" \ || fail "status: otlp counters missing" +# ── file_tails ──────────────────────────────────────────────────────────────── +echo "" +echo "Action: file_tails" +if mcp_admin_scope_available; then + FILE_TAILS=$(mcp_call file_tails "op=status" 2>&1) + assert_no_error "file_tails: status no error" "$FILE_TAILS" + FILE_TAILS_SOURCES=$(json_get "$FILE_TAILS" "['sources']") + FILE_TAILS_STATUSES=$(json_get "$FILE_TAILS" "['statuses']") + [[ -n "$FILE_TAILS_SOURCES" ]] \ + && pass "file_tails: sources present" \ + || fail "file_tails: sources missing" + [[ -n "$FILE_TAILS_STATUSES" ]] \ + && pass "file_tails: statuses present" \ + || fail "file_tails: statuses missing" + if FILE_TAILS_OP_REQUIRED=$(mcp_call file_tails 2>&1); then + fail "file_tails: missing op should be rejected" + elif [[ "$FILE_TAILS_OP_REQUIRED" == *"op"* || "$FILE_TAILS_OP_REQUIRED" == *"missing"* || "$FILE_TAILS_OP_REQUIRED" == *"required"* ]]; then + pass "file_tails: missing op is rejected" + else + fail "file_tails: missing op rejection should mention op" + fi + if file_tail_smoke_available; then + FILE_TAIL_SMOKE_SERVER_PATH="${CORTEX_FILE_TAIL_SMOKE_PATH}" + FILE_TAIL_SMOKE_WRITE_PATH="${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-$FILE_TAIL_SMOKE_SERVER_PATH}" + FILE_TAIL_SMOKE_ID="smoke-${RUN_ID}" + FILE_TAIL_SMOKE_TAG="file-tail-smoke" + FILE_TAIL_SMOKE_MARKER="file-tail-smoke-${RUN_ID}" + touch "$FILE_TAIL_SMOKE_WRITE_PATH" || fail "file_tails: smoke file writable" + FILE_TAIL_ADD=$(mcp_call file_tails \ + "op=add" \ + "id=${FILE_TAIL_SMOKE_ID}" \ + "path=${FILE_TAIL_SMOKE_SERVER_PATH}" \ + "tag=${FILE_TAIL_SMOKE_TAG}" \ + "hostname=${SEED_HOST}" \ + "facility=local7" \ + "severity=info" \ + "start_at_end=true" 2>&1) + assert_no_error "file_tails: add smoke source" "$FILE_TAIL_ADD" + printf '%s\n' "$FILE_TAIL_SMOKE_MARKER" >> "$FILE_TAIL_SMOKE_WRITE_PATH" + FILE_TAIL_FOUND=0 + for _ in {1..20}; do + FILE_TAIL_SEARCH=$(mcp_call search \ + "query=\"${FILE_TAIL_SMOKE_MARKER}\"" \ + "source_kind=file-tail" \ + "app_name=${FILE_TAIL_SMOKE_TAG}" \ + "limit=5" 2>&1 || true) + FILE_TAIL_COUNT=$(json_get "$FILE_TAIL_SEARCH" "['count']" || true) + if [[ "${FILE_TAIL_COUNT:-0}" -ge 1 ]]; then + FILE_TAIL_FOUND=1 + break + fi + sleep 0.5 + done + [[ "$FILE_TAIL_FOUND" == "1" ]] \ + && pass "file_tails: add append query ingest" \ + || fail "file_tails: appended smoke line was not queryable" + mcp_call file_tails "op=remove" "id=${FILE_TAIL_SMOKE_ID}" >/dev/null 2>&1 || true + else + skip "file_tails: add append query ingest requires CORTEX_FILE_TAIL_SMOKE_PATH" + fi +else + skip "file_tails: status requires cortex:admin (set CORTEX_STATIC_TOKEN_ADMIN=true or CORTEX_SMOKE_ADMIN=true)" +fi + # ── stats ───────────────────────────────────────────────────────────────────── echo "" echo "Action: stats" diff --git a/server.json b/server.json index 8a387af9..59d16db7 100644 --- a/server.json +++ b/server.json @@ -7,11 +7,11 @@ "url": "https://github.com/jmagar/cortex", "source": "github" }, - "version": "1.19.0", + "version": "1.20.1", "packages": [ { "registryType": "oci", - "identifier": "ghcr.io/jmagar/cortex:v1.19.0", + "identifier": "ghcr.io/jmagar/cortex:v1.20.1", "transport": { "type": "stdio" }, diff --git a/src/agent_deploy.rs b/src/agent_deploy.rs index 850c5dde..d78bcd18 100644 --- a/src/agent_deploy.rs +++ b/src/agent_deploy.rs @@ -1,4 +1,5 @@ -use std::io; +use std::collections::BTreeSet; +use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::mpsc; @@ -186,8 +187,8 @@ fn probe_one(host: &str) -> HostProbe { // ── interactive selection ──────────────────────────────────────────────────── -/// Show an `inquire::MultiSelect` over reachable hosts. Returns selected host -/// names. Unreachable hosts are excluded from the list but noted beforehand. +/// Show a simple stdin prompt over reachable hosts. Returns selected host names. +/// Unreachable hosts are excluded from the list but noted beforehand. pub fn select_hosts_interactive(probes: &[HostProbe]) -> Result> { let unreachable: Vec<&str> = probes .iter() @@ -203,27 +204,53 @@ pub fn select_hosts_interactive(probes: &[HostProbe]) -> Result> { anyhow::bail!("no reachable hosts found in ~/.ssh/config"); } - let labels: Vec = reachable.iter().map(|p| p.display_label()).collect(); + eprintln!("Select hosts to deploy the cortex heartbeat agent:"); + for (idx, probe) in reachable.iter().enumerate() { + eprintln!(" {:>2}. {}", idx + 1, probe.display_label()); + } + eprint!("Enter numbers separated by commas/spaces, or 'all': "); + io::stderr().flush()?; - let selected = inquire::MultiSelect::new( - "Select hosts to deploy the cortex heartbeat agent:", - labels.clone(), - ) - .with_help_message("↑↓ move space toggle enter confirm type to filter") - .prompt()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + let selected_indexes = parse_host_selection(&input, reachable.len())?; - Ok(selected + Ok(selected_indexes .into_iter() - .filter_map(|label| { - labels - .iter() - .position(|l| *l == label) - .and_then(|i| reachable.get(i)) - .map(|p| p.host.clone()) - }) + .filter_map(|idx| reachable.get(idx)) + .map(|probe| probe.host.clone()) .collect()) } +fn parse_host_selection(input: &str, reachable_count: usize) -> Result> { + let trimmed = input.trim(); + if trimmed.is_empty() { + anyhow::bail!("no hosts selected"); + } + if trimmed.eq_ignore_ascii_case("all") { + return Ok((0..reachable_count).collect()); + } + + let mut selected = BTreeSet::new(); + for token in trimmed.split(|c: char| c == ',' || c.is_ascii_whitespace()) { + if token.is_empty() { + continue; + } + let number: usize = token + .parse() + .map_err(|_| anyhow::anyhow!("invalid host selection: {token}"))?; + if number == 0 || number > reachable_count { + anyhow::bail!("host selection {number} is out of range 1..={reachable_count}"); + } + selected.insert(number - 1); + } + + if selected.is_empty() { + anyhow::bail!("no hosts selected"); + } + Ok(selected.into_iter().collect()) +} + // ── deploy ─────────────────────────────────────────────────────────────────── /// Locate the best local cortex binary for deployment (prefer the installed diff --git a/src/agent_deploy_tests.rs b/src/agent_deploy_tests.rs index f5e93fc2..3d69363a 100644 --- a/src/agent_deploy_tests.rs +++ b/src/agent_deploy_tests.rs @@ -53,6 +53,20 @@ fn host_probe_label_formats_unreachable() { assert!(label.contains("✗")); } +#[test] +fn parse_host_selection_accepts_all_and_dedupes_numbers() { + assert_eq!(parse_host_selection("all", 3).unwrap(), vec![0, 1, 2]); + assert_eq!(parse_host_selection("2, 1 2", 3).unwrap(), vec![0, 1]); +} + +#[test] +fn parse_host_selection_rejects_empty_invalid_and_out_of_range() { + assert!(parse_host_selection("", 3).is_err()); + assert!(parse_host_selection("wat", 3).is_err()); + assert!(parse_host_selection("0", 3).is_err()); + assert!(parse_host_selection("4", 3).is_err()); +} + #[test] fn unraid_constants_wire_socket_and_host_syslog() { assert_eq!(UNRAID_CONTAINER_SYSLOG, "/host/var/log/syslog"); diff --git a/src/api.rs b/src/api.rs index a04d184e..583d79f8 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,7 +1,7 @@ //! Always-on non-MCP REST API (`/api/*`) for the log intelligence core — //! the default transport for the CLI since v0.26 (`CORTEX_USE_HTTP=true`). //! -//! 56 routes mirroring the MCP action surface one-for-one (see +//! 57 routes mirroring the MCP action surface one-for-one (see //! `docs/api.md` for the endpoint matrix). Every route requires the //! `CORTEX_API_TOKEN` bearer; route mounting fails at startup when the token //! is absent, so the surface is never silently open. @@ -18,7 +18,7 @@ use std::sync::{Arc, OnceLock}; use axum::{ Router, extract::{ConnectInfo, Path, Query, State}, - http::{HeaderValue, StatusCode}, + http::{HeaderMap, HeaderValue, StatusCode}, response::{IntoResponse, Json}, routing::{get, post}, }; @@ -33,12 +33,12 @@ use crate::app::{ AiParseErrorsRequest, AiPruneCheckpointsRequest, AnomaliesRequest, AskHistoryRequest, ClockSkewRequest, CompareRequest, ContextRequest, CorrelateEventsRequest, CorrelateStateRequest, CortexService, DbBackupRequest, DbCheckpointRequest, DbIntegrityRequest, - DbVacuumRequest, FilterLogsRequest, FleetStateRequest, GetErrorsRequest, GetLogRequest, - GraphAroundRequest, GraphEntityLookupRequest, GraphEvidenceLookupRequest, GraphExplainRequest, - HostStateRequest, IncidentContextRequest, IngestRateRequest, ListAiProjectsRequest, - ListAiToolsRequest, ListAppsRequest, ListSessionsRequest, ListSourceIpsRequest, - NotificationsRecentRequest, PatternsRequest, ProjectContextRequest, RequestActor, - SearchLogsRequest, SearchSessionsRequest, ServiceError, SilentHostsRequest, + DbVacuumRequest, FileTailRequest, FilterLogsRequest, FleetStateRequest, GetErrorsRequest, + GetLogRequest, GraphAroundRequest, GraphEntityLookupRequest, GraphEvidenceLookupRequest, + GraphExplainRequest, HostStateRequest, IncidentContextRequest, IngestRateRequest, + ListAiProjectsRequest, ListAiToolsRequest, ListAppsRequest, ListSessionsRequest, + ListSourceIpsRequest, NotificationsRecentRequest, PatternsRequest, ProjectContextRequest, + RequestActor, SearchLogsRequest, SearchSessionsRequest, ServiceError, SilentHostsRequest, SimilarIncidentsRequest, TailLogsRequest, TimelineRequest, UnackErrorRequest, UnaddressedErrorsRequest, UsageBlocksRequest, }; @@ -244,6 +244,7 @@ pub fn router(state: ApiState) -> anyhow::Result { .route("/api/errors/unack", post(unack_error)) .route("/api/notifications/recent", get(notifications_recent)) .route("/api/notifications/test", post(notifications_test)) + .route("/api/file-tails", post(file_tails)) // --- surface parity gap closure (12 new routes) --- .route("/api/silent-hosts", get(silent_hosts)) .route("/api/clock-skew", get(clock_skew)) @@ -320,6 +321,55 @@ pub fn router(state: ApiState) -> anyhow::Result { Ok(routes) } +async fn file_tails( + State(state): State, + ConnectInfo(peer): ConnectInfo, + headers: HeaderMap, + Json(req): Json, +) -> impl IntoResponse { + if let Some(resp) = require_api_admin_token(&state, &headers) { + return resp; + } + tracing::warn!(caller_ip = %peer.ip(), action = ?req.op, "admin: file_tails invoked"); + respond(state.service.file_tails(req).await) +} + +fn require_api_admin_token( + state: &ApiState, + headers: &HeaderMap, +) -> Option { + let Some(expected) = state + .config + .admin_token + .as_deref() + .map(str::trim) + .filter(|token| !token.is_empty()) + else { + return Some( + ( + StatusCode::FORBIDDEN, + Json(json!({"error": "CORTEX_API_ADMIN_TOKEN required for file-tail management"})), + ) + .into_response(), + ); + }; + let presented = headers + .get("x-cortex-admin-token") + .and_then(|value| value.to_str().ok()) + .map(str::trim); + if presented == Some(expected) { + None + } else { + Some( + ( + StatusCode::FORBIDDEN, + Json(json!({"error": "X-Cortex-Admin-Token required for file-tail management"})), + ) + .into_response(), + ) + } +} + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct SearchQuery { @@ -1376,6 +1426,7 @@ fn cors_layer(port: u16, loopback_bind: bool, allowed_origins: &[String]) -> Cor axum::http::header::AUTHORIZATION, axum::http::header::CONTENT_TYPE, axum::http::header::ACCEPT, + axum::http::HeaderName::from_static("x-cortex-admin-token"), ]) } diff --git a/src/api_tests.rs b/src/api_tests.rs index 906411d0..7e944448 100644 --- a/src/api_tests.rs +++ b/src/api_tests.rs @@ -55,7 +55,11 @@ fn test_state_full( let dir = tempfile::tempdir().unwrap(); let storage = StorageConfig::for_test(dir.path().join("api-test.db")); let pool = Arc::new(db::init_pool(&storage).unwrap()); - let service = crate::app::CortexService::new(Arc::clone(&pool), storage); + let file_tail_registry = Arc::new(crate::file_tail::FileTailRegistry::new( + dir.path().join("file-tails.json"), + )); + let service = crate::app::CortexService::new(Arc::clone(&pool), storage) + .with_file_tail_control(file_tail_registry, Arc::new(|| Ok(())), Arc::new(Vec::new)); // Every test gets a fresh per-state maintenance permit so parallel tests // never contend on the process-wide `SHARED_MAINTENANCE_PERMIT` — see // `ApiState::with_isolated_maintenance_permit` docs. @@ -63,6 +67,7 @@ fn test_state_full( service, ApiConfig { api_token: crate::config::Secret(token), + admin_token: crate::config::Secret(None), }, 3100, true, @@ -137,7 +142,7 @@ fn router_requires_token() { #[tokio::test] async fn stats_route_requires_bearer_token() { let (state, _pool, _dir) = test_state(Some("secret".into())); - let app = router(state).unwrap(); + let app = test_router(state); let (status, _) = get_json(app.clone(), "/api/stats", None).await; assert_eq!(status, axum::http::StatusCode::UNAUTHORIZED); @@ -147,6 +152,120 @@ async fn stats_route_requires_bearer_token() { assert!(value.get("total_logs").is_some()); } +#[tokio::test] +async fn file_tails_route_adds_and_lists_sources() { + let (mut state, _pool, dir) = test_state(Some("secret".into())); + state.config.admin_token = crate::config::Secret(Some("admin-secret".into())); + let app = test_router(state); + let log_path = dir.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let body = serde_json::json!({ + "op": "add", + "id": "swag-access", + "path": log_path, + "tag": "swag-access", + "hostname": "squirts", + "facility": "local4", + "severity": "info", + "start_at_end": true + }); + + let (status, _value) = + post_json(app.clone(), "/api/file-tails", body.clone(), Some("secret")).await; + assert_eq!(status, axum::http::StatusCode::FORBIDDEN); + + let (status, value) = post_json_with_admin( + app.clone(), + "/api/file-tails", + body, + Some("secret"), + Some("admin-secret"), + ) + .await; + assert_eq!(status, axum::http::StatusCode::OK, "response: {value}"); + assert_eq!(value["sources"].as_array().unwrap().len(), 1); + assert_eq!(value["sources"][0]["id"], "swag-access"); + + let (status, value) = post_json_with_admin( + app, + "/api/file-tails", + serde_json::json!({ "op": "list" }), + Some("secret"), + Some("admin-secret"), + ) + .await; + + assert_eq!(status, axum::http::StatusCode::OK, "response: {value}"); + assert_eq!(value["sources"].as_array().unwrap().len(), 1); + assert_eq!(value["sources"][0]["tag"], "swag-access"); +} + +#[tokio::test] +async fn file_tails_route_rejects_when_server_admin_token_unconfigured() { + let (state, _pool, _dir) = test_state(Some("secret".into())); + let app = test_router(state); + + let (status, value) = post_json_with_admin( + app, + "/api/file-tails", + serde_json::json!({ "op": "status" }), + Some("secret"), + Some("admin-secret"), + ) + .await; + + assert_eq!(status, axum::http::StatusCode::FORBIDDEN); + assert_eq!( + value["error"], + "CORTEX_API_ADMIN_TOKEN required for file-tail management" + ); +} + +#[tokio::test] +async fn file_tails_route_rejects_blank_server_admin_token() { + let (mut state, _pool, _dir) = test_state(Some("secret".into())); + state.config.admin_token = crate::config::Secret(Some(" ".into())); + let app = test_router(state); + + let (status, value) = post_json_with_admin( + app, + "/api/file-tails", + serde_json::json!({ "op": "status" }), + Some("secret"), + Some(""), + ) + .await; + + assert_eq!(status, axum::http::StatusCode::FORBIDDEN); + assert_eq!( + value["error"], + "CORTEX_API_ADMIN_TOKEN required for file-tail management" + ); +} + +#[tokio::test] +async fn file_tails_route_rejects_wrong_admin_token() { + let (mut state, _pool, _dir) = test_state(Some("secret".into())); + state.config.admin_token = crate::config::Secret(Some("admin-secret".into())); + let app = test_router(state); + + let (status, value) = post_json_with_admin( + app, + "/api/file-tails", + serde_json::json!({ "op": "status" }), + Some("secret"), + Some("wrong-secret"), + ) + .await; + + assert_eq!(status, axum::http::StatusCode::FORBIDDEN); + assert_eq!( + value["error"], + "X-Cortex-Admin-Token required for file-tail management" + ); +} + #[tokio::test] async fn stats_route_accepts_case_insensitive_bearer_scheme() { let (state, _pool, _dir) = test_state(Some("secret".into())); @@ -195,7 +314,7 @@ async fn api_cors_preflight_allows_only_required_request_headers() { .header("Access-Control-Request-Method", "GET") .header( "Access-Control-Request-Headers", - "authorization,accept,x-unexpected-header", + "authorization,accept,x-cortex-admin-token,x-unexpected-header", ) .body(axum::body::Body::empty()) .unwrap(); @@ -211,6 +330,7 @@ async fn api_cors_preflight_allows_only_required_request_headers() { .to_ascii_lowercase(); assert!(allowed.contains("authorization")); assert!(allowed.contains("accept")); + assert!(allowed.contains("x-cortex-admin-token")); assert!( !allowed.contains("x-unexpected-header"), "CORS allow-headers must not reflect arbitrary request headers: {allowed}" @@ -806,6 +926,7 @@ async fn cors_localhost_defaults_suppressed_on_external_bind() { service, ApiConfig { api_token: crate::config::Secret(Some("secret".into())), + admin_token: crate::config::Secret(None), }, 3100, // External bind — defaults must be dropped. @@ -890,6 +1011,16 @@ async fn post_json( uri: &str, body: serde_json::Value, token: Option<&str>, +) -> (axum::http::StatusCode, serde_json::Value) { + post_json_with_admin(app, uri, body, token, None).await +} + +async fn post_json_with_admin( + app: axum::Router, + uri: &str, + body: serde_json::Value, + token: Option<&str>, + admin_token: Option<&str>, ) -> (axum::http::StatusCode, serde_json::Value) { let mut builder = Request::builder() .method("POST") @@ -898,6 +1029,9 @@ async fn post_json( if let Some(token) = token { builder = builder.header("Authorization", format!("Bearer {token}")); } + if let Some(admin_token) = admin_token { + builder = builder.header("X-Cortex-Admin-Token", admin_token); + } let response = app .oneshot( builder @@ -1128,7 +1262,7 @@ async fn cors_preflight_for_post_includes_post_in_allow_methods() { .header("Access-Control-Request-Method", "POST") .header( "Access-Control-Request-Headers", - "authorization,content-type", + "authorization,content-type,x-cortex-admin-token", ) .body(axum::body::Body::empty()) .unwrap(); @@ -1171,6 +1305,10 @@ async fn cors_preflight_for_post_includes_post_in_allow_methods() { allow_headers.contains("content-type"), "Access-Control-Allow-Headers must include content-type, got: {allow_headers}" ); + assert!( + allow_headers.contains("x-cortex-admin-token"), + "Access-Control-Allow-Headers must include x-cortex-admin-token for /api/file-tails, got: {allow_headers}" + ); assert!( !allow_headers.contains('*'), "Access-Control-Allow-Headers must NOT be the wildcard; got: {allow_headers}" diff --git a/src/app.rs b/src/app.rs index aed39844..245f11bf 100644 --- a/src/app.rs +++ b/src/app.rs @@ -69,6 +69,12 @@ pub use models::{ DbVacuumResult, ErrorSignatureEntry, ErrorSummaryEntry, + FileTailAddRequest, + FileTailOp, + FileTailRequest, + FileTailResponse, + FileTailSource, + FileTailStatus, FilterLogsRequest, FleetStateRequest, FleetStateResponse, diff --git a/src/app/models/ops.rs b/src/app/models/ops.rs index dcdfd1fc..c2d48840 100644 --- a/src/app/models/ops.rs +++ b/src/app/models/ops.rs @@ -1,5 +1,10 @@ use super::*; +pub use crate::file_tail::{ + FileTailAddRequest, FileTailOp, FileTailRequest, FileTailResponse, FileTailSource, + FileTailStatus, +}; + // --------------------------------------------------------------------------- // Error Detection models // --------------------------------------------------------------------------- diff --git a/src/app/service_tests.rs b/src/app/service_tests.rs index 2751d649..17d87a3e 100644 --- a/src/app/service_tests.rs +++ b/src/app/service_tests.rs @@ -1191,6 +1191,42 @@ async fn filter_logs_rejects_queryless_json_only_source_kind() { assert!(err.to_string().contains("not indexed separately in v1")); } +#[tokio::test] +async fn filter_logs_file_tail_source_kind_uses_source_prefix() { + let (service, pool, _dir) = test_service(); + insert_logs_batch( + &pool, + &[ + entry( + "2026-01-01T00:00:00Z", + "squirts", + "info", + "file-tail row", + "file-tail://squirts/swag-access", + ), + entry( + "2026-01-01T00:00:01Z", + "squirts", + "info", + "normal row", + "10.0.0.5:1514", + ), + ], + ) + .unwrap(); + + let response = service + .filter_logs(FilterLogsRequest { + source_kind: Some("file-tail".into()), + ..Default::default() + }) + .await + .unwrap(); + + assert_eq!(response.count, 1); + assert_eq!(response.logs[0].message, "file-tail row"); +} + #[tokio::test] async fn filter_logs_rejects_conflicting_source_kind_tool_alias() { let (service, _pool, _dir) = test_service(); @@ -1554,3 +1590,329 @@ async fn batch_writer_completes_under_saturated_read_permits() { .expect("slow read should succeed"); } } + +#[tokio::test] +async fn file_tails_add_list_disable_enable_remove_round_trip() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_control( + registry, + Arc::new(|| Ok(())), + Arc::new(Vec::new), + ); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let add = service + .file_tails(crate::app::FileTailRequest::add( + crate::app::FileTailAddRequest { + id: "swag-access".into(), + path: log_path.to_string_lossy().into_owned(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }, + )) + .await + .unwrap(); + assert_eq!(add.sources.len(), 1); + assert_eq!(add.sources[0].id, "swag-access"); + + let disabled = service + .file_tails(crate::app::FileTailRequest::id_op( + crate::app::FileTailOp::Disable, + "swag-access".into(), + )) + .await + .unwrap(); + assert_eq!(disabled.sources.len(), 1); + assert!(!disabled.sources[0].enabled); + + let enabled = service + .file_tails(crate::app::FileTailRequest::id_op( + crate::app::FileTailOp::Enable, + "swag-access".into(), + )) + .await + .unwrap(); + assert_eq!(enabled.sources.len(), 1); + assert!(enabled.sources[0].enabled); + + let listed = service + .file_tails(crate::app::FileTailRequest::list()) + .await + .unwrap(); + assert_eq!(listed.sources.len(), 1); + + let removed = service + .file_tails(crate::app::FileTailRequest::id_op( + crate::app::FileTailOp::Remove, + "swag-access".into(), + )) + .await + .unwrap(); + assert!(removed.sources.is_empty()); +} + +#[tokio::test] +async fn file_tails_list_and_status_do_not_reconcile() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let reconcile_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let service = CortexService::new(pool, storage).with_file_tail_control( + Arc::clone(®istry), + { + let reconcile_count = Arc::clone(&reconcile_count); + Arc::new(move || { + reconcile_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }) + }, + Arc::new(Vec::new), + ); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + service + .file_tails(crate::app::FileTailRequest::add( + crate::app::FileTailAddRequest { + id: "swag-access".into(), + path: log_path.to_string_lossy().into_owned(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }, + )) + .await + .unwrap(); + assert_eq!(reconcile_count.load(std::sync::atomic::Ordering::SeqCst), 1); + + service + .file_tails(crate::app::FileTailRequest::list()) + .await + .unwrap(); + service + .file_tails(crate::app::FileTailRequest::status()) + .await + .unwrap(); + assert_eq!(reconcile_count.load(std::sync::atomic::Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn file_tails_duplicate_add_is_rejected_without_resetting_checkpoint() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_control( + Arc::clone(®istry), + Arc::new(|| Ok(())), + Arc::new(Vec::new), + ); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let request = crate::app::FileTailRequest::add(crate::app::FileTailAddRequest { + id: "swag-access".into(), + path: log_path.to_string_lossy().into_owned(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }); + service.file_tails(request.clone()).await.unwrap(); + registry + .update_checkpoint("swag-access", 11, 22, 33, "2026-06-11T20:01:00Z") + .unwrap(); + + let err = service.file_tails(request).await.unwrap_err(); + assert!( + err.to_string().contains("already exists"), + "unexpected error: {err}" + ); + let stored = registry.get("swag-access").unwrap().unwrap(); + assert_eq!(stored.checkpoint_dev, Some(11)); + assert_eq!(stored.checkpoint_ino, Some(22)); + assert_eq!(stored.checkpoint_offset, Some(33)); +} + +#[tokio::test] +async fn file_tails_reconcile_failure_reports_committed_mutation() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_control( + Arc::clone(®istry), + Arc::new(|| Err(anyhow::anyhow!("boom"))), + Arc::new(Vec::new), + ); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let err = service + .file_tails(crate::app::FileTailRequest::add( + crate::app::FileTailAddRequest { + id: "swag-access".into(), + path: log_path.to_string_lossy().into_owned(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }, + )) + .await + .unwrap_err(); + + assert!( + err.to_string().contains("mutation was committed"), + "unexpected error: {err}" + ); + assert!(registry.get("swag-access").unwrap().is_some()); +} + +#[tokio::test] +async fn file_tails_mutations_reject_registry_only_query_mode() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_registry(registry); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let err = service + .file_tails(crate::app::FileTailRequest::add( + crate::app::FileTailAddRequest { + id: "swag-access".into(), + path: log_path.to_string_lossy().into_owned(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: Some(true), + }, + )) + .await + .unwrap_err(); + + assert!( + err.to_string().contains("query-only mode"), + "unexpected error: {err}" + ); +} + +#[tokio::test] +async fn file_tails_missing_source_maps_to_not_found() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_control( + registry, + Arc::new(|| Ok(())), + Arc::new(Vec::new), + ); + + let err = service + .file_tails(crate::app::FileTailRequest::id_op( + crate::app::FileTailOp::Disable, + "missing-source".into(), + )) + .await + .unwrap_err(); + + assert!( + matches!(err, ServiceError::NotFound(_)), + "missing source should be NotFound, got {err:?}" + ); +} + +fn add_file_tail_request( + id: &str, + path: impl Into, + facility: Option<&str>, + severity: Option<&str>, +) -> crate::app::FileTailRequest { + crate::app::FileTailRequest::add(crate::app::FileTailAddRequest { + id: id.into(), + path: path.into(), + tag: id.into(), + hostname: Some("squirts".into()), + facility: facility.map(str::to_string), + severity: severity.map(str::to_string), + start_at_end: Some(true), + }) +} + +#[tokio::test] +async fn file_tails_rejects_invalid_facility_severity_and_disallowed_paths() { + let temp = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(temp.path().join("file-tail-test.db")); + let pool = Arc::new(init_pool(&storage).unwrap()); + let registry = Arc::new(crate::file_tail::FileTailRegistry::new( + temp.path().join("file-tails.json"), + )); + let service = CortexService::new(pool, storage).with_file_tail_control( + registry, + Arc::new(|| Ok(())), + Arc::new(Vec::new), + ); + let log_path = temp.path().join("access.log"); + std::fs::write(&log_path, "seed\n").unwrap(); + + let invalid_severity = service + .file_tails(add_file_tail_request( + "bad-sev", + log_path.to_string_lossy().into_owned(), + Some("local4"), + Some("bogus"), + )) + .await + .unwrap_err(); + assert!(invalid_severity.to_string().contains("severity")); + + let invalid_facility = service + .file_tails(add_file_tail_request( + "bad-facility", + log_path.to_string_lossy().into_owned(), + Some("notafacility"), + Some("info"), + )) + .await + .unwrap_err(); + assert!(invalid_facility.to_string().contains("facility")); + + let disallowed = service + .file_tails(add_file_tail_request( + "hosts", + "/etc/hosts", + Some("local4"), + Some("info"), + )) + .await + .unwrap_err(); + assert!(disallowed.to_string().contains("allowed roots")); +} diff --git a/src/app/services.rs b/src/app/services.rs index 99291df2..648fbb0c 100644 --- a/src/app/services.rs +++ b/src/app/services.rs @@ -50,6 +50,7 @@ use crate::assessment::{GeminiAssessConfig, build_assessment_prompt, run_gemini_ use crate::command_log::{self, CommandLogImportResult}; use crate::config::StorageConfig; use crate::db::{self, Bucket, ContextRef, DbPool, SearchParams, TimelineGroupBy}; +use crate::file_tail::{FileTailRegistry, FileTailStatus}; use crate::scanner; mod ai; @@ -58,6 +59,7 @@ mod analytics; mod assessment; mod compose; mod error_detection; +mod file_tails; mod filters; mod graph; mod graph_limits; @@ -91,6 +93,9 @@ pub struct CortexService { acquire_timeout: Duration, /// OS-level adapter for journalctl / systemd shell-outs. pub(super) os: Arc, + file_tail_registry: Option>, + file_tail_reconcile: Option anyhow::Result<()> + Send + Sync>>, + file_tail_statuses: Option Vec + Send + Sync>>, } /// Number of read permits issued for a given r2d2 pool size. @@ -115,6 +120,9 @@ impl CortexService { db_permits: Arc::new(Semaphore::new(permits)), acquire_timeout: DB_ACQUIRE_TIMEOUT, os: Arc::new(SystemOsAdapter), + file_tail_registry: None, + file_tail_reconcile: None, + file_tail_statuses: None, } } @@ -133,9 +141,29 @@ impl CortexService { db_permits: Arc::new(Semaphore::new(permits)), acquire_timeout: DB_ACQUIRE_TIMEOUT, os, + file_tail_registry: None, + file_tail_reconcile: None, + file_tail_statuses: None, } } + pub(crate) fn with_file_tail_registry(mut self, registry: Arc) -> Self { + self.file_tail_registry = Some(registry); + self + } + + pub(crate) fn with_file_tail_control( + mut self, + registry: Arc, + reconcile: Arc anyhow::Result<()> + Send + Sync>, + statuses: Arc Vec + Send + Sync>, + ) -> Self { + self.file_tail_registry = Some(registry); + self.file_tail_reconcile = Some(reconcile); + self.file_tail_statuses = Some(statuses); + self + } + /// One-shot SQLite schema-version probe. Sync because callers run during /// startup construction (e.g. `ApiState::new` caches it for /api/version) /// before the runtime serves requests. Exists so transport layers never diff --git a/src/app/services/file_tails.rs b/src/app/services/file_tails.rs new file mode 100644 index 00000000..2f0be0e3 --- /dev/null +++ b/src/app/services/file_tails.rs @@ -0,0 +1,122 @@ +use crate::app::{ServiceError, ServiceResult}; +use crate::file_tail::path_policy::validate_file_tail_path; +use crate::file_tail::{FileTailOp, FileTailRequest, FileTailResponse, FileTailSource}; + +use super::CortexService; + +impl CortexService { + pub async fn file_tails(&self, req: FileTailRequest) -> ServiceResult { + req.validate_shape().map_err(ServiceError::InvalidInput)?; + let registry = self.file_tail_registry.as_ref().ok_or_else(|| { + ServiceError::InvalidInput("file-tail registry is not mounted".into()) + })?; + let now = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + + let mut should_reconcile = false; + match req.op { + FileTailOp::List | FileTailOp::Status => {} + FileTailOp::Add => { + if self.file_tail_reconcile.is_none() { + return Err(ServiceError::InvalidInput( + "file-tail mutations require the long-running server; query-only mode cannot manage tailers".into(), + )); + } + let add = req.into_add().map_err(ServiceError::InvalidInput)?; + validate_file_tail_path(&add.path) + .map_err(|err| ServiceError::InvalidInput(err.to_string()))?; + let source = + FileTailSource::from_add(add, &now).map_err(ServiceError::InvalidInput)?; + if registry + .get(&source.id) + .map_err(|err| ServiceError::Internal(anyhow::anyhow!(err)))? + .is_some() + { + return Err(ServiceError::InvalidInput(format!( + "file tail source already exists: {}", + source.id + ))); + } + registry + .upsert(source) + .map_err(|err| ServiceError::Internal(anyhow::anyhow!(err)))?; + should_reconcile = true; + } + FileTailOp::Remove => { + if self.file_tail_reconcile.is_none() { + return Err(ServiceError::InvalidInput( + "file-tail mutations require the long-running server; query-only mode cannot manage tailers".into(), + )); + } + let id = req.required_id().map_err(ServiceError::InvalidInput)?; + registry.remove(id).map_err(map_registry_mutation_error)?; + should_reconcile = true; + } + FileTailOp::Enable => { + if self.file_tail_reconcile.is_none() { + return Err(ServiceError::InvalidInput( + "file-tail mutations require the long-running server; query-only mode cannot manage tailers".into(), + )); + } + let id = req.required_id().map_err(ServiceError::InvalidInput)?; + registry + .set_enabled(id, true, &now) + .map_err(map_registry_mutation_error)?; + should_reconcile = true; + } + FileTailOp::Disable => { + if self.file_tail_reconcile.is_none() { + return Err(ServiceError::InvalidInput( + "file-tail mutations require the long-running server; query-only mode cannot manage tailers".into(), + )); + } + let id = req.required_id().map_err(ServiceError::InvalidInput)?; + registry + .set_enabled(id, false, &now) + .map_err(map_registry_mutation_error)?; + should_reconcile = true; + } + } + + if should_reconcile { + if let Some(reconcile) = &self.file_tail_reconcile { + reconcile().map_err(|err| { + ServiceError::Internal(anyhow::anyhow!( + "file-tail mutation was committed, but reconcile failed: {err}" + )) + })?; + } + } + + let sources = registry.list().map_err(|err| { + if should_reconcile { + ServiceError::Internal(anyhow::anyhow!( + "file-tail mutation was committed, but refresh failed: {err}" + )) + } else { + ServiceError::Internal(anyhow::anyhow!(err)) + } + })?; + let statuses = self + .file_tail_statuses + .as_ref() + .map(|statuses| statuses()) + .unwrap_or_default(); + Ok(FileTailResponse { sources, statuses }) + } + + pub(crate) fn file_tail_statuses_snapshot(&self) -> Vec { + self.file_tail_statuses + .as_ref() + .map(|statuses| statuses()) + .unwrap_or_default() + } +} + +fn map_registry_mutation_error(err: anyhow::Error) -> ServiceError { + let message = err.to_string(); + if message.contains("file tail source not found:") { + ServiceError::NotFound(message) + } else { + ServiceError::Internal(err) + } +} diff --git a/src/app/services/filters.rs b/src/app/services/filters.rs index b7016db4..fcc7769f 100644 --- a/src/app/services/filters.rs +++ b/src/app/services/filters.rs @@ -141,6 +141,9 @@ fn apply_log_filter_aliases( Some("shell-history") => { params.source_ip_prefix = Some("shell-history://".to_string()); } + Some("file-tail") => { + params.source_ip_prefix = Some("file-tail://".to_string()); + } Some("claude") | Some("claude-transcript") => { apply_source_kind_tool_alias(params, "claude")?; } @@ -161,7 +164,7 @@ fn apply_log_filter_aliases( } Some(other) => { return Err(ServiceError::InvalidInput(format!( - "unsupported source_kind '{other}'. Supported: docker-stream, docker-event, agent-command, shell-history, transcript, claude, codex, gemini" + "unsupported source_kind '{other}'. Supported: docker-stream, docker-event, agent-command, shell-history, file-tail, transcript, claude, codex, gemini" ))); } } diff --git a/src/cli.rs b/src/cli.rs index d5824ae7..73a1b402 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -13,7 +13,8 @@ pub(crate) use args::{ AiIndexArgs, AiInvestigateArgs, AiListArgs, AiOutputDetail, AiPruneCheckpointsArgs, AiSearchArgs, AiSimilarArgs, AiWatchArgs, CliCommand, ComposeArgs, ComposeCommand, ComposeLogsArgs, ComposeMutationArgs, CorrelateArgs, DbBackupArgs, DbCheckpointArgs, DbCommand, - DbIntegrityArgs, DbIntegrityStatusArgs, DbStatusArgs, DbVacuumArgs, EntityArgs, FilterArgs, + DbIntegrityArgs, DbIntegrityStatusArgs, DbStatusArgs, DbVacuumArgs, EntityArgs, + FileTailAddArgs, FileTailCommand, FileTailIdArgs, FileTailListArgs, FilterArgs, GraphAroundArgs, GraphCommand, GraphEvidenceArgs, GraphExplainArgs, GraphRebuildArgs, GraphStatusArgs, HeartbeatAgentArgs, HeartbeatCommand, IncidentArgs, IngestRateArgs, InventoryArgs, InventoryCommand, NotifyRecentArgs, NotifyTestArgs, OutputArgs, PatternsArgs, diff --git a/src/cli/args.rs b/src/cli/args.rs index 89b18bec..99c3f4a6 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -52,6 +52,40 @@ pub(crate) enum CliCommand { CorrelateState(CorrelateStateArgs), Entity(EntityArgs), Graph(GraphCommand), + FileTail(FileTailCommand), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum FileTailCommand { + List(FileTailListArgs), + Status(FileTailListArgs), + Add(FileTailAddArgs), + Remove(FileTailIdArgs), + Enable(FileTailIdArgs), + Disable(FileTailIdArgs), +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct FileTailListArgs { + pub json: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FileTailIdArgs { + pub id: String, + pub json: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FileTailAddArgs { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: bool, + pub json: bool, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/cli/commands.rs b/src/cli/commands.rs index 74592e7a..e5d89cf6 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -26,6 +26,7 @@ pub(crate) mod apps; pub(crate) mod clock_skew; pub(crate) mod compare; pub(crate) mod correlate_state; +pub(crate) mod file_tails; pub(crate) mod fleet_state; pub(crate) mod graph; pub(crate) mod host_state; diff --git a/src/cli/commands/file_tails.rs b/src/cli/commands/file_tails.rs new file mode 100644 index 00000000..0122b63f --- /dev/null +++ b/src/cli/commands/file_tails.rs @@ -0,0 +1,158 @@ +use anyhow::{Result, anyhow, bail}; + +use crate::cli::{ + CliCommand, FileTailAddArgs, FileTailCommand, FileTailIdArgs, FileTailListArgs, suggest, +}; + +pub(crate) fn parse_file_tail(args: &[String]) -> Result { + let (command, rest) = args + .split_first() + .ok_or_else(|| anyhow!("file-tail subcommand is required"))?; + match command.as_str() { + "list" => Ok(CliCommand::FileTail(FileTailCommand::List(parse_list( + rest, + )?))), + "status" => Ok(CliCommand::FileTail(FileTailCommand::Status(parse_list( + rest, + )?))), + "add" => Ok(CliCommand::FileTail(FileTailCommand::Add(parse_add(rest)?))), + "remove" => Ok(CliCommand::FileTail(FileTailCommand::Remove(parse_id( + rest, + )?))), + "enable" => Ok(CliCommand::FileTail(FileTailCommand::Enable(parse_id( + rest, + )?))), + "disable" => Ok(CliCommand::FileTail(FileTailCommand::Disable(parse_id( + rest, + )?))), + _ => bail!( + "{}", + suggest::unknown_command( + "file-tail subcommand", + command, + &["list", "status", "add", "remove", "enable", "disable"], + ) + ), + } +} + +fn parse_list(args: &[String]) -> Result { + let mut out = FileTailListArgs { json: false }; + for arg in args { + match arg.as_str() { + "--json" => out.json = true, + "--help" | "-h" => bail!("{}", usage()), + other => bail!( + "{}", + suggest::unknown_option("file-tail list", other, &["--json"]) + ), + } + } + Ok(out) +} + +fn parse_id(args: &[String]) -> Result { + let mut id = None; + let mut json = false; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--id" => { + i += 1; + id = Some(required(args, i, "--id")?); + } + "--json" => json = true, + "--help" | "-h" => bail!("{}", usage()), + other => bail!( + "{}", + suggest::unknown_option("file-tail", other, &["--id", "--json"]) + ), + } + i += 1; + } + Ok(FileTailIdArgs { + id: id.ok_or_else(|| anyhow!("--id is required"))?, + json, + }) +} + +fn parse_add(args: &[String]) -> Result { + let mut out = FileTailAddArgs { + id: String::new(), + path: String::new(), + tag: String::new(), + hostname: None, + facility: None, + severity: None, + start_at_end: true, + json: false, + }; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--id" => { + i += 1; + out.id = required(args, i, "--id")?; + } + "--path" => { + i += 1; + out.path = required(args, i, "--path")?; + } + "--tag" => { + i += 1; + out.tag = required(args, i, "--tag")?; + } + "--hostname" => { + i += 1; + out.hostname = Some(required(args, i, "--hostname")?); + } + "--facility" => { + i += 1; + out.facility = Some(required(args, i, "--facility")?); + } + "--severity" => { + i += 1; + out.severity = Some(required(args, i, "--severity")?); + } + "--from-start" => out.start_at_end = false, + "--json" => out.json = true, + "--help" | "-h" => bail!("{}", usage()), + other => bail!( + "{}", + suggest::unknown_option( + "file-tail add", + other, + &[ + "--id", + "--path", + "--tag", + "--hostname", + "--facility", + "--severity", + "--from-start", + "--json", + ], + ) + ), + } + i += 1; + } + if out.id.is_empty() || out.path.is_empty() || out.tag.is_empty() || out.hostname.is_none() { + bail!("file-tail add requires --id, --path, --tag, and --hostname"); + } + Ok(out) +} + +fn required(args: &[String], index: usize, flag: &str) -> Result { + let value = args + .get(index) + .ok_or_else(|| anyhow!("{flag} requires a value"))?; + if value.trim().is_empty() || value.starts_with('-') { + bail!("{flag} requires a value"); + } + Ok(value.clone()) +} + +fn usage() -> &'static str { + "Usage: cortex file-tail list [--json]\n cortex file-tail status [--json]\n cortex file-tail add --id ID --path PATH --tag TAG --hostname HOST [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json]\n cortex file-tail remove --id ID [--json]\n cortex file-tail enable --id ID [--json]\n cortex file-tail disable --id ID [--json]" +} diff --git a/src/cli/dispatch.rs b/src/cli/dispatch.rs index 0c1ebd5f..490caf1f 100644 --- a/src/cli/dispatch.rs +++ b/src/cli/dispatch.rs @@ -20,8 +20,9 @@ use anyhow::{Result, bail}; use cortex::app::{ - CorrelateEventsRequest, FilterLogsRequest, GetErrorsRequest, IncidentRequest, - ListSessionsRequest, SearchLogsRequest, TailLogsRequest, + CorrelateEventsRequest, FileTailAddRequest, FileTailOp, FileTailRequest, FileTailResponse, + FilterLogsRequest, GetErrorsRequest, IncidentRequest, ListSessionsRequest, SearchLogsRequest, + TailLogsRequest, }; use std::future::Future; @@ -31,8 +32,8 @@ use super::output_logs::{ print_sessions_response, print_stats_response, }; use super::{ - CliMode, CorrelateArgs, FilterArgs, IncidentArgs, SearchArgs, SessionsArgs, TailArgs, - TimeRangeArgs, + CliMode, CorrelateArgs, FileTailCommand, FileTailIdArgs, FilterArgs, IncidentArgs, SearchArgs, + SessionsArgs, TailArgs, TimeRangeArgs, }; // ─── Arg → Request conversions ────────────────────────────────────────────── @@ -277,6 +278,67 @@ pub(crate) async fn run_sessions(mode: &CliMode, args: SessionsArgs) -> Result<( print_sessions_response(&response, json) } +pub(crate) async fn run_file_tail(mode: &CliMode, command: FileTailCommand) -> Result<()> { + let (req, json) = match command { + FileTailCommand::List(args) => (FileTailRequest::list(), args.json), + FileTailCommand::Status(args) => (FileTailRequest::status(), args.json), + FileTailCommand::Add(args) => ( + FileTailRequest::add(FileTailAddRequest { + id: args.id, + path: args.path, + tag: args.tag, + hostname: args.hostname, + facility: args.facility, + severity: args.severity, + start_at_end: Some(args.start_at_end), + }), + args.json, + ), + FileTailCommand::Remove(args) => id_request(FileTailOp::Remove, args), + FileTailCommand::Enable(args) => id_request(FileTailOp::Enable, args), + FileTailCommand::Disable(args) => id_request(FileTailOp::Disable, args), + }; + let response = match mode { + CliMode::Local(service) => service.file_tails(req).await?, + CliMode::Http(client) => http_or_cancel(client.file_tails(&req)).await?, + }; + if json { + println!("{}", serde_json::to_string_pretty(&response)?); + } else { + print!("{}", format_file_tail_response(&response)); + } + Ok(()) +} + +fn format_file_tail_response(response: &FileTailResponse) -> String { + let mut out = String::new(); + for source in &response.sources { + out.push_str(&format!( + "{}\t{}\t{}\t{}\n", + source.id, + if source.enabled { + "enabled" + } else { + "disabled" + }, + source.tag, + source.path + )); + } + for status in &response.statuses { + let last_error = status.last_error.as_deref().unwrap_or("-"); + out.push_str(&format!( + "{}\t{}\t{}\n", + status.id, status.running, last_error + )); + } + out +} + +fn id_request(op: FileTailOp, args: FileTailIdArgs) -> (FileTailRequest, bool) { + (FileTailRequest::id_op(op, args.id), args.json) +} + pub(crate) use super::dispatch_ai::{ run_ai_abuse, run_ai_add, run_ai_ask_history, run_ai_assess, run_ai_blocks, run_ai_checkpoints, run_ai_context, run_ai_correlate, run_ai_doctor, run_ai_errors, run_ai_incident_context, diff --git a/src/cli/dispatch_tests.rs b/src/cli/dispatch_tests.rs index 25a28ab8..af388fa5 100644 --- a/src/cli/dispatch_tests.rs +++ b/src/cli/dispatch_tests.rs @@ -15,25 +15,26 @@ //! #A29). use super::{ - http_or_cancel_with, run_ai_abuse, run_ai_add, run_ai_blocks, run_ai_checkpoints, - run_ai_context, run_ai_correlate, run_ai_doctor, run_ai_errors, run_ai_index, run_ai_projects, - run_ai_prune_checkpoints, run_ai_search, run_ai_smoke_watch, run_ai_tools, run_ai_watch, - run_ai_watch_status, run_correlate, run_db_backup, run_db_checkpoint, run_db_integrity, - run_db_status, run_db_vacuum, run_errors, run_hosts, run_search, run_sessions, run_stats, - run_tail, + format_file_tail_response, http_or_cancel_with, run_ai_abuse, run_ai_add, run_ai_blocks, + run_ai_checkpoints, run_ai_context, run_ai_correlate, run_ai_doctor, run_ai_errors, + run_ai_index, run_ai_projects, run_ai_prune_checkpoints, run_ai_search, run_ai_smoke_watch, + run_ai_tools, run_ai_watch, run_ai_watch_status, run_correlate, run_db_backup, + run_db_checkpoint, run_db_integrity, run_db_status, run_db_vacuum, run_errors, run_file_tail, + run_hosts, run_search, run_sessions, run_stats, run_tail, }; use crate::cli::http_client::HttpClient; use crate::cli::{ AiAbuseArgs, AiAddArgs, AiBlocksArgs, AiCheckpointsArgs, AiContextArgs, AiCorrelateArgs, AiDoctorArgs, AiErrorsArgs, AiIndexArgs, AiListArgs, AiPruneCheckpointsArgs, AiSearchArgs, AiWatchArgs, CliMode, CorrelateArgs, DbBackupArgs, DbCheckpointArgs, DbIntegrityArgs, - DbStatusArgs, DbVacuumArgs, EntityArgs, FilterArgs, GraphAroundArgs, GraphEvidenceArgs, - GraphExplainArgs, IngestRateArgs, OutputArgs, PatternsArgs, SearchArgs, SessionsArgs, - SigAckArgs, SigListArgs, SigUnackArgs, SourceIpsArgs, TailArgs, TimeRangeArgs, TimelineArgs, + DbStatusArgs, DbVacuumArgs, EntityArgs, FileTailCommand, FileTailListArgs, FilterArgs, + GraphAroundArgs, GraphEvidenceArgs, GraphExplainArgs, IngestRateArgs, OutputArgs, PatternsArgs, + SearchArgs, SessionsArgs, SigAckArgs, SigListArgs, SigUnackArgs, SourceIpsArgs, TailArgs, + TimeRangeArgs, TimelineArgs, }; use anyhow::{Result, bail}; use std::time::Duration; -use wiremock::matchers::{method, path}; +use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; // ─── helpers ──────────────────────────────────────────────────────────────── @@ -42,6 +43,10 @@ async fn http_mode() -> (MockServer, CliMode) { let server = MockServer::start().await; let client = HttpClient::discover(Some(server.uri()), Some("test-token".into())).expect("discover ok"); + http_mode_with_client(server, client).await +} + +async fn http_mode_with_client(server: MockServer, client: HttpClient) -> (MockServer, CliMode) { // Catch-all guard: any request that doesn't match a per-test // mock returns 404 and counts against `expect(0)`. Combined with the // per-test `expect(1)` on the actual endpoint, this asserts EXACTLY @@ -58,6 +63,14 @@ async fn http_mode() -> (MockServer, CliMode) { (server, CliMode::Http(client)) } +async fn http_mode_with_admin_token(admin_token: &str) -> (MockServer, CliMode) { + let server = MockServer::start().await; + let client = HttpClient::discover(Some(server.uri()), Some("test-token".into())) + .expect("discover ok") + .with_api_admin_token_for_test(admin_token); + http_mode_with_client(server, client).await +} + fn empty_search_logs_body() -> serde_json::Value { serde_json::json!({"count": 0, "logs": []}) } @@ -378,6 +391,50 @@ async fn run_sessions_http_sends_exactly_one_request() { .expect("sessions ok"); } +#[tokio::test] +async fn run_file_tail_http_sends_exactly_one_request() { + let (server, mode) = http_mode_with_admin_token("admin-token").await; + Mock::given(method("POST")) + .and(path("/api/file-tails")) + .and(header("x-cortex-admin-token", "admin-token")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "sources": [], + "statuses": [], + }))) + .expect(1) + .mount(&server) + .await; + + run_file_tail( + &mode, + FileTailCommand::List(FileTailListArgs { json: true }), + ) + .await + .expect("file-tail ok"); +} + +#[test] +fn file_tail_status_text_includes_healthy_statuses() { + let response = cortex::app::FileTailResponse { + sources: vec![], + statuses: vec![cortex::app::FileTailStatus { + id: "swag-access".into(), + running: true, + last_line_at: None, + last_read_at: None, + last_checkpoint_at: None, + blocked_on_writer_since: None, + last_error: None, + }], + }; + + let out = format_file_tail_response(&response); + assert!( + out.contains("swag-access\ttrue\t-"), + "healthy status should be visible even without last_error: {out}" + ); +} + #[tokio::test] async fn run_correlate_http_sends_exactly_one_request() { let (server, mode) = http_mode().await; diff --git a/src/cli/help.rs b/src/cli/help.rs index d17aa660..160625b6 100644 --- a/src/cli/help.rs +++ b/src/cli/help.rs @@ -75,7 +75,13 @@ const SECTIONS: &[(&str, &[&str])] = &[ ("Signals & Alerts", &["sig", "notify"]), ( "Ingestion", - &["shell", "agent-command", "heartbeat", "inventory"], + &[ + "shell", + "agent-command", + "heartbeat", + "inventory", + "file-tail", + ], ), ( "Runtime & Setup", @@ -317,6 +323,18 @@ const CATALOG: &[CommandDoc] = &[ "cortex inventory status [--json]", ], }, + CommandDoc { + name: "file-tail", + summary: "Manage file-tail log ingest sources", + usage: &[ + "cortex file-tail list [--json]", + "cortex file-tail status [--json]", + "cortex file-tail add --id ID --path PATH --tag TAG --hostname HOST [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json]", + "cortex file-tail remove --id ID [--json]", + "cortex file-tail enable --id ID [--json]", + "cortex file-tail disable --id ID [--json]", + ], + }, // ── Runtime & Setup ──────────────────────────────────────────────────── CommandDoc { name: "serve", @@ -683,6 +701,38 @@ const NESTED_CATALOG: &[NestedCommandDoc] = &[ "cortex heartbeat agent [--target URL] [--token TOKEN] [--interval-secs N] [--probe-deadline-ms N] [--collection-deadline-ms N] [--retry-buffer N] [--host-id-path PATH] [--once|--emit] [--json]", ], }, + NestedCommandDoc { + path: "file-tail list", + summary: "List configured file-tail sources", + usage: &["cortex file-tail list [--json]"], + }, + NestedCommandDoc { + path: "file-tail status", + summary: "List configured file-tail sources with runtime state", + usage: &["cortex file-tail status [--json]"], + }, + NestedCommandDoc { + path: "file-tail add", + summary: "Add or update a managed file-tail source", + usage: &[ + "cortex file-tail add --id ID --path PATH --tag TAG --hostname HOST [--facility FACILITY] [--severity SEVERITY] [--from-start] [--json]", + ], + }, + NestedCommandDoc { + path: "file-tail remove", + summary: "Remove a managed file-tail source", + usage: &["cortex file-tail remove --id ID [--json]"], + }, + NestedCommandDoc { + path: "file-tail enable", + summary: "Enable a managed file-tail source", + usage: &["cortex file-tail enable --id ID [--json]"], + }, + NestedCommandDoc { + path: "file-tail disable", + summary: "Disable a managed file-tail source", + usage: &["cortex file-tail disable --id ID [--json]"], + }, ]; const GLOBAL_OPTIONS: &[(&str, &str)] = &[ diff --git a/src/cli/help_tests.rs b/src/cli/help_tests.rs index 6cc23f0b..99ed5ada 100644 --- a/src/cli/help_tests.rs +++ b/src/cli/help_tests.rs @@ -40,6 +40,7 @@ const PARSER_TOKENS: &[&str] = &[ "host-state", "fleet-state", "correlate-state", + "file-tail", // Mode-level (src/main.rs) "serve", "mcp", @@ -93,6 +94,7 @@ fn top_level_help_plain_lists_sections_and_commands() { assert!(out.contains("Commands")); assert!(out.contains("Search & Logs")); assert!(out.contains("source-ips")); + assert!(out.contains("file-tail")); assert!(out.contains("→ Run cortex --help")); } @@ -134,6 +136,10 @@ fn nested_help_shows_subcommand_specific_usage() { out.contains("cortex inventory status [--json]"), "got: {out}" ); + + let out = render_command("file-tail add", false).expect("file-tail add is known"); + assert!(out.contains("cortex file-tail add --id ID"), "got: {out}"); + assert!(out.contains("--from-start"), "got: {out}"); } #[test] diff --git a/src/cli/http_client.rs b/src/cli/http_client.rs index 7956c522..d9f41475 100644 --- a/src/cli/http_client.rs +++ b/src/cli/http_client.rs @@ -53,7 +53,7 @@ use std::env; use std::time::Duration; use anyhow::{Context, Result, anyhow, bail}; -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue}; use reqwest::{Method, Response, StatusCode}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -69,19 +69,19 @@ use cortex::app::{ CorrelateEventsRequest, CorrelateEventsResponse, CorrelateStateRequest, CorrelateStateResponse, DbBackupRequest, DbBackupResult, DbCheckpointRequest, DbCheckpointResult, DbIntegrityJobStarted, DbIntegrityRequest, DbIntegrityResult, DbMaintenanceStatus, DbStats, - DbVacuumRequest, DbVacuumResult, FilterLogsRequest, FleetStateRequest, FleetStateResponse, - GetErrorsRequest, GetErrorsResponse, GetLogRequest, GetLogResponse, GraphAroundRequest, - GraphAroundResponse, GraphEntityLookupRequest, GraphEntityLookupResponse, - GraphEvidenceLookupRequest, GraphEvidenceLookupResponse, GraphExplainRequest, - GraphExplainResponse, HostStateRequest, HostStateResponse, IncidentContextRequest, - IncidentContextResponse, IngestRateRequest, IngestRateResponse, ListAiProjectsRequest, - ListAiProjectsResponse, ListAiToolsRequest, ListAiToolsResponse, ListAppsRequest, - ListAppsResponse, ListHostsResponse, ListSessionsRequest, ListSessionsResponse, - ListSourceIpsRequest, ListSourceIpsResponse, MaintenanceJobStatus, PatternsRequest, - PatternsResponse, ProjectContextRequest, ProjectContextResponse, SearchLogsRequest, - SearchLogsResponse, SearchSessionsRequest, SearchSessionsResponse, SilentHostsRequest, - SilentHostsResponse, SimilarIncidentsRequest, SimilarIncidentsResponse, TailLogsRequest, - TimelineRequest, TimelineResponse, UnackErrorRequest, UnackErrorResponse, + DbVacuumRequest, DbVacuumResult, FileTailRequest, FileTailResponse, FilterLogsRequest, + FleetStateRequest, FleetStateResponse, GetErrorsRequest, GetErrorsResponse, GetLogRequest, + GetLogResponse, GraphAroundRequest, GraphAroundResponse, GraphEntityLookupRequest, + GraphEntityLookupResponse, GraphEvidenceLookupRequest, GraphEvidenceLookupResponse, + GraphExplainRequest, GraphExplainResponse, HostStateRequest, HostStateResponse, + IncidentContextRequest, IncidentContextResponse, IngestRateRequest, IngestRateResponse, + ListAiProjectsRequest, ListAiProjectsResponse, ListAiToolsRequest, ListAiToolsResponse, + ListAppsRequest, ListAppsResponse, ListHostsResponse, ListSessionsRequest, + ListSessionsResponse, ListSourceIpsRequest, ListSourceIpsResponse, MaintenanceJobStatus, + PatternsRequest, PatternsResponse, ProjectContextRequest, ProjectContextResponse, + SearchLogsRequest, SearchLogsResponse, SearchSessionsRequest, SearchSessionsResponse, + SilentHostsRequest, SilentHostsResponse, SimilarIncidentsRequest, SimilarIncidentsResponse, + TailLogsRequest, TimelineRequest, TimelineResponse, UnackErrorRequest, UnackErrorResponse, UnaddressedErrorsRequest, UnaddressedErrorsResponse, UsageBlocksRequest, UsageBlocksResponse, }; use cortex::scanner::{CheckpointEntry, ParseErrorEntry, PruneCheckpointsResult}; @@ -127,10 +127,10 @@ pub struct ServerVersion { /// see the populated value or race the init future fairly). The dispatch layer /// (bead 0p8r.7) is responsible for wrapping these in `tokio::select!` against /// `tokio::signal::ctrl_c()`. -#[derive(Debug)] pub struct HttpClient { base_url: Url, inner: reqwest::Client, + api_admin_token: Option, /// **LAZY ON 404 ONLY. Do NOT pre-populate or refresh after success.** The /// whole point of `/api/version` is detecting upgrades after a deploy; /// caching beyond 404 enrichment defeats it (eng-review #A33). Populated @@ -146,6 +146,23 @@ pub struct HttpClient { server_version_cache: OnceCell>, } +impl std::fmt::Debug for HttpClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HttpClient") + .field("base_url", &self.base_url) + .field("inner", &self.inner) + .field( + "api_admin_token", + &self + .api_admin_token + .as_ref() + .map(|_| ""), + ) + .field("server_version_cache", &self.server_version_cache) + .finish() + } +} + impl HttpClient { /// Resolve the base URL and bearer token from CLI flags / env vars / defaults, /// then construct a `reqwest::Client` with our connect + request timeouts @@ -178,10 +195,20 @@ impl HttpClient { Ok(Self { base_url, inner, + api_admin_token: env::var("CORTEX_API_ADMIN_TOKEN") + .ok() + .filter(|token| !token.trim().is_empty()), server_version_cache: OnceCell::new(), }) } + #[cfg(test)] + pub(crate) fn with_api_admin_token_for_test(mut self, token: impl Into) -> Self { + let token = token.into(); + self.api_admin_token = (!token.trim().is_empty()).then_some(token); + self + } + // ─── HTTP plumbing ────────────────────────────────────────────────────── /// Build a full URL from a path like `/api/search` joined onto the @@ -250,6 +277,43 @@ impl HttpClient { self.execute_with_retry(send, path).await } + async fn post_json_with_admin_no_retry(&self, path: &str, body: &Req) -> Result + where + Req: Serialize + ?Sized, + Resp: DeserializeOwned, + { + let token = self.api_admin_token.as_deref().ok_or_else(|| { + anyhow!("CORTEX_API_ADMIN_TOKEN is required for this HTTP API mutation") + })?; + let mut admin_value = + HeaderValue::from_str(token).context("failed to construct admin token header")?; + admin_value.set_sensitive(true); + let admin_header = HeaderName::from_static("x-cortex-admin-token"); + let url = self.url(path)?; + let send = || async { + self.inner + .request(Method::POST, url.clone()) + .header(admin_header.clone(), admin_value.clone()) + .json(body) + .send() + .await + }; + self.execute_once(send, path).await + } + + async fn execute_once(&self, send: F, path: &str) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + Resp: DeserializeOwned, + { + let resp = match send().await { + Ok(r) => r, + Err(err) => return Err(map_send_error(err, &self.base_url)), + }; + self.handle_response(resp, path).await + } + /// Send a request, handling the 503 retry and final response classification. /// /// The closure is invoked at most twice — once initially, and once more @@ -515,6 +579,11 @@ impl HttpClient { self.post_json("/api/ai/prune-checkpoints", req).await } + pub async fn file_tails(&self, req: &FileTailRequest) -> Result { + self.post_json_with_admin_no_retry("/api/file-tails", req) + .await + } + // ─── REST surface: bead 0p8r.4 (DB ops) ───────────────────────────────── pub async fn db_status(&self) -> Result { diff --git a/src/cli/http_client_tests.rs b/src/cli/http_client_tests.rs index ac699982..f80ef64b 100644 --- a/src/cli/http_client_tests.rs +++ b/src/cli/http_client_tests.rs @@ -426,6 +426,31 @@ async fn double_503_error_includes_both_bodies() { assert_eq!(counter.load(Ordering::SeqCst), 2); } +#[tokio::test] +async fn file_tails_post_does_not_retry_503() { + let server = MockServer::start().await; + let client = HttpClient::discover(Some(server.uri()), Some("test-value".into())) + .unwrap() + .with_api_admin_token_for_test("admin-value"); + Mock::given(method("POST")) + .and(path("/api/file-tails")) + .and(header("x-cortex-admin-token", "admin-value")) + .respond_with(ResponseTemplate::new(503).set_body_string("committed but unavailable")) + .expect(1) + .mount(&server) + .await; + + let err = client + .file_tails(&cortex::app::FileTailRequest::status()) + .await + .expect_err("stateful admin POST must not retry"); + + assert!( + err.to_string().contains("committed but unavailable"), + "expected first 503 body in error: {err}" + ); +} + // ─── Malformed JSON: serde_path_to_error surfaces field path + preview ────── #[tokio::test] diff --git a/src/cli/parse.rs b/src/cli/parse.rs index 4787b19d..c3bc9bde 100644 --- a/src/cli/parse.rs +++ b/src/cli/parse.rs @@ -43,6 +43,7 @@ const TOP_LEVEL_COMMANDS: &[&str] = &[ "host-state", "fleet-state", "correlate-state", + "file-tail", ]; pub(crate) fn parse_command(args: Vec) -> Result { @@ -87,6 +88,7 @@ pub(crate) fn parse_command(args: Vec) -> Result { "host-state" => commands::host_state::parse_host_state(rest), "fleet-state" => commands::fleet_state::parse_fleet_state(rest), "correlate-state" => commands::correlate_state::parse_correlate_state(rest), + "file-tail" => commands::file_tails::parse_file_tail(rest), _ => bail!( "{}", suggest::unknown_command("CLI command", command, TOP_LEVEL_COMMANDS) diff --git a/src/cli/parse_tests.rs b/src/cli/parse_tests.rs index 37fac168..9b0ce085 100644 --- a/src/cli/parse_tests.rs +++ b/src/cli/parse_tests.rs @@ -1,5 +1,6 @@ use super::super::{ - HeartbeatAgentArgs, HeartbeatCommand, InventoryArgs, InventoryCommand, OutputArgs, + FileTailAddArgs, FileTailCommand, FileTailListArgs, HeartbeatAgentArgs, HeartbeatCommand, + InventoryArgs, InventoryCommand, OutputArgs, }; use super::*; @@ -11,6 +12,69 @@ fn parse_routes_stats() { ); } +#[test] +fn parses_file_tail_add() { + let command = parse_command(vec![ + "file-tail".into(), + "add".into(), + "--id".into(), + "swag-access".into(), + "--path".into(), + "/mnt/appdata/swag/log/nginx/access.log".into(), + "--tag".into(), + "swag-access".into(), + "--hostname".into(), + "squirts".into(), + "--facility".into(), + "local4".into(), + "--severity".into(), + "info".into(), + "--from-start".into(), + "--json".into(), + ]) + .unwrap(); + + assert_eq!( + command, + CliCommand::FileTail(FileTailCommand::Add(FileTailAddArgs { + id: "swag-access".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: Some("info".into()), + start_at_end: false, + json: true, + })) + ); +} + +#[test] +fn file_tail_add_requires_hostname() { + let err = parse_command(vec![ + "file-tail".into(), + "add".into(), + "--id".into(), + "swag-access".into(), + "--path".into(), + "/mnt/appdata/swag/log/nginx/access.log".into(), + "--tag".into(), + "swag-access".into(), + ]) + .unwrap_err(); + + assert!(err.to_string().contains("--hostname")); +} + +#[test] +fn parses_file_tail_list() { + let command = parse_command(vec!["file-tail".into(), "list".into(), "--json".into()]).unwrap(); + assert_eq!( + command, + CliCommand::FileTail(FileTailCommand::List(FileTailListArgs { json: true })) + ); +} + #[test] fn parse_routes_heartbeat_agent_defaults() { assert_eq!( diff --git a/src/cli/run.rs b/src/cli/run.rs index e487bfe8..3fc95845 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -55,6 +55,7 @@ pub(crate) async fn run(mode: CliMode, command: CliCommand) -> Result<()> { CliCommand::Correlate(args) => dispatch::run_correlate(&mode, args).await, CliCommand::Stats(args) => dispatch::run_stats(&mode, args).await, CliCommand::Sessions(args) => dispatch::run_sessions(&mode, args).await, + CliCommand::FileTail(command) => dispatch::run_file_tail(&mode, command).await, // AI commands (bead 0p8r.8). 10 are HTTP-capable; 6 are LOCAL-only // and bail in HTTP mode with a per-command inline message. CliCommand::Ai(ai) => match ai { diff --git a/src/config.rs b/src/config.rs index c1f94655..15772b02 100644 --- a/src/config.rs +++ b/src/config.rs @@ -452,6 +452,11 @@ pub struct ApiConfig { /// Provisioned by `cortex setup repair`. The server fails to start without it. #[serde(default)] pub api_token: Secret, + /// Optional stronger token for REST file-tail management. + /// All `/api/file-tails` operations require this token because list/status + /// expose configured filesystem paths. + #[serde(default)] + pub admin_token: Secret, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -843,6 +848,7 @@ impl Config { )?; env_override_opt_str("CORTEX_API_TOKEN", &mut config.api.api_token.0); + env_override_opt_str("CORTEX_API_ADMIN_TOKEN", &mut config.api.admin_token.0); env_override_opt_str( "CORTEX_AUTHELIA_SOURCE_IP", @@ -1140,6 +1146,9 @@ pub(crate) fn validate_auth_config(config: &Config, check_bind: bool) -> anyhow: if token_is_set_but_blank(&config.api.api_token.0) { return Err(anyhow::anyhow!("api.api_token must not be empty")); } + if token_is_set_but_blank(&config.api.admin_token.0) { + return Err(anyhow::anyhow!("api.admin_token must not be empty")); + } // Note: CORTEX_API_TOKEN being entirely unset is enforced at // route-mount time by `api::router` (anyhow::bail) rather than here. // Failing in `Config::load()` would break stdio-mode invocations diff --git a/src/config_tests.rs b/src/config_tests.rs index 00df8a7c..3ff0a602 100644 --- a/src/config_tests.rs +++ b/src/config_tests.rs @@ -39,6 +39,19 @@ fn api_token_env_sets_api_token_not_mcp_token() { assert_eq!(cfg.mcp.api_token, None); } +#[test] +#[serial] +fn api_admin_token_env_sets_admin_token() { + unsafe { std::env::set_var("CORTEX_HOST", "127.0.0.1") }; + unsafe { std::env::set_var("CORTEX_API_ADMIN_TOKEN", "api-admin-token") }; + let result = Config::load(); + unsafe { std::env::remove_var("CORTEX_API_ADMIN_TOKEN") }; + unsafe { std::env::remove_var("CORTEX_HOST") }; + + let cfg = result.expect("Config::load() should succeed"); + assert_eq!(cfg.api.admin_token, Some("api-admin-token".into())); +} + #[test] #[serial] fn env_var_overrides_mcp_port() { @@ -339,6 +352,14 @@ fn auth_validation_rejects_blank_api_token() { assert!(err.to_string().contains("api.api_token")); } +#[test] +fn auth_validation_rejects_blank_api_admin_token() { + let mut cfg = Config::default(); + cfg.api.admin_token = Some(" ".into()).into(); + let err = validate_auth_config(&cfg, true).unwrap_err(); + assert!(err.to_string().contains("api.admin_token")); +} + #[test] #[serial] fn host_with_port_is_rejected() { diff --git a/src/enrich/dispatch.rs b/src/enrich/dispatch.rs index 3158258f..9e70a16a 100644 --- a/src/enrich/dispatch.rs +++ b/src/enrich/dispatch.rs @@ -176,6 +176,7 @@ fn to_source_kind(raw: Option<&str>) -> SourceKind { Some("agent") => SourceKind::Agent, Some("shell-history") => SourceKind::ShellHistory, Some("agent-command") => SourceKind::AgentCommand, + Some("file-tail") => SourceKind::FileTail, _ => SourceKind::SyslogTcp, } } diff --git a/src/enrich/parser.rs b/src/enrich/parser.rs index c5678ae9..53677ca2 100644 --- a/src/enrich/parser.rs +++ b/src/enrich/parser.rs @@ -27,6 +27,7 @@ use thiserror::Error; /// - `Agent` → `"agent"` (per-host agent WebSocket) /// - `ShellHistory` → `"shell-history"` (local shell history backfill) /// - `AgentCommand` → `"agent-command"` (AI agent-launched shell command spool) +/// - `FileTail` → `"file-tail"` (Cortex-managed local file-tail ingest) /// /// **History:** prior versions of this contract used `snake_case` with a /// bare `Syslog` variant. Both were corrected during the cross-cutting audit @@ -45,6 +46,7 @@ pub enum SourceKind { Agent, ShellHistory, AgentCommand, + FileTail, } impl SourceKind { @@ -63,6 +65,7 @@ impl SourceKind { SourceKind::Agent => "agent", SourceKind::ShellHistory => "shell-history", SourceKind::AgentCommand => "agent-command", + SourceKind::FileTail => "file-tail", } } diff --git a/src/enrich/parser_tests.rs b/src/enrich/parser_tests.rs index f3b04d31..392aa485 100644 --- a/src/enrich/parser_tests.rs +++ b/src/enrich/parser_tests.rs @@ -9,6 +9,7 @@ fn source_kind_as_str_matches_serde() { assert_eq!(SourceKind::UnifiApi.as_str(), "unifi-api"); assert_eq!(SourceKind::ShellHistory.as_str(), "shell-history"); assert_eq!(SourceKind::AgentCommand.as_str(), "agent-command"); + assert_eq!(SourceKind::FileTail.as_str(), "file-tail"); } #[test] diff --git a/src/file_tail.rs b/src/file_tail.rs new file mode 100644 index 00000000..9996265d --- /dev/null +++ b/src/file_tail.rs @@ -0,0 +1,27 @@ +pub(crate) mod models; +pub(crate) mod path_policy; +pub(crate) mod registry; +pub(crate) mod supervisor; + +pub use models::{ + FileTailAddRequest, FileTailOp, FileTailRequest, FileTailResponse, FileTailSource, + FileTailStatus, +}; +pub(crate) use registry::FileTailRegistry; +pub(crate) use supervisor::FileTailSupervisor; + +#[cfg(test)] +#[path = "file_tail/models_tests.rs"] +mod models_tests; + +#[cfg(test)] +#[path = "file_tail/path_policy_tests.rs"] +mod path_policy_tests; + +#[cfg(test)] +#[path = "file_tail/registry_tests.rs"] +mod registry_tests; + +#[cfg(test)] +#[path = "file_tail/supervisor_tests.rs"] +mod supervisor_tests; diff --git a/src/file_tail/models.rs b/src/file_tail/models.rs new file mode 100644 index 00000000..891a5e0a --- /dev/null +++ b/src/file_tail/models.rs @@ -0,0 +1,333 @@ +use serde::{Deserialize, Serialize}; + +const SYSLOG_FACILITIES: &[&str] = &[ + "kern", + "user", + "mail", + "daemon", + "auth", + "syslog", + "lpr", + "news", + "uucp", + "cron", + "authpriv", + "ftp", + "ntp", + "security", + "console", + "solaris-cron", + "local0", + "local1", + "local2", + "local3", + "local4", + "local5", + "local6", + "local7", +]; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FileTailSource { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: String, + pub start_at_end: bool, + pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkpoint_dev: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkpoint_ino: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub checkpoint_offset: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FileTailOp { + List, + Add, + Remove, + Enable, + Disable, + Status, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FileTailRequest { + pub op: FileTailOp, + pub id: Option, + pub path: Option, + pub tag: Option, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct FileTailAddRequest { + pub id: String, + pub path: String, + pub tag: String, + pub hostname: Option, + pub facility: Option, + pub severity: Option, + pub start_at_end: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileTailStatus { + pub id: String, + pub running: bool, + pub last_line_at: Option, + pub last_read_at: Option, + pub last_checkpoint_at: Option, + pub blocked_on_writer_since: Option, + pub last_error: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FileTailResponse { + pub sources: Vec, + pub statuses: Vec, +} + +impl FileTailSource { + pub(crate) fn from_add(req: FileTailAddRequest, now: &str) -> Result { + validate_id(&req.id)?; + if req.path.is_empty() || req.tag.is_empty() || req.hostname.is_none() { + return Err("file_tails op=add requires id, path, tag, and hostname".into()); + } + if let Some(facility) = req.facility.as_deref() { + validate_facility(facility)?; + } + let severity = req + .severity + .as_deref() + .map(|severity| { + normalize_severity(Some(severity)).ok_or_else(|| { + "file_tails severity must be one of emerg, alert, crit, err, warning, notice, info, debug".to_string() + }) + }) + .transpose()? + .unwrap_or_else(|| "info".to_string()); + + let hostname = req + .hostname + .as_deref() + .ok_or_else(|| "file_tails op=add requires id, path, tag, and hostname".to_string()) + .and_then(normalize_hostname)?; + + Ok(Self { + id: req.id, + path: req.path, + tag: req.tag, + hostname: Some(hostname), + facility: Some(req.facility.unwrap_or_else(|| "local7".to_string())), + severity, + start_at_end: req.start_at_end.unwrap_or(true), + enabled: true, + checkpoint_dev: None, + checkpoint_ino: None, + checkpoint_offset: None, + created_at: now.to_string(), + updated_at: now.to_string(), + }) + } + + pub(crate) fn same_definition(&self, other: &Self) -> bool { + self.id == other.id + && self.path == other.path + && self.tag == other.tag + && self.hostname == other.hostname + && self.facility == other.facility + && self.severity == other.severity + && self.start_at_end == other.start_at_end + && self.enabled == other.enabled + } +} + +impl FileTailRequest { + pub fn list() -> Self { + Self { + op: FileTailOp::List, + id: None, + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + } + } + + pub fn status() -> Self { + Self { + op: FileTailOp::Status, + ..Self::list() + } + } + + pub fn id_op(op: FileTailOp, id: String) -> Self { + Self { + op, + id: Some(id), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + } + } + + pub fn add(add: FileTailAddRequest) -> Self { + Self { + op: FileTailOp::Add, + id: Some(add.id), + path: Some(add.path), + tag: Some(add.tag), + hostname: add.hostname, + facility: add.facility, + severity: add.severity, + start_at_end: add.start_at_end, + } + } + + pub(crate) fn required_id(&self) -> Result<&str, String> { + let id = self + .id + .as_deref() + .ok_or_else(|| format!("file_tails op={:?} requires id", self.op).to_lowercase())?; + validate_id(id)?; + Ok(id) + } + + pub(crate) fn validate_shape(&self) -> Result<(), String> { + match self.op { + FileTailOp::List | FileTailOp::Status => { + if self.id.is_some() + || self.path.is_some() + || self.tag.is_some() + || self.hostname.is_some() + || self.facility.is_some() + || self.severity.is_some() + || self.start_at_end.is_some() + { + return Err(format!( + "file_tails op={:?} does not accept source fields", + self.op + ) + .to_lowercase()); + } + Ok(()) + } + FileTailOp::Add => Ok(()), + FileTailOp::Remove | FileTailOp::Enable | FileTailOp::Disable => { + if self.path.is_some() + || self.tag.is_some() + || self.hostname.is_some() + || self.facility.is_some() + || self.severity.is_some() + || self.start_at_end.is_some() + { + return Err( + format!("file_tails op={:?} accepts only id", self.op).to_lowercase() + ); + } + Ok(()) + } + } + } + + pub(crate) fn into_add(self) -> Result { + let id = self + .id + .ok_or_else(|| "file_tails op=add requires id, path, tag, and hostname".to_string())?; + validate_id(&id)?; + let path = self + .path + .ok_or_else(|| "file_tails op=add requires id, path, tag, and hostname".to_string())?; + let tag = self + .tag + .ok_or_else(|| "file_tails op=add requires id, path, tag, and hostname".to_string())?; + let hostname = self + .hostname + .ok_or_else(|| "file_tails op=add requires id, path, tag, and hostname".to_string())?; + if path.is_empty() || tag.is_empty() || hostname.trim().is_empty() { + return Err("file_tails op=add requires id, path, tag, and hostname".into()); + } + Ok(FileTailAddRequest { + id, + path, + tag, + hostname: Some(hostname), + facility: self.facility, + severity: self.severity, + start_at_end: self.start_at_end, + }) + } +} + +fn normalize_severity(severity: Option<&str>) -> Option { + let severity = severity?; + match severity.to_ascii_lowercase().as_str() { + "emerg" | "emergency" => Some("emerg".to_string()), + "alert" => Some("alert".to_string()), + "crit" | "critical" => Some("crit".to_string()), + "err" | "error" | "fatal" | "panic" => Some("err".to_string()), + "warning" | "warn" => Some("warning".to_string()), + "notice" => Some("notice".to_string()), + "info" | "informational" => Some("info".to_string()), + "debug" => Some("debug".to_string()), + _ => None, + } +} + +fn validate_facility(facility: &str) -> Result<(), String> { + if SYSLOG_FACILITIES.contains(&facility) { + return Ok(()); + } + Err("file_tails facility must be a canonical syslog facility".into()) +} + +fn validate_id(id: &str) -> Result<(), String> { + if id.is_empty() + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + { + return Err( + "file_tails id must contain only ASCII letters, digits, dot, underscore, or dash" + .into(), + ); + } + Ok(()) +} + +fn normalize_hostname(hostname: &str) -> Result { + let hostname = hostname.trim().to_ascii_lowercase(); + if hostname.is_empty() + || hostname.len() > 255 + || !hostname + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) + || hostname.starts_with(['.', '-', '_']) + || hostname.ends_with(['.', '-', '_']) + { + return Err( + "file_tails hostname must be URI-safe ASCII letters, digits, dot, underscore, or dash" + .into(), + ); + } + Ok(hostname) +} diff --git a/src/file_tail/models_tests.rs b/src/file_tail/models_tests.rs new file mode 100644 index 00000000..b4a2848d --- /dev/null +++ b/src/file_tail/models_tests.rs @@ -0,0 +1,142 @@ +use super::models::*; + +#[test] +fn add_request_builds_enabled_source_with_defaults() { + let req = FileTailAddRequest { + id: "swag-access".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: None, + severity: None, + start_at_end: None, + }; + + let source = FileTailSource::from_add(req, "2026-06-11T20:00:00Z").unwrap(); + + assert_eq!(source.id, "swag-access"); + assert_eq!(source.path, "/mnt/appdata/swag/log/nginx/access.log"); + assert_eq!(source.tag, "swag-access"); + assert_eq!(source.hostname.as_deref(), Some("squirts")); + assert_eq!(source.facility.as_deref(), Some("local7")); + assert_eq!(source.severity, "info"); + assert!(source.start_at_end); + assert!(source.enabled); + assert_eq!(source.created_at, "2026-06-11T20:00:00Z"); + assert_eq!(source.updated_at, "2026-06-11T20:00:00Z"); +} + +#[test] +fn add_request_normalizes_and_validates_hostname() { + let source = FileTailSource::from_add( + FileTailAddRequest { + id: "swag-access".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "swag-access".into(), + hostname: Some(" Squirts.LOCAL ".into()), + facility: None, + severity: None, + start_at_end: None, + }, + "2026-06-11T20:00:00Z", + ) + .unwrap(); + assert_eq!(source.hostname.as_deref(), Some("squirts.local")); + + let err = FileTailSource::from_add( + FileTailAddRequest { + id: "bad-host".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "bad-host".into(), + hostname: Some("bad host/name".into()), + facility: None, + severity: None, + start_at_end: None, + }, + "2026-06-11T20:00:00Z", + ) + .unwrap_err(); + assert!(err.contains("hostname must be URI-safe")); +} + +#[test] +fn file_tail_request_rejects_missing_fields_for_add() { + let req = FileTailRequest { + op: FileTailOp::Add, + id: None, + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }; + + assert_eq!( + req.into_add().unwrap_err(), + "file_tails op=add requires id, path, tag, and hostname" + ); +} + +#[test] +fn add_request_rejects_missing_hostname() { + let err = FileTailSource::from_add( + FileTailAddRequest { + id: "swag-access".into(), + path: "/mnt/appdata/swag/log/nginx/access.log".into(), + tag: "swag-access".into(), + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }, + "2026-06-11T20:00:00Z", + ) + .unwrap_err(); + + assert_eq!( + err, + "file_tails op=add requires id, path, tag, and hostname" + ); +} + +#[test] +fn file_tail_request_rejects_path_traversal_ids() { + let req = FileTailRequest { + op: FileTailOp::Remove, + id: Some("../swag".into()), + path: None, + tag: None, + hostname: None, + facility: None, + severity: None, + start_at_end: None, + }; + + assert_eq!( + req.required_id().unwrap_err(), + "file_tails id must contain only ASCII letters, digits, dot, underscore, or dash" + ); +} + +#[test] +fn file_tail_request_rejects_extra_fields_for_id_ops() { + let mut req = FileTailRequest::id_op(FileTailOp::Remove, "swag".into()); + req.path = Some("/tmp/access.log".into()); + + assert_eq!( + req.validate_shape().unwrap_err(), + "file_tails op=remove accepts only id" + ); +} + +#[test] +fn file_tail_request_rejects_extra_fields_for_list_ops() { + let mut req = FileTailRequest::list(); + req.tag = Some("swag".into()); + + assert_eq!( + req.validate_shape().unwrap_err(), + "file_tails op=list does not accept source fields" + ); +} diff --git a/src/file_tail/path_policy.rs b/src/file_tail/path_policy.rs new file mode 100644 index 00000000..f3fa3c15 --- /dev/null +++ b/src/file_tail/path_policy.rs @@ -0,0 +1,103 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Result, bail}; +use std::os::unix::fs::MetadataExt; + +pub(crate) fn validate_file_tail_path(path: &str) -> Result<()> { + let raw = Path::new(path); + if !raw.is_absolute() { + bail!("file-tail path must be absolute"); + } + let symlink_metadata = std::fs::symlink_metadata(raw) + .map_err(|err| anyhow::anyhow!("file-tail path is not readable: {path}: {err}"))?; + if symlink_metadata.file_type().is_symlink() { + bail!("file-tail path must not be a symlink"); + } + if !symlink_metadata.file_type().is_file() { + bail!("file-tail path must be a regular file"); + } + + let canonical = std::fs::canonicalize(raw) + .map_err(|err| anyhow::anyhow!("file-tail path could not be canonicalized: {err}"))?; + let denied = [ + "/data", + "/cortex-home", + "/home/cortex/.ssh", + "/home/cortex/workspace", + ]; + if denied + .iter() + .any(|root| canonical.starts_with(Path::new(root))) + { + bail!("file-tail path is under a sensitive cortex mount"); + } + + let allowed_roots = canonical_allowed_file_tail_roots(); + if allowed_roots.iter().any(|root| canonical.starts_with(root)) { + return Ok(()); + } + + bail!( + "file-tail path is outside allowed roots: {}", + canonical.display() + ); +} + +pub(crate) fn validate_opened_file_tail_path( + path: &str, + opened_metadata: &std::fs::Metadata, +) -> Result<()> { + if !opened_metadata.file_type().is_file() { + bail!("file-tail opened path must be a regular file"); + } + validate_file_tail_path(path)?; + let path_metadata = std::fs::symlink_metadata(path) + .map_err(|err| anyhow::anyhow!("file-tail path is not readable: {path}: {err}"))?; + if opened_metadata.dev() != path_metadata.dev() || opened_metadata.ino() != path_metadata.ino() + { + bail!("file-tail path changed while opening"); + } + Ok(()) +} + +fn canonical_allowed_file_tail_roots() -> Vec { + allowed_file_tail_roots() + .into_iter() + .filter_map(|root| std::fs::canonicalize(root).ok()) + .collect() +} + +#[cfg(not(test))] +fn allowed_file_tail_roots() -> Vec { + std::env::var("CORTEX_FILE_TAIL_ALLOWED_ROOTS") + .ok() + .filter(|value| !value.trim().is_empty()) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|root| !root.is_empty()) + .map(PathBuf::from) + .collect() + }) + .unwrap_or_else(|| { + let roots = vec![PathBuf::from("/file-tail-root")]; + roots + }) +} + +#[cfg(test)] +fn allowed_file_tail_roots() -> Vec { + std::env::var("CORTEX_FILE_TAIL_ALLOWED_ROOTS") + .ok() + .filter(|value| !value.trim().is_empty()) + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|root| !root.is_empty()) + .map(PathBuf::from) + .collect() + }) + .unwrap_or_else(|| vec![PathBuf::from("/file-tail-root"), std::env::temp_dir()]) +} diff --git a/src/file_tail/path_policy_tests.rs b/src/file_tail/path_policy_tests.rs new file mode 100644 index 00000000..3c4e92a5 --- /dev/null +++ b/src/file_tail/path_policy_tests.rs @@ -0,0 +1,80 @@ +use serial_test::serial; + +use super::path_policy::validate_file_tail_path; + +struct EnvGuard { + key: &'static str, + value: Option, +} + +impl EnvGuard { + fn set(key: &'static str, value: String) -> Self { + let guard = Self { + key, + value: std::env::var(key).ok(), + }; + unsafe { + std::env::set_var(key, value); + } + guard + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + unsafe { + if let Some(value) = &self.value { + std::env::set_var(self.key, value); + } else { + std::env::remove_var(self.key); + } + } + } +} + +#[test] +#[serial] +fn env_allowed_root_allows_file_inside_root() { + let temp = tempfile::tempdir().unwrap(); + let log_path = temp.path().join("app.log"); + std::fs::write(&log_path, "hello\n").unwrap(); + let _guard = EnvGuard::set( + "CORTEX_FILE_TAIL_ALLOWED_ROOTS", + temp.path().to_string_lossy().into_owned(), + ); + + validate_file_tail_path(&log_path.to_string_lossy()).unwrap(); +} + +#[test] +#[serial] +fn symlink_allowed_root_is_canonicalized() { + let temp = tempfile::tempdir().unwrap(); + let real_root = temp.path().join("real"); + let link_root = temp.path().join("link"); + std::fs::create_dir(&real_root).unwrap(); + std::os::unix::fs::symlink(&real_root, &link_root).unwrap(); + let log_path = real_root.join("app.log"); + std::fs::write(&log_path, "hello\n").unwrap(); + let _guard = EnvGuard::set( + "CORTEX_FILE_TAIL_ALLOWED_ROOTS", + link_root.to_string_lossy().into_owned(), + ); + + validate_file_tail_path(&log_path.to_string_lossy()).unwrap(); +} + +#[test] +#[serial] +fn sensitive_mount_is_denied_even_when_env_allows_parent() { + let Ok(temp) = tempfile::tempdir_in("/data") else { + return; + }; + let log_path = temp.path().join("app.log"); + std::fs::write(&log_path, "hello\n").unwrap(); + let _guard = EnvGuard::set("CORTEX_FILE_TAIL_ALLOWED_ROOTS", "/".to_string()); + + let err = validate_file_tail_path(&log_path.to_string_lossy()).unwrap_err(); + + assert!(err.to_string().contains("sensitive cortex mount")); +} diff --git a/src/file_tail/registry.rs b/src/file_tail/registry.rs new file mode 100644 index 00000000..a07e4d24 --- /dev/null +++ b/src/file_tail/registry.rs @@ -0,0 +1,116 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; + +use super::models::FileTailSource; + +#[derive(Debug)] +pub(crate) struct FileTailRegistry { + path: PathBuf, + lock: Mutex<()>, +} + +impl FileTailRegistry { + pub(crate) fn new(path: PathBuf) -> Self { + Self { + path, + lock: Mutex::new(()), + } + } + + pub(crate) fn path_from_storage_db(db_path: &Path) -> PathBuf { + db_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join("file-tails.json") + } + + pub(crate) fn list(&self) -> Result> { + let _guard = self.lock.lock(); + self.read_locked() + } + + pub(crate) fn get(&self, id: &str) -> Result> { + let _guard = self.lock.lock(); + Ok(self + .read_locked()? + .into_iter() + .find(|source| source.id == id)) + } + + pub(crate) fn upsert(&self, source: FileTailSource) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + sources.retain(|existing| existing.id != source.id); + sources.push(source); + sources.sort_by(|a, b| a.id.cmp(&b.id)); + self.write_locked(&sources) + } + + pub(crate) fn remove(&self, id: &str) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + let before = sources.len(); + sources.retain(|existing| existing.id != id); + if sources.len() == before { + anyhow::bail!("file tail source not found: {id}"); + } + self.write_locked(&sources) + } + + pub(crate) fn set_enabled(&self, id: &str, enabled: bool, now: &str) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + let source = sources + .iter_mut() + .find(|source| source.id == id) + .with_context(|| format!("file tail source not found: {id}"))?; + source.enabled = enabled; + source.updated_at = now.to_string(); + self.write_locked(&sources) + } + + pub(crate) fn update_checkpoint( + &self, + id: &str, + dev: u64, + ino: u64, + offset: u64, + now: &str, + ) -> Result<()> { + let _guard = self.lock.lock(); + let mut sources = self.read_locked()?; + let source = sources + .iter_mut() + .find(|source| source.id == id) + .with_context(|| format!("file tail source not found: {id}"))?; + source.checkpoint_dev = Some(dev); + source.checkpoint_ino = Some(ino); + source.checkpoint_offset = Some(offset); + source.updated_at = now.to_string(); + self.write_locked(&sources) + } + + fn read_locked(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let raw = std::fs::read_to_string(&self.path) + .with_context(|| format!("read {}", self.path.display()))?; + serde_json::from_str(&raw).with_context(|| format!("parse {}", self.path.display())) + } + + fn write_locked(&self, sources: &[FileTailSource]) -> Result<()> { + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("create {}", parent.display()))?; + } + let tmp = self.path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(sources)?; + std::fs::write(&tmp, body).with_context(|| format!("write {}", tmp.display()))?; + std::fs::rename(&tmp, &self.path) + .with_context(|| format!("replace {}", self.path.display()))?; + Ok(()) + } +} diff --git a/src/file_tail/registry_tests.rs b/src/file_tail/registry_tests.rs new file mode 100644 index 00000000..c87bac36 --- /dev/null +++ b/src/file_tail/registry_tests.rs @@ -0,0 +1,68 @@ +use super::models::{FileTailAddRequest, FileTailSource}; +use super::registry::FileTailRegistry; + +#[test] +fn registry_adds_lists_and_removes_sources() { + let temp = tempfile::tempdir().unwrap(); + let registry = FileTailRegistry::new(temp.path().join("file-tails.json")); + let source = FileTailSource::from_add( + FileTailAddRequest { + id: "swag-access".into(), + path: "/tmp/access.log".into(), + tag: "swag-access".into(), + hostname: Some("squirts".into()), + facility: None, + severity: None, + start_at_end: None, + }, + "2026-06-11T20:00:00Z", + ) + .unwrap(); + + registry.upsert(source.clone()).unwrap(); + assert_eq!(registry.list().unwrap(), vec![source]); + + registry.remove("swag-access").unwrap(); + assert!(registry.list().unwrap().is_empty()); +} + +#[test] +fn registry_persists_across_instances() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("file-tails.json"); + let registry = FileTailRegistry::new(path.clone()); + registry + .upsert( + FileTailSource::from_add( + FileTailAddRequest { + id: "authelia".into(), + path: "/tmp/authelia.log".into(), + tag: "authelia".into(), + hostname: Some("squirts".into()), + facility: Some("local5".into()), + severity: Some("info".into()), + start_at_end: Some(false), + }, + "2026-06-11T20:00:00Z", + ) + .unwrap(), + ) + .unwrap(); + + let reloaded = FileTailRegistry::new(path); + let sources = reloaded.list().unwrap(); + assert_eq!(sources.len(), 1); + assert_eq!(sources[0].id, "authelia"); + assert_eq!(sources[0].facility.as_deref(), Some("local5")); + assert!(!sources[0].start_at_end); +} + +#[test] +fn registry_remove_missing_source_returns_error() { + let temp = tempfile::tempdir().unwrap(); + let registry = FileTailRegistry::new(temp.path().join("file-tails.json")); + + let err = registry.remove("missing").unwrap_err(); + + assert!(err.to_string().contains("not found")); +} diff --git a/src/file_tail/supervisor.rs b/src/file_tail/supervisor.rs new file mode 100644 index 00000000..bb50573f --- /dev/null +++ b/src/file_tail/supervisor.rs @@ -0,0 +1,685 @@ +use std::collections::HashMap; +use std::io::ErrorKind; +use std::os::unix::fs::MetadataExt; +use std::os::unix::fs::OpenOptionsExt; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncReadExt, AsyncSeekExt, BufReader}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use crate::db::LogBatchEntry; +use crate::enrich::{SourceKind, stamp_source_kind}; +use crate::ingest::IngestTx; +use crate::ingest_metadata::bounded_metadata_json; + +use super::models::{FileTailSource, FileTailStatus}; +use super::path_policy::{validate_file_tail_path, validate_opened_file_tail_path}; +use super::registry::FileTailRegistry; + +const FILE_TAIL_FINGERPRINT_BYTES: usize = 256; +const FILE_TAIL_ROTATION_GRACE: Duration = Duration::from_millis(1000); + +#[derive(Clone)] +pub(crate) struct FileTailSupervisor { + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + tasks: Arc>>, + max_line_bytes: usize, +} + +struct TailTask { + handle: JoinHandle<()>, + status: Arc>, + source: FileTailSource, +} + +impl FileTailSupervisor { + pub(crate) fn new( + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + max_line_bytes: usize, + ) -> Self { + Self { + registry, + ingest, + token, + tasks: Arc::new(Mutex::new(HashMap::new())), + max_line_bytes, + } + } + + pub(crate) fn statuses(&self) -> Vec { + let mut out: Vec<_> = self + .tasks + .lock() + .values() + .map(|task| task.status.lock().clone()) + .collect(); + out.sort_by(|a, b| a.id.cmp(&b.id)); + out + } + + pub(crate) fn shutdown(&self) { + self.token.cancel(); + let mut tasks = self.tasks.lock(); + for (_, task) in tasks.drain() { + task.status.lock().running = false; + task.handle.abort(); + } + } + + pub(crate) fn reconcile(&self) -> Result<()> { + let sources = self.registry.list()?; + let enabled: HashMap = sources + .iter() + .filter(|source| source.enabled) + .map(|source| (source.id.clone(), source.clone())) + .collect(); + + let mut tasks = self.tasks.lock(); + tasks.retain(|id, task| { + let keep_running = enabled + .get(id) + .is_some_and(|source| source.same_definition(&task.source)); + if !keep_running { + task.status.lock().running = false; + task.handle.abort(); + } + keep_running + }); + for source in sources { + if source.enabled && !tasks.contains_key(&source.id) { + self.ensure_initial_checkpoint(&source)?; + let (id, task) = self.build_task(source); + tasks.insert(id, task); + } + } + Ok(()) + } + + fn ensure_initial_checkpoint(&self, source: &FileTailSource) -> Result<()> { + let has_checkpoint = source.checkpoint_dev.is_some() + || source.checkpoint_ino.is_some() + || source.checkpoint_offset.is_some(); + if has_checkpoint { + return Ok(()); + } + + let file = open_validated_tail_file_sync(&source.path)?; + let metadata = file.metadata()?; + let offset = if source.start_at_end { + metadata.len() + } else { + 0 + }; + self.registry.update_checkpoint( + &source.id, + metadata.dev(), + metadata.ino(), + offset, + &now_iso(), + ) + } + + fn build_task(&self, source: FileTailSource) -> (String, TailTask) { + let id = source.id.clone(); + let task_source = source.clone(); + let status = Arc::new(Mutex::new(FileTailStatus { + id: id.clone(), + running: true, + last_line_at: None, + last_read_at: None, + last_checkpoint_at: None, + blocked_on_writer_since: None, + last_error: None, + })); + let task_status = Arc::clone(&status); + let ingest = self.ingest.clone(); + let token = self.token.clone(); + let registry = Arc::clone(&self.registry); + let max_line_bytes = self.max_line_bytes; + let task_id = id.clone(); + let handle = tokio::spawn(async move { + tail_file_loop( + task_id, + registry, + ingest, + token, + task_status, + max_line_bytes, + ) + .await; + }); + ( + id, + TailTask { + handle, + status, + source: task_source, + }, + ) + } + + #[cfg(test)] + pub(crate) fn running_source_for_test(&self, id: &str) -> Option { + self.tasks.lock().get(id).map(|task| task.source.clone()) + } +} + +async fn tail_file_loop( + source_id: String, + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + status: Arc>, + max_line_bytes: usize, +) { + loop { + if token.is_cancelled() { + status.lock().running = false; + return; + } + let source = match registry.get(&source_id) { + Ok(Some(source)) if source.enabled => source, + Ok(_) => { + status.lock().running = false; + return; + } + Err(err) => { + tracing::error!( + source_id = %source_id, + error = %err, + "file-tail source reload failed; retrying" + ); + status.lock().last_error = Some(err.to_string()); + tokio::select! { + _ = token.cancelled() => { + status.lock().running = false; + return; + } + _ = tokio::time::sleep(Duration::from_secs(5)) => {} + } + continue; + } + }; + match tail_file_until_cancelled( + &source, + Arc::clone(®istry), + ingest.clone(), + token.clone(), + Arc::clone(&status), + max_line_bytes, + ) + .await + { + Ok(()) => { + status.lock().running = false; + return; + } + Err(err) => { + tracing::error!( + source_id = %source.id, + path = %source.path, + error = %err, + "file-tail source failed; retrying" + ); + status.lock().last_error = Some(err.to_string()); + tokio::select! { + _ = token.cancelled() => { + status.lock().running = false; + return; + } + _ = tokio::time::sleep(Duration::from_secs(5)) => {} + } + } + } + } +} + +async fn tail_file_until_cancelled( + source: &FileTailSource, + registry: Arc, + ingest: IngestTx, + token: CancellationToken, + status: Arc>, + max_line_bytes: usize, +) -> Result<()> { + let opened = open_tail_file(source, true) + .await + .with_context(|| format!("open {}", source.path))?; + let mut reader = BufReader::new(opened.file); + let mut position = opened.position; + let mut identity = opened.identity; + let mut fingerprint = opened.fingerprint; + let mut line = Vec::new(); + let mut pending_rotation_since: Option = None; + loop { + tokio::select! { + _ = token.cancelled() => return Ok(()), + read = read_bounded_line(&mut reader, &mut line, max_line_bytes) => { + let read = read?; + if read.bytes_read == 0 { + if path_identity_changed(source, identity).await? { + let since = pending_rotation_since.get_or_insert_with(Instant::now); + if since.elapsed() < FILE_TAIL_ROTATION_GRACE { + tokio::time::sleep(Duration::from_millis(200)).await; + continue; + } + } else { + pending_rotation_since = None; + } + if let Some(next) = reopen_if_rotated_or_truncated(source, identity, position, &fingerprint).await? { + if !line.is_empty() { + let now = now_iso(); + let partial = PartialLineBeforeReopen { + source, + registry: ®istry, + ingest: &ingest, + status: &status, + line: &line, + identity, + position, + now: &now, + }; + ingest_partial_line_before_reopen(partial).await?; + } + reader = BufReader::new(next.file); + position = next.position; + identity = next.identity; + fingerprint = next.fingerprint; + pending_rotation_since = None; + line.clear(); + } else { + tokio::time::sleep(Duration::from_millis(500)).await; + } + continue; + } + position = position.saturating_add(read.bytes_read as u64); + pending_rotation_since = None; + if !read.complete { + tokio::time::sleep(Duration::from_millis(500)).await; + continue; + } + let msg = String::from_utf8_lossy(&line); + let msg = msg.trim_end_matches(['\r', '\n']); + if msg.is_empty() { + line.clear(); + continue; + } + let now = now_iso(); + let entry = file_tail_line_to_entry(source, msg, &now); + { + let mut status = status.lock(); + status.last_read_at = Some(now.clone()); + status.blocked_on_writer_since = Some(now.clone()); + } + ingest.send_durable(entry).await?; + registry.update_checkpoint(&source.id, identity.dev, identity.ino, position, &now)?; + line.clear(); + let mut status = status.lock(); + status.last_line_at = Some(now); + status.last_checkpoint_at = status.last_line_at.clone(); + status.blocked_on_writer_since = None; + status.last_error = if read.truncated { + Some(format!( + "truncated oversized line from {} to {max_line_bytes} bytes", + source.path + )) + } else { + None + }; + } + } + } +} + +struct PartialLineBeforeReopen<'a> { + source: &'a FileTailSource, + registry: &'a FileTailRegistry, + ingest: &'a IngestTx, + status: &'a Mutex, + line: &'a [u8], + identity: FileIdentity, + position: u64, + now: &'a str, +} + +async fn ingest_partial_line_before_reopen(partial: PartialLineBeforeReopen<'_>) -> Result<()> { + let msg = String::from_utf8_lossy(partial.line); + let msg = msg.trim_end_matches(['\r', '\n']); + if msg.is_empty() { + return Ok(()); + } + partial + .ingest + .send_durable(file_tail_line_to_entry(partial.source, msg, partial.now)) + .await?; + partial.registry.update_checkpoint( + &partial.source.id, + partial.identity.dev, + partial.identity.ino, + partial.position, + partial.now, + )?; + let mut status = partial.status.lock(); + status.last_line_at = Some(partial.now.to_string()); + status.last_error = Some(format!( + "ingested unterminated partial line before rotation/truncation for {}", + partial.source.path + )); + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct FileIdentity { + pub(crate) dev: u64, + pub(crate) ino: u64, +} + +#[derive(Debug)] +pub(crate) struct OpenedTailFile { + pub(crate) file: tokio::fs::File, + pub(crate) identity: FileIdentity, + pub(crate) position: u64, + pub(crate) fingerprint: Vec, +} + +pub(crate) struct BoundedLine { + pub(crate) bytes_read: usize, + pub(crate) truncated: bool, + pub(crate) complete: bool, +} + +pub(crate) async fn open_tail_file( + source: &FileTailSource, + first_open: bool, +) -> Result { + let mut file = open_validated_tail_file(&source.path).await?; + let metadata = file.metadata().await?; + let identity = FileIdentity { + dev: metadata.dev(), + ino: metadata.ino(), + }; + let fingerprint = file_prefix_fingerprint(&mut file).await?; + let checkpoint_matches = source.checkpoint_dev == Some(identity.dev) + && source.checkpoint_ino == Some(identity.ino) + && source + .checkpoint_offset + .is_some_and(|offset| offset <= metadata.len()); + let has_checkpoint = source.checkpoint_dev.is_some() + || source.checkpoint_ino.is_some() + || source.checkpoint_offset.is_some(); + let position = if checkpoint_matches { + source.checkpoint_offset.unwrap_or(0) + } else if has_checkpoint { + 0 + } else if first_open && source.start_at_end { + metadata.len() + } else { + 0 + }; + file.seek(std::io::SeekFrom::Start(position)).await?; + Ok(OpenedTailFile { + file, + identity, + position, + fingerprint, + }) +} + +pub(crate) async fn reopen_if_rotated_or_truncated( + source: &FileTailSource, + identity: FileIdentity, + position: u64, + fingerprint: &[u8], +) -> Result> { + let metadata = match tokio::fs::metadata(&source.path).await { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => { + anyhow::bail!("file-tail source disappeared: {}", source.path); + } + Err(err) => return Err(err.into()), + }; + let current = FileIdentity { + dev: metadata.dev(), + ino: metadata.ino(), + }; + if current != identity || metadata.len() < position { + return reopen_from_start(source).await.map(Some); + } + if position > 0 { + let mut file = open_validated_tail_file(&source.path).await?; + let current_fingerprint = file_prefix_fingerprint(&mut file).await?; + if current_fingerprint != fingerprint { + let metadata = file.metadata().await?; + file.seek(std::io::SeekFrom::Start(0)).await?; + return Ok(Some(OpenedTailFile { + file, + identity: FileIdentity { + dev: metadata.dev(), + ino: metadata.ino(), + }, + position: 0, + fingerprint: current_fingerprint, + })); + } + } + Ok(None) +} + +async fn path_identity_changed(source: &FileTailSource, identity: FileIdentity) -> Result { + let metadata = match tokio::fs::metadata(&source.path).await { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => { + anyhow::bail!("file-tail source disappeared: {}", source.path); + } + Err(err) => return Err(err.into()), + }; + Ok(FileIdentity { + dev: metadata.dev(), + ino: metadata.ino(), + } != identity) +} + +async fn reopen_from_start(source: &FileTailSource) -> Result { + let mut file = open_validated_tail_file(&source.path).await?; + let metadata = file.metadata().await?; + let fingerprint = file_prefix_fingerprint(&mut file).await?; + file.seek(std::io::SeekFrom::Start(0)).await?; + Ok(OpenedTailFile { + file, + identity: FileIdentity { + dev: metadata.dev(), + ino: metadata.ino(), + }, + position: 0, + fingerprint, + }) +} + +async fn file_prefix_fingerprint(file: &mut tokio::fs::File) -> std::io::Result> { + let mut buf = vec![0; FILE_TAIL_FINGERPRINT_BYTES]; + file.seek(std::io::SeekFrom::Start(0)).await?; + let n = file.read(&mut buf).await?; + buf.truncate(n); + file.seek(std::io::SeekFrom::Start(0)).await?; + Ok(buf) +} + +async fn open_validated_tail_file(path: &str) -> Result { + validate_file_tail_path(path)?; + let path = path.to_string(); + let std_file = tokio::task::spawn_blocking({ + let path = path.clone(); + move || { + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(&path) + } + }) + .await??; + let metadata = std_file.metadata()?; + validate_opened_file_tail_path(&path, &metadata)?; + Ok(tokio::fs::File::from_std(std_file)) +} + +fn open_validated_tail_file_sync(path: &str) -> Result { + validate_file_tail_path(path)?; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + let metadata = file.metadata()?; + validate_opened_file_tail_path(path, &metadata)?; + Ok(file) +} + +pub(crate) async fn read_bounded_line( + reader: &mut R, + out: &mut Vec, + max_line_bytes: usize, +) -> std::io::Result { + let mut bytes_read = 0; + let mut truncated = false; + + loop { + let available = reader.fill_buf().await?; + if available.is_empty() { + return Ok(BoundedLine { + bytes_read, + truncated, + complete: false, + }); + } + + let newline_pos = available.iter().position(|byte| *byte == b'\n'); + let consume_len = newline_pos.map_or(available.len(), |pos| pos + 1); + let remaining = max_line_bytes.saturating_sub(out.len()); + let copy_len = remaining.min(consume_len); + out.extend_from_slice(&available[..copy_len]); + if copy_len < consume_len { + truncated = true; + } + reader.consume(consume_len); + bytes_read += consume_len; + + if newline_pos.is_some() { + return Ok(BoundedLine { + bytes_read, + truncated, + complete: true, + }); + } + } +} + +pub(crate) fn file_tail_line_to_entry( + source: &FileTailSource, + line: &str, + now: &str, +) -> LogBatchEntry { + let hostname = source.hostname.clone().unwrap_or_else(local_hostname); + let source_hostname = source_identity_component(&hostname); + let path_basename = std::path::Path::new(&source.path) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("unknown"); + let metadata_json = bounded_metadata_json(serde_json::json!({ + "source_type": "file_tail", + "source_kind": SourceKind::FileTail.as_str(), + "file_tail_id": source.id, + "tag": source.tag, + "path_basename": path_basename, + })); + let mut entry = LogBatchEntry { + timestamp: now.to_string(), + hostname: hostname.clone(), + facility: source.facility.clone(), + severity: source.severity.clone(), + app_name: Some(source.tag.clone()), + process_id: None, + message: line.to_string(), + raw: line.to_string(), + source_ip: format!("file-tail://{source_hostname}/{}", source.id), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: Some(metadata_json), + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + stamp_source_kind(&mut entry, SourceKind::FileTail); + entry +} + +#[cfg(test)] +pub(crate) async fn tail_file_once_for_test( + source: FileTailSource, + ingest: IngestTx, +) -> Result<()> { + let file = tokio::fs::File::open(&source.path).await?; + let mut reader = BufReader::new(file); + let mut line = String::new(); + while reader.read_line(&mut line).await? > 0 { + let msg = line.trim_end_matches(['\r', '\n']); + if !msg.is_empty() { + ingest + .send(file_tail_line_to_entry( + &source, + msg, + "2026-06-11T20:01:00Z", + )) + .await?; + } + line.clear(); + } + Ok(()) +} + +fn now_iso() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +fn local_hostname() -> String { + std::env::var("HOSTNAME") + .ok() + .filter(|host| !host.trim().is_empty()) + .unwrap_or_else(|| "localhost".to_string()) +} + +fn source_identity_component(hostname: &str) -> String { + let normalized = hostname + .trim() + .to_ascii_lowercase() + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b'_') { + byte as char + } else { + '_' + } + }) + .collect::() + .trim_matches(['.', '-', '_']) + .to_string() + .chars() + .take(255) + .collect::(); + if normalized.is_empty() { + "localhost".to_string() + } else { + normalized + } +} diff --git a/src/file_tail/supervisor_tests.rs b/src/file_tail/supervisor_tests.rs new file mode 100644 index 00000000..97b35cc4 --- /dev/null +++ b/src/file_tail/supervisor_tests.rs @@ -0,0 +1,620 @@ +use std::os::unix::fs::MetadataExt; + +use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; + +use crate::db::LogBatchEntry; +use crate::ingest::IngestTx; + +use super::models::FileTailSource; +use super::registry::FileTailRegistry; +use super::supervisor::{ + FileTailSupervisor, file_tail_line_to_entry, open_tail_file, read_bounded_line, + reopen_if_rotated_or_truncated, tail_file_once_for_test, +}; + +fn source(id: &str, path: &str, tag: &str) -> FileTailSource { + FileTailSource { + id: id.into(), + path: path.into(), + tag: tag.into(), + hostname: Some("squirts".into()), + facility: Some("local4".into()), + severity: "info".into(), + start_at_end: true, + enabled: true, + checkpoint_dev: None, + checkpoint_ino: None, + checkpoint_offset: None, + created_at: "2026-06-11T20:00:00Z".into(), + updated_at: "2026-06-11T20:00:00Z".into(), + } +} + +#[test] +fn file_tail_line_to_entry_sets_expected_envelope() { + let source = source("swag-access", "/tmp/access.log", "swag-access"); + + let entry = file_tail_line_to_entry(&source, "GET / HTTP/1.1\" 401", "2026-06-11T20:01:00Z"); + + assert_eq!(entry.timestamp, "2026-06-11T20:01:00Z"); + assert_eq!(entry.hostname, "squirts"); + assert_eq!(entry.facility.as_deref(), Some("local4")); + assert_eq!(entry.severity, "info"); + assert_eq!(entry.app_name.as_deref(), Some("swag-access")); + assert_eq!(entry.message, "GET / HTTP/1.1\" 401"); + assert_eq!(entry.raw, "GET / HTTP/1.1\" 401"); + assert_eq!(entry.source_ip, "file-tail://squirts/swag-access"); + let metadata: serde_json::Value = + serde_json::from_str(entry.metadata_json.as_deref().unwrap()).unwrap(); + assert_eq!(metadata["source_kind"], "file-tail"); + assert_eq!(metadata["file_tail_id"], "swag-access"); + assert_eq!(metadata["tag"], "swag-access"); + assert_eq!(metadata["path_basename"], "access.log"); +} + +#[tokio::test] +async fn reconcile_restarts_task_when_source_definition_changes() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, _rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + tokio_util::sync::CancellationToken::new(), + 8192, + ); + + let first_path = temp.path().join("one.log"); + let second_path = temp.path().join("two.log"); + tokio::fs::write(&first_path, b"one\n").await.unwrap(); + tokio::fs::write(&second_path, b"two\n").await.unwrap(); + + registry + .upsert(source( + "swag-access", + &first_path.to_string_lossy(), + "swag-access", + )) + .unwrap(); + supervisor.reconcile().unwrap(); + assert_eq!( + supervisor + .running_source_for_test("swag-access") + .unwrap() + .path, + first_path.to_string_lossy() + ); + + registry + .upsert(source( + "swag-access", + &second_path.to_string_lossy(), + "swag-access", + )) + .unwrap(); + supervisor.reconcile().unwrap(); + assert_eq!( + supervisor + .running_source_for_test("swag-access") + .unwrap() + .path, + second_path.to_string_lossy() + ); + supervisor.shutdown(); +} + +#[tokio::test] +async fn reconcile_does_not_restart_task_for_checkpoint_updates() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, _rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + tokio_util::sync::CancellationToken::new(), + 8192, + ); + + let log_path = temp.path().join("one.log"); + tokio::fs::write(&log_path, b"one\n").await.unwrap(); + + registry + .upsert(source( + "swag-access", + &log_path.to_string_lossy(), + "swag-access", + )) + .unwrap(); + supervisor.reconcile().unwrap(); + registry + .update_checkpoint("swag-access", 1, 2, 3, "2026-06-11T20:01:00Z") + .unwrap(); + supervisor.reconcile().unwrap(); + assert_eq!( + supervisor + .running_source_for_test("swag-access") + .unwrap() + .path, + log_path.to_string_lossy() + ); + supervisor.shutdown(); +} + +#[tokio::test] +async fn supervisor_ingests_appended_line_and_updates_checkpoint() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let token = tokio_util::sync::CancellationToken::new(); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + token.clone(), + 8192, + ); + let log_path = temp.path().join("loop.log"); + tokio::fs::write(&log_path, b"").await.unwrap(); + + let mut source = source("loop", &log_path.to_string_lossy(), "loop"); + source.start_at_end = false; + registry.upsert(source).unwrap(); + supervisor.reconcile().unwrap(); + + let mut writer = tokio::fs::OpenOptions::new() + .append(true) + .open(&log_path) + .await + .unwrap(); + writer.write_all(b"hello from loop\n").await.unwrap(); + writer.flush().await.unwrap(); + + let entry = tokio::time::timeout(std::time::Duration::from_secs(3), rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(entry.message, "hello from loop"); + + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + let stored = registry.get("loop").unwrap().unwrap(); + if stored.checkpoint_offset.unwrap_or_default() > 0 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + let statuses = supervisor.statuses(); + assert_eq!(statuses.len(), 1); + assert!(statuses[0].last_line_at.is_some()); + + token.cancel(); + supervisor.shutdown(); +} + +#[tokio::test] +async fn reconcile_initializes_start_at_end_checkpoint_before_returning() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let token = tokio_util::sync::CancellationToken::new(); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + token.clone(), + 8192, + ); + let log_path = temp.path().join("loop.log"); + tokio::fs::write(&log_path, b"already here\n") + .await + .unwrap(); + + registry + .upsert(source("loop", &log_path.to_string_lossy(), "loop")) + .unwrap(); + supervisor.reconcile().unwrap(); + let initial = registry.get("loop").unwrap().unwrap(); + assert_eq!( + initial.checkpoint_offset, + Some("already here\n".len() as u64) + ); + + let mut writer = tokio::fs::OpenOptions::new() + .append(true) + .open(&log_path) + .await + .unwrap(); + writer.write_all(b"after reconcile\n").await.unwrap(); + writer.flush().await.unwrap(); + + let entry = tokio::time::timeout(std::time::Duration::from_secs(3), rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(entry.message, "after reconcile"); + + token.cancel(); + supervisor.shutdown(); +} + +#[tokio::test] +async fn supervisor_waits_for_durable_ack_before_checkpointing() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_envelope_sender_for_test(tx); + let token = tokio_util::sync::CancellationToken::new(); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + token.clone(), + 8192, + ); + let log_path = temp.path().join("loop.log"); + tokio::fs::write(&log_path, b"").await.unwrap(); + + let mut src = source("loop", &log_path.to_string_lossy(), "loop"); + src.start_at_end = false; + registry.upsert(src).unwrap(); + supervisor.reconcile().unwrap(); + + let mut writer = tokio::fs::OpenOptions::new() + .append(true) + .open(&log_path) + .await + .unwrap(); + writer.write_all(b"hello durable\n").await.unwrap(); + writer.flush().await.unwrap(); + + let envelope = tokio::time::timeout(std::time::Duration::from_secs(3), rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(envelope.entry.message, "hello durable"); + assert_eq!( + registry.get("loop").unwrap().unwrap().checkpoint_offset, + Some(0) + ); + + envelope.ack_success(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if registry + .get("loop") + .unwrap() + .unwrap() + .checkpoint_offset + .is_some_and(|offset| offset > 0) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + + token.cancel(); + supervisor.shutdown(); +} + +#[tokio::test] +async fn supervisor_reconcile_stops_disabled_and_removed_sources() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, _rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + tokio_util::sync::CancellationToken::new(), + 8192, + ); + let first_path = temp.path().join("one.log"); + let second_path = temp.path().join("two.log"); + tokio::fs::write(&first_path, b"").await.unwrap(); + tokio::fs::write(&second_path, b"").await.unwrap(); + + registry + .upsert(source("one", &first_path.to_string_lossy(), "one")) + .unwrap(); + registry + .upsert(source("two", &second_path.to_string_lossy(), "two")) + .unwrap(); + supervisor.reconcile().unwrap(); + assert!(supervisor.running_source_for_test("one").is_some()); + assert!(supervisor.running_source_for_test("two").is_some()); + + registry + .set_enabled("one", false, "2026-06-11T20:01:00Z") + .unwrap(); + supervisor.reconcile().unwrap(); + assert!(supervisor.running_source_for_test("one").is_none()); + assert!(supervisor.running_source_for_test("two").is_some()); + + registry.remove("two").unwrap(); + supervisor.reconcile().unwrap(); + assert!(supervisor.running_source_for_test("two").is_none()); + supervisor.shutdown(); +} + +#[tokio::test] +async fn open_tail_file_resumes_matching_checkpoint_before_start_at_end() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"old\nnew\n").await.unwrap(); + let metadata = std::fs::metadata(&file_path).unwrap(); + let mut src = source("app", &file_path.to_string_lossy(), "app"); + src.start_at_end = true; + src.checkpoint_dev = Some(metadata.dev()); + src.checkpoint_ino = Some(metadata.ino()); + src.checkpoint_offset = Some(4); + + let mut opened = open_tail_file(&src, true).await.unwrap(); + assert_eq!(opened.position, 4); + let mut rest = String::new(); + opened.file.read_to_string(&mut rest).await.unwrap(); + assert_eq!(rest, "new\n"); +} + +#[tokio::test] +async fn open_tail_file_restarts_at_beginning_when_checkpoint_identity_mismatches() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"replacement\n") + .await + .unwrap(); + let mut src = source("app", &file_path.to_string_lossy(), "app"); + src.start_at_end = true; + src.checkpoint_dev = Some(1); + src.checkpoint_ino = Some(2); + src.checkpoint_offset = Some(3); + + let mut opened = open_tail_file(&src, true).await.unwrap(); + + assert_eq!(opened.position, 0); + let mut rest = String::new(); + opened.file.read_to_string(&mut rest).await.unwrap(); + assert_eq!(rest, "replacement\n"); +} + +#[tokio::test] +async fn open_tail_file_rejects_symlink_paths() { + let temp = tempfile::tempdir().unwrap(); + let target_path = temp.path().join("target.log"); + let symlink_path = temp.path().join("link.log"); + tokio::fs::write(&target_path, b"secret\n").await.unwrap(); + std::os::unix::fs::symlink(&target_path, &symlink_path).unwrap(); + let src = source("app", &symlink_path.to_string_lossy(), "app"); + + let err = open_tail_file(&src, false).await.unwrap_err(); + + assert!(err.to_string().contains("must not be a symlink")); +} + +#[tokio::test] +async fn reopen_if_rotated_or_truncated_detects_rename_create_rotation() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"old\n").await.unwrap(); + let src = source("app", &file_path.to_string_lossy(), "app"); + let old = open_tail_file(&src, false).await.unwrap(); + + tokio::fs::rename(&file_path, temp.path().join("app.log.1")) + .await + .unwrap(); + tokio::fs::write(&file_path, b"new\n").await.unwrap(); + + let reopened = + reopen_if_rotated_or_truncated(&src, old.identity, old.position, &old.fingerprint) + .await + .unwrap() + .expect("rotation should reopen"); + assert_eq!(reopened.position, 0); + assert_ne!(reopened.identity, old.identity); +} + +#[tokio::test] +async fn reopen_if_rotated_or_truncated_detects_copytruncate() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"first line\nsecond line\n") + .await + .unwrap(); + let src = source("app", &file_path.to_string_lossy(), "app"); + let mut opened = open_tail_file(&src, false).await.unwrap(); + opened.position = 22; + tokio::fs::write(&file_path, b"new\n").await.unwrap(); + + let reopened = + reopen_if_rotated_or_truncated(&src, opened.identity, opened.position, &opened.fingerprint) + .await + .unwrap() + .expect("truncate should reopen"); + assert_eq!(reopened.position, 0); +} + +#[tokio::test] +async fn reopen_if_rotated_or_truncated_detects_same_inode_copytruncate_regrow() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + let old_contents = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\nbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n"; + tokio::fs::write(&file_path, old_contents).await.unwrap(); + let src = source("app", &file_path.to_string_lossy(), "app"); + let mut opened = open_tail_file(&src, false).await.unwrap(); + opened.position = 40; + tokio::fs::write( + &file_path, + b"cccccccccccccccccccccccccccccccccccccccccccccccccccccccc\n", + ) + .await + .unwrap(); + + let reopened = + reopen_if_rotated_or_truncated(&src, opened.identity, opened.position, &opened.fingerprint) + .await + .unwrap() + .expect("same-inode replacement should reopen"); + assert_eq!(reopened.position, 0); +} + +#[tokio::test] +async fn reopen_if_rotated_or_truncated_errors_when_file_disappears() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"old\n").await.unwrap(); + let src = source("app", &file_path.to_string_lossy(), "app"); + let old = open_tail_file(&src, false).await.unwrap(); + tokio::fs::remove_file(&file_path).await.unwrap(); + + let err = reopen_if_rotated_or_truncated(&src, old.identity, old.position, &old.fingerprint) + .await + .unwrap_err(); + + assert!(err.to_string().contains("disappeared")); +} + +#[tokio::test] +async fn supervisor_ingests_partial_eof_buffer_before_rotation() { + let temp = tempfile::tempdir().unwrap(); + let registry = std::sync::Arc::new(FileTailRegistry::new(temp.path().join("file-tails.json"))); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_envelope_sender_for_test(tx); + let token = tokio_util::sync::CancellationToken::new(); + let supervisor = FileTailSupervisor::new( + std::sync::Arc::clone(®istry), + ingest, + token.clone(), + 8192, + ); + let log_path = temp.path().join("app.log"); + tokio::fs::write(&log_path, b"partial").await.unwrap(); + let mut src = source("app", &log_path.to_string_lossy(), "app"); + src.start_at_end = false; + registry.upsert(src).unwrap(); + supervisor.reconcile().unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + tokio::fs::rename(&log_path, temp.path().join("app.log.1")) + .await + .unwrap(); + tokio::fs::write(&log_path, b"next\n").await.unwrap(); + + let envelope = tokio::time::timeout(std::time::Duration::from_secs(3), rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(envelope.entry.message, "partial"); + envelope.ack_success(); + + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + let statuses = supervisor.statuses(); + if statuses.iter().any(|status| { + status + .last_error + .as_deref() + .is_some_and(|err| err.contains("unterminated partial line")) + }) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + + token.cancel(); + supervisor.shutdown(); +} + +#[tokio::test] +async fn read_bounded_line_truncates_oversized_records() { + let cursor = std::io::Cursor::new(b"abcdef\nnext\n".to_vec()); + let mut reader = BufReader::new(cursor); + let mut out = Vec::new(); + + let first = read_bounded_line(&mut reader, &mut out, 3).await.unwrap(); + assert_eq!(first.bytes_read, 7); + assert!(first.truncated); + assert!(first.complete); + assert_eq!(out, b"abc"); + + out.clear(); + let second = read_bounded_line(&mut reader, &mut out, 3).await.unwrap(); + assert_eq!(second.bytes_read, 5); + assert!(second.truncated); + assert!(second.complete); + assert_eq!(out, b"nex"); +} + +#[tokio::test] +async fn read_bounded_line_buffers_partial_eof_until_newline() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("app.log"); + tokio::fs::write(&file_path, b"abc").await.unwrap(); + let file = tokio::fs::File::open(&file_path).await.unwrap(); + let mut reader = BufReader::new(file); + let mut out = Vec::new(); + + let partial = read_bounded_line(&mut reader, &mut out, 8192) + .await + .unwrap(); + assert_eq!(partial.bytes_read, 3); + assert!(!partial.complete); + assert_eq!(out, b"abc"); + + let mut writer = tokio::fs::OpenOptions::new() + .append(true) + .open(&file_path) + .await + .unwrap(); + writer.write_all(b"def\n").await.unwrap(); + writer.flush().await.unwrap(); + + let complete = read_bounded_line(&mut reader, &mut out, 8192) + .await + .unwrap(); + assert_eq!(complete.bytes_read, 4); + assert!(complete.complete); + assert_eq!(out, b"abcdef\n"); +} + +#[tokio::test] +async fn tail_file_once_sends_existing_lines_when_not_starting_at_end() { + let temp = tempfile::tempdir().unwrap(); + let file_path = temp.path().join("authelia.log"); + let mut file = tokio::fs::File::create(&file_path).await.unwrap(); + file.write_all(b"time=one level=info\n").await.unwrap(); + file.write_all(b"time=two level=error\n").await.unwrap(); + file.flush().await.unwrap(); + + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let ingest = IngestTx::from_sender_for_test(tx); + let source = FileTailSource { + id: "authelia".into(), + path: file_path.to_string_lossy().into_owned(), + tag: "authelia".into(), + hostname: Some("squirts".into()), + facility: Some("local5".into()), + severity: "info".into(), + start_at_end: false, + enabled: true, + checkpoint_dev: None, + checkpoint_ino: None, + checkpoint_offset: None, + created_at: "2026-06-11T20:00:00Z".into(), + updated_at: "2026-06-11T20:00:00Z".into(), + }; + + tail_file_once_for_test(source, ingest).await.unwrap(); + + assert_eq!(rx.recv().await.unwrap().message, "time=one level=info"); + assert_eq!(rx.recv().await.unwrap().message, "time=two level=error"); + assert!(rx.try_recv().is_err()); +} diff --git a/src/ingest.rs b/src/ingest.rs index 7577ff62..013b89cf 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use parking_lot::Mutex; -use tokio::sync::{mpsc, watch}; +use tokio::sync::{mpsc, oneshot, watch}; use tokio::task::JoinHandle; use crate::config::{ReceiverConfig, StorageConfig}; @@ -22,7 +22,7 @@ pub(crate) enum TrySendErr { #[derive(Clone)] pub(crate) struct IngestTx { - tx: mpsc::Sender, + tx: mpsc::Sender, observability: Arc, channel_capacity: usize, /// Shutdown signal. Sending `true` tells the batch writer to drain and @@ -35,6 +35,49 @@ pub(crate) struct IngestTx { writer_handle: Arc>>>, } +pub(crate) type DurableAckResult = Result<(), String>; + +pub(crate) struct IngestEnvelope { + pub(crate) entry: db::LogBatchEntry, + durable_ack: Option>, +} + +impl IngestEnvelope { + pub(crate) fn best_effort(entry: db::LogBatchEntry) -> Self { + Self { + entry, + durable_ack: None, + } + } + + fn durable(entry: db::LogBatchEntry) -> (Self, oneshot::Receiver) { + let (tx, rx) = oneshot::channel(); + ( + Self { + entry, + durable_ack: Some(tx), + }, + rx, + ) + } + + pub(crate) fn ack_success(self) { + if let Some(ack) = self.durable_ack { + let _ = ack.send(Ok(())); + } + } + + pub(crate) fn ack_failure(self, error: impl Into) { + if let Some(ack) = self.durable_ack { + let _ = ack.send(Err(error.into())); + } + } + + pub(crate) fn requires_durable_ack(&self) -> bool { + self.durable_ack.is_some() + } +} + struct WriterTuning { batch_size: usize, flush_interval_ms: u64, @@ -55,8 +98,8 @@ impl IngestTx { pub(crate) async fn send( &self, entry: db::LogBatchEntry, - ) -> Result<(), mpsc::error::SendError> { - let result = self.tx.send(entry).await; + ) -> Result<(), mpsc::error::SendError> { + let result = self.tx.send(IngestEnvelope::best_effort(entry)).await; let depth = self.queue_depth(); match &result { Ok(()) => self.observability.record_enqueue_ok(depth), @@ -65,12 +108,24 @@ impl IngestTx { result } + pub(crate) async fn send_durable(&self, entry: db::LogBatchEntry) -> anyhow::Result<()> { + let (envelope, ack) = IngestEnvelope::durable(entry); + self.tx + .send(envelope) + .await + .map_err(|_| anyhow::anyhow!("ingest writer is closed"))?; + self.observability.record_enqueue_ok(self.queue_depth()); + ack.await + .map_err(|_| anyhow::anyhow!("ingest writer dropped durable acknowledgement"))? + .map_err(anyhow::Error::msg) + } + /// Non-blocking send. Returns `Err(TrySendErr::Full)` when the channel is /// at capacity so the OTLP HTTP handler can return 503 instead of awaiting /// and holding the connection open. The dropped entry is not returned — /// the caller's contract is "best effort, drop on backpressure." pub(crate) fn try_send(&self, entry: db::LogBatchEntry) -> Result<(), TrySendErr> { - match self.tx.try_send(entry) { + match self.tx.try_send(IngestEnvelope::best_effort(entry)) { Ok(()) => { self.observability.record_enqueue_ok(self.queue_depth()); Ok(()) @@ -126,6 +181,39 @@ impl IngestTx { /// don't have to spawn a real batch writer. #[cfg(test)] pub(crate) fn from_sender_for_test(tx: mpsc::Sender) -> Self { + let channel_capacity = tx.max_capacity(); + let (envelope_tx, mut envelope_rx) = mpsc::channel::(channel_capacity); + std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test ingest bridge runtime"); + runtime.block_on(async move { + while let Some(envelope) = envelope_rx.recv().await { + match tx.send(envelope.entry.clone()).await { + Ok(()) => envelope.ack_success(), + Err(_) => { + envelope.ack_failure("test ingest receiver is closed"); + break; + } + } + } + }); + }); + let observability = Arc::new(RuntimeObservability::default()); + observability.set_queue_capacity(channel_capacity); + let (shutdown_tx, _) = watch::channel(false); + Self { + tx: envelope_tx, + observability, + channel_capacity, + shutdown_tx: Arc::new(shutdown_tx), + writer_handle: Arc::new(Mutex::new(None)), + } + } + + #[cfg(test)] + pub(crate) fn from_envelope_sender_for_test(tx: mpsc::Sender) -> Self { let observability = Arc::new(RuntimeObservability::default()); let channel_capacity = tx.max_capacity(); observability.set_queue_capacity(channel_capacity); @@ -153,7 +241,7 @@ fn start_writer( flush_interval_ms, channel_capacity, } = tuning; - let (tx, rx) = mpsc::channel::(channel_capacity); + let (tx, rx) = mpsc::channel::(channel_capacity); let (shutdown_tx, shutdown_rx) = watch::channel(false); observability.set_queue_capacity(channel_capacity); let writer_observability = Arc::clone(&observability); diff --git a/src/lib.rs b/src/lib.rs index 8e37ee84..4cf8c2b2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod config; pub mod deploy; pub mod doctor; pub mod enrich; +pub mod file_tail; pub mod heartbeat; pub mod heartbeat_agent; pub mod inventory; diff --git a/src/main.rs b/src/main.rs index 1d6a1037..57963a3c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -645,6 +645,7 @@ impl Mode { | "host-state" | "fleet-state" | "correlate-state" + | "file-tail" ) => { let mut cli_args = Vec::with_capacity(rest.len() + 1); @@ -662,7 +663,7 @@ impl Mode { // ignored for `serve mcp`, `setup`, etc. anyhow::bail!( "--http / --server / --token only apply to CLI query commands \ - (search, tail, errors, hosts, sessions, incident, entity, graph, ai, shell, agent-command, heartbeat, correlate, stats, db); \ + (search, tail, errors, hosts, sessions, incident, entity, graph, ai, shell, agent-command, heartbeat, correlate, stats, db, file-tail); \ compose, service, setup, inventory, and deploy are local-only and reject HTTP flags; \ got: {}", args.join(" ") diff --git a/src/mcp/actions.rs b/src/mcp/actions.rs index f17f427c..aec40441 100644 --- a/src/mcp/actions.rs +++ b/src/mcp/actions.rs @@ -89,6 +89,7 @@ pub(super) enum ActionHandler { AckError, UnackError, NotificationsRecent, + FileTails, NotificationsTest, SimilarIncidents, AskHistory, @@ -428,6 +429,13 @@ pub(super) const ACTION_SPECS: &[ActionSpec] = &[ Write, UnackError ), + action_spec!( + "file_tails", + Admin, + "Manage Cortex-owned file-tail ingest sources", + Write, + FileTails + ), action_spec!( "notifications_test", Admin, diff --git a/src/mcp/schemas.rs b/src/mcp/schemas.rs index 710b097c..77dfce42 100644 --- a/src/mcp/schemas.rs +++ b/src/mcp/schemas.rs @@ -132,13 +132,13 @@ pub(super) fn tool_definitions() -> Vec { }, "source_kind": { "type": "string", - "enum": ["docker-stream", "docker-event", "agent-command", "shell-history", "transcript", "claude", "codex", "gemini"], + "enum": ["docker-stream", "docker-event", "file-tail", "agent-command", "shell-history", "transcript", "claude", "codex", "gemini"], "description": "For action=filter: structured source alias. syslog-udp, syslog-tcp, and otlp are rejected in v1 because transport is not indexed separately." }, "severity": { "type": "string", "enum": SEVERITY_LEVELS, - "description": "For action=search or filter: syslog severity filter." + "description": "For action=search or filter: syslog severity filter. For action=file_tails op=add: severity assigned to tailed lines." }, "severity_min": { "type": "string", @@ -163,7 +163,7 @@ pub(super) fn tool_definitions() -> Vec { }, "facility": { "type": "string", - "description": "For action=search or filter: syslog facility filter, e.g. kern, auth, daemon, clockd." + "description": "For action=search or filter: syslog facility filter, e.g. kern, auth, daemon, clockd. For action=file_tails op=add: facility assigned to tailed lines." }, "exclude_facility": { "type": "string", @@ -316,8 +316,32 @@ pub(super) fn tool_definitions() -> Vec { "description": "For action=ai_correlate: max non-AI related log events per AI anchor, default 25, max 200." }, "id": { - "type": "integer", - "description": "For action=get: log id to fetch." + "oneOf": [ + {"type": "integer"}, + {"type": "string"} + ], + "description": "For action=get: integer log id to fetch. For action=file_tails: string source id for op=add|remove|enable|disable." + }, + "op": { + "type": "string", + "enum": ["list", "add", "remove", "enable", "disable", "status"], + "description": "For action=file_tails: required operation, one of list, add, remove, enable, disable, or status." + }, + "path": { + "type": "string", + "description": "For action=file_tails op=add: local log file path to tail." + }, + "tag": { + "type": "string", + "description": "For action=file_tails op=add: app/tag stored as app_name for tailed lines." + }, + "hostname": { + "type": "string", + "description": "For action=file_tails op=add: required source hostname assigned to tailed lines." + }, + "start_at_end": { + "type": "boolean", + "description": "For action=file_tails op=add: true starts at EOF, false backfills existing file content." }, "by_host": { "type": "boolean", @@ -489,6 +513,59 @@ pub(super) fn tool_definitions() -> Vec { } } } + }, + { + "if": { + "properties": { + "action": { "const": "get" } + }, + "required": ["action"] + }, + "then": { + "properties": { + "id": { "type": "integer" } + }, + "required": ["id"] + } + }, + { + "if": { + "properties": { + "action": { "const": "file_tails" } + }, + "required": ["action"] + }, + "then": { + "properties": { + "id": { "type": "string" }, + "op": { "enum": ["list", "add", "remove", "enable", "disable", "status"] } + }, + "required": ["op"], + "allOf": [ + { + "if": { + "properties": { + "op": { "const": "add" } + }, + "required": ["op"] + }, + "then": { + "required": ["id", "path", "tag", "hostname"] + } + }, + { + "if": { + "properties": { + "op": { "enum": ["remove", "enable", "disable"] } + }, + "required": ["op"] + }, + "then": { + "required": ["id"] + } + } + ] + } } ] } diff --git a/src/mcp/schemas_tests.rs b/src/mcp/schemas_tests.rs index 4b58b3cc..f3baf9e6 100644 --- a/src/mcp/schemas_tests.rs +++ b/src/mcp/schemas_tests.rs @@ -65,6 +65,76 @@ fn schema_source_ips_exposes_pagination() { ); } +#[test] +fn schema_includes_file_tails_action() { + let tool = tool_definitions() + .into_iter() + .find(|tool| tool["name"] == "cortex") + .expect("cortex tool"); + let schema = serde_json::to_value(tool["inputSchema"].clone()).unwrap(); + let properties = &schema["properties"]; + let action_enum = properties["action"]["enum"].as_array().unwrap(); + assert!(action_enum.iter().any(|value| value == "file_tails")); + assert_eq!( + properties["op"]["description"], + "For action=file_tails: required operation, one of list, add, remove, enable, disable, or status." + ); + assert_eq!( + properties["op"]["enum"], + serde_json::json!(["list", "add", "remove", "enable", "disable", "status"]) + ); + let source_kind_enum = properties["source_kind"]["enum"].as_array().unwrap(); + assert!(source_kind_enum.iter().any(|value| value == "file-tail")); + + let all_of = schema["allOf"].as_array().unwrap(); + assert!(all_of.iter().any(|rule| { + rule["if"]["properties"]["action"]["const"] == "get" + && rule["then"]["properties"]["id"]["type"] == "integer" + && rule["then"]["required"] + .as_array() + .unwrap() + .iter() + .any(|value| value == "id") + })); + let file_tails_rule = all_of + .iter() + .find(|rule| rule["if"]["properties"]["action"]["const"] == "file_tails") + .expect("file_tails conditional"); + assert_eq!( + file_tails_rule["then"]["properties"]["id"]["type"], + "string" + ); + assert_eq!( + file_tails_rule["then"]["properties"]["op"]["enum"], + serde_json::json!(["list", "add", "remove", "enable", "disable", "status"]) + ); + assert!( + file_tails_rule["then"]["required"] + .as_array() + .unwrap() + .iter() + .any(|value| value == "op") + ); + let nested = file_tails_rule["then"]["allOf"].as_array().unwrap(); + assert!(nested.iter().any(|rule| { + let required = rule["then"]["required"].as_array().unwrap(); + rule["if"]["properties"]["op"]["const"] == "add" + && ["id", "path", "tag", "hostname"] + .iter() + .all(|name| required.iter().any(|value| value == name)) + })); + assert!(nested.iter().any(|rule| { + rule["if"]["properties"]["op"]["enum"] + .as_array() + .is_some_and(|values| values.iter().any(|value| value == "remove")) + && rule["then"]["required"] + .as_array() + .unwrap() + .iter() + .any(|value| value == "id") + })); +} + #[test] fn schema_apps_exposes_pagination_and_total() { let tools = tool_definitions(); diff --git a/src/mcp/tools.rs b/src/mcp/tools.rs index 0ab7d606..fd5c4022 100644 --- a/src/mcp/tools.rs +++ b/src/mcp/tools.rs @@ -110,6 +110,7 @@ async fn dispatch_cortex_action( H::AckError => tool_ack_error(state, args, auth).await, H::UnackError => tool_unack_error(state, args, auth).await, H::NotificationsRecent => tool_notifications_recent(state, args).await, + H::FileTails => tool_file_tails(state, args).await, H::NotificationsTest => tool_notifications_test(state, args, auth).await, H::SimilarIncidents => tool_similar_incidents(state, args).await, H::AskHistory => tool_ask_history(state, args).await, @@ -408,10 +409,24 @@ pub(super) async fn tool_get_stats(state: &AppState, _args: Value) -> anyhow::Re pub(super) async fn tool_get_status(state: &AppState, _args: Value) -> anyhow::Result { let db_ok = state.service.health_check().await.is_ok(); let db_maintenance = state.service.db_status().await.ok(); + let file_tail_statuses = state.service.file_tail_statuses_snapshot(); + let file_tail_blocked_count = file_tail_statuses + .iter() + .filter(|status| status.blocked_on_writer_since.is_some()) + .count(); + let degraded = db_ok && file_tail_blocked_count > 0; Ok(json!({ - "status": if db_ok { "ok" } else { "error" }, + "status": if db_ok { + if degraded { "degraded" } else { "ok" } + } else { + "error" + }, "db_ok": db_ok, "db_maintenance": db_maintenance, + "file_tails": { + "blocked_count": file_tail_blocked_count, + "statuses": file_tail_statuses, + }, "runtime_observability": state.observability.snapshot(), "otlp": { "logs_received": state.otlp_counters.logs_received.load(std::sync::atomic::Ordering::Relaxed), @@ -505,6 +520,12 @@ async fn tool_notifications_recent(state: &AppState, args: Value) -> anyhow::Res Ok(serde_json::to_value(firings)?) } +async fn tool_file_tails(state: &AppState, args: Value) -> anyhow::Result { + let req: crate::app::FileTailRequest = action_payload(args, "file_tails")?; + let resp = state.service.file_tails(req).await?; + Ok(serde_json::to_value(resp)?) +} + async fn tool_notifications_test( state: &AppState, args: Value, @@ -544,6 +565,17 @@ const ADMIN_ACTION_HELP: &[AdminActionHelp] = &[ "`reason` (string, optional) — reason for removing the acknowledgement (max 4096 chars)", ], }, + AdminActionHelp { + action: "file_tails", + description: "Manage Cortex-owned file-tail ingest sources. Sources are stored in the local file-tail registry and reconciled by the runtime supervisor.", + parameters: &[ + "`op` (string, **required**) — list, add, remove, enable, disable, or status", + "`id` (string, required for add/remove/enable/disable) — stable file-tail source id", + "`path` (string, required for add) — local log file path", + "`tag` (string, required for add) — app/tag stored on ingested rows", + "`hostname`, `facility`, `severity`, `start_at_end` (optional) — row envelope defaults", + ], + }, AdminActionHelp { action: "notifications_test", description: "Send a test notification via the server-configured Apprise URLs. Rate-limited to 10 per minute per actor.\nCaller-supplied Apprise URLs are ignored for security; the server uses its own configured URLs.", diff --git a/src/mcp/tools_tests.rs b/src/mcp/tools_tests.rs index 1dc325a4..c87c5391 100644 --- a/src/mcp/tools_tests.rs +++ b/src/mcp/tools_tests.rs @@ -16,9 +16,13 @@ fn test_state_with_token(token: Option) -> (AppState, Arc, t let dir = tempfile::tempdir().unwrap(); let storage = StorageConfig::for_test(dir.path().join("mcp-test.db")); let pool = Arc::new(db::init_pool(&storage).unwrap()); + let file_tail_registry = Arc::new(crate::file_tail::FileTailRegistry::new( + dir.path().join("file-tails.json"), + )); ( AppState { - service: CortexService::new(Arc::clone(&pool), storage.clone()), + service: CortexService::new(Arc::clone(&pool), storage.clone()) + .with_file_tail_registry(file_tail_registry), config: McpConfig { host: "127.0.0.1".into(), port: 3100, @@ -196,6 +200,16 @@ fn graph_inventory_without_route_target_fixture() -> HomelabInventory { inventory } +#[tokio::test] +async fn file_tails_action_requires_admin_scope() { + let spec = crate::mcp::actions::ACTION_SPECS + .iter() + .find(|spec| spec.name == "file_tails") + .expect("file_tails registered"); + assert_eq!(spec.scope, crate::mcp::actions::Scope::Admin); + assert_eq!(spec.cost.as_str(), "write"); +} + fn project_graph_fixture(pool: &db::DbPool) { db::graph::refresh_graph_projection(pool).unwrap(); let inventory = graph_inventory_fixture(); @@ -264,6 +278,8 @@ async fn tool_get_status_returns_runtime_observability() { let value = tool_get_status(&h.state, json!({})).await.unwrap(); assert_eq!(value["status"], "ok"); assert_eq!(value["db_ok"], true); + assert_eq!(value["file_tails"]["blocked_count"], 0); + assert!(value["file_tails"]["statuses"].as_array().is_some()); assert!(value["runtime_observability"]["ingest_queue_depth"].is_number()); assert!(value["otlp"]["logs_received"].is_number()); } @@ -1096,6 +1112,7 @@ fn sample_args_for_action(action: &str) -> Option { "incident_context" => { json!({"action": action, "from": "2026-01-01T00:00:00Z", "to": "2026-01-01T01:00:00Z"}) } + "file_tails" => json!({"action": action, "op": "status"}), "filter" => json!({"action": action, "hostname": "schema-test-host"}), "map" => json!({"action": action, "mode": "snapshot"}), "abuse" => json!({"action": action, "terms": ["schema"]}), diff --git a/src/receiver/writer.rs b/src/receiver/writer.rs index 02de0d3f..fc6933a5 100644 --- a/src/receiver/writer.rs +++ b/src/receiver/writer.rs @@ -11,6 +11,7 @@ use super::enrichment::{EnrichmentConfig, enrich_entry}; use crate::config::StorageConfig; use crate::db::{self, DbPool}; use crate::enrich::EnrichmentPipeline; +use crate::ingest::IngestEnvelope; use crate::observability::RuntimeObservability; const INGEST_SUMMARY_INTERVAL_SECS: u64 = 60; @@ -50,13 +51,13 @@ impl WriterContext { } pub(crate) async fn batch_writer( - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver, context: WriterContext, batch_size: usize, flush_interval: tokio::time::Duration, mut shutdown: watch::Receiver, ) { - let mut batch: Vec = Vec::with_capacity(batch_size); + let mut batch: Vec = Vec::with_capacity(batch_size); let mut storage_blocked = false; let mut summary = IngestSummary::default(); let mut summary_deadline = tokio::time::Instant::now() @@ -76,6 +77,7 @@ pub(crate) async fn batch_writer( msg = rx.recv() => { match msg { Some(parsed) => { + let requires_durable_ack = parsed.requires_durable_ack(); batch.push(parsed); debug!( batch_len = batch.len(), @@ -83,7 +85,7 @@ pub(crate) async fn batch_writer( queue_capacity = rx.max_capacity(), "Queued parsed syslog entry" ); - if !batch.is_empty() && batch.len() % batch_size == 0 { + if requires_durable_ack || (!batch.is_empty() && batch.len() % batch_size == 0) { break; } } @@ -138,7 +140,7 @@ pub(crate) async fn batch_writer( } pub(super) async fn flush_batch( - batch: &mut Vec, + batch: &mut Vec, storage_blocked: &mut bool, summary: &mut IngestSummary, context: &WriterContext, @@ -162,12 +164,12 @@ pub(super) async fn flush_batch( // framework (metadata_json parsed twice per entry) was fixed in the // new framework's dispatch() — it now parses once and passes a // reference to all helpers (Arch-H5 partial fix, 2026-05-22). - let batch_to_write: Vec = std::mem::take(batch) + let batch_to_write: Vec = std::mem::take(batch) .into_iter() - .map(|e| { - let mut e = enrich_entry(e, &context.enrichment); - context.pipeline.dispatch(&mut e); - e + .map(|mut envelope| { + envelope.entry = enrich_entry(envelope.entry, &context.enrichment); + context.pipeline.dispatch(&mut envelope.entry); + envelope }) .collect(); let count = batch_to_write.len(); @@ -201,19 +203,20 @@ pub(super) async fn flush_batch( return; } } - match tokio::task::spawn_blocking( - move || match db::insert_logs_batch(&pool, &batch_to_write) { + match tokio::task::spawn_blocking(move || { + match insert_envelopes_batch(&pool, &batch_to_write) { Ok(n) => Ok((n, batch_to_write)), Err(e) => { let outcome = handle_failed_batch(&pool, batch_to_write, &e); Err((e, outcome)) } - }, - ) + } + }) .await { Ok(Ok((n, inserted_batch))) => { - summary.record_batch(&inserted_batch[..n.min(inserted_batch.len())]); + let inserted_count = n.min(inserted_batch.len()); + summary.record_envelopes(&inserted_batch[..inserted_count]); context.observability.record_writer_flushed(n); if *storage_blocked { info!( @@ -228,11 +231,14 @@ pub(super) async fn flush_batch( elapsed_ms = started.elapsed().as_millis(), "Flushed log batch" ); + for envelope in inserted_batch.into_iter().take(inserted_count) { + envelope.ack_success(); + } } Ok(Err((e, outcome))) => { if !outcome.inserted_entries.is_empty() { let inserted_count = outcome.inserted_count.min(outcome.inserted_entries.len()); - summary.record_batch(&outcome.inserted_entries[..inserted_count]); + summary.record_envelopes(&outcome.inserted_entries[..inserted_count]); context.observability.record_writer_flushed(inserted_count); debug!( inserted_count, @@ -241,6 +247,10 @@ pub(super) async fn flush_batch( ); } + for envelope in outcome.inserted_entries { + envelope.ack_success(); + } + if !outcome.retained_entries.is_empty() { context .observability @@ -297,11 +307,19 @@ pub(super) async fn flush_batch( } } +fn insert_envelopes_batch(pool: &DbPool, envelopes: &[IngestEnvelope]) -> anyhow::Result { + let entries = envelopes + .iter() + .map(|envelope| envelope.entry.clone()) + .collect::>(); + db::insert_logs_batch(pool, &entries) +} + #[derive(Default)] struct FailedBatchOutcome { inserted_count: usize, - inserted_entries: Vec, - retained_entries: Vec, + inserted_entries: Vec, + retained_entries: Vec, retained_chunks: usize, discarded_count: usize, discarded_chunks: usize, @@ -309,7 +327,7 @@ struct FailedBatchOutcome { fn handle_failed_batch( pool: &DbPool, - failed_batch: Vec, + failed_batch: Vec, error: &anyhow::Error, ) -> FailedBatchOutcome { let mut outcome = FailedBatchOutcome::default(); @@ -318,18 +336,21 @@ fn handle_failed_batch( return outcome; } - for chunk in failed_batch.chunks(FAILED_BATCH_RETRY_CHUNK_SIZE) { - retry_failed_chunk(pool, chunk.to_vec(), &mut outcome); + let mut remaining = failed_batch; + while !remaining.is_empty() { + let next = if remaining.len() > FAILED_BATCH_RETRY_CHUNK_SIZE { + remaining.split_off(FAILED_BATCH_RETRY_CHUNK_SIZE) + } else { + Vec::new() + }; + retry_failed_chunk(pool, remaining, &mut outcome); + remaining = next; } outcome } -fn retry_failed_chunk( - pool: &DbPool, - chunk: Vec, - outcome: &mut FailedBatchOutcome, -) { - match db::insert_logs_batch(pool, &chunk) { +fn retry_failed_chunk(pool: &DbPool, chunk: Vec, outcome: &mut FailedBatchOutcome) { + match insert_envelopes_batch(pool, &chunk) { Ok(inserted) => { outcome.inserted_count += inserted; outcome.inserted_entries.extend(chunk); @@ -347,22 +368,27 @@ fn retry_failed_chunk( Err(_) => { outcome.discarded_count += chunk.len(); outcome.discarded_chunks += 1; + for envelope in chunk { + envelope.ack_failure("unrecoverable database insert failure"); + } } } } -fn retain_or_discard_entries(outcome: &mut FailedBatchOutcome, entries: Vec) { +fn retain_or_discard_entries(outcome: &mut FailedBatchOutcome, entries: Vec) { let retain_remaining = FAILED_BATCH_RETAIN_LIMIT.saturating_sub(outcome.retained_entries.len()); if retain_remaining == 0 { - for entry in &entries { + let discarded = entries.len(); + for envelope in entries { tracing::warn!( - hostname = %entry.hostname, - severity = %entry.severity, - timestamp = %entry.timestamp, + hostname = %envelope.entry.hostname, + severity = %envelope.entry.severity, + timestamp = %envelope.entry.timestamp, "Discarding log entry: retain limit reached" ); + envelope.ack_failure("ingest writer retain limit reached"); } - outcome.discarded_count += entries.len(); + outcome.discarded_count += discarded; outcome.discarded_chunks += 1; return; } @@ -379,13 +405,14 @@ fn retain_or_discard_entries(outcome: &mut FailedBatchOutcome, entries: Vec db:: } } +fn envelope_batch(entries: Vec) -> Vec { + entries + .into_iter() + .map(crate::ingest::IngestEnvelope::best_effort) + .collect() +} + #[tokio::test] async fn flush_batch_retains_entries_while_storage_is_write_blocked() { let (pool, mut storage, _dir) = test_pool(); @@ -65,7 +72,7 @@ async fn flush_batch_retains_entries_while_storage_is_write_blocked() { metrics: db::get_storage_metrics(&pool, &storage).unwrap(), write_blocked: true, }); - let mut batch = vec![make_entry("blocked write")]; + let mut batch = envelope_batch(vec![make_entry("blocked write")]); let mut storage_blocked = false; let mut summary = IngestSummary::default(); let observability = Arc::new(crate::observability::RuntimeObservability::default()); @@ -92,7 +99,7 @@ async fn flush_batch_resumes_after_storage_recovers() { metrics: db::get_storage_metrics(&pool, &storage).unwrap(), write_blocked: false, }))); - let mut batch = vec![make_entry("resumed write")]; + let mut batch = envelope_batch(vec![make_entry("resumed write")]); let mut storage_blocked = true; let mut summary = IngestSummary::default(); let observability = Arc::new(crate::observability::RuntimeObservability::default()); @@ -131,11 +138,11 @@ async fn flush_batch_isolates_bad_rows_and_writes_remaining_entries() { metrics: db::get_storage_metrics(&pool, &storage).unwrap(), write_blocked: false, }))); - let mut batch = vec![ + let mut batch = envelope_batch(vec![ make_entry("good row one"), make_entry("bad row"), make_entry("good row two"), - ]; + ]); let mut storage_blocked = false; let mut summary = IngestSummary::default(); let observability = Arc::new(crate::observability::RuntimeObservability::default()); @@ -167,9 +174,11 @@ async fn flush_batch_retains_bounded_entries_for_large_retryable_failures() { metrics, write_blocked: false, }))); - let mut batch = (0..(FAILED_BATCH_RETAIN_LIMIT + 5)) - .map(|i| make_entry(&format!("locked row {i}"))) - .collect::>(); + let mut batch = envelope_batch( + (0..(FAILED_BATCH_RETAIN_LIMIT + 5)) + .map(|i| make_entry(&format!("locked row {i}"))) + .collect::>(), + ); let mut storage_blocked = false; let mut summary = IngestSummary::default(); let observability = Arc::new(crate::observability::RuntimeObservability::default()); @@ -193,7 +202,7 @@ async fn flush_batch_retains_bounded_entries_for_large_retryable_failures() { #[test] fn failed_batch_retains_disk_full_errors_instead_of_discarding_rows() { let (pool, _storage, _dir) = test_pool(); - let batch = vec![make_entry("disk full retained")]; + let batch = envelope_batch(vec![make_entry("disk full retained")]); let error = anyhow::anyhow!(rusqlite::Error::SqliteFailure( rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_FULL), Some("database or disk is full".to_string()), diff --git a/src/runtime.rs b/src/runtime.rs index ce6008c3..82df083b 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -27,6 +27,7 @@ use tokio_util::sync::CancellationToken; use crate::app::CortexService; use crate::config::{AuthMode, Config, mcp_bind_is_loopback, validate_auth_config}; use crate::db::{self, DbPool, StorageBudgetState}; +use crate::file_tail::{FileTailRegistry, FileTailSupervisor}; use crate::heartbeat::HeartbeatState; use crate::ingest::IngestTx; use crate::mcp::AuthPolicy; @@ -50,6 +51,7 @@ pub struct RuntimeCore { /// HTTP back-pressure from starving the DB maintenance tasks. dispatcher_permit: Arc, ingest: IngestTx, + file_tail_supervisor: FileTailSupervisor, otlp_counters: Arc, auth_policy: AuthPolicy, observability: Arc, @@ -65,6 +67,7 @@ pub struct MaintenanceHandles { purge: Option>, storage: Option>, docker_ingest: Vec>, + file_tail: Option>, error_scan: Option>, notification_dispatcher: Option>, notification_evaluator: Option>, @@ -131,6 +134,7 @@ impl MaintenanceHandles { self.timeline_rollup, self.optimize, self.syslog_monitor, + self.file_tail, ] .into_iter() .flatten() @@ -230,7 +234,6 @@ impl RuntimeCore { "Initial storage budget check completed" ); } - let service = CortexService::new(Arc::clone(&pool), config.storage.clone()); let enrichment = EnrichmentConfig { authelia_source_ip: config.enrichment.authelia_source_ip.clone(), adguard_source_ip: config.enrichment.adguard_source_ip.clone(), @@ -246,6 +249,27 @@ impl RuntimeCore { enrichment, Arc::clone(&observability), ); + let file_tail_registry = Arc::new(FileTailRegistry::new( + FileTailRegistry::path_from_storage_db(&config.storage.db_path), + )); + let file_tail_supervisor = FileTailSupervisor::new( + Arc::clone(&file_tail_registry), + ingest.clone(), + CancellationToken::new(), + config.receiver.max_message_size, + ); + let mut service = CortexService::new(Arc::clone(&pool), config.storage.clone()); + if is_stdio { + service = service.with_file_tail_registry(file_tail_registry); + } else { + let reconcile_supervisor = file_tail_supervisor.clone(); + let status_supervisor = file_tail_supervisor.clone(); + service = service.with_file_tail_control( + file_tail_registry, + Arc::new(move || reconcile_supervisor.reconcile()), + Arc::new(move || status_supervisor.statuses()), + ); + } let auth_policy = build_auth_policy(&config, is_stdio).await?; @@ -257,6 +281,7 @@ impl RuntimeCore { maintenance_permit: Arc::new(Semaphore::new(1)), dispatcher_permit: Arc::new(Semaphore::new(1)), ingest, + file_tail_supervisor, otlp_counters: Arc::new(OtlpCounters::default()), auth_policy, observability, @@ -402,11 +427,13 @@ impl RuntimeCore { let session_rollup = self.spawn_session_rollup_task(token.clone()); let timeline_rollup = self.spawn_timeline_rollup_task(token.clone()); let optimize = self.spawn_optimize_task(token.clone()); + let file_tail = self.spawn_file_tail_task(token.clone()); MaintenanceHandles { token, purge, storage, docker_ingest, + file_tail, error_scan, notification_dispatcher, notification_evaluator, @@ -420,6 +447,31 @@ impl RuntimeCore { } } + fn spawn_file_tail_task(&self, token: CancellationToken) -> Option> { + let supervisor = self.file_tail_supervisor.clone(); + Some(tokio::spawn(async move { + if let Err(err) = supervisor.reconcile() { + tracing::warn!(error = %err, "initial file-tail reconcile failed"); + } + let mut interval = background_interval(tokio::time::Duration::from_secs(30)); + loop { + tokio::select! { + biased; + _ = token.cancelled() => { + supervisor.shutdown(); + tracing::debug!("file_tail: cooperative shutdown"); + break; + } + _ = interval.tick() => { + if let Err(err) = supervisor.reconcile() { + tracing::warn!(error = %err, "file-tail reconcile failed"); + } + } + } + } + })) + } + /// Periodically run `PRAGMA optimize` so the query planner keeps fresh /// `sqlite_stat1` statistics as the DB grows. /// diff --git a/tests/mcporter/test-tools.sh b/tests/mcporter/test-tools.sh index 4515c7cd..f6d32ca3 100755 --- a/tests/mcporter/test-tools.sh +++ b/tests/mcporter/test-tools.sh @@ -11,7 +11,7 @@ # cortex get, cortex ingest_rate, cortex silent_hosts, cortex clock_skew, # cortex anomalies, cortex compare, cortex compose_status, # cortex compose_doctor, cortex unaddressed_errors, cortex ack_error, -# cortex unack_error, cortex notifications_recent, cortex notifications_test, +# cortex unack_error, cortex notifications_recent, cortex file_tails, cortex notifications_test, # cortex similar_incidents, cortex ask_history, cortex incident_context, cortex graph, # cortex help # @@ -405,6 +405,21 @@ except Exception as e: return 0 } +pass_test() { + local label="${1:?label required}" + printf "${C_GREEN}[PASS]${C_RESET} %-60s\n" "${label}" | tee -a "${LOG_FILE}" + PASS_COUNT=$(( PASS_COUNT + 1 )) +} + +fail_test() { + local label="${1:?label required}" + local reason="${2:-check failed}" + printf "${C_RED}[FAIL]${C_RESET} %-60s\n" "${label}" | tee -a "${LOG_FILE}" + printf ' %s\n' "${reason}" | tee -a "${LOG_FILE}" + FAIL_COUNT=$(( FAIL_COUNT + 1 )) + FAIL_NAMES+=("${label}") +} + # --------------------------------------------------------------------------- # Skip helper # --------------------------------------------------------------------------- @@ -415,6 +430,18 @@ skip_test() { SKIP_COUNT=$(( SKIP_COUNT + 1 )) } +mcp_admin_scope_available() { + local token="${CORTEX_TOKEN:-${CORTEX_API_TOKEN:-}}" + token="${token//[[:space:]]/}" + [[ -n "${token}" \ + && ( "${CORTEX_STATIC_TOKEN_ADMIN:-false}" == "true" \ + || "${CORTEX_SMOKE_ADMIN:-false}" == "true" ) ]] +} + +file_tail_smoke_available() { + [[ -n "${CORTEX_FILE_TAIL_SMOKE_PATH:-}" && -n "${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-${CORTEX_FILE_TAIL_SMOKE_PATH:-}}" ]] +} + # --------------------------------------------------------------------------- # Safe JSON payload builder # Usage: _json_payload '' key1=value1 key2=value2 ... @@ -488,6 +515,57 @@ suite_meta() { run_test "cortex stats: returns database statistics" cortex stats '{}' "total_logs" run_test "cortex stats: write_blocked field present" cortex stats '{}' "write_blocked" run_test "cortex stats: free_disk_mb field present" cortex stats '{}' "free_disk_mb" + if mcp_admin_scope_available; then + run_test "cortex file_tails: returns registry status" cortex file_tails '{"op":"status"}' "sources" + local missing_op + if missing_op="$(mcporter_call cortex file_tails '{}' 2>&1)"; then + fail_test "cortex file_tails: missing op rejected" "request unexpectedly succeeded: ${missing_op}" + elif [[ "${missing_op}" == *"op"* || "${missing_op}" == *"missing"* || "${missing_op}" == *"required"* ]]; then + pass_test "cortex file_tails: missing op rejected" + else + fail_test "cortex file_tails: missing op rejected" "response did not mention op: ${missing_op}" + fi + if file_tail_smoke_available; then + local server_path write_path source_id tag marker add_output search_output count attempt + server_path="${CORTEX_FILE_TAIL_SMOKE_PATH}" + write_path="${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-${server_path}}" + source_id="smoke-$(date +%s)-$$" + tag="file-tail-smoke" + marker="file-tail-smoke-${source_id}" + touch "${write_path}" || fail_test "cortex file_tails: smoke file writable" "could not write ${write_path}" + add_output="$(mcporter_call cortex file_tails "$(jq -nc \ + --arg id "${source_id}" \ + --arg path "${server_path}" \ + --arg tag "${tag}" \ + '{"op":"add","id":$id,"path":$path,"tag":$tag,"hostname":"mcporter-smoke","facility":"local7","severity":"info","start_at_end":true}')")" || add_output="" + if printf '%s' "${add_output}" | jq -e '.sources | type == "array"' >/dev/null 2>&1; then + pass_test "cortex file_tails: add smoke source" + else + fail_test "cortex file_tails: add smoke source" "${add_output}" + fi + printf '%s\n' "${marker}" >> "${write_path}" + count=0 + for attempt in {1..20}; do + search_output="$(mcporter_call cortex search "$(jq -nc \ + --arg q "\"${marker}\"" \ + --arg tag "${tag}" \ + '{"query":$q,"source_kind":"file-tail","app_name":$tag,"limit":5}')")" || search_output="" + count="$(printf '%s' "${search_output}" | jq -r '.count // 0' 2>/dev/null)" || count=0 + [[ "${count}" -ge 1 ]] && break + sleep 0.5 + done + if [[ "${count}" -ge 1 ]]; then + pass_test "cortex file_tails: add append query ingest" + else + fail_test "cortex file_tails: add append query ingest" "marker was not queryable" + fi + mcporter_call cortex file_tails "$(jq -nc --arg id "${source_id}" '{"op":"remove","id":$id}')" >/dev/null 2>&1 || true + else + skip_test "cortex file_tails: add append query ingest" "requires CORTEX_FILE_TAIL_SMOKE_PATH" + fi + else + skip_test "cortex file_tails: returns registry status" "requires cortex:admin (set CORTEX_STATIC_TOKEN_ADMIN=true or CORTEX_SMOKE_ADMIN=true)" + fi run_test "cortex compose_status: redacted diagnostics" cortex compose_status '{}' "runtime_state" local compose_status compose_runtime compose_ownership diff --git a/tests/test_live.sh b/tests/test_live.sh index 03a471d0..2303e248 100755 --- a/tests/test_live.sh +++ b/tests/test_live.sh @@ -25,7 +25,7 @@ # cortex get, cortex ingest_rate, cortex silent_hosts, cortex clock_skew, # cortex anomalies, cortex compare, cortex compose_status, # cortex compose_doctor, cortex unaddressed_errors, cortex ack_error, -# cortex unack_error, cortex notifications_recent, cortex notifications_test, +# cortex unack_error, cortex notifications_recent, cortex file_tails, cortex notifications_test, # cortex similar_incidents, cortex ask_history, cortex incident_context, cortex graph, # cortex help # @@ -59,6 +59,9 @@ AI_SMOKE_PROJECT="/tmp/cortex-ai-smoke" AI_SMOKE_QUERY='"ai-smoke-authentication"' AI_SEEDED=false CLI_PARITY_CONTAINER="" +FILE_TAIL_SMOKE_DIR="" +FILE_TAIL_SMOKE_HOST_PATH="" +FILE_TAIL_SMOKE_SERVER_PATH="/file-tail-root/smoke.log" # --------------------------------------------------------------------------- # Counters @@ -170,6 +173,19 @@ _skip() { SKIP_COUNT=$(( SKIP_COUNT + 1 )) } +mcp_admin_scope_available() { + local token="${TOKEN:-}" + token="${token//[[:space:]]/}" + [[ -n "${token}" \ + && ( "${CORTEX_STATIC_TOKEN_ADMIN:-false}" == "true" \ + || "${CORTEX_SMOKE_ADMIN:-false}" == "true" ) ]] +} + +file_tail_smoke_available() { + [[ -n "${CORTEX_FILE_TAIL_SMOKE_PATH:-${FILE_TAIL_SMOKE_SERVER_PATH:-}}" \ + && -n "${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-${FILE_TAIL_SMOKE_HOST_PATH:-${CORTEX_FILE_TAIL_SMOKE_PATH:-}}}" ]] +} + section() { printf '\n%b=== %s ===%b\n' "${C_BOLD}" "$*" "${C_RESET}" } @@ -474,6 +490,59 @@ phase_tools() { assert_jq "cortex stats — total_logs is a number >= 0" "${stats_result}" '.total_logs >= 0' assert_jq "cortex stats — total_hosts is a number >= 0" "${stats_result}" '.total_hosts >= 0' + # --- cortex file_tails --- + section " cortex file_tails" + if mcp_admin_scope_available; then + local file_tails_result + file_tails_result="$(call_tool cortex '{"action":"file_tails","op":"status"}')" || file_tails_result="" + assert_jq "cortex file_tails — sources array present" "${file_tails_result}" '.sources | type == "array"' + assert_jq "cortex file_tails — statuses array present" "${file_tails_result}" '.statuses | type == "array"' + local file_tails_missing_op + if file_tails_missing_op="$(call_tool cortex '{"action":"file_tails"}' 2>&1)"; then + _fail "cortex file_tails — missing op rejected" "request unexpectedly succeeded: ${file_tails_missing_op}" + elif [[ "${file_tails_missing_op}" == *"op"* || "${file_tails_missing_op}" == *"missing"* || "${file_tails_missing_op}" == *"required"* ]]; then + _pass "cortex file_tails — missing op rejected" + else + _fail "cortex file_tails — missing op rejected" "response did not mention op: ${file_tails_missing_op}" + fi + if file_tail_smoke_available; then + local server_path write_path source_id tag marker add_result search_result count attempt + server_path="${CORTEX_FILE_TAIL_SMOKE_PATH:-${FILE_TAIL_SMOKE_SERVER_PATH}}" + write_path="${CORTEX_FILE_TAIL_SMOKE_WRITE_PATH:-${FILE_TAIL_SMOKE_HOST_PATH:-${server_path}}}" + source_id="smoke-${CONTAINER_NAME:-$$}" + tag="file-tail-smoke" + marker="file-tail-smoke-${CONTAINER_NAME:-$$}" + touch "${write_path}" || _fail "cortex file_tails — smoke file writable" "could not write ${write_path}" + add_result="$(call_tool cortex "$(jq -nc \ + --arg id "${source_id}" \ + --arg path "${server_path}" \ + --arg tag "${tag}" \ + '{"action":"file_tails","op":"add","id":$id,"path":$path,"tag":$tag,"hostname":"live-smoke","facility":"local7","severity":"info","start_at_end":true}')")" || add_result="" + assert_jq "cortex file_tails — add smoke source" "${add_result}" '.sources | type == "array"' + printf '%s\n' "${marker}" >> "${write_path}" + count=0 + for attempt in {1..20}; do + search_result="$(call_tool cortex "$(jq -nc \ + --arg q "\"${marker}\"" \ + --arg tag "${tag}" \ + '{"action":"search","query":$q,"source_kind":"file-tail","app_name":$tag,"limit":5}')")" || search_result="" + count="$(printf '%s' "${search_result}" | jq -r '.count // 0' 2>/dev/null)" || count=0 + [[ "${count}" -ge 1 ]] && break + sleep 0.5 + done + if [[ "${count}" -ge 1 ]]; then + _pass "cortex file_tails — add append query ingest" + else + _fail "cortex file_tails — add append query ingest" "marker was not queryable" + fi + call_tool cortex "$(jq -nc --arg id "${source_id}" '{"action":"file_tails","op":"remove","id":$id}')" >/dev/null 2>&1 || true + else + _skip "cortex file_tails — add append query ingest" "requires CORTEX_FILE_TAIL_SMOKE_PATH" + fi + else + _skip "cortex file_tails — registry status" "requires cortex:admin (set CORTEX_STATIC_TOKEN_ADMIN=true or CORTEX_SMOKE_ADMIN=true)" + fi + # --- compose diagnostics --- section " cortex compose diagnostics" local compose_status_result @@ -699,6 +768,9 @@ docker_cleanup() { log_info "Removing test container ${CONTAINER_NAME}..." docker rm -f "${CONTAINER_NAME}" &>/dev/null || true fi + if [[ -n "${FILE_TAIL_SMOKE_DIR}" && -d "${FILE_TAIL_SMOKE_DIR}" ]]; then + rm -rf "${FILE_TAIL_SMOKE_DIR}" + fi } run_docker_mode() { @@ -733,6 +805,14 @@ run_docker_mode() { # uid=1000,gid=1000 matches the 'syslog' user in the container image "--tmpfs" "/data:rw,noexec,nosuid,size=64m,uid=1000,gid=1000" ) + FILE_TAIL_SMOKE_DIR="$(mktemp -d /tmp/cortex-file-tail-smoke.XXXXXX)" + FILE_TAIL_SMOKE_HOST_PATH="${FILE_TAIL_SMOKE_DIR}/smoke.log" + : > "${FILE_TAIL_SMOKE_HOST_PATH}" + chmod 755 "${FILE_TAIL_SMOKE_DIR}" + chmod 644 "${FILE_TAIL_SMOKE_HOST_PATH}" + docker_args+=("-v" "${FILE_TAIL_SMOKE_DIR}:/file-tail-root:ro") + CORTEX_FILE_TAIL_SMOKE_PATH="${FILE_TAIL_SMOKE_SERVER_PATH}" + CORTEX_FILE_TAIL_SMOKE_WRITE_PATH="${FILE_TAIL_SMOKE_HOST_PATH}" # /api/* is always mounted post-v0.26, so the container will refuse to # start without CORTEX_API_TOKEN. Fail fast here with a clear message @@ -744,6 +824,8 @@ run_docker_mode() { docker_args+=("-e" "CORTEX_HOST=0.0.0.0") docker_args+=("-e" "CORTEX_TOKEN=${TOKEN}") docker_args+=("-e" "CORTEX_API_TOKEN=${TOKEN}") + docker_args+=("-e" "CORTEX_STATIC_TOKEN_ADMIN=true") + CORTEX_STATIC_TOKEN_ADMIN=true # Remove storage budget env vars that conflict with tmpfs size limits docker_args+=( @@ -795,6 +877,12 @@ run_docker_mode() { sleep 1 done + if ! docker exec "${CONTAINER_NAME}" test -r "${FILE_TAIL_SMOKE_SERVER_PATH}"; then + log_error "file-tail smoke path is not readable in container: ${FILE_TAIL_SMOKE_SERVER_PATH}" + docker exec "${CONTAINER_NAME}" sh -c 'id; ls -ld /file-tail-root; ls -l /file-tail-root' 2>&1 || true + return 2 + fi + section "Docker — Seed AI transcript fixture" seed_ai_fixture_container "${project_dir}" || { log_error "AI transcript fixture seed failed"