diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..0a03870 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,4 @@ +[advisories] +# bincode remains read-only for v0.9 graph compatibility. New snapshots use +# postcard; remove this exception when legacy graph support is dropped. +ignore = ["RUSTSEC-2025-0141"] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dad0eba..9a8740a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,17 +22,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: fetch-depth: 0 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: components: rustfmt, clippy - name: Cache cargo - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cargo/registry/index @@ -53,8 +53,52 @@ jobs: - name: Verify generated docs are in sync run: cargo run -p xtask -- gen-skill --check + - name: Enforce v0.9 performance gates + run: cargo run -p xtask -- perf-gate + + - name: Audit Rust dependencies + uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + with: + version: 10 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + cache: pnpm + cache-dependency-path: www/pnpm-lock.yaml + + - name: Build website + working-directory: www + run: | + pnpm install --frozen-lockfile + pnpm build + pnpm audit --audit-level moderate + - name: Check installer syntax - run: sh -n install.sh + run: | + sh -n install.sh + shellcheck install.sh + + - name: Check PowerShell installer syntax + shell: pwsh + run: | + $tokens = $null + $errors = $null + [System.Management.Automation.Language.Parser]::ParseFile( + (Resolve-Path install.ps1), + [ref]$tokens, + [ref]$errors + ) | Out-Null + if ($errors.Count -gt 0) { + $errors | Format-List | Out-String | Write-Error + exit 1 + } - name: Verify release tag is on main if: startsWith(github.ref, 'refs/tags/v') @@ -67,10 +111,29 @@ jobs: exit 1 fi + test-platforms: + name: Test ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable + + - name: Test + run: cargo test --locked + build: name: Build ${{ matrix.name }} if: startsWith(github.ref, 'refs/tags/v') - needs: check + needs: [check, test-platforms] strategy: fail-fast: false matrix: @@ -99,15 +162,15 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Rust - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 # stable with: targets: ${{ matrix.target }} - name: Cache cargo - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cargo/registry/index @@ -141,7 +204,7 @@ jobs: fi - name: Upload artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ matrix.archive }} path: | @@ -157,12 +220,27 @@ jobs: contents: write runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Download release artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: release-assets merge-multiple: true + - name: Add installers and checksums + run: | + cp install.sh install.ps1 release-assets/ + cd release-assets + for asset in *; do + if [ "$asset" != "SHA256SUMS" ]; then + sha256sum "$asset" + fi + done > SHA256SUMS + - name: Create or update release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bad1781..ed1045b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## Unreleased + +### Added + +- Added v3 graph snapshots with full BLAKE3 payload identities, hydrated search and dependency indexes, and transparent v0.9-v2 loading. +- Added a reproducible performance gate requiring 20% faster 500-file indexing and warm exact search than v0.9.0. +- Added release-asset checksum verification to the Unix and Windows installers. +- Added cross-platform, website, dependency-audit, and installer checks to CI. + +### Fixed + +- Detect same-size source changes even when file modification times are preserved. +- Preserve line endings and executable permissions during atomic line edits. +- Bound newline-delimited MCP messages, headers, and agent-facing result counts. +- Report one deterministic finding per dependency cycle and downgrade cycles contained within one Rust module family. + +### Changed + +- Centralized shared CLI/MCP read, patch, create, and audit workflows plus MCP graph lifecycle persistence. +- Reduced index memory and CPU overhead by removing duplicate content caching and deduplicating word and trigram data. +- Parallelized parsing and search-index preparation for projects with at least 64 files. +- Limited exact-word hit materialization to the requested result count. +- Pinned every external GitHub Action to an immutable full-length commit SHA. + ## v0.9.0 - 2026-06-30 ### Added diff --git a/Cargo.lock b/Cargo.lock index 0ed428f..e539787 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,9 +75,21 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "autocfg" @@ -106,6 +118,20 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + [[package]] name = "bstr" version = "1.12.1" @@ -211,12 +237,36 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "criterion" version = "0.5.1" @@ -274,9 +324,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -299,6 +349,18 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + [[package]] name = "equivalent" version = "1.0.2" @@ -578,11 +640,14 @@ version = "0.9.0" dependencies = [ "anyhow", "bincode", + "blake3", "clap", "criterion", "hashbrown 0.15.5", "ignore", "notify", + "postcard", + "rayon", "regex", "serde", "serde_json", @@ -740,6 +805,18 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + [[package]] name = "prettyplease" version = "0.2.37" diff --git a/Cargo.toml b/Cargo.toml index 0d8defa..cd1db08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ serde_json = "1" toon-format = { version = "0.5.0", default-features = false } toml = "0.8" bincode = "1.3" +postcard = { version = "1", default-features = false, features = ["alloc"] } +blake3 = "1" hashbrown = "0.15" walkdir = "2" ignore = "0.4" @@ -25,6 +27,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } anyhow = "1" regex = "1" +rayon = "1" notify = "6" tree-sitter = "0.24" tree-sitter-zig = "1.1" diff --git a/benches/engine.rs b/benches/engine.rs index 2ac6e7b..5d3a426 100644 --- a/benches/engine.rs +++ b/benches/engine.rs @@ -1,5 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; use lexa::engine::{Engine, SearchOptions}; +use lexa::freshness; use lexa::snapshot; use lexa::store::Op; use std::fs; @@ -219,6 +220,19 @@ fn bench_snapshot(c: &mut Criterion) { }); }); + group.bench_function("strict_refresh_unchanged", |b| { + b.iter_batched( + || build_engine_from_project(project.path()), + |mut engine| { + black_box( + freshness::refresh_project(&mut engine, project.path()) + .expect("refresh project"), + ); + }, + BatchSize::LargeInput, + ); + }); + group.finish(); } diff --git a/docs/install.md b/docs/install.md index 2d322f4..ba929a4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -32,6 +32,8 @@ By default, `upgrade` installs into the directory containing the running `lexa` `lexa upgrade` updates the binary, not a project index. Use `lexa index .` to refresh a project's graph. +Release installers verify the selected archive against the release's `SHA256SUMS` file before extraction. If checksum verification fails, installation stops without replacing the existing binary. + ## Troubleshooting If `lexa --version` is unavailable after install, verify that the install directory is on `PATH`. diff --git a/docs/tools.md b/docs/tools.md index 9937254..46be9c4 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -20,12 +20,14 @@ }, "max": { "description": "Alias for max_results.", + "maximum": 200, "type": "integer" }, "max_lines": { "type": "integer" }, "max_results": { + "maximum": 200, "type": "integer" }, "min_lines": { @@ -121,9 +123,11 @@ "properties": { "max": { "description": "Alias for max_results.", + "maximum": 200, "type": "integer" }, "max_results": { + "maximum": 200, "type": "integer" }, "name": { @@ -230,9 +234,11 @@ "properties": { "max": { "description": "Alias for max_results.", + "maximum": 200, "type": "integer" }, "max_results": { + "maximum": 200, "type": "integer" }, "name": { @@ -278,10 +284,12 @@ }, "max": { "description": "Alias for max_results.", + "maximum": 200, "minimum": 1, "type": "integer" }, "max_results": { + "maximum": 200, "minimum": 1, "type": "integer" }, @@ -323,6 +331,7 @@ "type": "boolean" }, "max_results": { + "maximum": 200, "type": "integer" }, "path_glob": { @@ -412,9 +421,11 @@ }, "max": { "description": "Alias for max_results.", + "maximum": 200, "type": "integer" }, "max_results": { + "maximum": 200, "type": "integer" }, "path": { @@ -638,6 +649,7 @@ { "properties": { "limit": { + "maximum": 200, "type": "integer" } }, @@ -719,9 +731,11 @@ "type": "array" }, "max": { + "maximum": 1000, "type": "integer" }, "max_results": { + "maximum": 1000, "type": "integer" }, "no_config": { diff --git a/install.ps1 b/install.ps1 index 4d76266..b69af58 100644 --- a/install.ps1 +++ b/install.ps1 @@ -47,6 +47,18 @@ New-Item -ItemType Directory -Path $tmpDir, $extractDir -Force | Out-Null try { Write-Host "Downloading $url..." Invoke-WebRequest -Uri $url -OutFile $zipPath + $checksumsPath = Join-Path $tmpDir "SHA256SUMS" + $checksumsUrl = "https://github.com/$Repo/releases/download/$tag/SHA256SUMS" + Invoke-WebRequest -Uri $checksumsUrl -OutFile $checksumsPath + $checksumLine = Get-Content $checksumsPath | Where-Object { $_ -match "^[0-9a-fA-F]+\s+$([regex]::Escape($archive))$" } | Select-Object -First 1 + if (-not $checksumLine) { + throw "Checksum file did not contain $archive." + } + $expectedChecksum = ($checksumLine -split "\s+")[0].ToLowerInvariant() + $actualChecksum = (Get-FileHash -Path $zipPath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualChecksum -ne $expectedChecksum) { + throw "Checksum mismatch for $archive." + } Expand-Archive -Path $zipPath -DestinationPath $extractDir -Force $binary = Join-Path $extractDir "lexa-windows-x86_64-$assetVersion\lexa.exe" diff --git a/install.sh b/install.sh index 4fe0217..8f7a424 100755 --- a/install.sh +++ b/install.sh @@ -71,6 +71,28 @@ trap cleanup EXIT INT TERM printf 'Downloading %s...\n' "$url" curl -fL --retry 3 --retry-delay 2 -o "$tmp_dir/$archive" "$url" +checksum_url="https://github.com/$repo/releases/download/$tag/SHA256SUMS" +curl -fL --retry 3 --retry-delay 2 -o "$tmp_dir/SHA256SUMS" "$checksum_url" +expected_checksum="$( + sed -n "s/^\\([[:xdigit:]]*\\)[[:space:]][[:space:]]*$archive$/\\1/p" "$tmp_dir/SHA256SUMS" | + sed -n '1p' +)" +if [ -z "$expected_checksum" ]; then + printf 'error: checksum file did not contain %s\n' "$archive" >&2 + exit 1 +fi +if command -v sha256sum >/dev/null 2>&1; then + actual_checksum="$(sha256sum "$tmp_dir/$archive" | sed 's/[[:space:]].*//')" +elif command -v shasum >/dev/null 2>&1; then + actual_checksum="$(shasum -a 256 "$tmp_dir/$archive" | sed 's/[[:space:]].*//')" +else + printf 'error: sha256sum or shasum is required to verify the download\n' >&2 + exit 1 +fi +if [ "$actual_checksum" != "$expected_checksum" ]; then + printf 'error: checksum mismatch for %s\n' "$archive" >&2 + exit 1 +fi tar -xzf "$tmp_dir/$archive" -C "$tmp_dir" binary="$tmp_dir/lexa-${platform}-${asset_version}/lexa" @@ -95,6 +117,8 @@ case ":$PATH:" in *":$install_dir:"*) ;; *) printf 'Add this directory to PATH to run lexa from anywhere:\n' + # $PATH must remain literal in the instruction printed for the user. + # shellcheck disable=SC2016 printf ' export PATH="%s:$PATH"\n' "$install_dir" ;; esac diff --git a/justfile b/justfile index 3e78ced..89c0802 100644 --- a/justfile +++ b/justfile @@ -13,6 +13,9 @@ build: bench: cargo bench --bench engine +perf-gate: + cargo run -p xtask -- perf-gate + verify: fmt lint test build gen-skill: diff --git a/src/application.rs b/src/application.rs new file mode 100644 index 0000000..518cc1e --- /dev/null +++ b/src/application.rs @@ -0,0 +1,204 @@ +use anyhow::Result; +use std::path::Path; + +use crate::edit::{self, AnchorPlacement, EditOp, PreviewMode}; +use crate::engine::{Engine, ReadFileResult}; +use crate::project_path::{normalize_project_path, project_target_path, PathMode}; +use crate::snapshot; +use crate::store; +use crate::{audit, audit::AuditOptions}; + +pub struct ProjectSession<'a> { + engine: &'a mut Engine, + root: &'a Path, + graph_path: &'a Path, + persist_graph: bool, +} + +pub struct ReadRequest<'a> { + pub path: &'a str, + pub existing_only: bool, + pub line_start: Option, + pub line_end: Option, + pub compact: bool, + pub if_hash: Option<&'a str>, +} + +pub struct ReadResult { + pub path: String, + pub file: Option, +} + +pub struct PatchRequest<'a> { + pub path: &'a str, + pub op: Option, + pub range_start: Option, + pub range_end: Option, + pub after: Option, + pub content: Option, + pub replace_text: Option, + pub anchor: Option, + pub placement: Option, + pub preview_mode: PreviewMode, + pub if_hash: Option, + pub dry_run: bool, +} + +pub struct PatchResult { + pub path: String, + pub edit: edit::EditResult, +} + +pub struct CreateRequest<'a> { + pub path: &'a str, + pub content: String, + pub overwrite: bool, + pub dry_run: bool, +} + +pub struct CreateResult { + pub path: String, + pub create: edit::CreateResult, + pub would_create: bool, +} + +impl<'a> ProjectSession<'a> { + pub fn new( + engine: &'a mut Engine, + root: &'a Path, + graph_path: &'a Path, + persist_graph: bool, + ) -> Self { + Self { + engine, + root, + graph_path, + persist_graph, + } + } + + pub fn read(&self, request: ReadRequest<'_>) -> Result { + let mode = if request.existing_only { + PathMode::Existing + } else { + PathMode::Create + }; + let path = normalize_project_path(self.root, request.path, mode)?; + let file = self.engine.read_file_rich( + &path, + request.line_start, + request.line_end, + request.compact, + request.if_hash, + ); + Ok(ReadResult { path, file }) + } + + pub fn patch(&mut self, request: PatchRequest<'_>) -> Result { + let path = normalize_project_path(self.root, request.path, PathMode::Existing)?; + let effective_op = effective_edit_op( + request.op, + request.replace_text.as_deref(), + request.anchor.as_deref(), + )?; + let edit_request = edit::EditRequest { + path: project_target_path(self.root, &path), + op: request.op, + range_start: request.range_start, + range_end: request.range_end, + after: request.after, + content: request.content, + replace_text: request.replace_text, + anchor: request.anchor, + placement: request.placement, + preview_mode: request.preview_mode, + if_hash: request.if_hash, + dry_run: request.dry_run, + }; + let edit = edit::apply_edit(&edit_request)?; + + if edit.changed && !edit_request.dry_run { + self.engine + .index_edited_file(&path, &edit.new_content, store_op(effective_op)); + self.persist()?; + } + + Ok(PatchResult { path, edit }) + } + + pub fn create(&mut self, request: CreateRequest<'_>) -> Result { + let path = normalize_project_path(self.root, request.path, PathMode::Create)?; + let target = project_target_path(self.root, &path); + let would_create = request.dry_run && !target.exists(); + let create_request = edit::CreateRequest { + path: target, + content: request.content.clone(), + overwrite: request.overwrite, + dry_run: request.dry_run, + }; + let create = edit::create_file(&create_request)?; + + if !request.dry_run { + self.engine + .index_edited_file(&path, &request.content, store::Op::Create); + self.persist()?; + } + + Ok(CreateResult { + path, + create, + would_create, + }) + } + + pub fn audit(&self, options: AuditOptions) -> audit::AuditReport { + audit::run_audit(self.engine, options) + } + + pub fn reindex(&mut self) -> Result { + let mut engine = Engine::new(16_384); + let count = engine.index_project(self.root); + if self.persist_graph { + snapshot::write_snapshot(&engine, self.graph_path)?; + } + *self.engine = engine; + Ok(count) + } + + pub fn clear_index(&mut self) -> Result { + let existed = self.graph_path.exists(); + if existed { + std::fs::remove_file(self.graph_path)?; + } + *self.engine = Engine::new(16_384); + Ok(existed) + } + + fn persist(&self) -> Result<()> { + if self.persist_graph { + snapshot::write_snapshot(self.engine, self.graph_path)?; + } + Ok(()) + } +} + +fn effective_edit_op( + op: Option, + replace_text: Option<&str>, + anchor: Option<&str>, +) -> Result { + match (op, replace_text.is_some(), anchor.is_some()) { + (Some(op), false, false) => Ok(op), + (None, true, false) => Ok(EditOp::Replace), + (None, false, true) => Ok(EditOp::Insert), + _ => anyhow::bail!("patch requires exactly one target: operation, replace_text, or anchor"), + } +} + +fn store_op(op: EditOp) -> store::Op { + match op { + EditOp::Replace => store::Op::Replace, + EditOp::Insert => store::Op::Insert, + EditOp::Delete => store::Op::Delete, + } +} diff --git a/src/audit.rs b/src/audit.rs index fde10ae..da49dd9 100644 --- a/src/audit.rs +++ b/src/audit.rs @@ -4,8 +4,8 @@ mod rules; mod scope; pub use config::{ - load_audit_config, AuditConfig, AuditIgnore, AuditRules, AuditThresholds, DeadCodeConfig, - RuleSetting, + load_audit_config, AuditConfig, AuditIgnore, AuditIncludes, AuditRules, AuditThresholds, + DeadCodeConfig, RuleSetting, }; pub use report::{ render_audit_report, AuditActionability, AuditFinding, AuditGroups, AuditNextStep, AuditReport, @@ -40,11 +40,6 @@ impl Default for AuditOptions { } } -#[derive(Debug, Clone, Copy, Default)] -pub struct AuditIncludes { - pub dead_code: bool, -} - pub fn run_audit(engine: &Engine, options: AuditOptions) -> AuditReport { let max_results = options.max_results.unwrap_or(options.config.max_findings); diff --git a/src/audit/config.rs b/src/audit/config.rs index d38efbd..8caa64f 100644 --- a/src/audit/config.rs +++ b/src/audit/config.rs @@ -6,6 +6,11 @@ use std::path::{Path, PathBuf}; use super::report::AuditSeverity; +#[derive(Debug, Clone, Copy, Default)] +pub struct AuditIncludes { + pub dead_code: bool, +} + pub(crate) const DEFAULT_MAX_FINDINGS: usize = 100; const LARGE_FILE_WARNING_LINES: u32 = 800; const LARGE_FILE_HIGH_LINES: u32 = 1500; diff --git a/src/audit/rules.rs b/src/audit/rules.rs index 6ccf4c3..83063e1 100644 --- a/src/audit/rules.rs +++ b/src/audit/rules.rs @@ -7,10 +7,9 @@ use crate::engine::Engine; use crate::glob::match_glob; use hashbrown::HashSet; -use super::config::{AuditConfig, AuditIgnore, DEFAULT_GENERATED_IGNORE_GLOBS}; +use super::config::{AuditConfig, AuditIgnore, AuditIncludes, DEFAULT_GENERATED_IGNORE_GLOBS}; use super::report::AuditFinding; use super::scope::AuditScope; -use super::AuditIncludes; pub(crate) fn collect_findings( engine: &Engine, diff --git a/src/audit/rules/architecture.rs b/src/audit/rules/architecture.rs index d2aefa8..f94258d 100644 --- a/src/audit/rules/architecture.rs +++ b/src/audit/rules/architecture.rs @@ -1,12 +1,10 @@ use crate::engine::Engine; -use hashbrown::HashSet; -use std::collections::HashMap; +use hashbrown::{HashMap, HashSet}; use crate::audit::config::AuditConfig; use crate::audit::report::{AuditActionability, AuditFinding, AuditNextStep, AuditSeverity}; use serde_json::json; -const MAX_CYCLE_DEPTH: usize = 32; const MAX_INTERNAL_CYCLES: usize = 1000; pub(crate) fn audit_cycles( @@ -27,20 +25,35 @@ pub(crate) fn audit_cycles( let Some(path) = cycle.first().cloned() else { continue; }; + let module_internal = is_internal_rust_module_cycle(&cycle); + let finding_severity = if module_internal && severity == AuditSeverity::High { + AuditSeverity::Warning + } else { + severity + }; let next_path = path.clone(); let trace_path = path.clone(); + let evidence_cycle = ordered_cycle(engine, &cycle).unwrap_or_else(|| { + let mut fallback = cycle.clone(); + fallback.push(path.clone()); + fallback + }); findings.push(AuditFinding { id: format!("architecture.cycle:{path}"), rule: "architecture.cycle".to_string(), - severity, + severity: finding_severity, actionability: AuditActionability::Actionable, secondary: false, - title: "Import cycle detected".to_string(), + title: if module_internal { + "Rust module dependency cycle detected".to_string() + } else { + "Import cycle detected".to_string() + }, path, line_start: None, line_end: None, message: "Files in this cycle depend on each other through parsed imports.".to_string(), - evidence: vec![cycle.join(" -> ")], + evidence: vec![evidence_cycle.join(" -> ")], related_paths: cycle, suggestion: "Break the cycle by moving shared types or behavior into a lower-level module." @@ -56,12 +69,51 @@ pub(crate) fn audit_cycles( } } +fn ordered_cycle(engine: &Engine, component: &[String]) -> Option> { + let start = component.first()?; + let members = component.iter().cloned().collect::>(); + let mut path = vec![start.clone()]; + if find_cycle_path(engine, start, start, &members, &mut path) { + Some(path) + } else { + None + } +} + +fn find_cycle_path( + engine: &Engine, + start: &str, + current: &str, + members: &HashSet, + path: &mut Vec, +) -> bool { + for neighbor in engine.get_depends_on(current) { + if !members.contains(&neighbor) { + continue; + } + if neighbor == start && path.len() > 1 { + path.push(neighbor); + return true; + } + if path.contains(&neighbor) { + continue; + } + path.push(neighbor.clone()); + if find_cycle_path(engine, start, &neighbor, members, path) { + return true; + } + path.pop(); + } + false +} + fn find_cycles(engine: &Engine) -> Vec> { - let paths = engine + let mut paths = engine .file_map() .into_iter() .map(|(path, _)| path) .collect::>(); + paths.sort(); let indexed = paths.iter().cloned().collect::>(); let mut adjacency = HashMap::new(); @@ -75,65 +127,140 @@ fn find_cycles(engine: &Engine) -> Vec> { adjacency.insert(path.clone(), deps); } - let mut cycles = Vec::new(); - let mut seen = HashSet::new(); + StronglyConnectedComponents::new(&adjacency) + .run(&paths) + .into_iter() + .filter(|component| component.len() > 1) + .take(MAX_INTERNAL_CYCLES) + .collect() +} - for start in &paths { - let mut stack = vec![start.clone()]; - dfs_cycles(start, start, &adjacency, &mut stack, &mut seen, &mut cycles); - if cycles.len() >= MAX_INTERNAL_CYCLES { - break; +struct StronglyConnectedComponents<'a> { + adjacency: &'a HashMap>, + next_index: usize, + indices: HashMap, + lowlinks: HashMap, + stack: Vec, + on_stack: HashSet, + components: Vec>, +} + +impl<'a> StronglyConnectedComponents<'a> { + fn new(adjacency: &'a HashMap>) -> Self { + Self { + adjacency, + next_index: 0, + indices: HashMap::new(), + lowlinks: HashMap::new(), + stack: Vec::new(), + on_stack: HashSet::new(), + components: Vec::new(), } } - cycles -} - -fn dfs_cycles( - start: &str, - current: &str, - adjacency: &HashMap>, - stack: &mut Vec, - seen: &mut HashSet, - cycles: &mut Vec>, -) { - if stack.len() > MAX_CYCLE_DEPTH || cycles.len() >= MAX_INTERNAL_CYCLES { - return; + fn run(mut self, paths: &[String]) -> Vec> { + for path in paths { + if !self.indices.contains_key(path) { + self.visit(path); + } + } + self.components.sort(); + self.components } - let Some(neighbors) = adjacency.get(current) else { - return; - }; + fn visit(&mut self, path: &str) { + let index = self.next_index; + self.next_index += 1; + self.indices.insert(path.to_string(), index); + self.lowlinks.insert(path.to_string(), index); + self.stack.push(path.to_string()); + self.on_stack.insert(path.to_string()); - for neighbor in neighbors { - if neighbor == start && stack.len() > 1 { - let key = canonical_cycle_key(stack); - if seen.insert(key) { - let mut cycle = stack.clone(); - cycle.push(start.to_string()); - cycles.push(cycle); + for neighbor in self.adjacency.get(path).cloned().unwrap_or_default() { + if !self.indices.contains_key(&neighbor) { + self.visit(&neighbor); + let neighbor_lowlink = self.lowlinks[&neighbor]; + let path_lowlink = self.lowlinks[path]; + self.lowlinks + .insert(path.to_string(), path_lowlink.min(neighbor_lowlink)); + } else if self.on_stack.contains(&neighbor) { + let neighbor_index = self.indices[&neighbor]; + let path_lowlink = self.lowlinks[path]; + self.lowlinks + .insert(path.to_string(), path_lowlink.min(neighbor_index)); } - continue; } - if stack.iter().any(|path| path == neighbor) { - continue; + if self.lowlinks[path] != self.indices[path] { + return; } - stack.push(neighbor.clone()); - dfs_cycles(start, neighbor, adjacency, stack, seen, cycles); - stack.pop(); + let mut component = Vec::new(); + while let Some(member) = self.stack.pop() { + self.on_stack.remove(&member); + let finished = member == path; + component.push(member); + if finished { + break; + } + } + component.sort(); + self.components.push(component); + } +} + +fn is_internal_rust_module_cycle(cycle: &[String]) -> bool { + if cycle.iter().any(|path| !path.ends_with(".rs")) { + return false; } + + cycle.iter().any(|candidate| { + let family = candidate + .strip_suffix("/mod.rs") + .or_else(|| candidate.strip_suffix(".rs")) + .unwrap_or(candidate); + cycle.iter().all(|path| { + path == &format!("{family}.rs") + || path == &format!("{family}/mod.rs") + || path.starts_with(&format!("{family}/")) + }) + }) } -fn canonical_cycle_key(cycle: &[String]) -> String { - let mut rotations = Vec::new(); - for index in 0..cycle.len() { - let mut rotated = Vec::with_capacity(cycle.len()); - rotated.extend_from_slice(&cycle[index..]); - rotated.extend_from_slice(&cycle[..index]); - rotations.push(rotated.join("\u{1f}")); +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rust_parent_and_child_files_are_one_module_family() { + assert!(is_internal_rust_module_cycle(&[ + "src/audit.rs".to_string(), + "src/audit/rules.rs".to_string(), + "src/audit/rules/size.rs".to_string(), + ])); + assert!(!is_internal_rust_module_cycle(&[ + "src/engine/mod.rs".to_string(), + "src/snapshot.rs".to_string(), + ])); + } + + #[test] + fn cycle_search_returns_one_component_for_overlapping_cycles() { + let mut engine = Engine::new(4); + engine.index_file("src/a.rs", "use crate::b;\nuse crate::c;\n"); + engine.index_file("src/b.rs", "use crate::a;\nuse crate::c;\n"); + engine.index_file("src/c.rs", "use crate::a;\n"); + + let cycles = find_cycles(&engine); + + assert_eq!(cycles.len(), 1); + assert_eq!( + cycles[0], + vec![ + "src/a.rs".to_string(), + "src/b.rs".to_string(), + "src/c.rs".to_string() + ] + ); } - rotations.sort(); - rotations.remove(0) } diff --git a/src/audit/rules/dead_code.rs b/src/audit/rules/dead_code.rs index 7e92aa5..f721c42 100644 --- a/src/audit/rules/dead_code.rs +++ b/src/audit/rules/dead_code.rs @@ -2,9 +2,8 @@ use crate::engine::Engine; use crate::glob::match_glob; use crate::types::{Language, Symbol, SymbolKind}; -use crate::audit::config::{AuditConfig, DeadCodeConfig, RuleSetting}; +use crate::audit::config::{AuditConfig, AuditIncludes, DeadCodeConfig, RuleSetting}; use crate::audit::report::{AuditActionability, AuditFinding, AuditNextStep, AuditSeverity}; -use crate::audit::AuditIncludes; use serde_json::json; pub(crate) fn audit_dead_code_candidates( diff --git a/src/cache.rs b/src/cache.rs deleted file mode 100644 index bd1ecea..0000000 --- a/src/cache.rs +++ /dev/null @@ -1,264 +0,0 @@ -use std::sync::atomic::{AtomicU64, Ordering}; - -const PROBE_LIMIT: u32 = 4; - -#[derive(Default)] -struct Slot { - key: String, - value: String, - key_hash: u64, - ref_bit: bool, - present: bool, -} - -pub struct ContentCache { - slots: Vec, - capacity: u32, - hand: u32, - count: u32, - hits: AtomicU64, - misses: AtomicU64, - evictions: AtomicU64, -} - -#[cfg(test)] -#[derive(Debug, Clone)] -pub struct CacheStats { - pub hits: u64, - pub misses: u64, - pub evictions: u64, - pub count: u32, - pub capacity: u32, -} - -impl ContentCache { - pub fn new(capacity: u32) -> Self { - assert!(capacity >= 1, "ContentCache capacity must be >= 1"); - let mut slots = Vec::with_capacity(capacity as usize); - slots.resize_with(capacity as usize, Slot::default); - Self { - slots, - capacity, - hand: 0, - count: 0, - hits: AtomicU64::new(0), - misses: AtomicU64::new(0), - evictions: AtomicU64::new(0), - } - } - - fn hash_key(key: &str) -> u64 { - let mut h: u64 = 14695981039346656037; - for b in key.bytes() { - h ^= b as u64; - h = h.wrapping_mul(1099511628211); - } - if h == 0 { - 1 - } else { - h - } - } - - pub fn get(&self, key: &str) -> Option<&str> { - let h = Self::hash_key(key); - let base = (h as u32) % self.capacity; - for i in 0..PROBE_LIMIT { - let slot_idx = (base.wrapping_add(i)) % self.capacity; - let slot = &self.slots[slot_idx as usize]; - if slot.present && slot.key_hash == h && slot.key == key { - self.hits.fetch_add(1, Ordering::Relaxed); - return Some(&slot.value); - } - if !slot.present && slot.key_hash == 0 { - break; - } - } - self.misses.fetch_add(1, Ordering::Relaxed); - None - } - - pub fn put(&mut self, key: String, value: String) { - let h = Self::hash_key(&key); - let base = (h as u32) % self.capacity; - let mut tombstone_idx = None; - - for i in 0..PROBE_LIMIT { - let slot_idx = (base.wrapping_add(i)) % self.capacity; - let slot = &mut self.slots[slot_idx as usize]; - if slot.present && slot.key_hash == h && slot.key == key { - slot.value = value; - slot.ref_bit = true; - return; - } - if !slot.present && slot.key_hash != 0 && tombstone_idx.is_none() { - tombstone_idx = Some(slot_idx); - } - if !slot.present && slot.key_hash == 0 { - let insert_idx = tombstone_idx.unwrap_or(slot_idx); - self.insert_at(insert_idx, key, value, h); - return; - } - } - - let insert_idx = tombstone_idx.unwrap_or_else(|| self.clock_evict()); - self.insert_at(insert_idx, key, value, h); - } - - pub fn remove(&mut self, key: &str) { - let h = Self::hash_key(key); - let base = (h as u32) % self.capacity; - for i in 0..PROBE_LIMIT { - let slot_idx = (base.wrapping_add(i)) % self.capacity; - let slot = &mut self.slots[slot_idx as usize]; - if slot.present && slot.key_hash == h && slot.key == key { - slot.present = false; - slot.key.clear(); - slot.value.clear(); - slot.ref_bit = false; - self.count -= 1; - return; - } - if !slot.present && slot.key_hash == 0 { - return; - } - } - } - - fn insert_at(&mut self, slot_idx: u32, key: String, value: String, key_hash: u64) { - let slot = &mut self.slots[slot_idx as usize]; - if slot.present { - self.count -= 1; - self.evictions.fetch_add(1, Ordering::Relaxed); - } - slot.key = key; - slot.value = value; - slot.key_hash = key_hash; - slot.ref_bit = true; - slot.present = true; - self.count += 1; - } - - pub fn clear(&mut self) { - for slot in &mut self.slots { - slot.present = false; - slot.key_hash = 0; - slot.ref_bit = false; - slot.key.clear(); - slot.value.clear(); - } - self.count = 0; - self.hand = 0; - } - - #[cfg(test)] - pub fn len(&self) -> u32 { - self.count - } - - #[cfg(test)] - pub fn is_empty(&self) -> bool { - self.count == 0 - } - - #[cfg(test)] - pub fn stats(&self) -> CacheStats { - CacheStats { - hits: self.hits.load(Ordering::Relaxed), - misses: self.misses.load(Ordering::Relaxed), - evictions: self.evictions.load(Ordering::Relaxed), - count: self.count, - capacity: self.capacity, - } - } - - fn clock_evict(&mut self) -> u32 { - let cap = self.capacity; - let mut sweeps = 0u32; - while sweeps < cap * 2 { - let slot_idx = self.hand % cap; - self.hand = (self.hand.wrapping_add(1)) % cap; - let slot = &self.slots[slot_idx as usize]; - if !slot.present { - return slot_idx; - } - if !slot.ref_bit { - return slot_idx; - } - self.slots[slot_idx as usize].ref_bit = false; - sweeps += 1; - } - let slot_idx = self.hand % cap; - self.hand = (self.hand.wrapping_add(1)) % cap; - slot_idx - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn basic_get_put() { - let mut cache = ContentCache::new(64); - cache.put("foo".to_string(), "bar".to_string()); - assert_eq!(cache.get("foo"), Some("bar")); - assert!(cache.get("missing").is_none()); - } - - #[test] - fn put_updates_existing() { - let mut cache = ContentCache::new(64); - cache.put("key".to_string(), "v1".to_string()); - cache.put("key".to_string(), "v2".to_string()); - assert_eq!(cache.get("key"), Some("v2")); - assert_eq!(cache.len(), 1); - } - - #[test] - fn clear_drops_all() { - let mut cache = ContentCache::new(64); - cache.put("a".to_string(), "1".to_string()); - cache.put("b".to_string(), "2".to_string()); - cache.clear(); - assert_eq!(cache.len(), 0); - assert!(cache.is_empty()); - assert!(cache.get("a").is_none()); - } - - #[test] - fn eviction_under_pressure() { - let mut cache = ContentCache::new(50); - for i in 0..100 { - cache.put(format!("file_{i}.zig"), format!("content_{i}")); - } - assert!(cache.len() <= 50); - let stats = cache.stats(); - assert!(stats.evictions > 0); - assert_eq!(stats.count, cache.len()); - assert_eq!(stats.capacity, 50); - assert!(stats.hits + stats.misses <= 100); - } - - #[test] - fn remove_preserves_collided_probe_chain() { - let capacity = 8; - let mut keys = Vec::new(); - for i in 0..1000 { - let key = format!("key_{i}"); - if (ContentCache::hash_key(&key) as u32).is_multiple_of(capacity) { - keys.push(key); - if keys.len() == 2 { - break; - } - } - } - - let mut cache = ContentCache::new(capacity); - cache.put(keys[0].clone(), "first".to_string()); - cache.put(keys[1].clone(), "second".to_string()); - cache.remove(&keys[0]); - - assert_eq!(cache.get(&keys[1]), Some("second")); - } -} diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..f819ea8 --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,383 @@ +use clap::{ArgAction, Parser, Subcommand, ValueEnum}; +use lexa::edit; +use std::path::PathBuf; + +#[derive(Parser)] +#[command( + name = "lexa", + disable_version_flag = true, + about = "Fast code intelligence engine for AI agents" +)] +pub(crate) struct Cli { + #[command(subcommand)] + pub(crate) command: Option, + + #[arg( + id = "print_version", + long = "version", + global = true, + action = ArgAction::SetTrue, + help = "Print version and check for updates" + )] + pub(crate) version: bool, + + #[arg(long, global = true)] + pub(crate) graph: Option, + + #[arg(long = "no-graph", global = true)] + pub(crate) no_graph: bool, + + #[arg(long, global = true, hide = true)] + pub(crate) json: bool, +} + +#[derive(Subcommand)] +pub(crate) enum Commands { + Index { + path: PathBuf, + + #[arg(short, long)] + output: Option, + }, + + Reindex { + #[arg(default_value = ".")] + path: PathBuf, + }, + + #[command(name = "clear-index")] + ClearIndex, + + #[command(name = "files")] + Files { + #[arg(default_value = "")] + path: String, + + #[arg(long)] + path_glob: Option, + + #[arg(long)] + language: Option, + + #[arg(long)] + min_lines: Option, + + #[arg(long)] + max_lines: Option, + + #[arg(long, alias = "max")] + max_results: Option, + }, + + List { + #[arg(default_value = "")] + path: String, + }, + + #[command(name = "path-search")] + PathSearch { + pattern: Option, + + #[arg(long)] + query: Option, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + }, + + #[command( + name = "text-search", + after_help = "Examples:\n lexa text-search \"uploadMutation\" --max 20\n lexa text-search --query \"uploadMutation\" --max-results 20\n lexa text-search \"useMutation\" --path-glob \"**/*.{ts,tsx}\"" + )] + TextSearch { + query: Option, + + #[arg(long = "query", value_name = "QUERY")] + query_flag: Option, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + + #[arg(short, long)] + regex: bool, + + #[arg(long)] + scope: bool, + + #[arg(short, long)] + compact: bool, + + #[arg(long)] + paths_only: bool, + + #[arg(long)] + path_glob: Option, + }, + + Outline { + path: String, + }, + + #[command(name = "symbol-defs")] + SymbolDefs { + name: String, + }, + + #[command(name = "symbol-search")] + SymbolSearch { + query: Option, + + #[arg(long = "query", value_name = "QUERY")] + query_flag: Option, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + }, + + #[command(name = "word-refs")] + WordRefs { + word: String, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + + #[arg(long, default_value = "0")] + cursor: usize, + + #[arg(long)] + path_prefix: Option, + + #[arg(long = "path")] + path: Option, + + #[arg(long)] + path_glob: Option, + }, + + #[command(name = "trace-deps")] + Deps { + path: String, + + #[arg(short, long)] + reverse: bool, + + #[arg(short, long)] + transitive: bool, + }, + + Recent { + #[arg(short, long, default_value = "10")] + limit: usize, + }, + + Callers { + name: Option, + + #[arg(long)] + query: Option, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + }, + + Brief { + task: Option, + + #[arg(long)] + query: Option, + + #[arg(short, long)] + max: Option, + + #[arg(long)] + max_results: Option, + + #[arg(long)] + path_prefix: Option, + + #[arg(long)] + path_glob: Option, + + #[arg(long)] + language: Option, + }, + + Changes { + #[arg(default_value = "0")] + since: u64, + }, + + #[command( + after_help = "Examples:\n lexa read src/main.rs -L 20-80 --hash\n lexa read src/main.rs --line-start 20 --line-end 80\n lexa read src/main.rs --if-hash " + )] + Read { + path: String, + + #[arg(short = 'L', long)] + line_range: Option, + + #[arg(long)] + line_start: Option, + + #[arg(long)] + line_end: Option, + + #[arg(short, long)] + compact: bool, + + #[arg(long)] + if_hash: Option, + + #[arg(long)] + hash: bool, + }, + + #[command( + after_help = "Examples:\n lexa patch src/main.rs replace -L 12 --content ' println!(\"updated\");'\n lexa patch src/main.rs insert --after 20 --content '// new comment' --preview compact --dry-run\n lexa patch src/main.rs --replace-text 'old block' --content 'new block'\n lexa patch src/main.rs --anchor 'const uploadMutation' --placement after --content 'const helper = ...;'" + )] + Patch { + path: String, + + #[arg(value_enum)] + op: Option, + + #[arg(short = 'L', long)] + line_range: Option, + + #[arg(long)] + after: Option, + + #[arg(long)] + replace_text: Option, + + #[arg(long)] + anchor: Option, + + #[arg(long, value_enum)] + placement: Option, + + #[arg(long, value_enum, default_value = "compact")] + preview: edit::PreviewMode, + + #[arg(long)] + content: Option, + + #[arg(long)] + content_file: Option, + + #[arg(long)] + if_hash: Option, + + #[arg(long)] + dry_run: bool, + }, + + Create { + path: String, + + #[arg(long)] + content: Option, + + #[arg(long)] + content_file: Option, + + #[arg(long)] + overwrite: bool, + + #[arg(long)] + dry_run: bool, + }, + + Glob { + pattern: String, + }, + + Status, + + Audit { + #[arg(short, long)] + max: Option, + + #[arg(long)] + since: Option, + + #[arg(long)] + strict: bool, + + #[arg(long)] + config: Option, + + #[arg(long)] + no_config: bool, + + #[arg(long, value_enum)] + include: Vec, + }, + + #[command( + alias = "update", + about = "Upgrade the Lexa binary, not the project index" + )] + Upgrade { + #[arg(id = "upgrade_version", default_value = "latest")] + version: String, + + #[arg(long, help = "Directory to install the upgraded Lexa binary into")] + install_dir: Option, + }, + + Watch { + #[arg(default_value = ".")] + path: String, + + #[arg(short, long, default_value = "500")] + debounce: u64, + }, + + Pipeline { + #[arg(trailing_var_arg = true)] + pipeline: Vec, + }, + + Mcp { + #[arg(default_value = ".")] + path: PathBuf, + + #[arg(long)] + no_refresh: bool, + + #[arg(long, default_value = "500")] + debounce: u64, + + #[arg(long = "structured-content", alias = "json-output", hide = true)] + structured_content: bool, + + #[arg(long = "log-file")] + log_file: Option, + }, + + /// Dump MCP tool specs as JSON for repository tooling. Internal use. + #[command(name = "dump-tools", hide = true)] + DumpTools, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum AuditInclude { + #[value(name = "dead-code")] + DeadCode, +} diff --git a/src/cli_tests.rs b/src/cli_tests.rs new file mode 100644 index 0000000..9260325 --- /dev/null +++ b/src/cli_tests.rs @@ -0,0 +1,114 @@ +use super::*; +use crate::cli::Commands; +use clap::Parser; +use std::path::PathBuf; + +#[test] +fn upgrade_default_version_does_not_conflict_with_global_version_flag() { + let cli = Cli::try_parse_from(["lexa", "upgrade"]).unwrap(); + + assert!(!cli.version); + match cli.command { + Some(Commands::Upgrade { + version, + install_dir, + }) => { + assert_eq!(version, "latest"); + assert!(install_dir.is_none()); + } + _ => panic!("expected upgrade command"), + } +} + +#[test] +fn mcp_defaults_to_refresh_with_standard_debounce() { + let cli = Cli::try_parse_from(["lexa", "mcp", "."]).unwrap(); + + match cli.command { + Some(Commands::Mcp { + no_refresh, + debounce, + structured_content, + .. + }) => { + assert!(!no_refresh); + assert_eq!(debounce, 500); + assert!(!structured_content); + } + _ => panic!("expected mcp command"), + } +} + +#[test] +fn mcp_accepts_no_refresh_and_custom_debounce() { + let cli = + Cli::try_parse_from(["lexa", "mcp", ".", "--no-refresh", "--debounce", "250"]).unwrap(); + + match cli.command { + Some(Commands::Mcp { + no_refresh, + debounce, + structured_content, + .. + }) => { + assert!(no_refresh); + assert_eq!(debounce, 250); + assert!(!structured_content); + } + _ => panic!("expected mcp command"), + } +} + +#[test] +fn removed_output_flags_are_detected_before_clap_parse() { + for flag in ["--json", "--structured-content", "--json-output"] { + assert_eq!(removed_output_flag(["lexa", "mcp", ".", flag]), Some(flag)); + } +} + +#[test] +fn mcp_accepts_log_file_flag() { + let cli = Cli::try_parse_from(["lexa", "mcp", ".", "--log-file", "/tmp/lexa-mcp.log"]).unwrap(); + + match cli.command { + Some(Commands::Mcp { log_file, .. }) => { + assert_eq!(log_file, Some(PathBuf::from("/tmp/lexa-mcp.log"))); + } + _ => panic!("expected mcp command"), + } +} + +#[test] +fn removed_output_flag_detects_equals_forms() { + assert_eq!(removed_output_flag(["lexa", "--json=true"]), Some("--json")); + assert_eq!( + removed_output_flag(["lexa", "mcp", ".", "--structured-content=true"]), + Some("--structured-content") + ); + assert_eq!( + removed_output_flag(["lexa", "mcp", ".", "--json-output=true"]), + Some("--json-output") + ); +} + +#[test] +fn removed_output_flag_respects_end_of_options_sentinel() { + assert_eq!( + removed_output_flag(["lexa", "pipeline", "--", "--json"]), + None + ); + assert_eq!( + removed_output_flag(["lexa", "--json", "--", "--structured-content"]), + Some("--json") + ); +} + +#[test] +fn parse_line_range_supports_single_bounded_and_open_ranges() { + assert_eq!(parse_line_range("7").unwrap(), (Some(7), Some(7))); + assert_eq!(parse_line_range("3-9").unwrap(), (Some(3), Some(9))); + assert_eq!(parse_line_range("-9").unwrap(), (None, Some(9))); + assert_eq!(parse_line_range("3-").unwrap(), (Some(3), None)); + assert!(parse_line_range("abc").is_err()); + assert!(parse_line_range("3-abc").is_err()); +} diff --git a/src/cli_upgrade.rs b/src/cli_upgrade.rs index 6f0adb4..6a45ea8 100644 --- a/src/cli_upgrade.rs +++ b/src/cli_upgrade.rs @@ -174,12 +174,19 @@ fn unix_secs() -> u64 { #[cfg(not(windows))] fn cmd_upgrade_unix(version: &str, install_dir: &Path, json_output: bool) -> Result<()> { - let script = r#"curl -fsSL https://raw.githubusercontent.com/anvia-hq/lexa/main/install.sh | sh -s -- "$1""#; + let tag = release_tag(version); + let installer_url = if version == "latest" { + format!("https://github.com/{LEXA_REPO}/releases/latest/download/install.sh") + } else { + format!("https://github.com/{LEXA_REPO}/releases/download/{tag}/install.sh") + }; + let script = r#"curl -fsSL "$1" | sh -s -- "$2""#; let mut command = std::process::Command::new("sh"); command .arg("-c") .arg(script) .arg("lexa-upgrade") + .arg(installer_url) .arg(version) .env("LEXA_INSTALL_DIR", install_dir); @@ -224,8 +231,14 @@ fn cmd_upgrade_windows(version: &str, install_dir: &Path, json_output: bool) -> let pid = std::process::id(); let escaped_version = version.replace('\'', "''"); let escaped_install_dir = install_dir.display().to_string().replace('\'', "''"); + let tag = release_tag(version); + let installer_url = if version == "latest" { + format!("https://github.com/{LEXA_REPO}/releases/latest/download/install.ps1") + } else { + format!("https://github.com/{LEXA_REPO}/releases/download/{tag}/install.ps1") + }; let command = format!( - "Wait-Process -Id {pid}; $env:LEXA_VERSION = '{escaped_version}'; $env:LEXA_INSTALL_DIR = '{escaped_install_dir}'; irm https://raw.githubusercontent.com/anvia-hq/lexa/main/install.ps1 | iex" + "Wait-Process -Id {pid}; $env:LEXA_VERSION = '{escaped_version}'; $env:LEXA_INSTALL_DIR = '{escaped_install_dir}'; irm '{installer_url}' | iex" ); std::process::Command::new("powershell") @@ -277,6 +290,14 @@ fn validate_upgrade_version(version: &str) -> Result<()> { bail!("upgrade version must contain only letters, numbers, '.', '_', or '-'") } +fn release_tag(version: &str) -> String { + if version == "latest" || version.starts_with('v') { + version.to_string() + } else { + format!("v{version}") + } +} + fn print_json(value: serde_json::Value) -> Result<()> { println!("{}", serde_json::to_string_pretty(&value)?); Ok(()) @@ -300,6 +321,13 @@ mod tests { assert!(validate_upgrade_version("$(echo bad)").is_err()); } + #[test] + fn release_tags_are_normalized_for_versioned_installers() { + assert_eq!(release_tag("latest"), "latest"); + assert_eq!(release_tag("v1.2.3"), "v1.2.3"); + assert_eq!(release_tag("1.2.3"), "v1.2.3"); + } + #[test] fn upgrade_install_dir_prefers_explicit_value() { let explicit = PathBuf::from("/tmp/lexa-explicit"); diff --git a/src/commands/graph.rs b/src/commands/graph.rs new file mode 100644 index 0000000..104f3fd --- /dev/null +++ b/src/commands/graph.rs @@ -0,0 +1,229 @@ +use crate::cli::Cli; +use anyhow::{Context, Result}; +use lexa::engine; +use lexa::{freshness, mcp, snapshot}; +use serde_json::json; +use std::io::IsTerminal; +use std::path::PathBuf; + +use super::shared::*; + +pub(crate) fn cmd_dump_tools() -> Result<()> { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + serde_json::to_writer_pretty(&mut handle, &*lexa::mcp::tool_spec::TOOL_SPECS)?; + use std::io::Write as _; + writeln!(handle)?; + Ok(()) +} + +pub(crate) fn cmd_index(root: &PathBuf, output: Option<&PathBuf>, cli: &Cli) -> Result<()> { + let root = std::fs::canonicalize(root)?; + let snap_path = if let Some(out) = output { + out.clone() + } else { + graph_path_for_root(&root, cli) + }; + + if !cli.json { + print_index_banner(&root, &snap_path); + } + + let mut engine = engine::Engine::new(16384); + let count = engine.index_project(&root); + + if cli.json { + if !cli.no_graph { + snapshot::write_snapshot(&engine, &snap_path)?; + } + return print_agent_result(json!({ + "root": root.display().to_string(), + "files_indexed": count, + "symbols_indexed": engine.symbol_index_count(), + "unique_words_indexed": engine.word_index_count(), + "word_indexed_files": engine.word_index_file_count(), + "graph": (!cli.no_graph).then(|| snap_path.display().to_string()), + "persisted": !cli.no_graph, + })); + } + + println!("Indexed {} files", count); + println!(" Symbols: {}", engine.symbol_index_count()); + println!(" Unique words: {}", engine.word_index_count()); + println!(" Word-indexed files: {}", engine.word_index_file_count()); + + if !cli.no_graph { + snapshot::write_snapshot(&engine, &snap_path)?; + println!("Graph saved to {}", snap_path.display()); + } else { + println!("Graph not saved (--no-graph)"); + } + + Ok(()) +} + +pub(crate) fn print_index_banner(root: &std::path::Path, graph: &std::path::Path) { + if std::io::stdout().is_terminal() + && std::env::var_os("NO_COLOR").is_none() + && std::env::var("TERM").ok().as_deref() != Some("dumb") + { + println!( + "\x1b[1;38;5;81mLexa\x1b[0m \x1b[38;5;245mFast code intelligence for AI agents\x1b[0m" + ); + println!("\x1b[38;5;245mroot\x1b[0m {}", root.display()); + println!("\x1b[38;5;245mgraph\x1b[0m {}", graph.display()); + println!(); + } else { + println!("Indexing {}...", root.display()); + } +} + +pub(crate) fn cmd_reindex(root: &PathBuf, cli: &Cli) -> Result<()> { + cmd_index(root, None, cli) +} + +pub(crate) fn cmd_clear_index(cli: &Cli) -> Result<()> { + let snap_path = graph_path(cli)?; + let existed = snap_path.exists(); + if existed { + std::fs::remove_file(&snap_path) + .with_context(|| format!("failed to remove graph {}", snap_path.display()))?; + } + + if cli.json { + return print_agent_result(json!({ + "graph": snap_path.display().to_string(), + "removed": existed, + })); + } + + if existed { + println!("Removed graph {}", snap_path.display()); + } else { + println!("No graph file found at {}", snap_path.display()); + } + Ok(()) +} + +pub(crate) fn cmd_mcp( + path: &PathBuf, + no_refresh: bool, + debounce_ms: u64, + log_file: Option<&PathBuf>, + cli: &Cli, +) -> Result<()> { + let mut diagnostics = match log_file { + Some(path) => mcp::Diagnostics::append_to_path(path)?, + None => mcp::Diagnostics::disabled(), + }; + diagnostics.info(format!( + "lexa {} starting MCP server", + env!("CARGO_PKG_VERSION") + )); + diagnostics.info(format!("requested_root={}", path.display())); + + let root = match std::fs::canonicalize(path) { + Ok(root) => root, + Err(err) => { + diagnostics.error(format!( + "failed to resolve MCP root {}: {err}", + path.display() + )); + return Err(err.into()); + } + }; + let snap_path = graph_path_for_root(&root, cli); + let mut engine = engine::Engine::new(16384); + diagnostics.info(format!( + "root={} graph={} persist_graph={} refresh={} watcher={} output=toon", + root.display(), + snap_path.display(), + !cli.no_graph, + !no_refresh, + !no_refresh + )); + + if !cli.no_graph && snap_path.exists() { + match snapshot::load_snapshot_into_engine(&mut engine, &snap_path) { + Ok(count) => mcp_info( + &mut diagnostics, + format!("Loaded {} files from graph", count), + ), + Err(err) => mcp_warn(&mut diagnostics, format!("Failed to load graph: {err}")), + } + } + + if engine.file_count() == 0 { + mcp_info( + &mut diagnostics, + format!("Indexing {} for MCP...", root.display()), + ); + let count = engine.index_project(&root); + mcp_info(&mut diagnostics, format!("Indexed {} files", count)); + if !cli.no_graph { + if let Err(err) = snapshot::write_snapshot(&engine, &snap_path) { + diagnostics.error(format!( + "failed to save graph {}: {err}", + snap_path.display() + )); + return Err(err); + } + mcp_info( + &mut diagnostics, + format!("Graph saved to {}", snap_path.display()), + ); + } + } else if !no_refresh { + mcp_info(&mut diagnostics, "Checking MCP graph freshness..."); + let summary = match freshness::refresh_project(&mut engine, &root) { + Ok(summary) => summary, + Err(err) => { + diagnostics.error(format!( + "failed to refresh MCP graph for {}: {err}", + root.display() + )); + return Err(err); + } + }; + if summary.changed() { + mcp_info( + &mut diagnostics, + format!( + "Refreshed MCP graph: {} indexed, {} removed", + summary.indexed, summary.removed + ), + ); + if !cli.no_graph { + if let Err(err) = snapshot::write_snapshot(&engine, &snap_path) { + diagnostics.error(format!( + "failed to save graph {}: {err}", + snap_path.display() + )); + return Err(err); + } + mcp_info( + &mut diagnostics, + format!("Graph saved to {}", snap_path.display()), + ); + } + } + } + + let mut server = mcp::McpServer::new(engine, root, snap_path, !cli.no_graph, diagnostics); + if !no_refresh { + server.enable_watcher(debounce_ms)?; + } + server.run() +} + +pub(crate) fn mcp_info(diagnostics: &mut mcp::Diagnostics, message: impl AsRef) { + let message = message.as_ref(); + eprintln!("{message}"); + diagnostics.info(message); +} + +pub(crate) fn mcp_warn(diagnostics: &mut mcp::Diagnostics, message: impl AsRef) { + let message = message.as_ref(); + eprintln!("Warning: {message}"); + diagnostics.warn(message); +} diff --git a/src/commands/maintenance.rs b/src/commands/maintenance.rs new file mode 100644 index 0000000..53da343 --- /dev/null +++ b/src/commands/maintenance.rs @@ -0,0 +1,304 @@ +use crate::cli::Cli; +use anyhow::{bail, Result}; +use lexa::application::ProjectSession; +use lexa::engine; +use lexa::output::format_unix_ms_utc; +use lexa::{audit, pipeline, snapshot}; +use serde_json::json; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use super::shared::*; + +use crate::cli::AuditInclude; + +pub(crate) fn cmd_glob(pattern: &str, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let results = engine.glob_files(pattern); + + if cli.json { + return print_agent_result(json!({ + "pattern": pattern, + "count": results.len(), + "paths": results, + })); + } + + if results.is_empty() { + println!("No files match '{}'", pattern); + } else { + println!("{} files match '{}':", results.len(), pattern); + for path in &results { + println!(" {}", path); + } + } + + Ok(()) +} + +pub(crate) fn cmd_ls(path: &str, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let entries = engine.list_dir(path); + + if cli.json { + return print_agent_result(json!({ + "path": path, + "count": entries.len(), + "entries": entries.into_iter().map(|(name, meta)| { + if let Some(meta) = meta { + json!({ + "name": name, + "kind": "file", + "language": meta.language.as_str(), + "line_count": meta.line_count, + "byte_size": meta.byte_size, + "symbol_count": meta.symbol_count, + "modified_ms": meta.modified_ms, + "modified_utc": format_unix_ms_utc(meta.modified_ms), + }) + } else { + json!({"name": name, "kind": "directory"}) + } + }).collect::>() + })); + } + + if entries.is_empty() { + println!("No files in '{}'", path); + } else { + for (name, meta) in &entries { + if let Some(m) = meta { + println!( + "{:<60} {:>8} {:>6}L {:>4} sym", + name, + m.language.as_str(), + m.line_count, + m.symbol_count + ); + } else { + println!("{}/", name); + } + } + } + + Ok(()) +} + +pub(crate) fn cmd_status(cli: &Cli) -> Result<()> { + let loaded = load_engine_for_root(¤t_root()?, cli)?; + let engine = loaded.engine; + let snap_path = graph_path(cli)?; + let graph = if snap_path.exists() { + let metadata = std::fs::metadata(&snap_path)?; + json!({ + "path": snap_path.display().to_string(), + "exists": true, + "size_bytes": metadata.len(), + "size_mb": metadata.len() as f64 / (1024.0 * 1024.0), + }) + } else { + json!({"path": snap_path.display().to_string(), "exists": false}) + }; + + if cli.json { + return print_agent_result(json!({ + "files_indexed": engine.file_count(), + "symbols_indexed": engine.symbol_index_count(), + "unique_words_indexed": engine.word_index_count(), + "word_indexed_files": engine.word_index_file_count(), + "seq": engine.store().current_seq(), + "change_history_persisted": false, + "graph": graph, + "refresh": { + "indexed": loaded.refresh.indexed, + "removed": loaded.refresh.removed, + "changed": loaded.refresh.changed(), + }, + })); + } + + println!("lexa status:"); + println!(" Files indexed: {}", engine.file_count()); + println!(" Symbols indexed: {}", engine.symbol_index_count()); + println!(" Unique words indexed: {}", engine.word_index_count()); + println!(" Word-indexed files: {}", engine.word_index_file_count()); + println!( + " Current sequence: {} (session-local)", + engine.store().current_seq() + ); + println!(" Change history persisted: false"); + + if snap_path.exists() { + let metadata = std::fs::metadata(&snap_path)?; + println!( + " Graph: {} ({:.1} MB)", + snap_path.display(), + metadata.len() as f64 / (1024.0 * 1024.0) + ); + } else { + println!(" Graph: not found"); + } + + Ok(()) +} + +pub(crate) fn cmd_audit( + max: Option, + since: Option<&str>, + strict: bool, + config_path: Option<&PathBuf>, + no_config: bool, + include: &[AuditInclude], + cli: &Cli, +) -> Result<()> { + let mut engine = load_engine(cli)?; + if engine.file_count() == 0 { + bail!("no files indexed; run 'lexa index .' before running audit"); + } + let root = std::env::current_dir()?; + let config = audit::load_audit_config(&root, config_path.map(PathBuf::as_path), no_config)?; + let scope = if let Some(base) = since { + audit::AuditScope::GitSince { + base: base.to_string(), + changed_files: audit::changed_files_since(&root, base)?, + } + } else { + audit::AuditScope::Project + }; + let report = + ProjectSession::new(&mut engine, &root, Path::new(""), false).audit(audit::AuditOptions { + max_results: max, + scope, + config, + includes: audit_includes(include), + }); + + if cli.json { + print_agent_result(json!(report))?; + } else { + print!("{}", audit::render_audit_report(&report)); + } + + if strict && report.summary.high > 0 { + std::io::stdout().flush()?; + std::process::exit(1); + } + + Ok(()) +} + +pub(crate) fn audit_includes(values: &[AuditInclude]) -> audit::AuditIncludes { + audit::AuditIncludes { + dead_code: values.contains(&AuditInclude::DeadCode), + } +} + +pub(crate) fn cmd_watch(path: &str, debounce_ms: u64, cli: &Cli) -> Result<()> { + use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; + use std::sync::mpsc::channel; + use std::time::Duration; + + let watch_path = std::fs::canonicalize(path)?; + println!("Watching {} for changes...", watch_path.display()); + println!("Press Ctrl+C to stop"); + + let (tx, rx) = channel(); + + let mut watcher = RecommendedWatcher::new( + tx, + notify::Config::default().with_poll_interval(Duration::from_millis(debounce_ms)), + )?; + + watcher.watch(&watch_path, RecursiveMode::Recursive)?; + + let mut engine = engine::Engine::new(16384); + let snap_path = graph_path_for_root(&watch_path, cli); + if !cli.no_graph { + if snap_path.exists() { + match snapshot::load_snapshot_into_engine(&mut engine, &snap_path) { + Ok(count) => eprintln!("Loaded {} files from graph", count), + Err(err) => bail!( + "failed to load graph {}: {err}. Run 'lexa reindex {}' to rebuild it or 'lexa clear-index' to remove it.", + snap_path.display(), + watch_path.display() + ), + } + } else { + bail!( + "no graph file found at {}. Run 'lexa index {}' first.", + snap_path.display(), + watch_path.display() + ); + } + } + + loop { + match rx.recv() { + Ok(Ok(event)) => { + let Event { kind, paths, .. } = event; + let should_reindex = matches!( + kind, + EventKind::Create(_) | EventKind::Modify(_) | EventKind::Remove(_) + ); + + if should_reindex { + for path in &paths { + if let Ok(relative) = path.strip_prefix(&watch_path) { + let relative_str = project_path_string(relative); + match kind { + EventKind::Create(_) | EventKind::Modify(_) => { + if let Ok(content) = std::fs::read_to_string(path) { + engine.index_file(&relative_str, &content); + println!("Updated: {}", relative_str); + } + } + EventKind::Remove(_) => { + engine.remove_file(&relative_str); + println!("Removed: {}", relative_str); + } + _ => {} + } + } + } + + if !cli.no_graph { + if let Err(e) = snapshot::write_snapshot(&engine, &snap_path) { + eprintln!("Warning: Failed to save graph: {}", e); + } + } + } + } + Ok(Err(e)) => { + eprintln!("Watch error: {}", e); + } + Err(e) => { + eprintln!("Channel error: {}", e); + break; + } + } + } + + Ok(()) +} + +fn project_path_string(path: &Path) -> String { + path.components() + .filter_map(|component| match component { + std::path::Component::Normal(value) => Some(value.to_string_lossy()), + _ => None, + }) + .collect::>() + .join("/") +} + +pub(crate) fn cmd_pipeline(pipeline: &[String], cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let pipeline_str = pipeline.join(" "); + let output = pipeline::run_output(&engine, &pipeline_str); + let text = output.render(); + if cli.json { + return print_agent_result(output.to_json(&pipeline_str)); + } + println!("{}", text); + Ok(()) +} diff --git a/src/commands/mod.rs b/src/commands/mod.rs new file mode 100644 index 0000000..f0ecaeb --- /dev/null +++ b/src/commands/mod.rs @@ -0,0 +1,255 @@ +mod graph; +mod maintenance; +mod mutation; +mod retrieval; +mod shared; + +pub(crate) use graph::*; +pub(crate) use maintenance::*; +pub(crate) use mutation::*; +pub(crate) use retrieval::*; +pub(crate) use shared::*; + +use anyhow::Result; +use clap::CommandFactory; +use lexa::engine::{ContextOptions, FileFilterOptions, SearchOptions}; + +use crate::cli::{Cli, Commands}; +use crate::cli_upgrade; + +pub(crate) fn run(cli: &Cli) -> Result<()> { + if cli.version { + return cli_upgrade::cmd_version(false); + } + + let Some(command) = &cli.command else { + Cli::command().print_help()?; + println!(); + return Ok(()); + }; + + match command { + Commands::Index { path, output } => cmd_index(path, output.as_ref(), cli), + Commands::Reindex { path } => cmd_reindex(path, cli), + Commands::ClearIndex => cmd_clear_index(cli), + Commands::Files { + path, + path_glob, + language, + min_lines, + max_lines, + max_results, + } => cmd_tree( + FileFilterOptions { + path_prefix: (!path.is_empty()).then(|| path.clone()), + path_glob: path_glob.clone(), + language: language.clone(), + min_lines: *min_lines, + max_lines: *max_lines, + max_results: *max_results, + }, + cli, + ), + Commands::List { path } => cmd_ls(path, cli), + Commands::PathSearch { + pattern, + query, + max, + max_results, + } => cmd_find( + &required_text(pattern.as_deref(), query.as_deref(), "path-search", "query")?, + max_limit(*max, *max_results, 20)?, + cli, + ), + Commands::TextSearch { + query, + query_flag, + max, + max_results, + regex, + scope, + compact, + paths_only, + path_glob, + } => cmd_search( + &required_text( + query.as_deref(), + query_flag.as_deref(), + "text-search", + "query", + )?, + SearchOptions { + max_results: max_limit(*max, *max_results, 20)?, + regex: *regex, + scope: *scope, + compact: *compact, + paths_only: *paths_only, + path_glob: path_glob.clone(), + }, + cli, + ), + Commands::Outline { path } => cmd_outline(path, cli), + Commands::SymbolDefs { name } => cmd_symbol(name, cli), + Commands::SymbolSearch { + query, + query_flag, + max, + max_results, + } => cmd_symbol_search( + &required_text( + query.as_deref(), + query_flag.as_deref(), + "symbol-search", + "query", + )?, + max_limit(*max, *max_results, 20)?, + cli, + ), + Commands::WordRefs { + word, + max, + max_results, + cursor, + path_prefix, + path, + path_glob, + } => cmd_word( + word, + max_limit(*max, *max_results, 50)?, + *cursor, + path_prefix.as_deref().or(path.as_deref()), + path_glob.as_deref(), + cli, + ), + Commands::Deps { + path, + reverse, + transitive, + } => cmd_deps(path, *reverse, *transitive, cli), + Commands::Recent { limit } => cmd_hot(*limit, cli), + Commands::Callers { + name, + query, + max, + max_results, + } => cmd_callers( + &required_text(name.as_deref(), query.as_deref(), "callers", "name")?, + max_limit(*max, *max_results, 20)?, + cli, + ), + Commands::Brief { + task, + query, + max, + max_results, + path_prefix, + path_glob, + language, + } => cmd_context( + &required_text(task.as_deref(), query.as_deref(), "brief", "task")?, + ContextOptions { + max_results: max_limit(*max, *max_results, 10)?, + path_prefix: path_prefix.clone(), + path_glob: path_glob.clone(), + language: language.clone(), + }, + cli, + ), + Commands::Changes { since } => cmd_changes(*since, cli), + Commands::Read { + path, + line_range, + line_start, + line_end, + compact, + if_hash, + hash, + } => { + let (line_start, line_end) = + resolve_line_range(line_range.as_deref(), *line_start, *line_end)?; + cmd_read( + path, + line_start, + line_end, + *compact, + if_hash.as_deref(), + *hash, + cli, + ) + } + Commands::Patch { + path, + op, + line_range, + after, + replace_text, + anchor, + placement, + preview, + content, + content_file, + if_hash, + dry_run, + } => cmd_edit( + path, + *op, + line_range.as_deref(), + *after, + replace_text.as_deref(), + anchor.as_deref(), + *placement, + *preview, + content.as_deref(), + content_file.as_ref(), + if_hash.as_deref(), + *dry_run, + cli, + ), + Commands::Create { + path, + content, + content_file, + overwrite, + dry_run, + } => cmd_create( + path, + content.as_deref(), + content_file.as_ref(), + *overwrite, + *dry_run, + cli, + ), + Commands::Glob { pattern } => cmd_glob(pattern, cli), + Commands::Status => cmd_status(cli), + Commands::Audit { + max, + since, + strict, + config, + no_config, + include, + } => cmd_audit( + *max, + since.as_deref(), + *strict, + config.as_ref(), + *no_config, + include, + cli, + ), + Commands::Upgrade { + version, + install_dir, + } => cli_upgrade::cmd_upgrade(version, install_dir.as_ref(), false), + Commands::Watch { path, debounce } => cmd_watch(path, *debounce, cli), + Commands::Pipeline { pipeline } => cmd_pipeline(pipeline, cli), + Commands::Mcp { + path, + no_refresh, + debounce, + structured_content: _, + log_file, + } => cmd_mcp(path, *no_refresh, *debounce, log_file.as_ref(), cli), + Commands::DumpTools => cmd_dump_tools(), + } +} diff --git a/src/commands/mutation.rs b/src/commands/mutation.rs new file mode 100644 index 0000000..c785dc1 --- /dev/null +++ b/src/commands/mutation.rs @@ -0,0 +1,280 @@ +use crate::cli::Cli; +use anyhow::{bail, Result}; +use lexa::application::{self, ProjectSession}; +use lexa::edit; +use lexa::engine; +use lexa::project_path::project_target_path; +use serde_json::json; +use std::path::{Path, PathBuf}; + +use super::shared::*; + +pub(crate) fn cmd_read( + path: &str, + line_start: Option, + line_end: Option, + compact: bool, + if_hash: Option<&str>, + show_hash: bool, + cli: &Cli, +) -> Result<()> { + let mut engine = load_engine(cli)?; + let root = std::env::current_dir()?; + let operation = ProjectSession::new(&mut engine, &root, Path::new(""), false).read( + application::ReadRequest { + path, + existing_only: false, + line_start, + line_end, + compact, + if_hash, + }, + )?; + let path = operation.path; + if engine.file_count() == 0 && project_target_path(&root, &path).exists() { + bail!("no files indexed; run 'lexa index .' before reading files"); + } + + match operation.file { + Some(result) => { + if cli.json { + return print_agent_result(json!({ + "path": path, + "hash": format!("{:x}", result.hash), + "unchanged": result.unchanged, + "line_start": line_start, + "line_end": line_end, + "compact": compact, + "content": result.content, + })); + } + if result.unchanged { + println!("unchanged:{:x}", result.hash); + return Ok(()); + } + if show_hash || if_hash.is_some() { + println!("hash:{:x}", result.hash); + } + print!("{}", result.content); + } + None => { + if cli.json { + return print_agent_result(json!({"error": "file_not_found", "path": path})); + } + println!("File not found: {}", path); + } + } + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn cmd_edit( + path: &str, + op: Option, + line_range: Option<&str>, + after: Option, + replace_text: Option<&str>, + anchor: Option<&str>, + placement: Option, + preview_mode: edit::PreviewMode, + content: Option<&str>, + content_file: Option<&PathBuf>, + if_hash: Option<&str>, + dry_run: bool, + cli: &Cli, +) -> Result<()> { + let root = current_root()?; + let (range_start, range_end) = if let Some(range) = line_range { + parse_line_range(range)? + } else { + (None, None) + }; + + let edit_content = if let Some(path) = content_file { + Some(std::fs::read_to_string(path)?) + } else { + content.map(ToString::to_string) + }; + + let (mut engine, snap_path) = if !dry_run && !cli.no_graph { + load_existing_engine_for_root(&root, cli)? + } else { + (engine::Engine::new(16), graph_path_for_root(&root, cli)) + }; + let operation = ProjectSession::new(&mut engine, &root, &snap_path, !cli.no_graph).patch( + application::PatchRequest { + path, + op, + range_start, + range_end, + after, + content: edit_content, + replace_text: replace_text.map(ToString::to_string), + anchor: anchor.map(ToString::to_string), + placement, + preview_mode, + if_hash: if_hash.map(ToString::to_string), + dry_run, + }, + )?; + let rel_path = operation.path; + let result = operation.edit; + let op_label = edit_op_label(op, replace_text, anchor); + + if dry_run { + if cli.json { + return print_agent_result(json!({ + "path": rel_path, + "op": op_label, + "dry_run": true, + "changed": result.changed, + "old_hash": format!("{:x}", result.old_hash), + "new_hash": format!("{:x}", result.new_hash), + "line_count": result.line_count, + "lines_added": result.lines_added, + "lines_removed": result.lines_removed, + "preview_mode": preview_mode_str(preview_mode), + "preview": result.preview, + })); + } + println!("{}", result.preview); + println!("old_hash:{:x}", result.old_hash); + println!("new_hash:{:x}", result.new_hash); + return Ok(()); + } + + if result.changed { + if cli.json { + return print_agent_result(json!({ + "path": rel_path, + "op": op_label, + "dry_run": false, + "changed": true, + "hash": format!("{:x}", result.new_hash), + "line_count": result.line_count, + "lines_added": result.lines_added, + "lines_removed": result.lines_removed, + "graph": (!cli.no_graph).then(|| snap_path.display().to_string()), + "persisted": !cli.no_graph, + "change_sequence": engine.store().current_seq(), + })); + } + println!( + "{}", + format_edit_applied(&rel_path, &result, result.new_hash) + ); + if !cli.no_graph { + println!("Graph saved to {}", snap_path.display()); + } + } else { + if cli.json { + return print_agent_result(json!({ + "path": rel_path, + "op": op_label, + "dry_run": false, + "changed": false, + "hash": format!("{:x}", result.new_hash), + "line_count": result.line_count, + "lines_added": result.lines_added, + "lines_removed": result.lines_removed, + })); + } + println!("edit unchanged: hash:{:x}", result.new_hash); + } + + Ok(()) +} + +pub(crate) fn cmd_create( + path: &str, + content: Option<&str>, + content_file: Option<&PathBuf>, + overwrite: bool, + dry_run: bool, + cli: &Cli, +) -> Result<()> { + let root = current_root()?; + let content = if let Some(path) = content_file { + std::fs::read_to_string(path)? + } else { + content.unwrap_or("").to_string() + }; + + let (mut engine, snap_path) = if !dry_run && !cli.no_graph { + load_existing_engine_for_root(&root, cli)? + } else { + (engine::Engine::new(16), graph_path_for_root(&root, cli)) + }; + let operation = ProjectSession::new(&mut engine, &root, &snap_path, !cli.no_graph).create( + application::CreateRequest { + path, + content, + overwrite, + dry_run, + }, + )?; + let rel_path = operation.path; + let result = operation.create; + let would_create = operation.would_create; + + if cli.json { + let mut payload = json!({ + "path": rel_path, + "op": "create", + "dry_run": dry_run, + "changed": result.changed, + "hash": format!("{:x}", result.hash), + "line_count": result.line_count, + "byte_size": result.byte_size, + }); + if would_create { + payload["would_create"] = json!(true); + } + return print_agent_result(payload); + } + + if dry_run { + println!( + "create dry-run: {} lines, hash:{:x}", + result.line_count, result.hash + ); + } else { + println!( + "file created: {} lines, hash:{:x}", + result.line_count, result.hash + ); + } + + Ok(()) +} + +pub(crate) fn format_edit_applied(path: &str, result: &edit::EditResult, hash: u64) -> String { + if result.lines_added == 0 && result.lines_removed == 0 { + return format!( + "edit applied to {path}: content changed without line-count change ({} total), hash:{hash:x}", + result.line_count + ); + } + + format!( + "edit applied to {path}: +{} -{} lines ({} total), hash:{hash:x}", + result.lines_added, result.lines_removed, result.line_count + ) +} + +pub(crate) fn edit_op_label( + op: Option, + replace_text: Option<&str>, + anchor: Option<&str>, +) -> &'static str { + if replace_text.is_some() { + "replace-text" + } else if anchor.is_some() { + "anchor" + } else if let Some(op) = op { + edit_op_str(op) + } else { + "unknown" + } +} diff --git a/src/commands/retrieval.rs b/src/commands/retrieval.rs new file mode 100644 index 0000000..268fea4 --- /dev/null +++ b/src/commands/retrieval.rs @@ -0,0 +1,567 @@ +use crate::cli::Cli; +use anyhow::Result; +use lexa::engine::{ContextOptions, FileFilterOptions, SearchOptions, WordSearchOptions}; +use lexa::output::{ + format_unix_ms_utc, rich_results_json, word_result_kind_facets, word_result_path_facets, +}; +use lexa::project_path::{normalize_project_path, project_target_path, PathMode}; +use serde_json::json; + +use super::shared::*; + +pub(crate) fn cmd_search(query: &str, options: SearchOptions, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + + let results = match engine.search_rich(query, &options) { + Ok(results) => results, + Err(e) => { + if cli.json { + return print_agent_result(json!({ + "error": "search_failed", + "message": e.to_string(), + })); + } + eprintln!("Error: {}", e); + return Ok(()); + } + }; + + if cli.json { + return print_agent_result(json!({ + "query": query, + "count": results.len(), + "limit": options.max_results, + "regex": options.regex, + "scope": options.scope, + "compact": options.compact, + "paths_only": options.paths_only, + "path_glob": options.path_glob, + "results": rich_results_json(&results), + })); + } + + if results.is_empty() { + println!("No results found for '{}'", query); + return Ok(()); + } + + println!("{} results for '{}':", results.len(), query); + for result in &results { + if options.paths_only { + println!(" {}:{}", result.path, result.line_num); + } else if let Some(scope) = &result.scope { + println!( + " {}:{}: {} [{} {}:{}-{}]", + result.path, + result.line_num, + result.line_text, + scope.kind, + scope.name, + scope.line_start, + scope.line_end + ); + } else { + println!( + " {}:{}: {}", + result.path, result.line_num, result.line_text + ); + } + } + + Ok(()) +} + +pub(crate) fn cmd_outline(path: &str, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let root = std::env::current_dir()?; + let path = match normalize_project_path(&root, path, PathMode::Existing) { + Ok(path) => path, + Err(_) if !project_target_path(&root, path).exists() => { + if cli.json { + return print_agent_result(json!({ + "error": "file_not_found", + "path": path, + "available": engine.list_dir("").into_iter().take(20).map(|(name, _)| name).collect::>(), + })); + } + println!("File not found: {}", path); + println!("Available files:"); + for file in engine.list_dir("").iter().take(20) { + println!(" {}", file.0); + } + return Ok(()); + } + Err(err) => return Err(err), + }; + + match engine.get_outline(&path) { + Some(outline) => { + let unresolved_imports = engine.get_unresolved_imports(&path); + if cli.json { + return print_agent_result(json!({ + "path": path, + "language": outline.language.as_str(), + "line_count": outline.line_count, + "byte_size": outline.byte_size, + "symbol_count": outline.symbols.len(), + "imports": &outline.imports, + "unresolved_imports": unresolved_imports, + "symbols": &outline.symbols, + })); + } + println!( + "{} ({} lines, {} symbols)", + path, + outline.line_count, + outline.symbols.len() + ); + println!("Language: {}", outline.language); + println!(); + + if !outline.imports.is_empty() { + println!("Imports:"); + for import in &outline.imports { + println!(" {}", import); + } + println!(); + } + + if !unresolved_imports.is_empty() { + println!("Unresolved local imports:"); + for import in &unresolved_imports { + let line = import + .line_start + .map(|line| format!("L{line}: ")) + .unwrap_or_default(); + println!(" {}{}", line, import.import); + } + println!(); + } + + if !outline.symbols.is_empty() { + println!("Symbols:"); + for sym in outline + .symbols + .iter() + .filter(|sym| sym.kind != lexa::types::SymbolKind::Import) + { + let detail = sym.detail.as_deref().unwrap_or(""); + let detail_str = if detail.is_empty() { + String::new() + } else { + format!(" {}", detail) + }; + println!( + " L{:<5} {:<12} {}{}", + sym.line_start, sym.kind, sym.name, detail_str + ); + } + } + } + None => { + if cli.json { + return print_agent_result(json!({ + "error": "file_not_found", + "path": path, + "available": engine.list_dir("").into_iter().take(20).map(|(name, _)| name).collect::>(), + })); + } + println!("File not found: {}", path); + println!("Available files:"); + for file in engine.list_dir("").iter().take(20) { + println!(" {}", file.0); + } + } + } + + Ok(()) +} + +pub(crate) fn cmd_symbol_search(query: &str, max: usize, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let results = engine.fuzzy_symbols(query, max); + + if cli.json { + return print_agent_result(json!({ + "query": query, + "count": results.len(), + "limit": max, + "results": results, + })); + } + + if results.is_empty() { + println!("No symbols found matching '{}'", query); + return Ok(()); + } + + println!("{} symbol(s) matching '{}':", results.len(), query); + for result in &results { + let detail = result.detail.as_deref().unwrap_or(""); + let detail_str = if detail.is_empty() { + String::new() + } else { + format!(" {}", detail) + }; + println!( + " {:.2} {}:{}-{} {} {}{}", + result.score, + result.path, + result.line_start, + result.line_end, + result.kind, + result.name, + detail_str + ); + } + + Ok(()) +} + +pub(crate) fn cmd_symbol(name: &str, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let results = engine.find_symbol(name); + + if cli.json { + return print_agent_result( + json!({"name": name, "count": results.len(), "results": results}), + ); + } + + if results.is_empty() { + println!("No symbols found for '{}'", name); + return Ok(()); + } + + println!("{} definition(s) for '{}':", results.len(), name); + for result in &results { + let detail = result.symbol.detail.as_deref().unwrap_or(""); + let detail_str = if detail.is_empty() { + String::new() + } else { + format!(" {}", detail) + }; + println!( + " {}:{}-{} {} {}{}", + result.path, + result.symbol.line_start, + result.symbol.line_end, + result.symbol.kind, + result.symbol.name, + detail_str + ); + } + + Ok(()) +} + +pub(crate) fn cmd_word( + word: &str, + limit: usize, + cursor: usize, + path_prefix: Option<&str>, + path_glob: Option<&str>, + cli: &Cli, +) -> Result<()> { + let engine = load_engine(cli)?; + let limit = limit.max(1); + let options = WordSearchOptions { + path_prefix: path_prefix.map(ToString::to_string), + path_glob: path_glob.map(ToString::to_string), + }; + let all_results = engine.search_word_with_options(word, &options); + let total = all_results.len(); + let start = cursor.min(total); + let end = start.saturating_add(limit).min(total); + let results = all_results[start..end].to_vec(); + let next_cursor = (end < total).then_some(end); + + if cli.json { + return print_agent_result(json!({ + "word": word, + "count": results.len(), + "total": total, + "limit": limit, + "cursor": start, + "truncated": next_cursor.is_some(), + "next_cursor": next_cursor, + "filters": { + "path_prefix": options.path_prefix, + "path_glob": options.path_glob, + }, + "facets": word_result_path_facets(&all_results), + "kind_facets": word_result_kind_facets(&all_results), + "results": results, + })); + } + + if all_results.is_empty() { + println!("No occurrences of '{}'", word); + return Ok(()); + } + + println!( + "{} occurrence(s) of '{}' (showing {} from cursor {}):", + total, + word, + results.len(), + start + ); + for result in &results { + println!( + " {}:{}: {}", + result.path, result.line_num, result.line_text + ); + } + if let Some(next_cursor) = next_cursor { + println!("Next: lexa word-refs {word} --cursor {next_cursor}"); + } + + Ok(()) +} + +pub(crate) fn cmd_find(pattern: &str, max: usize, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let results = engine.fuzzy_find(pattern, max); + + if cli.json { + return print_agent_result(json!({ + "query": pattern, + "count": results.len(), + "limit": max, + "results": results.into_iter().map(|(path, score)| json!({ + "path": path, + "score": score, + })).collect::>() + })); + } + + if results.is_empty() { + println!("No files found matching '{}'", pattern); + return Ok(()); + } + + println!("{} file(s) matching '{}':", results.len(), pattern); + for (path, score) in &results { + println!(" {} (score: {:.1})", path, score); + } + + Ok(()) +} + +pub(crate) fn cmd_tree(filters: FileFilterOptions, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let (files, total, truncated) = engine.filtered_files(&filters); + + if cli.json { + return print_agent_result(json!({ + "count": files.len(), + "total": total, + "truncated": truncated, + "limit": filters.max_results, + "filters": { + "path_prefix": filters.path_prefix, + "path_glob": filters.path_glob, + "language": filters.language, + "min_lines": filters.min_lines, + "max_lines": filters.max_lines, + }, + "files": files.into_iter().map(|(path, meta)| json!({ + "path": path, + "language": meta.language.as_str(), + "line_count": meta.line_count, + "byte_size": meta.byte_size, + "symbol_count": meta.symbol_count, + "modified_ms": meta.modified_ms, + "modified_utc": format_unix_ms_utc(meta.modified_ms), + })).collect::>() + })); + } + + if files.is_empty() { + println!("No indexed files match filters"); + } else { + for (path, meta) in &files { + println!( + "{:<60} {:>8} {:>6}L {:>4} sym", + path, + meta.language.as_str(), + meta.line_count, + meta.symbol_count + ); + } + if truncated { + println!("showing {} of {} matched files", files.len(), total); + } + } + + Ok(()) +} + +pub(crate) fn cmd_deps(path: &str, reverse: bool, transitive: bool, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let root = std::env::current_dir()?; + let path = normalize_project_path(&root, path, PathMode::Existing)?; + + let deps = if transitive { + if reverse { + engine.get_transitive_imported_by(&path) + } else { + engine.get_transitive_depends_on(&path) + } + } else if reverse { + engine.get_imported_by(&path) + } else { + engine.get_depends_on(&path) + }; + let unresolved_imports = if reverse { + Vec::new() + } else { + engine.get_unresolved_imports(&path) + }; + + let label = if reverse { "imported by" } else { "depends on" }; + let transitive_label = if transitive { " (transitive)" } else { "" }; + + if cli.json { + return print_agent_result(json!({ + "path": path, + "direction": if reverse { "imported_by" } else { "depends_on" }, + "transitive": transitive, + "count": deps.len(), + "dependencies": deps, + "unresolved_imports": unresolved_imports, + })); + } + + if deps.is_empty() { + println!("No {} dependencies for {}{}", label, path, transitive_label); + } else { + println!("{} {}{}: ", deps.len(), label, transitive_label); + for dep in &deps { + println!(" {}", dep); + } + } + if !unresolved_imports.is_empty() { + println!("Unresolved local import(s):"); + for import in &unresolved_imports { + let line = import + .line_start + .map(|line| format!("L{line}: ")) + .unwrap_or_default(); + println!(" {}{}", line, import.import); + } + } + + Ok(()) +} + +pub(crate) fn cmd_hot(limit: usize, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let files = engine.get_hot_files(limit); + + if cli.json { + return print_agent_result(json!({ + "count": files.len(), + "limit": limit, + "files": files.into_iter().map(|(path, meta)| json!({ + "path": path, + "language": meta.language.as_str(), + "line_count": meta.line_count, + "byte_size": meta.byte_size, + "symbol_count": meta.symbol_count, + "modified_ms": meta.modified_ms, + "modified_utc": format_unix_ms_utc(meta.modified_ms), + })).collect::>() + })); + } + + if files.is_empty() { + println!("No files indexed"); + return Ok(()); + } + + println!("{} recently modified file(s):", files.len().min(limit)); + for (path, meta) in &files { + println!( + " {} {:>6}L {}", + format_unix_ms_utc(meta.modified_ms), + meta.line_count, + path + ); + } + + Ok(()) +} + +pub(crate) fn cmd_callers(name: &str, max: usize, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let results = engine.find_callers(name, max); + + if cli.json { + return print_agent_result(json!({ + "name": name, + "count": results.len(), + "limit": max, + "results": results, + })); + } + + if results.is_empty() { + println!("No callers found for '{}'", name); + return Ok(()); + } + + println!("{} caller(s) of '{}':", results.len(), name); + for result in &results { + println!( + " {}:{}: {}", + result.path, result.line_num, result.line_text + ); + } + + Ok(()) +} + +pub(crate) fn cmd_context(task: &str, options: ContextOptions, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let details = engine.build_context_details_with_options(task, &options); + if cli.json { + return print_agent_result(json!(details)); + } + let context = engine.build_context_with_options(task, &options); + println!("{}", context); + Ok(()) +} + +pub(crate) fn cmd_changes(since: u64, cli: &Cli) -> Result<()> { + let engine = load_engine(cli)?; + let changes = engine.get_changes(since); + + if cli.json { + return print_agent_result(json!({ + "since": since, + "count": changes.len(), + "change_history_persisted": false, + "note": "Change history is session-local and is not restored from graph snapshots.", + "changes": changes.into_iter().map(|(path, seq, op)| json!({ + "path": path, + "seq": seq, + "op": op, + })).collect::>() + })); + } + + if changes.is_empty() { + println!("No changes since sequence {} in this session", since); + println!("Note: change history is session-local and is not restored from graph snapshots."); + return Ok(()); + } + + println!("{} change(s) since sequence {}:", changes.len(), since); + for (path, seq, op) in &changes { + println!(" {} (seq {}): {}", path, seq, op); + } + + Ok(()) +} diff --git a/src/commands/shared.rs b/src/commands/shared.rs new file mode 100644 index 0000000..3c7cf1f --- /dev/null +++ b/src/commands/shared.rs @@ -0,0 +1,272 @@ +use crate::cli::Cli; +use anyhow::{bail, Context, Result}; +use lexa::engine; +use lexa::output::agent_toon; +use lexa::{edit, freshness, snapshot}; +use std::path::{Path, PathBuf}; + +const DEFAULT_GRAPH_PATH: &str = ".lexa/graph.lexa"; + +pub(crate) fn reject_removed_output_flags() { + if let Some(flag) = removed_output_flag(std::env::args().skip(1)) { + eprintln!("{flag} was removed; Lexa command results are always TOON structured text."); + std::process::exit(2); + } +} + +pub(crate) fn removed_output_flag( + args: impl IntoIterator>, +) -> Option<&'static str> { + for arg in args { + let arg = arg.as_ref(); + if arg == "--" { + break; + } + match arg { + "--json" => return Some("--json"), + "--structured-content" => return Some("--structured-content"), + "--json-output" => return Some("--json-output"), + _ if arg.starts_with("--json=") => return Some("--json"), + _ if arg.starts_with("--structured-content=") => return Some("--structured-content"), + _ if arg.starts_with("--json-output=") => return Some("--json-output"), + _ => {} + } + } + None +} + +pub(crate) fn current_root() -> Result { + std::fs::canonicalize(std::env::current_dir()?) + .context("failed to canonicalize current directory") +} + +pub(crate) fn graph_path_for_root(root: &std::path::Path, cli: &Cli) -> PathBuf { + cli.graph + .clone() + .unwrap_or_else(|| root.join(DEFAULT_GRAPH_PATH)) +} + +pub(crate) fn graph_path(cli: &Cli) -> Result { + let root = current_root()?; + Ok(graph_path_for_root(&root, cli)) +} + +pub(crate) struct LoadedEngine { + pub(crate) engine: engine::Engine, + pub(crate) refresh: freshness::RefreshSummary, +} + +pub(crate) fn load_engine(cli: &Cli) -> Result { + let root = current_root()?; + Ok(load_engine_for_root(&root, cli)?.engine) +} + +pub(crate) fn load_engine_for_root(root: &Path, cli: &Cli) -> Result { + let mut engine = engine::Engine::new(16384); + let path = graph_path_for_root(root, cli); + let mut loaded_graph = false; + let mut refresh = freshness::RefreshSummary::default(); + + if !cli.no_graph { + if path.exists() { + match snapshot::load_snapshot_into_engine(&mut engine, &path) { + Ok(count) => { + eprintln!("Loaded {} files from graph", count); + loaded_graph = true; + } + Err(e) => { + bail!( + "failed to load graph {}: {e}. Run 'lexa reindex .' to rebuild it or 'lexa clear-index' to remove it.", + path.display() + ); + } + } + } else { + eprintln!( + "No graph file found at {}. Run 'lexa index .' first.", + path.display() + ); + } + } + + if loaded_graph { + refresh = refresh_loaded_graph(&mut engine, root, &path, !cli.no_graph)?; + } + + Ok(LoadedEngine { engine, refresh }) +} + +pub(crate) fn load_existing_engine_for_root( + root: &Path, + cli: &Cli, +) -> Result<(engine::Engine, PathBuf)> { + let snap_path = graph_path_for_root(root, cli); + if !snap_path.exists() { + bail!( + "no graph file found at {}. Run 'lexa index .' first.", + snap_path.display() + ); + } + + let mut engine = engine::Engine::new(16384); + snapshot::load_snapshot_into_engine(&mut engine, &snap_path).with_context(|| { + format!( + "failed to load graph {}. Run 'lexa reindex .' to rebuild it or 'lexa clear-index' to remove it.", + snap_path.display() + ) + })?; + refresh_loaded_graph(&mut engine, root, &snap_path, true)?; + Ok((engine, snap_path)) +} + +pub(crate) fn refresh_loaded_graph( + engine: &mut engine::Engine, + root: &Path, + snap_path: &Path, + persist_graph: bool, +) -> Result { + eprintln!("Checking graph freshness..."); + let refresh = freshness::refresh_project(engine, root) + .with_context(|| format!("failed to refresh graph for {}", root.display()))?; + if refresh.changed() { + eprintln!( + "Refreshed graph: {} indexed, {} removed", + refresh.indexed, refresh.removed + ); + if persist_graph { + snapshot::write_snapshot(engine, snap_path)?; + } + } + Ok(refresh) +} + +pub(crate) fn required_text( + positional: Option<&str>, + flag: Option<&str>, + command: &str, + label: &str, +) -> Result { + match (positional, flag) { + (Some(_), Some(_)) => { + bail!("{command} accepts either positional {label} or --query, not both") + } + (Some(value), None) | (None, Some(value)) => Ok(value.to_string()), + (None, None) => bail!("{command} requires {label}. Example: lexa {command} <{label}>"), + } +} + +pub(crate) fn max_limit( + max: Option, + max_results: Option, + default: usize, +) -> Result { + match (max, max_results) { + (Some(_), Some(_)) => bail!("use either --max or --max-results, not both"), + (Some(value), None) | (None, Some(value)) => Ok(value), + (None, None) => Ok(default), + } +} + +pub(crate) fn resolve_line_range( + line_range: Option<&str>, + line_start: Option, + line_end: Option, +) -> Result<(Option, Option)> { + if line_range.is_some() && (line_start.is_some() || line_end.is_some()) { + bail!("use either --line-range or --line-start/--line-end, not both"); + } + if let Some(range) = line_range { + return parse_line_range(range); + } + Ok((line_start, line_end)) +} + +pub(crate) fn parse_line_range(range: &str) -> Result<(Option, Option)> { + if let Some((start, end)) = range.split_once('-') { + let start = if start.is_empty() { + None + } else { + Some(start.parse::()?) + }; + let end = if end.is_empty() { + None + } else { + Some(end.parse::()?) + }; + Ok((start, end)) + } else { + let line = range.parse::()?; + Ok((Some(line), Some(line))) + } +} + +pub(crate) fn print_agent_result(value: serde_json::Value) -> Result<()> { + println!("{}", agent_toon(¤t_command_tool(), value)?); + Ok(()) +} + +pub(crate) fn current_command_tool() -> String { + let command_names = [ + ("index", "index"), + ("reindex", "reindex"), + ("clear-index", "clear_index"), + ("files", "files"), + ("list", "list"), + ("path-search", "path_search"), + ("text-search", "text_search"), + ("outline", "outline"), + ("symbol-defs", "symbol_defs"), + ("symbol-search", "symbol_search"), + ("word-refs", "word_refs"), + ("trace-deps", "trace_deps"), + ("recent", "recent"), + ("callers", "callers"), + ("brief", "brief"), + ("changes", "changes"), + ("read", "read"), + ("patch", "patch"), + ("create", "create"), + ("glob", "glob"), + ("status", "status"), + ("audit", "audit"), + ("pipeline", "pipeline"), + ("mcp", "mcp"), + ("upgrade", "upgrade"), + ("update", "upgrade"), + ]; + + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + if arg == "--graph" { + let _ = args.next(); + continue; + } + if arg == "--no-graph" + || arg == "--version" + || arg.starts_with("--graph=") + || arg.starts_with('-') + { + continue; + } + if let Some((_, tool)) = command_names.iter().find(|(command, _)| *command == arg) { + return (*tool).to_string(); + } + } + + "result".to_string() +} + +pub(crate) fn edit_op_str(op: edit::EditOp) -> &'static str { + match op { + edit::EditOp::Replace => "replace", + edit::EditOp::Insert => "insert", + edit::EditOp::Delete => "delete", + } +} + +pub(crate) fn preview_mode_str(mode: edit::PreviewMode) -> &'static str { + match mode { + edit::PreviewMode::Compact => "compact", + edit::PreviewMode::Full => "full", + } +} diff --git a/src/edit.rs b/src/edit.rs index 12e7560..60119d8 100644 --- a/src/edit.rs +++ b/src/edit.rs @@ -2,9 +2,12 @@ use anyhow::{anyhow, bail, Context, Result}; use clap::ValueEnum; use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use crate::engine::hash_content; +static TEMP_FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0); + #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] pub enum EditOp { Replace, @@ -146,8 +149,8 @@ fn build_new_content(old_content: &str, req: &EditRequest) -> Result { return insert_at_anchor(old_content, anchor, placement, content); } - let mut lines: Vec = old_content.lines().map(ToString::to_string).collect(); - let had_trailing_newline = old_content.ends_with('\n'); + let lines = line_spans(old_content); + let newline = dominant_newline(old_content); match req .op @@ -156,27 +159,26 @@ fn build_new_content(old_content: &str, req: &EditRequest) -> Result { EditOp::Replace => { let (start, end) = concrete_range(req)?; ensure_range_in_bounds(start, end, lines.len())?; - let replacement = replacement_lines(req)?; - lines.splice(start..end, replacement); + let replacement = required_content(req)?; + replace_line_range(old_content, &lines, start, end, replacement, newline) } EditOp::Insert => { - let replacement = replacement_lines(req)?; + let replacement = required_content(req)?; let after = req.after.unwrap_or(0) as usize; - let insert_at = after.min(lines.len()); - lines.splice(insert_at..insert_at, replacement); + insert_after_line( + old_content, + &lines, + after.min(lines.len()), + replacement, + newline, + ) } EditOp::Delete => { let (start, end) = concrete_range(req)?; ensure_range_in_bounds(start, end, lines.len())?; - lines.drain(start..end); + replace_line_range(old_content, &lines, start, end, "", newline) } } - - let mut result = lines.join("\n"); - if had_trailing_newline && !result.is_empty() { - result.push('\n'); - } - Ok(result) } fn validate_target_shape(req: &EditRequest) -> Result<()> { @@ -294,12 +296,159 @@ fn concrete_range(req: &EditRequest) -> Result<(usize, usize)> { Ok(((start - 1) as usize, end as usize)) } -fn replacement_lines(req: &EditRequest) -> Result> { - let content = req - .content +fn required_content(req: &EditRequest) -> Result<&str> { + req.content .as_deref() - .ok_or_else(|| anyhow!("replace/insert requires --content or --content-file"))?; - Ok(content.lines().map(ToString::to_string).collect()) + .ok_or_else(|| anyhow!("replace/insert requires --content or --content-file")) +} + +#[derive(Clone, Copy)] +struct LineSpan { + start: usize, + content_end: usize, + end: usize, +} + +fn line_spans(content: &str) -> Vec { + let bytes = content.as_bytes(); + let mut spans = Vec::new(); + let mut start = 0; + + for (index, byte) in bytes.iter().enumerate() { + if *byte != b'\n' { + continue; + } + let content_end = if index > start && bytes[index - 1] == b'\r' { + index - 1 + } else { + index + }; + spans.push(LineSpan { + start, + content_end, + end: index + 1, + }); + start = index + 1; + } + + if start < bytes.len() { + spans.push(LineSpan { + start, + content_end: bytes.len(), + end: bytes.len(), + }); + } + + spans +} + +fn dominant_newline(content: &str) -> &'static str { + let bytes = content.as_bytes(); + let mut crlf = 0usize; + let mut lf = 0usize; + let mut first = None; + for (index, byte) in bytes.iter().enumerate() { + if *byte == b'\n' { + if index > 0 && bytes[index - 1] == b'\r' { + crlf += 1; + first.get_or_insert("\r\n"); + } else { + lf += 1; + first.get_or_insert("\n"); + } + } + } + match crlf.cmp(&lf) { + std::cmp::Ordering::Greater => "\r\n", + std::cmp::Ordering::Less => "\n", + std::cmp::Ordering::Equal => first.unwrap_or("\n"), + } +} + +fn normalize_newlines(content: &str, newline: &str) -> String { + let normalized = content.replace("\r\n", "\n").replace('\r', "\n"); + if newline == "\n" { + normalized + } else { + normalized.replace('\n', newline) + } +} + +fn replace_line_range( + old_content: &str, + lines: &[LineSpan], + start: usize, + end: usize, + replacement: &str, + newline: &str, +) -> Result { + ensure_range_in_bounds(start, end, lines.len())?; + let replace_start = lines[start].start; + let replace_end = lines[end - 1].end; + let target_had_ending = lines[end - 1].end > lines[end - 1].content_end; + let suffix = &old_content[replace_end..]; + let mut replacement = normalize_newlines(replacement, newline); + + if !replacement.is_empty() + && !has_line_ending(&replacement) + && (!suffix.is_empty() || target_had_ending) + { + replacement.push_str(newline); + } + + let mut result = String::with_capacity( + old_content.len() - (replace_end - replace_start) + replacement.len(), + ); + result.push_str(&old_content[..replace_start]); + result.push_str(&replacement); + result.push_str(suffix); + Ok(result) +} + +fn insert_after_line( + old_content: &str, + lines: &[LineSpan], + after: usize, + content: &str, + newline: &str, +) -> Result { + if content.is_empty() { + return Ok(old_content.to_string()); + } + + let insert_at = if after == 0 { 0 } else { lines[after - 1].end }; + let prefix = &old_content[..insert_at]; + let suffix = &old_content[insert_at..]; + let insertion = normalize_newlines(content, newline); + let needs_prefix_separator = + !prefix.is_empty() && !has_line_ending(prefix) && !starts_with_line_ending(&insertion); + let needs_suffix_separator = !suffix.is_empty() && !has_line_ending(&insertion); + let preserve_trailing_ending = + suffix.is_empty() && has_line_ending(old_content) && !has_line_ending(&insertion); + + let mut result = String::with_capacity( + old_content.len() + + insertion.len() + + newline.len() * usize::from(needs_prefix_separator || needs_suffix_separator), + ); + result.push_str(prefix); + if needs_prefix_separator { + result.push_str(newline); + } + result.push_str(&insertion); + if needs_suffix_separator || preserve_trailing_ending { + result.push_str(newline); + } + result.push_str(suffix); + Ok(result) +} + +fn has_line_ending(content: &str) -> bool { + content.ends_with('\n') || content.ends_with('\r') +} + +fn starts_with_line_ending(content: &str) -> bool { + content.starts_with('\n') || content.starts_with('\r') } fn atomic_write(path: &Path, content: &str) -> Result<()> { @@ -308,32 +457,69 @@ fn atomic_write(path: &Path, content: &str) -> Result<()> { .file_name() .and_then(|name| name.to_str()) .unwrap_or("file"); - let tmp_path = temp_path(parent, filename, content); - - { - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp_path) - .with_context(|| format!("failed to create {}", tmp_path.display()))?; + let existing_permissions = std::fs::metadata(path) + .ok() + .map(|metadata| metadata.permissions()); + let (tmp_path, mut file) = create_temp_file(parent, filename)?; + + let write_result = (|| -> Result<()> { + if let Some(permissions) = existing_permissions { + file.set_permissions(permissions).with_context(|| { + format!("failed to preserve permissions for {}", path.display()) + })?; + } file.write_all(content.as_bytes()) .with_context(|| format!("failed to write {}", tmp_path.display()))?; - file.sync_all().ok(); + file.sync_all() + .with_context(|| format!("failed to sync {}", tmp_path.display()))?; + Ok(()) + })(); + if let Err(error) = write_result { + let _ = std::fs::remove_file(&tmp_path); + return Err(error); } + drop(file); std::fs::rename(&tmp_path, path).with_context(|| { let _ = std::fs::remove_file(&tmp_path); format!("failed to replace {}", path.display()) })?; + + #[cfg(unix)] + std::fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("failed to sync directory {}", parent.display()))?; + Ok(()) } -fn temp_path(parent: &Path, filename: &str, content: &str) -> PathBuf { - parent.join(format!( - ".{filename}.lexa-edit-{}-{:x}.tmp", - std::process::id(), - hash_content(content) - )) +fn create_temp_file(parent: &Path, filename: &str) -> Result<(PathBuf, std::fs::File)> { + for _ in 0..16 { + let sequence = TEMP_FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let path = parent.join(format!( + ".{filename}.lexa-edit-{}-{nonce:x}-{sequence:x}.tmp", + std::process::id() + )); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(file) => return Ok((path, file)), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", path.display())) + } + } + } + bail!( + "failed to allocate a unique temporary file in {}", + parent.display() + ) } fn build_preview(old_content: &str, new_content: &str, mode: PreviewMode) -> String { @@ -538,6 +724,52 @@ mod tests { assert!(result.preview.contains("+ 2: TWO")); } + #[test] + fn line_edits_preserve_crlf_endings() { + let dir = tempfile::tempdir().unwrap(); + let path = write_file(&dir, "app.rs", "one\r\ntwo\r\nthree\r\n"); + let mut req = request(path.clone(), EditOp::Replace); + req.range_start = Some(2); + req.range_end = Some(2); + req.content = Some("TWO\nextra".to_string()); + + let result = apply_edit(&req).unwrap(); + + assert_eq!(result.new_content, "one\r\nTWO\r\nextra\r\nthree\r\n"); + assert_eq!(std::fs::read(path).unwrap(), result.new_content.as_bytes()); + } + + #[test] + fn line_edits_leave_unaffected_mixed_endings_byte_identical() { + let dir = tempfile::tempdir().unwrap(); + let path = write_file(&dir, "mixed.txt", "one\r\ntwo\nthree\r\n"); + let mut req = request(path, EditOp::Replace); + req.range_start = Some(2); + req.range_end = Some(2); + req.content = Some("TWO".to_string()); + + let result = apply_edit(&req).unwrap(); + + assert_eq!(result.new_content, "one\r\nTWO\r\nthree\r\n"); + } + + #[cfg(unix)] + #[test] + fn atomic_edit_preserves_executable_permissions() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let path = write_file(&dir, "tool.sh", "#!/bin/sh\necho old\n"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + let mut req = request(path.clone(), EditOp::Replace); + req.range_start = Some(2); + req.content = Some("echo new".to_string()); + + apply_edit(&req).unwrap(); + + assert_eq!(path.metadata().unwrap().permissions().mode() & 0o777, 0o755); + } + #[test] fn insert_supports_start_middle_and_after_end_positions() { let dir = tempfile::tempdir().unwrap(); @@ -565,6 +797,20 @@ mod tests { ); } + #[test] + fn inserting_a_newline_after_unterminated_last_line_adds_one_ending() { + let dir = tempfile::tempdir().unwrap(); + let path = write_file(&dir, "notes.txt", "one\ntwo"); + let mut req = request(path, EditOp::Insert); + req.after = Some(99); + req.content = Some("\n".to_string()); + + let result = apply_edit(&req).unwrap(); + + assert_eq!(result.new_content, "one\ntwo\n"); + assert_eq!(result.line_count, 2); + } + #[test] fn delete_removes_line_range() { let dir = tempfile::tempdir().unwrap(); diff --git a/src/engine/context.rs b/src/engine/context.rs new file mode 100644 index 0000000..12f7ea6 --- /dev/null +++ b/src/engine/context.rs @@ -0,0 +1,620 @@ +use crate::glob::match_glob; +use crate::types::*; +use hashbrown::HashSet; + +use super::context_helpers::*; +use super::core::*; +use super::ranking::*; +use super::shared::*; + +impl Engine { + pub fn build_context_with_options(&self, task: &str, options: &ContextOptions) -> String { + let details = self.build_context_details_with_options(task, options); + render_context_details(&details) + } +} + +fn render_context_details(details: &ContextDetails) -> String { + let mut output = String::new(); + output.push_str(&format!("## Context for: {}\n\n", details.task)); + if let Some(note) = &details.note { + output.push_str(&format!("{}\n\n", note)); + } + if !details.suggested_next_steps.is_empty() { + output.push_str("### Suggested Next Steps\n\n"); + for step in &details.suggested_next_steps { + output.push_str(&format!("- {step}\n")); + } + output.push('\n'); + } + + if !details.relevant_symbols.is_empty() { + output.push_str("### Relevant Symbols\n\n"); + for sym in &details.relevant_symbols { + output.push_str(&format!( + "- {} ({}): {}:{}-{}\n", + sym.name, sym.kind, sym.path, sym.line_start, sym.line_end + )); + } + output.push('\n'); + + output.push_str("### Relevant Symbol Bodies\n\n"); + for sym in &details.relevant_symbols { + output.push_str(&format!( + "#### {}:{}-{} {}\n\n", + sym.path, sym.content_line_start, sym.content_line_end, sym.name + )); + output.push_str("```text\n"); + output.push_str(&sym.content); + if !sym.content.ends_with('\n') { + output.push('\n'); + } + output.push_str("```\n\n"); + } + } + + if !details.snippets.is_empty() { + output.push_str("### Relevant Code Snippets\n\n"); + for result in &details.snippets { + output.push_str(&format!( + "{}:{}: {}\n", + result.path, result.line_num, result.line_text + )); + } + } + + output +} + +impl Engine { + pub fn build_context_details(&self, task: &str, max_results: usize) -> ContextDetails { + self.build_context_details_with_options( + task, + &ContextOptions { + max_results, + ..ContextOptions::default() + }, + ) + } + + pub fn build_context_details_with_options( + &self, + task: &str, + options: &ContextOptions, + ) -> ContextDetails { + let keywords = context_keywords(task); + let max_results = options.max_results.max(1); + let allow_test_context = context_allows_test_context(task, options); + let relevant_symbols = + self.ranked_context_symbols(&keywords, options, 5, allow_test_context); + + let mut snippets = self.ranked_context_snippets( + &keywords, + &relevant_symbols, + options, + max_results, + allow_test_context, + ); + let confidence = context_confidence(task, &relevant_symbols, &snippets); + let low_confidence = confidence == "low"; + + ContextDetails { + task: task.to_string(), + keywords, + max_results, + confidence: confidence.to_string(), + note: low_confidence.then(|| { + "Low-confidence brief: this tool bundles context from explicit symbols, path fragments, and scoped keywords; it is not natural-language QA.".to_string() + }), + suggested_next_steps: if low_confidence { + vec![ + "Add --path-prefix or --path-glob to scope the search.".to_string(), + "Run symbol-search for likely symbol names.".to_string(), + "Run text-search for concrete terms from the task.".to_string(), + ] + } else { + Vec::new() + }, + relevant_symbols, + snippets: std::mem::take(&mut snippets), + } + } +} + +impl Engine { + fn ranked_context_symbols( + &self, + keywords: &[String], + options: &ContextOptions, + max_symbols: usize, + allow_test_context: bool, + ) -> Vec { + let mut scored: Vec = Vec::new(); + let mut seen: HashSet<(String, String, SymbolKind, u32)> = HashSet::new(); + + for keyword in keywords { + for result in self.find_symbol(keyword) { + if !self.context_path_allowed(&result.path, options) { + continue; + } + push_context_symbol_candidate( + &mut scored, + &mut seen, + self.context_symbol_score(keyword, &result, keywords, allow_test_context) + + CONTEXT_INDEXED_SYMBOL_SOURCE_BONUS, + result, + ); + } + + for (path, outline) in &self.outlines { + if !self.context_path_allowed(path, options) { + continue; + } + for symbol in &outline.symbols { + if !symbol.name.eq_ignore_ascii_case(keyword) { + continue; + } + push_context_symbol_candidate( + &mut scored, + &mut seen, + self.context_symbol_score( + keyword, + &SymbolResult { + path: path.clone(), + symbol: symbol.clone(), + }, + keywords, + allow_test_context, + ) + CONTEXT_OUTLINE_SYMBOL_SOURCE_BONUS, + SymbolResult { + path: path.clone(), + symbol: symbol.clone(), + }, + ); + } + } + + if keyword.len() < 3 { + continue; + } + + for (path, path_score) in self.fuzzy_find(keyword, 5) { + if !self.context_path_allowed(&path, options) { + continue; + } + let Some(outline) = self.outlines.get(&path) else { + continue; + }; + for symbol in &outline.symbols { + if symbol.kind == SymbolKind::Import { + continue; + } + let result = SymbolResult { + path: path.clone(), + symbol: symbol.clone(), + }; + let score = + self.context_symbol_score(keyword, &result, keywords, allow_test_context) + + path_score.round() as i32; + if score < 80 { + continue; + } + push_context_symbol_candidate(&mut scored, &mut seen, score, result); + } + } + } + + let core_terms = context_core_terms(keywords); + if !core_terms.is_empty() { + for (path, outline) in &self.outlines { + if !self.context_path_allowed(path, options) { + continue; + } + for symbol in &outline.symbols { + if symbol.kind == SymbolKind::Import { + continue; + } + let result = SymbolResult { + path: path.clone(), + symbol: symbol.clone(), + }; + let score = self.context_multi_term_symbol_score( + &result, + &core_terms, + allow_test_context, + ); + if score >= 260 { + push_context_symbol_candidate(&mut scored, &mut seen, score, result); + } + } + } + } + + suppress_test_context_symbols(&mut scored, allow_test_context); + scored.sort_by(|a, b| { + b.score + .cmp(&a.score) + .then_with(|| { + context_path_rank(&a.result.path).cmp(&context_path_rank(&b.result.path)) + }) + .then_with(|| a.result.path.cmp(&b.result.path)) + .then_with(|| a.result.symbol.line_start.cmp(&b.result.symbol.line_start)) + .then_with(|| a.result.symbol.name.cmp(&b.result.symbol.name)) + }); + + scored + .into_iter() + .filter_map(|entry| { + self.context_symbol_from_result(entry.result, MAX_CONTEXT_SYMBOL_LINES) + }) + .take(max_symbols) + .collect() + } + + fn context_symbol_from_result( + &self, + result: SymbolResult, + max_lines: u32, + ) -> Option { + let (content_line_start, content_line_end, content) = + self.symbol_source_bounded(&result.path, &result.symbol, max_lines)?; + Some(ContextSymbol { + path: result.path, + name: result.symbol.name, + kind: result.symbol.kind.to_string(), + line_start: result.symbol.line_start, + line_end: result.symbol.line_end, + detail: result.symbol.detail, + content_line_start, + content_line_end, + content, + }) + } +} + +impl Engine { + fn context_symbol_score( + &self, + keyword: &str, + result: &SymbolResult, + keywords: &[String], + allow_test_context: bool, + ) -> i32 { + let symbol_name = &result.symbol.name; + let symbol_norm = context_normalize(symbol_name); + let keyword_norm = context_normalize(keyword); + let path_norm = context_normalize(&result.path); + let basename_norm = result + .path + .rsplit('/') + .next() + .map(|name| name.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(name)) + .map(context_normalize) + .unwrap_or_default(); + let callable = matches!( + result.symbol.kind, + SymbolKind::Function | SymbolKind::Method + ); + let mut score = symbol_kind_context_score(result.symbol.kind); + score += context_path_score(&result.path, allow_test_context); + + if symbol_name == keyword { + score += CONTEXT_EXACT_SYMBOL_BONUS; + } else if has_identifier_case_signal(keyword) && symbol_name.eq_ignore_ascii_case(keyword) { + score += CONTEXT_CASE_INSENSITIVE_SYMBOL_BONUS; + } + + if !keyword_norm.is_empty() { + if symbol_norm == keyword_norm { + score += CONTEXT_NORMALIZED_EXACT_BONUS; + } else if symbol_norm.contains(&keyword_norm) { + score += CONTEXT_NORMALIZED_CONTAINS_BONUS; + } else if keyword_norm.len() >= 5 && keyword_norm.contains(&symbol_norm) { + score += CONTEXT_REVERSE_CONTAINS_BONUS; + } + + if callable + && symbol_norm.ends_with(&keyword_norm) + && symbol_norm.len() > keyword_norm.len() + { + score += CONTEXT_CALLABLE_SUFFIX_BONUS; + } + + if path_norm.contains(&keyword_norm) { + score += CONTEXT_PATH_KEYWORD_BONUS; + } + } + + if callable && !basename_norm.is_empty() && symbol_norm == basename_norm { + score += CONTEXT_BASENAME_CALLABLE_BONUS; + } + + if !callable + && !symbol_norm.is_empty() + && self.outlines.get(&result.path).is_some_and(|outline| { + outline.symbols.iter().any(|symbol| { + matches!(symbol.kind, SymbolKind::Function | SymbolKind::Method) + && context_normalize(&symbol.name).ends_with(&symbol_norm) + && context_normalize(&symbol.name).len() > symbol_norm.len() + }) + }) + { + score -= CONTEXT_NONCALLABLE_SHADOW_PENALTY; + } + + let mut matched_context_terms = 0; + for term in context_terms(keywords) { + if term.len() < 3 { + continue; + } + if symbol_norm.contains(&term) { + matched_context_terms += 1; + score += CONTEXT_SYMBOL_TERM_BONUS; + } + if path_norm.contains(&term) { + score += CONTEXT_PATH_TERM_BONUS; + } + } + + if callable && matched_context_terms >= 2 { + score += CONTEXT_MULTI_TERM_CALLABLE_BONUS; + if symbol_norm.starts_with("create") || symbol_norm.starts_with("use") { + score += CONTEXT_ACTION_NAME_BONUS; + } + } + + let core_terms = context_core_terms(keywords); + if !core_terms.is_empty() { + let symbol_core_matches = core_terms + .iter() + .filter(|term| symbol_norm.contains(term.as_str())) + .count(); + let path_core_matches = core_terms + .iter() + .filter(|term| path_norm.contains(term.as_str())) + .count(); + if symbol_core_matches == 0 { + score -= CONTEXT_NO_CORE_SYMBOL_PENALTY; + } else if symbol_core_matches == 1 && path_core_matches == 0 && core_terms.len() >= 3 { + score -= CONTEXT_WEAK_CORE_MATCH_PENALTY; + } else if symbol_core_matches >= 2 { + score += CONTEXT_STRONG_CORE_MATCH_BONUS; + } else if path_core_matches >= 1 { + score += CONTEXT_PATH_CORE_MATCH_BONUS; + } + if symbol_core_matches == 0 && path_core_matches < 2 { + score -= CONTEXT_POOR_CORE_PATH_PENALTY; + } else { + score += (symbol_core_matches as i32 * CONTEXT_SYMBOL_CORE_TERM_BONUS) + + (path_core_matches as i32 * CONTEXT_PATH_CORE_TERM_BONUS); + } + score += context_action_term_score(&symbol_norm, &path_norm, callable, &core_terms); + } + + if is_test_like_path(&result.path) && !allow_test_context { + score -= CONTEXT_TEST_PATH_PENALTY; + } + + score + } + + fn context_multi_term_symbol_score( + &self, + result: &SymbolResult, + terms: &[String], + allow_test_context: bool, + ) -> i32 { + let symbol_norm = context_normalize(&result.symbol.name); + let path_norm = context_normalize(&result.path); + let mut matched_symbol_terms = 0; + let mut matched_path_terms = 0; + let mut score = symbol_kind_context_score(result.symbol.kind); + score += context_path_score(&result.path, allow_test_context); + + for term in terms { + if symbol_norm.contains(term) { + matched_symbol_terms += 1; + score += CONTEXT_MULTI_TERM_SYMBOL_BONUS; + } + if path_norm.contains(term) { + matched_path_terms += 1; + score += CONTEXT_MULTI_TERM_PATH_BONUS; + } + } + + if matched_symbol_terms == 0 && matched_path_terms < 2 { + return 0; + } + if matched_symbol_terms + matched_path_terms < 2 { + return 0; + } + let is_callable = matches!( + result.symbol.kind, + SymbolKind::Function | SymbolKind::Method + ); + if is_callable { + score += CONTEXT_MULTI_TERM_CALLABLE_KIND_BONUS; + } + if symbol_norm.starts_with("create") || symbol_norm.starts_with("run") { + score += CONTEXT_MULTI_TERM_ACTION_BONUS; + } + if symbol_norm.contains("runtime") { + score += CONTEXT_MULTI_TERM_RUNTIME_BONUS; + } + score += context_action_term_score(&symbol_norm, &path_norm, is_callable, terms); + if is_test_like_path(&result.path) && !allow_test_context { + score -= CONTEXT_TEST_PATH_PENALTY; + } + score + } +} + +impl Engine { + fn ranked_context_snippets( + &self, + keywords: &[String], + relevant_symbols: &[ContextSymbol], + options: &ContextOptions, + max_results: usize, + allow_test_context: bool, + ) -> Vec { + let mut scored: Vec = Vec::new(); + for (rank, symbol) in relevant_symbols.iter().enumerate() { + if !self.context_path_allowed(&symbol.path, options) { + continue; + } + if scored + .iter() + .any(|x| x.result.path == symbol.path && x.result.line_num == symbol.line_start) + { + continue; + } + let Some(line_text) = self.get_line(&symbol.path, symbol.line_start) else { + continue; + }; + let rank_penalty = (rank as i32) * CONTEXT_SNIPPET_SYMBOL_RANK_STEP; + scored.push(ScoredSearchResult { + score: CONTEXT_SNIPPET_SYMBOL_DEFINITION_BONUS.saturating_sub(rank_penalty), + result: SearchResult { + path: symbol.path.clone(), + line_num: symbol.line_start, + line_text, + }, + }); + } + + for keyword in keywords { + let per_keyword_limit = if allow_test_context { 8 } else { 24 }; + let results = self.search(keyword, per_keyword_limit); + for result in results { + if !self.context_path_allowed(&result.path, options) { + continue; + } + if scored + .iter() + .any(|x| x.result.path == result.path && x.result.line_num == result.line_num) + { + continue; + } + let score = self.context_snippet_score( + keyword, + &result, + relevant_symbols, + allow_test_context, + ); + scored.push(ScoredSearchResult { score, result }); + } + } + suppress_test_context_snippets(&mut scored, allow_test_context); + scored.sort_by(|a, b| { + b.score + .cmp(&a.score) + .then_with(|| { + context_path_rank(&a.result.path).cmp(&context_path_rank(&b.result.path)) + }) + .then_with(|| a.result.path.cmp(&b.result.path)) + .then_with(|| a.result.line_num.cmp(&b.result.line_num)) + }); + scored + .into_iter() + .take(max_results) + .map(|entry| entry.result) + .collect() + } + + fn context_path_allowed(&self, path: &str, options: &ContextOptions) -> bool { + if let Some(prefix) = options + .path_prefix + .as_deref() + .filter(|prefix| !prefix.is_empty()) + .map(normalize_filter_prefix) + { + if path != prefix && !path.starts_with(&format!("{prefix}/")) { + return false; + } + } + if let Some(glob) = options.path_glob.as_deref() { + if !match_glob(glob, path) { + return false; + } + } + if let Some(language) = options.language.as_deref() { + let language = language.to_lowercase(); + if self + .file_meta + .get(path) + .is_none_or(|meta| meta.language.as_str() != language) + { + return false; + } + } + true + } +} + +impl Engine { + fn context_snippet_score( + &self, + keyword: &str, + result: &SearchResult, + relevant_symbols: &[ContextSymbol], + allow_test_context: bool, + ) -> i32 { + let keyword_lower = keyword.to_lowercase(); + let path_lower = result.path.to_lowercase(); + let line_lower = result.line_text.to_lowercase(); + let mut score = 0; + if is_source_context_path(&result.path) { + score += CONTEXT_SNIPPET_SOURCE_PATH_BONUS; + } else if is_doc_path(&result.path) { + score -= CONTEXT_SNIPPET_DOC_PATH_PENALTY; + } else if is_example_context_path(&result.path) { + score -= CONTEXT_SNIPPET_EXAMPLE_PATH_PENALTY; + } + if is_test_like_path(&result.path) && !allow_test_context { + score -= CONTEXT_SNIPPET_TEST_PATH_PENALTY; + } + + if line_lower.contains(&keyword_lower) { + score += CONTEXT_SNIPPET_LINE_MATCH_BONUS; + } + if result + .line_text + .split(|c: char| !c.is_alphanumeric() && c != '_' && c != '-') + .any(|word| word.eq_ignore_ascii_case(keyword)) + { + score += CONTEXT_SNIPPET_WORD_MATCH_BONUS; + } + if path_lower.contains(&keyword_lower) { + score += CONTEXT_SNIPPET_PATH_MATCH_BONUS; + } + if relevant_symbols + .iter() + .any(|symbol| symbol.path == result.path || symbol.name.eq_ignore_ascii_case(keyword)) + { + score += CONTEXT_SNIPPET_RELEVANT_SYMBOL_BONUS; + } + if relevant_symbols + .first() + .is_some_and(|symbol| symbol.path == result.path) + { + score += CONTEXT_SNIPPET_TOP_SYMBOL_FILE_BONUS; + } + + let language = self + .file_meta + .get(&result.path) + .map(|meta| meta.language) + .unwrap_or_else(|| detect_language(&result.path)); + if is_comment_or_blank(&result.line_text, language) { + score -= CONTEXT_SNIPPET_COMMENT_PENALTY; + } + if is_import_line(&result.line_text) { + score -= CONTEXT_SNIPPET_IMPORT_PENALTY; + } + if keyword.len() <= 3 { + score -= CONTEXT_SNIPPET_SHORT_KEYWORD_PENALTY; + } + score + } +} diff --git a/src/engine/context_helpers.rs b/src/engine/context_helpers.rs new file mode 100644 index 0000000..8563d13 --- /dev/null +++ b/src/engine/context_helpers.rs @@ -0,0 +1,471 @@ +use crate::types::*; +use hashbrown::HashSet; + +use super::core::{ContextOptions, ContextSymbol}; +use super::ranking::*; +use super::shared::*; + +pub(super) struct ScoredSearchResult { + pub(super) score: i32, + pub(super) result: SearchResult, +} + +pub(super) struct ScoredContextSymbol { + pub(super) score: i32, + pub(super) result: SymbolResult, +} + +pub(super) fn push_context_symbol_candidate( + scored: &mut Vec, + seen: &mut HashSet<(String, String, SymbolKind, u32)>, + score: i32, + result: SymbolResult, +) { + if score <= 0 { + return; + } + let key = ( + result.path.clone(), + result.symbol.name.clone(), + result.symbol.kind, + result.symbol.line_start, + ); + if let Some(existing) = scored.iter_mut().find(|existing| { + existing.result.path == key.0 + && existing.result.symbol.name == key.1 + && existing.result.symbol.kind == key.2 + && existing.result.symbol.line_start == key.3 + }) { + if score > existing.score { + existing.score = score; + existing.result = result; + } + return; + } + if seen.insert(key) { + scored.push(ScoredContextSymbol { score, result }); + } +} + +pub(super) fn suppress_test_context_symbols( + scored: &mut Vec, + allow_test_context: bool, +) { + if allow_test_context + || !scored + .iter() + .any(|entry| !is_test_like_path(&entry.result.path)) + { + return; + } + scored.retain(|entry| !is_test_like_path(&entry.result.path)); +} + +pub(super) fn suppress_test_context_snippets( + scored: &mut Vec, + allow_test_context: bool, +) { + if allow_test_context + || !scored + .iter() + .any(|entry| !is_test_like_path(&entry.result.path)) + { + return; + } + scored.retain(|entry| !is_test_like_path(&entry.result.path)); +} + +pub(super) fn context_keywords(task: &str) -> Vec { + let mut keywords = Vec::new(); + let mut seen = HashSet::new(); + + for quoted in quoted_segments(task) { + add_context_keyword_variants(&mut keywords, &mut seen, "ed); + } + + let tokens = context_tokens(task); + for token in &tokens { + if is_identifier_like_context_token(token) { + add_context_keyword_variants(&mut keywords, &mut seen, token); + } + } + + for window_size in 2..=3 { + for window in tokens.windows(window_size) { + if window + .iter() + .all(|token| context_normalize(token).len() >= 3) + { + add_context_phrase_variants(&mut keywords, &mut seen, window); + } + } + } + + for token in &tokens { + if is_context_content_token(token) { + add_context_keyword_variants(&mut keywords, &mut seen, token); + } + } + + if keywords.is_empty() { + for token in &tokens { + if context_normalize(token).len() >= 3 { + add_context_keyword_variants(&mut keywords, &mut seen, token); + } + } + } + + keywords +} + +pub(super) fn context_query_is_explicit(task: &str) -> bool { + !quoted_segments(task).is_empty() + || context_tokens(task).into_iter().any(|token| { + token.contains(['_', '-', '/', '.', ':']) + || has_lower_to_upper_transition(&token) + || token.chars().any(|ch| ch.is_ascii_digit()) + }) +} + +pub(super) fn context_confidence( + task: &str, + relevant_symbols: &[ContextSymbol], + snippets: &[SearchResult], +) -> &'static str { + let has_source_symbol = relevant_symbols + .iter() + .any(|symbol| is_source_context_path(&symbol.path)); + let has_source_snippet = snippets + .iter() + .any(|snippet| is_source_context_path(&snippet.path)); + + if context_query_is_explicit(task) && has_source_symbol { + "high" + } else if has_source_symbol || has_source_snippet { + "medium" + } else { + "low" + } +} + +pub(super) fn context_allows_test_context(task: &str, options: &ContextOptions) -> bool { + context_task_mentions_test_context(task) + || options + .path_prefix + .as_deref() + .is_some_and(context_filter_targets_test_context) + || options + .path_glob + .as_deref() + .is_some_and(context_filter_targets_test_context) +} + +pub(super) fn context_task_mentions_test_context(task: &str) -> bool { + context_tokens(task).into_iter().any(|token| { + matches!( + context_normalize(&token).as_str(), + "test" | "tests" | "testing" | "spec" | "specs" + ) + }) +} + +pub(super) fn context_filter_targets_test_context(value: &str) -> bool { + let lowered = value.to_ascii_lowercase(); + let normalized = lowered.trim_matches('/'); + is_test_like_path(normalized) + || normalized.split('/').any(|segment| { + let segment = + segment.trim_matches(|ch| matches!(ch, '*' | '?' | '[' | ']' | '{' | '}')); + matches!( + segment, + "test" | "tests" | "__tests__" | "spec" | "specs" | "__specs__" + ) + }) +} + +pub(super) fn is_context_content_token(token: &str) -> bool { + let normalized = context_normalize(token); + normalized.len() >= 3 && !is_low_signal_context_term(&normalized) +} + +pub(super) fn quoted_segments(text: &str) -> Vec { + let mut segments = Vec::new(); + let mut chars = text.char_indices().peekable(); + while let Some((_, ch)) = chars.next() { + if !matches!(ch, '"' | '\'' | '`') { + continue; + } + let quote = ch; + let start = chars.peek().map(|(idx, _)| *idx).unwrap_or(text.len()); + for (end, current) in chars.by_ref() { + if current == quote { + if end > start { + segments.push(text[start..end].to_string()); + } + break; + } + } + } + segments +} + +pub(super) fn context_tokens(task: &str) -> Vec { + task.split_whitespace() + .map(|word| { + word.trim_matches(|c: char| { + !c.is_alphanumeric() && !matches!(c, '_' | '-' | '/' | '.' | ':') + }) + .to_string() + }) + .filter(|word| !word.is_empty()) + .collect() +} + +pub(super) fn add_context_keyword_variants( + keywords: &mut Vec, + seen: &mut HashSet, + keyword: &str, +) { + let keyword = keyword.trim(); + if keyword.is_empty() { + return; + } + + push_context_keyword(keywords, seen, keyword.to_string()); + + let normalized = context_normalize(keyword); + if normalized.len() >= 3 { + push_context_keyword(keywords, seen, normalized.clone()); + if let Some(singular) = singular_context_term(&normalized) { + push_context_keyword(keywords, seen, singular); + } + } + + if keyword.contains(['-', '_', '/', '.', ':']) { + let separator_normalized = keyword + .replace(['/', '.', ':'], "-") + .replace('_', "-") + .trim_matches('-') + .to_string(); + if !separator_normalized.is_empty() { + push_context_keyword(keywords, seen, separator_normalized); + } + } +} + +pub(super) fn add_context_phrase_variants( + keywords: &mut Vec, + seen: &mut HashSet, + terms: &[String], +) { + if terms.is_empty() { + return; + } + + let joined_dash = terms.join("-"); + let joined_underscore = terms.join("_"); + let joined_space = terms.join(" "); + push_context_keyword(keywords, seen, joined_dash.clone()); + push_context_keyword(keywords, seen, joined_underscore); + push_context_keyword(keywords, seen, joined_space); + + let singular_terms = terms + .iter() + .map(|term| singular_context_term(term).unwrap_or_else(|| term.clone())) + .collect::>(); + if singular_terms != terms { + push_context_keyword(keywords, seen, singular_terms.join("-")); + push_context_keyword(keywords, seen, singular_terms.join("_")); + push_context_keyword(keywords, seen, singular_terms.join("")); + } + + let joined_normalized = context_normalize(&joined_dash); + if joined_normalized.len() >= 3 { + push_context_keyword(keywords, seen, joined_normalized); + } +} + +pub(super) fn push_context_keyword( + keywords: &mut Vec, + seen: &mut HashSet, + keyword: String, +) { + if keyword.len() < 3 { + return; + } + let key = keyword.to_lowercase(); + if seen.insert(key) { + keywords.push(keyword); + } +} + +pub(super) fn is_identifier_like_context_token(token: &str) -> bool { + token.contains(['_', '-', '/', '.', ':']) + || has_lower_to_upper_transition(token) + || token + .chars() + .filter(|ch| ch.is_ascii_alphabetic()) + .take(8) + .count() + >= 3 + && token.chars().all(|ch| { + !ch.is_ascii_alphabetic() || ch.is_ascii_uppercase() || ch.is_ascii_digit() + }) +} + +pub(super) fn has_lower_to_upper_transition(token: &str) -> bool { + let mut previous_lower = false; + for ch in token.chars() { + if previous_lower && ch.is_ascii_uppercase() { + return true; + } + previous_lower = ch.is_ascii_lowercase(); + } + false +} + +pub(super) fn has_identifier_case_signal(token: &str) -> bool { + has_lower_to_upper_transition(token) + || token + .chars() + .any(|ch| ch.is_ascii_uppercase() || matches!(ch, '_' | '-' | '/' | '.' | ':')) +} + +pub(super) fn context_terms(keywords: &[String]) -> Vec { + let mut terms = Vec::new(); + let mut seen = HashSet::new(); + for keyword in keywords { + let normalized = context_normalize(keyword); + if normalized.len() >= 3 && seen.insert(normalized.clone()) { + terms.push(normalized.clone()); + } + if let Some(singular) = singular_context_term(&normalized) { + if singular.len() >= 3 && seen.insert(singular.clone()) { + terms.push(singular); + } + } + + for raw_term in keyword.split(|ch: char| !ch.is_alphanumeric()) { + let term = context_normalize(raw_term); + if term.len() >= 3 && seen.insert(term.clone()) { + terms.push(term.clone()); + } + if let Some(singular) = singular_context_term(&term) { + if singular.len() >= 3 && seen.insert(singular.clone()) { + terms.push(singular); + } + } + } + } + terms +} + +pub(super) fn context_core_terms(keywords: &[String]) -> Vec { + context_terms(keywords) + .into_iter() + .filter(|term| (term.len() >= 4 || term == "run") && !is_low_signal_context_term(term)) + .collect() +} + +pub(super) fn is_low_signal_context_term(term: &str) -> bool { + matches!( + term, + "what" + | "when" + | "where" + | "which" + | "with" + | "this" + | "that" + | "from" + | "into" + | "does" + | "work" + | "works" + | "look" + | "find" + | "show" + | "how" + | "why" + | "the" + | "and" + | "for" + | "application" + ) +} + +pub(super) fn context_action_term_score( + symbol_norm: &str, + path_norm: &str, + callable: bool, + terms: &[String], +) -> i32 { + let has_action_term = terms + .iter() + .any(|term| matches!(term.as_str(), "create" | "build" | "make" | "use" | "run")); + if !has_action_term { + return 0; + } + + if terms + .iter() + .filter(|term| matches!(term.as_str(), "create" | "build" | "make" | "use" | "run")) + .any(|term| symbol_norm.contains(term) || path_norm.contains(term)) + { + if callable { + CONTEXT_ACTION_TERM_MATCH_BONUS + } else { + CONTEXT_ACTION_TERM_MATCH_BONUS / 2 + } + } else { + -CONTEXT_MISSING_ACTION_TERM_PENALTY + } +} + +pub(super) fn singular_context_term(term: &str) -> Option { + if term.len() > 3 && term.ends_with('s') { + Some(term.trim_end_matches('s').to_string()) + } else { + None + } +} + +pub(super) fn is_source_context_path(path: &str) -> bool { + path.starts_with("src/") + || (path.starts_with("packages/") && path.contains("/src/")) + || (path.starts_with("apps/") && path.contains("/src/")) +} + +pub(super) fn is_example_context_path(path: &str) -> bool { + path.starts_with("examples/") || path.contains("/examples/") +} + +pub(super) fn context_path_score(path: &str, allow_test_context: bool) -> i32 { + if is_source_context_path(path) { + CONTEXT_SOURCE_PATH_BONUS + } else if is_doc_path(path) { + -CONTEXT_DOC_PATH_PENALTY + } else if is_example_context_path(path) { + -CONTEXT_EXAMPLE_PATH_PENALTY + } else if is_test_like_path(path) && !allow_test_context { + -CONTEXT_TEST_PATH_PENALTY + } else { + 0 + } +} + +pub(super) fn context_path_rank(path: &str) -> u8 { + if is_source_context_path(path) { + 0 + } else if is_test_like_path(path) { + 4 + } else if is_example_context_path(path) { + 5 + } else if is_doc_path(path) { + 6 + } else if path.starts_with("packages/") || path.starts_with("apps/") { + 1 + } else { + 3 + } +} diff --git a/src/engine/core.rs b/src/engine/core.rs new file mode 100644 index 0000000..b737945 --- /dev/null +++ b/src/engine/core.rs @@ -0,0 +1,143 @@ +use crate::index::symbol::SymbolIndex; +use crate::index::trigram::TrigramIndex; +use crate::index::word::WordIndex; +use crate::store::Store; +use crate::types::*; +use hashbrown::HashMap; +use serde::Serialize; + +use super::dep_graph::DepGraph; + +#[derive(Debug, Clone, Default)] +pub struct SearchOptions { + pub max_results: usize, + pub regex: bool, + pub scope: bool, + pub compact: bool, + pub paths_only: bool, + pub path_glob: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct FileFilterOptions { + pub path_prefix: Option, + pub path_glob: Option, + pub language: Option, + pub min_lines: Option, + pub max_lines: Option, + pub max_results: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct WordSearchOptions { + pub path_prefix: Option, + pub path_glob: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct WordSearchResult { + pub path: String, + pub line_num: u32, + pub line_text: String, + pub kind: String, + pub score: i32, +} + +#[derive(Debug, Clone, Serialize)] +pub struct RichSearchResult { + pub path: String, + pub line_num: u32, + pub line_text: String, + pub scope: Option, +} + +#[derive(Debug, Clone)] +pub struct ReadFileResult { + pub content: String, + pub hash: u64, + pub unchanged: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ContextDetails { + pub task: String, + pub keywords: Vec, + pub max_results: usize, + pub confidence: String, + pub note: Option, + pub suggested_next_steps: Vec, + pub relevant_symbols: Vec, + pub snippets: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct ContextOptions { + pub max_results: usize, + pub path_prefix: Option, + pub path_glob: Option, + pub language: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ContextSymbol { + pub path: String, + pub name: String, + pub kind: String, + pub line_start: u32, + pub line_end: u32, + pub detail: Option, + pub content_line_start: u32, + pub content_line_end: u32, + pub content: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct SymbolSearchResult { + pub path: String, + pub name: String, + pub kind: String, + pub line_start: u32, + pub line_end: u32, + pub detail: Option, + pub score: f32, + pub raw_score: i32, +} + +pub struct Engine { + pub(super) outlines: HashMap, + pub(super) file_meta: HashMap, + pub(super) contents: HashMap, + pub(super) symbol_index: SymbolIndex, + pub(super) trigram_index: TrigramIndex, + pub(super) word_index: WordIndex, + pub(super) dep_graph: DepGraph, + pub(super) store: Store, + pub(super) freshness_watermark_ns: Option, +} + +impl Engine { + pub fn new(_cache_capacity: u32) -> Self { + Self { + outlines: HashMap::new(), + file_meta: HashMap::new(), + contents: HashMap::new(), + symbol_index: SymbolIndex::new(), + trigram_index: TrigramIndex::new(), + word_index: WordIndex::new(), + dep_graph: DepGraph::new(), + store: Store::new(), + freshness_watermark_ns: None, + } + } + + pub(crate) fn set_freshness_watermark(&mut self, watermark_ns: Option) { + self.freshness_watermark_ns = watermark_ns; + } + + pub(crate) fn content_unchanged_since_snapshot(&self, change_ns: Option) -> bool { + matches!( + (change_ns, self.freshness_watermark_ns), + (Some(change), Some(watermark)) if change < watermark + ) + } +} diff --git a/src/engine/dep_graph.rs b/src/engine/dep_graph.rs index fd5b5a0..04aca5d 100644 --- a/src/engine/dep_graph.rs +++ b/src/engine/dep_graph.rs @@ -142,6 +142,43 @@ impl DepGraph { deps.sort_by(|a, b| a.0.cmp(&b.0)); deps } + + pub(crate) fn unresolved_imports_by_path(&self) -> Vec<(String, Vec)> { + let mut unresolved = self + .unresolved + .iter() + .map(|(path, imports)| (path.clone(), imports.clone())) + .collect::>(); + unresolved.sort_by(|left, right| left.0.cmp(&right.0)); + unresolved + } + + pub(crate) fn from_snapshot( + forward: Vec<(String, Vec)>, + unresolved: Vec<(String, Vec)>, + ) -> Option { + let mut unresolved_by_path = HashMap::new(); + for (path, imports) in unresolved { + if unresolved_by_path.insert(path, imports).is_some() { + return None; + } + } + let mut graph = Self::new(); + for (path, deps) in forward { + let path_unresolved = unresolved_by_path.get(&path).cloned().unwrap_or_default(); + if graph.forward.contains_key(&path) { + return None; + } + graph.set_resolution(&path, deps, path_unresolved); + } + if unresolved_by_path + .keys() + .any(|path| !graph.forward.contains_key(path)) + { + return None; + } + Some(graph) + } } impl Default for DepGraph { diff --git a/src/engine/files.rs b/src/engine/files.rs new file mode 100644 index 0000000..8f40d1a --- /dev/null +++ b/src/engine/files.rs @@ -0,0 +1,410 @@ +use crate::glob::match_glob; +use crate::store::Store; +use crate::types::*; +use hashbrown::HashSet; + +use super::core::*; +use super::hash_content; +use super::shared::*; + +impl Engine { + pub fn file_map(&self) -> Vec<(String, FileMeta)> { + let mut entries: Vec<(&String, &FileMeta)> = self.file_meta.iter().collect(); + entries.sort_by_key(|(path, _)| path.as_str()); + entries + .into_iter() + .map(|(path, meta)| (path.clone(), meta.clone())) + .collect() + } + + pub fn filtered_files( + &self, + options: &FileFilterOptions, + ) -> (Vec<(String, FileMeta)>, usize, bool) { + let language = options.language.as_ref().map(|value| value.to_lowercase()); + let path_prefix = options + .path_prefix + .as_deref() + .filter(|value| !value.is_empty()) + .map(normalize_filter_prefix); + + let mut entries = + self.file_map() + .into_iter() + .filter(|(path, meta)| { + path_prefix.as_ref().is_none_or(|prefix| { + path == prefix || path.starts_with(&format!("{prefix}/")) + }) && options + .path_glob + .as_deref() + .is_none_or(|glob| match_glob(glob, path)) + && language + .as_deref() + .is_none_or(|language| meta.language.as_str() == language) + && options + .min_lines + .is_none_or(|min_lines| meta.line_count >= min_lines) + && options + .max_lines + .is_none_or(|max_lines| meta.line_count <= max_lines) + }) + .collect::>(); + let total = entries.len(); + if let Some(max_results) = options.max_results { + entries.truncate(max_results); + } + let truncated = entries.len() < total; + (entries, total, truncated) + } + + pub fn get_imported_by(&self, path: &str) -> Vec { + self.dep_graph.get_imported_by(path) + } + + pub fn get_depends_on(&self, path: &str) -> Vec { + self.dep_graph.get_depends_on(path) + } + + pub fn get_unresolved_imports(&self, path: &str) -> Vec { + self.dep_graph.get_unresolved_imports(path) + } + + pub fn unresolved_imports(&self) -> Vec { + let mut imports = self.dep_graph.unresolved_imports(); + imports.sort_by(|a, b| { + a.path + .cmp(&b.path) + .then_with(|| a.line_start.cmp(&b.line_start)) + .then_with(|| a.import.cmp(&b.import)) + }); + imports + } + + pub fn get_transitive_imported_by(&self, path: &str) -> Vec { + self.dep_graph.get_transitive(path, true) + } + + pub fn get_transitive_depends_on(&self, path: &str) -> Vec { + self.dep_graph.get_transitive(path, false) + } + + pub fn get_hot_files(&self, limit: usize) -> Vec<(String, FileMeta)> { + let mut entries: Vec<(&String, &FileMeta)> = self.file_meta.iter().collect(); + entries.sort_by(|a, b| { + b.1.modified_ms + .cmp(&a.1.modified_ms) + .then_with(|| b.1.byte_size.cmp(&a.1.byte_size)) + .then_with(|| a.0.cmp(b.0)) + }); + entries + .into_iter() + .take(limit) + .map(|(p, m)| (p.clone(), m.clone())) + .collect() + } + + pub fn find_callers(&self, symbol_name: &str, max_results: usize) -> Vec { + let mut results = Vec::new(); + let mut seen: HashSet<(String, u32)> = HashSet::new(); + + let definitions = self.symbol_index.find_all(symbol_name); + let def_locations: HashSet<(String, u32)> = definitions + .iter() + .map(|loc| (loc.path.clone(), loc.line_start)) + .collect(); + + let occurrences = self.word_index.search(symbol_name); + + for (path, line_num) in occurrences { + if results.len() >= max_results { + break; + } + + if def_locations.contains(&(path.clone(), line_num)) { + continue; + } + + if seen.insert((path.clone(), line_num)) { + if let Some(line_text) = self.get_line(&path, line_num) { + results.push(SearchResult { + path, + line_num, + line_text, + }); + } + } + } + + results + } +} + +impl Engine { + pub fn get_changes(&self, since_seq: u64) -> Vec<(String, u64, String)> { + let changes = self.store.changes_since_detailed(since_seq); + changes + .into_iter() + .map(|c| { + let op_str = format!("{:?}", c.op); + (c.path, c.seq, op_str) + }) + .collect() + } + + pub fn read_file( + &self, + path: &str, + line_start: Option, + line_end: Option, + ) -> Option { + let stub; + let content = if let Some(content) = self.content_for(path) { + content + } else { + let meta = self.file_meta.get(path)?; + if meta.indexed { + return None; + } + stub = unindexed_file_stub(path, meta); + &stub + }; + let lines: Vec<&str> = content.lines().collect(); + + match (line_start, line_end) { + (Some(start), Some(end)) => { + let start_idx = (start.saturating_sub(1)) as usize; + let end_idx = (end as usize).min(lines.len()); + if start_idx >= lines.len() || start_idx >= end_idx { + return Some(String::new()); + } + Some(lines[start_idx..end_idx].join("\n")) + } + (Some(start), None) => { + let start_idx = (start.saturating_sub(1)) as usize; + if start_idx >= lines.len() { + return Some(String::new()); + } + Some(lines[start_idx..].join("\n")) + } + (None, Some(end)) => { + let end_idx = (end as usize).min(lines.len()); + Some(lines[..end_idx].join("\n")) + } + (None, None) => Some(content.to_string()), + } + } + + pub fn read_file_rich( + &self, + path: &str, + line_start: Option, + line_end: Option, + compact: bool, + if_hash: Option<&str>, + ) -> Option { + let stub; + let content = if let Some(content) = self.content_for(path) { + content + } else { + let meta = self.file_meta.get(path)?; + if meta.indexed { + return None; + } + stub = unindexed_file_stub(path, meta); + &stub + }; + let hash = hash_content(content); + let hash_hex = format!("{hash:x}"); + if if_hash.is_some_and(|expected| expected.eq_ignore_ascii_case(&hash_hex)) { + return Some(ReadFileResult { + content: String::new(), + hash, + unchanged: true, + }); + } + + let mut selected = self.read_file(path, line_start, line_end)?; + if compact { + let language = self + .file_meta + .get(path) + .map(|meta| meta.language) + .unwrap_or_else(|| detect_language(path)); + selected = selected + .lines() + .filter(|line| !is_comment_or_blank(line, language)) + .collect::>() + .join("\n"); + } + + Some(ReadFileResult { + content: selected, + hash, + unchanged: false, + }) + } +} + +impl Engine { + pub fn glob_files(&self, pattern: &str) -> Vec { + let mut results: Vec = self + .file_meta + .keys() + .filter(|path| match_glob(pattern, path)) + .cloned() + .collect(); + results.sort(); + results + } + + pub fn fuzzy_find(&self, pattern: &str, max_results: usize) -> Vec<(String, f32)> { + let pattern_lower = pattern.to_lowercase(); + let pattern_chars: Vec = pattern_lower.chars().collect(); + let mut results: Vec<(String, f32)> = Vec::new(); + + for path in self.file_meta.keys() { + let path_lower = path.to_lowercase(); + let filename = path_lower.rsplit('/').next().unwrap_or(&path_lower); + + let (score, matched) = if let Some(s) = fuzzy_match(&pattern_chars, filename) { + (s + 10.0, true) + } else if let Some(s) = fuzzy_match(&pattern_chars, &path_lower) { + (s, true) + } else { + (0.0, false) + }; + + if matched && score > 0.0 { + results.push((path.clone(), score)); + } + } + + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + results.truncate(max_results); + results + } + + pub fn list_dir(&self, dir: &str) -> Vec<(String, Option<&FileMeta>)> { + let prefix = if dir.is_empty() || dir == "." { + String::new() + } else { + format!("{}/", dir.trim_end_matches('/')) + }; + + let mut dirs: HashSet = HashSet::new(); + let mut files: Vec<(String, Option<&FileMeta>)> = Vec::new(); + + for (path, meta) in &self.file_meta { + if let Some(rest) = path.strip_prefix(&prefix) { + if let Some(slash_pos) = rest.find('/') { + let dir_name = &rest[..slash_pos]; + dirs.insert(dir_name.to_string()); + } else { + files.push((rest.to_string(), Some(meta))); + } + } + } + + let mut result: Vec<(String, Option<&FileMeta>)> = Vec::new(); + let mut sorted_dirs: Vec = dirs.into_iter().collect(); + sorted_dirs.sort(); + for d in sorted_dirs { + result.push((format!("{}/", d), None)); + } + files.sort_by(|a, b| a.0.cmp(&b.0)); + result.extend(files); + + result + } +} + +impl Engine { + pub fn file_count(&self) -> usize { + self.file_meta.len() + } + + pub fn symbol_index_count(&self) -> usize { + self.symbol_index.symbol_count() + } + + pub fn word_index_count(&self) -> usize { + self.word_index.unique_word_count() + } + + pub fn word_index_file_count(&self) -> usize { + self.word_index.file_count() + } + + pub fn store(&self) -> &Store { + &self.store + } + + pub fn content(&self, path: &str) -> Option<&str> { + self.content_for(path) + } + + pub fn enclosing_symbol(&self, path: &str, line_num: u32) -> Option<&Symbol> { + self.outlines.get(path).and_then(|outline| { + outline + .symbols + .iter() + .filter(|sym| { + sym.kind != SymbolKind::Import + && sym.line_start <= line_num + && sym.line_end >= line_num + }) + .max_by_key(|sym| sym.line_start) + }) + } + + pub(super) fn symbol_source_bounded( + &self, + path: &str, + symbol: &Symbol, + max_lines: u32, + ) -> Option<(u32, u32, String)> { + let outline = self.outlines.get(path)?; + let start = symbol.line_start.max(1); + let natural_end = symbol.line_end.min(outline.line_count); + let capped_end = start + .saturating_add(max_lines.saturating_sub(1)) + .min(natural_end) + .max(start); + self.read_file(path, Some(start), Some(capped_end)) + .map(|content| (start, capped_end, content)) + } + + pub(super) fn get_line(&self, path: &str, line_num: u32) -> Option { + let content = self.content_for(path)?; + content + .lines() + .nth((line_num - 1) as usize) + .map(|s| s.to_string()) + } + + pub(super) fn content_for(&self, path: &str) -> Option<&str> { + self.contents.get(path).map(String::as_str) + } +} + +fn unindexed_file_stub(path: &str, meta: &FileMeta) -> String { + let kind = path + .rsplit_once('.') + .map(|(_, ext)| ext) + .filter(|ext| !ext.is_empty()) + .unwrap_or("unknown"); + format!( + "unindexed {kind} file: {} bytes\npath: {path}\nmodified_ms: {}\n", + meta.byte_size, meta.modified_ms + ) +} + +pub(super) fn matches_path_glob(pattern: &str, path: &str) -> bool { + if match_glob(pattern, path) { + return true; + } + if !pattern.contains('/') { + return match_glob(&format!("**/{pattern}"), path); + } + false +} diff --git a/src/engine/indexing.rs b/src/engine/indexing.rs new file mode 100644 index 0000000..c761483 --- /dev/null +++ b/src/engine/indexing.rs @@ -0,0 +1,175 @@ +use crate::index::trigram::{extract_trigrams, Trigram}; +use crate::index::word::{prepare_word_index, PreparedWordIndex}; +use crate::parser; +use crate::store::Op; +use crate::types::*; + +use super::core::Engine; +use super::hash_content; +use super::shared::now_ms; + +pub(super) struct PreparedIndexFile { + path: String, + content: String, + modified_ms: u64, + language: Language, + line_count: u32, + byte_size: u64, + outline: FileOutline, + trigrams: Vec, + words: PreparedWordIndex, +} + +pub(super) fn prepare_index_file( + path: &str, + content: String, + modified_ms: u64, +) -> PreparedIndexFile { + let language = detect_language(path); + let line_count = content.lines().count().max(1) as u32; + let byte_size = content.len() as u64; + let outline = parser::parse_file(path, language, &content).unwrap_or_else(|| { + let mut outline = FileOutline::new(path.to_string(), language); + outline.line_count = line_count; + outline.byte_size = byte_size; + outline + }); + let trigrams = extract_trigrams(&content); + let words = prepare_word_index(&content); + + PreparedIndexFile { + path: path.to_string(), + content, + modified_ms, + language, + line_count, + byte_size, + outline, + trigrams, + words, + } +} + +impl Engine { + pub fn index_file(&mut self, path: &str, content: &str) { + self.index_file_with_modified(path, content, now_ms()); + } + + pub fn index_file_with_modified(&mut self, path: &str, content: &str, modified_ms: u64) { + self.index_file_with_op(path, content, modified_ms, Op::Snapshot, true); + } + + pub(crate) fn index_file_with_modified_no_rebuild( + &mut self, + path: &str, + content: &str, + modified_ms: u64, + ) { + self.index_file_with_op(path, content, modified_ms, Op::Snapshot, false); + } + + pub fn index_edited_file(&mut self, path: &str, content: &str, op: Op) { + self.index_file_with_op(path, content, now_ms(), op, true); + } + + pub(super) fn index_file_with_op( + &mut self, + path: &str, + content: &str, + modified_ms: u64, + op: Op, + rebuild_deps: bool, + ) { + let prepared = prepare_index_file(path, content.to_string(), modified_ms); + self.index_prepared_file(prepared, op, rebuild_deps); + } + + pub(super) fn index_prepared_file( + &mut self, + prepared: PreparedIndexFile, + op: Op, + rebuild_deps: bool, + ) { + let PreparedIndexFile { + path, + content, + modified_ms, + language, + line_count, + byte_size, + outline, + trigrams, + words, + } = prepared; + let content_hash = hash_content(&content); + + self.symbol_index.index_file(&outline); + self.trigram_index.index_prepared(&path, trigrams); + self.word_index.index_prepared(&path, words); + self.contents.insert(path.clone(), content); + self.file_meta.insert( + path.clone(), + FileMeta { + language, + line_count, + byte_size, + symbol_count: outline.symbol_count() as u32, + modified_ms, + indexed: true, + }, + ); + self.outlines.insert(path.clone(), outline); + if rebuild_deps { + self.rebuild_dep_graph(); + } + match op { + Op::Snapshot => { + self.store.record_snapshot(&path, byte_size, content_hash); + } + Op::Replace | Op::Insert | Op::Delete | Op::Create => { + self.store + .record_edit(&path, 0, op, content_hash, byte_size); + } + Op::Tombstone => { + self.store.record_delete(&path, 0); + } + } + } + + pub fn index_file_meta_only(&mut self, path: &str, byte_size: u64, modified_ms: u64) { + self.index_file_meta_only_no_rebuild(path, byte_size, modified_ms); + self.rebuild_dep_graph(); + } + + pub(crate) fn index_file_meta_only_no_dep_rebuild( + &mut self, + path: &str, + byte_size: u64, + modified_ms: u64, + ) { + self.index_file_meta_only_no_rebuild(path, byte_size, modified_ms); + } + + pub(crate) fn update_file_metadata(&mut self, path: &str, byte_size: u64, modified_ms: u64) { + if let Some(meta) = self.file_meta.get_mut(path) { + meta.byte_size = byte_size; + meta.modified_ms = modified_ms; + } + } + + pub fn remove_file(&mut self, path: &str) { + self.remove_file_no_dep_rebuild(path); + self.rebuild_dep_graph(); + } + + pub(crate) fn remove_file_no_dep_rebuild(&mut self, path: &str) { + self.outlines.remove(path); + self.file_meta.remove(path); + self.contents.remove(path); + self.symbol_index.remove_file(path); + self.trigram_index.remove_file(path); + self.word_index.remove_file(path); + self.dep_graph.remove(path); + self.store.record_delete(path, 0); + } +} diff --git a/src/engine/mod.rs b/src/engine/mod.rs index 1c2f3f0..0df2f12 100644 --- a/src/engine/mod.rs +++ b/src/engine/mod.rs @@ -1,1813 +1,18 @@ -use crate::cache::ContentCache; -use crate::glob::match_glob; -use crate::index::symbol::SymbolIndex; -use crate::index::trigram::TrigramIndex; -use crate::index::word::WordIndex; -use crate::parser; -use crate::snapshot; -use crate::store::{Op, Store}; -use crate::types::*; -use hashbrown::{HashMap, HashSet}; -use regex::Regex; -use serde::Serialize; -use std::path::Path; -use std::time::{SystemTime, UNIX_EPOCH}; - +mod context; +mod context_helpers; +mod core; mod dep_graph; +mod files; mod imports; +mod indexing; +mod persistence; mod ranking; +mod search; +mod shared; +pub use core::*; pub use dep_graph::DepGraph; -use ranking::*; - -#[derive(Debug, Clone, Default)] -pub struct SearchOptions { - pub max_results: usize, - pub regex: bool, - pub scope: bool, - pub compact: bool, - pub paths_only: bool, - pub path_glob: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct FileFilterOptions { - pub path_prefix: Option, - pub path_glob: Option, - pub language: Option, - pub min_lines: Option, - pub max_lines: Option, - pub max_results: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct WordSearchOptions { - pub path_prefix: Option, - pub path_glob: Option, -} - -#[derive(Debug, Clone, Serialize)] -pub struct WordSearchResult { - pub path: String, - pub line_num: u32, - pub line_text: String, - pub kind: String, - pub score: i32, -} - -#[derive(Debug, Clone, Serialize)] -pub struct RichSearchResult { - pub path: String, - pub line_num: u32, - pub line_text: String, - pub scope: Option, -} - -#[derive(Debug, Clone)] -pub struct ReadFileResult { - pub content: String, - pub hash: u64, - pub unchanged: bool, -} - -#[derive(Debug, Clone, Serialize)] -pub struct ContextDetails { - pub task: String, - pub keywords: Vec, - pub max_results: usize, - pub confidence: String, - pub note: Option, - pub suggested_next_steps: Vec, - pub relevant_symbols: Vec, - pub snippets: Vec, -} - -#[derive(Debug, Clone, Default)] -pub struct ContextOptions { - pub max_results: usize, - pub path_prefix: Option, - pub path_glob: Option, - pub language: Option, -} - -#[derive(Debug, Clone, Serialize)] -pub struct ContextSymbol { - pub path: String, - pub name: String, - pub kind: String, - pub line_start: u32, - pub line_end: u32, - pub detail: Option, - pub content_line_start: u32, - pub content_line_end: u32, - pub content: String, -} - -#[derive(Debug, Clone, Serialize)] -pub struct SymbolSearchResult { - pub path: String, - pub name: String, - pub kind: String, - pub line_start: u32, - pub line_end: u32, - pub detail: Option, - pub score: f32, - pub raw_score: i32, -} - -struct ScoredSearchResult { - score: i32, - result: SearchResult, -} - -struct ScoredContextSymbol { - score: i32, - result: SymbolResult, -} - -pub struct Engine { - outlines: HashMap, - file_meta: HashMap, - contents: HashMap, - content_cache: ContentCache, - symbol_index: SymbolIndex, - trigram_index: TrigramIndex, - word_index: WordIndex, - dep_graph: DepGraph, - store: Store, -} - -impl Engine { - pub fn new(cache_capacity: u32) -> Self { - Self { - outlines: HashMap::new(), - file_meta: HashMap::new(), - contents: HashMap::new(), - content_cache: ContentCache::new(cache_capacity), - symbol_index: SymbolIndex::new(), - trigram_index: TrigramIndex::new(), - word_index: WordIndex::new(), - dep_graph: DepGraph::new(), - store: Store::new(), - } - } - - pub fn index_file(&mut self, path: &str, content: &str) { - self.index_file_with_modified(path, content, now_ms()); - } - - pub fn index_file_with_modified(&mut self, path: &str, content: &str, modified_ms: u64) { - self.index_file_with_op(path, content, modified_ms, Op::Snapshot, true); - } - - pub(crate) fn index_file_with_modified_no_rebuild( - &mut self, - path: &str, - content: &str, - modified_ms: u64, - ) { - self.index_file_with_op(path, content, modified_ms, Op::Snapshot, false); - } - - pub fn index_edited_file(&mut self, path: &str, content: &str, op: Op) { - self.index_file_with_op(path, content, now_ms(), op, true); - } - - fn index_file_with_op( - &mut self, - path: &str, - content: &str, - modified_ms: u64, - op: Op, - rebuild_deps: bool, - ) { - let language = detect_language(path); - let line_count = content.lines().count().max(1) as u32; - let byte_size = content.len() as u64; - - let outline = parser::parse_file(path, language, content).unwrap_or_else(|| { - let mut o = FileOutline::new(path.to_string(), language); - o.line_count = line_count; - o.byte_size = byte_size; - o - }); - - self.symbol_index.index_file(&outline); - self.trigram_index.index_file(path, content); - self.word_index.index_file(path, content); - self.content_cache - .put(path.to_string(), content.to_string()); - self.contents.insert(path.to_string(), content.to_string()); - self.file_meta.insert( - path.to_string(), - FileMeta { - language, - line_count, - byte_size, - symbol_count: outline.symbol_count() as u32, - modified_ms, - indexed: true, - }, - ); - self.outlines.insert(path.to_string(), outline); - if rebuild_deps { - self.rebuild_dep_graph(); - } - match op { - Op::Snapshot => { - self.store - .record_snapshot(path, byte_size, hash_content(content)); - } - Op::Replace | Op::Insert | Op::Delete | Op::Create => { - self.store - .record_edit(path, 0, op, hash_content(content), byte_size); - } - Op::Tombstone => { - self.store.record_delete(path, 0); - } - } - } - - pub fn index_file_meta_only(&mut self, path: &str, byte_size: u64, modified_ms: u64) { - self.index_file_meta_only_no_rebuild(path, byte_size, modified_ms); - self.rebuild_dep_graph(); - } - - pub(crate) fn index_file_meta_only_no_dep_rebuild( - &mut self, - path: &str, - byte_size: u64, - modified_ms: u64, - ) { - self.index_file_meta_only_no_rebuild(path, byte_size, modified_ms); - } - - pub fn remove_file(&mut self, path: &str) { - self.remove_file_no_dep_rebuild(path); - self.rebuild_dep_graph(); - } - - pub(crate) fn remove_file_no_dep_rebuild(&mut self, path: &str) { - self.outlines.remove(path); - self.file_meta.remove(path); - self.contents.remove(path); - self.content_cache.remove(path); - self.symbol_index.remove_file(path); - self.trigram_index.remove_file(path); - self.word_index.remove_file(path); - self.dep_graph.remove(path); - self.store.record_delete(path, 0); - } - - pub fn get_outline(&self, path: &str) -> Option<&FileOutline> { - self.outlines.get(path) - } - - pub fn find_symbol(&self, name: &str) -> Vec { - self.symbol_index - .find_all(name) - .into_iter() - .map(|loc| { - let symbol = self - .outlines - .get(&loc.path) - .and_then(|o| { - o.symbols - .iter() - .find(|s| s.name == name && s.kind == loc.kind) - }) - .cloned() - .unwrap_or_else(|| Symbol { - name: name.to_string(), - kind: loc.kind, - line_start: loc.line_start, - line_end: loc.line_end, - detail: None, - }); - SymbolResult { - path: loc.path, - symbol, - } - }) - .collect() - } - - pub fn fuzzy_symbols(&self, query: &str, max_results: usize) -> Vec { - let query_norm = context_normalize(query); - if query_norm.is_empty() { - return Vec::new(); - } - - let mut scored = Vec::new(); - for (path, outline) in &self.outlines { - for symbol in &outline.symbols { - if symbol.kind == SymbolKind::Import { - continue; - } - let symbol_norm = context_normalize(&symbol.name); - let path_norm = context_normalize(path); - let mut score = symbol_kind_context_score(symbol.kind); - - if symbol_norm == query_norm { - score += 1000; - } else if symbol_norm.contains(&query_norm) { - score += 700; - } else if query_norm.contains(&symbol_norm) && symbol_norm.len() >= 4 { - score += 320; - } else if fuzzy_match_score(&query_norm, &symbol_norm).is_some() { - score += 220; - } else { - continue; - } - - if path_norm.contains(&query_norm) { - score += 80; - } - if is_test_like_path(path) { - score -= 60; - } - - scored.push((score, path.clone(), symbol.clone())); - } - } - - scored.sort_by(|a, b| { - b.0.cmp(&a.0) - .then_with(|| a.1.cmp(&b.1)) - .then_with(|| a.2.line_start.cmp(&b.2.line_start)) - .then_with(|| a.2.name.cmp(&b.2.name)) - }); - - let max_score = scored.first().map(|entry| entry.0.max(1)).unwrap_or(1) as f32; - scored - .into_iter() - .take(max_results) - .map(|(raw_score, path, symbol)| SymbolSearchResult { - path, - name: symbol.name, - kind: symbol.kind.to_string(), - line_start: symbol.line_start, - line_end: symbol.line_end, - detail: symbol.detail, - score: raw_score as f32 / max_score, - raw_score, - }) - .collect() - } - - pub fn search(&self, query: &str, max_results: usize) -> Vec { - let query_lower = query.to_lowercase(); - let words: Vec<&str> = query_lower.split_whitespace().collect(); - - if words.len() > 1 { - return self.search_multi_word(&words, max_results); - } - - let single_word = words.first().copied().unwrap_or(&query_lower); - let mut results = Vec::new(); - let mut seen: HashSet<(String, u32)> = HashSet::new(); - - let word_hits = self.word_index.search(single_word); - for (path, line_num) in word_hits { - if results.len() >= max_results { - return results; - } - if seen.insert((path.clone(), line_num)) { - if let Some(line_text) = self.get_line(&path, line_num) { - results.push(SearchResult { - path, - line_num, - line_text, - }); - } - } - } - - if results.len() < max_results { - let prefix_hits = self.word_index.search_prefix(single_word); - for (path, line_num, _word) in prefix_hits { - if results.len() >= max_results { - return results; - } - if seen.insert((path.clone(), line_num)) { - if let Some(line_text) = self.get_line(&path, line_num) { - results.push(SearchResult { - path, - line_num, - line_text, - }); - } - } - } - } - - if results.len() < max_results { - let candidates = self.trigram_index.candidates(single_word); - for path in candidates { - if results.len() >= max_results { - return results; - } - if let Some(content) = self.content_for(&path) { - for (line_idx, line) in content.lines().enumerate() { - if results.len() >= max_results { - return results; - } - let line_num = (line_idx + 1) as u32; - if line.to_lowercase().contains(single_word) - && seen.insert((path.clone(), line_num)) - { - results.push(SearchResult { - path: path.clone(), - line_num, - line_text: line.to_string(), - }); - } - } - } - } - } - - if results.is_empty() && self.outlines.len() < 100 { - for (path, _) in &self.outlines { - if results.len() >= max_results { - return results; - } - if let Some(content) = self.content_for(path) { - for (line_idx, line) in content.lines().enumerate() { - if results.len() >= max_results { - return results; - } - let line_num = (line_idx + 1) as u32; - if line.to_lowercase().contains(single_word) - && seen.insert((path.clone(), line_num)) - { - results.push(SearchResult { - path: path.clone(), - line_num, - line_text: line.to_string(), - }); - } - } - } - } - } - - results - } - - fn search_multi_word(&self, words: &[&str], max_results: usize) -> Vec { - let mut results = Vec::new(); - let mut seen: HashSet<(String, u32)> = HashSet::new(); - - let first_word = words[0]; - let mut candidate_paths: Vec = Vec::new(); - - let word_hits = self.word_index.search(first_word); - for (path, _line_num) in &word_hits { - if !candidate_paths.contains(path) { - candidate_paths.push(path.clone()); - } - } - - let trigram_candidates = self.trigram_index.candidates(first_word); - for path in trigram_candidates { - if !candidate_paths.contains(&path) { - candidate_paths.push(path); - } - } - - if candidate_paths.is_empty() { - candidate_paths = self.file_meta.keys().cloned().collect(); - } - - for path in candidate_paths { - if results.len() >= max_results { - return results; - } - if let Some(content) = self.content_for(&path) { - for (line_idx, line) in content.lines().enumerate() { - if results.len() >= max_results { - return results; - } - let line_lower = line.to_lowercase(); - let contains_all = words.iter().all(|w| line_lower.contains(w)); - if contains_all { - let line_num = (line_idx + 1) as u32; - if seen.insert((path.clone(), line_num)) { - results.push(SearchResult { - path: path.clone(), - line_num, - line_text: line.to_string(), - }); - } - } - } - } - } - - results - } - - pub fn search_regex( - &self, - pattern: &str, - max_results: usize, - ) -> Result, String> { - let re = Regex::new(pattern).map_err(|e| format!("Invalid regex: {}", e))?; - let mut results = Vec::new(); - let mut seen: HashSet<(String, u32)> = HashSet::new(); - - for (path, _) in &self.outlines { - if results.len() >= max_results { - break; - } - if let Some(content) = self.content_for(path) { - for (line_idx, line) in content.lines().enumerate() { - if results.len() >= max_results { - break; - } - if re.is_match(line) { - let line_num = (line_idx + 1) as u32; - if seen.insert((path.clone(), line_num)) { - results.push(SearchResult { - path: path.clone(), - line_num, - line_text: line.to_string(), - }); - } - } - } - } - } - - Ok(results) - } - - pub fn search_rich( - &self, - query: &str, - options: &SearchOptions, - ) -> Result, String> { - let max_results = options.max_results.max(1); - let raw = if options.regex { - self.search_regex(query, max_results.saturating_mul(4))? - } else { - self.search(query, max_results.saturating_mul(4)) - }; - - let mut results = Vec::new(); - for result in raw { - if let Some(pattern) = &options.path_glob { - if !matches_path_glob(pattern, &result.path) { - continue; - } - } - - let language = self - .file_meta - .get(&result.path) - .map(|meta| meta.language) - .unwrap_or_else(|| detect_language(&result.path)); - if options.compact && is_comment_or_blank(&result.line_text, language) { - continue; - } - - let scope = if options.scope { - self.enclosing_symbol(&result.path, result.line_num) - .cloned() - } else { - None - }; - - results.push(RichSearchResult { - path: result.path, - line_num: result.line_num, - line_text: result.line_text, - scope, - }); - - if results.len() >= max_results { - break; - } - } - - Ok(results) - } - - pub fn search_word(&self, word: &str) -> Vec { - self.word_index - .search(word) - .into_iter() - .filter_map(|(path, line_num)| { - self.get_line(&path, line_num) - .map(|line_text| SearchResult { - path, - line_num, - line_text, - }) - }) - .collect() - } - - pub fn search_word_with_options( - &self, - word: &str, - options: &WordSearchOptions, - ) -> Vec { - let path_prefix = options - .path_prefix - .as_deref() - .filter(|value| !value.is_empty()) - .map(normalize_filter_prefix); - - let mut results = self - .search_word(word) - .into_iter() - .filter(|result| { - path_prefix.as_ref().is_none_or(|prefix| { - result.path == *prefix || result.path.starts_with(&format!("{prefix}/")) - }) && options - .path_glob - .as_deref() - .is_none_or(|glob| matches_path_glob(glob, &result.path)) - }) - .map(|result| self.classified_word_result(word, result)) - .collect::>(); - - results.sort_by(|left, right| { - right - .score - .cmp(&left.score) - .then_with(|| { - word_result_path_rank(&left.path).cmp(&word_result_path_rank(&right.path)) - }) - .then_with(|| left.path.cmp(&right.path)) - .then_with(|| left.line_num.cmp(&right.line_num)) - }); - results - } - - fn classified_word_result(&self, word: &str, result: SearchResult) -> WordSearchResult { - let kind = self.word_occurrence_kind(word, &result); - let score = word_occurrence_score(kind, &result.path); - WordSearchResult { - path: result.path, - line_num: result.line_num, - line_text: result.line_text, - kind: kind.to_string(), - score, - } - } - - fn word_occurrence_kind(&self, word: &str, result: &SearchResult) -> &'static str { - if is_doc_path(&result.path) { - return "doc"; - } - - let semantic = if self.is_word_definition(word, result) { - "definition" - } else if is_import_line(&result.line_text) { - "import" - } else if is_export_line(&result.line_text) { - "export" - } else if is_call_like_occurrence(word, &result.line_text) { - "call" - } else { - "reference" - }; - - if is_test_like_path(&result.path) && matches!(semantic, "call" | "reference") { - "test" - } else { - semantic - } - } - - fn is_word_definition(&self, word: &str, result: &SearchResult) -> bool { - self.outlines.get(&result.path).is_some_and(|outline| { - outline.symbols.iter().any(|symbol| { - symbol.name == word - && symbol.line_start == result.line_num - && !matches!(symbol.kind, SymbolKind::Import | SymbolKind::CommentBlock) - }) - }) - } - - pub fn file_map(&self) -> Vec<(String, FileMeta)> { - let mut entries: Vec<(&String, &FileMeta)> = self.file_meta.iter().collect(); - entries.sort_by_key(|(path, _)| path.as_str()); - entries - .into_iter() - .map(|(path, meta)| (path.clone(), meta.clone())) - .collect() - } - - pub fn filtered_files( - &self, - options: &FileFilterOptions, - ) -> (Vec<(String, FileMeta)>, usize, bool) { - let language = options.language.as_ref().map(|value| value.to_lowercase()); - let path_prefix = options - .path_prefix - .as_deref() - .filter(|value| !value.is_empty()) - .map(normalize_filter_prefix); - - let mut entries = - self.file_map() - .into_iter() - .filter(|(path, meta)| { - path_prefix.as_ref().is_none_or(|prefix| { - path == prefix || path.starts_with(&format!("{prefix}/")) - }) && options - .path_glob - .as_deref() - .is_none_or(|glob| match_glob(glob, path)) - && language - .as_deref() - .is_none_or(|language| meta.language.as_str() == language) - && options - .min_lines - .is_none_or(|min_lines| meta.line_count >= min_lines) - && options - .max_lines - .is_none_or(|max_lines| meta.line_count <= max_lines) - }) - .collect::>(); - let total = entries.len(); - if let Some(max_results) = options.max_results { - entries.truncate(max_results); - } - let truncated = entries.len() < total; - (entries, total, truncated) - } - - pub fn get_imported_by(&self, path: &str) -> Vec { - self.dep_graph.get_imported_by(path) - } - - pub fn get_depends_on(&self, path: &str) -> Vec { - self.dep_graph.get_depends_on(path) - } - - pub fn get_unresolved_imports(&self, path: &str) -> Vec { - self.dep_graph.get_unresolved_imports(path) - } - - pub fn unresolved_imports(&self) -> Vec { - let mut imports = self.dep_graph.unresolved_imports(); - imports.sort_by(|a, b| { - a.path - .cmp(&b.path) - .then_with(|| a.line_start.cmp(&b.line_start)) - .then_with(|| a.import.cmp(&b.import)) - }); - imports - } - - pub fn get_transitive_imported_by(&self, path: &str) -> Vec { - self.dep_graph.get_transitive(path, true) - } - - pub fn get_transitive_depends_on(&self, path: &str) -> Vec { - self.dep_graph.get_transitive(path, false) - } - - pub fn get_hot_files(&self, limit: usize) -> Vec<(String, FileMeta)> { - let mut entries: Vec<(&String, &FileMeta)> = self.file_meta.iter().collect(); - entries.sort_by(|a, b| { - b.1.modified_ms - .cmp(&a.1.modified_ms) - .then_with(|| b.1.byte_size.cmp(&a.1.byte_size)) - .then_with(|| a.0.cmp(b.0)) - }); - entries - .into_iter() - .take(limit) - .map(|(p, m)| (p.clone(), m.clone())) - .collect() - } - - pub fn find_callers(&self, symbol_name: &str, max_results: usize) -> Vec { - let mut results = Vec::new(); - let mut seen: HashSet<(String, u32)> = HashSet::new(); - - let definitions = self.symbol_index.find_all(symbol_name); - let def_locations: HashSet<(String, u32)> = definitions - .iter() - .map(|loc| (loc.path.clone(), loc.line_start)) - .collect(); - - let occurrences = self.word_index.search(symbol_name); - - for (path, line_num) in occurrences { - if results.len() >= max_results { - break; - } - - if def_locations.contains(&(path.clone(), line_num)) { - continue; - } - - if seen.insert((path.clone(), line_num)) { - if let Some(line_text) = self.get_line(&path, line_num) { - results.push(SearchResult { - path, - line_num, - line_text, - }); - } - } - } - - results - } - - pub fn build_context_with_options(&self, task: &str, options: &ContextOptions) -> String { - let details = self.build_context_details_with_options(task, options); - render_context_details(&details) - } -} - -fn render_context_details(details: &ContextDetails) -> String { - let mut output = String::new(); - output.push_str(&format!("## Context for: {}\n\n", details.task)); - if let Some(note) = &details.note { - output.push_str(&format!("{}\n\n", note)); - } - if !details.suggested_next_steps.is_empty() { - output.push_str("### Suggested Next Steps\n\n"); - for step in &details.suggested_next_steps { - output.push_str(&format!("- {step}\n")); - } - output.push('\n'); - } - - if !details.relevant_symbols.is_empty() { - output.push_str("### Relevant Symbols\n\n"); - for sym in &details.relevant_symbols { - output.push_str(&format!( - "- {} ({}): {}:{}-{}\n", - sym.name, sym.kind, sym.path, sym.line_start, sym.line_end - )); - } - output.push('\n'); - - output.push_str("### Relevant Symbol Bodies\n\n"); - for sym in &details.relevant_symbols { - output.push_str(&format!( - "#### {}:{}-{} {}\n\n", - sym.path, sym.content_line_start, sym.content_line_end, sym.name - )); - output.push_str("```text\n"); - output.push_str(&sym.content); - if !sym.content.ends_with('\n') { - output.push('\n'); - } - output.push_str("```\n\n"); - } - } - - if !details.snippets.is_empty() { - output.push_str("### Relevant Code Snippets\n\n"); - for result in &details.snippets { - output.push_str(&format!( - "{}:{}: {}\n", - result.path, result.line_num, result.line_text - )); - } - } - - output -} - -impl Engine { - pub fn build_context_details(&self, task: &str, max_results: usize) -> ContextDetails { - self.build_context_details_with_options( - task, - &ContextOptions { - max_results, - ..ContextOptions::default() - }, - ) - } - - pub fn build_context_details_with_options( - &self, - task: &str, - options: &ContextOptions, - ) -> ContextDetails { - let keywords = context_keywords(task); - let max_results = options.max_results.max(1); - let allow_test_context = context_allows_test_context(task, options); - let relevant_symbols = - self.ranked_context_symbols(&keywords, options, 5, allow_test_context); - - let mut snippets = self.ranked_context_snippets( - &keywords, - &relevant_symbols, - options, - max_results, - allow_test_context, - ); - let confidence = context_confidence(task, &relevant_symbols, &snippets); - let low_confidence = confidence == "low"; - - ContextDetails { - task: task.to_string(), - keywords, - max_results, - confidence: confidence.to_string(), - note: low_confidence.then(|| { - "Low-confidence brief: this tool bundles context from explicit symbols, path fragments, and scoped keywords; it is not natural-language QA.".to_string() - }), - suggested_next_steps: if low_confidence { - vec![ - "Add --path-prefix or --path-glob to scope the search.".to_string(), - "Run symbol-search for likely symbol names.".to_string(), - "Run text-search for concrete terms from the task.".to_string(), - ] - } else { - Vec::new() - }, - relevant_symbols, - snippets: std::mem::take(&mut snippets), - } - } - - fn ranked_context_symbols( - &self, - keywords: &[String], - options: &ContextOptions, - max_symbols: usize, - allow_test_context: bool, - ) -> Vec { - let mut scored: Vec = Vec::new(); - let mut seen: HashSet<(String, String, SymbolKind, u32)> = HashSet::new(); - - for keyword in keywords { - for result in self.find_symbol(keyword) { - if !self.context_path_allowed(&result.path, options) { - continue; - } - push_context_symbol_candidate( - &mut scored, - &mut seen, - self.context_symbol_score(keyword, &result, keywords, allow_test_context) - + CONTEXT_INDEXED_SYMBOL_SOURCE_BONUS, - result, - ); - } - - for (path, outline) in &self.outlines { - if !self.context_path_allowed(path, options) { - continue; - } - for symbol in &outline.symbols { - if !symbol.name.eq_ignore_ascii_case(keyword) { - continue; - } - push_context_symbol_candidate( - &mut scored, - &mut seen, - self.context_symbol_score( - keyword, - &SymbolResult { - path: path.clone(), - symbol: symbol.clone(), - }, - keywords, - allow_test_context, - ) + CONTEXT_OUTLINE_SYMBOL_SOURCE_BONUS, - SymbolResult { - path: path.clone(), - symbol: symbol.clone(), - }, - ); - } - } - - if keyword.len() < 3 { - continue; - } - - for (path, path_score) in self.fuzzy_find(keyword, 5) { - if !self.context_path_allowed(&path, options) { - continue; - } - let Some(outline) = self.outlines.get(&path) else { - continue; - }; - for symbol in &outline.symbols { - if symbol.kind == SymbolKind::Import { - continue; - } - let result = SymbolResult { - path: path.clone(), - symbol: symbol.clone(), - }; - let score = - self.context_symbol_score(keyword, &result, keywords, allow_test_context) - + path_score.round() as i32; - if score < 80 { - continue; - } - push_context_symbol_candidate(&mut scored, &mut seen, score, result); - } - } - } - - let core_terms = context_core_terms(keywords); - if !core_terms.is_empty() { - for (path, outline) in &self.outlines { - if !self.context_path_allowed(path, options) { - continue; - } - for symbol in &outline.symbols { - if symbol.kind == SymbolKind::Import { - continue; - } - let result = SymbolResult { - path: path.clone(), - symbol: symbol.clone(), - }; - let score = self.context_multi_term_symbol_score( - &result, - &core_terms, - allow_test_context, - ); - if score >= 260 { - push_context_symbol_candidate(&mut scored, &mut seen, score, result); - } - } - } - } - - suppress_test_context_symbols(&mut scored, allow_test_context); - scored.sort_by(|a, b| { - b.score - .cmp(&a.score) - .then_with(|| { - context_path_rank(&a.result.path).cmp(&context_path_rank(&b.result.path)) - }) - .then_with(|| a.result.path.cmp(&b.result.path)) - .then_with(|| a.result.symbol.line_start.cmp(&b.result.symbol.line_start)) - .then_with(|| a.result.symbol.name.cmp(&b.result.symbol.name)) - }); - - scored - .into_iter() - .filter_map(|entry| { - self.context_symbol_from_result(entry.result, MAX_CONTEXT_SYMBOL_LINES) - }) - .take(max_symbols) - .collect() - } - - fn context_symbol_from_result( - &self, - result: SymbolResult, - max_lines: u32, - ) -> Option { - let (content_line_start, content_line_end, content) = - self.symbol_source_bounded(&result.path, &result.symbol, max_lines)?; - Some(ContextSymbol { - path: result.path, - name: result.symbol.name, - kind: result.symbol.kind.to_string(), - line_start: result.symbol.line_start, - line_end: result.symbol.line_end, - detail: result.symbol.detail, - content_line_start, - content_line_end, - content, - }) - } - - fn context_symbol_score( - &self, - keyword: &str, - result: &SymbolResult, - keywords: &[String], - allow_test_context: bool, - ) -> i32 { - let symbol_name = &result.symbol.name; - let symbol_norm = context_normalize(symbol_name); - let keyword_norm = context_normalize(keyword); - let path_norm = context_normalize(&result.path); - let basename_norm = result - .path - .rsplit('/') - .next() - .map(|name| name.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(name)) - .map(context_normalize) - .unwrap_or_default(); - let callable = matches!( - result.symbol.kind, - SymbolKind::Function | SymbolKind::Method - ); - let mut score = symbol_kind_context_score(result.symbol.kind); - score += context_path_score(&result.path, allow_test_context); - - if symbol_name == keyword { - score += CONTEXT_EXACT_SYMBOL_BONUS; - } else if has_identifier_case_signal(keyword) && symbol_name.eq_ignore_ascii_case(keyword) { - score += CONTEXT_CASE_INSENSITIVE_SYMBOL_BONUS; - } - - if !keyword_norm.is_empty() { - if symbol_norm == keyword_norm { - score += CONTEXT_NORMALIZED_EXACT_BONUS; - } else if symbol_norm.contains(&keyword_norm) { - score += CONTEXT_NORMALIZED_CONTAINS_BONUS; - } else if keyword_norm.len() >= 5 && keyword_norm.contains(&symbol_norm) { - score += CONTEXT_REVERSE_CONTAINS_BONUS; - } - - if callable - && symbol_norm.ends_with(&keyword_norm) - && symbol_norm.len() > keyword_norm.len() - { - score += CONTEXT_CALLABLE_SUFFIX_BONUS; - } - - if path_norm.contains(&keyword_norm) { - score += CONTEXT_PATH_KEYWORD_BONUS; - } - } - - if callable && !basename_norm.is_empty() && symbol_norm == basename_norm { - score += CONTEXT_BASENAME_CALLABLE_BONUS; - } - - if !callable - && !symbol_norm.is_empty() - && self.outlines.get(&result.path).is_some_and(|outline| { - outline.symbols.iter().any(|symbol| { - matches!(symbol.kind, SymbolKind::Function | SymbolKind::Method) - && context_normalize(&symbol.name).ends_with(&symbol_norm) - && context_normalize(&symbol.name).len() > symbol_norm.len() - }) - }) - { - score -= CONTEXT_NONCALLABLE_SHADOW_PENALTY; - } - - let mut matched_context_terms = 0; - for term in context_terms(keywords) { - if term.len() < 3 { - continue; - } - if symbol_norm.contains(&term) { - matched_context_terms += 1; - score += CONTEXT_SYMBOL_TERM_BONUS; - } - if path_norm.contains(&term) { - score += CONTEXT_PATH_TERM_BONUS; - } - } - - if callable && matched_context_terms >= 2 { - score += CONTEXT_MULTI_TERM_CALLABLE_BONUS; - if symbol_norm.starts_with("create") || symbol_norm.starts_with("use") { - score += CONTEXT_ACTION_NAME_BONUS; - } - } - - let core_terms = context_core_terms(keywords); - if !core_terms.is_empty() { - let symbol_core_matches = core_terms - .iter() - .filter(|term| symbol_norm.contains(term.as_str())) - .count(); - let path_core_matches = core_terms - .iter() - .filter(|term| path_norm.contains(term.as_str())) - .count(); - if symbol_core_matches == 0 { - score -= CONTEXT_NO_CORE_SYMBOL_PENALTY; - } else if symbol_core_matches == 1 && path_core_matches == 0 && core_terms.len() >= 3 { - score -= CONTEXT_WEAK_CORE_MATCH_PENALTY; - } else if symbol_core_matches >= 2 { - score += CONTEXT_STRONG_CORE_MATCH_BONUS; - } else if path_core_matches >= 1 { - score += CONTEXT_PATH_CORE_MATCH_BONUS; - } - if symbol_core_matches == 0 && path_core_matches < 2 { - score -= CONTEXT_POOR_CORE_PATH_PENALTY; - } else { - score += (symbol_core_matches as i32 * CONTEXT_SYMBOL_CORE_TERM_BONUS) - + (path_core_matches as i32 * CONTEXT_PATH_CORE_TERM_BONUS); - } - score += context_action_term_score(&symbol_norm, &path_norm, callable, &core_terms); - } - - if is_test_like_path(&result.path) && !allow_test_context { - score -= CONTEXT_TEST_PATH_PENALTY; - } - - score - } - - fn context_multi_term_symbol_score( - &self, - result: &SymbolResult, - terms: &[String], - allow_test_context: bool, - ) -> i32 { - let symbol_norm = context_normalize(&result.symbol.name); - let path_norm = context_normalize(&result.path); - let mut matched_symbol_terms = 0; - let mut matched_path_terms = 0; - let mut score = symbol_kind_context_score(result.symbol.kind); - score += context_path_score(&result.path, allow_test_context); - - for term in terms { - if symbol_norm.contains(term) { - matched_symbol_terms += 1; - score += CONTEXT_MULTI_TERM_SYMBOL_BONUS; - } - if path_norm.contains(term) { - matched_path_terms += 1; - score += CONTEXT_MULTI_TERM_PATH_BONUS; - } - } - - if matched_symbol_terms == 0 && matched_path_terms < 2 { - return 0; - } - if matched_symbol_terms + matched_path_terms < 2 { - return 0; - } - let is_callable = matches!( - result.symbol.kind, - SymbolKind::Function | SymbolKind::Method - ); - if is_callable { - score += CONTEXT_MULTI_TERM_CALLABLE_KIND_BONUS; - } - if symbol_norm.starts_with("create") || symbol_norm.starts_with("run") { - score += CONTEXT_MULTI_TERM_ACTION_BONUS; - } - if symbol_norm.contains("runtime") { - score += CONTEXT_MULTI_TERM_RUNTIME_BONUS; - } - score += context_action_term_score(&symbol_norm, &path_norm, is_callable, terms); - if is_test_like_path(&result.path) && !allow_test_context { - score -= CONTEXT_TEST_PATH_PENALTY; - } - score - } - - fn ranked_context_snippets( - &self, - keywords: &[String], - relevant_symbols: &[ContextSymbol], - options: &ContextOptions, - max_results: usize, - allow_test_context: bool, - ) -> Vec { - let mut scored: Vec = Vec::new(); - for (rank, symbol) in relevant_symbols.iter().enumerate() { - if !self.context_path_allowed(&symbol.path, options) { - continue; - } - if scored - .iter() - .any(|x| x.result.path == symbol.path && x.result.line_num == symbol.line_start) - { - continue; - } - let Some(line_text) = self.get_line(&symbol.path, symbol.line_start) else { - continue; - }; - let rank_penalty = (rank as i32) * CONTEXT_SNIPPET_SYMBOL_RANK_STEP; - scored.push(ScoredSearchResult { - score: CONTEXT_SNIPPET_SYMBOL_DEFINITION_BONUS.saturating_sub(rank_penalty), - result: SearchResult { - path: symbol.path.clone(), - line_num: symbol.line_start, - line_text, - }, - }); - } - - for keyword in keywords { - let per_keyword_limit = if allow_test_context { 8 } else { 24 }; - let results = self.search(keyword, per_keyword_limit); - for result in results { - if !self.context_path_allowed(&result.path, options) { - continue; - } - if scored - .iter() - .any(|x| x.result.path == result.path && x.result.line_num == result.line_num) - { - continue; - } - let score = self.context_snippet_score( - keyword, - &result, - relevant_symbols, - allow_test_context, - ); - scored.push(ScoredSearchResult { score, result }); - } - } - suppress_test_context_snippets(&mut scored, allow_test_context); - scored.sort_by(|a, b| { - b.score - .cmp(&a.score) - .then_with(|| { - context_path_rank(&a.result.path).cmp(&context_path_rank(&b.result.path)) - }) - .then_with(|| a.result.path.cmp(&b.result.path)) - .then_with(|| a.result.line_num.cmp(&b.result.line_num)) - }); - scored - .into_iter() - .take(max_results) - .map(|entry| entry.result) - .collect() - } - - fn context_path_allowed(&self, path: &str, options: &ContextOptions) -> bool { - if let Some(prefix) = options - .path_prefix - .as_deref() - .filter(|prefix| !prefix.is_empty()) - .map(normalize_filter_prefix) - { - if path != prefix && !path.starts_with(&format!("{prefix}/")) { - return false; - } - } - if let Some(glob) = options.path_glob.as_deref() { - if !match_glob(glob, path) { - return false; - } - } - if let Some(language) = options.language.as_deref() { - let language = language.to_lowercase(); - if self - .file_meta - .get(path) - .is_none_or(|meta| meta.language.as_str() != language) - { - return false; - } - } - true - } - - fn context_snippet_score( - &self, - keyword: &str, - result: &SearchResult, - relevant_symbols: &[ContextSymbol], - allow_test_context: bool, - ) -> i32 { - let keyword_lower = keyword.to_lowercase(); - let path_lower = result.path.to_lowercase(); - let line_lower = result.line_text.to_lowercase(); - let mut score = 0; - if is_source_context_path(&result.path) { - score += CONTEXT_SNIPPET_SOURCE_PATH_BONUS; - } else if is_doc_path(&result.path) { - score -= CONTEXT_SNIPPET_DOC_PATH_PENALTY; - } else if is_example_context_path(&result.path) { - score -= CONTEXT_SNIPPET_EXAMPLE_PATH_PENALTY; - } - if is_test_like_path(&result.path) && !allow_test_context { - score -= CONTEXT_SNIPPET_TEST_PATH_PENALTY; - } - - if line_lower.contains(&keyword_lower) { - score += CONTEXT_SNIPPET_LINE_MATCH_BONUS; - } - if result - .line_text - .split(|c: char| !c.is_alphanumeric() && c != '_' && c != '-') - .any(|word| word.eq_ignore_ascii_case(keyword)) - { - score += CONTEXT_SNIPPET_WORD_MATCH_BONUS; - } - if path_lower.contains(&keyword_lower) { - score += CONTEXT_SNIPPET_PATH_MATCH_BONUS; - } - if relevant_symbols - .iter() - .any(|symbol| symbol.path == result.path || symbol.name.eq_ignore_ascii_case(keyword)) - { - score += CONTEXT_SNIPPET_RELEVANT_SYMBOL_BONUS; - } - if relevant_symbols - .first() - .is_some_and(|symbol| symbol.path == result.path) - { - score += CONTEXT_SNIPPET_TOP_SYMBOL_FILE_BONUS; - } - - let language = self - .file_meta - .get(&result.path) - .map(|meta| meta.language) - .unwrap_or_else(|| detect_language(&result.path)); - if is_comment_or_blank(&result.line_text, language) { - score -= CONTEXT_SNIPPET_COMMENT_PENALTY; - } - if is_import_line(&result.line_text) { - score -= CONTEXT_SNIPPET_IMPORT_PENALTY; - } - if keyword.len() <= 3 { - score -= CONTEXT_SNIPPET_SHORT_KEYWORD_PENALTY; - } - score - } - - pub fn get_changes(&self, since_seq: u64) -> Vec<(String, u64, String)> { - let changes = self.store.changes_since_detailed(since_seq); - changes - .into_iter() - .map(|c| { - let op_str = format!("{:?}", c.op); - (c.path, c.seq, op_str) - }) - .collect() - } - - pub fn read_file( - &self, - path: &str, - line_start: Option, - line_end: Option, - ) -> Option { - let stub; - let content = if let Some(content) = self.content_for(path) { - content - } else { - let meta = self.file_meta.get(path)?; - if meta.indexed { - return None; - } - stub = unindexed_file_stub(path, meta); - &stub - }; - let lines: Vec<&str> = content.lines().collect(); - - match (line_start, line_end) { - (Some(start), Some(end)) => { - let start_idx = (start.saturating_sub(1)) as usize; - let end_idx = (end as usize).min(lines.len()); - if start_idx >= lines.len() || start_idx >= end_idx { - return Some(String::new()); - } - Some(lines[start_idx..end_idx].join("\n")) - } - (Some(start), None) => { - let start_idx = (start.saturating_sub(1)) as usize; - if start_idx >= lines.len() { - return Some(String::new()); - } - Some(lines[start_idx..].join("\n")) - } - (None, Some(end)) => { - let end_idx = (end as usize).min(lines.len()); - Some(lines[..end_idx].join("\n")) - } - (None, None) => Some(content.to_string()), - } - } - - pub fn read_file_rich( - &self, - path: &str, - line_start: Option, - line_end: Option, - compact: bool, - if_hash: Option<&str>, - ) -> Option { - let stub; - let content = if let Some(content) = self.content_for(path) { - content - } else { - let meta = self.file_meta.get(path)?; - if meta.indexed { - return None; - } - stub = unindexed_file_stub(path, meta); - &stub - }; - let hash = hash_content(content); - let hash_hex = format!("{hash:x}"); - if if_hash.is_some_and(|expected| expected.eq_ignore_ascii_case(&hash_hex)) { - return Some(ReadFileResult { - content: String::new(), - hash, - unchanged: true, - }); - } - - let mut selected = self.read_file(path, line_start, line_end)?; - if compact { - let language = self - .file_meta - .get(path) - .map(|meta| meta.language) - .unwrap_or_else(|| detect_language(path)); - selected = selected - .lines() - .filter(|line| !is_comment_or_blank(line, language)) - .collect::>() - .join("\n"); - } - - Some(ReadFileResult { - content: selected, - hash, - unchanged: false, - }) - } - - pub fn glob_files(&self, pattern: &str) -> Vec { - let mut results: Vec = self - .file_meta - .keys() - .filter(|path| match_glob(pattern, path)) - .cloned() - .collect(); - results.sort(); - results - } - - pub fn fuzzy_find(&self, pattern: &str, max_results: usize) -> Vec<(String, f32)> { - let pattern_lower = pattern.to_lowercase(); - let pattern_chars: Vec = pattern_lower.chars().collect(); - let mut results: Vec<(String, f32)> = Vec::new(); - - for path in self.file_meta.keys() { - let path_lower = path.to_lowercase(); - let filename = path_lower.rsplit('/').next().unwrap_or(&path_lower); - - let (score, matched) = if let Some(s) = fuzzy_match(&pattern_chars, filename) { - (s + 10.0, true) - } else if let Some(s) = fuzzy_match(&pattern_chars, &path_lower) { - (s, true) - } else { - (0.0, false) - }; - - if matched && score > 0.0 { - results.push((path.clone(), score)); - } - } - - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - results.truncate(max_results); - results - } - - pub fn list_dir(&self, dir: &str) -> Vec<(String, Option<&FileMeta>)> { - let prefix = if dir.is_empty() || dir == "." { - String::new() - } else { - format!("{}/", dir.trim_end_matches('/')) - }; - - let mut dirs: HashSet = HashSet::new(); - let mut files: Vec<(String, Option<&FileMeta>)> = Vec::new(); - - for (path, meta) in &self.file_meta { - if let Some(rest) = path.strip_prefix(&prefix) { - if let Some(slash_pos) = rest.find('/') { - let dir_name = &rest[..slash_pos]; - dirs.insert(dir_name.to_string()); - } else { - files.push((rest.to_string(), Some(meta))); - } - } - } - - let mut result: Vec<(String, Option<&FileMeta>)> = Vec::new(); - let mut sorted_dirs: Vec = dirs.into_iter().collect(); - sorted_dirs.sort(); - for d in sorted_dirs { - result.push((format!("{}/", d), None)); - } - files.sort_by(|a, b| a.0.cmp(&b.0)); - result.extend(files); - - result - } - - pub fn file_count(&self) -> usize { - self.file_meta.len() - } - - pub fn symbol_index_count(&self) -> usize { - self.symbol_index.symbol_count() - } - - pub fn word_index_count(&self) -> usize { - self.word_index.unique_word_count() - } - - pub fn word_index_file_count(&self) -> usize { - self.word_index.file_count() - } - - pub fn store(&self) -> &Store { - &self.store - } - - pub fn content(&self, path: &str) -> Option<&str> { - self.content_for(path) - } - - pub fn enclosing_symbol(&self, path: &str, line_num: u32) -> Option<&Symbol> { - self.outlines.get(path).and_then(|outline| { - outline - .symbols - .iter() - .filter(|sym| { - sym.kind != SymbolKind::Import - && sym.line_start <= line_num - && sym.line_end >= line_num - }) - .max_by_key(|sym| sym.line_start) - }) - } - - fn symbol_source_bounded( - &self, - path: &str, - symbol: &Symbol, - max_lines: u32, - ) -> Option<(u32, u32, String)> { - let outline = self.outlines.get(path)?; - let start = symbol.line_start.max(1); - let natural_end = symbol.line_end.min(outline.line_count); - let capped_end = start - .saturating_add(max_lines.saturating_sub(1)) - .min(natural_end) - .max(start); - self.read_file(path, Some(start), Some(capped_end)) - .map(|content| (start, capped_end, content)) - } - - fn get_line(&self, path: &str, line_num: u32) -> Option { - let content = self.content_for(path)?; - content - .lines() - .nth((line_num - 1) as usize) - .map(|s| s.to_string()) - } - - fn content_for(&self, path: &str) -> Option<&str> { - self.content_cache - .get(path) - .or_else(|| self.contents.get(path).map(String::as_str)) - } - - fn rebuild_dep_graph(&mut self) { - let outlines: Vec<(String, FileOutline)> = self - .outlines - .iter() - .map(|(path, outline)| (path.clone(), outline.clone())) - .collect(); - - self.dep_graph.clear(); - for (path, outline) in outlines { - let resolution = imports::resolve_imports( - &path, - &outline.imports, - outline.language, - &self.file_meta, - ); - let unresolved = resolution - .unresolved - .into_iter() - .map(|import| unresolved_import_record(&path, &outline, import)) - .collect(); - self.dep_graph - .set_resolution(&path, resolution.deps, unresolved); - } - } - - pub(crate) fn rebuild_dep_graph_after_batch(&mut self) { - self.rebuild_dep_graph(); - } - - pub fn to_snapshot_data(&self) -> snapshot::SnapshotDataRaw { - snapshot::SnapshotDataRaw { - outlines: self - .outlines - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - file_meta: self - .file_meta - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), - contents: self - .contents - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(), - forward_deps: self.dep_graph.forward_deps(), - } - } - - pub fn load_from_snapshot(&mut self, data: snapshot::SnapshotData) { - let raw = data.into_raw(); - - self.outlines.clear(); - self.file_meta.clear(); - self.contents.clear(); - self.content_cache.clear(); - self.symbol_index = SymbolIndex::new(); - self.trigram_index = TrigramIndex::new(); - self.word_index = WordIndex::new(); - self.dep_graph.clear(); - self.store = Store::new(); - - for (path, outline) in raw.outlines { - self.symbol_index.index_file(&outline); - self.outlines.insert(path, outline); - } - - for (path, meta) in raw.file_meta { - self.file_meta.insert(path, meta); - } - - for (path, content) in raw.contents { - self.trigram_index.index_file(&path, &content); - self.word_index.index_file(&path, &content); - self.content_cache.put(path.clone(), content.clone()); - self.contents.insert(path, content); - } - - let _ = raw.forward_deps; - self.rebuild_dep_graph(); - } - - pub fn index_project(&mut self, root: impl AsRef) -> usize { - let root = root.as_ref(); - let files = crate::walker::walk_project_meta(root); - let count = files.len(); - - for file in &files { - if file.indexable { - let abs_path = root.join(&file.path); - match std::fs::read_to_string(&abs_path) { - Ok(content) => self.index_file_with_op( - &file.path, - &content, - file.modified_ms, - Op::Snapshot, - false, - ), - Err(_) => self.index_file_meta_only_no_rebuild( - &file.path, - file.byte_size, - file.modified_ms, - ), - } - } else { - self.index_file_meta_only_no_rebuild(&file.path, file.byte_size, file.modified_ms); - } - } - self.rebuild_dep_graph(); - - count - } - - fn index_file_meta_only_no_rebuild(&mut self, path: &str, byte_size: u64, modified_ms: u64) { - self.outlines.remove(path); - self.contents.remove(path); - self.content_cache.remove(path); - self.symbol_index.remove_file(path); - self.trigram_index.remove_file(path); - self.word_index.remove_file(path); - self.file_meta.insert( - path.to_string(), - FileMeta { - language: detect_language(path), - line_count: 0, - byte_size, - symbol_count: 0, - modified_ms, - indexed: false, - }, - ); - self.store.record_snapshot( - path, - byte_size, - hash_content(&format!("{path}\0{byte_size}\0{modified_ms}")), - ); - } -} +pub use shared::is_comment_or_blank; pub fn hash_content(content: &str) -> u64 { let mut hash: u64 = 14695981039346656037; @@ -1818,694 +23,6 @@ pub fn hash_content(content: &str) -> u64 { hash } -fn unindexed_file_stub(path: &str, meta: &FileMeta) -> String { - let kind = path - .rsplit_once('.') - .map(|(_, ext)| ext) - .filter(|ext| !ext.is_empty()) - .unwrap_or("unknown"); - format!( - "unindexed {kind} file: {} bytes\npath: {path}\nmodified_ms: {}\n", - meta.byte_size, meta.modified_ms - ) -} - -fn push_context_symbol_candidate( - scored: &mut Vec, - seen: &mut HashSet<(String, String, SymbolKind, u32)>, - score: i32, - result: SymbolResult, -) { - if score <= 0 { - return; - } - let key = ( - result.path.clone(), - result.symbol.name.clone(), - result.symbol.kind, - result.symbol.line_start, - ); - if let Some(existing) = scored.iter_mut().find(|existing| { - existing.result.path == key.0 - && existing.result.symbol.name == key.1 - && existing.result.symbol.kind == key.2 - && existing.result.symbol.line_start == key.3 - }) { - if score > existing.score { - existing.score = score; - existing.result = result; - } - return; - } - if seen.insert(key) { - scored.push(ScoredContextSymbol { score, result }); - } -} - -fn suppress_test_context_symbols(scored: &mut Vec, allow_test_context: bool) { - if allow_test_context - || !scored - .iter() - .any(|entry| !is_test_like_path(&entry.result.path)) - { - return; - } - scored.retain(|entry| !is_test_like_path(&entry.result.path)); -} - -fn suppress_test_context_snippets(scored: &mut Vec, allow_test_context: bool) { - if allow_test_context - || !scored - .iter() - .any(|entry| !is_test_like_path(&entry.result.path)) - { - return; - } - scored.retain(|entry| !is_test_like_path(&entry.result.path)); -} - -fn context_keywords(task: &str) -> Vec { - let mut keywords = Vec::new(); - let mut seen = HashSet::new(); - - for quoted in quoted_segments(task) { - add_context_keyword_variants(&mut keywords, &mut seen, "ed); - } - - let tokens = context_tokens(task); - for token in &tokens { - if is_identifier_like_context_token(token) { - add_context_keyword_variants(&mut keywords, &mut seen, token); - } - } - - for window_size in 2..=3 { - for window in tokens.windows(window_size) { - if window - .iter() - .all(|token| context_normalize(token).len() >= 3) - { - add_context_phrase_variants(&mut keywords, &mut seen, window); - } - } - } - - for token in &tokens { - if is_context_content_token(token) { - add_context_keyword_variants(&mut keywords, &mut seen, token); - } - } - - if keywords.is_empty() { - for token in &tokens { - if context_normalize(token).len() >= 3 { - add_context_keyword_variants(&mut keywords, &mut seen, token); - } - } - } - - keywords -} - -fn context_query_is_explicit(task: &str) -> bool { - !quoted_segments(task).is_empty() - || context_tokens(task).into_iter().any(|token| { - token.contains(['_', '-', '/', '.', ':']) - || has_lower_to_upper_transition(&token) - || token.chars().any(|ch| ch.is_ascii_digit()) - }) -} - -fn context_confidence( - task: &str, - relevant_symbols: &[ContextSymbol], - snippets: &[SearchResult], -) -> &'static str { - let has_source_symbol = relevant_symbols - .iter() - .any(|symbol| is_source_context_path(&symbol.path)); - let has_source_snippet = snippets - .iter() - .any(|snippet| is_source_context_path(&snippet.path)); - - if context_query_is_explicit(task) && has_source_symbol { - "high" - } else if has_source_symbol || has_source_snippet { - "medium" - } else { - "low" - } -} - -fn context_allows_test_context(task: &str, options: &ContextOptions) -> bool { - context_task_mentions_test_context(task) - || options - .path_prefix - .as_deref() - .is_some_and(context_filter_targets_test_context) - || options - .path_glob - .as_deref() - .is_some_and(context_filter_targets_test_context) -} - -fn context_task_mentions_test_context(task: &str) -> bool { - context_tokens(task).into_iter().any(|token| { - matches!( - context_normalize(&token).as_str(), - "test" | "tests" | "testing" | "spec" | "specs" - ) - }) -} - -fn context_filter_targets_test_context(value: &str) -> bool { - let lowered = value.to_ascii_lowercase(); - let normalized = lowered.trim_matches('/'); - is_test_like_path(normalized) - || normalized.split('/').any(|segment| { - let segment = - segment.trim_matches(|ch| matches!(ch, '*' | '?' | '[' | ']' | '{' | '}')); - matches!( - segment, - "test" | "tests" | "__tests__" | "spec" | "specs" | "__specs__" - ) - }) -} - -fn is_context_content_token(token: &str) -> bool { - let normalized = context_normalize(token); - normalized.len() >= 3 && !is_low_signal_context_term(&normalized) -} - -fn quoted_segments(text: &str) -> Vec { - let mut segments = Vec::new(); - let mut chars = text.char_indices().peekable(); - while let Some((_, ch)) = chars.next() { - if !matches!(ch, '"' | '\'' | '`') { - continue; - } - let quote = ch; - let start = chars.peek().map(|(idx, _)| *idx).unwrap_or(text.len()); - for (end, current) in chars.by_ref() { - if current == quote { - if end > start { - segments.push(text[start..end].to_string()); - } - break; - } - } - } - segments -} - -fn context_tokens(task: &str) -> Vec { - task.split_whitespace() - .map(|word| { - word.trim_matches(|c: char| { - !c.is_alphanumeric() && !matches!(c, '_' | '-' | '/' | '.' | ':') - }) - .to_string() - }) - .filter(|word| !word.is_empty()) - .collect() -} - -fn add_context_keyword_variants( - keywords: &mut Vec, - seen: &mut HashSet, - keyword: &str, -) { - let keyword = keyword.trim(); - if keyword.is_empty() { - return; - } - - push_context_keyword(keywords, seen, keyword.to_string()); - - let normalized = context_normalize(keyword); - if normalized.len() >= 3 { - push_context_keyword(keywords, seen, normalized.clone()); - if let Some(singular) = singular_context_term(&normalized) { - push_context_keyword(keywords, seen, singular); - } - } - - if keyword.contains(['-', '_', '/', '.', ':']) { - let separator_normalized = keyword - .replace(['/', '.', ':'], "-") - .replace('_', "-") - .trim_matches('-') - .to_string(); - if !separator_normalized.is_empty() { - push_context_keyword(keywords, seen, separator_normalized); - } - } -} - -fn add_context_phrase_variants( - keywords: &mut Vec, - seen: &mut HashSet, - terms: &[String], -) { - if terms.is_empty() { - return; - } - - let joined_dash = terms.join("-"); - let joined_underscore = terms.join("_"); - let joined_space = terms.join(" "); - push_context_keyword(keywords, seen, joined_dash.clone()); - push_context_keyword(keywords, seen, joined_underscore); - push_context_keyword(keywords, seen, joined_space); - - let singular_terms = terms - .iter() - .map(|term| singular_context_term(term).unwrap_or_else(|| term.clone())) - .collect::>(); - if singular_terms != terms { - push_context_keyword(keywords, seen, singular_terms.join("-")); - push_context_keyword(keywords, seen, singular_terms.join("_")); - push_context_keyword(keywords, seen, singular_terms.join("")); - } - - let joined_normalized = context_normalize(&joined_dash); - if joined_normalized.len() >= 3 { - push_context_keyword(keywords, seen, joined_normalized); - } -} - -fn push_context_keyword(keywords: &mut Vec, seen: &mut HashSet, keyword: String) { - if keyword.len() < 3 { - return; - } - let key = keyword.to_lowercase(); - if seen.insert(key) { - keywords.push(keyword); - } -} - -fn is_identifier_like_context_token(token: &str) -> bool { - token.contains(['_', '-', '/', '.', ':']) - || has_lower_to_upper_transition(token) - || token - .chars() - .filter(|ch| ch.is_ascii_alphabetic()) - .take(8) - .count() - >= 3 - && token.chars().all(|ch| { - !ch.is_ascii_alphabetic() || ch.is_ascii_uppercase() || ch.is_ascii_digit() - }) -} - -fn has_lower_to_upper_transition(token: &str) -> bool { - let mut previous_lower = false; - for ch in token.chars() { - if previous_lower && ch.is_ascii_uppercase() { - return true; - } - previous_lower = ch.is_ascii_lowercase(); - } - false -} - -fn has_identifier_case_signal(token: &str) -> bool { - has_lower_to_upper_transition(token) - || token - .chars() - .any(|ch| ch.is_ascii_uppercase() || matches!(ch, '_' | '-' | '/' | '.' | ':')) -} - -fn context_terms(keywords: &[String]) -> Vec { - let mut terms = Vec::new(); - let mut seen = HashSet::new(); - for keyword in keywords { - let normalized = context_normalize(keyword); - if normalized.len() >= 3 && seen.insert(normalized.clone()) { - terms.push(normalized.clone()); - } - if let Some(singular) = singular_context_term(&normalized) { - if singular.len() >= 3 && seen.insert(singular.clone()) { - terms.push(singular); - } - } - - for raw_term in keyword.split(|ch: char| !ch.is_alphanumeric()) { - let term = context_normalize(raw_term); - if term.len() >= 3 && seen.insert(term.clone()) { - terms.push(term.clone()); - } - if let Some(singular) = singular_context_term(&term) { - if singular.len() >= 3 && seen.insert(singular.clone()) { - terms.push(singular); - } - } - } - } - terms -} - -fn context_core_terms(keywords: &[String]) -> Vec { - context_terms(keywords) - .into_iter() - .filter(|term| (term.len() >= 4 || term == "run") && !is_low_signal_context_term(term)) - .collect() -} - -fn is_low_signal_context_term(term: &str) -> bool { - matches!( - term, - "what" - | "when" - | "where" - | "which" - | "with" - | "this" - | "that" - | "from" - | "into" - | "does" - | "work" - | "works" - | "look" - | "find" - | "show" - | "how" - | "why" - | "the" - | "and" - | "for" - | "application" - ) -} - -fn context_action_term_score( - symbol_norm: &str, - path_norm: &str, - callable: bool, - terms: &[String], -) -> i32 { - let has_action_term = terms - .iter() - .any(|term| matches!(term.as_str(), "create" | "build" | "make" | "use" | "run")); - if !has_action_term { - return 0; - } - - if terms - .iter() - .filter(|term| matches!(term.as_str(), "create" | "build" | "make" | "use" | "run")) - .any(|term| symbol_norm.contains(term) || path_norm.contains(term)) - { - if callable { - CONTEXT_ACTION_TERM_MATCH_BONUS - } else { - CONTEXT_ACTION_TERM_MATCH_BONUS / 2 - } - } else { - -CONTEXT_MISSING_ACTION_TERM_PENALTY - } -} - -fn context_normalize(value: &str) -> String { - value - .chars() - .filter(|ch| ch.is_alphanumeric()) - .flat_map(char::to_lowercase) - .collect() -} - -fn singular_context_term(term: &str) -> Option { - if term.len() > 3 && term.ends_with('s') { - Some(term.trim_end_matches('s').to_string()) - } else { - None - } -} - -fn symbol_kind_context_score(kind: SymbolKind) -> i32 { - match kind { - SymbolKind::Function | SymbolKind::Method => 70, - SymbolKind::StructDef - | SymbolKind::ClassDef - | SymbolKind::InterfaceDef - | SymbolKind::TraitDef - | SymbolKind::ImplBlock - | SymbolKind::UnionDef - | SymbolKind::TypeAlias => 55, - SymbolKind::EnumDef | SymbolKind::Module | SymbolKind::MacroDef => 40, - SymbolKind::Constant | SymbolKind::Variable => 25, - SymbolKind::TestDecl => 10, - SymbolKind::CommentBlock => -20, - SymbolKind::Import => -100, - } -} - -fn is_test_like_path(path: &str) -> bool { - let path = path.to_ascii_lowercase(); - let file_name = path.rsplit('/').next().unwrap_or(&path); - path.split('/').any(|segment| { - matches!( - segment, - "test" | "tests" | "__tests__" | "spec" | "specs" | "__specs__" - ) - }) || file_name.ends_with("_test.rs") - || file_name.contains(".test.") - || file_name.contains(".spec.") -} - -fn is_source_context_path(path: &str) -> bool { - path.starts_with("src/") - || (path.starts_with("packages/") && path.contains("/src/")) - || (path.starts_with("apps/") && path.contains("/src/")) -} - -fn is_example_context_path(path: &str) -> bool { - path.starts_with("examples/") || path.contains("/examples/") -} - -fn context_path_score(path: &str, allow_test_context: bool) -> i32 { - if is_source_context_path(path) { - CONTEXT_SOURCE_PATH_BONUS - } else if is_doc_path(path) { - -CONTEXT_DOC_PATH_PENALTY - } else if is_example_context_path(path) { - -CONTEXT_EXAMPLE_PATH_PENALTY - } else if is_test_like_path(path) && !allow_test_context { - -CONTEXT_TEST_PATH_PENALTY - } else { - 0 - } -} - -fn context_path_rank(path: &str) -> u8 { - if is_source_context_path(path) { - 0 - } else if is_test_like_path(path) { - 4 - } else if is_example_context_path(path) { - 5 - } else if is_doc_path(path) { - 6 - } else if path.starts_with("packages/") || path.starts_with("apps/") { - 1 - } else { - 3 - } -} - -fn now_ms() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -fn matches_path_glob(pattern: &str, path: &str) -> bool { - if match_glob(pattern, path) { - return true; - } - if !pattern.contains('/') { - return match_glob(&format!("**/{pattern}"), path); - } - false -} - -fn word_result_path_rank(path: &str) -> u8 { - if is_test_like_path(path) { - 5 - } else if path.starts_with("packages/") && path.contains("/src/") { - 0 - } else if path.starts_with("apps/") && path.contains("/src/") { - 1 - } else if path.starts_with("src/") { - 2 - } else if path.starts_with("packages/") || path.starts_with("apps/") { - 3 - } else if path.starts_with("examples/") { - 4 - } else if is_doc_path(path) { - 6 - } else { - 4 - } -} - -fn word_occurrence_score(kind: &str, path: &str) -> i32 { - occurrence_kind_score(kind) + word_path_score(path) -} - -fn occurrence_kind_score(kind: &str) -> i32 { - match kind { - "definition" => 100, - "export" => 90, - "import" => 75, - "call" => 65, - "reference" => 50, - "test" => 30, - "doc" => 10, - _ => 0, - } -} - -fn word_path_score(path: &str) -> i32 { - match word_result_path_rank(path) { - 0 => 20, - 1 => 16, - 2 => 14, - 3 => 8, - 4 => 0, - 5 => -40, - 6 => -30, - _ => 0, - } -} - -fn is_import_line(line: &str) -> bool { - let trimmed = line.trim_start(); - trimmed.starts_with("import ") - || trimmed.starts_with("import type ") - || trimmed.starts_with("use ") - || trimmed.starts_with("from ") -} - -fn is_export_line(line: &str) -> bool { - let trimmed = line.trim_start(); - trimmed.starts_with("export ") -} - -fn is_call_like_occurrence(word: &str, line: &str) -> bool { - let patterns = [ - format!("{word}("), - format!("new {word}("), - format!("{word}::"), - format!(".{word}("), - ]; - patterns.iter().any(|pattern| line.contains(pattern)) -} - -fn is_doc_path(path: &str) -> bool { - path.starts_with("docs/") - || path.eq_ignore_ascii_case("readme.md") - || path.ends_with(".md") - || path.ends_with(".mdx") -} - -pub fn is_comment_or_blank(line: &str, language: Language) -> bool { - let trimmed = line.trim(); - if trimmed.is_empty() { - return true; - } - - match language { - Language::Python - | Language::Ruby - | Language::R - | Language::Shell - | Language::Hcl - | Language::Yaml => trimmed.starts_with('#'), - Language::Sql => trimmed.starts_with("--"), - Language::Css | Language::Scss => { - trimmed.starts_with("/*") || trimmed.starts_with('*') || trimmed.ends_with("*/") - } - Language::Markdown => trimmed.starts_with("\n"; const END: &str = "\n"; @@ -16,10 +17,20 @@ struct ToolSpec { input_schema: Value, } +#[derive(Debug, Deserialize)] +struct CriterionEstimates { + mean: CriterionMean, +} + +#[derive(Debug, Deserialize)] +struct CriterionMean { + point_estimate: f64, +} + fn main() -> Result<()> { let mut args = std::env::args().skip(1); let Some(command) = args.next() else { - bail!("usage: xtask gen-skill [--check]"); + bail!("usage: xtask "); }; match command.as_str() { @@ -28,10 +39,150 @@ fn main() -> Result<()> { [flag] if flag == "--check" => gen_skill(true), _ => bail!("usage: xtask gen-skill [--check]"), }, + "perf-gate" => match args.collect::>().as_slice() { + [] => perf_gate(), + _ => bail!("usage: xtask perf-gate"), + }, other => bail!("unknown xtask command: {other}"), } } +fn perf_gate() -> Result<()> { + const BASELINE_REF: &str = "v0.9.0"; + const REQUIRED_IMPROVEMENT: f64 = 0.20; + const BENCH_FILTER: &str = "project_index/500|search/exact_word"; + const METRICS: [(&str, &str); 2] = [ + ("500-file indexing", "project_index/500"), + ("warm exact search", "search/exact_word"), + ]; + + let repo_root = find_repo_root()?; + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + let temp_root = + std::env::temp_dir().join(format!("lexa-perf-gate-{}-{nonce}", std::process::id())); + let baseline_root = temp_root.join("baseline"); + let target_dir = temp_root.join("target"); + std::fs::create_dir_all(&temp_root) + .with_context(|| format!("failed to create {}", temp_root.display()))?; + + let result = (|| -> Result<()> { + run_command( + Command::new("git") + .args(["worktree", "add", "--detach"]) + .arg(&baseline_root) + .arg(BASELINE_REF) + .current_dir(&repo_root), + "create v0.9 performance worktree", + )?; + + run_bench(&baseline_root, &target_dir, BENCH_FILTER)?; + let baseline = METRICS + .iter() + .map(|(_, path)| read_criterion_mean(&target_dir, path)) + .collect::>>()?; + + run_bench(&repo_root, &target_dir, BENCH_FILTER)?; + let current = METRICS + .iter() + .map(|(_, path)| read_criterion_mean(&target_dir, path)) + .collect::>>()?; + + let mut failed = false; + for (((label, _), baseline_ns), current_ns) in METRICS.iter().zip(baseline).zip(current) { + let improvement = 1.0 - current_ns / baseline_ns; + let passed = improvement >= REQUIRED_IMPROVEMENT; + println!( + "{label}: v0.9 {}, current {}, improvement {:.1}% [{}]", + format_duration_ns(baseline_ns), + format_duration_ns(current_ns), + improvement * 100.0, + if passed { "PASS" } else { "FAIL" } + ); + failed |= !passed; + } + + if failed { + bail!( + "performance gate failed: every metric must improve by at least {:.0}% over {BASELINE_REF}", + REQUIRED_IMPROVEMENT * 100.0 + ); + } + Ok(()) + })(); + + let cleanup = Command::new("git") + .args(["worktree", "remove"]) + .arg(&baseline_root) + .current_dir(&repo_root) + .status(); + let cleanup_failed = match cleanup { + Ok(status) => !status.success(), + Err(_) => true, + }; + if baseline_root.exists() && cleanup_failed { + eprintln!( + "warning: failed to remove temporary worktree {}", + baseline_root.display() + ); + } + if let Err(err) = std::fs::remove_dir_all(&temp_root) { + eprintln!( + "warning: failed to remove temporary benchmark data {}: {err}", + temp_root.display() + ); + } + + result +} + +fn run_bench(repo_root: &Path, target_dir: &Path, filter: &str) -> Result<()> { + run_command( + Command::new("cargo") + .args(["bench", "--bench", "engine", "--"]) + .arg(filter) + .arg("--noplot") + .env("CARGO_TARGET_DIR", target_dir) + .current_dir(repo_root), + &format!("run performance benchmarks in {}", repo_root.display()), + ) +} + +fn run_command(command: &mut Command, description: &str) -> Result<()> { + let status = command + .status() + .with_context(|| format!("failed to {description}"))?; + if !status.success() { + bail!("{description} failed with {status}"); + } + Ok(()) +} + +fn read_criterion_mean(target_dir: &Path, benchmark: &str) -> Result { + let path = target_dir + .join("criterion") + .join(benchmark) + .join("new") + .join("estimates.json"); + let encoded = std::fs::read(&path) + .with_context(|| format!("failed to read benchmark result {}", path.display()))?; + let estimates: CriterionEstimates = serde_json::from_slice(&encoded) + .with_context(|| format!("failed to parse benchmark result {}", path.display()))?; + Ok(estimates.mean.point_estimate) +} + +fn format_duration_ns(nanoseconds: f64) -> String { + if nanoseconds >= 1_000_000.0 { + format!("{:.2} ms", nanoseconds / 1_000_000.0) + } else if nanoseconds >= 1_000.0 { + format!("{:.2} µs", nanoseconds / 1_000.0) + } else { + format!("{nanoseconds:.2} ns") + } +} + fn gen_skill(check: bool) -> Result<()> { let repo_root = find_repo_root()?; let specs = load_tool_specs(&repo_root)?;