diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2e4d5f23..471927a6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -41,18 +41,8 @@ fixes). Newest entries at the bottom. comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe). - Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section. -## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386) - -- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling - back to a full rebuild (any modified or removed file, since HNSW only - supports add-only incremental updates) rebuilt the index from only the - changed files instead of the full corpus, silently dropping every untouched - file. Root cause: the fallback reused `all_texts` from the incremental path - (which intentionally loads only the delta) instead of reloading the full - corpus via `load_documents(docs_paths, ...)`. -- This was fixed on `main` in commit `956beb5` (merged via #279) but never - reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to - 0.3.8 to ship the fix. -- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an - HNSW index, modifies one file, and asserts untouched files are still present - in the rebuilt corpus. +## 2026-07-27: Bounded `leann list` index discovery + +- Bound App-format metadata discovery to a configurable directory depth; `leann list` defaults to depth 5 and accepts `--max-depth` for deeper projects. +- Prune common dependency, virtual-environment, cache, and system directories during discovery. +- Preserve discovery of registered CLI-format and App-format projects without recursively scanning an entire home directory. diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py index 967342c0..7b6adf3b 100644 --- a/packages/leann-core/src/leann/cli.py +++ b/packages/leann-core/src/leann/cli.py @@ -19,7 +19,7 @@ from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcher from .embedding_server_manager import EmbeddingServerManager from .interactive_utils import create_cli_session -from .registry import register_project_directory +from .registry import DEFAULT_INDEX_SCAN_DEPTH, iter_index_meta_files, register_project_directory from .settings import ( resolve_anthropic_base_url, resolve_atlascloud_api_key, @@ -33,6 +33,13 @@ from .sync import DEFAULT_INDEX_EXTENSIONS, FileSynchronizer, parse_include_extensions +def _non_negative_int(value: str) -> int: + parsed_value = int(value) + if parsed_value < 0: + raise argparse.ArgumentTypeError("must be non-negative") + return parsed_value + + def _default_embedding_model() -> str: """Pick a sensible default embedding model based on platform. @@ -856,7 +863,16 @@ def _add_index_args(p, default_name): ) # List command - subparsers.add_parser("list", help="List all indexes") + list_parser = subparsers.add_parser("list", help="List all indexes") + list_parser.add_argument( + "--max-depth", + type=_non_negative_int, + default=DEFAULT_INDEX_SCAN_DEPTH, + help=( + "Maximum directory depth for app index discovery " + f"(default: {DEFAULT_INDEX_SCAN_DEPTH})" + ), + ) # Remove command remove_parser = subparsers.add_parser("remove", help="Remove an index") @@ -947,7 +963,24 @@ def _is_git_submodule(self, path: Path) -> bool: # If anything goes wrong, assume it's not a submodule return False - def list_indexes(self): + @staticmethod + def _project_has_discoverable_indexes(project_path: Path, max_depth: int) -> bool: + """Return whether a project contains an index within the scan boundary.""" + cli_indexes_dir = project_path / ".leann" / "indexes" + return cli_indexes_dir.exists() or any( + iter_index_meta_files(project_path, max_depth=max_depth) + ) + + @staticmethod + def _nested_project_dirs(project_path: Path, projects: list[Path]) -> list[Path]: + """Return registered project roots nested below a project.""" + return [ + candidate + for candidate in projects + if candidate != project_path and candidate.is_relative_to(project_path) + ] + + def list_indexes(self, max_depth: int = DEFAULT_INDEX_SCAN_DEPTH): # Get all project directories with .leann global_registry = Path.home() / ".leann" / "projects.json" all_projects = [] @@ -961,16 +994,21 @@ def list_indexes(self): except Exception: pass - # Filter to only existing directories with .leann + # Filter to existing projects with a discoverable CLI- or App-format index. valid_projects = [] for project_dir in all_projects: project_path = Path(project_dir) - if project_path.exists() and (project_path / ".leann" / "indexes").exists(): + if project_path.exists() and self._project_has_discoverable_indexes( + project_path, max_depth + ): valid_projects.append(project_path) - # Add current project if it has .leann but not in registry + # Add current project if it has a discoverable index but is not in registry. current_path = Path.cwd() - if (current_path / ".leann" / "indexes").exists() and current_path not in valid_projects: + if ( + self._project_has_discoverable_indexes(current_path, max_depth) + and current_path not in valid_projects + ): valid_projects.append(current_path) # Separate current and other projects @@ -980,6 +1018,10 @@ def list_indexes(self): if project_path != current_path: other_projects.append(project_path) + # Exclude only registered projects nested below the current project. + # Registered ancestors must not hide indexes owned by the current directory. + current_exclude_dirs = self._nested_project_dirs(current_path, valid_projects) + print("📚 LEANN Indexes") print("=" * 50) @@ -992,7 +1034,7 @@ def list_indexes(self): print(" " + "─" * 45) current_indexes = self._discover_indexes_in_project( - current_path, exclude_dirs=other_projects + current_path, exclude_dirs=current_exclude_dirs, max_depth=max_depth ) if current_indexes: for idx in current_indexes: @@ -1011,7 +1053,10 @@ def list_indexes(self): print(" " + "─" * 45) for project_path in other_projects: - project_indexes = self._discover_indexes_in_project(project_path) + nested_projects = self._nested_project_dirs(project_path, valid_projects) + project_indexes = self._discover_indexes_in_project( + project_path, exclude_dirs=nested_projects, max_depth=max_depth + ) if not project_indexes: continue @@ -1035,9 +1080,14 @@ def list_indexes(self): projects_count = 0 for p in valid_projects: if p == current_path: - discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects) + discovered = self._discover_indexes_in_project( + p, exclude_dirs=current_exclude_dirs, max_depth=max_depth + ) else: - discovered = self._discover_indexes_in_project(p) + nested_projects = self._nested_project_dirs(p, valid_projects) + discovered = self._discover_indexes_in_project( + p, exclude_dirs=nested_projects, max_depth=max_depth + ) if len(discovered) > 0: projects_count += 1 print(f"📊 Total: {total_indexes} indexes across {projects_count} projects") @@ -1057,13 +1107,18 @@ def list_indexes(self): print(" leann build my-docs --docs ./documents") def _discover_indexes_in_project( - self, project_path: Path, exclude_dirs: Optional[list[Path]] = None + self, + project_path: Path, + exclude_dirs: Optional[list[Path]] = None, + max_depth: Optional[int] = None, ): """Discover all indexes in a project directory (both CLI and apps formats) exclude_dirs: when provided, skip any APP-format index files that are located under these directories. This prevents duplicates when the current project is a parent directory of other registered projects. + max_depth: maximum number of directories to descend for APP-format indexes. + None preserves full-depth discovery for non-CLI callers. """ indexes = [] exclude_dirs = exclude_dirs or [] @@ -1100,9 +1155,9 @@ def _discover_indexes_in_project( } ) - # 2. Apps format: *.leann.meta.json files anywhere in the project + # 2. Apps format: *.leann.meta.json files within the configured scan depth cli_indexes_dir = project_path / ".leann" / "indexes" - for meta_file in project_path.rglob("*.leann.meta.json"): + for meta_file in iter_index_meta_files(project_path, max_depth=max_depth): if meta_file.is_file(): # Skip CLI-built indexes (which store meta under .leann/indexes//) try: @@ -3732,7 +3787,7 @@ async def run(self, args=None): suppress = not getattr(args, "verbose", False) if args.command == "list": - self.list_indexes() + self.list_indexes(args.max_depth) elif args.command == "remove": self.remove_index(args.index_name, args.force) elif args.command == "build": diff --git a/packages/leann-core/src/leann/registry.py b/packages/leann-core/src/leann/registry.py index 2b74a203..b950ee65 100644 --- a/packages/leann-core/src/leann/registry.py +++ b/packages/leann-core/src/leann/registry.py @@ -4,6 +4,8 @@ import importlib.metadata import json import logging +import os +from collections.abc import Iterator from pathlib import Path from typing import TYPE_CHECKING, Optional, Union @@ -14,6 +16,35 @@ logger = logging.getLogger(__name__) BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {} +DEFAULT_INDEX_SCAN_DEPTH = 5 +INDEX_SCAN_SKIP_DIRS = frozenset( + {".git", ".venv", "venv", "node_modules", "__pycache__", "Library"} +) + + +def iter_index_meta_files( + root: Union[str, Path], max_depth: Optional[int] = DEFAULT_INDEX_SCAN_DEPTH +) -> Iterator[Path]: + """Yield LEANN metadata files within an optionally bounded directory tree. + + The root directory is depth zero. A max_depth of None scans all depths. + Known dependency, cache, and system directories are always pruned before traversal. + """ + if max_depth is not None and max_depth < 0: + raise ValueError("max_depth must be non-negative") + + root_path = Path(root) + for current_dir, dirnames, filenames in os.walk(root_path): + current_path = Path(current_dir) + depth = len(current_path.relative_to(root_path).parts) + if max_depth is not None and depth >= max_depth: + dirnames.clear() + else: + dirnames[:] = [name for name in dirnames if name not in INDEX_SCAN_SKIP_DIRS] + + for filename in filenames: + if filename.endswith(".leann.meta.json"): + yield current_path / filename def register_backend(name: str): @@ -49,7 +80,10 @@ def autodiscover_backends(): # print("INFO: Backend auto-discovery finished.") -def register_project_directory(project_dir: Optional[Union[str, Path]] = None): +def register_project_directory( + project_dir: Optional[Union[str, Path]] = None, + max_depth: Optional[int] = None, +): """ Register a project directory in the global LEANN registry. @@ -57,6 +91,8 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None): Args: project_dir: Directory to register. If None, uses current working directory. + max_depth: Maximum directory depth used when looking for App-format indexes. + None preserves full-depth discovery for existing API callers. """ if project_dir is None: project_dir = Path.cwd() @@ -64,9 +100,9 @@ def register_project_directory(project_dir: Optional[Union[str, Path]] = None): project_dir = Path(project_dir) # Only register directories that have some kind of LEANN content. - # Check CLI-format first to avoid an expensive rglob on large directories. + # Check CLI-format first to avoid even a bounded scan when it is unnecessary. has_cli_indexes = (project_dir / ".leann" / "indexes").exists() - if not has_cli_indexes and not any(project_dir.rglob("*.leann.meta.json")): + if not has_cli_indexes and not any(iter_index_meta_files(project_dir, max_depth=max_depth)): # Don't register if there are no LEANN indexes return diff --git a/tests/test_cli_list_performance.py b/tests/test_cli_list_performance.py new file mode 100644 index 00000000..de93fabc --- /dev/null +++ b/tests/test_cli_list_performance.py @@ -0,0 +1,220 @@ +import asyncio +import json +import sys + +import pytest +from leann import registry +from leann.cli import LeannCLI +from leann.registry import iter_index_meta_files + + +def test_index_discovery_finds_metadata_within_default_depth(tmp_path): + meta_file = tmp_path / "project" / "data" / "sample.leann.meta.json" + meta_file.parent.mkdir(parents=True) + meta_file.touch() + + assert list(iter_index_meta_files(tmp_path)) == [meta_file] + + +def test_index_discovery_does_not_descend_past_max_depth(tmp_path): + shallow_meta = tmp_path / "one" / "shallow.leann.meta.json" + shallow_meta.parent.mkdir() + shallow_meta.touch() + deep_meta = tmp_path / "one" / "two" / "deep.leann.meta.json" + deep_meta.parent.mkdir() + deep_meta.touch() + + assert list(iter_index_meta_files(tmp_path, max_depth=1)) == [shallow_meta] + + +@pytest.mark.parametrize("excluded_dir", [".git", ".venv", "node_modules", "Library"]) +def test_index_discovery_skips_large_irrelevant_directories(tmp_path, excluded_dir): + hidden_meta = tmp_path / excluded_dir / "hidden.leann.meta.json" + hidden_meta.parent.mkdir() + hidden_meta.touch() + + assert list(iter_index_meta_files(tmp_path)) == [] + + +def test_index_discovery_rejects_negative_max_depth(tmp_path): + with pytest.raises(ValueError, match="max_depth must be non-negative"): + list(iter_index_meta_files(tmp_path, max_depth=-1)) + + +def test_list_parser_accepts_custom_max_depth(): + args = LeannCLI().create_parser().parse_args(["list", "--max-depth", "5"]) + + assert args.max_depth == 5 + + +def test_list_parser_rejects_negative_max_depth(capsys): + with pytest.raises(SystemExit): + LeannCLI().create_parser().parse_args(["list", "--max-depth", "-1"]) + + assert "must be non-negative" in capsys.readouterr().err + + +def test_cli_project_discovery_respects_max_depth(tmp_path): + shallow_meta = tmp_path / "shallow" / "index.leann.meta.json" + shallow_meta.parent.mkdir() + shallow_meta.touch() + deep_meta = tmp_path / "one" / "two" / "index.leann.meta.json" + deep_meta.parent.mkdir(parents=True) + deep_meta.touch() + + indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=1) + + assert [index["name"] for index in indexes] == ["shallow"] + + +def test_cli_project_discovery_without_max_depth_finds_deep_app_index(tmp_path): + deep_meta = ( + tmp_path / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json" + ) + deep_meta.parent.mkdir(parents=True) + deep_meta.touch() + + indexes = LeannCLI()._discover_indexes_in_project(tmp_path) + + assert [index["name"] for index in indexes] == ["six"] + + +def test_list_command_passes_max_depth_to_discovery(monkeypatch): + cli = LeannCLI() + received = {} + + def capture_max_depth(max_depth): + received["max_depth"] = max_depth + + monkeypatch.setattr(cli, "list_indexes", capture_max_depth) + monkeypatch.setattr(sys, "argv", ["leann", "list", "--max-depth", "7"]) + + asyncio.run(cli.run()) + + assert received == {"max_depth": 7} + + +def test_project_registration_respects_explicit_max_depth(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "project" + deep_meta = ( + project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json" + ) + deep_meta.parent.mkdir(parents=True) + deep_meta.touch() + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + + registry.register_project_directory(project, max_depth=registry.DEFAULT_INDEX_SCAN_DEPTH) + + assert not (home / ".leann" / "projects.json").exists() + + +def test_project_registration_without_max_depth_finds_deep_app_index(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "project" + deep_meta = ( + project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json" + ) + deep_meta.parent.mkdir(parents=True) + deep_meta.touch() + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + + registry.register_project_directory(project) + + registry_file = home / ".leann" / "projects.json" + assert json.loads(registry_file.read_text()) == [str(project.resolve())] + + +def test_project_registration_accepts_custom_max_depth(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "project" + deep_meta = ( + project / "one" / "two" / "three" / "four" / "five" / "six" / "index.leann.meta.json" + ) + deep_meta.parent.mkdir(parents=True) + deep_meta.touch() + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + + registry.register_project_directory(project, max_depth=6) + + registry_file = home / ".leann" / "projects.json" + assert json.loads(registry_file.read_text()) == [str(project.resolve())] + + +def test_registered_app_project_is_listed(tmp_path, monkeypatch, capsys): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "registered-project" + meta_file = project / "app-index" / "documents.leann.meta.json" + meta_file.parent.mkdir(parents=True) + meta_file.touch() + current = tmp_path / "current-project" + current.mkdir() + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + registry.register_project_directory(project) + monkeypatch.chdir(current) + + LeannCLI().list_indexes() + + output = capsys.readouterr().out + assert "registered-project" in output + assert "app-index" in output + + +def test_registered_app_project_respects_requested_max_depth(tmp_path, monkeypatch, capsys): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "registered-project" + meta_file = ( + project / "one" / "two" / "three" / "four" / "five" / "six" / "deep-app.leann.meta.json" + ) + meta_file.parent.mkdir(parents=True) + meta_file.touch() + registry_file = home / ".leann" / "projects.json" + registry_file.parent.mkdir() + registry_file.write_text(json.dumps([str(project.resolve())])) + current = tmp_path / "current-project" + current.mkdir() + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + monkeypatch.chdir(current) + + LeannCLI().list_indexes(max_depth=6) + + output = capsys.readouterr().out + assert "registered-project" in output + assert "six" in output + + +def test_registered_ancestor_does_not_hide_current_app_index(tmp_path, monkeypatch, capsys): + home = tmp_path / "home" + home.mkdir() + registered_ancestor = tmp_path / "registered-ancestor" + current = registered_ancestor / "current-project" + meta_file = current / "current-app" / "documents.leann.meta.json" + meta_file.parent.mkdir(parents=True) + meta_file.touch() + registry_file = home / ".leann" / "projects.json" + registry_file.parent.mkdir() + registry_file.write_text(json.dumps([str(registered_ancestor.resolve())])) + monkeypatch.setattr(registry.Path, "home", classmethod(lambda cls: home)) + monkeypatch.chdir(current) + + LeannCLI().list_indexes() + + output = capsys.readouterr().out + current_section = output.split("Other Projects", maxsplit=1)[0] + assert "current-app" in current_section + assert output.count("current-app") == 1 + + +def test_cli_format_index_discovery_is_not_limited_by_app_scan_depth(tmp_path): + index_dir = tmp_path / ".leann" / "indexes" / "sample" + index_dir.mkdir(parents=True) + (index_dir / "documents.leann.meta.json").touch() + + indexes = LeannCLI()._discover_indexes_in_project(tmp_path, max_depth=0) + + assert [(index["name"], index["type"]) for index in indexes] == [("sample", "cli")]