Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 5 additions & 15 deletions docs/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,8 @@ fixes). Newest entries at the bottom.
comparable recall (GPU latency stays ~flat while CPU grows linearly with nprobe).
- Docs: `docs/flashlib_backend_guide.md` gains a `flashlib_ivf` section.

## 2026-07-24: `leann-core` 0.3.8 — HNSW full-rebuild fallback corpus wipe (#386)

- Fixed a bug where, on the HNSW backend, `leann build` / `leann watch` falling
back to a full rebuild (any modified or removed file, since HNSW only
supports add-only incremental updates) rebuilt the index from only the
changed files instead of the full corpus, silently dropping every untouched
file. Root cause: the fallback reused `all_texts` from the incremental path
(which intentionally loads only the delta) instead of reloading the full
corpus via `load_documents(docs_paths, ...)`.
- This was fixed on `main` in commit `956beb5` (merged via #279) but never
reached PyPI — `leann-core` 0.3.7 was still affected. Bumped `leann-core` to
0.3.8 to ship the fix.
- Added `tests/test_hnsw_rebuild_fallback.py`: regression test that builds an
HNSW index, modifies one file, and asserts untouched files are still present
in the rebuilt corpus.
## 2026-07-27: Bounded `leann list` index discovery

- Bound App-format metadata discovery to a configurable directory depth; `leann list` defaults to depth 5 and accepts `--max-depth` for deeper projects.
- Prune common dependency, virtual-environment, cache, and system directories during discovery.
- Preserve discovery of registered CLI-format and App-format projects without recursively scanning an entire home directory.
85 changes: 70 additions & 15 deletions packages/leann-core/src/leann/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .api import Fts5BM25Index, LeannBuilder, LeannChat, LeannSearcher
from .embedding_server_manager import EmbeddingServerManager
from .interactive_utils import create_cli_session
from .registry import register_project_directory
from .registry import DEFAULT_INDEX_SCAN_DEPTH, iter_index_meta_files, register_project_directory
from .settings import (
resolve_anthropic_base_url,
resolve_atlascloud_api_key,
Expand All @@ -33,6 +33,13 @@
from .sync import DEFAULT_INDEX_EXTENSIONS, FileSynchronizer, parse_include_extensions


def _non_negative_int(value: str) -> int:
parsed_value = int(value)
if parsed_value < 0:
raise argparse.ArgumentTypeError("must be non-negative")
return parsed_value


def _default_embedding_model() -> str:
"""Pick a sensible default embedding model based on platform.

Expand Down Expand Up @@ -856,7 +863,16 @@ def _add_index_args(p, default_name):
)

# List command
subparsers.add_parser("list", help="List all indexes")
list_parser = subparsers.add_parser("list", help="List all indexes")
list_parser.add_argument(
"--max-depth",
type=_non_negative_int,
default=DEFAULT_INDEX_SCAN_DEPTH,
help=(
"Maximum directory depth for app index discovery "
f"(default: {DEFAULT_INDEX_SCAN_DEPTH})"
),
)

# Remove command
remove_parser = subparsers.add_parser("remove", help="Remove an index")
Expand Down Expand Up @@ -947,7 +963,24 @@ def _is_git_submodule(self, path: Path) -> bool:
# If anything goes wrong, assume it's not a submodule
return False

def list_indexes(self):
@staticmethod
def _project_has_discoverable_indexes(project_path: Path, max_depth: int) -> bool:
"""Return whether a project contains an index within the scan boundary."""
cli_indexes_dir = project_path / ".leann" / "indexes"
return cli_indexes_dir.exists() or any(
iter_index_meta_files(project_path, max_depth=max_depth)
)

@staticmethod
def _nested_project_dirs(project_path: Path, projects: list[Path]) -> list[Path]:
"""Return registered project roots nested below a project."""
return [
candidate
for candidate in projects
if candidate != project_path and candidate.is_relative_to(project_path)
]

def list_indexes(self, max_depth: int = DEFAULT_INDEX_SCAN_DEPTH):
# Get all project directories with .leann
global_registry = Path.home() / ".leann" / "projects.json"
all_projects = []
Expand All @@ -961,16 +994,21 @@ def list_indexes(self):
except Exception:
pass

# Filter to only existing directories with .leann
# Filter to existing projects with a discoverable CLI- or App-format index.
valid_projects = []
for project_dir in all_projects:
project_path = Path(project_dir)
if project_path.exists() and (project_path / ".leann" / "indexes").exists():
if project_path.exists() and self._project_has_discoverable_indexes(
project_path, max_depth
):
valid_projects.append(project_path)

# Add current project if it has .leann but not in registry
# Add current project if it has a discoverable index but is not in registry.
current_path = Path.cwd()
if (current_path / ".leann" / "indexes").exists() and current_path not in valid_projects:
if (
self._project_has_discoverable_indexes(current_path, max_depth)
and current_path not in valid_projects
):
valid_projects.append(current_path)

# Separate current and other projects
Expand All @@ -980,6 +1018,10 @@ def list_indexes(self):
if project_path != current_path:
other_projects.append(project_path)

# Exclude only registered projects nested below the current project.
# Registered ancestors must not hide indexes owned by the current directory.
current_exclude_dirs = self._nested_project_dirs(current_path, valid_projects)

print("📚 LEANN Indexes")
print("=" * 50)

Expand All @@ -992,7 +1034,7 @@ def list_indexes(self):
print(" " + "─" * 45)

current_indexes = self._discover_indexes_in_project(
current_path, exclude_dirs=other_projects
current_path, exclude_dirs=current_exclude_dirs, max_depth=max_depth
)
if current_indexes:
for idx in current_indexes:
Expand All @@ -1011,7 +1053,10 @@ def list_indexes(self):
print(" " + "─" * 45)

for project_path in other_projects:
project_indexes = self._discover_indexes_in_project(project_path)
nested_projects = self._nested_project_dirs(project_path, valid_projects)
project_indexes = self._discover_indexes_in_project(
project_path, exclude_dirs=nested_projects, max_depth=max_depth
)
if not project_indexes:
continue

Expand All @@ -1035,9 +1080,14 @@ def list_indexes(self):
projects_count = 0
for p in valid_projects:
if p == current_path:
discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects)
discovered = self._discover_indexes_in_project(
p, exclude_dirs=current_exclude_dirs, max_depth=max_depth
)
else:
discovered = self._discover_indexes_in_project(p)
nested_projects = self._nested_project_dirs(p, valid_projects)
discovered = self._discover_indexes_in_project(
p, exclude_dirs=nested_projects, max_depth=max_depth
)
if len(discovered) > 0:
projects_count += 1
print(f"📊 Total: {total_indexes} indexes across {projects_count} projects")
Expand All @@ -1057,13 +1107,18 @@ def list_indexes(self):
print(" leann build my-docs --docs ./documents")

def _discover_indexes_in_project(
self, project_path: Path, exclude_dirs: Optional[list[Path]] = None
self,
project_path: Path,
exclude_dirs: Optional[list[Path]] = None,
max_depth: Optional[int] = None,
):
"""Discover all indexes in a project directory (both CLI and apps formats)

exclude_dirs: when provided, skip any APP-format index files that are
located under these directories. This prevents duplicates when the
current project is a parent directory of other registered projects.
max_depth: maximum number of directories to descend for APP-format indexes.
None preserves full-depth discovery for non-CLI callers.
"""
indexes = []
exclude_dirs = exclude_dirs or []
Expand Down Expand Up @@ -1100,9 +1155,9 @@ def _discover_indexes_in_project(
}
)

# 2. Apps format: *.leann.meta.json files anywhere in the project
# 2. Apps format: *.leann.meta.json files within the configured scan depth
cli_indexes_dir = project_path / ".leann" / "indexes"
for meta_file in project_path.rglob("*.leann.meta.json"):
for meta_file in iter_index_meta_files(project_path, max_depth=max_depth):
if meta_file.is_file():
# Skip CLI-built indexes (which store meta under .leann/indexes/<name>/)
try:
Expand Down Expand Up @@ -3732,7 +3787,7 @@ async def run(self, args=None):
suppress = not getattr(args, "verbose", False)

if args.command == "list":
self.list_indexes()
self.list_indexes(args.max_depth)
elif args.command == "remove":
self.remove_index(args.index_name, args.force)
elif args.command == "build":
Expand Down
42 changes: 39 additions & 3 deletions packages/leann-core/src/leann/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import importlib.metadata
import json
import logging
import os
from collections.abc import Iterator
from pathlib import Path
from typing import TYPE_CHECKING, Optional, Union

Expand All @@ -14,6 +16,35 @@
logger = logging.getLogger(__name__)

BACKEND_REGISTRY: dict[str, "LeannBackendFactoryInterface"] = {}
DEFAULT_INDEX_SCAN_DEPTH = 5
INDEX_SCAN_SKIP_DIRS = frozenset(
{".git", ".venv", "venv", "node_modules", "__pycache__", "Library"}
)


def iter_index_meta_files(
root: Union[str, Path], max_depth: Optional[int] = DEFAULT_INDEX_SCAN_DEPTH
) -> Iterator[Path]:
"""Yield LEANN metadata files within an optionally bounded directory tree.

The root directory is depth zero. A max_depth of None scans all depths.
Known dependency, cache, and system directories are always pruned before traversal.
"""
if max_depth is not None and max_depth < 0:
raise ValueError("max_depth must be non-negative")

root_path = Path(root)
for current_dir, dirnames, filenames in os.walk(root_path):
current_path = Path(current_dir)
depth = len(current_path.relative_to(root_path).parts)
if max_depth is not None and depth >= max_depth:
dirnames.clear()
else:
dirnames[:] = [name for name in dirnames if name not in INDEX_SCAN_SKIP_DIRS]

for filename in filenames:
if filename.endswith(".leann.meta.json"):
yield current_path / filename


def register_backend(name: str):
Expand Down Expand Up @@ -49,24 +80,29 @@ def autodiscover_backends():
# print("INFO: Backend auto-discovery finished.")


def register_project_directory(project_dir: Optional[Union[str, Path]] = None):
def register_project_directory(
project_dir: Optional[Union[str, Path]] = None,
max_depth: Optional[int] = None,
):
"""
Register a project directory in the global LEANN registry.

This allows `leann list` to discover indexes created by apps or other tools.

Args:
project_dir: Directory to register. If None, uses current working directory.
max_depth: Maximum directory depth used when looking for App-format indexes.
None preserves full-depth discovery for existing API callers.
"""
if project_dir is None:
project_dir = Path.cwd()
else:
project_dir = Path(project_dir)

# Only register directories that have some kind of LEANN content.
# Check CLI-format first to avoid an expensive rglob on large directories.
# Check CLI-format first to avoid even a bounded scan when it is unnecessary.
has_cli_indexes = (project_dir / ".leann" / "indexes").exists()
if not has_cli_indexes and not any(project_dir.rglob("*.leann.meta.json")):
if not has_cli_indexes and not any(iter_index_meta_files(project_dir, max_depth=max_depth)):
# Don't register if there are no LEANN indexes
return

Expand Down
Loading
Loading