diff --git a/src/semble/index/create.py b/src/semble/index/create.py index 9d7e5b387..168f8ef3c 100644 --- a/src/semble/index/create.py +++ b/src/semble/index/create.py @@ -1,4 +1,5 @@ import contextlib +from collections.abc import Sequence from pathlib import Path import bm25s @@ -6,7 +7,8 @@ from semble.chunking import chunk_source from semble.index.dense import SelectableBasicBackend, embed_chunks -from semble.index.file_walker import filter_extensions, language_for_path, walk_files +from semble.index.file_walker import walk_files +from semble.index.files import detect_language, get_extensions from semble.index.sparse import enrich_for_bm25 from semble.tokens import tokenize from semble.types import Chunk, Encoder @@ -17,8 +19,7 @@ def create_index_from_path( path: Path, model: Encoder, - extensions: frozenset[str] | None = None, - ignore: frozenset[str] | None = None, + extensions: Sequence[str] | None = None, include_text_files: bool = False, display_root: Path | None = None, ) -> tuple[bm25s.BM25, SelectableBasicBackend, list[Chunk]]: @@ -27,18 +28,15 @@ def create_index_from_path( :param path: Resolved absolute path to index. :param model: The model to use for indexing. :param extensions: File extensions to include. - :param ignore: Directory names to skip. :param include_text_files: If True, also index non-code text files (.md, .yaml, .json, etc.). :param display_root: If set, chunk file paths are stored relative to this root. :raises ValueError: if no items were found, no index can be created. :return: A bm25 index, vicinity index and list of chunks """ - extensions = filter_extensions(extensions, include_text_files=include_text_files) - chunks: list[Chunk] = [] - - for file_path in walk_files(path, extensions, ignore): - language = language_for_path(file_path) + extensions = get_extensions(include_text_files, extensions) + for file_path in walk_files(path, extensions): + language = detect_language(file_path) with contextlib.suppress(OSError): if file_path.stat().st_size > _MAX_FILE_BYTES: continue diff --git a/src/semble/index/file_walker.py b/src/semble/index/file_walker.py index 3e4542a91..d6fe659ac 100644 --- a/src/semble/index/file_walker.py +++ b/src/semble/index/file_walker.py @@ -1,149 +1,131 @@ -import os -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from dataclasses import dataclass -from enum import Enum from pathlib import Path from pathspec import GitIgnoreSpec -class FileCategory(str, Enum): - CODE = "CODE" - DOCUMENT = "DOCUMENT" +@dataclass(frozen=True) +class IgnoreSpec: + base: Path + spec: GitIgnoreSpec -@dataclass(frozen=True) -class FileType: - """Language and indexing policy for a file extension.""" - - language: str - category: FileCategory - - -FILE_TYPES: dict[str, FileType] = { - ".py": FileType("python", FileCategory.CODE), - ".js": FileType("javascript", FileCategory.CODE), - ".jsx": FileType("javascript", FileCategory.CODE), - ".ts": FileType("typescript", FileCategory.CODE), - ".tsx": FileType("typescript", FileCategory.CODE), - ".go": FileType("go", FileCategory.CODE), - ".rs": FileType("rust", FileCategory.CODE), - ".java": FileType("java", FileCategory.CODE), - ".kt": FileType("kotlin", FileCategory.CODE), - ".kts": FileType("kotlin", FileCategory.CODE), - ".rb": FileType("ruby", FileCategory.CODE), - ".php": FileType("php", FileCategory.CODE), - ".c": FileType("c", FileCategory.CODE), - ".h": FileType("c", FileCategory.CODE), - ".cpp": FileType("cpp", FileCategory.CODE), - ".hpp": FileType("cpp", FileCategory.CODE), - ".cs": FileType("csharp", FileCategory.CODE), - ".swift": FileType("swift", FileCategory.CODE), - ".scala": FileType("scala", FileCategory.CODE), - ".sbt": FileType("scala", FileCategory.CODE), - ".ex": FileType("elixir", FileCategory.CODE), - ".exs": FileType("elixir", FileCategory.CODE), - ".dart": FileType("dart", FileCategory.CODE), - ".lua": FileType("lua", FileCategory.CODE), - ".sql": FileType("sql", FileCategory.CODE), - ".sh": FileType("bash", FileCategory.CODE), - ".bash": FileType("bash", FileCategory.CODE), - ".zig": FileType("zig", FileCategory.CODE), - ".hs": FileType("haskell", FileCategory.CODE), - ".md": FileType("markdown", FileCategory.DOCUMENT), - ".yaml": FileType("yaml", FileCategory.DOCUMENT), - ".yml": FileType("yaml", FileCategory.DOCUMENT), - ".toml": FileType("toml", FileCategory.DOCUMENT), - ".json": FileType("json", FileCategory.DOCUMENT), -} - -DEFAULT_IGNORED_DIRS: frozenset[str] = frozenset( +_DEFAULT_IGNORED_DIRS: frozenset[str] = frozenset( { - ".git", - ".hg", - ".svn", - "__pycache__", - "node_modules", - ".venv", - "venv", - ".tox", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".cache", - ".semble", - ".next", - "dist", - "build", - ".eggs", + ".git/", + ".hg/", + ".svn/", + "__pycache__/", + "node_modules/", + ".venv/", + "venv/", + ".tox/", + ".mypy_cache/", + ".pytest_cache/", + ".ruff_cache/", + ".cache/", + ".semble/", + ".next/", + "dist/", + "build/", + ".eggs/", } ) -def language_for_path(path: Path) -> str | None: - """Return the language for a file path, or None for unknown extensions.""" - if spec := FILE_TYPES.get(path.suffix.lower()): - return spec.language - return None - +def _load_ignore_for_dir(directory: Path) -> GitIgnoreSpec | None: + """Loads a gitignore and sembleignore for a dir.""" + gitignore = directory / ".gitignore" + sembleignore = directory / ".sembleignore" -def filter_extensions(extensions: frozenset[str] | None, *, include_text_files: bool) -> frozenset[str]: - """Return the set of file extensions to index.""" - if extensions is not None: - return extensions - # Always index code files - categories_to_include = {FileCategory.CODE} - if include_text_files: - categories_to_include.add(FileCategory.DOCUMENT) - # Return a default set of extensions - return frozenset(ext for ext, spec in FILE_TYPES.items() if spec.category in categories_to_include) - - -def _load_root_gitignore(root: Path) -> GitIgnoreSpec | None: - """Load the root-level .gitignore as a spec, if present.""" - gitignore = root / ".gitignore" - if not gitignore.is_file(): - return None - return GitIgnoreSpec.from_lines(gitignore.read_text(encoding="utf-8", errors="ignore").splitlines()) - - -def _dir_is_gitignored(gitignore: GitIgnoreSpec, rel: str) -> bool: - """Return True if rel (a POSIX path relative to the gitignore root) matches a gitignore pattern for directories.""" - ignored = False - for pattern in gitignore.patterns: - if pattern.include is not None and pattern.match_file(rel): - ignored = pattern.include - return ignored + lines = [] + if gitignore.is_file(): + lines.extend(gitignore.read_text(encoding="utf-8", errors="ignore").splitlines()) + if sembleignore.is_file(): + lines.extend(sembleignore.read_text(encoding="utf-8", errors="ignore").splitlines()) + if lines: + return GitIgnoreSpec.from_lines(lines) + return None -def walk_files(root: Path, extensions: frozenset[str], ignore: frozenset[str] | None = None) -> Iterator[Path]: +def walk_files(root: Path, extensions: Sequence[str], ignore: Sequence[str] | None = None) -> Iterator[Path]: """Yield files under root matching extensions, skipping ignored paths. Directories matching DEFAULT_IGNORED_DIRS plus any names in ignore are always skipped. If the root contains a .gitignore, its patterns are also honoured. :param root: Root directory to walk. - :param extensions: Set of file extensions to include (e.g. {".py", ".js"}). - :param ignore: Additional directory names to ignore (e.g. {"build", "dist"}). + :param extensions: List of file extensions to match. + :param ignore: Additional patterns to ignore. :yield: Path to each file under root matching the criteria. :ytype: Path """ - ignore_dirs = DEFAULT_IGNORED_DIRS | (ignore or frozenset()) - gitignore = _load_root_gitignore(root) - for dirpath, dirnames, filenames in os.walk(root): - rel_dir = Path(dirpath).relative_to(root) - kept: list[str] = [] - for dirname in dirnames: - if dirname in ignore_dirs: - continue - if gitignore is not None and _dir_is_gitignored(gitignore, (rel_dir / dirname).as_posix() + "/"): + # This should be a list. Traversal is done in order, so the order matters. + ignored = [] + extensions_as_patterns = [f"!*{ext}" for ext in extensions] + ignored.extend(extensions_as_patterns) + ignored.extend(_DEFAULT_IGNORED_DIRS) + # Always give user patterns preference + ignored.extend(ignore or []) + base_spec = GitIgnoreSpec.from_lines(ignored, backend="simple") + s = IgnoreSpec(base=root, spec=base_spec) + yield from _walk(root, [s]) + + +def _is_ignored(path: Path, specs: list[IgnoreSpec]) -> bool: + """Check if a path is ignored by any of the provided ignore specs.""" + is_dir = path.is_dir() + # Everything starts off as unignored + ignored = not is_dir + + for ignore_spec in specs: + try: + # If there is no relative path, this is invalid. + relative = path.relative_to(ignore_spec.base) + except ValueError: + continue + + relative_str = relative.as_posix() + # We need to add a trailing slash. Gitignore + # matches dirs as trailing '/'. + if is_dir: + relative_str += "/" + + # Loop over all the patterns + for pattern in ignore_spec.spec.patterns: + # This pattern doesn't do anything. + if pattern.include is None: continue - kept.append(dirname) - dirnames[:] = kept - for filename in sorted(filenames): - file_path = Path(dirpath) / filename - if file_path.suffix.lower() not in extensions: - continue - if gitignore is not None and gitignore.match_file((rel_dir / filename).as_posix()): - continue - yield file_path + + if pattern.match_file(relative_str) is not None: + ignored = pattern.include + + return ignored + + +def _walk( + directory: Path, + inherited_specs: list[IgnoreSpec], +) -> Iterator[Path]: + """Recursive function for walking files under a directory.""" + active_specs = inherited_specs + + spec = _load_ignore_for_dir(directory) + if spec is not None: + active_specs = [ + *inherited_specs, + IgnoreSpec(base=directory, spec=spec), + ] + + for item in directory.iterdir(): + # Don't follow symlinks + if item.is_symlink(): + continue + if _is_ignored(item, active_specs): + continue + + if item.is_dir(): + yield from _walk(item, active_specs) + elif item.is_file(): + yield item diff --git a/src/semble/index/files.py b/src/semble/index/files.py new file mode 100644 index 000000000..8e80b5299 --- /dev/null +++ b/src/semble/index/files.py @@ -0,0 +1,462 @@ +from collections import defaultdict +from collections.abc import Sequence +from pathlib import Path + +_EXTENSION_TO_LANGUAGE = { + ".4th": "forth", + ".ada": "ada", + ".adb": "ada", + ".adoc": "asciidoc", + ".ads": "ada", + ".agda": "agda", + ".al": "al", + ".as": "actionscript", + ".asciidoc": "asciidoc", + ".asm": "asm", + ".astro": "astro", + ".awk": "awk", + ".axi": "netlinx", + ".axs": "netlinx", + ".bash": "bash", + ".bat": "batch", + ".bb": "bitbake", + ".bbappend": "bitbake", + ".bbclass": "bitbake", + ".beancount": "beancount", + ".bib": "bibtex", + ".bicep": "bicep", + ".blade": "blade", + ".bq": "sql_bigquery", + ".brs": "brightscript", + ".bsl": "bsl", + ".bzl": "starlark", + ".c": "c", + ".c3": "c3", + ".c3i": "c3", + ".c3t": "c3", + ".caddyfile": "caddy", + ".cairo": "cairo", + ".capnp": "capnp", + ".cbl": "cobol", + ".cc": "cpp", + ".cedar": "cedar", + ".cedarschema": "cedarschema", + ".cel": "cel", + ".cfc": "cfml", + ".cfg": "ini", + ".chatito": "chatito", + ".circom": "circom", + ".cjs": "javascript", + ".ck": "chuck", + ".cl": "commonlisp", + ".clar": "clarity", + ".clj": "clojure", + ".cljc": "clojure", + ".cljs": "clojure", + ".cls": "abl", + ".cmake": "cmake", + ".cmd": "batch", + ".cob": "cobol", + ".cobol": "cobol", + ".conf": "nginx", + ".cook": "cooklang", + ".corn": "corn", + ".cpon": "cpon", + ".cpp": "cpp", + ".cr": "crystal", + ".cs": "csharp", + ".cshtml": "razor", + ".css": "css", + ".cst": "cst", + ".csv": "csv", + ".cts": "typescript", + ".cu": "cuda", + ".cuda": "cuda", + ".cue": "cue", + ".cxx": "cpp", + ".cylc": "cylc", + ".d": "d", + ".dart": "dart", + ".desktop": "desktop", + ".dhall": "dhall", + ".diff": "diff", + ".dj": "djot", + ".dl": "souffle", + ".dockerfile": "dockerfile", + ".dot": "dot", + ".dsp": "faust", + ".dtd": "dtd", + ".dts": "devicetree", + ".dtsi": "devicetree", + ".ebnf": "ebnf", + ".eds": "eds", + ".eex": "eex", + ".el": "elisp", + ".elm": "elm", + ".elv": "elvish", + ".enforce": "enforce", + ".eps": "postscript", + ".erb": "embeddedtemplate", + ".erl": "erlang", + ".ex": "elixir", + ".exs": "elixir", + ".f": "fortran", + ".f03": "fortran", + ".f08": "fortran", + ".f90": "fortran", + ".f95": "fortran", + ".fc": "func", + ".fidl": "fidl", + ".filter": "poe_filter", + ".fir": "firrtl", + ".fish": "fish", + ".fnl": "fennel", + ".fs": "fsharp", + ".fsd": "facility", + ".fsi": "fsharp_signature", + ".fsx": "fsharp", + ".fth": "forth", + ".fun": "sml", + ".g": "gap", + ".gd": "gdscript", + ".gdshader": "gdshader", + ".gi": "gap", + ".gitattributes": "gitattributes", + ".gitignore": "gitignore", + ".gleam": "gleam", + ".glsl": "glsl", + ".gn": "gn", + ".gni": "gn", + ".gnuplot": "gnuplot", + ".go": "go", + ".gotmpl": "gotmpl", + ".gp": "gnuplot", + ".gql": "graphql", + ".gradle": "groovy", + ".graphql": "graphql", + ".gren": "gren", + ".groovy": "groovy", + ".gv": "dot", + ".h": "c", + ".hack": "hack", + ".hare": "hare", + ".hbs": "glimmer", + ".hcl": "hcl", + ".heex": "heex", + ".hjson": "hjson", + ".hlsl": "hlsl", + ".hocon": "hocon", + ".hoon": "hoon", + ".hpp": "cpp", + ".hrl": "erlang", + ".hs": "haskell", + ".htm": "html", + ".html": "html", + ".http": "http", + ".hurl": "hurl", + ".hx": "haxe", + ".hxx": "cpp", + ".idr": "idris", + ".inc": "sourcepawn", + ".ini": "ini", + ".ino": "arduino", + ".ispc": "ispc", + ".j2": "jinja2", + ".jai": "jai", + ".janet": "janet", + ".java": "java", + ".jinja2": "jinja2", + ".jl": "julia", + ".journal": "ledger", + ".jq": "jq", + ".js": "javascript", + ".json": "json", + ".json5": "json5", + ".jsonnet": "jsonnet", + ".jsx": "javascript", + ".just": "just", + ".k": "kcl", + ".kdl": "kdl", + ".kt": "kotlin", + ".kts": "kotlin", + ".lc": "elsa", + ".ldg": "ledger", + ".lds": "linkerscript", + ".lean": "lean", + ".ledger": "ledger", + ".leex": "eex", + ".less": "less", + ".libsonnet": "jsonnet", + ".liquid": "liquid", + ".lisp": "commonlisp", + ".ll": "llvm", + ".lua": "lua", + ".luau": "luau", + ".m": "objc", + ".magik": "magik", + ".makefile": "make", + ".markdown": "markdown", + ".matlab": "matlab", + ".md": "markdown", + ".mermaid": "mermaid", + ".meson": "meson", + ".mjs": "javascript", + ".mk": "make", + ".ml": "ocaml", + ".mli": "ocaml_interface", + ".mlir": "mlir", + ".mll": "ocamllex", + ".mmd": "mermaid", + ".mod": "gomod", + ".mojo": "mojo", + ".move": "move", + ".mts": "typescript", + ".nasm": "nasm", + ".ncl": "nickel", + ".nginx": "nginx", + ".nim": "nim", + ".nims": "nim", + ".ninja": "ninja", + ".nix": "nix", + ".norg": "norg", + ".nqc": "nqc", + ".nu": "nushell", + ".nut": "squirrel", + ".odin": "odin", + ".org": "org", + ".p": "abl", + ".pas": "pascal", + ".patch": "diff", + ".pbtxt": "textproto", + ".pem": "pem", + ".pgn": "pgn", + ".php": "php", + ".pkl": "pkl", + ".pl": "perl", + ".plt": "gnuplot", + ".pm": "perl", + ".po": "po", + ".pony": "pony", + ".pot": "po", + ".pp": "puppet", + ".prisma": "prisma", + ".pro": "prolog", + ".promql": "promql", + ".properties": "properties", + ".proto": "proto", + ".prql": "prql", + ".ps": "postscript", + ".ps1": "powershell", + ".psd1": "powershell", + ".psm1": "powershell", + ".psv": "psv", + ".pug": "pug", + ".purs": "purescript", + ".py": "python", + ".pyi": "python", + ".pyw": "python", + ".ql": "ql", + ".qml": "qmljs", + ".r": "r", + ".rasi": "rasi", + ".razor": "razor", + ".rb": "ruby", + ".rbs": "rbs", + ".re": "re2c", + ".rego": "rego", + ".res": "rescript", + ".resi": "rescript", + ".rkt": "racket", + ".robot": "robot", + ".roc": "roc", + ".ron": "ron", + ".rs": "rust", + ".rst": "rst", + ".rtf": "rtf", + ".s": "asm", + ".scad": "openscad", + ".scala": "scala", + ".scm": "scheme", + ".scss": "scss", + ".sh": "bash", + ".shtml": "superhtml", + ".sig": "sml", + ".slang": "slang", + ".smali": "smali", + ".smithy": "smithy", + ".smk": "snakemake", + ".sml": "sml", + ".sol": "solidity", + ".sp": "sourcepawn", + ".sparql": "sparql", + ".sql": "sql", + ".squirrel": "squirrel", + ".st": "smalltalk", + ".stan": "stan", + ".star": "starlark", + ".sv": "systemverilog", + ".svelte": "svelte", + ".svh": "systemverilog", + ".sw": "sway", + ".swift": "swift", + ".tact": "tact", + ".tal": "uxntal", + ".tape": "vhs", + ".tcl": "tcl", + ".td": "tablegen", + ".templ": "templ", + ".tera": "tera", + ".tex": "latex", + ".textproto": "textproto", + ".tf": "terraform", + ".tfvars": "terraform", + ".thrift": "thrift", + ".tl": "teal", + ".tla": "tlaplus", + ".todotxt": "todotxt", + ".toml": "toml", + ".tres": "godot_resource", + ".trigger": "apex", + ".ts": "typescript", + ".tscn": "godot_resource", + ".tsconfig": "typoscript", + ".tsp": "typespec", + ".tsv": "tsv", + ".tsx": "tsx", + ".ttl": "turtle", + ".twig": "twig", + # Overly broad + # ".txt": "vimdoc", + ".typoscript": "typoscript", + ".typst": "typst", + ".v": "v", + ".vb": "vb", + ".verilog": "verilog", + ".vhd": "vhdl", + ".vhdl": "vhdl", + ".vim": "vim", + ".vrl": "vrl", + ".vue": "vue", + ".w": "abl", + ".wast": "wast", + ".wat": "wat", + ".wgsl": "wgsl", + ".wit": "wit", + ".wl": "wolfram", + ".xml": "xml", + ".xsl": "xml", + ".xslt": "xml", + ".yaml": "yaml", + ".yml": "yaml", + ".yuck": "yuck", + ".zig": "zig", + ".ziggy": "ziggy", + ".zsh": "zsh", +} + + +_DOC_LANGUAGES = { + "asciidoc", + "beancount", + "bibtex", + "capnp", + "cedarschema", + "comment", + "cooklang", + "cpon", + "csv", + "desktop", + "devicetree", + "diff", + "djot", + "doxygen", + "dtd", + "editorconfig", + "ebnf", + "git_config", + "gitattributes", + "gitcommit", + "gitignore", + "godot_resource", + "gomod", + "gosum", + "gowork", + "gpg", + "hjson", + "hocon", + "html", + "ini", + "javadoc", + "jsdoc", + "json", + "json5", + "kdl", + "latex", + "ledger", + "luadoc", + "markdown", + "markdown_inline", + "mermaid", + "norg", + "norg_meta", + "org", + "pem", + "pgn", + "phpdoc", + "po", + "properties", + "proto", + "psv", + "requirements", + "ron", + "rst", + "rtf", + "smithy", + "ssh_config", + "textproto", + "thrift", + "todotxt", + "toml", + "tsv", + "turtle", + "typespec", + "vimdoc", + "wit", + "xcompose", + "xml", + "yaml", + "ziggy_schema", +} + + +def _inv_mapping(mapping: dict[str, str]) -> dict[str, list[str]]: + """Invert a mapping, taking into account duplicate values.""" + inv: defaultdict[str, list[str]] = defaultdict(list) + for key, value in mapping.items(): + inv[value].append(key) + return dict(inv) + + +_ALL_LANGUAGES = frozenset(_EXTENSION_TO_LANGUAGE.values()) +_WITHOUT_DOC = _ALL_LANGUAGES - _DOC_LANGUAGES +_LANGUAGE_TO_EXTENSION = _inv_mapping(_EXTENSION_TO_LANGUAGE) + + +def detect_language(file_name: Path) -> str | None: + """Detects the language of a file.""" + return _EXTENSION_TO_LANGUAGE.get(file_name.suffix.lower()) + + +def get_extensions(include_text_files: bool, extensions: Sequence[str] | None) -> list[str]: + """Returns a list of supported file extensions.""" + if include_text_files: + languages = _ALL_LANGUAGES + else: + languages = _WITHOUT_DOC + all_extensions: set[str] = set() + for language in languages: + all_extensions.update(_LANGUAGE_TO_EXTENSION.get(language, set())) + if extensions is not None: + all_extensions.update(extensions) + + return sorted(all_extensions) diff --git a/src/semble/index/index.py b/src/semble/index/index.py index 24c1f36e4..b3be69a21 100644 --- a/src/semble/index/index.py +++ b/src/semble/index/index.py @@ -4,6 +4,7 @@ import subprocess import tempfile from collections import defaultdict +from collections.abc import Sequence from pathlib import Path import numpy as np @@ -89,8 +90,7 @@ def from_path( cls, path: str | Path, model: Encoder | None = None, - extensions: frozenset[str] | None = None, - ignore: frozenset[str] | None = None, + extensions: Sequence[str] | None = None, include_text_files: bool = False, ) -> SembleIndex: """Create and index a SembleIndex from a directory. @@ -98,7 +98,6 @@ def from_path( :param path: Root directory to index. :param model: Embedding model to use. Defaults to potion-code-16M. :param extensions: File extensions to include. Defaults to a standard set of code extensions. - :param ignore: Directory names to skip. Defaults to common VCS and build dirs. :param include_text_files: If True, also index non-code text files (.md, .yaml, .json, etc.). :return: An indexed SembleIndex. Chunk file paths are relative to ``path``. :raises FileNotFoundError: If `path` does not exist. @@ -115,7 +114,6 @@ def from_path( path, model=model, extensions=extensions, - ignore=ignore, include_text_files=include_text_files, display_root=path, ) @@ -128,8 +126,7 @@ def from_git( url: str, ref: str | None = None, model: Encoder | None = None, - extensions: frozenset[str] | None = None, - ignore: frozenset[str] | None = None, + extensions: Sequence[str] | None = None, include_text_files: bool = False, ) -> SembleIndex: """Clone a git repository and index it. @@ -143,7 +140,6 @@ def from_git( :param ref: Branch or tag to check out. Defaults to the remote HEAD. :param model: Embedding model to use. Defaults to potion-code-16M. :param extensions: File extensions to include. Defaults to a standard set of code extensions. - :param ignore: Directory names to skip. Defaults to common VCS and build dirs. :param include_text_files: If True, also index non-code text files (.md, .yaml, .json, etc.). :return: An indexed SembleIndex. Chunk file paths are repo-relative (e.g. ``src/foo.py``). :raises RuntimeError: If git is not on PATH, the clone fails, or times out. @@ -167,7 +163,6 @@ def from_git( resolved_path, model=model, extensions=extensions, - ignore=ignore, include_text_files=include_text_files, display_root=resolved_path, ) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 29aa7c731..1f7e5fde8 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -2,7 +2,6 @@ from semble.chunking.chunking import Chunk, chunk_lines, chunk_source from semble.chunking.core import ChunkBoundary, chunk -from semble.index.file_walker import filter_extensions def test_chunk_lines() -> None: @@ -74,10 +73,3 @@ def test_core_chunk_leaf_node_exceeds_desired_length() -> None: assert chunks[0].start == 0 for c in chunks: assert 0 <= c.start < c.end <= len(code) - - -def test_filter_extensions_explicit() -> None: - """filter_extensions returns the provided set unchanged when extensions is not None.""" - explicit: frozenset[str] = frozenset({".py", ".ts"}) - result = filter_extensions(explicit, include_text_files=False) - assert result == explicit diff --git a/tests/test_file_walker.py b/tests/test_file_walker.py index ef807fd45..b93fb1d48 100644 --- a/tests/test_file_walker.py +++ b/tests/test_file_walker.py @@ -1,17 +1,8 @@ -import os -from collections.abc import Iterator from pathlib import Path import pytest -from semble.index.file_walker import language_for_path, walk_files - - -def test_languagef_for_path() -> None: - """Test language_for_path returns the correct language for a given path.""" - assert language_for_path(Path("foo.py")) == "python" - assert language_for_path(Path("bar.js")) == "javascript" - assert language_for_path(Path("dangerous.exe")) is None +from semble.index.file_walker import walk_files def _touch(path: Path, content: str = "x = 1\n") -> None: @@ -21,64 +12,137 @@ def _touch(path: Path, content: str = "x = 1\n") -> None: @pytest.mark.parametrize( - ("files", "gitignore", "expected"), + ("files", "gitignore", "sembleignore", "expected"), [ # Default-ignored dirs (.venv, node_modules, .cache) are always skipped. ( ["src/a.py", ".venv/lib/b.py", "node_modules/pkg/c.py", ".cache/uv/d.py"], None, + None, {"src/a.py"}, ), # Root .gitignore excludes both directories and files. ( ["src/keep.py", "local/ignored.py", "generated.py"], - "local/\ngenerated.py\n", + "local/\ngenerated.py\n# comment", + None, {"src/keep.py"}, ), # Negation (`!`) patterns re-include previously ignored files. ( ["out/a.py", "out/keep.py"], "out/*\n!out/keep.py\n", + None, {"out/keep.py"}, ), # Allow-list style gitignore (`*` + `!*/` + `!*.py`) must not prune subdirs. ( ["main.py", "internal/pkg/foo.py", "internal/pkg/bar.py"], "*\n!*/\n!*.py\n", + None, {"main.py", "internal/pkg/foo.py", "internal/pkg/bar.py"}, ), # Ignored-parent negation: out/* prunes out/deep/, so out/deep/keep.py must not leak. ( ["out/deep/keep.py"], "out/*\n!out/deep/keep.py\n", + None, + set(), + ), + # Ignored-parent negation: out/* prunes out/deep/, so out/deep/keep.py must not leak. + ( + ["out/deep/keep.py"], + None, + "out/*\n!out/deep/keep.py\n", set(), ), ], ) -def test_walk_files_filtering(tmp_path: Path, files: list[str], gitignore: str | None, expected: set[str]) -> None: +def test_walk_files_filtering( + tmp_path: Path, files: list[str], gitignore: str | None, sembleignore: str | None, expected: set[str] +) -> None: """Directory defaults, gitignore patterns, and negations filter the yielded files.""" for rel in files: _touch(tmp_path / rel) if gitignore is not None: (tmp_path / ".gitignore").write_text(gitignore) + if sembleignore is not None: + (tmp_path / ".sembleignore").write_text(sembleignore) - found = {p.relative_to(tmp_path).as_posix() for p in walk_files(tmp_path, frozenset({".py"}))} + found = {p.relative_to(tmp_path).as_posix() for p in walk_files(tmp_path, [".py"])} assert found == expected -def test_walk_files_prunes_ignored_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_walk_files_prunes_ignored_dirs(tmp_path: Path) -> None: """Ignored directories are pruned so os.walk never descends into them.""" _touch(tmp_path / "src" / "a.py") _touch(tmp_path / "node_modules" / "deep" / "deeper" / "b.js") - visited: list[str] = [] - real_walk = os.walk + visited = list(walk_files(tmp_path, [".py", ".js"])) + assert not any("node_modules" in str(v) for v in visited), visited + + +def test_is_ignored_skips_spec_with_unrelated_base(tmp_path: Path) -> None: + """An IgnoreSpec whose base is not an ancestor of the path is silently skipped. + + Files default to ignored in _is_ignored. When the first spec has an + unrelated base, the ValueError is caught and the spec is skipped without + crashing. A second spec with the correct base can still un-ignore the file. + """ + from pathspec import GitIgnoreSpec + + from semble.index.file_walker import IgnoreSpec, _is_ignored + + # Create two unrelated directory trees + project_a = tmp_path / "project_a" + project_b = tmp_path / "project_b" + project_a.mkdir() + project_b.mkdir() + + target_file = project_a / "keep.py" + target_file.write_text("x = 1\n") + + # Spec rooted at project_b — unrelated to target_file + unrelated_spec = IgnoreSpec( + base=project_b, + spec=GitIgnoreSpec.from_lines(["*.py"]), + ) + + # With only the unrelated spec the file stays in its default state (ignored) + # and, crucially, no exception is raised. + assert _is_ignored(target_file, [unrelated_spec]) is True + + # Spec rooted at project_a that un-ignores .py files + matching_spec = IgnoreSpec( + base=project_a, + spec=GitIgnoreSpec.from_lines(["!*.py"]), + ) + + # The unrelated spec is safely skipped; the matching spec un-ignores the file. + assert _is_ignored(target_file, [unrelated_spec, matching_spec]) is False + + +def test_walk_files_skips_symlinks(tmp_path: Path) -> None: + """Symlinked files and directories are skipped; real paths are still walked.""" + # Real directory with a file + real_dir = tmp_path / "real_pkg" / "src" + _touch(real_dir / "mod.py") + + # A symlink to that directory from another location + link_parent = tmp_path / "wrapper" / "src" + link_parent.mkdir(parents=True) + (link_parent / "linked").symlink_to(real_dir) + + # A symlink to a single file + _touch(tmp_path / "original.py") + (tmp_path / "link_to_original.py").symlink_to(tmp_path / "original.py") + + found = {p.relative_to(tmp_path).as_posix() for p in walk_files(tmp_path, [".py"])} - def tracking_walk(top: str) -> Iterator[tuple[str, list[str], list[str]]]: - for dirpath, dirnames, filenames in real_walk(top): - visited.append(dirpath) - yield dirpath, dirnames, filenames + # Real paths are present + assert "real_pkg/src/mod.py" in found + assert "original.py" in found - monkeypatch.setattr("semble.index.file_walker.os.walk", tracking_walk) - list(walk_files(tmp_path, frozenset({".py", ".js"}))) - assert not any("node_modules" in v for v in visited[1:]), visited + # Symlink-based paths are absent + assert "wrapper/src/linked/mod.py" not in found + assert "link_to_original.py" not in found diff --git a/tests/test_files.py b/tests/test_files.py new file mode 100644 index 000000000..71ede4c4b --- /dev/null +++ b/tests/test_files.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from semble.index.files import _DOC_LANGUAGES, _EXTENSION_TO_LANGUAGE, detect_language, get_extensions + + +def test_detect_language() -> None: + """Test the detect_language function.""" + assert detect_language(Path("a.py")) == "python" + assert detect_language(Path("b.js")) == "javascript" + assert detect_language(Path("c.txt")) is None + + +def test_get_extensions() -> None: + """Test the get_extensions function.""" + all_extensions = get_extensions(True, None) + without_doc_extensions = get_extensions(False, None) + + doc_extensions = set(all_extensions) - set(without_doc_extensions) + + for extension in doc_extensions: + assert _EXTENSION_TO_LANGUAGE[extension] in _DOC_LANGUAGES + for extension in without_doc_extensions: + assert _EXTENSION_TO_LANGUAGE[extension] not in _DOC_LANGUAGES + + +def test_get_extensions_additional() -> None: + """Test the get_extensions function.""" + all_extensions = get_extensions(True, None) + all_extensions_extra = get_extensions(True, [".kjs"]) + + assert set(all_extensions_extra) == set(all_extensions) | {".kjs"} + + all_extensions = get_extensions(False, None) + all_extensions_extra = get_extensions(False, [".kjs"]) + + assert set(all_extensions_extra) == set(all_extensions) | {".kjs"} + + all_extensions = get_extensions(False, None) + all_extensions_extra = get_extensions(False, [".py"]) + + assert set(all_extensions_extra) == set(all_extensions) diff --git a/uv.lock b/uv.lock index 0330d1422..c51680ce6 100644 --- a/uv.lock +++ b/uv.lock @@ -12,7 +12,7 @@ resolution-markers = [ [options] exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. -exclude-newer-span = "P1W" +exclude-newer-span = "P3D" [[package]] name = "annotated-doc"