diff --git a/src/contextualize/manifest/hydrate.py b/src/contextualize/manifest/hydrate.py index dd3f06c..c937f8a 100644 --- a/src/contextualize/manifest/hydrate.py +++ b/src/contextualize/manifest/hydrate.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta from pathlib import Path -from typing import Any +from typing import Any, Callable from urllib.parse import unquote, urlparse from ..concurrency import run_indexed_tasks_fail_fast @@ -39,10 +39,20 @@ from .manifest import ( GROUP_BASE_KEY, GROUP_PATH_KEY, + SET_KEY, coerce_file_spec, normalize_components, ) -from .source import ManifestFormat, ManifestSlice, load_manifest_source +from .source import ( + ManifestFormat, + ManifestSlice, + frontmatter_has_manifest_tag, + iter_active_leaves, + load_manifest_source, + path_contains_manifest, +) + +_MANIFEST_REFERENCE_EXTENSIONS = {".md", ".markdown", ".yaml", ".yml"} _EXTERNAL_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") _SOURCE_IGNORE_PATTERNS = [ @@ -497,6 +507,13 @@ def build_hydration_plan_data( plugin_seen_counter = 0 resolved_spec_cache: dict[tuple[Any, ...], list[ResolvedItem]] = {} linked_manifest_failures: list[LinkedManifestFailure] = [] + reference_edges: list[dict[str, Any]] = [] + registry_sources_cache: list[set[Path]] = [] + + def get_registry_sources() -> set[Path]: + if not registry_sources_cache: + registry_sources_cache.append(_load_registry_manifest_sources()) + return registry_sources_cache[0] global_strip_prefix: Path | None = None if context_cfg["path_strategy"] == "by-component": @@ -510,9 +527,20 @@ def build_hydration_plan_data( context_cfg["agents_text"], ) + active_outline_nodes: list[dict[str, Any]] = ( + list(iter_active_leaves(manifest_format.outline)) + if manifest_format is not None + else [] + ) + log_progress("hydrate", "component", "total", count=len(components)) - for comp in components: + for comp_index, comp in enumerate(components): comp_name = comp["name"] + outline_node = ( + active_outline_nodes[comp_index] + if comp_index < len(active_outline_nodes) + else None + ) log_progress("hydrate", "component", "start", target=comp_name) comp_files = comp.get("files") comp_repos = comp.get("repos") @@ -595,6 +623,7 @@ def build_hydration_plan_data( str | None, dict[str, Any], str | None, + bool, ] ] = [] spec_jobs = get_payload_spec_jobs() @@ -750,6 +779,7 @@ def build_hydration_plan_data( extract_ranges(item.content, ranges) if ranges else item.content ) suffix = _build_suffix(ranges, symbols) + from_files = not spec_cache_key[1] resolved_items.append( ( item, @@ -761,6 +791,7 @@ def build_hydration_plan_data( spec_comment, file_opts, spec_root, + from_files, ) ) @@ -778,44 +809,130 @@ def build_hydration_plan_data( effective_strip_prefix = global_strip_prefix skip_external_root = False - for ( - item, - content, - suffix, - ranges, - symbols, - range_spec, - spec_comment, - file_opts, - spec_root, - ) in resolved_items: - rel_path, should_write = _resolve_context_path( - comp_name, + if comp.get(SET_KEY): + set_rel_path = _dedupe_path( + _build_set_output_path( + comp_name, + comp.get(GROUP_PATH_KEY), + comp.get(GROUP_BASE_KEY), + ), + used_paths, + ) + _ensure_relative(set_rel_path) + fused_text, parts = _build_set_fusion(resolved_items, outline_node) + files_to_write.append((context_dir / set_rel_path, fused_text)) + index_components.setdefault(comp_name, []).append( + _build_set_index_entry(set_rel_path, fused_text, parts) + ) + if outline_node is not None: + outline_node["set"] = { + "context_path": set_rel_path.as_posix(), + "parts": parts, + } + normalized_files.extend( + _build_manifest_file_entry( + item, range_spec, symbols, spec_comment, file_opts + ) + for ( + item, + _content, + _suffix, + _ranges, + symbols, + range_spec, + spec_comment, + file_opts, + _spec_root, + _from_files, + ) in resolved_items + ) + else: + for ( item, + content, suffix, - used_paths, - identity_paths, - context_cfg["path_strategy"], ranges, symbols, - component_root=component_root, - use_external_root=use_external_root, - strip_prefix=effective_strip_prefix, - skip_external_root=skip_external_root, - external_root_prefix=spec_root, - ) - plugin_identity = _plugin_pending_key(item, ranges, symbols) - if plugin_identity is not None: - plugin_pending.setdefault(plugin_identity, []).append( - _PluginPendingWrite( - rel_path=rel_path, - content=content, - should_write=should_write, + range_spec, + spec_comment, + file_opts, + spec_root, + from_files, + ) in resolved_items: + rel_path, should_write = _resolve_context_path( + comp_name, + item, + suffix, + used_paths, + identity_paths, + context_cfg["path_strategy"], + ranges, + symbols, + component_root=component_root, + use_external_root=use_external_root, + strip_prefix=effective_strip_prefix, + skip_external_root=skip_external_root, + external_root_prefix=spec_root, + ) + if from_files: + _record_recognized_reference( + reference_edges, + component_name=comp_name, item=item, - encounter_index=plugin_seen_counter, + context_path=rel_path, + registry_sources=get_registry_sources, ) - ) - plugin_seen_counter += 1 + plugin_identity = _plugin_pending_key(item, ranges, symbols) + if plugin_identity is not None: + plugin_pending.setdefault(plugin_identity, []).append( + _PluginPendingWrite( + rel_path=rel_path, + content=content, + should_write=should_write, + item=item, + encounter_index=plugin_seen_counter, + ) + ) + plugin_seen_counter += 1 + index_components.setdefault(comp_name, []).append( + _build_index_entry( + rel_path, + item, + ranges, + symbols, + content, + ) + ) + normalized_files.append( + _build_manifest_file_entry( + item, + range_spec, + symbols, + spec_comment, + file_opts, + ) + ) + continue + if should_write: + _queue_hydrated_file( + context_dir / rel_path, + item, + content, + copy_files=context_cfg["copy"], + ranges=ranges, + symbols=symbols, + files_to_write=files_to_write, + files_to_copy=files_to_copy, + files_to_symlink=files_to_symlink, + ) + file_ts = _item_file_ts(item) + if file_ts: + file_timestamps[context_dir / rel_path] = file_ts + dir_ts = _item_dir_ts(item) + if dir_ts: + file_timestamps.setdefault( + (context_dir / rel_path).parent, dir_ts + ) index_components.setdefault(comp_name, []).append( _build_index_entry( rel_path, @@ -834,45 +951,6 @@ def build_hydration_plan_data( file_opts, ) ) - continue - if should_write: - _queue_hydrated_file( - context_dir / rel_path, - item, - content, - copy_files=context_cfg["copy"], - ranges=ranges, - symbols=symbols, - files_to_write=files_to_write, - files_to_copy=files_to_copy, - files_to_symlink=files_to_symlink, - ) - file_ts = _item_file_ts(item) - if file_ts: - file_timestamps[context_dir / rel_path] = file_ts - dir_ts = _item_dir_ts(item) - if dir_ts: - file_timestamps.setdefault( - (context_dir / rel_path).parent, dir_ts - ) - index_components.setdefault(comp_name, []).append( - _build_index_entry( - rel_path, - item, - ranges, - symbols, - content, - ) - ) - normalized_files.append( - _build_manifest_file_entry( - item, - range_spec, - symbols, - spec_comment, - file_opts, - ) - ) if comp_manifests: from .contexts import ensure_manifest_link_hydrated @@ -936,6 +1014,17 @@ def build_hydration_plan_data( dest_rel = _dedupe_path(manifest_root / slug, used_paths) _ensure_relative(dest_rel) dirs_to_symlink.append((context_dir / dest_rel, canonical_dir)) + reference_edges.append( + { + "component": comp_name, + "form": "full", + "spec": str(manifest_spec), + "target_path": str(resolved_link_path), + "detected_via": "manifests-key", + "payload": "pointer", + "context_path": dest_rel.as_posix(), + } + ) if normalized_files: normalized_comp["files"] = _dedupe_manifest_entries(normalized_files) @@ -1006,6 +1095,10 @@ def build_hydration_plan_data( ) if manifest_source is not None: index_data["manifest_source"] = manifest_source + if manifest_format is not None and manifest_format.outline: + index_data["outline"] = manifest_format.outline + if reference_edges: + index_data["references"] = {"out": reference_edges} index_text = _dump_index(index_data) files_to_write.append((context_dir / "index.json", index_text)) @@ -1387,10 +1480,14 @@ def _build_normalized_component(comp: dict[str, Any], name: str) -> dict[str, An normalized = { k: v for k, v in comp.items() - if k not in {"files", "repos", "name", GROUP_PATH_KEY, GROUP_BASE_KEY} + if k + not in {"files", "repos", "name", GROUP_PATH_KEY, GROUP_BASE_KEY, SET_KEY} and v is not None } - normalized["name"] = name + if comp.get(SET_KEY): + normalized["set"] = name + else: + normalized["name"] = name return normalized @@ -1590,6 +1687,79 @@ def _find_global_subpath_prefix( return _find_common_subpath_prefix(all_subpaths) +def _record_recognized_reference( + edges: list[dict[str, Any]], + *, + component_name: str, + item: ResolvedItem, + context_path: Path, + registry_sources: Callable[[], set[Path]], +) -> None: + if item.source_type != "local" or not item.source_full_path: + return + path = Path(item.source_full_path) + if path.suffix.lower() not in _MANIFEST_REFERENCE_EXTENSIONS: + return + detected_via = _detect_manifest_reference(path, registry_sources) + if detected_via is None: + return + edges.append( + { + "component": component_name, + "form": "recognized", + "spec": item.manifest_spec, + "target_path": str(path), + "detected_via": detected_via, + "payload": "copy", + "context_path": context_path.as_posix(), + } + ) + + +def _detect_manifest_reference( + path: Path, registry_sources: Callable[[], set[Path]] +) -> str | None: + # registry_sources() can shell out to zk; local checks must run first + try: + text = path.read_text(encoding="utf-8") + except OSError: + text = None + if text is not None and frontmatter_has_manifest_tag(text): + return "frontmatter-tag" + if path_contains_manifest(path): + return "fenced-block" + try: + resolved = path.resolve() + except OSError: + resolved = path + if resolved in registry_sources(): + return "registry" + return None + + +def _load_registry_manifest_sources() -> set[Path]: + from .contexts import load_context_registry + + sources: set[Path] = set() + try: + registry = load_context_registry() + except Exception: + return sources + for entry in registry.values(): + manifest = entry.manifest if isinstance(entry.manifest, dict) else {} + raw_source = manifest.get("source") + if not isinstance(raw_source, str) or not raw_source: + continue + candidate = Path(os.path.expanduser(raw_source)) + if not candidate.is_absolute(): + candidate = entry.target_dir / candidate + try: + sources.add(candidate.resolve(strict=False)) + except OSError: + continue + return sources + + def _manifest_has_local_sources(components: list[dict[str, Any]]) -> bool: for comp in components: files = comp.get("files") @@ -2377,6 +2547,94 @@ def _build_component_root( return Path(component_name) +def _build_set_output_path( + component_name: str, + group_path: Any | None, + base_name: Any | None, +) -> Path: + if group_path: + parts = [group_path] if isinstance(group_path, str) else list(group_path) + name = base_name if isinstance(base_name, str) and base_name else component_name + return Path(*parts, f"{name}.md") + return Path(f"{component_name}.md") + + +def _format_set_part_header(item: ResolvedItem, title: str | None) -> str: + bits = [item.manifest_spec] + if title: + bits.append(title) + timestamp = item.source_modified or item.source_created + if timestamp: + bits.append(timestamp) + return f"--- {' · '.join(bits)} ---" + + +def _set_member_titles(outline_node: dict[str, Any] | None) -> list[str | None]: + if outline_node is None: + return [] + files = outline_node.get("members", {}).get("files", []) + return [member["inline_comment"] for member in files if not member["disabled"]] + + +def _build_set_fusion( + resolved_items: list[ + tuple[ + ResolvedItem, + str, + str | None, + list[tuple[int, int]] | None, + list[str] | None, + list[tuple[int, int]] | None, + str | None, + dict[str, Any], + str | None, + bool, + ] + ], + outline_node: dict[str, Any] | None = None, +) -> tuple[str, list[dict[str, Any]]]: + member_titles = _set_member_titles(outline_node) + parts: list[dict[str, Any]] = [] + blocks: list[str] = [] + line_cursor = 1 + for order, (item, content, *_rest) in enumerate(resolved_items): + spec_comment = _rest[4] + title = spec_comment or ( + member_titles[order] if order < len(member_titles) else None + ) + header = _format_set_part_header(item, title) + body = content if content.endswith("\n") else f"{content}\n" + block = f"{header}\n\n{body}" + line_start = line_cursor + line_end = line_start + block.count("\n") - 1 + line_cursor = line_end + 2 + blocks.append(block) + parts.append( + { + "order": order, + "key": item.manifest_spec, + "title": title, + "source_created": item.source_created, + "source_modified": item.source_modified, + "line_start": line_start, + "line_end": line_end, + } + ) + return "\n".join(blocks), parts + + +def _build_set_index_entry( + rel_path: Path, fused_text: str, parts: list[dict[str, Any]] +) -> dict[str, Any]: + digest = hashlib.sha256(fused_text.encode("utf-8")).hexdigest() + return { + "context_path": rel_path.as_posix(), + "kind": "set", + "hash": f"sha256:{digest}", + "parts": parts, + } + + def _resolve_context_path( component_name: str, item: ResolvedItem, diff --git a/src/contextualize/manifest/manifest.py b/src/contextualize/manifest/manifest.py index 5dddf1d..0f6ff30 100644 --- a/src/contextualize/manifest/manifest.py +++ b/src/contextualize/manifest/manifest.py @@ -4,6 +4,7 @@ GROUP_DELIMITER = "." GROUP_PATH_KEY = "__group_path" GROUP_BASE_KEY = "__group_base" +SET_KEY = "__is_set" _DEFAULT_KEYS = { "wrap", @@ -62,34 +63,55 @@ def collect_group_defaults(entry: dict[str, Any]) -> dict[str, Any]: return defaults def add_component( - entry: dict[str, Any], group_path: list[str], defaults: dict[str, Any] + entry: dict[str, Any], + group_path: list[str], + defaults: dict[str, Any], + *, + is_set: bool = False, ) -> None: + kind = "Set" if is_set else "Component" if "group" in entry: - raise ValueError("Component cannot define 'group'") + raise ValueError(f"{kind} cannot define 'group'") if "components" in entry: - raise ValueError("Component cannot define 'components' without 'group'") + raise ValueError(f"{kind} cannot define 'components' without 'group'") comp = dict(entry) - name = comp.get("name") + if is_set: + if "name" in comp: + raise ValueError("Set cannot also define 'name'") + name = comp.pop("set") + else: + name = comp.get("name") if name is None: name = next_auto_name() if not isinstance(name, str): - raise ValueError("Component name must be a string") + raise ValueError(f"{kind} name must be a string") name = name.strip() if not name: - raise ValueError("Component name must be a non-empty string") - validate_name(name, kind="Component name", allow_delimiter=True) + raise ValueError(f"{kind} name must be a non-empty string") + validate_name(name, kind=f"{kind} name", allow_delimiter=True) full_name = join_group_name(group_path, name) if full_name in used_names: raise ValueError(f"Duplicate component name: {full_name}") used_names.add(full_name) + if is_set: + for unsupported in ("repos", "manifests"): + if unsupported in comp: + raise ValueError( + f"Set '{full_name}' does not support '{unsupported}'" + ) + if not comp.get("files"): + raise ValueError(f"Set '{full_name}' must define 'files'") + for key, value in defaults.items(): if key not in comp: comp[key] = value comp["name"] = full_name + if is_set: + comp[SET_KEY] = True if group_path: comp[GROUP_PATH_KEY] = tuple(group_path) comp[GROUP_BASE_KEY] = name @@ -127,6 +149,8 @@ def process( merged_defaults = dict(defaults) merged_defaults.update(group_defaults) process(group_components, group_path + [group_name], merged_defaults) + elif "set" in entry: + add_component(entry, group_path, defaults, is_set=True) else: add_component(entry, group_path, defaults) diff --git a/src/contextualize/manifest/source.py b/src/contextualize/manifest/source.py index 6e39400..4ac3cd9 100644 --- a/src/contextualize/manifest/source.py +++ b/src/contextualize/manifest/source.py @@ -8,7 +8,7 @@ import subprocess from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Iterator import yaml @@ -25,6 +25,7 @@ class ManifestFormat: line: int source_path: str | None group_slices: dict[tuple[str, ...], ManifestSlice] + outline: list[dict[str, Any]] @dataclass(frozen=True) @@ -39,6 +40,11 @@ class ManifestSource: r"(?ms)^```(?P