perf(setup): unzip only manual html bake inputs (#42)

Author: Razpines

docs/README.md

@@ -11,7 +11,7 @@ This file contains the advanced, developer-oriented details that were removed fr
 
 ## Where files are stored / disk usage
 - Zip file: `data/unity/<version>/raw/UnityDocumentation.zip`
-- Unzipped HTML: `data/unity/<version>/raw/UnityDocumentation/`
+- Unzipped HTML: `data/unity/<version>/raw/UnityDocumentation/` (currently Manual HTML pages only)
 - Baked artifacts: `data/unity/<version>/baked/`
 - Index artifacts: `data/unity/<version>/index/`
 

src/unity_docs_mcp/setup/ensure_artifacts.py

@@ -12,6 +12,11 @@
 from unity_docs_mcp.setup.download import download_zip
 from unity_docs_mcp.setup.unzip import safe_unzip
 
+_BAKE_INPUT_GLOBS = [
+    "Documentation/en/Manual/*.html",
+    "Documentation/en/Manual/**/*.html",
+]
+
 
 def _manifest_matches(path: Path, signature: str) -> bool:
     if not path.exists():
@@ -24,17 +29,23 @@ def _manifest_matches(path: Path, signature: str) -> bool:
 
 
 def _raw_docs_ready(raw_unzipped: Path) -> bool:
-    return (raw_unzipped / "Documentation" / "en").is_dir()
+    return (raw_unzipped / "Documentation" / "en" / "Manual" / "index.html").is_file()
 
 
-def _recover_unzip(download_url: str, raw_zip: Path, raw_unzipped: Path, error: Exception) -> None:
+def _recover_unzip(
+    download_url: str,
+    raw_zip: Path,
+    raw_unzipped: Path,
+    include_globs: list[str],
+    error: Exception,
+) -> None:
     print(f"[setup] Unzip failed ({error}). Re-downloading zip and retrying once...")
     if raw_unzipped.exists():
         shutil.rmtree(raw_unzipped, ignore_errors=True)
     if raw_zip.exists():
         raw_zip.unlink()
     download_zip(download_url, raw_zip, overwrite=True)
-    safe_unzip(raw_zip, raw_unzipped)
+    safe_unzip(raw_zip, raw_unzipped, include_globs=include_globs)
 
 
 def ensure(config: Config) -> None:
@@ -54,9 +65,15 @@ def ensure(config: Config) -> None:
             if paths.raw_unzipped.exists():
                 shutil.rmtree(paths.raw_unzipped, ignore_errors=True)
             try:
-                safe_unzip(paths.raw_zip, paths.raw_unzipped)
+                safe_unzip(paths.raw_zip, paths.raw_unzipped, include_globs=_BAKE_INPUT_GLOBS)
             except Exception as unzip_error:
-                _recover_unzip(config.download_url, paths.raw_zip, paths.raw_unzipped, unzip_error)
+                _recover_unzip(
+                    config.download_url,
+                    paths.raw_zip,
+                    paths.raw_unzipped,
+                    include_globs=_BAKE_INPUT_GLOBS,
+                    error=unzip_error,
+                )
 
         print("==> Baking docs (HTML -> cleaned text + chunks)...")
         bake(config)

src/unity_docs_mcp/setup/unzip.py

@@ -1,29 +1,73 @@
 from __future__ import annotations
 
+import fnmatch
 import zipfile
 from pathlib import Path
+from typing import Iterable, Sequence
 
 from tqdm import tqdm
 
 
-def safe_unzip(zip_path: Path, target_dir: Path) -> Path:
+def _normalize_patterns(patterns: Sequence[str] | None) -> list[str]:
+    if not patterns:
+        return []
+    normalized: list[str] = []
+    for pattern in patterns:
+        value = str(pattern).strip().replace("\\", "/").lstrip("/")
+        if value:
+            normalized.append(value)
+    return normalized
+
+
+def _glob_variants(pattern: str) -> Iterable[str]:
+    # pathlib/fnmatch treat `/**/` as "one or more" directories; include a
+    # zero-directory variant so `Manual/**/*.html` also matches `Manual/index.html`.
+    yield pattern
+    if "/**/" in pattern:
+        yield pattern.replace("/**/", "/")
+
+
+def _member_selected(member_name: str, include_globs: Sequence[str] | None) -> bool:
+    if not include_globs:
+        return True
+    normalized = member_name.replace("\\", "/").lstrip("/")
+    if normalized.endswith("/"):
+        return False
+    for pattern in include_globs:
+        for variant in _glob_variants(pattern):
+            if fnmatch.fnmatch(normalized, variant):
+                return True
+    return False
+
+
+def safe_unzip(
+    zip_path: Path,
+    target_dir: Path,
+    include_globs: Sequence[str] | None = None,
+) -> Path:
     """
     Safely extract zip contents, preventing zip-slip by validating paths.
     """
     if target_dir.exists() and any(target_dir.iterdir()):
         return target_dir
 
-    print(f"==> Unzipping {zip_path}...")
+    selected_globs = _normalize_patterns(include_globs)
     target_dir.mkdir(parents=True, exist_ok=True)
     with zipfile.ZipFile(zip_path, "r") as zf:
         members = zf.infolist()
-        for member in members:
+        selected_members = [m for m in members if _member_selected(m.filename, selected_globs)]
+        print(
+            f"==> Unzipping {zip_path} "
+            f"({len(selected_members)}/{len(members)} files selected)..."
+        )
+        target_root = target_dir.resolve()
+        for member in selected_members:
             extracted_path = target_dir / member.filename
             resolved_path = extracted_path.resolve()
-            if target_dir.resolve() not in resolved_path.parents and target_dir.resolve() != resolved_path:
+            if target_root not in resolved_path.parents and target_root != resolved_path:
                 raise ValueError(f"Unsafe path detected in zip: {member.filename}")
-        bar = tqdm(total=len(members), unit="file", unit_scale=False)
-        for member in members:
+        bar = tqdm(total=len(selected_members), unit="file", unit_scale=False)
+        for member in selected_members:
             zf.extract(member, target_dir)
             bar.update(1)
         bar.close()

tests/test_setup_recovery.py

@@ -32,13 +32,16 @@ def fake_download(url: str, destination: Path, overwrite: bool = False, progress
         destination.write_bytes(b"fresh")
         return destination
 
-    def fake_unzip(zip_path: Path, target_dir: Path) -> Path:
+    def fake_unzip(zip_path: Path, target_dir: Path, include_globs=None) -> Path:
         calls["unzip"] += 1
+        assert include_globs == ensure_artifacts._BAKE_INPUT_GLOBS
         if calls["unzip"] == 1:
             target_dir.mkdir(parents=True, exist_ok=True)
             (target_dir / "partial.txt").write_text("partial", encoding="utf-8")
             raise RuntimeError("bad zip payload")
-        (target_dir / "Documentation" / "en").mkdir(parents=True, exist_ok=True)
+        manual_dir = target_dir / "Documentation" / "en" / "Manual"
+        manual_dir.mkdir(parents=True, exist_ok=True)
+        (manual_dir / "index.html").write_text("<html></html>", encoding="utf-8")
         return target_dir
 
     monkeypatch.setattr(ensure_artifacts, "download_zip", fake_download)
@@ -50,7 +53,7 @@ def fake_unzip(zip_path: Path, target_dir: Path) -> Path:
 
     assert calls["unzip"] == 2
     assert calls["download_overwrite"] == [True]
-    assert (raw_unzipped / "Documentation" / "en").is_dir()
+    assert (raw_unzipped / "Documentation" / "en" / "Manual" / "index.html").is_file()
     assert not (raw_unzipped / "partial.txt").exists()
 
 
@@ -65,10 +68,13 @@ def test_nonready_unzip_dir_is_cleared_before_unzip(monkeypatch, tmp_path: Path)
 
     calls = {"unzip": 0}
 
-    def fake_unzip(zip_path: Path, target_dir: Path) -> Path:
+    def fake_unzip(zip_path: Path, target_dir: Path, include_globs=None) -> Path:
         calls["unzip"] += 1
+        assert include_globs == ensure_artifacts._BAKE_INPUT_GLOBS
         assert not (target_dir / "stale.tmp").exists()
-        (target_dir / "Documentation" / "en").mkdir(parents=True, exist_ok=True)
+        manual_dir = target_dir / "Documentation" / "en" / "Manual"
+        manual_dir.mkdir(parents=True, exist_ok=True)
+        (manual_dir / "index.html").write_text("<html></html>", encoding="utf-8")
         return target_dir
 
     monkeypatch.setattr(ensure_artifacts, "safe_unzip", fake_unzip)
@@ -79,4 +85,4 @@ def fake_unzip(zip_path: Path, target_dir: Path) -> Path:
     ensure_artifacts.ensure(cfg)
 
     assert calls["unzip"] == 1
-    assert (raw_unzipped / "Documentation" / "en").is_dir()
+    assert (raw_unzipped / "Documentation" / "en" / "Manual" / "index.html").is_file()

tests/test_unzip_filtering.py

@@ -0,0 +1,55 @@
+import zipfile
+from pathlib import Path
+
+import pytest
+
+from unity_docs_mcp.setup.unzip import safe_unzip
+
+
+def _write_zip(path: Path, members: dict[str, str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(path, "w") as zf:
+        for name, body in members.items():
+            zf.writestr(name, body)
+
+
+def test_safe_unzip_extracts_only_selected_manual_html(tmp_path: Path):
+    zip_path = tmp_path / "UnityDocumentation.zip"
+    target_dir = tmp_path / "unzipped"
+    _write_zip(
+        zip_path,
+        {
+            "Documentation/en/Manual/index.html": "<html>manual index</html>",
+            "Documentation/en/Manual/Sub/page.html": "<html>manual page</html>",
+            "Documentation/en/Manual/readme.txt": "not html",
+            "Documentation/en/ScriptReference/index.html": "<html>script ref</html>",
+        },
+    )
+
+    safe_unzip(
+        zip_path,
+        target_dir,
+        include_globs=[
+            "Documentation/en/Manual/*.html",
+            "Documentation/en/Manual/**/*.html",
+        ],
+    )
+
+    assert (target_dir / "Documentation" / "en" / "Manual" / "index.html").exists()
+    assert (target_dir / "Documentation" / "en" / "Manual" / "Sub" / "page.html").exists()
+    assert not (target_dir / "Documentation" / "en" / "Manual" / "readme.txt").exists()
+    assert not (target_dir / "Documentation" / "en" / "ScriptReference" / "index.html").exists()
+
+
+def test_safe_unzip_keeps_zip_slip_protection(tmp_path: Path):
+    zip_path = tmp_path / "UnityDocumentation.zip"
+    target_dir = tmp_path / "unzipped"
+    _write_zip(
+        zip_path,
+        {
+            "../escape.txt": "bad",
+        },
+    )
+
+    with pytest.raises(ValueError, match="Unsafe path detected"):
+        safe_unzip(zip_path, target_dir)