feat(harvester): requirement extract and GitHub URL ingest

Optional requirement extract (off/auto/on) plus GitHub tarball/URL ingest
wiring for Module B → C. Stacked on #1088 librarian base.

Author: northdpole

application/cmd/cre_main.py

@@ -911,6 +911,25 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
             sys.exit(1)
         return
 
+    if str(getattr(args, "ingest_github", "") or "").strip():
+        import sys
+
+        from application import sqla
+        from application.utils.harvester.github_url_ingest import ingest_github_url
+
+        db_connect(args.cache_file)
+        counts = ingest_github_url(
+            str(args.ingest_github).strip(),
+            session=sqla.session,
+            cache_file=args.cache_file,
+            branch_override=(getattr(args, "ingest_branch", "") or "").strip() or None,
+            keep_all=bool(getattr(args, "ingest_keep_all", False)),
+            model_override=(getattr(args, "ingest_model", "") or "").strip() or None,
+        )
+        if counts.errors or (counts.knowledge + counts.uncertain) == 0:
+            sys.exit(1)
+        return
+
     if getattr(args, "run_noise_filter", False):
         import sys
 

application/tests/harvester_test/chunk_merger_test.py

@@ -0,0 +1,188 @@
+"""Tests for Module A.2 chunk merge profiles."""
+
+from __future__ import annotations
+
+import unittest
+from datetime import datetime, timezone
+
+from application.utils.harvester.chunk_merger import merge_chunks, requirement_ids
+from application.utils.harvester.models import (
+    ChunkInfo,
+    Document,
+    HeadingNode,
+    Locator,
+    SourceInfo,
+)
+from application.utils.harvester.schemas import ChunkingConfig
+
+
+def _doc(text: str, headings: list[HeadingNode] | None = None) -> Document:
+    return Document(
+        schema_version="0.2.0",
+        artifact_id="art:OWASP/Test:x.md",
+        pipeline_run_id="test",
+        text=text,
+        source=SourceInfo(
+            type="github",
+            repository="OWASP/Test",
+            commit_sha="abc",
+            committed_at=datetime.now(timezone.utc),
+        ),
+        locator=Locator(kind="repo_path", id="x.md", path="x.md"),
+        heading_structure=headings or [],
+    )
+
+
+class ChunkMergerTests(unittest.TestCase):
+    def test_requirement_ids_extracted(self) -> None:
+        self.assertEqual(
+            requirement_ids("**2.1.1** Verify passwords"), frozenset({"2.1.1"})
+        )
+        self.assertIn("2.1.1", requirement_ids("V2.1.1 something"))
+        self.assertEqual(
+            requirement_ids("AC-2 Account Management"), frozenset({"ac-2"})
+        )
+        self.assertIn("a.5.1", requirement_ids("ISO A.5.1 policies"))
+        self.assertIn("3.4", requirement_ids("Req 3.4 Protect stored"))
+        self.assertEqual(
+            requirement_ids("Requirement 3.4.1 hashing"), frozenset({"3.4.1"})
+        )
+
+    def test_requirements_does_not_merge_nist_or_iso_ids(self) -> None:
+        t1 = ("AC-2 Account Management. " * 20).strip()
+        t2 = ("AC-3 Access Enforcement. " * 20).strip()
+        text = t1 + "\n\n" + t2
+        mid = len(t1) + 2
+        chunks = [
+            ChunkInfo(text=t1, start_char_idx=0, end_char_idx=len(t1)),
+            ChunkInfo(text=t2, start_char_idx=mid, end_char_idx=len(text)),
+        ]
+        headings = [
+            HeadingNode(level=2, text="Access Control", start_line=1, end_line=99)
+        ]
+        doc = _doc(text, headings)
+        cfg = ChunkingConfig(
+            strategy="docling",
+            max_tokens=1200,
+            overlap_tokens=10,
+            merge_profile="requirements",
+        )
+        out = merge_chunks(doc, chunks, cfg)
+        self.assertEqual(len(out), 2)
+
+    def test_none_profile_is_noop(self) -> None:
+        text = "aaaa " * 50 + "\n\n" + "bbbb " * 50
+        mid = text.index("bbbb")
+        chunks = [
+            ChunkInfo(text=text[:mid], start_char_idx=0, end_char_idx=mid),
+            ChunkInfo(text=text[mid:], start_char_idx=mid, end_char_idx=len(text)),
+        ]
+        cfg = ChunkingConfig(
+            strategy="docling", max_tokens=1200, overlap_tokens=10, merge_profile="none"
+        )
+        doc = _doc(text)
+        out = merge_chunks(doc, chunks, cfg)
+        self.assertEqual(len(out), 2)
+
+    def test_requirements_does_not_merge_across_ids(self) -> None:
+        t1 = ("Verify 2.1.1 passwords are long. " * 20).strip()
+        t2 = ("Verify 2.1.2 passwords allow unicode. " * 20).strip()
+        text = t1 + "\n\n" + t2
+        mid = len(t1) + 2
+        chunks = [
+            ChunkInfo(text=t1, start_char_idx=0, end_char_idx=len(t1)),
+            ChunkInfo(text=t2, start_char_idx=mid, end_char_idx=len(text)),
+        ]
+        headings = [
+            HeadingNode(
+                level=2, text="V2.1 Password Security", start_line=1, end_line=99
+            )
+        ]
+        doc = _doc(text, headings)
+        cfg = ChunkingConfig(
+            strategy="docling",
+            max_tokens=1200,
+            overlap_tokens=10,
+            merge_profile="requirements",
+        )
+        out = merge_chunks(doc, chunks, cfg)
+        self.assertEqual(len(out), 2)
+
+    def test_narrative_merges_same_heading_crumbs(self) -> None:
+        # Same heading, tiny crumbs → one merged chunk under narrative profile.
+        a = "Short intro."
+        b = "More detail on the same topic."
+        text = a + "\n\n" + b
+        mid = len(a) + 2
+        chunks = [
+            ChunkInfo(text=a, start_char_idx=0, end_char_idx=len(a)),
+            ChunkInfo(text=b, start_char_idx=mid, end_char_idx=len(text)),
+        ]
+        headings = [
+            HeadingNode(level=2, text="Password Storage", start_line=1, end_line=99)
+        ]
+        doc = _doc(text, headings)
+        cfg = ChunkingConfig(
+            strategy="docling",
+            max_tokens=1000,
+            overlap_tokens=10,
+            merge_profile="narrative",
+        )
+        out = merge_chunks(doc, chunks, cfg)
+        self.assertEqual(len(out), 1)
+        self.assertIn("Short intro", out[0].text)
+        self.assertIn("More detail", out[0].text)
+
+    def test_narrative_merges_h3_under_same_h2(self) -> None:
+        text = (
+            "## Auth\n\n"
+            "### Intro\n\n"
+            "First crumb about auth.\n\n"
+            "### Detail\n\n"
+            "Second crumb still under Auth."
+        )
+        # Approximate char ranges for two leaves under different ### but same ##.
+        first = "First crumb about auth."
+        second = "Second crumb still under Auth."
+        i1 = text.index(first)
+        i2 = text.index(second)
+        chunks = [
+            ChunkInfo(text=first, start_char_idx=i1, end_char_idx=i1 + len(first)),
+            ChunkInfo(text=second, start_char_idx=i2, end_char_idx=i2 + len(second)),
+        ]
+        headings = [
+            HeadingNode(level=2, text="Auth", start_line=1, end_line=20),
+            HeadingNode(level=3, text="Intro", start_line=3, end_line=6),
+            HeadingNode(level=3, text="Detail", start_line=8, end_line=20),
+        ]
+        doc = _doc(text, headings)
+        cfg = ChunkingConfig(
+            strategy="docling",
+            max_tokens=1000,
+            overlap_tokens=10,
+            merge_profile="narrative",
+        )
+        out = merge_chunks(doc, chunks, cfg)
+        self.assertEqual(len(out), 1)
+
+    def test_profile_defaults_on_config(self) -> None:
+        req = ChunkingConfig(
+            strategy="fixed_size",
+            max_tokens=500,
+            overlap_tokens=10,
+            merge_profile="requirements",
+        )
+        self.assertTrue(req.split_on_requirement_id)
+        self.assertEqual(req.merge_min_tokens, 100)
+        nar = ChunkingConfig(
+            strategy="fixed_size",
+            max_tokens=500,
+            overlap_tokens=10,
+            merge_profile="narrative",
+        )
+        self.assertFalse(nar.split_on_requirement_id)
+        self.assertEqual(nar.merge_min_tokens, 400)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/chunk_record_builder_test.py

@@ -56,11 +56,46 @@ def test_builds_change_record_shaped_payload(self) -> None:
         self.assertEqual(record.source_repo, "OWASP/ASVS")
         self.assertEqual(record.locator_path, "README.md")
         self.assertEqual(record.span.heading_path, ["Root"])
+        self.assertTrue(record.text.startswith("Standard: ASVS\n"))
+        self.assertIn("Source: README.md\n", record.text)
+        self.assertIn("Section: Root\n", record.text)
+        self.assertIn("First paragraph.", record.text)
 
         ChunkRecordValidator().validate(record)
         payload = ingest_record_to_payload(record)
         ChangeRecord.model_validate(payload)
 
+    def test_prefixes_section_id_from_filename(self) -> None:
+        text = "# Broken Access Control\n\nDeny by default."
+        document = Document(
+            schema_version="0.2.0",
+            artifact_id="art:B2/owasp_top10_2025:A01",
+            pipeline_run_id="run-1",
+            text=text,
+            source=SourceInfo(
+                type="github",
+                repository="B2/owasp_top10_2025",
+                commit_sha="abc1234",
+                committed_at=datetime(2026, 2, 1, 1, 0, 0, tzinfo=timezone.utc),
+            ),
+            locator=Locator(
+                kind="repo_path",
+                id="b2/owasp_top10_2025/A01.txt",
+                path="b2/owasp_top10_2025/A01.txt",
+            ),
+            heading_structure=[
+                HeadingNode(
+                    level=1, text="Broken Access Control", start_line=1, end_line=3
+                )
+            ],
+        )
+        chunk = ChunkInfo(text=text, start_char_idx=0, end_char_idx=len(text))
+        record = ChunkRecordBuilder().build(document, [chunk])[0]
+        self.assertIn("Standard: owasp_top10_2025", record.text)
+        self.assertIn("Version: 2025", record.text)
+        self.assertIn("Section-ID: A01", record.text)
+        self.assertIn("Section: Broken Access Control", record.text)
+
     def test_indexes_multiple_chunks(self) -> None:
         text = "AAAA\n\nBBBB"
         document = self._document(text, [])
@@ -74,6 +109,52 @@ def test_indexes_multiple_chunks(self) -> None:
         self.assertEqual(records[1].span.index, 1)
         self.assertEqual(records[1].chunk_id, "chk:art:OWASP/ASVS:README.md:1")
 
+    def test_requirement_extract_chunk_keeps_single_section_id(self) -> None:
+        """Do not re-scrape heading/path ids over a clean extractor prefix."""
+        body = (
+            "Source: V1.1.2\n"
+            "Section-ID: V1.1.2\n"
+            "Section: Verify that output encoding is applied.\n\n"
+            "Verify that output encoding is applied."
+        )
+        document = Document(
+            schema_version="0.2.0",
+            artifact_id="art:OWASP/ASVS:0x10-V1.md",
+            pipeline_run_id="run-1",
+            text="# V1 Encoding\n\n## V1.1 Architecture\n\n" + body,
+            source=SourceInfo(
+                type="github",
+                repository="OWASP/ASVS",
+                commit_sha="abc1234deadbeef",
+                committed_at=datetime(2026, 2, 1, 1, 0, 0, tzinfo=timezone.utc),
+            ),
+            locator=Locator(
+                kind="repo_path",
+                id="0x10-V1.md",
+                path="5.0/en/0x10-V1-Encoding-and-Sanitization.md",
+            ),
+            heading_structure=[
+                HeadingNode(level=1, text="V1 Encoding", start_line=1, end_line=10),
+                HeadingNode(
+                    level=2, text="V1.1 Architecture", start_line=3, end_line=10
+                ),
+            ],
+        )
+        chunk = ChunkInfo(text=body, start_char_idx=0, end_char_idx=len(body))
+        record = ChunkRecordBuilder().build(document, [chunk])[0]
+        sid_lines = [
+            ln for ln in record.text.splitlines() if ln.startswith("Section-ID:")
+        ]
+        src_lines = [ln for ln in record.text.splitlines() if ln.startswith("Source:")]
+        # Do not prepend a second Source:/Section-ID: over extractor identity.
+        self.assertEqual(len(sid_lines), 1)
+        self.assertEqual(sid_lines[0], "Section-ID: V1.1.2")
+        self.assertEqual(len(src_lines), 1)
+        self.assertEqual(src_lines[0], "Source: V1.1.2")
+        self.assertNotIn("V1.1,", record.text)
+        self.assertNotIn("5.0,", record.text)
+        self.assertNotIn("0x10-V1.md", record.text)
+
 
 if __name__ == "__main__":
     unittest.main()

application/tests/harvester_test/config_loader_test.py

@@ -99,9 +99,8 @@ def test_load_packaged_repos_yaml(self):
 
         config = load_repo_config(config_path)
 
-        self.assertEqual(len(config.repositories), 2)
-        self.assertEqual(config.repositories[0].id, "owasp-asvs")
-        self.assertEqual(config.repositories[1].id, "owasp-cheatsheets")
+        ids = [repo.id for repo in config.repositories]
+        self.assertEqual(ids, ["owasp-asvs", "owasp-aisvs", "owasp-cheatsheets"])
 
     def test_empty_owner(self):
         config_path = FIXTURES_DIR / "empty_owner.yaml"

application/tests/harvester_test/github_url_ingest_test.py

@@ -0,0 +1,51 @@
+"""Unit tests for point-and-shoot GitHub URL ingest helpers."""
+
+from __future__ import annotations
+
+import unittest
+
+from application.utils.harvester.github_url_ingest import (
+    format_knowledge_item,
+    parse_github_url,
+)
+
+
+class ParseGitHubUrlTests(unittest.TestCase):
+    def test_owner_repo(self) -> None:
+        p = parse_github_url("https://github.com/OWASP/ASVS")
+        self.assertEqual(p.owner, "OWASP")
+        self.assertEqual(p.repo, "ASVS")
+        self.assertIsNone(p.branch)
+        self.assertIsNone(p.path_prefix)
+
+    def test_git_suffix_and_www(self) -> None:
+        p = parse_github_url("https://www.github.com/OWASP/ASVS.git")
+        self.assertEqual(p.owner, "OWASP")
+        self.assertEqual(p.repo, "ASVS")
+
+    def test_tree_branch_and_path(self) -> None:
+        p = parse_github_url("https://github.com/OWASP/ASVS/tree/master/5.0/en")
+        self.assertEqual(p.branch, "master")
+        self.assertEqual(p.path_prefix, "5.0/en")
+
+    def test_rejects_non_github(self) -> None:
+        with self.assertRaises(ValueError):
+            parse_github_url("https://gitlab.com/OWASP/ASVS")
+
+
+class FormatKnowledgeItemTests(unittest.TestCase):
+    def test_includes_path_and_label(self) -> None:
+        out = format_knowledge_item(
+            path="5.0/en/V1.md",
+            label="KNOWLEDGE",
+            confidence=0.91,
+            text="Verify that input is sanitized.",
+            reasoning="looks like a control",
+        )
+        self.assertIn("path: 5.0/en/V1.md", out)
+        self.assertIn("label: KNOWLEDGE", out)
+        self.assertIn("Verify that input is sanitized.", out)
+
+
+if __name__ == "__main__":
+    unittest.main()