diff --git a/docs/DOCUMENT_CITATIONS.md b/docs/DOCUMENT_CITATIONS.md new file mode 100644 index 000000000..9bc095b38 --- /dev/null +++ b/docs/DOCUMENT_CITATIONS.md @@ -0,0 +1,65 @@ +# PDF page citations + +Document enrichment can attach an optional `locator` to an observation: + +```json +{ + "category": "fact", + "content": "Revenue increased.", + "locator": {"page": 2, "page_label": "iv"} +} +``` + +`page` is the one-based physical PDF page, not the printed page label. Trusted +assembly checks it against the extraction's page count and derives the destination +from the source file path. The agent does not supply a citation URL. + +The resulting document retains its existing `source` checksum and storage-version +provenance and adds OKF-compatible `sources` entries and Markdown footnotes: + +```yaml +sources: + - id: document-page-2 + resource: /docs/report.pdf#page=2 + title: report.pdf, p. iv + locator: + page: 2 + page_label: iv +``` + +```markdown +## Observations + +- [fact] Revenue increased. [^document-page-2] + +[^document-page-2]: [PDF page 2](/docs/report.pdf#page=2) +``` + +The footnote label joins to `sources[].id`, not the entry's array position. IDs are +scoped to this document note and its single trusted source PDF. Multiple +observations on the same page share an entry. Reordering observations or sources +does not change the target. Conflicting printed labels for the same page are +rejected. Uncited documents retain their existing serialized shape. + +The leading slash denotes a project/bundle-root-relative resource. Consumers must +resolve it in that scope; it is not a new Cloud HTTP route. The fragment requests +the physical page in a supporting PDF viewer. The existing source checksum and +storage version identify the bytes used for extraction; the link itself does not +retrieve a historical version or automatically detect replacement of the PDF. + +## Scope and prior art + +This implements page-level addressing for #1366 and the citation convention from +the amended SPEC-89. It does not implement an extraction offset map, quote +highlighting, generic OKF conformance (#1246), evidence watermarks, or contradiction +detection. Cloud prompt adoption and viewer routing are separate integration work. + +- [RFC 8118, section 3](https://www.rfc-editor.org/rfc/rfc8118.html#section-3) + defines the PDF `page=N` fragment with one-based page numbering. +- [OKF provenance sources](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md#51-provenance-sources) + provides `sources[].id`, resource references, and footnote-label joins. +- [Zotero PDF reader](https://www.zotero.org/support/pdf_reader) demonstrates the + annotation/note-to-source-page navigation experience. +- [W3C TextQuoteSelector](https://www.w3.org/TR/annotation-model/#text-quote-selector) + is prior art for a later, more precise exact-text/context selector; no W3C + selector is implemented here. diff --git a/src/basic_memory/schemas/__init__.py b/src/basic_memory/schemas/__init__.py index b4f734dd3..9c7bf45b5 100644 --- a/src/basic_memory/schemas/__init__.py +++ b/src/basic_memory/schemas/__init__.py @@ -59,6 +59,7 @@ DocumentAgentObservationV1, DocumentAgentOutputV1, DocumentAgentRelationV1, + DocumentCitationSourceV1, DocumentExtractionStatus, DocumentExtractionV1, DocumentIngestionFailureV1, @@ -71,6 +72,7 @@ DocumentMarkdownV1, DocumentMetadataV1, DocumentNoteFrontmatterV1, + DocumentPageLocatorV1, DocumentRevisionKind, DocumentRevisionReferenceV1, DocumentSourceV1, @@ -124,6 +126,7 @@ "DocumentAgentObservationV1", "DocumentAgentOutputV1", "DocumentAgentRelationV1", + "DocumentCitationSourceV1", "DocumentExtractionStatus", "DocumentExtractionV1", "DocumentIngestionFailureV1", @@ -136,6 +139,7 @@ "DocumentMarkdownV1", "DocumentMetadataV1", "DocumentNoteFrontmatterV1", + "DocumentPageLocatorV1", "DocumentRevisionKind", "DocumentRevisionReferenceV1", "DocumentSourceV1", diff --git a/src/basic_memory/schemas/document.py b/src/basic_memory/schemas/document.py index 6c5e8eef5..c0535fc36 100644 --- a/src/basic_memory/schemas/document.py +++ b/src/basic_memory/schemas/document.py @@ -13,6 +13,7 @@ from enum import StrEnum from pathlib import PurePosixPath, PureWindowsPath from typing import TYPE_CHECKING, Annotated, Literal +from urllib.parse import quote from uuid import NAMESPACE_URL, UUID, uuid5 from frontmatter import Post @@ -196,6 +197,28 @@ def require_unique_authors(cls, value: tuple[str, ...]) -> tuple[str, ...]: return value +class DocumentPageLocatorV1(_DocumentContractModel): + """A one-based physical PDF page, separate from its printed page label.""" + + page: int = Field(ge=1, strict=True) + page_label: Annotated[NonEmptyText, StringConstraints(pattern=r"^[^\r\n]+$")] | None = None + + @property + def source_id(self) -> str: + # IDs are note-scoped and keyed to physical pages, never the position of + # an entry in sources. Reordering observations cannot misattribute them. + return f"document-page-{self.page}" + + +class DocumentCitationSourceV1(_DocumentContractModel): + """OKF source entry for a cited page of the note's trusted source PDF.""" + + id: NonEmptyText + resource: NonEmptyText + title: NonEmptyText + locator: DocumentPageLocatorV1 + + class DocumentNoteFrontmatterV1(_DocumentContractModel): """Authoritative nested frontmatter for a ``type: document`` note.""" @@ -212,6 +235,8 @@ class DocumentNoteFrontmatterV1(_DocumentContractModel): created: datetime | None = None modified: datetime | None = None source: DocumentSourceV1 + # None keeps legacy uncited document serialization and checksums unchanged. + sources: tuple[DocumentCitationSourceV1, ...] | None = None extraction: DocumentExtractionV1 ingestion: DocumentIngestionV1 document: DocumentMetadataV1 = Field(default_factory=DocumentMetadataV1) @@ -235,6 +260,13 @@ def require_aware_canonical_timestamps(cls, value: datetime | None) -> datetime def validate_trusted_envelope(self) -> "DocumentNoteFrontmatterV1": if self.source.checksum != self.ingestion.input_checksum: raise ValueError("ingestion input_checksum must match the source checksum") + if self.sources: + if len({citation.id for citation in self.sources}) != len(self.sources): + raise ValueError("citation source IDs must be unique") + for citation in self.sources: + expected = _document_citation_source(self.source, self.extraction, citation.locator) + if citation != expected: + raise ValueError("citation must reference its trusted source PDF page") _require_deterministic_run_id( source=self.source, extraction=self.extraction, @@ -447,6 +479,7 @@ class DocumentAgentObservationV1(_DocumentContractModel): StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^\r\n]+$"), ] tags: tuple[AgentTag, ...] = () + locator: DocumentPageLocatorV1 | None = None context: ( Annotated[ StrictStr, @@ -463,6 +496,8 @@ def require_exact_parser_semantics(self) -> "DocumentAgentObservationV1": parsed_observation = parsed.observations[0] expected_content = self.content + if self.locator is not None: + expected_content += f" [^{self.locator.source_id}]" if self.tags: expected_content += " " + " ".join(f"#{tag}" for tag in self.tags) if ( @@ -546,6 +581,20 @@ def reject_unstructured_semantics(cls, value: str) -> str: @model_validator(mode="after") def require_exact_assembled_semantics(self) -> "DocumentAgentOutputV1": + if any(observation.locator is not None for observation in self.observations): + # Reserve both references and definitions: otherwise undeclared text + # can borrow a generated citation without supplying a locator. + agent_text = [self.title, self.body] + for observation in self.observations: + agent_text.extend((observation.content, observation.context or "")) + for relation in self.relations: + agent_text.extend((relation.target, relation.context or "")) + if any( + re.search(r"\[\^document-page-[0-9]+\]", text, re.IGNORECASE) for text in agent_text + ): + raise ValueError( + "agent text cannot define generated document-page citations or references" + ) parsed = _parse_agent_semantics(_assemble_agent_body(self)) expected_observations = [ _parse_agent_semantics(_format_agent_observation(observation)).observations[0] @@ -720,6 +769,22 @@ def enrich_document_markdown( raise ValueError("enrichment cannot replace trusted ingestion identity or version fields") tags = tuple(dict.fromkeys((*raw_frontmatter.tags, *agent_output.tags))) + citations: dict[str, DocumentCitationSourceV1] = {} + for observation in agent_output.observations: + if observation.locator is None: + continue + citation = _document_citation_source( + raw_frontmatter.source, raw_frontmatter.extraction, observation.locator + ) + existing = citations.get(citation.id) + if existing is not None and existing.locator.page_label is not None: + if citation.locator.page_label not in {None, existing.locator.page_label}: + raise ValueError("observations citing the same page must agree on its page label") + # An omitted label contributes no conflicting information. Keep the + # explicit label regardless of observation order. + continue + citations[citation.id] = citation + citation_sources = tuple(citations.values()) return DocumentMarkdownV1( frontmatter=DocumentNoteFrontmatterV1( title=agent_output.title, @@ -728,16 +793,42 @@ def enrich_document_markdown( created=raw_frontmatter.created, modified=raw_frontmatter.modified, source=raw_frontmatter.source, + sources=citation_sources or None, extraction=raw_frontmatter.extraction, ingestion=target_ingestion, document=agent_output.document, bm_parse_semantics=True, ), - body=_assemble_agent_body(agent_output), + body=_assemble_agent_body(agent_output, citation_sources), + ) + + +def _document_citation_source( + source: DocumentSourceV1, + extraction: DocumentExtractionV1, + locator: DocumentPageLocatorV1, +) -> DocumentCitationSourceV1: + """Build a standard PDF fragment from trusted provenance, not an agent URL.""" + if source.media_type != "application/pdf": + raise ValueError("page citations require a PDF source") + if locator.page > extraction.page_count: + raise ValueError("citation page is outside the source PDF") + # RFC 8118 page= uses physical one-based pages, not printed page labels. + # Encode filename delimiters so a literal # or % cannot change the target. + resource = f"/{quote(source.file_path, safe='/')}#page={locator.page}" + label = locator.page_label or str(locator.page) + return DocumentCitationSourceV1( + id=locator.source_id, + resource=resource, + title=f"{PurePosixPath(source.file_path).name}, p. {label}", + locator=locator, ) -def _assemble_agent_body(agent_output: DocumentAgentOutputV1) -> str: +def _assemble_agent_body( + agent_output: DocumentAgentOutputV1, + citations: tuple[DocumentCitationSourceV1, ...] = (), +) -> str: sections: list[str] = [] if normalized_body := _normalize_markdown_body(agent_output.body): sections.append(normalized_body.rstrip("\n")) @@ -749,11 +840,21 @@ def _assemble_agent_body(agent_output: DocumentAgentOutputV1) -> str: if agent_output.relations: relations = [_format_agent_relation(relation) for relation in agent_output.relations] sections.append("## Relations\n\n" + "\n".join(relations)) + if citations: + sections.append( + "\n".join( + f"[^{citation.id}]: [PDF page {citation.locator.page}]({citation.resource})" + for citation in citations + ) + ) return "\n\n".join(sections) + ("\n" if sections else "") def _format_agent_observation(observation: DocumentAgentObservationV1) -> str: line = f"- [{observation.category}] {observation.content}" + if observation.locator is not None: + # A space prevents a trailing Markdown escape from swallowing the marker. + line += f" [^{observation.locator.source_id}]" if observation.tags: line += " " + " ".join(f"#{tag}" for tag in observation.tags) if observation.context: diff --git a/test-int/test_document_page_citations.py b/test-int/test_document_page_citations.py new file mode 100644 index 000000000..bbf618c41 --- /dev/null +++ b/test-int/test_document_page_citations.py @@ -0,0 +1,454 @@ +"""PDF extraction -> enrichment -> portable citations -> real note API regressions.""" + +import hashlib +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import unquote, urlsplit +from uuid import UUID + +from httpx import AsyncClient +from markdown_it import MarkdownIt +from mdit_py_plugins.footnote import footnote_plugin +from pydantic import ValidationError +import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from basic_memory.document_ingestion.pdf_inspector import PdfInspectorLimits +from basic_memory.document_ingestion.pdf_inspector_worker import inspect_pdf_bytes +from basic_memory.document_ingestion.raw_document import ( + DocumentSourceEntity, + DocumentSourceSnapshot, + build_raw_document_artifacts, +) +from basic_memory.markdown.entity_parser import parse +from basic_memory.models import Observation, Project +from basic_memory.schemas.document import ( + DocumentAgentObservationV1, + DocumentAgentOutputV1, + DocumentAgentRelationV1, + DocumentIngestionStage, + DocumentIngestionV1, + DocumentMarkdownV1, + DocumentPageLocatorV1, + assemble_document_markdown, + document_markdown_checksum, + enrich_document_markdown, + parse_document_markdown, +) + + +@pytest.fixture +def raw_pdf(test_project: Project) -> DocumentMarkdownV1: + pdf_bytes = (Path(__file__).parents[1] / "tests/Non-MarkdownFileSupport.pdf").read_bytes() + source_path = Path(test_project.path) / "docs/report #1%.pdf" + source_path.parent.mkdir() + source_path.write_bytes(pdf_bytes) + source = DocumentSourceSnapshot( + entity=DocumentSourceEntity( + entity_id=1, + external_id=UUID("11111111-1111-1111-1111-111111111111"), + file_path="docs/report #1%.pdf", + media_type="application/pdf", + ), + content=pdf_bytes, + checksum=f"sha256:{hashlib.sha256(pdf_bytes).hexdigest()}", + size_bytes=len(pdf_bytes), + storage_etag="original-pdf-etag", + storage_version_id="original-pdf-version", + ) + extracted = inspect_pdf_bytes(pdf_bytes, max_pages=10, max_output_bytes=5_000_000) + assert extracted.page_count == 3 + assert "" in extracted.markdown + now = datetime.now(UTC) + artifacts = build_raw_document_artifacts( + source, extracted, limits=PdfInspectorLimits(), started_at=now, extracted_at=now + ) + return parse_document_markdown(artifacts.document_markdown) + + +def enrich( + raw: DocumentMarkdownV1, observations: tuple[DocumentAgentObservationV1, ...] +) -> DocumentMarkdownV1: + ingestion = raw.frontmatter.ingestion + return enrich_document_markdown( + raw, + DocumentAgentOutputV1( + title="PDF citations", body="Extracted evidence.", observations=observations + ), + DocumentIngestionV1( + stage=DocumentIngestionStage.ready, + pipeline_version=ingestion.pipeline_version, + prompt_version=ingestion.prompt_version, + run_id=ingestion.run_id, + input_checksum=ingestion.input_checksum, + base_checksum=document_markdown_checksum(assemble_document_markdown(raw)), + ), + ) + + +@pytest.mark.asyncio +async def test_pdf_page_citation_survives_real_note_api( + client: AsyncClient, + test_project: Project, + raw_pdf: DocumentMarkdownV1, + engine_factory: tuple[AsyncEngine, async_sessionmaker[AsyncSession]], +) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", + content="The document describes non-Markdown file support.", + tags=("pdf",), + context="Source document", + locator=DocumentPageLocatorV1(page=2, page_label="iv"), + ), + ), + ) + response = await client.post( + f"/v2/projects/{test_project.external_id}/knowledge/entities", + json={ + "title": "PDF citations", + "directory": "docs", + "content": assemble_document_markdown(enriched), + }, + ) + assert response.status_code == 202, response.text + entity = response.json() + read = await client.get( + f"/v2/projects/{test_project.external_id}/knowledge/entities/{entity['external_id']}" + ) + assert read.status_code == 200, read.text + stored = parse_document_markdown(read.json()["content"]) + assert stored.frontmatter.source == raw_pdf.frontmatter.source + assert stored.frontmatter.sources is not None + citation = stored.frontmatter.sources[0] + assert citation.id == "document-page-2" + assert citation.resource == "/docs/report%20%231%25.pdf#page=2" + assert citation.locator.page_label == "iv" + assert citation.title.endswith("p. iv") + target = urlsplit(citation.resource) + assert target.fragment == "page=2" + source_bytes = (Path(test_project.path) / unquote(target.path).lstrip("/")).read_bytes() + assert ( + f"sha256:{hashlib.sha256(source_bytes).hexdigest()}" == stored.frontmatter.source.checksum + ) + assert "[^document-page-2]: [PDF page 2](/docs/report%20%231%25.pdf#page=2)" in stored.body + # The accepted-content GET returns canonical Markdown; inspect the real + # database separately to prove the write also indexed the cited observation. + _, session_maker = engine_factory + async with session_maker() as session: + observations = ( + await session.scalars( + select(Observation).where(Observation.project_id == test_project.id) + ) + ).all() + assert len(observations) == 1 + assert observations[0].content.endswith("[^document-page-2] #pdf") + assert observations[0].context == "Source document" + assert (Path(test_project.path) / entity["file_path"]).read_text( + encoding="utf-8" + ) == read.json()["content"] + + +def test_pdf_citation_ids_survive_source_and_observation_reordering( + raw_pdf: DocumentMarkdownV1, +) -> None: + first = DocumentAgentObservationV1( + category="fact", content="First.", locator=DocumentPageLocatorV1(page=1) + ) + second = DocumentAgentObservationV1( + category="fact", content="Second.", locator=DocumentPageLocatorV1(page=2) + ) + original = enrich(raw_pdf, (first, second)) + reordered = enrich(raw_pdf, (second, first)) + assert original.frontmatter.sources is not None + assert reordered.frontmatter.sources is not None + expected = {source.id: source.resource for source in original.frontmatter.sources} + reordered = reordered.model_copy( + update={ + "frontmatter": reordered.frontmatter.model_copy( + update={"sources": tuple(reversed(reordered.frontmatter.sources))} + ) + } + ) + parsed = parse_document_markdown(assemble_document_markdown(reordered)) + assert parsed.frontmatter.sources is not None + assert {source.id: source.resource for source in parsed.frontmatter.sources} == expected + assert "First. [^document-page-1]" in parsed.body + assert "Second. [^document-page-2]" in parsed.body + + +def test_trailing_backslash_does_not_escape_rendered_page_citation( + raw_pdf: DocumentMarkdownV1, +) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", + content="The separator is \\", + locator=DocumentPageLocatorV1(page=2), + ), + ), + ) + rendered = MarkdownIt().use(footnote_plugin).render(enriched.body) + assert 'class="footnote-ref"' in rendered + assert 'href="/docs/report%20%231%25.pdf#page=2"' in rendered + + +def test_multiple_observations_share_one_page_citation(raw_pdf: DocumentMarkdownV1) -> None: + locator = DocumentPageLocatorV1(page=2) + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1(category="fact", content="First.", locator=locator), + DocumentAgentObservationV1(category="fact", content="Second.", locator=locator), + ), + ) + assert enriched.frontmatter.sources is not None + assert len(enriched.frontmatter.sources) == 1 + assert enriched.body.count("[^document-page-2]:") == 1 + assert len(parse(enriched.body).observations) == 2 + + +def test_uncited_document_retains_legacy_markdown_shape(raw_pdf: DocumentMarkdownV1) -> None: + enriched = enrich( + raw_pdf, (DocumentAgentObservationV1(category="fact", content="No locator."),) + ) + markdown = assemble_document_markdown(enriched) + assert "sources:" not in markdown + assert "[^document-page-" not in markdown + assert enriched.body.endswith("- [fact] No locator.\n") + assert parse_document_markdown(markdown) == enriched + + +def test_pdf_citation_rejects_page_outside_extracted_document(raw_pdf: DocumentMarkdownV1) -> None: + with pytest.raises(ValueError, match="outside the source PDF"): + enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", content="Missing page.", locator=DocumentPageLocatorV1(page=4) + ), + ), + ) + + +def test_pdf_citation_rejects_zero_based_page() -> None: + with pytest.raises(ValidationError, match="greater than or equal to 1"): + DocumentAgentOutputV1.model_validate( + { + "title": "Bad", + "body": "", + "observations": [{"category": "fact", "content": "Zero.", "locator": {"page": 0}}], + } + ) + + +def test_pdf_citation_does_not_coerce_printed_page_label_into_page() -> None: + with pytest.raises(ValidationError, match="valid integer"): + DocumentAgentOutputV1.model_validate( + { + "title": "Bad", + "body": "", + "observations": [ + {"category": "fact", "content": "Label.", "locator": {"page": "iv"}} + ], + } + ) + + +def test_pdf_citation_rejects_conflicting_labels_for_same_page(raw_pdf: DocumentMarkdownV1) -> None: + with pytest.raises(ValueError, match="agree on its page label"): + enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", + content="First.", + locator=DocumentPageLocatorV1(page=2, page_label="iv"), + ), + DocumentAgentObservationV1( + category="fact", + content="Second.", + locator=DocumentPageLocatorV1(page=2, page_label="v"), + ), + ), + ) + + +def test_agent_cannot_redirect_generated_pdf_footnote() -> None: + with pytest.raises(ValidationError, match="cannot define generated"): + DocumentAgentOutputV1( + title="Redirect", + body="[^document-page-2]: https://example.com/unrelated.pdf", + observations=( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + + +def test_pdf_citation_rejects_source_entry_that_disagrees_with_provenance( + raw_pdf: DocumentMarkdownV1, +) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + markdown = assemble_document_markdown(enriched) + redirected = markdown.replace( + "resource: /docs/report%20%231%25.pdf#page=2", "resource: /other.pdf#page=2" + ) + assert redirected != markdown + with pytest.raises(ValidationError, match="trusted source PDF page"): + parse_document_markdown(redirected) + + +def test_relation_context_cannot_borrow_declared_page_citation() -> None: + with pytest.raises(ValidationError, match="cannot define generated"): + DocumentAgentOutputV1( + title="Citations", + body="", + observations=( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + relations=( + DocumentAgentRelationV1( + relation_type="supports", + target="Evidence", + context="Borrowed [^document-page-2]", + ), + ), + ) + + +def test_pdf_citation_rejects_duplicate_source_ids(raw_pdf: DocumentMarkdownV1) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + assert enriched.frontmatter.sources is not None + duplicate = enriched.model_copy( + update={ + "frontmatter": enriched.frontmatter.model_copy( + update={"sources": enriched.frontmatter.sources * 2} + ) + } + ) + with pytest.raises(ValidationError, match="citation source IDs must be unique"): + parse_document_markdown(assemble_document_markdown(duplicate)) + + +def test_page_citation_rejects_non_pdf_source(raw_pdf: DocumentMarkdownV1) -> None: + markdown = assemble_document_markdown(raw_pdf).replace( + "media_type: application/pdf", "media_type: text/plain" + ) + non_pdf = parse_document_markdown(markdown) + with pytest.raises(ValueError, match="page citations require a PDF source"): + enrich( + non_pdf, + ( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + + +def test_page_label_is_retained_when_later_observation_omits_it( + raw_pdf: DocumentMarkdownV1, +) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", + content="First.", + locator=DocumentPageLocatorV1(page=2, page_label="iv"), + ), + DocumentAgentObservationV1( + category="fact", content="Second.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + parsed = parse_document_markdown(assemble_document_markdown(enriched)) + assert parsed.frontmatter.sources is not None + assert len(parsed.frontmatter.sources) == 1 + assert parsed.frontmatter.sources[0].locator.page_label == "iv" + + +def test_page_label_is_added_when_earlier_observation_omits_it(raw_pdf: DocumentMarkdownV1) -> None: + enriched = enrich( + raw_pdf, + ( + DocumentAgentObservationV1( + category="fact", content="First.", locator=DocumentPageLocatorV1(page=2) + ), + DocumentAgentObservationV1( + category="fact", + content="Second.", + locator=DocumentPageLocatorV1(page=2, page_label="iv"), + ), + ), + ) + parsed = parse_document_markdown(assemble_document_markdown(enriched)) + assert parsed.frontmatter.sources is not None + assert len(parsed.frontmatter.sources) == 1 + assert parsed.frontmatter.sources[0].locator.page_label == "iv" + + +def test_agent_body_cannot_borrow_declared_page_citation() -> None: + with pytest.raises(ValidationError, match="cannot define generated"): + DocumentAgentOutputV1( + title="Citations", + body="Unsupported claim [^document-page-2]", + observations=( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + ), + ) + + +def test_unlocated_observation_cannot_borrow_declared_page_citation() -> None: + with pytest.raises(ValidationError, match="cannot define generated"): + DocumentAgentOutputV1( + title="Citations", + body="", + observations=( + DocumentAgentObservationV1( + category="fact", content="Claim.", locator=DocumentPageLocatorV1(page=2) + ), + DocumentAgentObservationV1(category="fact", content="Unlocated [^document-page-2]"), + ), + ) + + +def test_observation_context_cannot_borrow_declared_page_citation() -> None: + with pytest.raises(ValidationError, match="cannot define generated"): + DocumentAgentOutputV1( + title="Citations", + body="", + observations=( + DocumentAgentObservationV1( + category="fact", + content="Claim.", + locator=DocumentPageLocatorV1(page=2), + context="Borrowed [^document-page-2]", + ), + ), + )