diff --git a/.github/workflows/dataset-agentdataflow-validate.yml b/.github/workflows/dataset-agentdataflow-validate.yml new file mode 100644 index 0000000..27bfcf9 --- /dev/null +++ b/.github/workflows/dataset-agentdataflow-validate.yml @@ -0,0 +1,46 @@ +name: dataset-agentdataflow-validate + +# Runs the dataset-local validator for agentdataflow_toolexchange_traces on +# every change to it, so "enforced by validate.py" in that dataset's README +# is true of the merged tree and not only of whoever remembered to run it +# locally. Path-filtered to the dataset, same pattern as scanner-selftest. +on: + push: + branches: [main] + paths: + - 'datasets/agentdataflow_toolexchange_traces/**' + - 'datasets/_shared/**' + - '.github/workflows/dataset-agentdataflow-validate.yml' + pull_request: + paths: + - 'datasets/agentdataflow_toolexchange_traces/**' + - 'datasets/_shared/**' + - '.github/workflows/dataset-agentdataflow-validate.yml' + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 + with: + python-version: '3.11' + + - name: Install jsonschema + run: python -m pip install --quiet jsonschema + + - name: Validate entries and example against schema.json + run: python validate.py + working-directory: datasets/agentdataflow_toolexchange_traces + + - name: index.csv is up to date with entries/ + run: | + python build_index.py + git diff --exit-code index.csv + working-directory: datasets/agentdataflow_toolexchange_traces diff --git a/datasets/agentdataflow_toolexchange_traces/README.md b/datasets/agentdataflow_toolexchange_traces/README.md index 232d70b..8249934 100644 --- a/datasets/agentdataflow_toolexchange_traces/README.md +++ b/datasets/agentdataflow_toolexchange_traces/README.md @@ -10,39 +10,113 @@ Structured traces capturing how data flows through agentic AI systems — betwee Categories include: -- **Tool call traces** — Sequences of tool invocations showing what data is sent to and received from external tools (APIs, databases, file systems, code execution environments) -- **Multi-agent delegation chains** — Traces showing how tasks and data are passed between agents, including what context is forwarded, filtered, or accumulated -- **Plugin data exchange logs** — Records of data shared with third-party plugins, including request/response payloads and metadata -- **Credential and permission flows** — How credentials, tokens, and permissions are scoped, delegated, and consumed across agent workflows -- **Context accumulation patterns** — Traces showing how agent context windows grow over multi-step tasks, what data persists across steps, and where over-sharing occurs -- **Memory read/write traces** — How agents interact with short-term and long-term memory stores, including what data is persisted and retrieved +- **Tool call traces** (`tool_call`) — Sequences of tool invocations showing what data is sent to and received from external tools (APIs, databases, file systems, code execution environments) +- **Multi-agent delegation chains** (`multi_agent_delegation`) — Traces showing how tasks and data are passed between agents, including what context is forwarded, filtered, or accumulated +- **Plugin data exchange logs** (`plugin_data_exchange`) — Records of data shared with third-party plugins, including request/response payloads and metadata +- **Credential and permission flows** (`credential_flow`) — How credentials, tokens, and permissions are scoped, delegated, and consumed across agent workflows +- **Context accumulation patterns** (`context_accumulation`) — Traces showing how agent context windows grow over multi-step tasks, what data persists across steps, and where over-sharing occurs +- **Memory read/write traces** (`memory_read_write`) — How agents interact with short-term and long-term memory stores, including what data is persisted and retrieved + +## Files + +| File | Purpose | +|---|---| +| `schema.json` | JSON Schema (draft 2020-12) every entry validates against | +| `example.json` | Worked example. Read it with the schema before writing an entry | +| `entries/` | One trace per file, named `.json`. Empty until the first entry merges; `validate.py` passes against `example.json` alone in the meantime. | +| `validate.py` | `python validate.py` - schema, provenance, span graph, and secret-scan checks | +| `build_index.py` | `python build_index.py` - regenerates `index.csv` | +| `index.csv` | Flat index of all entries, for filtering and citation. Generated once `entries/` is non-empty. | ## Data Format - +One trace per file in `entries/`, named after its `trace_id` (`DSGAI-TRACE-.json`). The authoritative definition is `schema.json`; the summary below is orientation. -Contributions should include: +Every entry carries: -- **Trace ID** -- **Category** — From the list above -- **DSGAI mapping** — Primary DSGAI entries relevant to this trace -- **Agent framework** — LangGraph, AutoGPT, CrewAI, custom, etc. (if disclosable) -- **Trace data** — The sequence of events, tool calls, and data exchanges in structured format (JSON, JSONL, or OpenTelemetry-compatible spans) -- **Data sensitivity annotations** — Flag any steps where sensitive data is present, over-shared, or inadequately scoped -- **Security observations** — What data security risks this trace illustrates -- **Benign / adversarial** — Whether this is a normal workflow trace or one demonstrating a security failure +- **`trace_id`, `title`, `description`** - what flows, and what makes it security-relevant +- **`category`** - one of the six above +- **`disposition`** - `benign`, `adversarial`, or `unintentional_failure` +- **`dsgai_mapping`** - DSGAI entries this trace illustrates, primary risk first +- **`provenance`** - where the trace came from, and what backs it (see below) +- **`sanitization`** - a hard attestation plus the techniques applied +- **`spans`** - the trace itself: an ordered sequence of events +- **`security_observations`** - what the trace demonstrates, one falsifiable claim per item + +Optional: `agent` (framework, protocol, topology, autonomy), `owasp_llm_top10_mapping`, `mitre_atlas_mapping`, `mitigations`, `contributor`, `tags`, `notes`. + +### Spans + +A span is one event. The shape is OpenTelemetry-compatible without requiring an OTel pipeline to produce it, so a trace can be exported from a real system or written by hand and still validate the same way. + +```json +{ + "span_id": "s7", + "parent_span_id": "s6", + "t_offset_ms": 176400910, + "actor": "tool", + "actor_id": "tool://server-b/lookup_record", + "operation": "tool.result", + "summary": "Tool returns the full record, three fields wider than the schema the agent bound to.", + "payload": { "contact_email": "", "internal_notes": "" }, + "data_classes": ["tool_output", "pii"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI01", + "note": "Entitlement was enforced once, against a schema that has since changed.", + "severity": "High" + } +} +``` + +Two conventions carry most of the weight: + +**Payloads are shapes, not content.** String values in `payload` should be typed placeholders - ``, ``, ``. Literal content belongs in a trace only where the literal *is* the published artifact, such as a prompt-injection string from a paper. This is not only a privacy rule: it is what makes traces from real systems shareable at all, because the finding almost always lives in the shape. + +**Findings hang off spans.** Attaching a risk to the span where it materializes lets a reader see exactly where a flow stops being safe, instead of inferring it from prose. `validate.py` requires every span-level `finding.dsgai_id` to appear in the trace's top-level `dsgai_mapping`. + +Prefer `t_offset_ms` over wall-clock timestamps. Relative offsets carry the ordering that matters without inventing precision or disclosing when a system was running. Use `timestamp` only where the absolute date is part of the finding. + +### Provenance tiers + +Most people who hold real agent traces cannot publish them, because production telemetry contains everything that must not be published. A dataset that accepts only production telemetry stays empty; a dataset that accepts anything fills with plausible fiction. The `provenance.tier` field is how this dataset takes both contributions without confusing them: + +| Tier | Meaning | +|---|---| +| `observed_production` | Sanitized telemetry from a real deployed system | +| `observed_lab` | Sandbox, test environment, CTF, or red-team exercise | +| `derived_from_public_documentation` | The mechanism is rendered as a trace, and its precondition is a documented public fact carried in `evidence` - a specification, an advisory, a CVE, third-party research. The source may be normative (what a protocol permits) or empirical (what was observed); either is fine, as long as it is checkable without the contributor. | +| `hypothetical` | Illustrative only, no empirical backing | + +Every tier except `hypothetical` requires at least one `provenance.evidence` item, enforced by `validate.py`, and every item has to say what it supports, narrowly. Acceptable backing is anything a reader can check without the contributor: a normative statement in a protocol specification, a published advisory or CVE, a vendor changelog, third-party research, a DOI. A source the contributor owns or controls - their own site, their own dataset, their own publication - does not establish a precondition here, whatever it says. This rule is not machine-checkable and is not enforced by `validate.py`; it is a review-time obligation, the same as the sanitization judgment call a reviewer makes on every entry. Where a contributor-controlled source is the only backing available, the honest tier is `hypothetical`. Statistics computed over this dataset should be reported per tier, never pooled across them. + +Choose the tier honestly. `derived_from_public_documentation` is not a lesser contribution; it is the tier that lets a documented phenomenon be studied as a data flow without anyone publishing their logs. A specification saying what a protocol permits and telemetry saying what happened are different claims with different weight, which is exactly why the field is named "documentation" and not "observation." + +### MITRE ATLAS mappings + +`mitre_atlas_mapping` is release-pinned by construction: an entry records the ATLAS release it was verified against and the name each identifier carried in that release. ATLAS identifiers and their names have both moved between releases, so an unpinned pair silently rots. Omit the property rather than guess. ## Sanitization Requirements All traces **must** be sanitized before submission: -- No real API keys, tokens, credentials, or secrets — replace with placeholder values (e.g., `sk-REDACTED`, `Bearer EXAMPLE_TOKEN`) +- No real API keys, tokens, credentials, or secrets — replace with placeholder values (e.g. ``) - No real PII, PHI, or proprietary data — use synthetic equivalents - No internal hostnames, IP addresses, or infrastructure details - Generalize organization-specific tool names if they could identify the source Traces from test environments, sandboxes, or CTF exercises are ideal. Production traces must be thoroughly sanitized. +Two things back this up rather than leaving it to good intentions. `sanitization.attestation` must be a literal `true`, so an entry cannot merge with the box quietly unticked. And `validate.py` scans every string in the entry, independently of that attestation: payloads, summaries, notes, evidence locators. Placeholders are exempt; values matching one of its patterns fail the run - common credential shapes, RFC1918 addresses, `.internal`/`.corp`/`.lan`/`.intra` hostnames, home directory paths, credential-shaped `key=value` assignments, and two or more addresses in one string that look like real routable public IPv4 addresses. The scan is deliberately wider than the payloads, because prose written while looking at a real trace is where a real hostname actually gets typed. It is pattern matching, not a guarantee: a contributor is still the one certifying `sanitization.attestation`, and a reviewer is still the backstop for anything the patterns miss. + +Naming a third party as the subject of a security failure is a disclosure act, not a dataset contribution. Generalize the counterparty unless you have gone through disclosure and are prepared to say so in `notes`. + ## Contributing -Add traces as individual JSON, JSONL, or YAML files and submit a pull request. See the [main datasets README](../README.md) for general contribution guidelines. +1. Copy `example.json` and edit it, or export spans from your own system into the same shape +2. Save as `entries/.json` +3. Run `python validate.py` - it must print OK +4. Run `python build_index.py` to refresh `index.csv` +5. Open a pull request describing what the trace demonstrates and which DSGAI entries it maps to + +See the [main datasets README](../README.md) for general contribution guidelines, and `#team-genai-data-security-initiative` on the [OWASP Slack workspace](https://owasp.slack.com) for anything that needs a conversation first. diff --git a/datasets/agentdataflow_toolexchange_traces/build_index.py b/datasets/agentdataflow_toolexchange_traces/build_index.py new file mode 100644 index 0000000..169c10d --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/build_index.py @@ -0,0 +1,100 @@ +"""Generate index.csv from every entry in ./entries/. + +Usage: python build_index.py +Writes ./index.csv with key flat columns; arrays are joined with '|'. +Stdlib only. +""" +from __future__ import annotations + +import csv +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +ENTRIES_DIR = ROOT / "entries" +INDEX_PATH = ROOT / "index.csv" + +SENSITIVITY_ORDER = ["none", "low", "moderate", "high"] + +COLUMNS = [ + "trace_id", + "title", + "category", + "disposition", + "provenance_tier", + "primary_dsgai", + "dsgai_mapping", + "owasp_llm_top10_mapping", + "agent_framework", + "tool_protocol", + "span_count", + "finding_count", + "max_sensitivity", + "data_classes", + "evidence_count", + "primary_evidence", + "date_added", + "tags", +] + + +def max_sensitivity(spans: list[dict]) -> str: + best = -1 + for span in spans: + value = span.get("sensitivity") + if value in SENSITIVITY_ORDER: + best = max(best, SENSITIVITY_ORDER.index(value)) + return SENSITIVITY_ORDER[best] if best >= 0 else "" + + +def row_for(entry: dict) -> dict[str, str]: + spans = entry.get("spans", []) or [] + prov = entry.get("provenance", {}) or {} + agent = entry.get("agent", {}) or {} + evidence = prov.get("evidence", []) or [] + dsgai = entry.get("dsgai_mapping", []) or [] + + data_classes: list[str] = [] + for span in spans: + for cls in span.get("data_classes", []) or []: + if cls not in data_classes: + data_classes.append(cls) + + return { + "trace_id": entry.get("trace_id", ""), + "title": entry.get("title", ""), + "category": entry.get("category", ""), + "disposition": entry.get("disposition", ""), + "provenance_tier": prov.get("tier", ""), + "primary_dsgai": dsgai[0] if dsgai else "", + "dsgai_mapping": "|".join(dsgai), + "owasp_llm_top10_mapping": "|".join(entry.get("owasp_llm_top10_mapping", []) or []), + "agent_framework": agent.get("framework", ""), + "tool_protocol": agent.get("tool_protocol", ""), + "span_count": str(len(spans)), + "finding_count": str(sum(1 for s in spans if s.get("finding"))), + "max_sensitivity": max_sensitivity(spans), + "data_classes": "|".join(sorted(data_classes)), + "evidence_count": str(len(evidence)), + "primary_evidence": evidence[0].get("citation", "") if evidence else "", + "date_added": entry.get("date_added", ""), + "tags": "|".join(entry.get("tags", []) or []), + } + + +def main() -> None: + rows = [] + for path in sorted(ENTRIES_DIR.glob("*.json")): + entry = json.loads(path.read_text(encoding="utf-8")) + rows.append(row_for(entry)) + + with INDEX_PATH.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=COLUMNS) + writer.writeheader() + writer.writerows(rows) + + print(f"Wrote {INDEX_PATH.name} with {len(rows)} entries.") + + +if __name__ == "__main__": + main() diff --git a/datasets/agentdataflow_toolexchange_traces/example.json b/datasets/agentdataflow_toolexchange_traces/example.json new file mode 100644 index 0000000..646295f --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/example.json @@ -0,0 +1,203 @@ +{ + "$schema": "./schema.json", + "trace_id": "DSGAI-TRACE-example-toolset-replacement-name-rebind", + "title": "Name-keyed binding survives a whole-toolset replacement and re-points at a different implementation", + "description": "This is the worked example for this dataset; read it alongside schema.json before writing an entry. A server replaces its entire toolset between two observations. The agent holds a binding keyed by tool name and a credential scoped to that name, so when a similarly named tool appears in the replacement set, the binding and the credential both survive a change to everything they referred to. The trace shows where the rebind happens and where the inherited scope is spent.", + "category": "tool_call", + "disposition": "unintentional_failure", + + "dsgai_mapping": ["DSGAI06", "DSGAI05", "DSGAI02"], + "owasp_llm_top10_mapping": ["LLM03:2025", "LLM06:2025"], + + "agent": { + "framework": "custom", + "tool_protocol": "mcp", + "topology": "single agent, name-keyed tool registry, per-tool credential scopes", + "autonomy": "fully_autonomous" + }, + + "provenance": { + "tier": "derived_from_public_documentation", + "collection_method": "Constructed around a rebind the Model Context Protocol permits: a tool is identified by its name alone, the tool list may be replaced during a session, and nothing binds a name to the implementation it referred to earlier. The agent, registry and credential model are synthetic.", + "evidence": [ + { + "type": "url", + "citation": "https://modelcontextprotocol.io/specification/2025-06-18/server/tools", + "locator": "Tools overview; Data Types > Tool; List Changed Notification", + "supports": "'Each tool is uniquely identified by a name'. The protocol adds a title, description and schemas alongside the name, but none of those is defined as an identity - name is what a client looks up a tool by. The tool list can also be replaced during a session, so a binding or a credential keyed to that name can outlive the implementation the name referred to when the binding was made." + }, + { + "type": "cve", + "citation": "CVE-2025-54136", + "locator": "NVD description; advisory GHSA-24mc-g4xr-4395", + "supports": "A shipped client carried this class of substitution: an MCP entry a collaborator had accepted could later be swapped for a different command with no re-prompt (Cursor 1.2.4 and below, fixed in 1.3). The CVE requires an attacker with write access to the shared repository or to the local file, so it is an adversarial route to the same end-state; this trace models the non-adversarial route, a toolset replacement with no attacker, reaching the same rebind." + } + ] + }, + + "sanitization": { + "attestation": true, + "techniques": ["not_applicable_fully_synthetic", "name_generalization", "shape_only_payloads"], + "notes": "Fully synthetic. Nothing here was taken from a real deployment." + }, + + "spans": [ + { + "span_id": "s1", + "parent_span_id": null, + "t_offset_ms": 0, + "actor": "orchestrator", + "actor_id": "orchestrator://ops-flow", + "operation": "tool.discover", + "summary": "Agent discovers a 12-tool set and registers each tool under its name.", + "payload": { + "server": "tool://server-c", + "tool_count": 12, + "registry_key": "name", + "toolset_digest": "" + }, + "data_classes": ["tool_schema"], + "sensitivity": "none" + }, + { + "span_id": "s2", + "parent_span_id": "s1", + "t_offset_ms": 90, + "actor": "policy_engine", + "actor_id": "policy://credentials", + "operation": "credential.issue", + "summary": "A scoped credential is minted for one tool and cached against the tool name.", + "payload": { + "subject": "server-c/export_report", + "scopes": ["reports.read", "reports.export"], + "credential": "", + "bound_to": "tool name", + "bound_to_toolset_digest": false + }, + "data_classes": ["credential", "token"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI02", + "note": "The credential is bound to a string, not to the implementation the string resolved to at issue time. Anything that later answers to that string inherits the scope.", + "severity": "Medium" + } + }, + { + "span_id": "s3", + "parent_span_id": null, + "t_offset_ms": 259200000, + "actor": "external_service", + "actor_id": "tool://server-c", + "operation": "contract.change", + "summary": "Server replaces its entire toolset. All 12 original tools are gone; 9 new ones appear, one of which reuses a prior name.", + "payload": { + "change_kind": "tool-removed", + "removal_scope": "toolset-replaced", + "tools_removed": 12, + "tools_added": 9, + "name_collisions": ["export_report"], + "declared_server_version_delta": "same" + }, + "data_classes": ["tool_schema"], + "sensitivity": "none", + "finding": { + "dsgai_id": "DSGAI05", + "note": "A whole-toolset swap under a stable server identity is an integrity failure at the binding layer: nothing in the client's state changed, but what the binding now points to did. The reused name is the vector; the stable endpoint and version are what make it look like continuity.", + "severity": "High" + } + }, + { + "span_id": "s4", + "parent_span_id": null, + "t_offset_ms": 262800000, + "actor": "orchestrator", + "actor_id": "orchestrator://ops-flow", + "operation": "tool.bind", + "summary": "Registry resolves the cached name to the new implementation. Because the lookup succeeds, no error, warning, or re-approval is produced.", + "payload": { + "requested": "server-c/export_report", + "resolved": "server-c/export_report", + "implementation_changed": true, + "input_schema_digest_changed": true, + "rebind_alert": false + }, + "data_classes": ["tool_schema"], + "sensitivity": "moderate", + "finding": { + "dsgai_id": "DSGAI06", + "note": "A successful name lookup is treated as evidence of continuity. It is only evidence that a name is in use.", + "severity": "High" + } + }, + { + "span_id": "s5", + "parent_span_id": "s4", + "t_offset_ms": 262801100, + "actor": "agent", + "actor_id": "agent://ops-assistant", + "operation": "credential.use", + "summary": "The cached credential from s2 is presented to the new implementation.", + "payload": { + "credential": "", + "issued_for_implementation": "s1", + "presented_to_implementation": "s3", + "scopes": ["reports.read", "reports.export"] + }, + "data_classes": ["credential", "token"], + "sensitivity": "high", + "finding": { + "dsgai_id": "DSGAI02", + "note": "Scope granted to one implementation is spent against another. The gap between issued_for_implementation and presented_to_implementation is the defect.", + "severity": "High" + } + }, + { + "span_id": "s6", + "parent_span_id": "s5", + "t_offset_ms": 262802400, + "actor": "tool", + "actor_id": "tool://server-c/export_report", + "operation": "tool.result", + "summary": "New implementation accepts the call and returns an export, with a different data reach than the tool the scope was granted for.", + "payload": { + "status": "ok", + "rows": "", + "includes_fields_outside_prior_reach": true + }, + "data_classes": ["tool_output", "proprietary"], + "sensitivity": "high" + }, + { + "span_id": "s7", + "parent_span_id": "s6", + "t_offset_ms": 262802500, + "actor": "orchestrator", + "actor_id": "orchestrator://ops-flow", + "operation": "context.append", + "summary": "Export summary is appended to context and the run completes as a success.", + "payload": { "appended_bytes": "", "presented_as": "completed action" }, + "data_classes": ["tool_output", "proprietary"], + "sensitivity": "moderate" + } + ], + + "security_observations": [ + "Tool identity in practice is a name plus an endpoint, and neither is stable enough to carry an authorization or credential binding.", + "A name is the only tool identifier the protocol defines, so a name that reappears after a toolset change is indistinguishable from continuity to a conformant client, with or without an adversary.", + "Every guard in this trace passed. The registry resolved, the credential validated, the tool returned success. Nothing in the run had a place to notice the substitution.", + "The same mechanism carries an adversarial variant, but the adversarial variant is not required for the failure." + ], + + "mitigations": [ + "Key the tool registry by a contract digest or a signed tool identity, and treat name-only resolution as a cache miss.", + "Bind credentials to the implementation digest they were issued against; refuse presentation when the digest has moved.", + "Emit a rebind event whenever a resolved name maps to a changed input-schema digest, and require re-approval before the first call.", + "Track toolset-level digests, not just per-tool ones, so a wholesale replacement is visible as one event rather than as a scatter of unrelated removals." + ], + + "date_added": "2026-08-07", + + "tags": ["mcp", "tool-contract-drift", "toolset-replacement", "name-rebind", "credential-scope", "supply-chain", "example"], + + "notes": "Template entry. Two conventions worth copying: (1) payload values are typed placeholders, never real content, which is what makes a trace of this class publishable at all; (2) findings hang off individual spans, so a reader can see exactly where a flow stops being safe rather than inferring it from a paragraph. The adversarial variant of this flow - an actor deliberately registering a name a known agent population is bound to - belongs in a separate entry at a different provenance tier, since what is public here is that the rebind is possible, not that anyone intends it. Keeping the two apart is the whole reason the tier field exists." +} diff --git a/datasets/agentdataflow_toolexchange_traces/schema.json b/datasets/agentdataflow_toolexchange_traces/schema.json new file mode 100644 index 0000000..4554671 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/schema.json @@ -0,0 +1,435 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://owasp.org/dsgai/datasets/agentdataflow_toolexchange_traces/schema.json", + "title": "DSGAI Agent Data Flow & Tool Exchange Trace", + "description": "Schema for a single trace entry in the OWASP DSGAI agentdataflow_toolexchange_traces dataset. Each entry documents one data flow through an agentic AI system as an ordered span sequence, with an explicit provenance tier and a machine-checkable sanitization attestation. One JSON file per entry, validated against this schema.", + "type": "object", + "additionalProperties": false, + "required": [ + "trace_id", + "title", + "description", + "category", + "disposition", + "dsgai_mapping", + "provenance", + "sanitization", + "spans", + "security_observations", + "date_added" + ], + "properties": { + "$schema": { + "description": "Optional pointer to the schema file for editor tooling. Ignored by validators.", + "type": "string" + }, + + "trace_id": { + "description": "Internal identifier. 'DSGAI-TRACE-', lowercase kebab-case slug.", + "type": "string", + "pattern": "^DSGAI-TRACE-[a-z0-9]+(-[a-z0-9]+)*$" + }, + + "title": { + "description": "Short human-readable name of the data flow this trace captures.", + "type": "string", + "minLength": 4, + "maxLength": 200 + }, + + "description": { + "description": "What flows through this trace, and what makes it security-relevant. 2-6 sentences.", + "type": "string", + "minLength": 40 + }, + + "category": { + "description": "Primary trace category, from the categories listed in this dataset's README.", + "type": "string", + "enum": [ + "tool_call", + "multi_agent_delegation", + "plugin_data_exchange", + "credential_flow", + "context_accumulation", + "memory_read_write" + ] + }, + + "disposition": { + "description": "What kind of run this is. 'benign' = a normal workflow with no security failure. 'adversarial' = an attacker is present and acting. 'unintentional_failure' = no adversary, but data is exposed, over-shared, or mis-scoped anyway.", + "type": "string", + "enum": ["benign", "adversarial", "unintentional_failure"] + }, + + "dsgai_mapping": { + "description": "DSGAI risk entries this trace illustrates. First entry should be the primary risk.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "DSGAI01", "DSGAI02", "DSGAI03", "DSGAI04", "DSGAI05", + "DSGAI06", "DSGAI07", "DSGAI08", "DSGAI09", "DSGAI10", + "DSGAI11", "DSGAI12", "DSGAI13", "DSGAI14", "DSGAI15", + "DSGAI16", "DSGAI17", "DSGAI18", "DSGAI19", "DSGAI20", "DSGAI21" + ] + }, + "minItems": 1, + "uniqueItems": true + }, + + "owasp_llm_top10_mapping": { + "description": "OWASP Top 10 for LLM Applications 2025 entries this trace aligns with.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "LLM01:2025", "LLM02:2025", "LLM03:2025", "LLM04:2025", "LLM05:2025", + "LLM06:2025", "LLM07:2025", "LLM08:2025", "LLM09:2025", "LLM10:2025" + ] + }, + "uniqueItems": true + }, + + "mitre_atlas_mapping": { + "description": "MITRE ATLAS alignment. Release-pinned by construction: ATLAS identifiers and their names have both moved across releases, so an entry must record which release it was verified against and the name the identifier carried in that release. Omit the property entirely rather than guess.", + "type": "object", + "additionalProperties": false, + "required": ["atlas_release", "techniques"], + "properties": { + "atlas_release": { + "description": "ATLAS data release the identifiers and names below were verified against, e.g. '4.9.0'.", + "type": "string", + "minLength": 1 + }, + "techniques": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name_at_release"], + "properties": { + "id": { + "type": "string", + "pattern": "^AML\\.T[0-9]{4}(\\.[0-9]{3})?$" + }, + "name_at_release": { + "description": "The technique name verbatim in the pinned release. Verified, not remembered.", + "type": "string", + "minLength": 3 + } + } + } + } + } + }, + + "agent": { + "description": "The agentic system the trace was taken from or models.", + "type": "object", + "additionalProperties": false, + "properties": { + "framework": { + "description": "LangGraph, CrewAI, AutoGen, OpenAI Agents SDK, custom, undisclosed, etc.", + "type": "string", + "minLength": 2 + }, + "tool_protocol": { + "description": "Protocol carrying the tool exchange.", + "type": "string", + "enum": ["mcp", "openai_tools", "anthropic_tools", "a2a", "openapi_plugin", "custom", "undisclosed"] + }, + "topology": { + "description": "Agent topology, e.g. 'single agent + tools', 'planner delegating to two workers'.", + "type": "string", + "minLength": 4 + }, + "autonomy": { + "description": "How much the system acts without a human in the loop for the actions in this trace.", + "type": "string", + "enum": ["human_approves_each_action", "human_approves_flagged_actions", "fully_autonomous", "undisclosed"] + } + } + }, + + "provenance": { + "description": "Where this trace came from. The tier is the load-bearing field: it lets consumers filter, and it lets contributors who cannot publish production telemetry still contribute without disguising extrapolation as observation.", + "type": "object", + "additionalProperties": false, + "required": ["tier", "collection_method"], + "properties": { + "tier": { + "description": "observed_production = sanitized telemetry from a real deployed system. observed_lab = sandbox, test environment, CTF, or red-team exercise. derived_from_public_documentation = the mechanism is rendered as a trace, and its precondition is a documented public fact carried in `evidence` - a specification, an advisory, a CVE, third-party research. The source may be normative (what a protocol permits) or empirical (what was observed); either way it is independently checkable, which is the property that matters. hypothetical = illustrative only, no empirical backing; excluded from any dataset statistic.", + "type": "string", + "enum": [ + "observed_production", + "observed_lab", + "derived_from_public_documentation", + "hypothetical" + ] + }, + "collection_method": { + "description": "How the trace was produced: exporter, harness, corpus, or construction method. One or two sentences.", + "type": "string", + "minLength": 8 + }, + "evidence": { + "description": "Citable public backing, independently verifiable by a reader without the contributor. Required for every tier except 'hypothetical' (enforced by validate.py). A source the contributor owns or controls - their own site, their own dataset, their own publication - does not establish a precondition here, whatever it says: this is not machine-checkable and is a review-time rule, not a validator one. Where a contributor-controlled source is the only backing available, the honest tier is 'hypothetical'.", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["type", "citation", "supports"], + "properties": { + "type": { + "type": "string", + "enum": ["doi", "url", "cve", "ghsa", "advisory", "publication", "dataset", "registry_record", "vendor_changelog"] + }, + "citation": { + "description": "The identifier or URL. For DOIs use the concept DOI, not a versioned one.", + "type": "string", + "minLength": 4 + }, + "locator": { + "description": "Where inside the source the supporting fact sits, e.g. a file and field, a section, a table.", + "type": "string", + "minLength": 2 + }, + "supports": { + "description": "The specific claim in this trace that this source backs. Keep it narrow.", + "type": "string", + "minLength": 8 + } + } + } + }, + "observation_window": { + "description": "For observed and derived tiers: the period the underlying observation covers.", + "type": "object", + "additionalProperties": false, + "properties": { + "start": { "type": "string", "format": "date" }, + "end": { "type": "string", "format": "date" }, + "note": { "type": "string", "minLength": 4 } + } + } + } + }, + + "sanitization": { + "description": "Machine-checkable form of the CONTRIBUTING.md anonymization rule. The attestation is a hard `true`, so an entry cannot be merged with the box silently unticked, and validate.py scans payloads for secret-shaped and PII-shaped strings independently of it.", + "type": "object", + "additionalProperties": false, + "required": ["attestation", "techniques"], + "properties": { + "attestation": { + "description": "Contributor attests that no real credentials, tokens, PII, PHI, proprietary data, internal hostnames, or internal IP addresses remain in this entry.", + "const": true + }, + "techniques": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "synthetic_substitution", + "redaction_placeholder", + "shape_only_payloads", + "name_generalization", + "hash_truncation", + "field_dropped", + "not_applicable_fully_synthetic" + ] + } + }, + "notes": { + "description": "Anything a reviewer should know about what was removed and why.", + "type": "string", + "minLength": 4 + } + } + }, + + "spans": { + "description": "The trace itself: an ordered sequence of events. OpenTelemetry-compatible in shape without requiring an OTel pipeline to produce it.", + "type": "array", + "minItems": 2, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["span_id", "actor", "operation"], + "properties": { + "span_id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "parent_span_id": { + "type": ["string", "null"], + "pattern": "^[a-z0-9][a-z0-9._-]*$" + }, + "t_offset_ms": { + "description": "Milliseconds from the first span. Preferred over wall-clock: it carries the ordering that matters without inventing precision or leaking when a system was running.", + "type": "integer", + "minimum": 0 + }, + "timestamp": { + "description": "Wall-clock time, for observed traces where the absolute date is part of the finding.", + "type": "string", + "format": "date-time" + }, + "actor": { + "type": "string", + "enum": [ + "user", + "orchestrator", + "agent", + "model", + "tool", + "memory_store", + "policy_engine", + "human_approver", + "external_service", + "adversary" + ] + }, + "actor_id": { + "description": "Stable synthetic identifier, e.g. 'agent://planner' or 'tool://server-a/lookup_record'. Generalize anything that would identify a real deployment.", + "type": "string", + "minLength": 1 + }, + "operation": { + "type": "string", + "enum": [ + "tool.discover", + "tool.bind", + "tool.call", + "tool.result", + "tool.error", + "contract.change", + "model.prompt", + "model.completion", + "context.append", + "memory.read", + "memory.write", + "delegation.handoff", + "policy.decision", + "approval.request", + "approval.grant", + "approval.deny", + "credential.issue", + "credential.use", + "egress" + ] + }, + "summary": { + "description": "One line, plain language: what happened in this span.", + "type": "string", + "minLength": 4 + }, + "payload": { + "description": "The SHAPE of the data moving, not its content. String values should be typed placeholders such as '', '', or ''. Literal content is acceptable only where the literal is itself the published research artifact. validate.py scans the whole entry, this object included, for secret-shaped and PII-shaped strings.", + "type": "object" + }, + "data_classes": { + "description": "Classes of data present in this span.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "none", + "system_prompt", + "user_prompt", + "tool_schema", + "tool_output", + "source_code", + "file_path", + "credential", + "token", + "pii", + "phi", + "financial", + "proprietary", + "telemetry", + "memory_state", + "retrieved_document" + ] + } + }, + "sensitivity": { + "description": "Sensitivity of the data in this span, in the contributor's judgement.", + "type": "string", + "enum": ["none", "low", "moderate", "high"] + }, + "finding": { + "description": "Attach the risk to the exact span where it materializes, rather than only to the trace as a whole.", + "type": "object", + "additionalProperties": false, + "required": ["dsgai_id", "note"], + "properties": { + "dsgai_id": { + "type": "string", + "pattern": "^DSGAI(0[1-9]|1[0-9]|2[01])$" + }, + "note": { "type": "string", "minLength": 8 }, + "severity": { + "type": "string", + "enum": ["Critical", "High", "Medium", "Low", "Informational"] + } + } + } + } + } + }, + + "security_observations": { + "description": "What this trace demonstrates. One claim per item; keep each falsifiable.", + "type": "array", + "items": { "type": "string", "minLength": 12 }, + "minItems": 1 + }, + + "mitigations": { + "description": "Controls that would break this flow, stated concretely enough to implement.", + "type": "array", + "items": { "type": "string", "minLength": 8 } + }, + + "contributor": { + "description": "Optional attribution, for research datasets that need a citable creator.", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 2 }, + "orcid": { "type": "string", "pattern": "^[0-9]{4}-[0-9]{4}-[0-9]{4}-[0-9]{3}[0-9X]$" }, + "affiliation": { "type": "string", "minLength": 2 } + } + }, + + "date_observed": { + "description": "Date the underlying flow was observed (ISO 8601 date). Omit for hypothetical traces.", + "type": "string", + "format": "date" + }, + + "date_added": { + "description": "Date this entry was added to the dataset (ISO 8601 date).", + "type": "string", + "format": "date" + }, + + "tags": { + "type": "array", + "items": { "type": "string", "minLength": 2 }, + "uniqueItems": true + }, + + "notes": { + "description": "Scope caveats, what this trace deliberately does not claim, and what a follow-up entry would need to cover.", + "type": "string" + } + } +} diff --git a/datasets/agentdataflow_toolexchange_traces/validate.py b/datasets/agentdataflow_toolexchange_traces/validate.py new file mode 100644 index 0000000..7e50f25 --- /dev/null +++ b/datasets/agentdataflow_toolexchange_traces/validate.py @@ -0,0 +1,372 @@ +"""Validate ./example.json and every entry in ./entries/ against ./schema.json. + +Usage: python validate.py +Exit code 0 on success, 1 on any validation failure. +Requires: jsonschema (pip install jsonschema) + +Beyond schema conformance this checks the things a schema cannot express: + * trace_id agrees with the filename + * a non-hypothetical provenance tier actually carries evidence, and each + evidence citation looks like the kind (doi/cve/ghsa/url) it claims to be + * the span graph is well formed: no duplicate span_id, and every + parent_span_id names a span that strictly precedes it (not itself, not a + later one) + * spans are listed in non-decreasing t_offset_ms order + * every span-level finding maps to a risk the trace declares at the top level + * date_added, date_observed, and every span timestamp are real ISO 8601 + values, in every environment - not only where optional format-checker + packages happen to be installed + * no string ANYWHERE in the entry looks like a real credential, key, + internal hostname, RFC1918 address, home directory path, or credential + -shaped key=value assignment, and no string carries two or more distinct + addresses that look like real routable public IPv4 addresses + +What this does NOT check, because it cannot be checked mechanically: whether +a `provenance.evidence` citation actually says what its `supports` field +claims, and whether that source is one the contributor owns or controls +(disallowed by the README's provenance rule). Both are reviewer judgment +calls, the same as the sanitization judgment a reviewer already makes on +every entry. + +The secret and IP scans are the machine-readable half of the CONTRIBUTING.md +anonymization rule: `sanitization.attestation` is a claim by the contributor, +and this scan is what stops that claim from being the only thing standing +between a real secret and a public dataset. It is pattern-based and will +miss anything that does not match one of the patterns below; it is a floor, +not a guarantee. +""" +from __future__ import annotations + +import json +import re +import sys +from datetime import date, datetime +from pathlib import Path + +try: + from jsonschema import Draft202012Validator +except ImportError: + sys.stderr.write("jsonschema not installed. Run: pip install jsonschema\n") + sys.exit(2) + +ROOT = Path(__file__).resolve().parent +SCHEMA_PATH = ROOT / "schema.json" +ENTRIES_DIR = ROOT / "entries" +TAXONOMY_PATH = ROOT.parent / "_shared" / "dsgai_taxonomy.json" + +# A value that is entirely a placeholder is exempt from the secret scan. +PLACEHOLDER_RE = re.compile(r"^<[^<>]+>$") + +# Each pattern needs a context cue, not just entropy: a bare hex or base64 run +# over-matches on digests, ids, and hashes that are perfectly fine to publish. +SECRET_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ + ("openai-style key", re.compile(r"\bsk-[A-Za-z0-9]{16,}")), + ("aws access key id", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("github token", re.compile(r"\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}")), + ("github fine-grained pat", re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}")), + ("slack token", re.compile(r"\bxox[abposr]-[A-Za-z0-9-]{10,}")), + ("google api key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")), + ("json web token", re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}")), + ("private key block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), + ("bearer credential", re.compile(r"\bBearer\s+[A-Za-z0-9._~+/-]{20,}")), + ("email address", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")), + ("rfc1918 address", re.compile(r"\b(10\.\d{1,3}|192\.168|172\.(1[6-9]|2\d|3[01]))\.\d{1,3}\.\d{1,3}\b")), + # .local is excluded on purpose: mDNS hostnames of this shape show up in + # legitimate lab and CTF write-ups, not only in production. .internal, + # .corp, .lan, and .intra are conventions with no ordinary public use. + ("internal hostname", re.compile(r"\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+\.(?:internal|corp|lan|intra)\b", re.IGNORECASE)), + # A key=value or key: value assignment for a credential-shaped field + # name, wherever it was typed - payload or prose. Excludes the case + # where the value itself is already a placeholder. + ("secret-looking assignment", re.compile(r"\b(?:password|passwd|secret|api[_-]?key|access[_-]?key|client[_-]?secret)\s*[:=]\s*(?!<)\S{4,}", re.IGNORECASE)), + # macOS/Linux only. The Windows arm this used to carry was written + # against a double backslash (r"[A-Za-z]:\\\\Users\\\\..."), which a + # JSON string decodes to a single backslash per separator, so it could + # never match a real entry. Dropped rather than shipped as a check that + # cannot fire; re-add as r"[A-Za-z]:\\Users\\[^\s\"]+" for real coverage. + ("user home path", re.compile(r"(/Users/|/home/)[A-Za-z0-9._-]+")), +] + +# A single non-loopback, non-RFC1918 IPv4 address is weak evidence on its +# own - plenty of legitimate strings contain four dot-separated numbers +# under 256 that are not addresses. This instead looks for two or more +# DISTINCT such addresses in one string, which is closer to the shape of +# someone pasting real network detail than of an incidental number. The +# RFC 5737 documentation ranges (192.0.2.0/24, 198.51.100.0/24, +# 203.0.113.0/24) are excluded outright: they are the correct way to write +# a fake-but-obviously-fake IP in an example, and this dataset should not +# penalize using them. +_IPV4_RE = re.compile(r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b") + + +def _is_routable_public_ip(octets: tuple[int, int, int, int]) -> bool: + a, b, c, d = octets + if any(o > 255 for o in (a, b, c, d)): + return False + if a == 10 or a == 127: + return False + if a == 192 and b == 168: + return False + if a == 172 and 16 <= b <= 31: + return False + if a == 192 and b == 0 and c == 2: # 192.0.2.0/24 (TEST-NET-1) + return False + if a == 198 and b == 51 and c == 100: # 198.51.100.0/24 (TEST-NET-2) + return False + if a == 203 and b == 0 and c == 113: # 203.0.113.0/24 (TEST-NET-3) + return False + if a == 0 or a >= 224: # this-network / multicast / reserved + return False + return True + + +def scan_for_public_ip_clusters(entry: dict, name: str, errors: list[str]) -> None: + for jpath, value in walk_strings(entry): + if PLACEHOLDER_RE.match(value.strip()): + continue + found = {m.group(0) for m in _IPV4_RE.finditer(value) if _is_routable_public_ip(tuple(int(g) for g in m.groups()))} + if len(found) >= 2: + errors.append( + f"{name}: {jpath}: value contains {len(found)} distinct addresses that look like " + f"real routable IPv4 addresses ({', '.join(sorted(found))}); use RFC 5737 documentation " + f"addresses (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) or a typed placeholder instead" + ) + + +# An identifier typed as a kind has to look like that kind, or the type field is +# decoration. Types with no single canonical form (advisory, publication, +# vendor_changelog, dataset, registry_record) are left unchecked on purpose. +CITATION_FORMATS: dict[str, tuple[re.Pattern[str], str]] = { + "doi": (re.compile(r"^10\.\d{4,9}/\S+$"), "a bare DOI (expected form: 10.xxxx/yyyy)"), + "cve": (re.compile(r"^CVE-\d{4}-\d{4,}$"), "a CVE id (expected form: CVE-YYYY-NNNNN)"), + "ghsa": ( + re.compile(r"^GHSA-[23456789cfghjmpqrvwx]{4}-[23456789cfghjmpqrvwx]{4}-[23456789cfghjmpqrvwx]{4}$"), + "a GHSA id (expected form: GHSA-xxxx-xxxx-xxxx)", + ), + "url": (re.compile(r"^https?://\S+$"), "an absolute http(s) URL"), +} + + +def load_taxonomy_ids() -> set[str]: + if not TAXONOMY_PATH.exists(): + return set() + data = json.loads(TAXONOMY_PATH.read_text(encoding="utf-8")) + return {e["id"] for e in data.get("entries", [])} + + +def walk_strings(node, path: str = ""): + """Yield (json_path, string) for every string anywhere under `node`.""" + if isinstance(node, dict): + for key, value in node.items(): + yield from walk_strings(value, f"{path}.{key}" if path else key) + elif isinstance(node, list): + for i, value in enumerate(node): + yield from walk_strings(value, f"{path}[{i}]") + elif isinstance(node, str): + yield path, node + + +def scan_for_secrets(entry: dict, name: str, errors: list[str]) -> None: + """Scan the WHOLE entry, not just span payloads. + + Payloads are the obvious place for a leaked value, but they are not the + likely one. A contributor writing `summary`, `notes`, `collection_method`, + or an evidence `locator` is writing prose while looking at a real trace, + which is exactly when a real hostname or address gets typed out. A scan + narrower than the attestation it backs is worse than no scan, because it + reads as coverage. + """ + for jpath, value in walk_strings(entry): + if PLACEHOLDER_RE.match(value.strip()): + continue + for label, pattern in SECRET_PATTERNS: + if pattern.search(value): + errors.append( + f"{name}: {jpath}: value looks like a real {label}; " + f"replace with a typed placeholder such as " + ) + break + + +def check_spans(entry: dict, name: str, errors: list[str]) -> None: + spans = entry.get("spans", []) or [] + seen: set[str] = set() + last_offset: int | None = None + + for index, span in enumerate(spans): + span_id = span.get("span_id", "") + if span_id in seen: + errors.append(f"{name}: duplicate span_id {span_id!r}") + + # Checked against the parents seen BEFORE this span, so a span + # cannot claim itself (or a later span) as its own parent - only a + # strictly preceding one, which is what "parent precedes child" + # requires. + parent = span.get("parent_span_id") + if parent is not None and parent not in seen: + errors.append( + f"{name}: span {span_id!r} has parent_span_id {parent!r}, " + f"which is not a preceding span" + ) + + seen.add(span_id) + + offset = span.get("t_offset_ms") + if isinstance(offset, int): + if last_offset is not None and offset < last_offset: + errors.append( + f"{name}: span {span_id!r} (index {index}) moves t_offset_ms backwards " + f"({last_offset} -> {offset}); spans should be listed in temporal order" + ) + last_offset = offset + + +def check_findings(entry: dict, name: str, errors: list[str]) -> None: + declared = set(entry.get("dsgai_mapping", []) or []) + for span in entry.get("spans", []) or []: + finding = span.get("finding") + if not finding: + continue + risk = finding.get("dsgai_id", "") + if risk not in declared: + errors.append( + f"{name}: span {span.get('span_id', '?')!r} has a finding for {risk}, " + f"which is not in the trace's dsgai_mapping {sorted(declared)}" + ) + + +def check_provenance(entry: dict, name: str, errors: list[str]) -> None: + prov = entry.get("provenance", {}) or {} + tier = prov.get("tier", "") + evidence = prov.get("evidence") or [] + if tier != "hypothetical" and not evidence: + errors.append( + f"{name}: provenance.tier is {tier!r}, which requires at least one " + f"entry in provenance.evidence. Use tier 'hypothetical' if the trace " + f"has no citable backing." + ) + for i, item in enumerate(evidence): + rule = CITATION_FORMATS.get(item.get("type", "")) + if rule is None: + continue + pattern, expected = rule + citation = item.get("citation", "") + if not pattern.match(citation): + errors.append( + f"{name}: provenance.evidence[{i}].citation {citation!r} is typed " + f"{item['type']!r} but is not {expected}" + ) + + +def check_dates(entry: dict, name: str, errors: list[str]) -> None: + """Check ISO dates with the stdlib. + + `"format": "date"` in the schema is an annotation, not a constraint, unless + a format checker is wired in - and the checkers for `date-time` and `uri` + need optional packages that may or may not be installed. FORMAT_CHECKER is + enabled below for whatever it can cover; this function guarantees the date + fields are checked in every environment regardless. + """ + for field in ("date_added", "date_observed"): + value = entry.get(field) + if value is None: + continue + try: + date.fromisoformat(value) + except (ValueError, TypeError): + errors.append(f"{name}: {field} {value!r} is not an ISO 8601 date (YYYY-MM-DD)") + + # spans[].timestamp is "format": "date-time" in the schema, which is the + # other field this dataset actually expects contributors to fill in with + # a real value (an observed or lab trace has wall-clock times; a derived + # or hypothetical one usually does not). The jsonschema format checker + # only validates date-time when the optional rfc3339-validator package is + # installed, so this is hand-rolled the same way date_added is, rather + # than trusting the schema to have caught it. + for index, span in enumerate(entry.get("spans", []) or []): + ts = span.get("timestamp") + if ts is None: + continue + try: + datetime.fromisoformat(str(ts).replace("Z", "+00:00")) + except (ValueError, TypeError): + span_id = span.get("span_id", f"index {index}") + errors.append( + f"{name}: span {span_id!r} timestamp {ts!r} is not an ISO 8601 date-time" + ) + + +def main() -> int: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft202012Validator( + schema, format_checker=Draft202012Validator.FORMAT_CHECKER + ) + taxonomy_ids = load_taxonomy_ids() + + errors: list[str] = [] + warnings: list[str] = [] + if not taxonomy_ids: + warnings.append( + f"{TAXONOMY_PATH} not found or empty - dsgai_mapping values were NOT " + f"cross-checked against the shared taxonomy this run" + ) + entry_files = sorted(ENTRIES_DIR.glob("*.json")) if ENTRIES_DIR.exists() else [] + + # The worked example is validated too. An example that does not validate is + # a trap for the next contributor, and it is the only thing to validate + # while entries/ is still empty, so an empty dataset is a pass and not an + # error. + example_path = ROOT / "example.json" + if not entry_files and not example_path.exists(): + sys.stderr.write(f"Nothing to validate: no entries in {ENTRIES_DIR}, no example.json\n") + return 1 + checked = entry_files + ([example_path] if example_path.exists() else []) + + for path in checked: + name = path.name + try: + entry = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + errors.append(f"{name}: invalid JSON - {e}") + continue + + for err in validator.iter_errors(entry): + loc = "/".join(str(p) for p in err.absolute_path) or "" + errors.append(f"{name}: {loc}: {err.message}") + + trace_id = entry.get("trace_id", "") + if trace_id and path.parent == ENTRIES_DIR and trace_id != path.stem: + errors.append( + f"{name}: trace_id {trace_id!r} does not match the filename stem {path.stem!r}" + ) + + if taxonomy_ids: + for dsgai_id in entry.get("dsgai_mapping", []) or []: + if dsgai_id not in taxonomy_ids: + errors.append(f"{name}: dsgai_mapping {dsgai_id!r} not in taxonomy") + + check_provenance(entry, name, errors) + check_spans(entry, name, errors) + check_findings(entry, name, errors) + check_dates(entry, name, errors) + scan_for_secrets(entry, name, errors) + scan_for_public_ip_clusters(entry, name, errors) + + for line in warnings: + print(f"WARN {line}") + + if errors: + for line in errors: + print(f"ERROR {line}") + print(f"\nFAIL: {len(errors)} issue(s) across {len(checked)} files.") + return 1 + + if entry_files: + print(f"OK: {len(entry_files)} entries + example.json validated against schema.") + else: + print("OK: example.json validated against schema. entries/ is empty.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())