Skip to content

Commit 52d8baa

Browse files
committed
chore(oie): NIST 800-53 and remaining B2 sources/gold
NIST OSCAL sample plus Top10/API/K8s/LLM/AISVS/agentic fixture sources and gold JSON. Stacked after the ASVS-only fixture PR.
1 parent c66eb52 commit 52d8baa

128 files changed

Lines changed: 16232 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎application/tests/fixtures/owasp_mappings/nist_800_53_v5.json‎

Lines changed: 722 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 328 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,328 @@
1+
#!/usr/bin/env python3
2+
"""Build NIST 800-53 v5 eval gold (opencre.org Links) + OSCAL Module A sources.
3+
4+
Gold (CRE answer key)::
5+
6+
application/tests/fixtures/owasp_mappings/nist_800_53_v5.json
7+
8+
Sources (control statement + guidance from usnistgov OSCAL catalog)::
9+
10+
scripts/oie_owasp_eval/fixtures/b2_sources/nist_800_53_v5/<SECTION_ID>.txt
11+
12+
Usage::
13+
14+
PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py
15+
PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py --sample 48
16+
"""
17+
18+
from __future__ import annotations
19+
20+
import argparse
21+
import json
22+
import re
23+
import sys
24+
import urllib.parse
25+
import urllib.request
26+
from collections import defaultdict
27+
from pathlib import Path
28+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
29+
30+
ROOT = Path(__file__).resolve().parents[2]
31+
DEFAULT_OUT_GOLD = (
32+
ROOT
33+
/ "application"
34+
/ "tests"
35+
/ "fixtures"
36+
/ "owasp_mappings"
37+
/ "nist_800_53_v5.json"
38+
)
39+
DEFAULT_SOURCES = (
40+
ROOT / "scripts" / "oie_owasp_eval" / "fixtures" / "b2_sources" / "nist_800_53_v5"
41+
)
42+
OSCAL_URL = (
43+
"https://raw.githubusercontent.com/usnistgov/oscal-content/main/"
44+
"nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json"
45+
)
46+
OSCAL_CACHE = ROOT / "tmp" / "oie_owasp_eval" / "nist_sp80053_rev5_catalog.json"
47+
BASE_URL = "https://opencre.org"
48+
USER_AGENT = "OpenCRE-NIST80053-EvalGold/1.0"
49+
STANDARD_NAME = "NIST 800-53 v5"
50+
51+
# Hub uses AC-2 / IA-2(1); OSCAL labels are AC-02 / IA-02(01).
52+
_LABEL_RE = re.compile(
53+
r"^([A-Z]{1,4})-0*(\d+)(?:\((0*)(\d+)\))?$",
54+
re.IGNORECASE,
55+
)
56+
57+
58+
def _get_json(url: str) -> Any:
59+
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
60+
with urllib.request.urlopen(req, timeout=120) as resp:
61+
return json.load(resp)
62+
63+
64+
def normalize_control_id(raw: str) -> str:
65+
"""Map ``AC-02`` / ``IA-02(01)`` / ``ac-2.1`` → hub-style ``AC-2`` / ``IA-2(1)``."""
66+
s = (raw or "").strip().upper().replace("_", "-")
67+
# OSCAL dotted enhancement id: ia-2.1 / IA-02.01
68+
dotted = re.match(r"^([A-Z]{1,4})-0*(\d+)\.0*(\d+)$", s)
69+
if dotted:
70+
return f"{dotted.group(1)}-{int(dotted.group(2))}({int(dotted.group(3))})"
71+
m = _LABEL_RE.match(s)
72+
if m:
73+
fam = m.group(1).upper()
74+
num = int(m.group(2))
75+
if m.group(4) is not None:
76+
return f"{fam}-{num}({int(m.group(4))})"
77+
return f"{fam}-{num}"
78+
# Already hub-ish: AC-2(1)
79+
m2 = re.match(r"^([A-Z]{1,4})-(\d+)(?:\((\d+)\))?$", s, re.IGNORECASE)
80+
if not m2:
81+
return s
82+
fam, num, enh = m2.group(1).upper(), m2.group(2), m2.group(3)
83+
return f"{fam}-{int(num)}" + (f"({int(enh)})" if enh else "")
84+
85+
86+
def sid_from_node(node: Mapping[str, Any]) -> Optional[str]:
87+
for key in ("sectionID", "section_id"):
88+
val = node.get(key)
89+
if val:
90+
return normalize_control_id(str(val))
91+
section = str(node.get("section") or "")
92+
m = re.match(r"^([A-Z]{1,4}-\d+(?:\([0-9]+\))?)\b", section, re.I)
93+
if m:
94+
return normalize_control_id(m.group(1))
95+
return None
96+
97+
98+
def cre_ids_from_node(node: Mapping[str, Any]) -> List[str]:
99+
out: List[str] = []
100+
for link in node.get("links") or []:
101+
if not isinstance(link, dict):
102+
continue
103+
doc = link.get("document") or {}
104+
if str(doc.get("doctype") or "").upper() != "CRE":
105+
continue
106+
cid = str(doc.get("id") or "").strip()
107+
if cid and cid not in out:
108+
out.append(cid)
109+
return out
110+
111+
112+
def fetch_hub_nodes(*, base_url: str = BASE_URL) -> List[Dict[str, Any]]:
113+
rows: List[Dict[str, Any]] = []
114+
page = 1
115+
while True:
116+
url = (
117+
f"{base_url.rstrip('/')}/rest/v1/Standard/"
118+
f"{urllib.parse.quote(STANDARD_NAME)}?page={page}"
119+
)
120+
data = _get_json(url)
121+
batch = data.get("standards") or []
122+
if not batch:
123+
break
124+
rows.extend(batch)
125+
total_pages = int(data.get("total_pages") or page)
126+
if page >= total_pages:
127+
break
128+
page += 1
129+
return rows
130+
131+
132+
def build_gold_rows(nodes: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]:
133+
rows: List[Dict[str, Any]] = []
134+
seen: set[str] = set()
135+
for node in nodes:
136+
sid = sid_from_node(node)
137+
cres = cre_ids_from_node(node)
138+
if not sid or not cres:
139+
continue
140+
if sid in seen:
141+
continue
142+
seen.add(sid)
143+
rows.append(
144+
{
145+
"section_id": sid,
146+
"section": str(node.get("section") or sid),
147+
"hyperlink": str(node.get("hyperlink") or ""),
148+
"cre_ids": cres,
149+
"gold_provenance": {
150+
"kind": "hub_links",
151+
"method": "opencre_org_standard_api",
152+
"standard": STANDARD_NAME,
153+
},
154+
}
155+
)
156+
rows.sort(key=lambda r: r["section_id"])
157+
return rows
158+
159+
160+
def stratified_sample(
161+
rows: Sequence[Dict[str, Any]], n: int
162+
) -> List[Dict[str, Any]]:
163+
"""Take up to ``n`` rows, round-robin across control families."""
164+
if n <= 0 or n >= len(rows):
165+
return list(rows)
166+
by_fam: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
167+
for row in rows:
168+
fam = str(row["section_id"]).split("-", 1)[0]
169+
by_fam[fam].append(row)
170+
families = sorted(by_fam.keys(), key=lambda f: (-len(by_fam[f]), f))
171+
out: List[Dict[str, Any]] = []
172+
idx = {f: 0 for f in families}
173+
while len(out) < n:
174+
progressed = False
175+
for fam in families:
176+
i = idx[fam]
177+
bucket = by_fam[fam]
178+
if i >= len(bucket):
179+
continue
180+
out.append(bucket[i])
181+
idx[fam] = i + 1
182+
progressed = True
183+
if len(out) >= n:
184+
break
185+
if not progressed:
186+
break
187+
out.sort(key=lambda r: r["section_id"])
188+
for row in out:
189+
prov = dict(row.get("gold_provenance") or {})
190+
prov["sample"] = True
191+
prov["sample_size"] = n
192+
row["gold_provenance"] = prov
193+
return out
194+
195+
196+
def _walk_prose(parts: Sequence[Mapping[str, Any]], chunks: List[str]) -> None:
197+
for part in parts or []:
198+
if not isinstance(part, dict):
199+
continue
200+
prose = part.get("prose")
201+
if isinstance(prose, str) and prose.strip():
202+
chunks.append(prose.strip())
203+
nested = part.get("parts")
204+
if isinstance(nested, list):
205+
_walk_prose(nested, chunks)
206+
207+
208+
def load_oscal_index(path: Path) -> Dict[str, Tuple[str, str]]:
209+
"""Map hub-style section id → (title, statement+guidance prose)."""
210+
catalog = json.loads(path.read_text(encoding="utf-8"))["catalog"]
211+
index: Dict[str, Tuple[str, str]] = {}
212+
213+
def add(control: Mapping[str, Any]) -> None:
214+
label = None
215+
for prop in control.get("props") or []:
216+
if isinstance(prop, dict) and prop.get("name") == "label":
217+
label = str(prop.get("value") or "")
218+
break
219+
raw_id = label or str(control.get("id") or "")
220+
sid = normalize_control_id(raw_id)
221+
title = str(control.get("title") or sid)
222+
chunks: List[str] = []
223+
# Prefer statement + guidance only (skip assessment noise).
224+
for part in control.get("parts") or []:
225+
if not isinstance(part, dict):
226+
continue
227+
name = str(part.get("name") or "")
228+
if name not in {"statement", "guidance"}:
229+
continue
230+
prose = part.get("prose")
231+
if isinstance(prose, str) and prose.strip():
232+
chunks.append(prose.strip())
233+
_walk_prose(part.get("parts") or [], chunks)
234+
text = "\n".join(chunks).strip()
235+
# Strip OSCAL parameter placeholders for cleaner retrieval.
236+
text = re.sub(r"\{\{\s*insert:[^}]+\}\}", "…", text)
237+
if sid and text:
238+
index[sid] = (title, text)
239+
for sub in control.get("controls") or []:
240+
if isinstance(sub, dict):
241+
add(sub)
242+
243+
for group in catalog.get("groups") or []:
244+
for control in group.get("controls") or []:
245+
if isinstance(control, dict):
246+
add(control)
247+
return index
248+
249+
250+
def ensure_oscal(cache: Path = OSCAL_CACHE) -> Path:
251+
cache.parent.mkdir(parents=True, exist_ok=True)
252+
if cache.is_file() and cache.stat().st_size > 1_000_000:
253+
return cache
254+
print(f"GET {OSCAL_URL}", flush=True)
255+
urllib.request.urlretrieve(OSCAL_URL, cache)
256+
return cache
257+
258+
259+
def write_sources(
260+
rows: Sequence[Mapping[str, Any]],
261+
oscal: Mapping[str, Tuple[str, str]],
262+
dest: Path,
263+
) -> Dict[str, int]:
264+
dest.mkdir(parents=True, exist_ok=True)
265+
written = 0
266+
missing = 0
267+
for row in rows:
268+
sid = str(row["section_id"])
269+
title, body = oscal.get(sid, ("", ""))
270+
if not body:
271+
missing += 1
272+
continue
273+
section = str(row.get("section") or title or sid)
274+
text = (
275+
f"Source: {sid}\n"
276+
f"Section-ID: {sid}\n"
277+
f"Section: {section}\n"
278+
f"Standard: {STANDARD_NAME}\n\n"
279+
f"{title}\n\n{body}\n"
280+
)
281+
# Match B2 loader: b2_sources/<stem>/<section_id>.txt
282+
(dest / f"{sid}.txt").write_text(text, encoding="utf-8")
283+
written += 1
284+
return {"written": written, "missing_oscal": missing}
285+
286+
287+
def main(argv: Optional[Sequence[str]] = None) -> int:
288+
parser = argparse.ArgumentParser(description=__doc__)
289+
parser.add_argument("--out-gold", type=Path, default=DEFAULT_OUT_GOLD)
290+
parser.add_argument("--out-sources", type=Path, default=DEFAULT_SOURCES)
291+
parser.add_argument(
292+
"--sample",
293+
type=int,
294+
default=0,
295+
help="If >0, stratified sample of N controls (default: all hub-linked).",
296+
)
297+
parser.add_argument("--base-url", default=BASE_URL)
298+
parser.add_argument("--skip-sources", action="store_true")
299+
args = parser.parse_args(list(argv) if argv is not None else None)
300+
301+
print(f"fetching {STANDARD_NAME} from {args.base_url} …", flush=True)
302+
nodes = fetch_hub_nodes(base_url=args.base_url)
303+
gold = build_gold_rows(nodes)
304+
print(f"hub nodes={len(nodes)} gold_rows={len(gold)}", flush=True)
305+
if args.sample > 0:
306+
gold = stratified_sample(gold, args.sample)
307+
print(f"sampled to {len(gold)}", flush=True)
308+
309+
args.out_gold.parent.mkdir(parents=True, exist_ok=True)
310+
args.out_gold.write_text(json.dumps(gold, indent=2) + "\n", encoding="utf-8")
311+
print(f"wrote {args.out_gold}", flush=True)
312+
313+
if not args.skip_sources:
314+
oscal_path = ensure_oscal()
315+
index = load_oscal_index(oscal_path)
316+
print(f"oscal controls indexed={len(index)}", flush=True)
317+
stats = write_sources(gold, index, args.out_sources)
318+
print(json.dumps({"sources_dir": str(args.out_sources), **stats}, indent=2))
319+
if stats["missing_oscal"]:
320+
print(
321+
f"warning: {stats['missing_oscal']} gold ids missing from OSCAL",
322+
file=sys.stderr,
323+
)
324+
return 0
325+
326+
327+
if __name__ == "__main__":
328+
raise SystemExit(main())
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
[
2+
{
3+
"section_id": "AAI01",
4+
"section": "Goal Hijacking and Agent Drift",
5+
"hyperlink": "",
6+
"cre_ids": ["177-260", "724-770"]
7+
},
8+
{
9+
"section_id": "AAI02",
10+
"section": "Tool Misuse and Unsafe Actuation",
11+
"hyperlink": "",
12+
"cre_ids": ["486-813", "117-371"]
13+
},
14+
{
15+
"section_id": "AAI03",
16+
"section": "Multi-Agent Collusion and Privilege Escalation",
17+
"hyperlink": "",
18+
"cre_ids": ["128-128", "633-428"]
19+
},
20+
{
21+
"section_id": "AAI04",
22+
"section": "Memory Poisoning Across Agent Sessions",
23+
"hyperlink": "",
24+
"cre_ids": ["340-375", "613-287"]
25+
},
26+
{
27+
"section_id": "AAI05",
28+
"section": "Unsupervised Cross-System Workflows",
29+
"hyperlink": "",
30+
"cre_ids": ["132-146", "515-021"]
31+
}
32+
]

0 commit comments

Comments
 (0)