Skip to content

Commit 9faed5c

Browse files
committed
chore(oie): NIST 800-53 and remaining B2 sources/gold
NIST OSCAL sample plus Top10/API/K8s/LLM/AISVS/agentic fixture sources and gold JSON. Stacked after the ASVS-only fixture PR.
1 parent 0345e20 commit 9faed5c

128 files changed

Lines changed: 16199 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎application/tests/fixtures/owasp_mappings/nist_800_53_v5.json‎

Lines changed: 722 additions & 0 deletions
Large diffs are not rendered by default.

‎application/tests/owasp_mapping_fixtures_test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010

1111
EXPECTED_FIXTURES = {
12+
"nist_800_53_v5.json",
1213
"owasp_aisvs_1_0.json",
1314
"owasp_api_top10_2023.json",
1415
"owasp_asvs_5_0_provisional.json",
Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
#!/usr/bin/env python3
2+
"""Build NIST 800-53 v5 eval gold (opencre.org Links) + OSCAL Module A sources.
3+
4+
Gold (CRE answer key)::
5+
6+
application/tests/fixtures/owasp_mappings/nist_800_53_v5.json
7+
8+
Sources (control statement + guidance from usnistgov OSCAL catalog)::
9+
10+
scripts/oie_owasp_eval/fixtures/b2_sources/nist_800_53_v5/<SECTION_ID>.txt
11+
12+
Usage::
13+
14+
PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py
15+
PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py --sample 48
16+
"""
17+
18+
from __future__ import annotations
19+
20+
import argparse
21+
import json
22+
import re
23+
import sys
24+
import urllib.parse
25+
import urllib.request
26+
from collections import defaultdict
27+
from pathlib import Path
28+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple
29+
30+
ROOT = Path(__file__).resolve().parents[2]
31+
DEFAULT_OUT_GOLD = (
32+
ROOT
33+
/ "application"
34+
/ "tests"
35+
/ "fixtures"
36+
/ "owasp_mappings"
37+
/ "nist_800_53_v5.json"
38+
)
39+
DEFAULT_SOURCES = (
40+
ROOT / "scripts" / "oie_owasp_eval" / "fixtures" / "b2_sources" / "nist_800_53_v5"
41+
)
42+
OSCAL_URL = (
43+
"https://raw.githubusercontent.com/usnistgov/oscal-content/main/"
44+
"nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json"
45+
)
46+
OSCAL_CACHE = ROOT / "tmp" / "oie_owasp_eval" / "nist_sp80053_rev5_catalog.json"
47+
BASE_URL = "https://opencre.org"
48+
USER_AGENT = "OpenCRE-NIST80053-EvalGold/1.0"
49+
STANDARD_NAME = "NIST 800-53 v5"
50+
51+
# Hub uses AC-2 / IA-2(1); OSCAL labels are AC-02 / IA-02(01).
52+
_LABEL_RE = re.compile(
53+
r"^([A-Z]{1,4})-0*(\d+)(?:\((0*)(\d+)\))?$",
54+
re.IGNORECASE,
55+
)
56+
57+
58+
def _get_json(url: str) -> Any:
59+
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
60+
with urllib.request.urlopen(req, timeout=120) as resp:
61+
return json.load(resp)
62+
63+
64+
def normalize_control_id(raw: str) -> str:
65+
"""Map ``AC-02`` / ``IA-02(01)`` / ``ac-2.1`` → hub-style ``AC-2`` / ``IA-2(1)``."""
66+
s = (raw or "").strip().upper().replace("_", "-")
67+
# OSCAL dotted enhancement id: ia-2.1 / IA-02.01
68+
dotted = re.match(r"^([A-Z]{1,4})-0*(\d+)\.0*(\d+)$", s)
69+
if dotted:
70+
return f"{dotted.group(1)}-{int(dotted.group(2))}({int(dotted.group(3))})"
71+
m = _LABEL_RE.match(s)
72+
if m:
73+
fam = m.group(1).upper()
74+
num = int(m.group(2))
75+
if m.group(4) is not None:
76+
return f"{fam}-{num}({int(m.group(4))})"
77+
return f"{fam}-{num}"
78+
# Already hub-ish: AC-2(1)
79+
m2 = re.match(r"^([A-Z]{1,4})-(\d+)(?:\((\d+)\))?$", s, re.IGNORECASE)
80+
if not m2:
81+
return s
82+
fam, num, enh = m2.group(1).upper(), m2.group(2), m2.group(3)
83+
return f"{fam}-{int(num)}" + (f"({int(enh)})" if enh else "")
84+
85+
86+
def sid_from_node(node: Mapping[str, Any]) -> Optional[str]:
87+
for key in ("sectionID", "section_id"):
88+
val = node.get(key)
89+
if val:
90+
return normalize_control_id(str(val))
91+
section = str(node.get("section") or "")
92+
m = re.match(r"^([A-Z]{1,4}-\d+(?:\([0-9]+\))?)\b", section, re.I)
93+
if m:
94+
return normalize_control_id(m.group(1))
95+
return None
96+
97+
98+
def cre_ids_from_node(node: Mapping[str, Any]) -> List[str]:
99+
out: List[str] = []
100+
for link in node.get("links") or []:
101+
if not isinstance(link, dict):
102+
continue
103+
doc = link.get("document") or {}
104+
if str(doc.get("doctype") or "").upper() != "CRE":
105+
continue
106+
cid = str(doc.get("id") or "").strip()
107+
if cid and cid not in out:
108+
out.append(cid)
109+
return out
110+
111+
112+
def fetch_hub_nodes(*, base_url: str = BASE_URL) -> List[Dict[str, Any]]:
113+
rows: List[Dict[str, Any]] = []
114+
page = 1
115+
while True:
116+
url = (
117+
f"{base_url.rstrip('/')}/rest/v1/Standard/"
118+
f"{urllib.parse.quote(STANDARD_NAME)}?page={page}"
119+
)
120+
data = _get_json(url)
121+
batch = data.get("standards") or []
122+
if not batch:
123+
break
124+
rows.extend(batch)
125+
total_pages = int(data.get("total_pages") or page)
126+
if page >= total_pages:
127+
break
128+
page += 1
129+
return rows
130+
131+
132+
def build_gold_rows(nodes: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]:
133+
rows: List[Dict[str, Any]] = []
134+
seen: set[str] = set()
135+
for node in nodes:
136+
sid = sid_from_node(node)
137+
cres = cre_ids_from_node(node)
138+
if not sid or not cres:
139+
continue
140+
if sid in seen:
141+
continue
142+
seen.add(sid)
143+
rows.append(
144+
{
145+
"section_id": sid,
146+
"section": str(node.get("section") or sid),
147+
"hyperlink": str(node.get("hyperlink") or ""),
148+
"cre_ids": cres,
149+
"gold_provenance": {
150+
"kind": "hub_links",
151+
"method": "opencre_org_standard_api",
152+
"standard": STANDARD_NAME,
153+
},
154+
}
155+
)
156+
rows.sort(key=lambda r: r["section_id"])
157+
return rows
158+
159+
160+
def stratified_sample(rows: Sequence[Dict[str, Any]], n: int) -> List[Dict[str, Any]]:
161+
"""Take up to ``n`` rows, round-robin across control families."""
162+
if n <= 0 or n >= len(rows):
163+
return list(rows)
164+
by_fam: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
165+
for row in rows:
166+
fam = str(row["section_id"]).split("-", 1)[0]
167+
by_fam[fam].append(row)
168+
families = sorted(by_fam.keys(), key=lambda f: (-len(by_fam[f]), f))
169+
out: List[Dict[str, Any]] = []
170+
idx = {f: 0 for f in families}
171+
while len(out) < n:
172+
progressed = False
173+
for fam in families:
174+
i = idx[fam]
175+
bucket = by_fam[fam]
176+
if i >= len(bucket):
177+
continue
178+
out.append(bucket[i])
179+
idx[fam] = i + 1
180+
progressed = True
181+
if len(out) >= n:
182+
break
183+
if not progressed:
184+
break
185+
out.sort(key=lambda r: r["section_id"])
186+
for row in out:
187+
prov = dict(row.get("gold_provenance") or {})
188+
prov["sample"] = True
189+
prov["sample_size"] = n
190+
row["gold_provenance"] = prov
191+
return out
192+
193+
194+
def _walk_prose(parts: Sequence[Mapping[str, Any]], chunks: List[str]) -> None:
195+
for part in parts or []:
196+
if not isinstance(part, dict):
197+
continue
198+
prose = part.get("prose")
199+
if isinstance(prose, str) and prose.strip():
200+
chunks.append(prose.strip())
201+
nested = part.get("parts")
202+
if isinstance(nested, list):
203+
_walk_prose(nested, chunks)
204+
205+
206+
def load_oscal_index(path: Path) -> Dict[str, Tuple[str, str]]:
207+
"""Map hub-style section id → (title, statement+guidance prose)."""
208+
catalog = json.loads(path.read_text(encoding="utf-8"))["catalog"]
209+
index: Dict[str, Tuple[str, str]] = {}
210+
211+
def add(control: Mapping[str, Any]) -> None:
212+
label = None
213+
for prop in control.get("props") or []:
214+
if isinstance(prop, dict) and prop.get("name") == "label":
215+
label = str(prop.get("value") or "")
216+
break
217+
raw_id = label or str(control.get("id") or "")
218+
sid = normalize_control_id(raw_id)
219+
title = str(control.get("title") or sid)
220+
chunks: List[str] = []
221+
# Prefer statement + guidance only (skip assessment noise).
222+
for part in control.get("parts") or []:
223+
if not isinstance(part, dict):
224+
continue
225+
name = str(part.get("name") or "")
226+
if name not in {"statement", "guidance"}:
227+
continue
228+
prose = part.get("prose")
229+
if isinstance(prose, str) and prose.strip():
230+
chunks.append(prose.strip())
231+
_walk_prose(part.get("parts") or [], chunks)
232+
text = "\n".join(chunks).strip()
233+
# Strip OSCAL parameter placeholders for cleaner retrieval.
234+
text = re.sub(r"\{\{\s*insert:[^}]+\}\}", "…", text)
235+
if sid and text:
236+
index[sid] = (title, text)
237+
for sub in control.get("controls") or []:
238+
if isinstance(sub, dict):
239+
add(sub)
240+
241+
for group in catalog.get("groups") or []:
242+
for control in group.get("controls") or []:
243+
if isinstance(control, dict):
244+
add(control)
245+
return index
246+
247+
248+
def ensure_oscal(cache: Path = OSCAL_CACHE) -> Path:
249+
cache.parent.mkdir(parents=True, exist_ok=True)
250+
if cache.is_file() and cache.stat().st_size > 1_000_000:
251+
return cache
252+
print(f"GET {OSCAL_URL}", flush=True)
253+
urllib.request.urlretrieve(OSCAL_URL, cache)
254+
return cache
255+
256+
257+
def write_sources(
258+
rows: Sequence[Mapping[str, Any]],
259+
oscal: Mapping[str, Tuple[str, str]],
260+
dest: Path,
261+
) -> Dict[str, int]:
262+
dest.mkdir(parents=True, exist_ok=True)
263+
written = 0
264+
missing = 0
265+
for row in rows:
266+
sid = str(row["section_id"])
267+
title, body = oscal.get(sid, ("", ""))
268+
if not body:
269+
missing += 1
270+
continue
271+
section = str(row.get("section") or title or sid)
272+
text = (
273+
f"Source: {sid}\n"
274+
f"Section-ID: {sid}\n"
275+
f"Section: {section}\n"
276+
f"Standard: {STANDARD_NAME}\n\n"
277+
f"{title}\n\n{body}\n"
278+
)
279+
# Match B2 loader: b2_sources/<stem>/<section_id>.txt
280+
(dest / f"{sid}.txt").write_text(text, encoding="utf-8")
281+
written += 1
282+
return {"written": written, "missing_oscal": missing}
283+
284+
285+
def main(argv: Optional[Sequence[str]] = None) -> int:
286+
parser = argparse.ArgumentParser(description=__doc__)
287+
parser.add_argument("--out-gold", type=Path, default=DEFAULT_OUT_GOLD)
288+
parser.add_argument("--out-sources", type=Path, default=DEFAULT_SOURCES)
289+
parser.add_argument(
290+
"--sample",
291+
type=int,
292+
default=0,
293+
help="If >0, stratified sample of N controls (default: all hub-linked).",
294+
)
295+
parser.add_argument("--base-url", default=BASE_URL)
296+
parser.add_argument("--skip-sources", action="store_true")
297+
args = parser.parse_args(list(argv) if argv is not None else None)
298+
299+
print(f"fetching {STANDARD_NAME} from {args.base_url} …", flush=True)
300+
nodes = fetch_hub_nodes(base_url=args.base_url)
301+
gold = build_gold_rows(nodes)
302+
print(f"hub nodes={len(nodes)} gold_rows={len(gold)}", flush=True)
303+
if args.sample > 0:
304+
gold = stratified_sample(gold, args.sample)
305+
print(f"sampled to {len(gold)}", flush=True)
306+
307+
args.out_gold.parent.mkdir(parents=True, exist_ok=True)
308+
args.out_gold.write_text(json.dumps(gold, indent=2) + "\n", encoding="utf-8")
309+
print(f"wrote {args.out_gold}", flush=True)
310+
311+
if not args.skip_sources:
312+
oscal_path = ensure_oscal()
313+
index = load_oscal_index(oscal_path)
314+
print(f"oscal controls indexed={len(index)}", flush=True)
315+
stats = write_sources(gold, index, args.out_sources)
316+
print(json.dumps({"sources_dir": str(args.out_sources), **stats}, indent=2))
317+
if stats["missing_oscal"]:
318+
print(
319+
f"warning: {stats['missing_oscal']} gold ids missing from OSCAL",
320+
file=sys.stderr,
321+
)
322+
return 0
323+
324+
325+
if __name__ == "__main__":
326+
raise SystemExit(main())

0 commit comments

Comments
 (0)