|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Build NIST 800-53 v5 eval gold (opencre.org Links) + OSCAL Module A sources. |
| 3 | +
|
| 4 | +Gold (CRE answer key):: |
| 5 | +
|
| 6 | + application/tests/fixtures/owasp_mappings/nist_800_53_v5.json |
| 7 | +
|
| 8 | +Sources (control statement + guidance from usnistgov OSCAL catalog):: |
| 9 | +
|
| 10 | + scripts/oie_owasp_eval/fixtures/b2_sources/nist_800_53_v5/<SECTION_ID>.txt |
| 11 | +
|
| 12 | +Usage:: |
| 13 | +
|
| 14 | + PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py |
| 15 | + PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py --sample 48 |
| 16 | +""" |
| 17 | + |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import argparse |
| 21 | +import json |
| 22 | +import re |
| 23 | +import sys |
| 24 | +import urllib.parse |
| 25 | +import urllib.request |
| 26 | +from collections import defaultdict |
| 27 | +from pathlib import Path |
| 28 | +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple |
| 29 | + |
| 30 | +ROOT = Path(__file__).resolve().parents[2] |
| 31 | +DEFAULT_OUT_GOLD = ( |
| 32 | + ROOT |
| 33 | + / "application" |
| 34 | + / "tests" |
| 35 | + / "fixtures" |
| 36 | + / "owasp_mappings" |
| 37 | + / "nist_800_53_v5.json" |
| 38 | +) |
| 39 | +DEFAULT_SOURCES = ( |
| 40 | + ROOT / "scripts" / "oie_owasp_eval" / "fixtures" / "b2_sources" / "nist_800_53_v5" |
| 41 | +) |
| 42 | +OSCAL_URL = ( |
| 43 | + "https://raw.githubusercontent.com/usnistgov/oscal-content/main/" |
| 44 | + "nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json" |
| 45 | +) |
| 46 | +OSCAL_CACHE = ROOT / "tmp" / "oie_owasp_eval" / "nist_sp80053_rev5_catalog.json" |
| 47 | +BASE_URL = "https://opencre.org" |
| 48 | +USER_AGENT = "OpenCRE-NIST80053-EvalGold/1.0" |
| 49 | +STANDARD_NAME = "NIST 800-53 v5" |
| 50 | + |
| 51 | +# Hub uses AC-2 / IA-2(1); OSCAL labels are AC-02 / IA-02(01). |
| 52 | +_LABEL_RE = re.compile( |
| 53 | + r"^([A-Z]{1,4})-0*(\d+)(?:\((0*)(\d+)\))?$", |
| 54 | + re.IGNORECASE, |
| 55 | +) |
| 56 | + |
| 57 | + |
| 58 | +def _get_json(url: str) -> Any: |
| 59 | + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) |
| 60 | + with urllib.request.urlopen(req, timeout=120) as resp: |
| 61 | + return json.load(resp) |
| 62 | + |
| 63 | + |
| 64 | +def normalize_control_id(raw: str) -> str: |
| 65 | + """Map ``AC-02`` / ``IA-02(01)`` / ``ac-2.1`` → hub-style ``AC-2`` / ``IA-2(1)``.""" |
| 66 | + s = (raw or "").strip().upper().replace("_", "-") |
| 67 | + # OSCAL dotted enhancement id: ia-2.1 / IA-02.01 |
| 68 | + dotted = re.match(r"^([A-Z]{1,4})-0*(\d+)\.0*(\d+)$", s) |
| 69 | + if dotted: |
| 70 | + return f"{dotted.group(1)}-{int(dotted.group(2))}({int(dotted.group(3))})" |
| 71 | + m = _LABEL_RE.match(s) |
| 72 | + if m: |
| 73 | + fam = m.group(1).upper() |
| 74 | + num = int(m.group(2)) |
| 75 | + if m.group(4) is not None: |
| 76 | + return f"{fam}-{num}({int(m.group(4))})" |
| 77 | + return f"{fam}-{num}" |
| 78 | + # Already hub-ish: AC-2(1) |
| 79 | + m2 = re.match(r"^([A-Z]{1,4})-(\d+)(?:\((\d+)\))?$", s, re.IGNORECASE) |
| 80 | + if not m2: |
| 81 | + return s |
| 82 | + fam, num, enh = m2.group(1).upper(), m2.group(2), m2.group(3) |
| 83 | + return f"{fam}-{int(num)}" + (f"({int(enh)})" if enh else "") |
| 84 | + |
| 85 | + |
| 86 | +def sid_from_node(node: Mapping[str, Any]) -> Optional[str]: |
| 87 | + for key in ("sectionID", "section_id"): |
| 88 | + val = node.get(key) |
| 89 | + if val: |
| 90 | + return normalize_control_id(str(val)) |
| 91 | + section = str(node.get("section") or "") |
| 92 | + m = re.match(r"^([A-Z]{1,4}-\d+(?:\([0-9]+\))?)\b", section, re.I) |
| 93 | + if m: |
| 94 | + return normalize_control_id(m.group(1)) |
| 95 | + return None |
| 96 | + |
| 97 | + |
| 98 | +def cre_ids_from_node(node: Mapping[str, Any]) -> List[str]: |
| 99 | + out: List[str] = [] |
| 100 | + for link in node.get("links") or []: |
| 101 | + if not isinstance(link, dict): |
| 102 | + continue |
| 103 | + doc = link.get("document") or {} |
| 104 | + if str(doc.get("doctype") or "").upper() != "CRE": |
| 105 | + continue |
| 106 | + cid = str(doc.get("id") or "").strip() |
| 107 | + if cid and cid not in out: |
| 108 | + out.append(cid) |
| 109 | + return out |
| 110 | + |
| 111 | + |
| 112 | +def fetch_hub_nodes(*, base_url: str = BASE_URL) -> List[Dict[str, Any]]: |
| 113 | + rows: List[Dict[str, Any]] = [] |
| 114 | + page = 1 |
| 115 | + while True: |
| 116 | + url = ( |
| 117 | + f"{base_url.rstrip('/')}/rest/v1/Standard/" |
| 118 | + f"{urllib.parse.quote(STANDARD_NAME)}?page={page}" |
| 119 | + ) |
| 120 | + data = _get_json(url) |
| 121 | + batch = data.get("standards") or [] |
| 122 | + if not batch: |
| 123 | + break |
| 124 | + rows.extend(batch) |
| 125 | + total_pages = int(data.get("total_pages") or page) |
| 126 | + if page >= total_pages: |
| 127 | + break |
| 128 | + page += 1 |
| 129 | + return rows |
| 130 | + |
| 131 | + |
| 132 | +def build_gold_rows(nodes: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]: |
| 133 | + rows: List[Dict[str, Any]] = [] |
| 134 | + seen: set[str] = set() |
| 135 | + for node in nodes: |
| 136 | + sid = sid_from_node(node) |
| 137 | + cres = cre_ids_from_node(node) |
| 138 | + if not sid or not cres: |
| 139 | + continue |
| 140 | + if sid in seen: |
| 141 | + continue |
| 142 | + seen.add(sid) |
| 143 | + rows.append( |
| 144 | + { |
| 145 | + "section_id": sid, |
| 146 | + "section": str(node.get("section") or sid), |
| 147 | + "hyperlink": str(node.get("hyperlink") or ""), |
| 148 | + "cre_ids": cres, |
| 149 | + "gold_provenance": { |
| 150 | + "kind": "hub_links", |
| 151 | + "method": "opencre_org_standard_api", |
| 152 | + "standard": STANDARD_NAME, |
| 153 | + }, |
| 154 | + } |
| 155 | + ) |
| 156 | + rows.sort(key=lambda r: r["section_id"]) |
| 157 | + return rows |
| 158 | + |
| 159 | + |
| 160 | +def stratified_sample(rows: Sequence[Dict[str, Any]], n: int) -> List[Dict[str, Any]]: |
| 161 | + """Take up to ``n`` rows, round-robin across control families.""" |
| 162 | + if n <= 0 or n >= len(rows): |
| 163 | + return list(rows) |
| 164 | + by_fam: Dict[str, List[Dict[str, Any]]] = defaultdict(list) |
| 165 | + for row in rows: |
| 166 | + fam = str(row["section_id"]).split("-", 1)[0] |
| 167 | + by_fam[fam].append(row) |
| 168 | + families = sorted(by_fam.keys(), key=lambda f: (-len(by_fam[f]), f)) |
| 169 | + out: List[Dict[str, Any]] = [] |
| 170 | + idx = {f: 0 for f in families} |
| 171 | + while len(out) < n: |
| 172 | + progressed = False |
| 173 | + for fam in families: |
| 174 | + i = idx[fam] |
| 175 | + bucket = by_fam[fam] |
| 176 | + if i >= len(bucket): |
| 177 | + continue |
| 178 | + out.append(bucket[i]) |
| 179 | + idx[fam] = i + 1 |
| 180 | + progressed = True |
| 181 | + if len(out) >= n: |
| 182 | + break |
| 183 | + if not progressed: |
| 184 | + break |
| 185 | + out.sort(key=lambda r: r["section_id"]) |
| 186 | + for row in out: |
| 187 | + prov = dict(row.get("gold_provenance") or {}) |
| 188 | + prov["sample"] = True |
| 189 | + prov["sample_size"] = n |
| 190 | + row["gold_provenance"] = prov |
| 191 | + return out |
| 192 | + |
| 193 | + |
| 194 | +def _walk_prose(parts: Sequence[Mapping[str, Any]], chunks: List[str]) -> None: |
| 195 | + for part in parts or []: |
| 196 | + if not isinstance(part, dict): |
| 197 | + continue |
| 198 | + prose = part.get("prose") |
| 199 | + if isinstance(prose, str) and prose.strip(): |
| 200 | + chunks.append(prose.strip()) |
| 201 | + nested = part.get("parts") |
| 202 | + if isinstance(nested, list): |
| 203 | + _walk_prose(nested, chunks) |
| 204 | + |
| 205 | + |
| 206 | +def load_oscal_index(path: Path) -> Dict[str, Tuple[str, str]]: |
| 207 | + """Map hub-style section id → (title, statement+guidance prose).""" |
| 208 | + catalog = json.loads(path.read_text(encoding="utf-8"))["catalog"] |
| 209 | + index: Dict[str, Tuple[str, str]] = {} |
| 210 | + |
| 211 | + def add(control: Mapping[str, Any]) -> None: |
| 212 | + label = None |
| 213 | + for prop in control.get("props") or []: |
| 214 | + if isinstance(prop, dict) and prop.get("name") == "label": |
| 215 | + label = str(prop.get("value") or "") |
| 216 | + break |
| 217 | + raw_id = label or str(control.get("id") or "") |
| 218 | + sid = normalize_control_id(raw_id) |
| 219 | + title = str(control.get("title") or sid) |
| 220 | + chunks: List[str] = [] |
| 221 | + # Prefer statement + guidance only (skip assessment noise). |
| 222 | + for part in control.get("parts") or []: |
| 223 | + if not isinstance(part, dict): |
| 224 | + continue |
| 225 | + name = str(part.get("name") or "") |
| 226 | + if name not in {"statement", "guidance"}: |
| 227 | + continue |
| 228 | + prose = part.get("prose") |
| 229 | + if isinstance(prose, str) and prose.strip(): |
| 230 | + chunks.append(prose.strip()) |
| 231 | + _walk_prose(part.get("parts") or [], chunks) |
| 232 | + text = "\n".join(chunks).strip() |
| 233 | + # Strip OSCAL parameter placeholders for cleaner retrieval. |
| 234 | + text = re.sub(r"\{\{\s*insert:[^}]+\}\}", "…", text) |
| 235 | + if sid and text: |
| 236 | + index[sid] = (title, text) |
| 237 | + for sub in control.get("controls") or []: |
| 238 | + if isinstance(sub, dict): |
| 239 | + add(sub) |
| 240 | + |
| 241 | + for group in catalog.get("groups") or []: |
| 242 | + for control in group.get("controls") or []: |
| 243 | + if isinstance(control, dict): |
| 244 | + add(control) |
| 245 | + return index |
| 246 | + |
| 247 | + |
| 248 | +def ensure_oscal(cache: Path = OSCAL_CACHE) -> Path: |
| 249 | + cache.parent.mkdir(parents=True, exist_ok=True) |
| 250 | + if cache.is_file() and cache.stat().st_size > 1_000_000: |
| 251 | + return cache |
| 252 | + print(f"GET {OSCAL_URL}", flush=True) |
| 253 | + urllib.request.urlretrieve(OSCAL_URL, cache) |
| 254 | + return cache |
| 255 | + |
| 256 | + |
| 257 | +def write_sources( |
| 258 | + rows: Sequence[Mapping[str, Any]], |
| 259 | + oscal: Mapping[str, Tuple[str, str]], |
| 260 | + dest: Path, |
| 261 | +) -> Dict[str, int]: |
| 262 | + dest.mkdir(parents=True, exist_ok=True) |
| 263 | + written = 0 |
| 264 | + missing = 0 |
| 265 | + for row in rows: |
| 266 | + sid = str(row["section_id"]) |
| 267 | + title, body = oscal.get(sid, ("", "")) |
| 268 | + if not body: |
| 269 | + missing += 1 |
| 270 | + continue |
| 271 | + section = str(row.get("section") or title or sid) |
| 272 | + text = ( |
| 273 | + f"Source: {sid}\n" |
| 274 | + f"Section-ID: {sid}\n" |
| 275 | + f"Section: {section}\n" |
| 276 | + f"Standard: {STANDARD_NAME}\n\n" |
| 277 | + f"{title}\n\n{body}\n" |
| 278 | + ) |
| 279 | + # Match B2 loader: b2_sources/<stem>/<section_id>.txt |
| 280 | + (dest / f"{sid}.txt").write_text(text, encoding="utf-8") |
| 281 | + written += 1 |
| 282 | + return {"written": written, "missing_oscal": missing} |
| 283 | + |
| 284 | + |
| 285 | +def main(argv: Optional[Sequence[str]] = None) -> int: |
| 286 | + parser = argparse.ArgumentParser(description=__doc__) |
| 287 | + parser.add_argument("--out-gold", type=Path, default=DEFAULT_OUT_GOLD) |
| 288 | + parser.add_argument("--out-sources", type=Path, default=DEFAULT_SOURCES) |
| 289 | + parser.add_argument( |
| 290 | + "--sample", |
| 291 | + type=int, |
| 292 | + default=0, |
| 293 | + help="If >0, stratified sample of N controls (default: all hub-linked).", |
| 294 | + ) |
| 295 | + parser.add_argument("--base-url", default=BASE_URL) |
| 296 | + parser.add_argument("--skip-sources", action="store_true") |
| 297 | + args = parser.parse_args(list(argv) if argv is not None else None) |
| 298 | + |
| 299 | + print(f"fetching {STANDARD_NAME} from {args.base_url} …", flush=True) |
| 300 | + nodes = fetch_hub_nodes(base_url=args.base_url) |
| 301 | + gold = build_gold_rows(nodes) |
| 302 | + print(f"hub nodes={len(nodes)} gold_rows={len(gold)}", flush=True) |
| 303 | + if args.sample > 0: |
| 304 | + gold = stratified_sample(gold, args.sample) |
| 305 | + print(f"sampled to {len(gold)}", flush=True) |
| 306 | + |
| 307 | + args.out_gold.parent.mkdir(parents=True, exist_ok=True) |
| 308 | + args.out_gold.write_text(json.dumps(gold, indent=2) + "\n", encoding="utf-8") |
| 309 | + print(f"wrote {args.out_gold}", flush=True) |
| 310 | + |
| 311 | + if not args.skip_sources: |
| 312 | + oscal_path = ensure_oscal() |
| 313 | + index = load_oscal_index(oscal_path) |
| 314 | + print(f"oscal controls indexed={len(index)}", flush=True) |
| 315 | + stats = write_sources(gold, index, args.out_sources) |
| 316 | + print(json.dumps({"sources_dir": str(args.out_sources), **stats}, indent=2)) |
| 317 | + if stats["missing_oscal"]: |
| 318 | + print( |
| 319 | + f"warning: {stats['missing_oscal']} gold ids missing from OSCAL", |
| 320 | + file=sys.stderr, |
| 321 | + ) |
| 322 | + return 0 |
| 323 | + |
| 324 | + |
| 325 | +if __name__ == "__main__": |
| 326 | + raise SystemExit(main()) |
0 commit comments