feat(vscode+data): unit tests, perf budgets, input validation, searchability tests

tests/vulnerability_db/test_searchability.py

@@ -0,0 +1,231 @@
+"""
+Unit tests + fixtures for data/ + schemas/ searchability and indexing (issue #666).
+
+Verifies that every vulnerability entry is structured so downstream consumers
+can reliably build search indexes, faceted filters, and full-text queries.
+"""
+import json
+import pathlib
+import re
+import unittest
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+DATABASES = [
+    ROOT / "data" / "vulnerability-db.json",
+    ROOT / "tooling" / "sanctifier-cli" / "data" / "vulnerability-db.json",
+]
+SCHEMA_PATH = ROOT / "schemas" / "vulnerability-db.json"
+
+SEVERITY_TAXONOMY_PATH = ROOT / "schemas" / "severity-taxonomy.schema.json"
+
+# Expected severity values for faceted-search filtering
+VALID_SEVERITIES = {"critical", "high", "medium", "low", "informational"}
+
+# ID format used as the primary index key.
+# Accepts both SOL-YYYY-NNN and shorter VULN-NNN conventions.
+ID_PATTERN = re.compile(r"^[A-Z]+-\d{3,}(-\d+)?$")
+
+# Fixture: minimal valid entry shape used by index-consumer tests
+VALID_ENTRY_FIXTURE: dict = {
+    "id": "SOL-2024-001",
+    "name": "Test Entry",
+    "description": "A test vulnerability entry.",
+    "severity": "high",
+    "category": "access-control",
+    "pattern": r"fn\s+init\s*\(",
+    "recommendation": "Add require_auth().",
+    "references": [],
+}
+
+
+def _load_db(path: pathlib.Path) -> list[dict]:
+    return json.loads(path.read_text(encoding="utf-8"))["vulnerabilities"]
+
+
+class IdIndexabilityTests(unittest.TestCase):
+    """Primary key index: IDs must be unique, non-empty, and follow a stable format."""
+
+    def test_ids_are_non_empty(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                entries = _load_db(db_path)
+                for entry in entries:
+                    self.assertTrue(entry["id"].strip(), f"Empty id in {db_path.name}")
+
+    def test_ids_are_unique_within_each_database(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                ids = [e["id"] for e in _load_db(db_path)]
+                self.assertEqual(len(ids), len(set(ids)), f"Duplicate IDs in {db_path.name}")
+
+    def test_ids_follow_indexable_format(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertRegex(
+                        entry["id"],
+                        ID_PATTERN,
+                        f"ID '{entry['id']}' does not match indexable format PREFIX-YYYY-NNN",
+                    )
+
+    def test_ids_are_lexicographically_sortable(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                ids = [e["id"] for e in _load_db(db_path)]
+                self.assertEqual(ids, sorted(ids), f"IDs in {db_path.name} are not sorted")
+
+
+class CategoryFacetTests(unittest.TestCase):
+    """Faceted-search index on category field."""
+
+    def test_category_field_present_and_non_empty(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertTrue(
+                        entry.get("category", "").strip(),
+                        f"Empty category in entry {entry['id']}",
+                    )
+
+    def test_each_category_has_at_least_one_entry(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                entries = _load_db(db_path)
+                by_category: dict[str, list] = {}
+                for entry in entries:
+                    by_category.setdefault(entry["category"], []).append(entry["id"])
+                for cat, ids in by_category.items():
+                    self.assertGreater(len(ids), 0, f"Category '{cat}' has no entries")
+
+    def test_categories_use_kebab_case(self) -> None:
+        kebab = re.compile(r"^[a-z][a-z0-9-]*$")
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertRegex(
+                        entry["category"],
+                        kebab,
+                        f"Category '{entry['category']}' in {entry['id']} is not kebab-case",
+                    )
+
+
+class SeverityFacetTests(unittest.TestCase):
+    """Faceted-search index on severity field."""
+
+    def test_severity_values_are_valid(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertIn(
+                        entry["severity"],
+                        VALID_SEVERITIES,
+                        f"Invalid severity '{entry['severity']}' in {entry['id']}",
+                    )
+
+    def test_severity_field_is_lowercase(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertEqual(
+                        entry["severity"],
+                        entry["severity"].lower(),
+                        f"Severity not lowercase in {entry['id']}",
+                    )
+
+
+class FullTextSearchFieldTests(unittest.TestCase):
+    """Fields used for full-text search must be present and non-empty."""
+
+    def test_name_is_searchable(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertTrue(
+                        entry.get("name", "").strip(),
+                        f"Empty name field in {entry['id']}",
+                    )
+
+    def test_description_is_searchable(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertTrue(
+                        entry.get("description", "").strip(),
+                        f"Empty description in {entry['id']}",
+                    )
+
+    def test_recommendation_is_searchable(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertTrue(
+                        entry.get("recommendation", "").strip(),
+                        f"Empty recommendation in {entry['id']}",
+                    )
+
+
+class PatternIndexabilityTests(unittest.TestCase):
+    """Pattern field must be valid regex for search/match indexing."""
+
+    def test_all_patterns_are_valid_regex(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    try:
+                        re.compile(entry["pattern"])
+                    except re.error as exc:
+                        self.fail(f"Invalid regex in {entry['id']}: {exc}")
+
+    def test_patterns_are_non_empty(self) -> None:
+        for db_path in DATABASES:
+            with self.subTest(db=db_path.name):
+                for entry in _load_db(db_path):
+                    self.assertTrue(
+                        entry.get("pattern", "").strip(),
+                        f"Empty pattern in {entry['id']}",
+                    )
+
+
+class SchemaIndexabilityTests(unittest.TestCase):
+    """The JSON schema itself must declare fields needed for indexing."""
+
+    def test_schema_declares_id_as_required(self) -> None:
+        schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+        item_required = schema["properties"]["vulnerabilities"]["items"]["required"]
+        self.assertIn("id", item_required)
+
+    def test_schema_declares_category_as_required(self) -> None:
+        schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+        item_required = schema["properties"]["vulnerabilities"]["items"]["required"]
+        self.assertIn("category", item_required)
+
+    def test_schema_declares_severity_as_required(self) -> None:
+        schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+        item_required = schema["properties"]["vulnerabilities"]["items"]["required"]
+        self.assertIn("severity", item_required)
+
+
+class FixtureShapeTests(unittest.TestCase):
+    """Validate that the canonical fixture entry matches the schema's required fields."""
+
+    def test_fixture_has_all_required_fields(self) -> None:
+        schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+        required_fields = schema["properties"]["vulnerabilities"]["items"]["required"]
+        for field in required_fields:
+            self.assertIn(field, VALID_ENTRY_FIXTURE, f"Fixture missing required field '{field}'")
+
+    def test_fixture_id_matches_indexable_format(self) -> None:
+        self.assertRegex(VALID_ENTRY_FIXTURE["id"], ID_PATTERN)
+
+    def test_fixture_severity_is_valid(self) -> None:
+        self.assertIn(VALID_ENTRY_FIXTURE["severity"], VALID_SEVERITIES)
+
+    def test_fixture_pattern_is_valid_regex(self) -> None:
+        try:
+            re.compile(VALID_ENTRY_FIXTURE["pattern"])
+        except re.error as exc:
+            self.fail(f"Fixture pattern is not valid regex: {exc}")
+
+
+if __name__ == "__main__":
+    unittest.main()

vscode-extension/package.json

@@ -81,6 +81,7 @@
     "vscode:prepublish": "npm run compile",
     "compile": "tsc -p ./",
     "watch": "tsc -watch -p ./",
+    "test": "node --test out/analyzer.test.js",
     "test": "npm run compile && node --test out/test/analyzer.test.js",
     "lint": "tsc --noEmit"
   },