diff --git a/tests/vulnerability_db/test_searchability.py b/tests/vulnerability_db/test_searchability.py new file mode 100644 index 00000000..e875d9cf --- /dev/null +++ b/tests/vulnerability_db/test_searchability.py @@ -0,0 +1,231 @@ +""" +Unit tests + fixtures for data/ + schemas/ searchability and indexing (issue #666). + +Verifies that every vulnerability entry is structured so downstream consumers +can reliably build search indexes, faceted filters, and full-text queries. +""" +import json +import pathlib +import re +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DATABASES = [ + ROOT / "data" / "vulnerability-db.json", + ROOT / "tooling" / "sanctifier-cli" / "data" / "vulnerability-db.json", +] +SCHEMA_PATH = ROOT / "schemas" / "vulnerability-db.json" + +SEVERITY_TAXONOMY_PATH = ROOT / "schemas" / "severity-taxonomy.schema.json" + +# Expected severity values for faceted-search filtering +VALID_SEVERITIES = {"critical", "high", "medium", "low", "informational"} + +# ID format used as the primary index key. +# Accepts both SOL-YYYY-NNN and shorter VULN-NNN conventions. +ID_PATTERN = re.compile(r"^[A-Z]+-\d{3,}(-\d+)?$") + +# Fixture: minimal valid entry shape used by index-consumer tests +VALID_ENTRY_FIXTURE: dict = { + "id": "SOL-2024-001", + "name": "Test Entry", + "description": "A test vulnerability entry.", + "severity": "high", + "category": "access-control", + "pattern": r"fn\s+init\s*\(", + "recommendation": "Add require_auth().", + "references": [], +} + + +def _load_db(path: pathlib.Path) -> list[dict]: + return json.loads(path.read_text(encoding="utf-8"))["vulnerabilities"] + + +class IdIndexabilityTests(unittest.TestCase): + """Primary key index: IDs must be unique, non-empty, and follow a stable format.""" + + def test_ids_are_non_empty(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + entries = _load_db(db_path) + for entry in entries: + self.assertTrue(entry["id"].strip(), f"Empty id in {db_path.name}") + + def test_ids_are_unique_within_each_database(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + ids = [e["id"] for e in _load_db(db_path)] + self.assertEqual(len(ids), len(set(ids)), f"Duplicate IDs in {db_path.name}") + + def test_ids_follow_indexable_format(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertRegex( + entry["id"], + ID_PATTERN, + f"ID '{entry['id']}' does not match indexable format PREFIX-YYYY-NNN", + ) + + def test_ids_are_lexicographically_sortable(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + ids = [e["id"] for e in _load_db(db_path)] + self.assertEqual(ids, sorted(ids), f"IDs in {db_path.name} are not sorted") + + +class CategoryFacetTests(unittest.TestCase): + """Faceted-search index on category field.""" + + def test_category_field_present_and_non_empty(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertTrue( + entry.get("category", "").strip(), + f"Empty category in entry {entry['id']}", + ) + + def test_each_category_has_at_least_one_entry(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + entries = _load_db(db_path) + by_category: dict[str, list] = {} + for entry in entries: + by_category.setdefault(entry["category"], []).append(entry["id"]) + for cat, ids in by_category.items(): + self.assertGreater(len(ids), 0, f"Category '{cat}' has no entries") + + def test_categories_use_kebab_case(self) -> None: + kebab = re.compile(r"^[a-z][a-z0-9-]*$") + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertRegex( + entry["category"], + kebab, + f"Category '{entry['category']}' in {entry['id']} is not kebab-case", + ) + + +class SeverityFacetTests(unittest.TestCase): + """Faceted-search index on severity field.""" + + def test_severity_values_are_valid(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertIn( + entry["severity"], + VALID_SEVERITIES, + f"Invalid severity '{entry['severity']}' in {entry['id']}", + ) + + def test_severity_field_is_lowercase(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertEqual( + entry["severity"], + entry["severity"].lower(), + f"Severity not lowercase in {entry['id']}", + ) + + +class FullTextSearchFieldTests(unittest.TestCase): + """Fields used for full-text search must be present and non-empty.""" + + def test_name_is_searchable(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertTrue( + entry.get("name", "").strip(), + f"Empty name field in {entry['id']}", + ) + + def test_description_is_searchable(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertTrue( + entry.get("description", "").strip(), + f"Empty description in {entry['id']}", + ) + + def test_recommendation_is_searchable(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertTrue( + entry.get("recommendation", "").strip(), + f"Empty recommendation in {entry['id']}", + ) + + +class PatternIndexabilityTests(unittest.TestCase): + """Pattern field must be valid regex for search/match indexing.""" + + def test_all_patterns_are_valid_regex(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + try: + re.compile(entry["pattern"]) + except re.error as exc: + self.fail(f"Invalid regex in {entry['id']}: {exc}") + + def test_patterns_are_non_empty(self) -> None: + for db_path in DATABASES: + with self.subTest(db=db_path.name): + for entry in _load_db(db_path): + self.assertTrue( + entry.get("pattern", "").strip(), + f"Empty pattern in {entry['id']}", + ) + + +class SchemaIndexabilityTests(unittest.TestCase): + """The JSON schema itself must declare fields needed for indexing.""" + + def test_schema_declares_id_as_required(self) -> None: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + item_required = schema["properties"]["vulnerabilities"]["items"]["required"] + self.assertIn("id", item_required) + + def test_schema_declares_category_as_required(self) -> None: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + item_required = schema["properties"]["vulnerabilities"]["items"]["required"] + self.assertIn("category", item_required) + + def test_schema_declares_severity_as_required(self) -> None: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + item_required = schema["properties"]["vulnerabilities"]["items"]["required"] + self.assertIn("severity", item_required) + + +class FixtureShapeTests(unittest.TestCase): + """Validate that the canonical fixture entry matches the schema's required fields.""" + + def test_fixture_has_all_required_fields(self) -> None: + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + required_fields = schema["properties"]["vulnerabilities"]["items"]["required"] + for field in required_fields: + self.assertIn(field, VALID_ENTRY_FIXTURE, f"Fixture missing required field '{field}'") + + def test_fixture_id_matches_indexable_format(self) -> None: + self.assertRegex(VALID_ENTRY_FIXTURE["id"], ID_PATTERN) + + def test_fixture_severity_is_valid(self) -> None: + self.assertIn(VALID_ENTRY_FIXTURE["severity"], VALID_SEVERITIES) + + def test_fixture_pattern_is_valid_regex(self) -> None: + try: + re.compile(VALID_ENTRY_FIXTURE["pattern"]) + except re.error as exc: + self.fail(f"Fixture pattern is not valid regex: {exc}") + + +if __name__ == "__main__": + unittest.main() diff --git a/vscode-extension/package.json b/vscode-extension/package.json index 11aab2a3..2e6f37ea 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -81,6 +81,7 @@ "vscode:prepublish": "npm run compile", "compile": "tsc -p ./", "watch": "tsc -watch -p ./", + "test": "node --test out/analyzer.test.js", "test": "npm run compile && node --test out/test/analyzer.test.js", "lint": "tsc --noEmit" }, diff --git a/vscode-extension/src/analyzer.test.ts b/vscode-extension/src/analyzer.test.ts new file mode 100644 index 00000000..0d3a4e35 --- /dev/null +++ b/vscode-extension/src/analyzer.test.ts @@ -0,0 +1,296 @@ +import { strict as assert } from 'node:assert'; +import { describe, it } from 'node:test'; +import { analyzeSorobanSource, looksLikeSorobanSource, CODES } from './analyzer'; + +// --------------------------------------------------------------------------- +// looksLikeSorobanSource +// --------------------------------------------------------------------------- + +describe('looksLikeSorobanSource', () => { + it('returns true for #[contractimpl]', () => { + assert.equal(looksLikeSorobanSource('#[contractimpl]\nimpl Foo {}'), true); + }); + + it('returns true for soroban_sdk reference', () => { + assert.equal(looksLikeSorobanSource('use soroban_sdk::Env;'), true); + }); + + it('returns true for #[contract]', () => { + assert.equal(looksLikeSorobanSource('#[contract]\npub struct Counter;'), true); + }); + + it('returns true for contractimpl keyword', () => { + assert.equal(looksLikeSorobanSource('contractimpl'), true); + }); + + it('returns false for plain Rust with no Soroban markers', () => { + assert.equal(looksLikeSorobanSource('fn main() { println!("hello"); }'), false); + }); + + it('returns false for empty string', () => { + assert.equal(looksLikeSorobanSource(''), false); + }); +}); + +// --------------------------------------------------------------------------- +// Auth-gap detection +// --------------------------------------------------------------------------- + +const AUTH_GAP_SRC = ` +#[contractimpl] +impl MyContract { + pub fn withdraw(env: Env, amount: i128) { + env.storage().persistent().set(&DataKey::Balance, &amount); + } +} +`; + +const AUTH_OK_SRC = ` +#[contractimpl] +impl MyContract { + pub fn withdraw(env: Env, user: Address, amount: i128) { + user.require_auth(); + env.storage().persistent().set(&DataKey::Balance, &amount); + } +} +`; + +const AUTH_FOR_ARGS_SRC = ` +#[contractimpl] +impl MyContract { + pub fn transfer(env: Env, from: Address, to: Address, amount: i128) { + from.require_auth_for_args(()); + env.storage().persistent().set(&DataKey::Balance, &amount); + } +} +`; + +const CROSS_CONTRACT_NO_AUTH = ` +#[contractimpl] +impl Proxy { + pub fn call_other(env: Env, contract: Address) { + env.invoke_contract::<()>(&contract, &Symbol::short("do_it"), vec![&env]); + } +} +`; + +describe('analyzeSorobanSource – auth gaps', () => { + it('flags pub fn with storage mutation and no require_auth', () => { + const findings = analyzeSorobanSource(AUTH_GAP_SRC); + const gaps = findings.filter((f) => f.code === CODES.AUTH_GAP); + assert.equal(gaps.length, 1); + assert.match(gaps[0].message, /withdraw/); + assert.equal(gaps[0].severity, 'warning'); + }); + + it('does not flag when require_auth is present', () => { + const gaps = analyzeSorobanSource(AUTH_OK_SRC).filter((f) => f.code === CODES.AUTH_GAP); + assert.equal(gaps.length, 0); + }); + + it('does not flag when require_auth_for_args is present', () => { + const gaps = analyzeSorobanSource(AUTH_FOR_ARGS_SRC).filter((f) => f.code === CODES.AUTH_GAP); + assert.equal(gaps.length, 0); + }); + + it('flags cross-contract invoke without auth', () => { + const gaps = analyzeSorobanSource(CROSS_CONTRACT_NO_AUTH).filter((f) => f.code === CODES.AUTH_GAP); + assert.equal(gaps.length, 1); + }); + + it('returns the correct 1-based line number for the flagged function', () => { + const findings = analyzeSorobanSource(AUTH_GAP_SRC).filter((f) => f.code === CODES.AUTH_GAP); + assert.ok(findings[0].line >= 1, 'line must be >= 1'); + }); +}); + +// --------------------------------------------------------------------------- +// Panic / unwrap / expect detection (S002, S006) +// --------------------------------------------------------------------------- + +describe('analyzeSorobanSource – panic patterns', () => { + it('flags panic! macro', () => { + const src = `fn foo() { panic!("boom"); }`; + const findings = analyzeSorobanSource(src); + assert.ok(findings.some((f) => f.code === CODES.PANIC_USAGE)); + }); + + it('flags .unwrap()', () => { + const src = `fn foo(x: Option) -> i32 { x.unwrap() }`; + assert.ok(analyzeSorobanSource(src).some((f) => f.code === CODES.UNSAFE_PATTERN)); + }); + + it('flags .expect("msg")', () => { + const src = `fn foo(x: Option) -> i32 { x.expect("never none") }`; + assert.ok(analyzeSorobanSource(src).some((f) => f.code === CODES.UNSAFE_PATTERN)); + }); + + it('does not flag commented-out panic!', () => { + const src = `fn foo() { // panic!("suppressed"); }`; + assert.equal( + analyzeSorobanSource(src).filter((f) => f.code === CODES.PANIC_USAGE).length, + 0, + ); + }); + + it('panic! finding has severity "error"', () => { + const src = `fn foo() { panic!(""); }`; + const f = analyzeSorobanSource(src).find((f) => f.code === CODES.PANIC_USAGE); + assert.ok(f); + assert.equal(f.severity, 'error'); + }); + + it('.unwrap() finding has severity "warning"', () => { + const src = `fn foo(x: Option<()>) { x.unwrap(); }`; + const f = analyzeSorobanSource(src).find((f) => f.code === CODES.UNSAFE_PATTERN); + assert.ok(f); + assert.equal(f.severity, 'warning'); + }); +}); + +// --------------------------------------------------------------------------- +// Arithmetic overflow detection (S003) +// --------------------------------------------------------------------------- + +const OVERFLOW_SRC = ` +#[contractimpl] +impl Counter { + pub fn add(env: Env, a: i128, b: i128) -> i128 { + a + b + } +} +`; + +const CHECKED_ADD_SRC = ` +#[contractimpl] +impl Counter { + pub fn add(env: Env, a: i128, b: i128) -> i128 { + a.checked_add(b).unwrap_or(0) + } +} +`; + +const SATURATING_SRC = ` +#[contractimpl] +impl Counter { + pub fn add(env: Env, a: i128, b: i128) -> i128 { + a.saturating_add(b) + } +} +`; + +describe('analyzeSorobanSource – arithmetic overflow', () => { + it('flags unchecked + inside contractimpl', () => { + assert.ok( + analyzeSorobanSource(OVERFLOW_SRC).some((f) => f.code === CODES.ARITHMETIC_OVERFLOW), + ); + }); + + it('does not flag checked_add', () => { + assert.equal( + analyzeSorobanSource(CHECKED_ADD_SRC).filter((f) => f.code === CODES.ARITHMETIC_OVERFLOW) + .length, + 0, + ); + }); + + it('does not flag saturating_add', () => { + assert.equal( + analyzeSorobanSource(SATURATING_SRC).filter((f) => f.code === CODES.ARITHMETIC_OVERFLOW) + .length, + 0, + ); + }); + + it('does not flag arithmetic outside contractimpl', () => { + const src = `fn helper(a: i128, b: i128) -> i128 { a + b }`; + assert.equal( + analyzeSorobanSource(src).filter((f) => f.code === CODES.ARITHMETIC_OVERFLOW).length, + 0, + ); + }); + + it('arithmetic finding has severity "warning"', () => { + const f = analyzeSorobanSource(OVERFLOW_SRC).find((f) => f.code === CODES.ARITHMETIC_OVERFLOW); + assert.ok(f); + assert.equal(f.severity, 'warning'); + }); +}); + +// --------------------------------------------------------------------------- +// Deduplication +// --------------------------------------------------------------------------- + +describe('analyzeSorobanSource – deduplication', () => { + it('deduplicates identical (line, code, message-prefix) findings', () => { + const src = `fn foo() {\n panic!("dup");\n panic!("dup");\n}`; + const findings = analyzeSorobanSource(src); + const panics = findings.filter((f) => f.code === CODES.PANIC_USAGE); + const keys = panics.map((f) => `${f.line}:${f.code}:${f.message.slice(0, 40)}`); + assert.equal(keys.length, new Set(keys).size, 'duplicate findings present'); + }); +}); + +// --------------------------------------------------------------------------- +// Clean contract produces no findings +// --------------------------------------------------------------------------- + +const CLEAN_SRC = ` +use soroban_sdk::{contract, contractimpl, Env, Address}; + +#[contract] +pub struct CleanToken; + +#[contractimpl] +impl CleanToken { + pub fn transfer(env: Env, from: Address, to: Address, amount: i128) { + from.require_auth(); + let from_balance: i128 = env.storage().persistent().get(&from).unwrap_or(0); + let to_balance: i128 = env.storage().persistent().get(&to).unwrap_or(0); + env.storage().persistent().set(&from, &(from_balance.checked_sub(amount).unwrap_or(0))); + env.storage().persistent().set(&to, &(to_balance.checked_add(amount).unwrap_or(0))); + } +} +`; + +describe('analyzeSorobanSource – clean contract', () => { + it('produces no auth-gap or arithmetic findings on well-written code', () => { + const findings = analyzeSorobanSource(CLEAN_SRC); + assert.equal(findings.filter((f) => f.code === CODES.AUTH_GAP).length, 0); + assert.equal(findings.filter((f) => f.code === CODES.ARITHMETIC_OVERFLOW).length, 0); + assert.equal(findings.filter((f) => f.code === CODES.PANIC_USAGE).length, 0); + }); +}); + +// --------------------------------------------------------------------------- +// Performance budget (#618) +// --------------------------------------------------------------------------- + +describe('analyzeSorobanSource – performance budget', () => { + it('analyzes a 500-line contract in under 100ms', () => { + const fns = Array.from( + { length: 90 }, + (_, i) => + ` pub fn fn_${i}(env: Env, user: Address, val: i128) -> i128 {\n` + + ` user.require_auth();\n` + + ` val.checked_add(1).unwrap_or(0)\n` + + ` }`, + ).join('\n'); + const src = `#[contractimpl]\nimpl BigContract {\n${fns}\n}`; + + const start = performance.now(); + analyzeSorobanSource(src); + const elapsed = performance.now() - start; + + assert.ok(elapsed < 100, `analysis took ${elapsed.toFixed(1)}ms, budget is 100ms`); + }); + + it('handles empty input without throwing', () => { + assert.doesNotThrow(() => analyzeSorobanSource('')); + }); + + it('handles very long single line without throwing', () => { + const src = `fn foo() { ${'x'.repeat(10_000)} }`; + assert.doesNotThrow(() => analyzeSorobanSource(src)); + }); +}); diff --git a/vscode-extension/src/extension.ts b/vscode-extension/src/extension.ts index 87ead770..3818fa61 100644 --- a/vscode-extension/src/extension.ts +++ b/vscode-extension/src/extension.ts @@ -1,4 +1,6 @@ import * as vscode from 'vscode'; +import * as fs from 'fs'; +import { analyzeSorobanSource, looksLikeSorobanSource, type EditorFinding } from './analyzer'; import { analyzeSorobanSource, looksLikeSorobanSource, filterBySeverity, type Severity } from './analyzer'; import { type EditorFinding, type SanctifierExtensionApi } from './types'; import { folderLooksLikeSorobanProject, invalidateWorkspaceCache } from './workspace'; @@ -45,6 +47,20 @@ function findingToDiagnostic(doc: vscode.TextDocument, f: EditorFinding): vscode return d; } +function validateSanctifierPath(exePath: string): void { + const trimmed = exePath.trim(); + if (!trimmed) { + return; + } + if (!fs.existsSync(trimmed)) { + vscode.window.showWarningMessage( + `Sanctifier: sanctifierPath "${trimmed}" was not found on disk. ` + + 'Update sanctifier.sanctifierPath to a valid CLI binary path.', + ); + } +} + +export async function activate(context: vscode.ExtensionContext): Promise { export async function activate(context: vscode.ExtensionContext): Promise { const collection = vscode.languages.createDiagnosticCollection(SOURCE); const outputChannel = vscode.window.createOutputChannel('Sanctifier'); @@ -74,6 +90,12 @@ export async function activate(context: vscode.ExtensionContext): Promise>(); const runAnalysis = (doc: vscode.TextDocument) => { @@ -105,6 +127,14 @@ export async function activate(context: vscode.ExtensionContext): Promise findingToDiagnostic(doc, f)); + collection.set(doc.uri, diags); + statusBar.text = + diags.length > 0 + ? `$(shield) Sanctifier (${diags.length} hint${diags.length === 1 ? '' : 's'})` + : '$(shield) Sanctifier'; findingsCache.set(doc.uri.toString(), findings); const diags = findings.map((f) => findingToDiagnostic(doc, f)); collection.set(doc.uri, diags); @@ -122,7 +152,7 @@ export async function activate(context: vscode.ExtensionContext): Promise('debounceMs') ?? 400; + const ms = Math.min(5000, Math.max(100, getConfig().get('debounceMs') ?? 400)); const key = doc.uri.toString(); const prev = debouncers.get(key); if (prev) { @@ -175,6 +205,10 @@ export async function activate(context: vscode.ExtensionContext): Promise { if (e.affectsConfiguration('sanctifier')) { + sorobanWorkspaceCache = null; + if (e.affectsConfiguration('sanctifier.sanctifierPath')) { + validateSanctifierPath(getConfig().get('sanctifierPath') ?? ''); + } invalidateWorkspaceCache(); contentCache.clear(); findingsCache.clear(); @@ -213,7 +247,14 @@ export async function activate(context: vscode.ExtensionContext): Promise('sanctifierPath')?.trim(); if (!exe) { vscode.window.showWarningMessage( - 'Set sanctifier.sanctifierPath to your sanctifier CLI binary, then run again.' + 'Set sanctifier.sanctifierPath to your sanctifier CLI binary, then run again.', + ); + return; + } + if (!fs.existsSync(exe)) { + vscode.window.showErrorMessage( + `Sanctifier: binary not found at "${exe}". ` + + 'Update sanctifier.sanctifierPath to a valid path and try again.', ); return; } @@ -222,6 +263,51 @@ export async function activate(context: vscode.ExtensionContext): Promise( + (resolve) => { + const p = spawn(exe, ['analyze', folder.uri.fsPath, '--format', 'json'], { + cwd: folder.uri.fsPath, + }); + let out = ''; + let err = ''; + p.stdout.on('data', (b: Buffer) => (out += b.toString())); + p.stderr.on('data', (b: Buffer) => (err += b.toString())); + p.on('close', () => resolve({ output: out || undefined, stderr: err })); + p.on('error', () => resolve({ output: undefined, stderr: '' })); + }, + ); + statusBar.text = '$(shield) Sanctifier'; + if (!output) { + const isWasmFailure = + /wasm32|wasm-unknown|target.*wasm|error\[E/i.test(stderr); + if (isWasmFailure) { + const choice = await vscode.window.showErrorMessage( + 'Sanctifier: WASM compilation failed. ' + + 'Ensure the wasm32 target is installed: `rustup target add wasm32-unknown-unknown`.', + 'Show Error Output', + ); + if (choice === 'Show Error Output') { + const errDoc = await vscode.workspace.openTextDocument({ + content: stderr, + language: 'text', + }); + await vscode.window.showTextDocument(errDoc, { preview: true }); + } + } else { + vscode.window.showErrorMessage( + 'Sanctifier CLI failed or produced no output. ' + + 'Check sanctifier.sanctifierPath and ensure the binary is executable.', + ); + } + return; + } + const token = output; + const doc = await vscode.workspace.openTextDocument({ + content: token, + language: 'json', + }); + await vscode.window.showTextDocument(doc, { preview: true }); // Remote workspaces (SSH, Codespaces, WSL) use non-file URIs; the CLI // must run on the same machine as the source files.