diff --git a/README.md b/README.md index a17d864..024f1d2 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,10 @@ docksec -i myapp:latest --image-only --json # Write a SARIF report for GitHub Code Scanning docksec Dockerfile --scan-only --sarif +# Save today's findings as a baseline, then only gate on new findings later +docksec -i myapp:latest --image-only --baseline .docksec-baseline.json --update-baseline +docksec -i myapp:latest --image-only --baseline .docksec-baseline.json --fail-on high + # Reduce output to warnings, errors, and the result summary docksec Dockerfile --scan-only --quiet @@ -171,6 +175,26 @@ directly on pull requests and in the Security tab: which report formats you've selected, since it targets CI/Code Scanning rather than local reading. +### Baseline / ratchet mode + +`--baseline FILE` lets you adopt `--fail-on` on an existing project without a wall of +pre-existing findings blocking every build. Run once with `--update-baseline` to snapshot +today's findings, then commit the baseline file; from then on, `--fail-on` only gates on +findings that aren't already in the baseline: + +```bash +# Snapshot current findings (does not gate) +docksec -i myapp:latest --image-only --baseline .docksec-baseline.json --update-baseline + +# Later runs only fail on NEW findings above the threshold +docksec -i myapp:latest --image-only --baseline .docksec-baseline.json --fail-on high +``` + +Findings are matched by vulnerability ID, target, and package name, so the baseline stays +valid as unrelated findings come and go. Re-run with `--update-baseline` whenever you want +to accept the current state as the new baseline (e.g. after triaging and deciding to defer +a finding). + ### Exit codes DockSec uses CI-friendly exit codes so builds and shells can react to results: diff --git a/docksec/baseline.py b/docksec/baseline.py new file mode 100644 index 0000000..3efd45d --- /dev/null +++ b/docksec/baseline.py @@ -0,0 +1,45 @@ +"""Baseline/ratchet mode: compare a scan's findings against a stored baseline +so --fail-on only gates on newly introduced findings. + +The baseline file is a flat JSON list of finding fingerprints. A fingerprint +identifies a finding by VulnerabilityID + Target + PkgName, which is stable +across scans of the same Dockerfile/image even as unrelated findings appear +or disappear. +""" + +import json +from typing import Dict, List + + +def fingerprint(vuln: Dict) -> str: + """Build a stable identity for a finding from its VulnerabilityID, Target, + and PkgName, so the same underlying issue matches across scans.""" + vuln_id = str(vuln.get("VulnerabilityID") or "UNKNOWN") + target = str(vuln.get("Target") or "") + pkg = str(vuln.get("PkgName") or "") + return f"{vuln_id}|{target}|{pkg}" + + +def load_baseline(path: str) -> List[str]: + """Load a baseline file's fingerprints. Returns [] if the file doesn't exist.""" + try: + with open(path, "r") as f: + data = json.load(f) + except FileNotFoundError: + return [] + return list(data.get("fingerprints", [])) + + +def save_baseline(path: str, results: Dict) -> None: + """Write the current scan's findings to the baseline file as fingerprints.""" + vulnerabilities = results.get("json_data", []) + fingerprints = sorted({fingerprint(v) for v in vulnerabilities}) + with open(path, "w") as f: + json.dump({"fingerprints": fingerprints}, f, indent=2) + + +def new_findings(results: Dict, baseline_fingerprints: List[str]) -> List[Dict]: + """Return the findings in results that are not present in the baseline.""" + baseline_set = set(baseline_fingerprints) + vulnerabilities = results.get("json_data", []) + return [v for v in vulnerabilities if fingerprint(v) not in baseline_set] diff --git a/docksec/cli.py b/docksec/cli.py index 5b01012..ee6bb63 100644 --- a/docksec/cli.py +++ b/docksec/cli.py @@ -58,6 +58,8 @@ def main() -> None: parser.add_argument('--output-dir', dest='output_dir', metavar='DIR', help='Directory to write reports to (default: ~/.docksec/results or DOCKSEC_RESULTS_DIR)') parser.add_argument('--json', dest='json_stdout', action='store_true', help='Print scan results as JSON to stdout (no report files unless --format is also given)') parser.add_argument('--sarif', dest='sarif', action='store_true', help='Write a SARIF 2.1.0 report for GitHub Code Scanning and other SARIF-compatible tools') + parser.add_argument('--baseline', dest='baseline', metavar='FILE', help='Path to a baseline file; with --fail-on, only findings not present in the baseline trigger the gate') + parser.add_argument('--update-baseline', dest='update_baseline', action='store_true', help='Write the current scan findings to --baseline instead of gating against it') parser.add_argument('--quiet', action='store_true', help='Reduce output to warnings, errors, and the result summary') parser.add_argument('--no-color', action='store_true', help='Disable colored output (also honors the NO_COLOR env var)') parser.add_argument('--version', action='version', version=f'DockSec {get_version()}') @@ -140,6 +142,10 @@ def main() -> None: output_dir = args.output_dir or RESULTS_DIR # Validate argument combinations + if args.update_baseline and not args.baseline: + output.error("--update-baseline requires --baseline FILE") + sys.exit(2) + if args.image_only and args.ai_only: output.error("--image-only and --ai-only cannot be used together (AI analysis requires a Dockerfile)") sys.exit(2) @@ -358,14 +364,28 @@ def main() -> None: _render_scan_summary(output, args, scanner, results, report_paths, run_ai, run_compose_analysis) + # --update-baseline: snapshot current findings and skip gating. + if args.update_baseline: + from docksec import baseline as baseline_mod + baseline_mod.save_baseline(args.baseline, results) + output.info(f"Baseline written to {args.baseline}") + # --fail-on gate: flag findings at or above the chosen threshold. - if args.fail_on: + # With --baseline (and not --update-baseline), only findings not + # already present in the baseline count toward the gate. + if args.fail_on and not args.update_baseline: triggering = _findings_at_or_above(results, args.fail_on) + if args.baseline: + from docksec import baseline as baseline_mod + baseline_fingerprints = baseline_mod.load_baseline(args.baseline) + new_ids = {baseline_mod.fingerprint(v) for v in baseline_mod.new_findings(results, baseline_fingerprints)} + triggering = [v for v in triggering if baseline_mod.fingerprint(v) in new_ids] if triggering: gate_triggered = True + suffix = " (new since baseline)" if args.baseline else "" output.warn( - f"{len(triggering)} finding(s) at or above {args.fail_on} " - f"(--fail-on {args.fail_on}) -> exit 1" + f"{len(triggering)} finding(s) at or above {args.fail_on}" + f"{suffix} (--fail-on {args.fail_on}) -> exit 1" ) except ValueError as e: diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0f05c96..5f45677 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `--json` flag: print scan results as a single JSON object to stdout for scripts and CI pipelines. All human-readable output (banner, sections, info/warn/error, the result summary) moves to stderr in `--json` mode, so stdout carries only the JSON payload. `--json` alone does not write report files; combine with `--format` to also write files. - `--sarif` flag: write a SARIF 2.1.0 report for GitHub Code Scanning and other SARIF-compatible tools. Independent of `--format`; findings map to one SARIF rule per unique vulnerability ID and one result per finding, with severity mapped to SARIF levels (`CRITICAL`/`HIGH` -> `error`, `MEDIUM` -> `warning`, `LOW`/`UNKNOWN` -> `note`). - GitHub Action inputs for the new CLI flags: `output_dir`, `severity`, `fail_on`, `format`, `sarif`. +- `--baseline ` and `--update-baseline` flags for ratchet-mode adoption: `--update-baseline` snapshots the current scan's findings to the baseline file; subsequent runs with `--baseline` and `--fail-on` only gate on findings not already present in the baseline, so `--fail-on` can be adopted on existing projects without pre-existing findings blocking every build. Findings are matched by vulnerability ID, target, and package name. ### Changed - **Cleaner terminal output**: internal logs now write to `stderr` instead of `stdout` and stay quiet in CLI mode, so raw location-tagged log lines no longer interleave with the tool's user-facing messages. Set `DOCKSEC_LOG_LEVEL` to restore verbose logging. diff --git a/tests/test_baseline.py b/tests/test_baseline.py new file mode 100644 index 0000000..23a3da4 --- /dev/null +++ b/tests/test_baseline.py @@ -0,0 +1,84 @@ +import json +import os +import tempfile +import unittest + +from docksec import baseline + + +class TestFingerprint(unittest.TestCase): + def test_fingerprint_stable_for_same_finding(self): + v = {"VulnerabilityID": "CVE-2024-1", "Target": "app", "PkgName": "openssl"} + self.assertEqual(baseline.fingerprint(v), baseline.fingerprint(dict(v))) + + def test_fingerprint_differs_on_any_field(self): + base = {"VulnerabilityID": "CVE-2024-1", "Target": "app", "PkgName": "openssl"} + other = {"VulnerabilityID": "CVE-2024-2", "Target": "app", "PkgName": "openssl"} + self.assertNotEqual(baseline.fingerprint(base), baseline.fingerprint(other)) + + def test_fingerprint_handles_missing_fields(self): + self.assertEqual(baseline.fingerprint({}), "UNKNOWN||") + + +class TestLoadSaveBaseline(unittest.TestCase): + def test_load_missing_file_returns_empty(self): + self.assertEqual(baseline.load_baseline("/nonexistent/path/baseline.json"), []) + + def test_save_then_load_round_trips(self): + results = { + "json_data": [ + {"VulnerabilityID": "CVE-1", "Target": "a", "PkgName": "pkg1", "Severity": "HIGH"}, + {"VulnerabilityID": "CVE-2", "Target": "a", "PkgName": "pkg2", "Severity": "LOW"}, + ] + } + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "baseline.json") + baseline.save_baseline(path, results) + loaded = baseline.load_baseline(path) + self.assertEqual(len(loaded), 2) + self.assertIn(baseline.fingerprint(results["json_data"][0]), loaded) + + def test_save_dedupes_identical_findings(self): + vuln = {"VulnerabilityID": "CVE-1", "Target": "a", "PkgName": "pkg1", "Severity": "HIGH"} + results = {"json_data": [vuln, dict(vuln)]} + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "baseline.json") + baseline.save_baseline(path, results) + with open(path) as f: + data = json.load(f) + self.assertEqual(len(data["fingerprints"]), 1) + + def test_save_writes_valid_json_structure(self): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "baseline.json") + baseline.save_baseline(path, {"json_data": []}) + with open(path) as f: + data = json.load(f) + self.assertEqual(data, {"fingerprints": []}) + + +class TestNewFindings(unittest.TestCase): + def test_new_findings_excludes_baselined(self): + v1 = {"VulnerabilityID": "CVE-1", "Target": "a", "PkgName": "pkg1"} + v2 = {"VulnerabilityID": "CVE-2", "Target": "a", "PkgName": "pkg2"} + results = {"json_data": [v1, v2]} + baseline_fps = [baseline.fingerprint(v1)] + result = baseline.new_findings(results, baseline_fps) + self.assertEqual(result, [v2]) + + def test_new_findings_all_new_when_baseline_empty(self): + v1 = {"VulnerabilityID": "CVE-1", "Target": "a", "PkgName": "pkg1"} + results = {"json_data": [v1]} + self.assertEqual(baseline.new_findings(results, []), [v1]) + + def test_new_findings_empty_when_all_baselined(self): + v1 = {"VulnerabilityID": "CVE-1", "Target": "a", "PkgName": "pkg1"} + results = {"json_data": [v1]} + self.assertEqual(baseline.new_findings(results, [baseline.fingerprint(v1)]), []) + + def test_new_findings_handles_no_findings(self): + self.assertEqual(baseline.new_findings({"json_data": []}, []), []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 807a577..a7765a8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ """Unit tests for CLI arguments and flags.""" import unittest +import json import os import sys import tempfile @@ -572,5 +573,80 @@ def test_sarif_flag_does_not_affect_default_format_bundle(self): self.assertEqual(kwargs.get('formats'), ['json']) +class TestBaselineGate(unittest.TestCase): + """Test cases for --baseline / --update-baseline wiring in the CLI.""" + + def setUp(self): + self._tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self._tmpdir.cleanup) + self.baseline_path = os.path.join(self._tmpdir.name, 'baseline.json') + + def _run_image_only_with(self, findings, extra_argv=None): + from docksec.cli import main + + argv = ['docksec', '--image-only', '-i', 'test:latest'] + (extra_argv or []) + with patch('sys.argv', argv), \ + patch('docksec.docker_scanner.DockerSecurityScanner') as cls: + scanner = Mock() + cls.return_value = scanner + scanner.run_image_only_scan.return_value = { + 'json_data': [ + {"VulnerabilityID": f"CVE-{i}", "Target": "app", "PkgName": "pkg", "Severity": s} + for i, s in enumerate(findings) + ], + 'dockerfile_scan': {'skipped': True}, + 'image_scan': {'skipped': False}, + 'scan_mode': 'image_only', + } + scanner.get_security_score.return_value = 80.0 + scanner.generate_all_reports.return_value = {'json': 'x'} + scanner.RESULTS_DIR = '/tmp' + code = 0 + try: + main() + except SystemExit as e: + code = e.code + return code + + def test_update_baseline_without_baseline_flag_exits_2(self): + code = self._run_image_only_with(["HIGH"], ['--update-baseline']) + self.assertEqual(code, 2) + + def test_update_baseline_writes_file_and_does_not_gate(self): + code = self._run_image_only_with( + ["CRITICAL"], + ['--baseline', self.baseline_path, '--update-baseline', '--fail-on', 'critical'], + ) + self.assertEqual(code, 0) + self.assertTrue(os.path.isfile(self.baseline_path)) + with open(self.baseline_path) as f: + data = json.load(f) + self.assertEqual(len(data['fingerprints']), 1) + + def test_baseline_suppresses_previously_seen_findings(self): + # First run establishes the baseline. + self._run_image_only_with( + ["CRITICAL"], ['--baseline', self.baseline_path, '--update-baseline'] + ) + # Same finding again, now gated: should NOT trigger since it's baselined. + code = self._run_image_only_with( + ["CRITICAL"], ['--baseline', self.baseline_path, '--fail-on', 'critical'] + ) + self.assertEqual(code, 0) + + def test_baseline_still_gates_on_new_findings(self): + # Baseline has no findings. + self._run_image_only_with([], ['--baseline', self.baseline_path, '--update-baseline']) + # A new CRITICAL finding appears: should trigger the gate. + code = self._run_image_only_with( + ["CRITICAL"], ['--baseline', self.baseline_path, '--fail-on', 'critical'] + ) + self.assertEqual(code, 1) + + def test_fail_on_without_baseline_gates_normally(self): + code = self._run_image_only_with(["CRITICAL"], ['--fail-on', 'critical']) + self.assertEqual(code, 1) + + if __name__ == '__main__': unittest.main()