From 0db053397c6c074b6839073d733c5dbee855568b Mon Sep 17 00:00:00 2001 From: Advait Patel Date: Wed, 1 Jul 2026 13:32:07 -0500 Subject: [PATCH] Phase 2b: add --fail-on gate and CI-friendly 0/1/2/3 exit codes --- README.md | 21 +++++++++-- docksec/cli.py | 74 ++++++++++++++++++++++++++++++++------ docksec/enums.py | 15 ++++++++ docs/CHANGELOG.md | 2 ++ tests/test_cli.py | 87 +++++++++++++++++++++++++++++++++++++++++---- tests/test_enums.py | 27 ++++++++++++++ 6 files changed, 207 insertions(+), 19 deletions(-) create mode 100644 tests/test_enums.py diff --git a/README.md b/README.md index 7b5d439..61ee88d 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,9 @@ docksec Dockerfile --scan-only # Choose which severity levels the image scan reports (default: CRITICAL,HIGH) docksec -i myapp:latest --image-only --severity CRITICAL,HIGH,MEDIUM +# Fail the build (exit 1) if any finding is HIGH or above +docksec -i myapp:latest --image-only --fail-on high + # Reduce output to warnings, errors, and the result summary docksec Dockerfile --scan-only --quiet @@ -115,8 +118,22 @@ docksec Dockerfile --no-color Every scan ends with a result summary: a severity table, the security score with a rating, a "Quick take" action block, the generated reports, and a suggested next -command. Use `--quiet` for a compact result and `--no-color` for plain output. A failed -scan exits non-zero, so it works cleanly in shells and CI. +command. Use `--quiet` for a compact result and `--no-color` for plain output. + +### Exit codes + +DockSec uses CI-friendly exit codes so builds and shells can react to results: + +| Code | Meaning | +|---|---| +| `0` | Success, no findings at or above `--fail-on` | +| `1` | Findings at or above the `--fail-on` threshold | +| `2` | Usage or argument error | +| `3` | Tool or runtime error (scan failed, image not found, missing tools) | + +`--fail-on` gates on the structured findings (image vulnerabilities and compose +misconfigurations). When `--fail-on` is below the requested `--severity`, the scan +severity is widened automatically so the gate can observe those findings. --- diff --git a/docksec/cli.py b/docksec/cli.py index 230cd7a..d692cf9 100644 --- a/docksec/cli.py +++ b/docksec/cli.py @@ -53,6 +53,7 @@ def main() -> None: parser.add_argument('--compact-output', action='store_true', help='Use compact output format (less verbose)') parser.add_argument('--skip-ai-scoring', action='store_true', help='Skip AI-based security scoring (use local scoring only)') parser.add_argument('--severity', help='Comma-separated severity levels to scan for (default: CRITICAL,HIGH; or set DOCKSEC_DEFAULT_SEVERITY)') + parser.add_argument('--fail-on', dest='fail_on', metavar='SEVERITY', help='Exit with code 1 if any finding is at or above this severity (CRITICAL, HIGH, MEDIUM, or LOW)') parser.add_argument('--quiet', action='store_true', help='Reduce output to warnings, errors, and the result summary') parser.add_argument('--no-color', action='store_true', help='Disable colored output (also honors the NO_COLOR env var)') parser.add_argument('--version', action='version', version=f'DockSec {get_version()}') @@ -89,17 +90,37 @@ def main() -> None: f"Invalid --severity value '{severity}'. " f"Valid levels: {', '.join(Severity.values())}" ) - sys.exit(1) + sys.exit(2) severity = ','.join(severity_list) + # Validate --fail-on and widen the scan severity so the gate can see the + # levels it needs (e.g. --fail-on medium with the default CRITICAL,HIGH scan + # would otherwise never observe MEDIUM findings). Compose static findings are + # always emitted at all severities, so widening only affects the image scan. + if args.fail_on: + args.fail_on = args.fail_on.strip().upper() + if args.fail_on not in Severity.gate_levels(): + output.error( + f"Invalid --fail-on value '{args.fail_on}'. " + f"Choose one of: {', '.join(Severity.gate_levels())}" + ) + sys.exit(2) + needed = {lvl for lvl in Severity.gate_levels() + if Severity.rank(lvl) >= Severity.rank(args.fail_on)} + widened = set(severity_list) | needed + if widened != set(severity_list): + severity_list = [lvl for lvl in Severity.values() if lvl in widened] + severity = ','.join(severity_list) + output.info(f"Widened scan severity to {severity} to satisfy --fail-on {args.fail_on}") + # Validate argument combinations if args.image_only and args.ai_only: output.error("--image-only and --ai-only cannot be used together (AI analysis requires a Dockerfile)") - sys.exit(1) + sys.exit(2) if args.image_only and args.scan_only: output.error("--image-only and --scan-only cannot be used together (use --image-only for image-only scanning)") - sys.exit(1) + sys.exit(2) # Validate Dockerfile requirement if not args.image_only and not args.compose and not args.dockerfile: @@ -109,18 +130,18 @@ def main() -> None: print(" docksec --image-only -i myapp:latest # Scan only the image") print(" docksec --compose docker-compose.yml # Scan compose file and its services") print(" docksec --ai-only Dockerfile # AI analysis only") - sys.exit(1) + sys.exit(2) # Validate that the Dockerfile exists (if provided) if args.dockerfile and not os.path.isfile(args.dockerfile): output.error(f"Dockerfile not found at {args.dockerfile}") - sys.exit(1) + sys.exit(2) # Validate image requirement for image-based operations if args.image_only and not args.image: output.error("Image name is required for image-only scanning. Use -i/--image to specify the Docker image.") print("Example: docksec --image-only -i myapp:latest") - sys.exit(1) + sys.exit(2) # In scan-only mode, if no image is provided, we'll only run Dockerfile analysis if args.scan_only and not args.image: @@ -142,11 +163,11 @@ def main() -> None: break if compose_path == 'auto': output.error("Could not auto-detect a docker-compose file in the current directory.") - sys.exit(1) + sys.exit(2) if not os.path.isfile(compose_path): output.error(f"Compose file not found at {compose_path}") - sys.exit(1) + sys.exit(2) args.compose = compose_path elif args.image_only: @@ -235,12 +256,13 @@ def main() -> None: except ImportError as e: output.error(f"Required modules not found - {e}") - sys.exit(1) + sys.exit(3) except Exception as e: output.error(f"AI analysis failed: {e}") # Run the scanner tool scan_ok = None # None = scan not run, True = success, False = failed + gate_triggered = False # True when findings meet the --fail-on threshold if run_scan: scan_title = "Compose" if run_compose_analysis else ("Image" if args.image_only else "Full") output.section(f"{scan_title} security scan") @@ -299,23 +321,37 @@ def main() -> None: _render_scan_summary(output, args, scanner, results, report_paths, run_ai, run_compose_analysis) + # --fail-on gate: flag findings at or above the chosen threshold. + if args.fail_on: + triggering = _findings_at_or_above(results, args.fail_on) + if triggering: + gate_triggered = True + output.warn( + f"{len(triggering)} finding(s) at or above {args.fail_on} " + f"(--fail-on {args.fail_on}) -> exit 1" + ) + except ValueError as e: # Expected, actionable failures (missing image, missing tools, bad input). output.error(str(e)) scan_ok = False except ImportError as e: output.error(f"Scanner modules not found - {e}") - sys.exit(1) + sys.exit(3) except Exception as e: output.error(f"Scanner failed: {e}") scan_ok = False + # Exit codes (CI-friendly): 0 clean, 1 findings at/above --fail-on, + # 2 usage error, 3 tool/runtime error. if not run_ai and not run_scan: output.warn("No analysis performed. Use --help for usage information.") sys.exit(2) - # Exit non-zero when the scan failed so CI and shells can react honestly. if scan_ok is False: + sys.exit(3) + + if gate_triggered: sys.exit(1) @@ -363,6 +399,22 @@ def _quick_take_lines(results, counts, run_ai): return lines +def _findings_at_or_above(results, threshold): + """Return the scan findings whose severity is at or above the threshold. + + Operates on the structured findings in ``json_data`` (image vulnerabilities + and compose misconfigurations). Hadolint lint warnings are not severity-ranked + and do not participate in the --fail-on gate. + """ + from docksec.enums import Severity + + threshold_rank = Severity.rank(threshold) + return [ + v for v in results.get("json_data", []) + if Severity.rank(v.get("Severity")) >= threshold_rank + ] + + def _format_hadolint_line(line): """Turn a raw Hadolint line into a compact, path-free summary. diff --git a/docksec/enums.py b/docksec/enums.py index f61d0fd..722bae6 100644 --- a/docksec/enums.py +++ b/docksec/enums.py @@ -17,6 +17,21 @@ def scored_levels(cls) -> list: """Severities that affect the security score.""" return [cls.CRITICAL, cls.HIGH, cls.MEDIUM, cls.LOW] + @classmethod + def gate_levels(cls) -> list: + """Severities usable as a --fail-on threshold, most severe first.""" + return [cls.CRITICAL.value, cls.HIGH.value, cls.MEDIUM.value, cls.LOW.value] + + @classmethod + def rank(cls, value) -> int: + """Numeric ordering for comparisons (higher is more severe). + + Unknown or unrecognized values rank 0 so they never trip a + --fail-on threshold set to a real severity level. + """ + order = {"CRITICAL": 4, "HIGH": 3, "MEDIUM": 2, "LOW": 1, "UNKNOWN": 0} + return order.get(str(value).strip().upper(), 0) + class LLMProvider(str, Enum): OPENAI = "openai" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 295fd28..6fcc4a6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `--quiet` flag to reduce output to warnings, errors, and the result summary. - `--no-color` flag (also honors the `NO_COLOR` environment variable) to disable colored output. - `--severity` flag to choose which severity levels the image vulnerability scan reports (default `CRITICAL,HIGH`; also settable via `DOCKSEC_DEFAULT_SEVERITY`). Invalid values are rejected with a clear error. +- `--fail-on ` flag: exit with code 1 when any finding is at or above the chosen severity (`CRITICAL`, `HIGH`, `MEDIUM`, or `LOW`). The scan severity is auto-widened when needed so the gate can observe those findings. +- CI-friendly exit codes: `0` clean, `1` findings at or above `--fail-on`, `2` usage/argument error, `3` tool or runtime error (scan failed, image not found, missing tools). ### Changed - **Cleaner terminal output**: internal logs now write to `stderr` instead of `stdout` and stay quiet in CLI mode, so raw location-tagged log lines no longer interleave with the tool's user-facing messages. Set `DOCKSEC_LOG_LEVEL` to restore verbose logging. diff --git a/tests/test_cli.py b/tests/test_cli.py index 7922b37..547b968 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -165,13 +165,13 @@ def test_model_flag_sets_env(self, mock_scanner_class): @patch('sys.argv', ['docksec', '--image-only', '-i', 'test:latest', '--severity', 'BOGUS']) def test_invalid_severity_is_rejected(self): - """An invalid --severity value should error out and exit non-zero.""" + """An invalid --severity value is a usage error and exits 2.""" from docksec.cli import main with patch('builtins.print'): with self.assertRaises(SystemExit) as ctx: main() - self.assertEqual(ctx.exception.code, 1) + self.assertEqual(ctx.exception.code, 2) @patch('sys.argv', ['docksec', '--image-only', '-i', 'test:latest', '--severity', 'critical']) @patch('docksec.docker_scanner.DockerSecurityScanner') @@ -196,15 +196,15 @@ def test_severity_flag_threads_to_scanner(self, mock_scanner_class): # 'critical' is normalized to 'CRITICAL' and passed to the image scan. scanner.run_image_only_scan.assert_called_once_with('CRITICAL') - @patch('sys.argv', ['docksec', '/no/such/docksec_dockerfile_xyz', '--quiet', '--no-color']) + @patch('sys.argv', ['docksec', '--image-only', '-i', 'docksec_missing_img_xyz:latest', '--quiet', '--no-color']) def test_quiet_and_no_color_flags_are_accepted(self): - """--quiet and --no-color must parse (argparse exits 2 on unknown flags).""" + """--quiet and --no-color must parse. Using a missing image makes the run + reach a tool/runtime error (exit 3), which is distinct from argparse's + exit 2 for an unknown flag - so a non-2 exit proves the flags parsed.""" from docksec.cli import main with patch('builtins.print'): with self.assertRaises(SystemExit) as ctx: - # The Dockerfile path does not exist, so main exits 1 during - # validation - well past argument parsing. main() self.assertNotEqual(ctx.exception.code, 2) @@ -274,5 +274,80 @@ class Args: self.assertEqual(cmd, "") +class TestFailOnGate(unittest.TestCase): + """Test cases for the --fail-on gate helper and exit codes.""" + + def _results(self, severities): + return {"json_data": [{"Severity": s} for s in severities]} + + def test_findings_at_or_above_includes_equal_and_higher(self): + from docksec.cli import _findings_at_or_above + + results = self._results(["CRITICAL", "HIGH", "MEDIUM", "LOW"]) + # threshold HIGH -> CRITICAL + HIGH + self.assertEqual(len(_findings_at_or_above(results, "HIGH")), 2) + # threshold LOW -> all four + self.assertEqual(len(_findings_at_or_above(results, "LOW")), 4) + # threshold CRITICAL -> only CRITICAL + self.assertEqual(len(_findings_at_or_above(results, "CRITICAL")), 1) + + def test_findings_at_or_above_ignores_unknown(self): + from docksec.cli import _findings_at_or_above + + results = self._results(["UNKNOWN", "UNKNOWN"]) + self.assertEqual(_findings_at_or_above(results, "LOW"), []) + + def test_findings_at_or_above_empty(self): + from docksec.cli import _findings_at_or_above + + self.assertEqual(_findings_at_or_above({"json_data": []}, "CRITICAL"), []) + + @patch('sys.argv', ['docksec', '--image-only', '-i', 'test:latest', '--fail-on', 'BOGUS']) + def test_invalid_fail_on_exits_2(self): + from docksec.cli import main + + with patch('builtins.print'): + with self.assertRaises(SystemExit) as ctx: + main() + self.assertEqual(ctx.exception.code, 2) + + def _run_image_only_with(self, findings, fail_on): + """Run main() image-only with a mocked scanner returning `findings`.""" + from docksec.cli import main + + argv = ['docksec', '--image-only', '-i', 'test:latest'] + if fail_on: + argv += ['--fail-on', fail_on] + with patch('sys.argv', argv), \ + patch('docksec.docker_scanner.DockerSecurityScanner') as cls: + scanner = Mock() + cls.return_value = scanner + scanner.run_image_only_scan.return_value = { + 'json_data': [{"Severity": s} for s in findings], + 'dockerfile_scan': {'skipped': True}, + 'image_scan': {'skipped': False}, + 'scan_mode': 'image_only', + } + scanner.get_security_score.return_value = 80.0 + scanner.generate_all_reports.return_value = {'json': 'x'} + scanner.RESULTS_DIR = '/tmp' + code = 0 + try: + main() + except SystemExit as e: + code = e.code + return code + + def test_gate_triggers_exit_1_on_matching_finding(self): + self.assertEqual(self._run_image_only_with(["HIGH"], "high"), 1) + + def test_gate_clean_exit_0_when_below_threshold(self): + # LOW finding, threshold CRITICAL -> nothing at/above -> exit 0 + self.assertEqual(self._run_image_only_with(["LOW"], "critical"), 0) + + def test_no_fail_on_never_gates(self): + self.assertEqual(self._run_image_only_with(["CRITICAL"], None), 0) + + if __name__ == '__main__': unittest.main() diff --git a/tests/test_enums.py b/tests/test_enums.py new file mode 100644 index 0000000..6a713fb --- /dev/null +++ b/tests/test_enums.py @@ -0,0 +1,27 @@ +"""Unit tests for docksec.enums helpers.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from docksec.enums import Severity + + +def test_rank_ordering_is_descending_by_severity(): + assert Severity.rank("CRITICAL") > Severity.rank("HIGH") + assert Severity.rank("HIGH") > Severity.rank("MEDIUM") + assert Severity.rank("MEDIUM") > Severity.rank("LOW") + assert Severity.rank("LOW") > Severity.rank("UNKNOWN") + + +def test_rank_is_case_insensitive_and_safe(): + assert Severity.rank("critical") == Severity.rank("CRITICAL") + assert Severity.rank(" High ") == Severity.rank("HIGH") + assert Severity.rank(None) == 0 + assert Severity.rank("not-a-severity") == 0 + + +def test_gate_levels_are_the_four_real_severities_most_severe_first(): + assert Severity.gate_levels() == ["CRITICAL", "HIGH", "MEDIUM", "LOW"] + assert "UNKNOWN" not in Severity.gate_levels()