fix(scanner): scan gitignored .env files (audit C1, critical) (#47)

Discovery used 'git ls-files --cached --others --exclude-standard', which drops
gitignored+untracked files. In a real repo .env is gitignored, so the flagship
value-bearing credential detection (hardcoded key in .env) silently never ran —
the report showed a clean bill over a live key. The committed fixture .env
(tracked) masked this in tests.

Fix: union git discovery with a targeted walk for ALWAYS_SCAN_GLOBS (.env, *.env,
.env.*, *.envrc) so credential files are scanned even when gitignored. Fixture
scan unchanged (32 findings, exact); added a regression test with a real-world
gitignored-.env repo layout (would have caught this).

Author: emmanuelgjr

dsgai_scanner_tool/cli/dsgai_scan.py

@@ -40,6 +40,10 @@
 }
 SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "env",
              "dist", "build", ".mypy_cache", ".pytest_cache", ".tox", ".idea"}
+# Credential-bearing files that MUST be scanned even when gitignored — a secret
+# scanner's whole job is to catch keys in exactly these, and they are almost
+# always gitignored (so `git ls-files` would drop them). Matched by basename.
+ALWAYS_SCAN_GLOBS = ["*.env", "*.env.*", ".env", ".env.*", "*.envrc"]
 
 # requires_nearby resolution: status when the requirement is violated (required
 # rule absent) vs satisfied (present). Read the require list / window from the
@@ -109,16 +113,25 @@ def read_version():
 # File discovery
 # --------------------------------------------------------------------------- #
 def discover_files(target, excludes):
-    """Return files under target as (abs_path, rel_path) honoring .gitignore.
+    """Return files under target as (abs_path, rel_path).
 
-    Uses `git ls-files` (tracked + untracked-not-ignored) when target is inside
-    a repo — this respects .gitignore yet still includes tracked files such as a
-    committed fixture .env. Falls back to a filtered os.walk otherwise.
+    Uses `git ls-files` (tracked + untracked-not-ignored) when target is inside a
+    repo, so general noise/build output respects .gitignore. BUT credential
+    files matching ALWAYS_SCAN_GLOBS (e.g. `.env`) are ALWAYS included even when
+    gitignored — a secret scanner must scan exactly those. Falls back to a
+    filtered os.walk (which already sees everything) outside a repo.
     """
     target = os.path.abspath(target)
     files = _git_files(target)
     if files is None:
         files = _walk_files(target)
+    else:
+        # Union in gitignored credential files git would otherwise drop.
+        seen = set(files)
+        for ap in _sensitive_walk_files(target):
+            if ap not in seen:
+                seen.add(ap)
+                files.append(ap)
     out = []
     for ap in files:
         rel = os.path.relpath(ap, target).replace(os.sep, "/")
@@ -129,6 +142,18 @@ def discover_files(target, excludes):
     return out
 
 
+def _sensitive_walk_files(target):
+    """Files under target whose basename matches ALWAYS_SCAN_GLOBS (skips noise
+    dirs). Used to re-include gitignored credential files like .env."""
+    hits = []
+    for dp, dirs, fns in os.walk(target):
+        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        for fn in fns:
+            if any(fnmatch.fnmatch(fn, g) for g in ALWAYS_SCAN_GLOBS):
+                hits.append(os.path.join(dp, fn))
+    return hits
+
+
 def _git_files(target):
     try:
         top = subprocess.run(["git", "-C", target, "rev-parse", "--show-toplevel"],

dsgai_scanner_tool/tests/test_runner.py

@@ -314,6 +314,31 @@ def test_atlas_map_valid():
         assert all(ctrl_re.match(c) for c in t["controls"])
 
 
+@requires_rg
+def test_gitignored_env_is_still_scanned(tmp_path):
+    """Real-world layout: a `.env` is gitignored + untracked. A secret scanner
+    MUST still scan it — regression test for the discovery bug where git
+    ls-files silently dropped it (the committed fixture .env masked this)."""
+    repo = tmp_path / "realrepo"
+    repo.mkdir()
+    subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
+    (repo / ".gitignore").write_text(".env\n", encoding="utf-8")
+    (repo / ".env").write_text(
+        "OPENAI_API_KEY=sk-proj-FAKE00000000000000000000000000\n", encoding="utf-8")
+    (repo / "app.py").write_text("x = 1\n", encoding="utf-8")
+    env = dict(os.environ)
+    subprocess.run(["git", "add", ".gitignore", "app.py"], cwd=repo, check=True)
+    subprocess.run(["git", "-c", "user.email=a@b.c", "-c", "user.name=t",
+                    "commit", "-qm", "init"], cwd=repo, check=True)
+    out = tmp_path / "s.json"
+    subprocess.run([sys.executable, CLI, "scan", str(repo), "--no-cve",
+                    "--json-out", str(out), "--format", "none"], env=env)
+    cp = json.loads(out.read_text(encoding="utf-8"))
+    env_findings = [f for f in cp["findings"] if ".env" in f["path"]]
+    assert env_findings, "gitignored .env credential was not scanned"
+    assert cp["controls"]["DSGAI02"] == "FAIL"
+
+
 def test_rules_json_in_sync():
     from_yaml = yaml.safe_load(open(RULES_YAML, encoding="utf-8"))
     rebuilt = json.dumps(from_yaml, indent=2, sort_keys=True, ensure_ascii=False) + "\n"