test: added script which finds new formatting bugs with pysource-codegen

Author: 15r10nk

.pre-commit-config.yaml

@@ -58,6 +58,8 @@ repos:
           - types-commonmark
           - urllib3
           - hypothesmith
+          - pysource-codegen
+          - pysource-minimize
       - id: mypy
         name: mypy (Python 3.10)
         files: scripts/generate_schema.py

scripts/find_issue.py

@@ -0,0 +1,199 @@
+import random
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, cast
+
+from pysource_codegen import generate
+from pysource_minimize import minimize
+
+import black
+
+base_path = Path(__file__).parent
+
+
+@dataclass()
+class Issue:
+    src: str
+    mode: black.FileMode
+
+
+def bug_in_code(issue: Issue) -> bool:
+    try:
+        dst_contents = black.format_str(issue.src, mode=issue.mode)
+
+        black.assert_equivalent(issue.src, dst_contents)
+        black.assert_stable(issue.src, dst_contents, mode=issue.mode)
+    except Exception:
+        return True
+    return False
+
+
+def current_target_version() -> black.TargetVersion:
+    v = sys.version_info
+    return cast(black.TargetVersion, getattr(black.TargetVersion, f"PY{v[0]}{v[1]}"))
+
+
+def find_issue() -> Optional[Issue]:
+    t = time.time()
+    print("search for new issue ", end="", flush=True)
+
+    while time.time() - t < 60 * 10:
+        for line_length in (100, 1):
+            for magic_trailing_comma in (True, False):
+                print(".", end="", flush=True)
+                mode = black.FileMode(
+                    line_length=line_length,
+                    string_normalization=True,
+                    is_pyi=False,
+                    magic_trailing_comma=magic_trailing_comma,
+                    target_versions={current_target_version()},
+                )
+                seed = random.randint(0, 100000000)
+
+                src_code = generate(seed)
+
+                issue = Issue(src_code, mode)
+
+                if bug_in_code(issue):
+                    print(f"found bug (seed={seed})")
+                    return issue
+    print("no new issue found in 10 minutes")
+    return None
+
+
+def minimize_code(issue: Issue) -> Issue:
+    minimized = Issue(
+        minimize(issue.src, lambda code: bug_in_code(Issue(code, issue.mode))),
+        issue.mode,
+    )
+    assert bug_in_code(minimized)
+
+    print("minimized code:")
+    print(minimized.src)
+
+    return minimized
+
+
+def mode_to_options(mode: black.FileMode) -> list[str]:
+    result = ["-l", str(mode.line_length)]
+    if not mode.magic_trailing_comma:
+        result.append("-C")
+    (v,) = list(mode.target_versions)
+    result += ["-t", v.name.lower()]
+    return result
+
+
+def create_issue(issue: Issue) -> str:
+
+    dir = tempfile.TemporaryDirectory()
+
+    cwd = Path(dir.name)
+    (cwd / "bug.py").write_text(issue.src)
+
+    multiline_code = "\n".join(["    " + repr(s + "\n") for s in issue.src.split("\n")])
+
+    parse_code = f"""\
+from ast import parse
+parse(
+{multiline_code}
+)
+"""
+    cwd = Path(dir.name)
+    (cwd / "parse_code.py").write_text(parse_code)
+    command = ["black", *mode_to_options(issue.mode), "bug.py"]
+
+    format_result = subprocess.run(
+        [sys.executable, "-m", *command], capture_output=True, cwd=cwd
+    )
+
+    error_output = format_result.stderr.decode()
+
+    m = re.search("This diff might be helpful: (/.*)", error_output)
+    reported_diff = ""
+    if m:
+        path = Path(m[1])
+        reported_diff = f"""
+the reported diff in {path} is:
+``` diff
+{path.read_text()}
+```
+"""
+
+    run_result = subprocess.run(
+        [sys.executable, "parse_code.py"], capture_output=True, cwd=cwd
+    )
+
+    git_ref = subprocess.run(
+        ["git", "rev-parse", "origin/main"], capture_output=True
+    ).stdout
+
+    return f"""
+**Describe the bug**
+
+The following code can not be parsed/formatted by black:
+
+``` python
+{issue.src}
+```
+
+black reported the following error:
+```
+> {" ".join(command)}
+{format_result.stderr.decode()}
+```
+{reported_diff}
+
+but it can be parsed by cpython:
+``` python
+{parse_code}
+```
+result:
+```
+{run_result.stderr.decode()}
+returncode: {run_result.returncode}
+```
+
+
+**Environment**
+
+<!-- Please complete the following information: -->
+
+- Black's version: current main ({git_ref.decode().strip()})
+- OS and Python version: Linux/Python {sys.version}
+
+**Additional context**
+
+The bug was found by pysource-codegen (see #3908)
+
+"""
+
+
+def main() -> None:
+    issue = find_issue()
+    if issue is None:
+        return
+
+    issue = minimize_code(issue)
+
+    while issue.mode.line_length > 1 and bug_in_code(issue):
+        issue.mode.line_length -= 1
+    issue.mode.line_length += 1
+
+    issue = minimize_code(issue)
+
+    while issue.mode.line_length <= 100 and bug_in_code(issue):
+        issue.mode.line_length += 1
+    issue.mode.line_length -= 1
+
+    issue = minimize_code(issue)
+
+    print(create_issue(issue))
+
+
+if __name__ == "__main__":
+    main()

test_requirements.txt

@@ -4,3 +4,5 @@ pytest >= 6.1.1
 pytest-xdist >= 3.0.2
 pytest-cov >= 4.1.0
 tox
+pysource_codegen >= 0.4.1
+pysource_minimize >= 0.4.0