pregenerate the pycodestyle plugin to avoid call overhead

Author: asottile

bin/gen-pycodestyle-plugin

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+import inspect
+import os.path
+from typing import Any
+from typing import Callable
+from typing import Generator
+from typing import NamedTuple
+from typing import Tuple
+
+import pycodestyle
+
+
+def _too_long(s: str) -> str:
+    if len(s) >= 80:
+        return f"{s}  # noqa: E501"
+    else:
+        return s
+
+
+class Call(NamedTuple):
+    name: str
+    is_generator: bool
+    params: Tuple[str, ...]
+
+    def to_src(self) -> str:
+        params_s = ", ".join(self.params)
+        if self.is_generator:
+            return _too_long(f"    yield from _{self.name}({params_s})")
+        else:
+            lines = (
+                _too_long(f"    ret = _{self.name}({params_s})"),
+                "    if ret is not None:",
+                "        yield ret",
+            )
+            return "\n".join(lines)
+
+    @classmethod
+    def from_func(cls, func: Callable[..., Any]) -> "Call":
+        spec = inspect.getfullargspec(func)
+        params = tuple(spec.args)
+        return cls(func.__name__, inspect.isgeneratorfunction(func), params)
+
+
+def lines() -> Generator[str, None, None]:
+    logical = []
+    physical = []
+
+    logical = [
+        Call.from_func(check) for check in pycodestyle._checks["logical_line"]
+    ]
+    physical = [
+        Call.from_func(check) for check in pycodestyle._checks["physical_line"]
+    ]
+    assert not pycodestyle._checks["tree"]
+
+    yield f'"""Generated using ./bin/{os.path.basename(__file__)}."""'
+    yield "# fmt: off"
+    yield "from typing import Any"
+    yield "from typing import Generator"
+    yield "from typing import Tuple"
+    yield ""
+    imports = sorted(call.name for call in logical + physical)
+    for name in imports:
+        yield _too_long(f"from pycodestyle import {name} as _{name}")
+    yield ""
+    yield ""
+
+    yield "def pycodestyle_logical("
+    logical_params = {param for call in logical for param in call.params}
+    for param in sorted(logical_params):
+        yield f"    {param}: Any,"
+    yield ") -> Generator[Tuple[int, str], None, None]:"
+    yield '    """Run pycodestyle logical checks."""'
+    for call in sorted(logical):
+        yield call.to_src()
+    yield ""
+    yield ""
+
+    yield "def pycodestyle_physical("
+    physical_params = {param for call in physical for param in call.params}
+    for param in sorted(physical_params):
+        yield f"    {param}: Any,"
+    yield ") -> Generator[Tuple[int, str], None, None]:"
+    yield '    """Run pycodestyle physical checks."""'
+    for call in sorted(physical):
+        yield call.to_src()
+
+
+def main() -> int:
+    for line in lines():
+        print(line)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

setup.cfg

@@ -53,42 +53,8 @@ console_scripts =
     flake8 = flake8.main.cli:main
 flake8.extension =
     F = flake8.plugins.pyflakes:FlakesChecker
-    pycodestyle.ambiguous_identifier = pycodestyle:ambiguous_identifier
-    pycodestyle.bare_except = pycodestyle:bare_except
-    pycodestyle.blank_lines = pycodestyle:blank_lines
-    pycodestyle.break_after_binary_operator = pycodestyle:break_after_binary_operator
-    pycodestyle.break_before_binary_operator = pycodestyle:break_before_binary_operator
-    pycodestyle.comparison_negative = pycodestyle:comparison_negative
-    pycodestyle.comparison_to_singleton = pycodestyle:comparison_to_singleton
-    pycodestyle.comparison_type = pycodestyle:comparison_type
-    pycodestyle.compound_statements = pycodestyle:compound_statements
-    pycodestyle.continued_indentation = pycodestyle:continued_indentation
-    pycodestyle.explicit_line_join = pycodestyle:explicit_line_join
-    pycodestyle.extraneous_whitespace = pycodestyle:extraneous_whitespace
-    pycodestyle.imports_on_separate_lines = pycodestyle:imports_on_separate_lines
-    pycodestyle.indentation = pycodestyle:indentation
-    pycodestyle.maximum_doc_length = pycodestyle:maximum_doc_length
-    pycodestyle.maximum_line_length = pycodestyle:maximum_line_length
-    pycodestyle.missing_whitespace = pycodestyle:missing_whitespace
-    pycodestyle.missing_whitespace_after_import_keyword = pycodestyle:missing_whitespace_after_import_keyword
-    pycodestyle.missing_whitespace_around_operator = pycodestyle:missing_whitespace_around_operator
-    pycodestyle.module_imports_on_top_of_file = pycodestyle:module_imports_on_top_of_file
-    pycodestyle.python_3000_async_await_keywords = pycodestyle:python_3000_async_await_keywords
-    pycodestyle.python_3000_backticks = pycodestyle:python_3000_backticks
-    pycodestyle.python_3000_has_key = pycodestyle:python_3000_has_key
-    pycodestyle.python_3000_invalid_escape_sequence = pycodestyle:python_3000_invalid_escape_sequence
-    pycodestyle.python_3000_not_equal = pycodestyle:python_3000_not_equal
-    pycodestyle.python_3000_raise_comma = pycodestyle:python_3000_raise_comma
-    pycodestyle.tabs_obsolete = pycodestyle:tabs_obsolete
-    pycodestyle.tabs_or_spaces = pycodestyle:tabs_or_spaces
-    pycodestyle.trailing_blank_lines = pycodestyle:trailing_blank_lines
-    pycodestyle.trailing_whitespace = pycodestyle:trailing_whitespace
-    pycodestyle.whitespace_around_comma = pycodestyle:whitespace_around_comma
-    pycodestyle.whitespace_around_keywords = pycodestyle:whitespace_around_keywords
-    pycodestyle.whitespace_around_named_parameter_equals = pycodestyle:whitespace_around_named_parameter_equals
-    pycodestyle.whitespace_around_operator = pycodestyle:whitespace_around_operator
-    pycodestyle.whitespace_before_comment = pycodestyle:whitespace_before_comment
-    pycodestyle.whitespace_before_parameters = pycodestyle:whitespace_before_parameters
+    E = flake8.plugins.pycodestyle:pycodestyle_logical
+    W = flake8.plugins.pycodestyle:pycodestyle_physical
 flake8.report =
     default = flake8.formatting.default:Default
     pylint = flake8.formatting.default:Pylint

src/flake8/plugins/pycodestyle.py

@@ -0,0 +1,123 @@
+"""Generated using ./bin/gen-pycodestyle-plugin."""
+# fmt: off
+from typing import Any
+from typing import Generator
+from typing import Tuple
+
+from pycodestyle import ambiguous_identifier as _ambiguous_identifier
+from pycodestyle import bare_except as _bare_except
+from pycodestyle import blank_lines as _blank_lines
+from pycodestyle import break_after_binary_operator as _break_after_binary_operator  # noqa: E501
+from pycodestyle import break_before_binary_operator as _break_before_binary_operator  # noqa: E501
+from pycodestyle import comparison_negative as _comparison_negative
+from pycodestyle import comparison_to_singleton as _comparison_to_singleton
+from pycodestyle import comparison_type as _comparison_type
+from pycodestyle import compound_statements as _compound_statements
+from pycodestyle import continued_indentation as _continued_indentation
+from pycodestyle import explicit_line_join as _explicit_line_join
+from pycodestyle import extraneous_whitespace as _extraneous_whitespace
+from pycodestyle import imports_on_separate_lines as _imports_on_separate_lines
+from pycodestyle import indentation as _indentation
+from pycodestyle import maximum_doc_length as _maximum_doc_length
+from pycodestyle import maximum_line_length as _maximum_line_length
+from pycodestyle import missing_whitespace as _missing_whitespace
+from pycodestyle import missing_whitespace_after_import_keyword as _missing_whitespace_after_import_keyword  # noqa: E501
+from pycodestyle import missing_whitespace_around_operator as _missing_whitespace_around_operator  # noqa: E501
+from pycodestyle import module_imports_on_top_of_file as _module_imports_on_top_of_file  # noqa: E501
+from pycodestyle import python_3000_async_await_keywords as _python_3000_async_await_keywords  # noqa: E501
+from pycodestyle import python_3000_backticks as _python_3000_backticks
+from pycodestyle import python_3000_has_key as _python_3000_has_key
+from pycodestyle import python_3000_invalid_escape_sequence as _python_3000_invalid_escape_sequence  # noqa: E501
+from pycodestyle import python_3000_not_equal as _python_3000_not_equal
+from pycodestyle import python_3000_raise_comma as _python_3000_raise_comma
+from pycodestyle import tabs_obsolete as _tabs_obsolete
+from pycodestyle import tabs_or_spaces as _tabs_or_spaces
+from pycodestyle import trailing_blank_lines as _trailing_blank_lines
+from pycodestyle import trailing_whitespace as _trailing_whitespace
+from pycodestyle import whitespace_around_comma as _whitespace_around_comma
+from pycodestyle import whitespace_around_keywords as _whitespace_around_keywords  # noqa: E501
+from pycodestyle import whitespace_around_named_parameter_equals as _whitespace_around_named_parameter_equals  # noqa: E501
+from pycodestyle import whitespace_around_operator as _whitespace_around_operator  # noqa: E501
+from pycodestyle import whitespace_before_comment as _whitespace_before_comment
+from pycodestyle import whitespace_before_parameters as _whitespace_before_parameters  # noqa: E501
+
+
+def pycodestyle_logical(
+    blank_before: Any,
+    blank_lines: Any,
+    checker_state: Any,
+    hang_closing: Any,
+    indent_char: Any,
+    indent_level: Any,
+    indent_size: Any,
+    line_number: Any,
+    lines: Any,
+    logical_line: Any,
+    max_doc_length: Any,
+    noqa: Any,
+    previous_indent_level: Any,
+    previous_logical: Any,
+    previous_unindented_logical_line: Any,
+    tokens: Any,
+    verbose: Any,
+) -> Generator[Tuple[int, str], None, None]:
+    """Run pycodestyle logical checks."""
+    yield from _ambiguous_identifier(logical_line, tokens)
+    yield from _bare_except(logical_line, noqa)
+    yield from _blank_lines(logical_line, blank_lines, indent_level, line_number, blank_before, previous_logical, previous_unindented_logical_line, previous_indent_level, lines)  # noqa: E501
+    yield from _break_after_binary_operator(logical_line, tokens)
+    yield from _break_before_binary_operator(logical_line, tokens)
+    yield from _comparison_negative(logical_line)
+    yield from _comparison_to_singleton(logical_line, noqa)
+    yield from _comparison_type(logical_line, noqa)
+    yield from _compound_statements(logical_line)
+    yield from _continued_indentation(logical_line, tokens, indent_level, hang_closing, indent_char, indent_size, noqa, verbose)  # noqa: E501
+    yield from _explicit_line_join(logical_line, tokens)
+    yield from _extraneous_whitespace(logical_line)
+    yield from _imports_on_separate_lines(logical_line)
+    yield from _indentation(logical_line, previous_logical, indent_char, indent_level, previous_indent_level, indent_size)  # noqa: E501
+    yield from _maximum_doc_length(logical_line, max_doc_length, noqa, tokens)
+    yield from _missing_whitespace(logical_line)
+    yield from _missing_whitespace_after_import_keyword(logical_line)
+    yield from _missing_whitespace_around_operator(logical_line, tokens)
+    yield from _module_imports_on_top_of_file(logical_line, indent_level, checker_state, noqa)  # noqa: E501
+    yield from _python_3000_async_await_keywords(logical_line, tokens)
+    yield from _python_3000_backticks(logical_line)
+    yield from _python_3000_has_key(logical_line, noqa)
+    yield from _python_3000_invalid_escape_sequence(logical_line, tokens, noqa)
+    yield from _python_3000_not_equal(logical_line)
+    yield from _python_3000_raise_comma(logical_line)
+    yield from _whitespace_around_comma(logical_line)
+    yield from _whitespace_around_keywords(logical_line)
+    yield from _whitespace_around_named_parameter_equals(logical_line, tokens)
+    yield from _whitespace_around_operator(logical_line)
+    yield from _whitespace_before_comment(logical_line, tokens)
+    yield from _whitespace_before_parameters(logical_line, tokens)
+
+
+def pycodestyle_physical(
+    indent_char: Any,
+    line_number: Any,
+    lines: Any,
+    max_line_length: Any,
+    multiline: Any,
+    noqa: Any,
+    physical_line: Any,
+    total_lines: Any,
+) -> Generator[Tuple[int, str], None, None]:
+    """Run pycodestyle physical checks."""
+    ret = _maximum_line_length(physical_line, max_line_length, multiline, line_number, noqa)  # noqa: E501
+    if ret is not None:
+        yield ret
+    ret = _tabs_obsolete(physical_line)
+    if ret is not None:
+        yield ret
+    ret = _tabs_or_spaces(physical_line, indent_char)
+    if ret is not None:
+        yield ret
+    ret = _trailing_blank_lines(physical_line, lines, line_number, total_lines)
+    if ret is not None:
+        yield ret
+    ret = _trailing_whitespace(physical_line)
+    if ret is not None:
+        yield ret

tests/integration/test_plugins.py

@@ -104,7 +104,7 @@ def test_local_plugin_can_add_option(local_config):
 
     args = aggregator.aggregate_options(option_manager, cfg, cfg_dir, argv)
 
-    assert args.extended_default_select == {"XE", "F", "E", "C90"}
+    assert args.extended_default_select == {"XE", "F", "E", "W", "C90"}
     assert args.anopt == "foo"
 
 

tests/unit/plugins/pycodestyle_test.py

@@ -0,0 +1,33 @@
+import importlib.machinery
+import importlib.util
+import os.path
+
+import flake8.plugins.pycodestyle
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+
+def test_up_to_date():
+    """Validate that the generated pycodestyle plugin is up to date.
+
+    We generate two "meta" plugins for pycodestyle to avoid calling overhead.
+
+    To regenerate run:
+
+        ./bin/gen-pycodestyle-plugin > src/flake8/plugins/pycodestyle.py
+    """
+
+    path = os.path.join(HERE, "../../../bin/gen-pycodestyle-plugin")
+    name = os.path.basename(path)
+    loader = importlib.machinery.SourceFileLoader(name, path)
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    assert spec is not None
+    mod = importlib.util.module_from_spec(spec)
+    loader.exec_module(mod)
+
+    expected = "".join(f"{line}\n" for line in mod.lines())
+
+    with open(flake8.plugins.pycodestyle.__file__) as f:
+        contents = f.read()
+
+    assert contents == expected