From 9f8fefec82df083c1a9784b7fe37278f843e20ed Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 17 Jan 2025 19:49:25 -0500 Subject: [PATCH 01/10] Initial commit for cli --- cli/clang-tidy-utils/.gitignore | 10 + cli/clang-tidy-utils/README.md | 31 ++++ cli/clang-tidy-utils/pyproject.toml | 50 +++++ .../src/yscope_clang_tidy_utils/cli.py | 175 ++++++++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 cli/clang-tidy-utils/.gitignore create mode 100644 cli/clang-tidy-utils/README.md create mode 100644 cli/clang-tidy-utils/pyproject.toml create mode 100644 cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py diff --git a/cli/clang-tidy-utils/.gitignore b/cli/clang-tidy-utils/.gitignore new file mode 100644 index 0000000..4b8b0df --- /dev/null +++ b/cli/clang-tidy-utils/.gitignore @@ -0,0 +1,10 @@ +# Environment +venv*/ +__pycache__ + +# Packaging +*.egg-info + +# Dev +.mypy_cache +.ruff_cache diff --git a/cli/clang-tidy-utils/README.md b/cli/clang-tidy-utils/README.md new file mode 100644 index 0000000..2d15176 --- /dev/null +++ b/cli/clang-tidy-utils/README.md @@ -0,0 +1,31 @@ +# yscope-clang-tidy-utils + +This project contains CLI scripts for running [clang-tidy][clang-tidy-home] checks. + +## Installation +```shell +python3 -m pip install . +``` +Note: +- Python 3.10 or higher is required. +- It is highly suggested to install in a virtual environment. + +## Usage +```shell +yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] +``` +Note: +- By default, the number of jobs will be set to the number of cores in the running environment. +- Anything after `--` will be considered as clang-tidy arguments and will be directly passed into + clang-tidy. + +## Development: +```shell +pip install -e .[dev] +mypy src +docformatter -i src +black src +ruff check --fix src +``` + +[clang-tidy-home]: https://clang.llvm.org/extra/clang-tidy/ \ No newline at end of file diff --git a/cli/clang-tidy-utils/pyproject.toml b/cli/clang-tidy-utils/pyproject.toml new file mode 100644 index 0000000..af428ff --- /dev/null +++ b/cli/clang-tidy-utils/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["setuptools >= 61.0", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "yscope-clang-tidy-utils" +version = "0.0.1" +authors = [ + { name="zhihao lin", email="lin.zhihao@yscope.com" }, +] +requires-python = ">=3.10" +dependencies = [ + "clang-tidy >= 19.1.0", +] + +[project.optional-dependencies] +dev = [ + "black >= 24.10.0", + "docformatter >= 1.7.5", + "mypy >= 1.14.1", + "ruff >= 0.9.2", +] + +[project.scripts] +yscope-clang-tidy-utils = "yscope_clang_tidy_utils.cli:main" + +[tool.black] +line-length = 100 +target-version = ["py311"] +color = true +preview = true + +[tool.docformatter] +make-summary-multi-line = true +pre-summary-newline = true +recursive = true +wrap-summaries = 100 +wrap-descriptions = 100 + +[tool.mypy] +explicit_package_bases = true +strict = true +pretty = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "I", "F"] +isort.order-by-type = false diff --git a/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py b/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py new file mode 100644 index 0000000..714085a --- /dev/null +++ b/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py @@ -0,0 +1,175 @@ +import argparse +import asyncio +import dataclasses +import multiprocessing +import subprocess +import sys +from typing import List, Optional + + +@dataclasses.dataclass +class ClangTidyResult: + """ + Class that represents clang-tidy's execution results. + """ + + file_name: str + ret_code: int + stdout: str + stderr: str + + +def create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> List[str]: + """ + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: A list of arguments to run clang-tidy to check the given file with the given args. + """ + args: List[str] = ["clang-tidy", file] + args.extend(clang_tidy_args) + return args + + +async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: + """ + Executes a single clang-tidy task by checking one file using a process managed by asyncio. + + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: Execution results represented by an instance of `ClangTidyResult`. + """ + task_args: List[str] = create_clang_tidy_task_arg_list(file, clang_tidy_args) + try: + process = await asyncio.create_subprocess_exec( + *task_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = await process.communicate() + except asyncio.CancelledError: + process.terminate() + await process.wait() + raise + + assert process.returncode is not None + return ClangTidyResult( + file, + process.returncode, + stdout.decode("UTF-8"), + stderr.decode("UTF-8"), + ) + + +async def execute_clang_tidy_task_with_sem( + sem: asyncio.Semaphore, file: str, clang_tidy_args: List[str] +) -> ClangTidyResult: + """ + Wrapper of `execute_clang_tidy_task` with a global semaphore for concurrency control. + + :param sem: The global semaphore for concurrency control. + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: Forwards `execute_clang_tidy_task`'s return values. + """ + async with sem: + return await execute_clang_tidy_task(file, clang_tidy_args) + + +async def clang_tidy_parallel_execution_entry( + num_jobs: int, + files: List[str], + clang_tidy_args: List[str], +) -> int: + """ + Async entry for running clang-tidy checks in parallel. + + :param num_jobs: The maximum number of jobs allowed to run in parallel. + :param files: The list of files to check. :clang_tidy_args: The clang-tidy cli arguments. + """ + sem: asyncio.Semaphore = asyncio.Semaphore(num_jobs) + tasks: List[asyncio.Task[ClangTidyResult]] = [ + asyncio.create_task(execute_clang_tidy_task_with_sem(sem, file, clang_tidy_args)) + for file in files + ] + num_total_files: int = len(files) + + ret_code: int = 0 + try: + for idx, clang_tidy_task in enumerate(asyncio.as_completed(tasks)): + result: ClangTidyResult = await clang_tidy_task + if 0 != result.ret_code: + ret_code = 1 + print(f"[{idx + 1}/{num_total_files}]: {result.file_name}") + print(result.stdout) + print(result.stderr) + except asyncio.CancelledError as e: + print(f"\nAll tasks cancelled: {e}") + for task in tasks: + task.cancel() + + return ret_code + + +def main() -> None: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="yscope-clang-tidy-utils cli options.", + ) + + parser.add_argument( + "-j", + "--num-jobs", + type=int, + help="Number of jobs to run for parallel processing.", + ) + + parser.add_argument( + "input_files", + metavar="FILE", + type=str, + nargs="+", + help="Input files to process.", + ) + + default_parser_usage: str = parser.format_usage() + if default_parser_usage.endswith("\n"): + default_parser_usage = default_parser_usage[:-1] + usage_prefix: str = "usage: " + if default_parser_usage.startswith(usage_prefix): + default_parser_usage = default_parser_usage[len(usage_prefix):] + usage: str = default_parser_usage + " [-- CLANG-TIDY-ARGS ...]" + parser.usage = usage + + args: List[str] = sys.argv[1:] + delimiter_idx: Optional[int] = None + try: + delimiter_idx = args.index("--") + except ValueError: + pass + + cli_args: List[str] = args + clang_tidy_args: List[str] = [] + if delimiter_idx is not None: + cli_args = args[:delimiter_idx] + clang_tidy_args = args[delimiter_idx + 1 :] + + parsed_cli_args: argparse.Namespace = parser.parse_args(cli_args) + + num_jobs: int + if parsed_cli_args.num_jobs is not None: + num_jobs = parsed_cli_args.num_jobs + else: + num_jobs = multiprocessing.cpu_count() + + ret_code: int = 0 + try: + ret_code = asyncio.run( + clang_tidy_parallel_execution_entry( + num_jobs, parsed_cli_args.input_files, clang_tidy_args + ) + ) + except KeyboardInterrupt: + pass + + exit(ret_code) + + +if "__main__" == __name__: + main() From 7a6002ca4dfecf83b8d5836fb46191a9752beadd Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 30 Jan 2025 19:37:18 -0500 Subject: [PATCH 02/10] Let's use uv --- cli/clang-tidy-utils/README.md | 27 ++++++++++++------- cli/clang-tidy-utils/pyproject.toml | 7 +---- .../src/yscope_clang_tidy_utils/cli.py | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cli/clang-tidy-utils/README.md b/cli/clang-tidy-utils/README.md index 2d15176..0692d2d 100644 --- a/cli/clang-tidy-utils/README.md +++ b/cli/clang-tidy-utils/README.md @@ -2,13 +2,20 @@ This project contains CLI scripts for running [clang-tidy][clang-tidy-home] checks. +## Requirements +- [uv] + ## Installation +To install the tool permanently: ```shell -python3 -m pip install . +uv tool install . +``` + +Using virtual environment: +```shell +uv venv +uv pip install . ``` -Note: -- Python 3.10 or higher is required. -- It is highly suggested to install in a virtual environment. ## Usage ```shell @@ -21,11 +28,11 @@ Note: ## Development: ```shell -pip install -e .[dev] -mypy src -docformatter -i src -black src -ruff check --fix src +uv tool run mypy src +uv tool run docformatter -i src +uv tool run black src +uv tool run ruff check --fix src ``` -[clang-tidy-home]: https://clang.llvm.org/extra/clang-tidy/ \ No newline at end of file +[clang-tidy-home]: https://clang.llvm.org/extra/clang-tidy/ +[uv]: https://docs.astral.sh/uv/getting-started/installation/ \ No newline at end of file diff --git a/cli/clang-tidy-utils/pyproject.toml b/cli/clang-tidy-utils/pyproject.toml index af428ff..a910ea6 100644 --- a/cli/clang-tidy-utils/pyproject.toml +++ b/cli/clang-tidy-utils/pyproject.toml @@ -1,7 +1,3 @@ -[build-system] -requires = ["setuptools >= 61.0", "setuptools_scm"] -build-backend = "setuptools.build_meta" - [project] name = "yscope-clang-tidy-utils" version = "0.0.1" @@ -13,7 +9,7 @@ dependencies = [ "clang-tidy >= 19.1.0", ] -[project.optional-dependencies] +[dependency-groups] dev = [ "black >= 24.10.0", "docformatter >= 1.7.5", @@ -26,7 +22,6 @@ yscope-clang-tidy-utils = "yscope_clang_tidy_utils.cli:main" [tool.black] line-length = 100 -target-version = ["py311"] color = true preview = true diff --git a/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py b/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py index 714085a..bcfc431 100644 --- a/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py +++ b/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py @@ -133,7 +133,7 @@ def main() -> None: default_parser_usage = default_parser_usage[:-1] usage_prefix: str = "usage: " if default_parser_usage.startswith(usage_prefix): - default_parser_usage = default_parser_usage[len(usage_prefix):] + default_parser_usage = default_parser_usage[len(usage_prefix) :] usage: str = default_parser_usage + " [-- CLANG-TIDY-ARGS ...]" parser.usage = usage From 5b12b0cbb63078ddb55d55223e96606af8805062 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 30 Jan 2025 19:57:19 -0500 Subject: [PATCH 03/10] Polish readme --- cli/clang-tidy-utils/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/clang-tidy-utils/README.md b/cli/clang-tidy-utils/README.md index 0692d2d..3e97592 100644 --- a/cli/clang-tidy-utils/README.md +++ b/cli/clang-tidy-utils/README.md @@ -1,17 +1,17 @@ # yscope-clang-tidy-utils -This project contains CLI scripts for running [clang-tidy][clang-tidy-home] checks. +This project is a CLI scripts for running [clang-tidy][clang-tidy-home] checks. ## Requirements - [uv] ## Installation -To install the tool permanently: +To install the tool system-wide (permanently), run: ```shell uv tool install . ``` -Using virtual environment: +Or, installing the tool in a virtual environment: ```shell uv venv uv pip install . @@ -19,7 +19,7 @@ uv pip install . ## Usage ```shell -yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] +uv run yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] ``` Note: - By default, the number of jobs will be set to the number of cores in the running environment. @@ -27,6 +27,7 @@ Note: clang-tidy. ## Development: +Run linting tools with the following commands: ```shell uv tool run mypy src uv tool run docformatter -i src @@ -35,4 +36,4 @@ uv tool run ruff check --fix src ``` [clang-tidy-home]: https://clang.llvm.org/extra/clang-tidy/ -[uv]: https://docs.astral.sh/uv/getting-started/installation/ \ No newline at end of file +[uv]: https://docs.astral.sh/uv/getting-started/installation/ From d74ac85c7e90b69e948355c137f691cf04739d4e Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Thu, 30 Jan 2025 20:21:00 -0500 Subject: [PATCH 04/10] Update readme --- cli/clang-tidy-utils/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/clang-tidy-utils/README.md b/cli/clang-tidy-utils/README.md index 3e97592..64197d7 100644 --- a/cli/clang-tidy-utils/README.md +++ b/cli/clang-tidy-utils/README.md @@ -16,10 +16,11 @@ Or, installing the tool in a virtual environment: uv venv uv pip install . ``` +Chech [here] for uv's virtual environment behaviour. ## Usage ```shell -uv run yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] +yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] ``` Note: - By default, the number of jobs will be set to the number of cores in the running environment. @@ -37,3 +38,4 @@ uv tool run ruff check --fix src [clang-tidy-home]: https://clang.llvm.org/extra/clang-tidy/ [uv]: https://docs.astral.sh/uv/getting-started/installation/ +[uv-venv]: https://docs.astral.sh/uv/pip/compatibility/#virtual-environments-by-default From cf407b9c89d061f7b6ee4c9c36f828f119f5f77a Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 10 Feb 2025 13:23:45 -0500 Subject: [PATCH 05/10] Renaming + readme update --- cli/clang-tidy-utils/README.md | 33 +++++++++++-------- cli/clang-tidy-utils/pyproject.toml | 4 +-- .../cli.py | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) rename cli/clang-tidy-utils/src/{yscope_clang_tidy_utils => clang_tidy_utils}/cli.py (98%) diff --git a/cli/clang-tidy-utils/README.md b/cli/clang-tidy-utils/README.md index 3e97592..6eed244 100644 --- a/cli/clang-tidy-utils/README.md +++ b/cli/clang-tidy-utils/README.md @@ -1,33 +1,38 @@ -# yscope-clang-tidy-utils +# clang-tidy-utils -This project is a CLI scripts for running [clang-tidy][clang-tidy-home] checks. +`clang-tidy-utils` is a command-line tool for running [clang-tidy][clang-tidy-home] checks efficiently. ## Requirements -- [uv] +- [uv] (for package and environment management) ## Installation -To install the tool system-wide (permanently), run: -```shell -uv tool install . -``` -Or, installing the tool in a virtual environment: +### Virtual environment installation (Recommended) + +For installation within a virtual environment: ```shell uv venv uv pip install . ``` +### System-wide installation + +To install the tool globally: +```shell +uv tool install . +``` + ## Usage + +Run the tool using: ```shell -uv run yscope-clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] +uv run clang-tidy-utils [-h] [-j NUM_JOBS] FILE [FILE ...] [-- CLANG-TIDY-ARGS ...] ``` -Note: -- By default, the number of jobs will be set to the number of cores in the running environment. -- Anything after `--` will be considered as clang-tidy arguments and will be directly passed into - clang-tidy. +- By default, the number of jobs (`-j`) is set to the number of available CPU cores. +- Arguments after `--` are directly passed to `clang-tidy`. ## Development: -Run linting tools with the following commands: +To format and lint the code, run: ```shell uv tool run mypy src uv tool run docformatter -i src diff --git a/cli/clang-tidy-utils/pyproject.toml b/cli/clang-tidy-utils/pyproject.toml index a910ea6..e06ea2e 100644 --- a/cli/clang-tidy-utils/pyproject.toml +++ b/cli/clang-tidy-utils/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "yscope-clang-tidy-utils" +name = "clang-tidy-utils" version = "0.0.1" authors = [ { name="zhihao lin", email="lin.zhihao@yscope.com" }, @@ -18,7 +18,7 @@ dev = [ ] [project.scripts] -yscope-clang-tidy-utils = "yscope_clang_tidy_utils.cli:main" +clang-tidy-utils = "clang_tidy_utils.cli:main" [tool.black] line-length = 100 diff --git a/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py b/cli/clang-tidy-utils/src/clang_tidy_utils/cli.py similarity index 98% rename from cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py rename to cli/clang-tidy-utils/src/clang_tidy_utils/cli.py index bcfc431..3772bba 100644 --- a/cli/clang-tidy-utils/src/yscope_clang_tidy_utils/cli.py +++ b/cli/clang-tidy-utils/src/clang_tidy_utils/cli.py @@ -110,7 +110,7 @@ async def clang_tidy_parallel_execution_entry( def main() -> None: parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="yscope-clang-tidy-utils cli options.", + description="clang-tidy-utils cli options.", ) parser.add_argument( From 27fcb24c42a04c3fab869e9fc248f779443408d1 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 10 Feb 2025 13:28:22 -0500 Subject: [PATCH 06/10] Project rename --- {cli => linters}/clang-tidy-utils/.gitignore | 0 {cli => linters}/clang-tidy-utils/README.md | 0 .../build/lib}/clang_tidy_utils/cli.py | 0 .../clang-tidy-utils/pyproject.toml | 0 .../src/clang_tidy_utils/cli.py | 175 ++++++++++++++++++ 5 files changed, 175 insertions(+) rename {cli => linters}/clang-tidy-utils/.gitignore (100%) rename {cli => linters}/clang-tidy-utils/README.md (100%) rename {cli/clang-tidy-utils/src => linters/clang-tidy-utils/build/lib}/clang_tidy_utils/cli.py (100%) rename {cli => linters}/clang-tidy-utils/pyproject.toml (100%) create mode 100644 linters/clang-tidy-utils/src/clang_tidy_utils/cli.py diff --git a/cli/clang-tidy-utils/.gitignore b/linters/clang-tidy-utils/.gitignore similarity index 100% rename from cli/clang-tidy-utils/.gitignore rename to linters/clang-tidy-utils/.gitignore diff --git a/cli/clang-tidy-utils/README.md b/linters/clang-tidy-utils/README.md similarity index 100% rename from cli/clang-tidy-utils/README.md rename to linters/clang-tidy-utils/README.md diff --git a/cli/clang-tidy-utils/src/clang_tidy_utils/cli.py b/linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py similarity index 100% rename from cli/clang-tidy-utils/src/clang_tidy_utils/cli.py rename to linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py diff --git a/cli/clang-tidy-utils/pyproject.toml b/linters/clang-tidy-utils/pyproject.toml similarity index 100% rename from cli/clang-tidy-utils/pyproject.toml rename to linters/clang-tidy-utils/pyproject.toml diff --git a/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py new file mode 100644 index 0000000..3772bba --- /dev/null +++ b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py @@ -0,0 +1,175 @@ +import argparse +import asyncio +import dataclasses +import multiprocessing +import subprocess +import sys +from typing import List, Optional + + +@dataclasses.dataclass +class ClangTidyResult: + """ + Class that represents clang-tidy's execution results. + """ + + file_name: str + ret_code: int + stdout: str + stderr: str + + +def create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> List[str]: + """ + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: A list of arguments to run clang-tidy to check the given file with the given args. + """ + args: List[str] = ["clang-tidy", file] + args.extend(clang_tidy_args) + return args + + +async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: + """ + Executes a single clang-tidy task by checking one file using a process managed by asyncio. + + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: Execution results represented by an instance of `ClangTidyResult`. + """ + task_args: List[str] = create_clang_tidy_task_arg_list(file, clang_tidy_args) + try: + process = await asyncio.create_subprocess_exec( + *task_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = await process.communicate() + except asyncio.CancelledError: + process.terminate() + await process.wait() + raise + + assert process.returncode is not None + return ClangTidyResult( + file, + process.returncode, + stdout.decode("UTF-8"), + stderr.decode("UTF-8"), + ) + + +async def execute_clang_tidy_task_with_sem( + sem: asyncio.Semaphore, file: str, clang_tidy_args: List[str] +) -> ClangTidyResult: + """ + Wrapper of `execute_clang_tidy_task` with a global semaphore for concurrency control. + + :param sem: The global semaphore for concurrency control. + :param file: The file to check. + :param clang_tidy_args: The clang-tidy cli arguments. + :return: Forwards `execute_clang_tidy_task`'s return values. + """ + async with sem: + return await execute_clang_tidy_task(file, clang_tidy_args) + + +async def clang_tidy_parallel_execution_entry( + num_jobs: int, + files: List[str], + clang_tidy_args: List[str], +) -> int: + """ + Async entry for running clang-tidy checks in parallel. + + :param num_jobs: The maximum number of jobs allowed to run in parallel. + :param files: The list of files to check. :clang_tidy_args: The clang-tidy cli arguments. + """ + sem: asyncio.Semaphore = asyncio.Semaphore(num_jobs) + tasks: List[asyncio.Task[ClangTidyResult]] = [ + asyncio.create_task(execute_clang_tidy_task_with_sem(sem, file, clang_tidy_args)) + for file in files + ] + num_total_files: int = len(files) + + ret_code: int = 0 + try: + for idx, clang_tidy_task in enumerate(asyncio.as_completed(tasks)): + result: ClangTidyResult = await clang_tidy_task + if 0 != result.ret_code: + ret_code = 1 + print(f"[{idx + 1}/{num_total_files}]: {result.file_name}") + print(result.stdout) + print(result.stderr) + except asyncio.CancelledError as e: + print(f"\nAll tasks cancelled: {e}") + for task in tasks: + task.cancel() + + return ret_code + + +def main() -> None: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="clang-tidy-utils cli options.", + ) + + parser.add_argument( + "-j", + "--num-jobs", + type=int, + help="Number of jobs to run for parallel processing.", + ) + + parser.add_argument( + "input_files", + metavar="FILE", + type=str, + nargs="+", + help="Input files to process.", + ) + + default_parser_usage: str = parser.format_usage() + if default_parser_usage.endswith("\n"): + default_parser_usage = default_parser_usage[:-1] + usage_prefix: str = "usage: " + if default_parser_usage.startswith(usage_prefix): + default_parser_usage = default_parser_usage[len(usage_prefix) :] + usage: str = default_parser_usage + " [-- CLANG-TIDY-ARGS ...]" + parser.usage = usage + + args: List[str] = sys.argv[1:] + delimiter_idx: Optional[int] = None + try: + delimiter_idx = args.index("--") + except ValueError: + pass + + cli_args: List[str] = args + clang_tidy_args: List[str] = [] + if delimiter_idx is not None: + cli_args = args[:delimiter_idx] + clang_tidy_args = args[delimiter_idx + 1 :] + + parsed_cli_args: argparse.Namespace = parser.parse_args(cli_args) + + num_jobs: int + if parsed_cli_args.num_jobs is not None: + num_jobs = parsed_cli_args.num_jobs + else: + num_jobs = multiprocessing.cpu_count() + + ret_code: int = 0 + try: + ret_code = asyncio.run( + clang_tidy_parallel_execution_entry( + num_jobs, parsed_cli_args.input_files, clang_tidy_args + ) + ) + except KeyboardInterrupt: + pass + + exit(ret_code) + + +if "__main__" == __name__: + main() From a6ca5f36994806100b44ec40cfb3cbc56ce97bc8 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 10 Feb 2025 13:34:32 -0500 Subject: [PATCH 07/10] Rename symbols --- .../clang-tidy-utils/src/clang_tidy_utils/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py index 3772bba..9130e62 100644 --- a/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py +++ b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py @@ -19,7 +19,7 @@ class ClangTidyResult: stderr: str -def create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> List[str]: +def _create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> List[str]: """ :param file: The file to check. :param clang_tidy_args: The clang-tidy cli arguments. @@ -30,7 +30,7 @@ def create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> Li return args -async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: +async def _execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: """ Executes a single clang-tidy task by checking one file using a process managed by asyncio. @@ -38,7 +38,7 @@ async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> Clan :param clang_tidy_args: The clang-tidy cli arguments. :return: Execution results represented by an instance of `ClangTidyResult`. """ - task_args: List[str] = create_clang_tidy_task_arg_list(file, clang_tidy_args) + task_args: List[str] = _create_clang_tidy_task_arg_list(file, clang_tidy_args) try: process = await asyncio.create_subprocess_exec( *task_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -58,7 +58,7 @@ async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> Clan ) -async def execute_clang_tidy_task_with_sem( +async def _execute_clang_tidy_task_with_sem( sem: asyncio.Semaphore, file: str, clang_tidy_args: List[str] ) -> ClangTidyResult: """ @@ -70,10 +70,10 @@ async def execute_clang_tidy_task_with_sem( :return: Forwards `execute_clang_tidy_task`'s return values. """ async with sem: - return await execute_clang_tidy_task(file, clang_tidy_args) + return await _execute_clang_tidy_task(file, clang_tidy_args) -async def clang_tidy_parallel_execution_entry( +async def _clang_tidy_parallel_execution_entry( num_jobs: int, files: List[str], clang_tidy_args: List[str], @@ -86,7 +86,7 @@ async def clang_tidy_parallel_execution_entry( """ sem: asyncio.Semaphore = asyncio.Semaphore(num_jobs) tasks: List[asyncio.Task[ClangTidyResult]] = [ - asyncio.create_task(execute_clang_tidy_task_with_sem(sem, file, clang_tidy_args)) + asyncio.create_task(_execute_clang_tidy_task_with_sem(sem, file, clang_tidy_args)) for file in files ] num_total_files: int = len(files) @@ -161,7 +161,7 @@ def main() -> None: ret_code: int = 0 try: ret_code = asyncio.run( - clang_tidy_parallel_execution_entry( + _clang_tidy_parallel_execution_entry( num_jobs, parsed_cli_args.input_files, clang_tidy_args ) ) From 817fdefdbbd78b94cd046e9ec2b9c71e8ec6b015 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Mon, 10 Feb 2025 13:37:48 -0500 Subject: [PATCH 08/10] Remove build dir --- .../build/lib/clang_tidy_utils/cli.py | 175 ------------------ 1 file changed, 175 deletions(-) delete mode 100644 linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py diff --git a/linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py b/linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py deleted file mode 100644 index 3772bba..0000000 --- a/linters/clang-tidy-utils/build/lib/clang_tidy_utils/cli.py +++ /dev/null @@ -1,175 +0,0 @@ -import argparse -import asyncio -import dataclasses -import multiprocessing -import subprocess -import sys -from typing import List, Optional - - -@dataclasses.dataclass -class ClangTidyResult: - """ - Class that represents clang-tidy's execution results. - """ - - file_name: str - ret_code: int - stdout: str - stderr: str - - -def create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> List[str]: - """ - :param file: The file to check. - :param clang_tidy_args: The clang-tidy cli arguments. - :return: A list of arguments to run clang-tidy to check the given file with the given args. - """ - args: List[str] = ["clang-tidy", file] - args.extend(clang_tidy_args) - return args - - -async def execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: - """ - Executes a single clang-tidy task by checking one file using a process managed by asyncio. - - :param file: The file to check. - :param clang_tidy_args: The clang-tidy cli arguments. - :return: Execution results represented by an instance of `ClangTidyResult`. - """ - task_args: List[str] = create_clang_tidy_task_arg_list(file, clang_tidy_args) - try: - process = await asyncio.create_subprocess_exec( - *task_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = await process.communicate() - except asyncio.CancelledError: - process.terminate() - await process.wait() - raise - - assert process.returncode is not None - return ClangTidyResult( - file, - process.returncode, - stdout.decode("UTF-8"), - stderr.decode("UTF-8"), - ) - - -async def execute_clang_tidy_task_with_sem( - sem: asyncio.Semaphore, file: str, clang_tidy_args: List[str] -) -> ClangTidyResult: - """ - Wrapper of `execute_clang_tidy_task` with a global semaphore for concurrency control. - - :param sem: The global semaphore for concurrency control. - :param file: The file to check. - :param clang_tidy_args: The clang-tidy cli arguments. - :return: Forwards `execute_clang_tidy_task`'s return values. - """ - async with sem: - return await execute_clang_tidy_task(file, clang_tidy_args) - - -async def clang_tidy_parallel_execution_entry( - num_jobs: int, - files: List[str], - clang_tidy_args: List[str], -) -> int: - """ - Async entry for running clang-tidy checks in parallel. - - :param num_jobs: The maximum number of jobs allowed to run in parallel. - :param files: The list of files to check. :clang_tidy_args: The clang-tidy cli arguments. - """ - sem: asyncio.Semaphore = asyncio.Semaphore(num_jobs) - tasks: List[asyncio.Task[ClangTidyResult]] = [ - asyncio.create_task(execute_clang_tidy_task_with_sem(sem, file, clang_tidy_args)) - for file in files - ] - num_total_files: int = len(files) - - ret_code: int = 0 - try: - for idx, clang_tidy_task in enumerate(asyncio.as_completed(tasks)): - result: ClangTidyResult = await clang_tidy_task - if 0 != result.ret_code: - ret_code = 1 - print(f"[{idx + 1}/{num_total_files}]: {result.file_name}") - print(result.stdout) - print(result.stderr) - except asyncio.CancelledError as e: - print(f"\nAll tasks cancelled: {e}") - for task in tasks: - task.cancel() - - return ret_code - - -def main() -> None: - parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="clang-tidy-utils cli options.", - ) - - parser.add_argument( - "-j", - "--num-jobs", - type=int, - help="Number of jobs to run for parallel processing.", - ) - - parser.add_argument( - "input_files", - metavar="FILE", - type=str, - nargs="+", - help="Input files to process.", - ) - - default_parser_usage: str = parser.format_usage() - if default_parser_usage.endswith("\n"): - default_parser_usage = default_parser_usage[:-1] - usage_prefix: str = "usage: " - if default_parser_usage.startswith(usage_prefix): - default_parser_usage = default_parser_usage[len(usage_prefix) :] - usage: str = default_parser_usage + " [-- CLANG-TIDY-ARGS ...]" - parser.usage = usage - - args: List[str] = sys.argv[1:] - delimiter_idx: Optional[int] = None - try: - delimiter_idx = args.index("--") - except ValueError: - pass - - cli_args: List[str] = args - clang_tidy_args: List[str] = [] - if delimiter_idx is not None: - cli_args = args[:delimiter_idx] - clang_tidy_args = args[delimiter_idx + 1 :] - - parsed_cli_args: argparse.Namespace = parser.parse_args(cli_args) - - num_jobs: int - if parsed_cli_args.num_jobs is not None: - num_jobs = parsed_cli_args.num_jobs - else: - num_jobs = multiprocessing.cpu_count() - - ret_code: int = 0 - try: - ret_code = asyncio.run( - clang_tidy_parallel_execution_entry( - num_jobs, parsed_cli_args.input_files, clang_tidy_args - ) - ) - except KeyboardInterrupt: - pass - - exit(ret_code) - - -if "__main__" == __name__: - main() From 62edc76c8e0156d2946e60840e1b3c4132d9e833 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Tue, 22 Apr 2025 20:46:09 -0400 Subject: [PATCH 09/10] Lock clang-tidy version to 19 --- linters/clang-tidy-utils/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linters/clang-tidy-utils/pyproject.toml b/linters/clang-tidy-utils/pyproject.toml index e06ea2e..404ef37 100644 --- a/linters/clang-tidy-utils/pyproject.toml +++ b/linters/clang-tidy-utils/pyproject.toml @@ -6,7 +6,7 @@ authors = [ ] requires-python = ">=3.10" dependencies = [ - "clang-tidy >= 19.1.0", + "clang-tidy ~= 19.1.0", ] [dependency-groups] From 548ff02ebf6dd70251085f19e4168735c37abf53 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Wed, 13 Aug 2025 12:37:06 -0400 Subject: [PATCH 10/10] Update the cli script. --- .../src/clang_tidy_utils/cli.py | 47 ++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py index 9130e62..3611b8e 100644 --- a/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py +++ b/linters/clang-tidy-utils/src/clang_tidy_utils/cli.py @@ -2,6 +2,7 @@ import asyncio import dataclasses import multiprocessing +import os import subprocess import sys from typing import List, Optional @@ -27,9 +28,34 @@ def _create_clang_tidy_task_arg_list(file: str, clang_tidy_args: List[str]) -> L """ args: List[str] = ["clang-tidy", file] args.extend(clang_tidy_args) + # Enforce `warning-as-error` for all checks: + args.append("-warnings-as-errors=*") return args +def _collect_target_files(input_paths: List[str]) -> List[str]: + """ + Collect all the target files to lint, including all C++ source/header files under the input + directory. + + :param input_paths: The input paths. + :return: A list of target files. + """ + target_files = [] + for path in input_paths: + if os.path.isfile(path): + target_files.append(os.path.abspath(path)) + continue + if not os.path.isdir(path): + continue + for root, _, files in os.walk(path): + for name in files: + if name.endswith('.cpp') or name.endswith('.hpp'): + full_path = os.path.join(root, name) + target_files.append(os.path.abspath(full_path)) + return target_files + + async def _execute_clang_tidy_task(file: str, clang_tidy_args: List[str]) -> ClangTidyResult: """ Executes a single clang-tidy task by checking one file using a process managed by asyncio. @@ -82,7 +108,8 @@ async def _clang_tidy_parallel_execution_entry( Async entry for running clang-tidy checks in parallel. :param num_jobs: The maximum number of jobs allowed to run in parallel. - :param files: The list of files to check. :clang_tidy_args: The clang-tidy cli arguments. + :param files: The list of files to check. + :param clang_tidy_args: The clang-tidy cli arguments. """ sem: asyncio.Semaphore = asyncio.Semaphore(num_jobs) tasks: List[asyncio.Task[ClangTidyResult]] = [ @@ -97,9 +124,11 @@ async def _clang_tidy_parallel_execution_entry( result: ClangTidyResult = await clang_tidy_task if 0 != result.ret_code: ret_code = 1 - print(f"[{idx + 1}/{num_total_files}]: {result.file_name}") - print(result.stdout) - print(result.stderr) + print(f"[{idx + 1}/{num_total_files}]: {result.file_name}") + print(result.stdout) + print(result.stderr) + else: + print(f"[{idx + 1}/{num_total_files}]: {result.file_name} [All check passed!]") except asyncio.CancelledError as e: print(f"\nAll tasks cancelled: {e}") for task in tasks: @@ -162,13 +191,19 @@ def main() -> None: try: ret_code = asyncio.run( _clang_tidy_parallel_execution_entry( - num_jobs, parsed_cli_args.input_files, clang_tidy_args + num_jobs, _collect_target_files(parsed_cli_args.input_files), clang_tidy_args ) ) except KeyboardInterrupt: pass - exit(ret_code) + if 0 != ret_code: + # Ideally, we should return the error code directly. However, this utility is used inside + # the GitHub action, and we don't want to fail the workflow. We log the error code instead + # if it's not 0. + print(f"\nclang-tidy-utils: Linter check failed with return code {ret_code}") + + exit(0) if "__main__" == __name__: