adding arc lint opt in for benchmarking (#2448)

Summary:
Pull Request resolved: #2448

lets see if arc lint runs here

Reviewed By: zertosh

Differential Revision: D61929352

fbshipit-source-id: 4651d3dfa2eb743f5b0e30954765c43c77447ee2

Author: adamomainz

.clang-format

@@ -0,0 +1,5 @@
+---
+Language: ObjC
+DisableFormat: true
+SortIncludes: false
+...

install.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 
 from userbenchmark import list_userbenchmarks
-from utils import get_pkg_versions, TORCH_DEPS, generate_pkg_constraints
+from utils import generate_pkg_constraints, get_pkg_versions, TORCH_DEPS
 from utils.python_utils import pip_install_requirements
 
 REPO_ROOT = Path(__file__).parent
@@ -24,26 +24,21 @@
         action="store_true",
         help="Run in test mode and check package versions",
     )
-    parser.add_argument(
-        "--skip",
-        nargs="*",
-        default=[],
-        help="Skip models to install."
-    )
+    parser.add_argument("--skip", nargs="*", default=[], help="Skip models to install.")
     parser.add_argument(
         "--torch",
         action="store_true",
-        help="Only require torch to be installed, ignore torchvision and torchaudio."
+        help="Only require torch to be installed, ignore torchvision and torchaudio.",
     )
     parser.add_argument(
         "--numpy",
         action="store_true",
-        help="Only require numpy to be installed, ignore torch, torchvision and torchaudio."
+        help="Only require numpy to be installed, ignore torch, torchvision and torchaudio.",
     )
     parser.add_argument(
         "--check-only",
         action="store_true",
-        help="Only run the version check and generate the contraints"
+        help="Only run the version check and generate the contraints",
     )
     parser.add_argument("--canary", action="store_true", help="Install canary model.")
     parser.add_argument("--continue_on_fail", action="store_true")
@@ -86,14 +81,18 @@
         # Install userbenchmark dependencies if exists
         userbenchmark_dir = REPO_ROOT.joinpath("userbenchmark", args.userbenchmark)
         cmd = [sys.executable, "install.py"]
-        print(f"Installing userbenchmark {args.userbenchmark} with extra args: {extra_args}")
+        print(
+            f"Installing userbenchmark {args.userbenchmark} with extra args: {extra_args}"
+        )
         cmd.extend(extra_args)
         if userbenchmark_dir.joinpath("install.py").is_file():
             # add the current run env to PYTHONPATH to load framework install utils
             run_env = os.environ.copy()
             run_env["PYTHONPATH"] = Path(REPO_ROOT).as_posix()
             subprocess.check_call(
-                cmd, cwd=userbenchmark_dir.absolute(), env=run_env,
+                cmd,
+                cwd=userbenchmark_dir.absolute(),
+                env=run_env,
             )
         sys.exit(0)
 

run_benchmark.py

@@ -24,7 +24,9 @@ def list_benchmarks() -> Dict[str, str]:
 
 def run():
     available_benchmarks = list_benchmarks()
-    parser = argparse.ArgumentParser(description="Run a TorchBench user benchmark", add_help=False)
+    parser = argparse.ArgumentParser(
+        description="Run a TorchBench user benchmark", add_help=False
+    )
     parser.add_argument(
         "bm_name",
         choices=available_benchmarks.keys(),

scripts/userbenchmark/upload_s3_csv.py

@@ -1,12 +1,13 @@
 import argparse
-import sys
 import os
 import re
-from pathlib import Path
+import sys
 from datetime import datetime
+from pathlib import Path
 
 REPO_ROOT = Path(__file__).parent.parent.parent.resolve()
 
+
 class add_path:
     def __init__(self, path):
         self.path = path
@@ -22,18 +23,17 @@ def __exit__(self, exc_type, exc_value, traceback):
 
 
 with add_path(str(REPO_ROOT)):
-    from utils.s3_utils import (
-        S3Client,
-        USERBENCHMARK_S3_BUCKET,
-    )
+    from utils.s3_utils import S3Client, USERBENCHMARK_S3_BUCKET
 
 
-def upload_s3(s3_object: str, 
-              ub_name: str,
-              workflow_run_id: str,
-              workflow_run_attempt: str,
-              file_path: Path,
-              dryrun: bool):
+def upload_s3(
+    s3_object: str,
+    ub_name: str,
+    workflow_run_id: str,
+    workflow_run_attempt: str,
+    file_path: Path,
+    dryrun: bool,
+):
     """S3 path:
     s3://ossci-metrics/<s3_object>/<ub_name>/<workflow_run_id>/<workflow_run_attempt>/file_name
     """
@@ -46,7 +46,12 @@ def upload_s3(s3_object: str,
 
 def _get_files_to_upload(file_path: str, match_filename: str):
     filename_regex = re.compile(match_filename)
-    return [ file_name for file_name in os.listdir(file_path) if filename_regex.match(file_name) ]
+    return [
+        file_name
+        for file_name in os.listdir(file_path)
+        if filename_regex.match(file_name)
+    ]
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description=__doc__)
@@ -83,9 +88,11 @@ def _get_files_to_upload(file_path: str, match_filename: str):
 
     for file in files_to_upload:
         file_path = Path(args.upload_path).joinpath(file)
-        upload_s3(s3_object=args.s3_prefix, 
-                  ub_name=args.userbenchmark,
-                  workflow_run_id=workflow_run_id,
-                  workflow_run_attempt=workflow_run_attempt,
-                  file_path=file_path,
-                  dryrun=args.dryrun)
+        upload_s3(
+            s3_object=args.s3_prefix,
+            ub_name=args.userbenchmark,
+            workflow_run_id=workflow_run_id,
+            workflow_run_attempt=workflow_run_attempt,
+            file_path=file_path,
+            dryrun=args.dryrun,
+        )

test_bench.py

@@ -113,11 +113,9 @@ def test_eval(self, model_path, device, benchmark, pytestconfig):
 
             benchmark(task.invoke)
             benchmark.extra_info["machine_state"] = get_machine_state()
-            benchmark.extra_info["batch_size"] = task.get_model_attribute(
-                 "batch_size"
-            )
+            benchmark.extra_info["batch_size"] = task.get_model_attribute("batch_size")
             benchmark.extra_info["precision"] = task.get_model_attribute(
-                 "dargs", "precision"
+                "dargs", "precision"
             )
             benchmark.extra_info["test"] = "eval"
 

torchbenchmark/__init__.py

@@ -42,6 +42,7 @@ def __exit__(self, exc_type, exc_value, traceback):
         except ValueError:
             pass
 
+
 class add_ld_library_path:
     def __init__(self, path):
         self.path = path
@@ -57,6 +58,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_value, traceback):
         os.environ = self.os_environ.copy()
 
+
 with add_path(str(REPO_PATH)):
     from utils import get_pkg_versions, TORCH_DEPS
 
@@ -192,7 +194,7 @@ def setup(
         model_paths = list(model_paths)
         model_paths.extend(canary_model_paths)
     skip_models = [] if not skip_models else skip_models
-    model_paths = [ x for x in model_paths if os.path.basename(x) not in skip_models ]
+    model_paths = [x for x in model_paths if os.path.basename(x) not in skip_models]
     for model_path in model_paths:
         print(f"running setup for {model_path}...", end="", flush=True)
         if test_mode:
@@ -346,6 +348,7 @@ def _maybe_import_model(package: str, model_name: str) -> Dict[str, Any]:
         import importlib
         import os
         import traceback
+
         from torchbenchmark import load_model_by_name
 
         diagnostic_msg = ""
@@ -589,7 +592,10 @@ def watch_cuda_memory(
 
 
 def list_models_details(workers: int = 1) -> List[ModelDetails]:
-    return [ModelTask(os.path.basename(model_path)).model_details for model_path in _list_model_paths()]
+    return [
+        ModelTask(os.path.basename(model_path)).model_details
+        for model_path in _list_model_paths()
+    ]
 
 
 def list_models(model_match=None):

torchbenchmark/_components/_impl/tasks/base.py

@@ -1,4 +1,5 @@
 """Add Task abstraction to reduce the friction of controlling a remote worker."""
+
 import abc
 import ast
 import functools
@@ -19,8 +20,7 @@ class TaskBase(abc.ABC):
     """
 
     @abc.abstractproperty
-    def worker(self) -> base.WorkerBase:
-        ...
+    def worker(self) -> base.WorkerBase: ...
 
 
 def parse_f(f: typing.Callable) -> typing.Tuple[inspect.Signature, str]:
@@ -41,12 +41,14 @@ def parse_f(f: typing.Callable) -> typing.Tuple[inspect.Signature, str]:
         if arg_parameter.kind == inspect.Parameter.VAR_POSITIONAL:
             raise TypeError(
                 f"Variadic positional argument `*{arg}` not permitted "
-                "for `run_in_worker` function.")
+                "for `run_in_worker` function."
+            )
 
         if arg_parameter.kind == inspect.Parameter.VAR_KEYWORD:
             raise TypeError(
                 f"Variadic keywork argument `**{arg}` not permitted "
-                "for `run_in_worker` function.")
+                "for `run_in_worker` function."
+            )
 
         if arg_parameter.annotation == inspect.Parameter.empty:
             raise TypeError(f"Missing type annotation for parameter `{arg}`")
@@ -84,7 +86,9 @@ def parse_f(f: typing.Callable) -> typing.Tuple[inspect.Signature, str]:
     # `functools.wraps` is not detectable (by design) and is thus caveat
     # emptor.
     if getattr(f, "__wrapped__", None):
-        raise TypeError(textwrap.dedent("""
+        raise TypeError(
+            textwrap.dedent(
+                """
             `f` cannot be decorated below `@run_in_worker` (except for
             @staticmethod) because the extraction logic would not carry through
             said decorator(s).
@@ -100,7 +104,9 @@ def foo() -> None:
                 @my_decorator
                 def foo() -> None:
                     ...
-        """).strip())
+        """
+            ).strip()
+        )
 
     # Dedent, as `f` may have been defined in a scoped context.
     f_src = textwrap.dedent(inspect.getsource(f))
@@ -127,10 +133,12 @@ def foo() -> None:
         # line comment), we simply elect to skip over them and index on the
         # first node that will give valid indices.
         if node.col_offset == -1:
-            assert isinstance(node.value, ast.Str), f"Expected `ast.Str`, got {type(node)}. ({node}) {node.lineno}"
+            assert isinstance(
+                node.value, ast.Str
+            ), f"Expected `ast.Str`, got {type(node)}. ({node}) {node.lineno}"
             continue
 
-        raw_body_lines = src_lines[node.lineno - 1:]
+        raw_body_lines = src_lines[node.lineno - 1 :]
         col_offset = node.col_offset
         break
 
@@ -229,16 +237,15 @@ def outer(f: typing.Callable[..., typing.Any]) -> typing.Callable[..., typing.An
             pass
 
         signature, f_body = parse_f(f)
-        has_return_value = (signature.return_annotation is not None)
+        has_return_value = signature.return_annotation is not None
         if has_return_value and not scoped:
             raise TypeError(
-                "Unscoped (globally executed) call can not have a return value.")
+                "Unscoped (globally executed) call can not have a return value."
+            )
 
         @functools.wraps(f)
         def inner(
-            self: TaskBase,
-            *args: typing.Any,
-            **kwargs: typing.Any
+            self: TaskBase, *args: typing.Any, **kwargs: typing.Any
         ) -> typing.Any:
             bound_signature = signature.bind(*args, **kwargs)
             bound_signature.apply_defaults()
@@ -250,13 +257,17 @@ def inner(
                 except ValueError:
                     raise ValueError(f"unmarshallable arg {arg_name}: {arg_value}")
 
-                body.append(f"{arg_name} = marshal.loads(bytes.fromhex({repr(arg_bytes.hex())}))  # {arg_value}")
+                body.append(
+                    f"{arg_name} = marshal.loads(bytes.fromhex({repr(arg_bytes.hex())}))  # {arg_value}"
+                )
             body.extend(["", "# Wrapped source"] + f_body.splitlines(keepends=False))
 
-            src = "\n".join([
-                "def _run_in_worker_f():",
-                textwrap.indent("\n".join(body), " " * 4),
-                textwrap.dedent("""
+            src = "\n".join(
+                [
+                    "def _run_in_worker_f():",
+                    textwrap.indent("\n".join(body), " " * 4),
+                    textwrap.dedent(
+                        """
                 try:
                     # Clear prior value if it exists.
                     del _run_in_worker_result
@@ -265,8 +276,10 @@ def inner(
                     pass
 
                 _run_in_worker_result = _run_in_worker_f()
-                """)
-            ])
+                """
+                    ),
+                ]
+            )
 
             # `worker.load` is not free, so for void functions we skip it.
             if has_return_value:
@@ -278,4 +291,5 @@ def inner(
                 self.worker.run(src)
 
         return inner
+
     return outer