Shift Left: Try building and testing selective_build in OSS (#14499)

Summary:
- Enable buck targets in OSS under codegen/tools/targets.bzl
- Build all buck targets in OSS under codegen/tools/... (Can't test yet due to torch not available in OSS buck)
- Run test_tools_selective_build.py in pytest
- Building on top of #14488

Pull Request resolved: #14499

Reviewed By: lucylq

Differential Revision: D83024265

Pulled By: mergennachin

Author: mergennachin

.ci/scripts/unittest-buck2.sh

@@ -15,7 +15,8 @@ buck2 query "//backends/apple/... + //backends/arm: + //backends/arm/debug/... +
 //backends/arm/_passes/... + //backends/arm/runtime/... + //backends/arm/tosa/... \
 + //backends/example/... + \
 //backends/mediatek/... + //backends/transforms/... + \
-//backends/xnnpack/... + //configurations/... + //extension/flat_tensor: + \
+//backends/xnnpack/... + //codegen/tools/... + \
+//configurations/... + //extension/flat_tensor: + \
 //extension/llm/runner: + //kernels/aten/... + //kernels/optimized/... + \
 //kernels/portable/... + //kernels/quantized/... + //kernels/test/... + \
 //runtime/... + //schema/... + //test/... + //util/..."
@@ -38,3 +39,6 @@ for op in "build" "test"; do
           $BUILDABLE_KERNELS_PRIM_OPS_TARGETS //runtime/backend/... //runtime/core/... \
           //runtime/executor: //runtime/kernel/... //runtime/platform/...
 done
+
+# Build only without testing
+buck2 build //codegen/tools/... # Needs torch for testing which we don't have in our OSS buck setup.

codegen/tools/targets.bzl

@@ -17,10 +17,8 @@ def define_common_targets(is_fbcode = False):
         ],
         deps = [
             "//executorch/codegen:gen_lib",
-        ] + ([] if runtime.is_oss else select({
-            "DEFAULT": [],
-            "ovr_config//os:linux": ["//executorch/codegen/tools:selective_build"],  # TODO(larryliu0820) :selective_build doesn't build in OSS yet
-        })),
+            "//executorch/codegen/tools:selective_build",
+        ],
     )
 
     runtime.python_binary(
@@ -29,7 +27,7 @@ def define_common_targets(is_fbcode = False):
         deps = [
             ":gen_oplist_lib",
         ],
-        preload_deps = [] if runtime.is_oss else ["//executorch/codegen/tools:selective_build"],  # TODO(larryliu0820) :selective_build doesn't build in OSS yet
+        preload_deps = ["//executorch/codegen/tools:selective_build"],
         package_style = "inplace",
         visibility = [
             "//executorch/...",
@@ -196,27 +194,27 @@ def define_common_targets(is_fbcode = False):
         _is_external_target = True,
     )
 
-    if not runtime.is_oss:
-        runtime.cxx_python_extension(
-            name = "selective_build",
-            srcs = [
-                "selective_build.cpp",
-            ],
-            base_module = "executorch.codegen.tools",
-            types = ["selective_build.pyi"],
-            preprocessor_flags = [
-                "-DEXECUTORCH_PYTHON_MODULE_NAME=selective_build",
-            ],
-            deps = [
-                "//executorch/runtime/core:core",
-                "//executorch/schema:program",
-            ],
-            external_deps = [
-                "pybind11",
-            ],
-            use_static_deps = True,
-            visibility = ["//executorch/codegen/..."],
-        )
+    
+    runtime.cxx_python_extension(
+        name = "selective_build",
+        srcs = [
+            "selective_build.cpp",
+        ],
+        base_module = "executorch.codegen.tools",
+        types = ["selective_build.pyi"],
+        preprocessor_flags = [
+            "-DEXECUTORCH_PYTHON_MODULE_NAME=selective_build",
+        ],
+        deps = [
+            "//executorch/runtime/core:core",
+            "//executorch/schema:program",
+        ],
+        external_deps = [
+            "pybind11",
+        ],
+        use_static_deps = True,
+        visibility = ["//executorch/codegen/..."],
+    )
 
 
     # TODO(larryliu0820): This is a hack to only run these two on fbcode. These targets depends on exir which is only available in fbcode.
@@ -255,10 +253,12 @@ def define_common_targets(is_fbcode = False):
             ],
         )
 
+    if runtime.is_oss or is_fbcode:
+        # Doesn't work on xplat. But works on fbcode and OSS.
         runtime.python_test(
-            name = "test_selective_build",
+            name = "test_tools_selective_build",
             srcs = [
-                "test/test_selective_build.py",
+                "test/test_tools_selective_build.py",
             ],
             package_style = "inplace",
             visibility = [

codegen/tools/test/test_selective_build.py renamed to codegen/tools/test/test_tools_selective_build.py

@@ -15,9 +15,11 @@ runtime.cxx_python_extension(
     ],
     base_module = "executorch.extension.pytree",
     deps = [
-        "fbsource//third-party/pybind11:pybind11",
         ":pytree",
     ],
+    external_deps = [
+        "pybind11",
+    ],
 )
 
 runtime.cxx_python_extension(
@@ -27,9 +29,11 @@ runtime.cxx_python_extension(
     ],
     base_module = "executorch.extension.pytree",
     deps = [
-        "fbsource//third-party/pybind11:pybind11",
         ":pytree",
     ],
+    external_deps = [
+        "pybind11",
+    ],
 )
 
 runtime.python_library(

extension/pytree/TARGETS

@@ -31,6 +31,7 @@ addopts =
 
     # codegen
     codegen/test
+    codegen/tools/test/test_tools_selective_build.py
 
     # devtools
     devtools/

pytest.ini

@@ -346,6 +346,7 @@ def _python_binary(*args, **kwargs):
 
 def _python_test(*args, **kwargs):
     _patch_kwargs_common(kwargs)
+    _remove_caffe2_deps(kwargs)
     env.python_test(*args, **kwargs)
 
 def get_oss_build_kwargs():