Write Linux CMake pytest test-run reports

.ci/scripts/tests/test_torchci_report_args.py

@@ -0,0 +1,172 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+HELPER = ROOT / ".ci/scripts/torchci-report-args.sh"
+PROBE = """
+import os
+import sys
+from types import ModuleType, SimpleNamespace
+
+sys.modules['pytest'] = SimpleNamespace(__version__=os.environ['PROBE_PYTEST'])
+for name in ('torch', 'torch.testing', 'torch.testing._internal',
+             'torch.testing._internal.torchci'):
+    sys.modules[name] = ModuleType(name)
+torchci = sys.modules['torch.testing._internal.torchci']
+def capture():
+    os.environ['TORCHCI_POLLUTION'] = 'child-only'
+    if os.environ.get('PROBE_CAPTURE_ERROR'):
+        raise RuntimeError('capture failed')
+    print('CAPTURE')
+torchci.environment = SimpleNamespace(capture=capture)
+torchci.report = SimpleNamespace(SCHEMA_VERSION=os.environ['PROBE_SCHEMA'])
+if not os.environ.get('PROBE_MISSING_PLUGIN'):
+    torchci.plugin = ModuleType('plugin')
+exec(sys.argv[2])
+"""
+
+
+class TorchciReportArgumentsTest(unittest.TestCase):
+    def run_helper(self, extra=None, command=None):
+        with tempfile.TemporaryDirectory(dir=ROOT, prefix="torchci-test-") as tmp:
+            probe = Path(tmp) / "probe.py"
+            probe.write_text(PROBE)
+            env = dict(os.environ)
+            for name in (
+                "RUNNER_TEST_RUN_REPORTS_DIR",
+                "JOB_ID",
+                "GITHUB_REPOSITORY",
+                "TORCHCI_PYTHON",
+                "PYTHON_EXECUTABLE",
+                "TORCHCI_POLLUTION",
+                "PYTHONOPTIMIZE",
+            ):
+                env.pop(name, None)
+            env.update(
+                PROBE_PYTEST="7.3.0",
+                PROBE_SCHEMA="1.0",
+                TORCHCI_PYTHON=f"{sys.executable} -B {probe}",
+            )
+            env.update(extra or {})
+            command = command or (
+                "printf 'ARG <%s>\\n' "
+                '${TORCHCI_REPORT_ARGS[@]+"${TORCHCI_REPORT_ARGS[@]}"}; '
+                'printf "POLLUTION <%s>\\n" "${TORCHCI_POLLUTION:-}"'
+            )
+            return subprocess.run(
+                [
+                    "bash",
+                    "-c",
+                    f"set -eu; source {shlex.quote(str(HELPER))} smoke; {command}",
+                ],
+                env=env,
+                capture_output=True,
+                text=True,
+            )
+
+    def opted_env(self, **extra):
+        return {
+            "RUNNER_TEST_RUN_REPORTS_DIR": "/tmp/torchci reports",
+            "JOB_ID": "51234567890",
+            "GITHUB_REPOSITORY": "pytorch/executorch",
+            **extra,
+        }
+
+    def test_unopted_run_does_not_start_an_interpreter(self):
+        for directory in (None, ""):
+            with self.subTest(directory=directory):
+                extra = {"TORCHCI_PYTHON": "/missing/python"}
+                if directory is not None:
+                    extra["RUNNER_TEST_RUN_REPORTS_DIR"] = directory
+                result = self.run_helper(extra)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                self.assertNotIn("CAPTURE", result.stdout)
+                self.assertNotIn("torchci.plugin", result.stdout)
+
+    def test_valid_preflight_preserves_argument_boundaries_and_parent_environment(self):
+        result = self.run_helper(self.opted_env())
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            result.stdout.splitlines(),
+            [
+                "CAPTURE",
+                "ARG <-p>",
+                "ARG <torch.testing._internal.torchci.plugin>",
+                "ARG <--torchci-report-prefix=/tmp/torchci reports/smoke>",
+                "POLLUTION <>",
+            ],
+        )
+
+    def test_invalid_preflight_stops_before_the_next_command(self):
+        cases = (
+            ({"PROBE_PYTEST": "7.2.0"}, "pytest >= 7.3"),
+            ({"PROBE_SCHEMA": "2.0"}, "schema 1.0"),
+            ({"JOB_ID": ""}, "positive JOB_ID"),
+            ({"JOB_ID": "0"}, "positive JOB_ID"),
+            ({"JOB_ID": "-1"}, "positive JOB_ID"),
+            ({"JOB_ID": "abc"}, "positive JOB_ID"),
+            ({"GITHUB_REPOSITORY": ""}, "GITHUB_REPOSITORY"),
+            ({"PROBE_MISSING_PLUGIN": "1"}, "plugin"),
+            ({"PROBE_CAPTURE_ERROR": "1"}, "capture failed"),
+        )
+        for extra, message in cases:
+            with self.subTest(extra=extra):
+                result = self.run_helper(self.opted_env(**extra), "echo TESTS_STARTED")
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn(message, result.stderr)
+                self.assertNotIn("TESTS_STARTED", result.stdout)
+
+    def test_python_optimization_cannot_bypass_preflight(self):
+        result = self.run_helper(self.opted_env(JOB_ID="0", PYTHONOPTIMIZE="1"))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("positive JOB_ID", result.stderr)
+        result = self.run_helper(self.opted_env(PROBE_SCHEMA="2.0", PYTHONOPTIMIZE="1"))
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("schema 1.0", result.stderr)
+
+    def test_linux_invocation_preserves_coverage_and_cpp_failure_order(self):
+        script = (ROOT / ".ci/scripts/unittest-linux-cmake.sh").read_text()
+        invocation = script.split("# Run pytest with coverage\n", 1)[1]
+        for status in (0, 1):
+            with self.subTest(status=status):
+                command = (
+                    "pytest() { printf 'PYTEST'; printf ' <%s>' \"$@\"; "
+                    f"printf '\\n'; return {status}; }}; "
+                    "test() { echo CPP; }; "
+                    + invocation.replace("test/run_oss_cpp_tests.sh", "test")
+                )
+                result = self.run_helper(self.opted_env(), command)
+                self.assertEqual(result.returncode, status, result.stderr)
+                self.assertIn(
+                    "PYTEST <-n> <auto> <--cov=./> <--cov-report=xml> <-p> "
+                    "<torch.testing._internal.torchci.plugin> "
+                    "<--torchci-report-prefix=/tmp/torchci reports/smoke>",
+                    result.stdout,
+                )
+                self.assertEqual("CPP" in result.stdout, status == 0)
+
+    def test_unopted_linux_invocation_has_no_reporting_arguments(self):
+        script = (ROOT / ".ci/scripts/unittest-linux-cmake.sh").read_text()
+        invocation = script.split("# Run pytest with coverage\n", 1)[1]
+        command = (
+            'pytest() { printf "ARG <%s>\\n" "$@"; }; '
+            "test() { :; }; " + invocation.replace("test/run_oss_cpp_tests.sh", "test")
+        )
+        result = self.run_helper({"TORCHCI_PYTHON": "/missing/python"}, command)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(
+            result.stdout.splitlines(),
+            ["ARG <-n>", "ARG <auto>", "ARG <--cov=./>", "ARG <--cov-report=xml>"],
+        )

.ci/scripts/torchci-report-args.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2034
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+TORCHCI_REPORT_ARGS=()
+if [[ -n "${RUNNER_TEST_RUN_REPORTS_DIR:-}" ]]; then
+  # The interpreter may include a launcher such as conda run.
+  # shellcheck disable=SC2086
+  ${TORCHCI_PYTHON:-${PYTHON_EXECUTABLE:-python}} -c '
+import os
+import pytest
+from torch.testing._internal.torchci import environment, plugin, report
+
+if report.SCHEMA_VERSION != "1.0":
+    raise RuntimeError("torchci requires report schema 1.0")
+if tuple(map(int, pytest.__version__.split(".")[:2])) < (7, 3):
+    raise RuntimeError("torchci requires pytest >= 7.3")
+job_id = os.environ.get("JOB_ID", "")
+if not job_id.isdigit() or int(job_id) <= 0:
+    raise RuntimeError("torchci requires a positive JOB_ID")
+if not os.environ.get("GITHUB_REPOSITORY"):
+    raise RuntimeError("torchci requires GITHUB_REPOSITORY")
+environment.capture()
+' || return 1
+  TORCHCI_REPORT_ARGS=(
+    -p torch.testing._internal.torchci.plugin
+    "--torchci-report-prefix=${RUNNER_TEST_RUN_REPORTS_DIR}/${1:-pytest}"
+  )
+fi

.ci/scripts/unittest-linux-cmake.sh

@@ -31,8 +31,11 @@ fi
 
 bash .ci/scripts/tests/vulkan/test_vulkan_cmake_package.sh
 
+source .ci/scripts/torchci-report-args.sh unittest-linux-cmake
+
 # Run pytest with coverage
-pytest -n auto --cov=./ --cov-report=xml
+pytest -n auto --cov=./ --cov-report=xml \
+  ${TORCHCI_REPORT_ARGS[@]+"${TORCHCI_REPORT_ARGS[@]}"}
 # Run gtest
 LLVM_PROFDATA=llvm-profdata-12 LLVM_COV=llvm-cov-12 \
 test/run_oss_cpp_tests.sh