Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/setup_multi_arch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ on:
description: "Build native Linux packages (deb/rpm) and run their install tests."
type: boolean
default: true
asan_presubmit:
description: >-
Run ASAN as an automatic presubmit gate. No 'ci:asan'/'ci:host-asan'
label is required, and ASAN sandbox test runners are allocated on
push/pull_request instead of nightly triggers only. Intended for
callers that also narrow the build via build_stages.
type: boolean
default: false
python_version:
description: "Single Python version for wheel builds; empty uses the default matrix."
type: string
Expand Down Expand Up @@ -212,6 +220,7 @@ jobs:
BUILD_PYTORCH: ${{ inputs.build_pytorch }}
BUILD_JAX: ${{ inputs.build_jax }}
BUILD_NATIVE_LINUX: ${{ inputs.build_native_linux }}
ASAN_PRESUBMIT: ${{ inputs.asan_presubmit }}
PYTHON_VERSION: ${{ inputs.python_version }}
LINUX_AMDGPU_FAMILIES: ${{ inputs.linux_amdgpu_families }}
WINDOWS_AMDGPU_FAMILIES: ${{ inputs.windows_amdgpu_families }}
Expand Down
54 changes: 27 additions & 27 deletions build_tools/github_actions/configure_multi_arch_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,11 @@ class CIInputs:
build_native_linux: bool = True
python_versions: list[str] = field(default_factory=list)

# When set, ASAN builds and tests are a first-class presubmit gate: no
# 'ci:asan'/'ci:host-asan' label is needed to run, and sandbox test runners
# are allocated on push/pull_request instead of nightly triggers only.
asan_presubmit: bool = False

# PR labels (from event payload for pull_request events)
pr_labels: list[str] = field(default_factory=list)

Expand Down Expand Up @@ -312,6 +317,7 @@ def from_environ() -> "CIInputs":
build_native_linux = (
os.environ.get("BUILD_NATIVE_LINUX", "true").lower() != "false"
)
asan_presubmit = os.environ.get("ASAN_PRESUBMIT", "false").lower() == "true"
python_version = os.environ.get("PYTHON_VERSION", "").strip()

pr_labels: list[str] = []
Expand Down Expand Up @@ -396,6 +402,7 @@ def from_environ() -> "CIInputs":
build_jax=build_jax,
build_native_linux=build_native_linux,
python_versions=[python_version] if python_version else [],
asan_presubmit=asan_presubmit,
pr_labels=pr_labels,
linux_amdgpu_families=_parse_comma_list(
os.environ.get("LINUX_AMDGPU_FAMILIES", "")
Expand Down Expand Up @@ -764,19 +771,25 @@ def should_skip_ci(
# This avoids running expensive ASAN builds on every PR.
# Labels that enable ASAN CI:
# - ci:asan / ci:host-asan: explicit opt-in for ASAN testing
# Callers that set asan_presubmit have already narrowed the build to a
# cost that is acceptable on every PR, so the opt-in gate does not apply.
has_asan_label = (
"ci:asan" in ci_inputs.pr_labels or "ci:host-asan" in ci_inputs.pr_labels
)
if (
ci_inputs.is_pull_request
and ci_inputs.build_variant == "asan"
and not has_asan_label
Comment on lines 776 to 782

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're going to have multiple build variants (ASan, TSan, Debug, Coverage, etc.) and I wouldn't want to add individual options like *_presubmit for each of them.

A few other ideas:

explicit opt-in label per entry point workflow

We could also generalize the ci:asan label by adding a pull_request_opt_in_labels input to .github/workflows/setup_multi_arch.yml then...

Dynamic enable/disable behavior

We could add an extension point to setup_multi_arch.yml where python ./build_tools/github_actions/configure_multi_arch_ci.py imports another python module that can inject its own logic. That would allow repositories like rocm-systems and rocm-libraries to customize the behavior without needing to fully fork the configure script (see the changes in ROCm/rocm-libraries#10752 for example)

and not ci_inputs.asan_presubmit
):
print(
" Skipping: ASAN PR without enabling label (add 'ci:asan' or 'ci:host-asan' to enable)"
)
return True

if ci_inputs.asan_presubmit and ci_inputs.build_variant == "asan":
print(" Running: ASAN configured as an automatic presubmit gate")

if has_asan_label and ci_inputs.build_variant == "asan":
print(" Running: ASAN CI triggered by PR label")

Expand Down Expand Up @@ -1140,7 +1153,7 @@ def decide_jobs(

# TODO(#3433): Plumb test_rocm.action through workflow outputs. Until then,
# the skip is enforced in _expand_build_config_for_platform() via test_runs_on.
if ci_inputs.build_variant == "asan":
if ci_inputs.build_variant == "asan" and not ci_inputs.asan_presubmit:
# Only run ASAN tests on scheduled or workflow_dispatch runs, to avoid impact on submodule bumps
if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch):
test_rocm = TestRocmDecision(
Expand Down Expand Up @@ -1241,42 +1254,29 @@ def _expand_build_config_for_platform(
)

# TODO(#3433): Remove once ASAN tests pass and test_rocm.action is plumbed.
if build_variant == "asan":
# Only run full ASAN tests on scheduled or workflow_dispatch runs
if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch):
test_runs_on = ""
print(
f" {family_name}: ASAN tests skipped for non-nightly trigger, "
f"disabling tests"
)
elif "test-runs-on-sandbox" in platform_info:
test_runs_on = platform_info["test-runs-on-sandbox"]
print(f" {family_name}: using ASAN sandbox runner: {test_runs_on}")
else:
test_runs_on = ""
print(
f" {family_name}: no ASAN sandbox runner available, "
f"disabling tests"
)
elif build_variant == "host-asan":
# Run host-asan tests only on nightly (schedule or workflow_dispatch)
# due to limited ASAN runner capacity and stability concerns.
if not (ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch):
if build_variant in ("asan", "host-asan"):
# ASAN tests normally run on nightly triggers only, because ASAN
# sandbox runner capacity is limited. Callers that set
# asan_presubmit have narrowed the build enough to make the
# sandbox affordable on every push/pull_request as well.
nightly_trigger = ci_inputs.is_schedule or ci_inputs.is_workflow_dispatch
if not (nightly_trigger or ci_inputs.asan_presubmit):
test_runs_on = ""
print(
f" {family_name}: host-asan tests only run on nightly, "
f"disabling tests"
f" {family_name}: {build_variant} tests skipped for "
f"non-nightly trigger, disabling tests"
)
elif "test-runs-on-sandbox" in platform_info:
test_runs_on = platform_info["test-runs-on-sandbox"]
print(
f" {family_name}: using host-asan sandbox runner: {test_runs_on}"
f" {family_name}: using {build_variant} sandbox runner: "
f"{test_runs_on}"
)
else:
test_runs_on = ""
print(
f" {family_name}: no host-asan sandbox runner available, "
f"disabling tests"
f" {family_name}: no {build_variant} sandbox runner "
f"available, disabling tests"
)

# If run-full-tests-only is set and test_type is "quick", disable testing
Expand Down
32 changes: 32 additions & 0 deletions build_tools/github_actions/tests/configure_multi_arch_ci_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,18 @@ def test_asan_non_pr_runs(self):
git = cm.GitContext(changed_files=None)
self.assertFalse(cm.should_skip_ci(inputs, git))

def test_asan_presubmit_bypasses_label_requirement(self):
"""asan_presubmit allows ASAN PRs without explicit label."""
inputs = self._inputs(
build_variant="asan",
pr_labels=[],
asan_presubmit=True,
)
git = cm.GitContext(
changed_files=["CMakeLists.txt"],
)
self.assertFalse(cm.should_skip_ci(inputs, git))

def test_release_pr_without_submodule_change_runs(self):
"""Release variant PR without submodule changes still runs (not skipped)."""
inputs = self._inputs(build_variant="release", pr_labels=[])
Expand Down Expand Up @@ -753,6 +765,26 @@ def test_asan_tests_only_run_on_nightly_triggers(self):
)
self.assertEqual(result.test_rocm.action, cm.JobAction.RUN)

def test_asan_presubmit_enables_tests_on_pr(self):
"""asan_presubmit allows ASAN tests on PR/push triggers."""
git_context = cm.GitContext()

for event in ["pull_request", "push"]:
result = cm.decide_jobs(
self._inputs(
event_name=event,
build_variant="asan",
asan_presubmit=True,
),
git_context=git_context,
targets=cm.TargetSelection(),
)
self.assertEqual(
result.test_rocm.action,
cm.JobAction.RUN,
f"asan_presubmit should enable ASAN tests on {event}",
)

@patch("configure_multi_arch_ci.compute_auto_stage_reuse")
def test_external_repo_stage_reuse_uses_repo_as_changed_file(self, mock_reuse):
"""External repo builds pass repo name to stage-impact analysis.
Expand Down
Loading