Skip to content

Reapply "[CI] Migrate to runtimes build" (#143612) #144033

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Reapply "[CI] Migrate to runtimes build" (#143612) #144033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5a36b9e
[𝘀𝗽𝗿] initial version
boomanaiden154 Jun 13, 2025
d818c7b
rebase
boomanaiden154 Jun 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 74 additions & 41 deletions .ci/compute_projects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@
},
"lld": {"bolt", "cross-project-tests"},
# TODO(issues/132795): LLDB should be enabled on clang changes.
"clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
"clang-tools-extra": {"libc"},
"clang": {"clang-tools-extra", "cross-project-tests"},
"mlir": {"flang"},
# Test everything if ci scripts are changed.
# FIXME: Figure out what is missing and add here.
Expand All @@ -64,7 +63,15 @@

# This mapping describes runtimes that should be tested when the key project is
# touched.
DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
DEPENDENT_RUNTIMES_TO_TEST = {
"clang": {"compiler-rt"},
"clang-tools-extra": {"libc"},
}
DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
"llvm": {"libcxx", "libcxxabi", "libunwind"},
"clang": {"libcxx", "libcxxabi", "libunwind"},
".ci": {"libcxx", "libcxxabi", "libunwind"},
}

EXCLUDE_LINUX = {
"cross-project-tests", # TODO(issues/132796): Tests are failing.
Expand Down Expand Up @@ -93,9 +100,6 @@
"cross-project-tests",
"flang",
"libc",
"libcxx",
"libcxxabi",
"libunwind",
"lldb",
"openmp",
"polly",
Expand All @@ -122,10 +126,10 @@
"polly": "check-polly",
}

RUNTIMES = {"libcxx", "libcxxabi", "libunwind"}
RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}


def _add_dependencies(projects: Set[str]) -> Set[str]:
def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:
projects_with_dependents = set(projects)
current_projects_count = 0
while current_projects_count != len(projects_with_dependents):
Expand All @@ -134,9 +138,25 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
if project not in PROJECT_DEPENDENCIES:
continue
projects_with_dependents.update(PROJECT_DEPENDENCIES[project])
for runtime in runtimes:
if runtime not in PROJECT_DEPENDENCIES:
continue
projects_with_dependents.update(PROJECT_DEPENDENCIES[runtime])
return projects_with_dependents


def _exclude_projects(current_projects: Set[str], platform: str) -> Set[str]:
if platform == "Linux":
to_exclude = EXCLUDE_LINUX
elif platform == "Windows":
to_exclude = EXCLUDE_WINDOWS
elif platform == "Darwin":
to_exclude = EXCLUDE_MAC
else:
raise ValueError(f"Unexpected platform: {platform}")
return current_projects.difference(to_exclude)


def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
projects_to_test = set()
for modified_project in modified_projects:
Expand All @@ -154,25 +174,14 @@ def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set
):
continue
projects_to_test.add(dependent_project)
if platform == "Linux":
for to_exclude in EXCLUDE_LINUX:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
elif platform == "Windows":
for to_exclude in EXCLUDE_WINDOWS:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
elif platform == "Darwin":
for to_exclude in EXCLUDE_MAC:
if to_exclude in projects_to_test:
projects_to_test.remove(to_exclude)
else:
raise ValueError("Unexpected platform.")
projects_to_test = _exclude_projects(projects_to_test, platform)
return projects_to_test


def _compute_projects_to_build(projects_to_test: Set[str]) -> Set[str]:
return _add_dependencies(projects_to_test)
def _compute_projects_to_build(
projects_to_test: Set[str], runtimes: Set[str]
) -> Set[str]:
return _add_dependencies(projects_to_test, runtimes)


def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
Expand All @@ -184,24 +193,36 @@ def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
return check_targets


def _compute_runtimes_to_test(projects_to_test: Set[str]) -> Set[str]:
def _compute_runtimes_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
runtimes_to_test = set()
for project_to_test in projects_to_test:
if project_to_test in DEPENDENT_RUNTIMES_TO_TEST:
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
if project_to_test in DEPENDENT_RUNTIMES_TO_BUILD:
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_BUILD[project_to_test])
return runtimes_to_test
for modified_project in modified_projects:
if modified_project not in DEPENDENT_RUNTIMES_TO_TEST:
continue
runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[modified_project])
return _exclude_projects(runtimes_to_test, platform)


def _compute_runtime_check_targets(projects_to_test: Set[str]) -> Set[str]:
check_targets = set()
for project_to_test in projects_to_test:
if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
def _compute_runtimes_to_test_needs_reconfig(
modified_projects: Set[str], platform: str
) -> Set[str]:
runtimes_to_test = set()
for modified_project in modified_projects:
if modified_project not in DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG:
continue
for runtime_to_test in DEPENDENT_RUNTIMES_TO_TEST[project_to_test]:
check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
return check_targets
runtimes_to_test.update(
DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG[modified_project]
)
return _exclude_projects(runtimes_to_test, platform)


def _compute_runtimes_to_build(
runtimes_to_test: Set[str], modified_projects: Set[str], platform: str
) -> Set[str]:
runtimes_to_build = set(runtimes_to_test)
for modified_project in modified_projects:
if modified_project in DEPENDENT_RUNTIMES_TO_BUILD:
runtimes_to_build.update(DEPENDENT_RUNTIMES_TO_BUILD[modified_project])
return _exclude_projects(runtimes_to_build, platform)


def _get_modified_projects(modified_files: list[str]) -> Set[str]:
Expand All @@ -225,10 +246,19 @@ def _get_modified_projects(modified_files: list[str]) -> Set[str]:
def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
modified_projects = _get_modified_projects(modified_files)
projects_to_test = _compute_projects_to_test(modified_projects, platform)
projects_to_build = _compute_projects_to_build(projects_to_test)
runtimes_to_test = _compute_runtimes_to_test(modified_projects, platform)
runtimes_to_test_needs_reconfig = _compute_runtimes_to_test_needs_reconfig(
modified_projects, platform
)
runtimes_to_build = _compute_runtimes_to_build(
runtimes_to_test | runtimes_to_test_needs_reconfig, modified_projects, platform
)
projects_to_build = _compute_projects_to_build(projects_to_test, runtimes_to_build)
projects_check_targets = _compute_project_check_targets(projects_to_test)
runtimes_to_build = _compute_runtimes_to_test(projects_to_test)
runtimes_check_targets = _compute_runtime_check_targets(projects_to_test)
runtimes_check_targets = _compute_project_check_targets(runtimes_to_test)
runtimes_check_targets_needs_reconfig = _compute_project_check_targets(
runtimes_to_test_needs_reconfig
)
# We use a semicolon to separate the projects/runtimes as they get passed
# to the CMake invocation and thus we need to use the CMake list separator
# (;). We use spaces to separate the check targets as they end up getting
Expand All @@ -238,6 +268,9 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
"project_check_targets": " ".join(sorted(projects_check_targets)),
"runtimes_to_build": ";".join(sorted(runtimes_to_build)),
"runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),
"runtimes_check_targets_needs_reconfig": " ".join(
sorted(runtimes_check_targets_needs_reconfig)
),
}


Expand Down
55 changes: 51 additions & 4 deletions .ci/compute_projects_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ def test_llvm(self):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -46,6 +50,10 @@ def test_llvm_windows(self):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -66,6 +74,10 @@ def test_llvm_mac(self):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -75,17 +87,21 @@ def test_clang(self):
)
self.assertEqual(
env_variables["projects_to_build"],
"clang;clang-tools-extra;compiler-rt;lld;llvm",
"clang;clang-tools-extra;lld;llvm",
)
self.assertEqual(
env_variables["project_check_targets"],
"check-clang check-clang-tools check-compiler-rt",
"check-clang check-clang-tools",
)
self.assertEqual(
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"check-compiler-rt",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -104,6 +120,10 @@ def test_clang_windows(self):
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -115,6 +135,7 @@ def test_bolt(self):
self.assertEqual(env_variables["project_check_targets"], "check-bolt")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_lldb(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -124,6 +145,7 @@ def test_lldb(self):
self.assertEqual(env_variables["project_check_targets"], "check-lldb")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_mlir(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -135,6 +157,7 @@ def test_mlir(self):
)
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_flang(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -144,6 +167,7 @@ def test_flang(self):
self.assertEqual(env_variables["project_check_targets"], "check-flang")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_invalid_subproject(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -153,13 +177,15 @@ def test_invalid_subproject(self):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_top_level_file(self):
env_variables = compute_projects.get_env_variables(["README.md"], "Linux")
self.assertEqual(env_variables["projects_to_build"], "")
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_exclude_runtiems_in_projects(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -169,6 +195,7 @@ def test_exclude_runtiems_in_projects(self):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_exclude_docs(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -178,6 +205,7 @@ def test_exclude_docs(self):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_exclude_gn(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -187,6 +215,7 @@ def test_exclude_gn(self):
self.assertEqual(env_variables["project_check_targets"], "")
self.assertEqual(env_variables["runtimes_to_build"], "")
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_ci(self):
env_variables = compute_projects.get_env_variables(
Expand All @@ -198,10 +227,15 @@ def test_ci(self):
"check-clang check-lld check-lldb check-llvm",
)
self.assertEqual(
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
env_variables["runtimes_to_build"],
"libcxx;libcxxabi;libunwind",
)
self.assertEqual(
env_variables["runtimes_check_targets"],
"",
)
self.assertEqual(
env_variables["runtimes_check_targets_needs_reconfig"],
"check-cxx check-cxxabi check-unwind",
)

Expand All @@ -215,6 +249,19 @@ def test_lldb(self):
env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
)
self.assertEqual(env_variables["runtimes_check_targets"], "")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")

def test_clang_tools_extra(self):
env_variables = compute_projects.get_env_variables(
["clang-tools-extra/CMakeLists.txt"], "Linux"
)
self.assertEqual(
env_variables["projects_to_build"], "clang;clang-tools-extra;lld;llvm"
)
self.assertEqual(env_variables["project_check_targets"], "check-clang-tools")
self.assertEqual(env_variables["runtimes_to_build"], "libc")
self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")


if __name__ == "__main__":
Expand Down
Loading
Loading