Consolidate PR and regular worker setup (#706)

This is a straightforward merging of the two loops, leaving
room for follow-ups.

One exception: skipping logic now uses the "wasm" tag
for all builders; previously PR builders used "wasi".
These tags currently always appear together.

Co-Authored-By: Zachary Ware <zach@python.org>

Author: encukou

master/custom/branches.py

@@ -117,7 +117,6 @@ def __lt__(self, other):
 
 
 BRANCHES = list(generate_branches())
-PR_BRANCH = BRANCHES[-1]
 
 # Verify that we've defined these in sort order
 assert BRANCHES == sorted(BRANCHES)

master/master.cfg

@@ -50,7 +50,7 @@ from custom.builders import (  # noqa: E402
     STABLE,
     ONLY_MAIN_BRANCH,
 )
-from custom.branches import BRANCHES, PR_BRANCH
+from custom.branches import BRANCHES
 
 
 def set_up_sentry():
@@ -186,32 +186,40 @@ def is_important_change(change):
     return any(is_important_file(filename) for filename in change.files)
 
 
+# Builders
+
 github_status_builders = []
 release_status_builders = []
 mail_status_builders = []
 
-# Regular builders
+stable_pull_request_builders = []
+all_pull_request_builders = []
 
 for branch in BRANCHES:
-    if not branch.git_branch:
-        continue
     buildernames = []
     refleakbuildernames = []
+    stable_builder_names = []
     for name, worker, buildfactory, stability, tier in BUILDERS:
         if any(
             pattern in name for pattern in ONLY_MAIN_BRANCH
-        ) and not branch.is_main:
+        ) and not branch.is_main and not branch.is_pr:
             # Workers known to be broken on older branches: let's focus on
             # supporting these platforms in the main branch.
             continue
 
-        if worker.not_branches and branch.name in worker.not_branches:
-            continue
-        if worker.branches and branch.name not in worker.branches:
-            continue
+        if not branch.is_pr:
+            if worker.not_branches and branch.name in worker.not_branches:
+                continue
+            if worker.branches and branch.name not in worker.branches:
+                continue
 
         buildername = name + " " + branch.name
-        source = Git(repourl=GIT_URL, branch=branch.git_branch, **GIT_KWDS)
+
+        if branch.is_pr:
+            source = GitHub(repourl=GIT_URL, **GIT_KWDS)
+        else:
+            source = Git(repourl=GIT_URL, branch=branch.git_branch, **GIT_KWDS)
+
         f = buildfactory(
             source,
             branch=branch,
@@ -225,6 +233,12 @@ for branch in BRANCHES:
         if "wasm" in tags:
             if branch.wasm_tier is None:
                 continue
+            elif branch.is_pr:
+                # Don't use the WASI buildbot meant for 3.11 and 3.12 on PRs;
+                # it's tier 3 only and getting it to work on PRs against `main`
+                # is too much work.
+                if "nondebug" in tags:
+                    continue
             elif "nondebug" in tags and branch.wasm_tier == 2:
                 continue
             elif "nondebug" not in tags and branch.wasm_tier == 3:
@@ -237,13 +251,17 @@ for branch in BRANCHES:
             refleakbuildernames.append(buildername)
         else:
             buildernames.append(buildername)
-        # disable notifications for unstable builders
-        # (all these lists are the same now, but we might need to
-        # diverge gain later.)
+
         if stability == STABLE:
-            mail_status_builders.append(buildername)
-            github_status_builders.append(buildername)
-            release_status_builders.append(buildername)
+            stable_builder_names.append(buildername)
+
+            # disable notifications for unstable & PR builders
+            # (all these lists are the same now, but we might need to
+            # diverge again later.)
+            if not branch.is_pr:
+                mail_status_builders.append(buildername)
+                github_status_builders.append(buildername)
+                release_status_builders.append(buildername)
 
         builder = util.BuilderConfig(
             name=buildername,
@@ -259,87 +277,54 @@ for branch in BRANCHES:
 
         c["builders"].append(builder)
 
-    c["schedulers"].append(
-        schedulers.SingleBranchScheduler(
-            name=branch.name,
-            change_filter=util.ChangeFilter(branch=branch.git_branch),
-            treeStableTimer=30,  # seconds
-            builderNames=buildernames,
-            fileIsImportant=is_important_change,
+    if branch.is_pr:
+        all_pull_request_builders = refleakbuildernames + buildernames
+        stable_pull_request_builders = stable_builder_names
+        c["schedulers"].append(
+            GitHubPrScheduler(
+                name="pull-request-scheduler",
+                change_filter=util.ChangeFilter(filter_fn=should_pr_be_tested),
+                treeStableTimer=30,  # seconds
+                builderNames=all_pull_request_builders,
+                stable_builder_names=set(stable_builder_names),
+            )
         )
-    )
-    if refleakbuildernames:
+    else:
         c["schedulers"].append(
             schedulers.SingleBranchScheduler(
-                name=branch.name + "-refleak",
+                name=branch.name,
                 change_filter=util.ChangeFilter(branch=branch.git_branch),
-                # Wait this many seconds for no commits before starting a build
-                # NB: During extremely busy times, this can cause the builders
-                # to never actually fire.  The current expectation is that it
-                # won't ever actually be that busy, but we need to keep an eye
-                # on that.
-                treeStableTimer=1 * 60 * 60,  # h * m * s
-                builderNames=refleakbuildernames,
+                treeStableTimer=30,  # seconds
+                builderNames=buildernames,
                 fileIsImportant=is_important_change,
             )
         )
+        if refleakbuildernames:
+            c["schedulers"].append(
+                schedulers.SingleBranchScheduler(
+                    name=branch.name + "-refleak",
+                    change_filter=util.ChangeFilter(branch=branch.git_branch),
+                    # Wait this many seconds for no commits before starting a build
+                    # NB: During extremely busy times, this can cause the builders
+                    # to never actually fire.  The current expectation is that it
+                    # won't ever actually be that busy, but we need to keep an eye
+                    # on that.
+                    treeStableTimer=1 * 60 * 60,  # h * m * s
+                    builderNames=refleakbuildernames,
+                    fileIsImportant=is_important_change,
+                )
+            )
 
+# Make sure all names are unique
+scheduler_names = [s.name for s in c["schedulers"]]
+assert len(scheduler_names) == len(set(scheduler_names))
 
-# Set up Pull Request builders
-
-stable_pull_request_builders = []
-all_pull_request_builders = []
-
-for name, worker, buildfactory, stability, tier in BUILDERS:
-    branch = PR_BRANCH
-    assert PR_BRANCH.is_pr
-
-    buildername = f"{name} PR"
-    source = GitHub(repourl=GIT_URL, **GIT_KWDS)
-
-    f = buildfactory(
-        source,
-        branch=branch,
-        worker=worker,
-    )
-
-    tags = [branch.builder_tag, stability, *f.tags]
-    if tier:
-        tags.append(tier)
-
-    # Don't use the WASI buildbot meant for 3.11 and 3.12 on PRs;
-    # it's tier 3 only and getting it to work on PRs against `main`
-    # is too much work.
-    if "wasi" in tags and "nondebug" in tags:
-        continue
-
-    all_pull_request_builders.append(buildername)
-    if stability == STABLE:
-        stable_pull_request_builders.append(buildername)
-
-    builder = util.BuilderConfig(
-        name=buildername,
-        workernames=[worker.name],
-        builddir=f"{branch.builddir_name}.{worker.name}{f.buildersuffix}",
-        factory=f,
-        tags=tags,
-        locks=[cpulock.access("counting")],
-    )
-
-    if worker.downtime:
-        builder.canStartBuild = worker.downtime
-
-    c["builders"].append(builder)
-
-c["schedulers"].append(
-    GitHubPrScheduler(
-        name="pull-request-scheduler",
-        change_filter=util.ChangeFilter(filter_fn=should_pr_be_tested),
-        treeStableTimer=30,  # seconds
-        builderNames=all_pull_request_builders,
-        stable_builder_names=set(stable_pull_request_builders),
-    )
-)
+# Make sure the lists got filled up
+assert github_status_builders
+assert release_status_builders
+assert mail_status_builders
+assert stable_pull_request_builders
+assert all_pull_request_builders
 
 
 # Set up aditional schedulers