Upgrade bazel to 4.2.1 (LTS), upgrade bazel toolchain to 4.1.0 (grpc#27410)

* update bazel to 4.2.1

* add bazel 4.2.1 to update_mirror.sh

* regenerate dockerfiles

* upgrade bazel toolchains to 4.1.0

* use create_rbe_exec_properties_dict instead of the deprecated create_exec_properties_dict

* RBE ubsan doesnt require the crosstool_top hack anymore

* update rules_apple and build_bazel_apple_support

Author: jtattermusch

WORKSPACE

@@ -18,23 +18,15 @@ register_toolchains(
     "//third_party/toolchains/bazel_0.26.0_rbe_windows:cc-toolchain-x64_windows",
 )
 
-load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict", "custom_exec_properties", "merge_dicts")
+load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict", "custom_exec_properties")
 
 custom_exec_properties(
     name = "grpc_custom_exec_properties",
     constants = {
-        "LARGE_MACHINE": merge_dicts(
-            create_exec_properties_dict(),
-            # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
-            # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
-            # and currently the toolchain we're using is too old for that. To be able to select worker
-            # pools through labels, we use a workaround and populate the corresponding label values
-            # manually (see create_exec_properties_dict logic for how labels get transformed)
-            # Remove this workaround once we transition to a new-enough bazel toolchain.
-            # The next line corresponds to 'labels = {"os": "ubuntu", "machine_size": "large"}'
-            {
-                "label:os": "ubuntu",
-                "label:machine_size": "large",
+        "LARGE_MACHINE": create_rbe_exec_properties_dict(
+            labels = {
+                "os": "ubuntu",
+                "machine_size": "large",
             },
         ),
     },
@@ -45,23 +37,14 @@ load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig")
 # Create toolchain configuration for remote execution.
 rbe_autoconfig(
     name = "rbe_default",
-    exec_properties = merge_dicts(
-        create_exec_properties_dict(
-            docker_add_capabilities = "SYS_PTRACE",
-            docker_privileged = True,
-            os_family = "Linux",
-        ),
-        # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
-        # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
-        # and currently the toolchain we're using is too old for that. To be able to select worker
-        # pools through labels, we use a workaround and populate the corresponding label values
-        # manually (see create_exec_properties_dict logic for how labels get transformed)
-        # Remove this workaround once we transition to a new-enough bazel toolchain.
-        # The next line corresponds to 'labels = {"os": "ubuntu", "machine_size": "small"}'
-        {
-            "label:os": "ubuntu",
-            "label:machine_size": "small",
+    exec_properties = create_rbe_exec_properties_dict(
+        docker_add_capabilities = "SYS_PTRACE",
+        docker_privileged = True,
+        labels = {
+            "os": "ubuntu",
+            "machine_size": "small",
         },
+        os_family = "Linux",
     ),
     # use exec_properties instead of deprecated remote_execution_properties
     use_legacy_platform_definition = False,

bazel/grpc_deps.bzl

@@ -295,11 +295,11 @@ def grpc_deps():
         # list of releases is at https://releases.bazel.build/bazel-toolchains.html
         http_archive(
             name = "bazel_toolchains",
-            sha256 = "0b36eef8a66f39c8dbae88e522d5bbbef49d5e66e834a982402c79962281be10",
-            strip_prefix = "bazel-toolchains-1.0.1",
+            sha256 = "179ec02f809e86abf56356d8898c8bd74069f1bd7c56044050c2cd3d79d0e024",
+            strip_prefix = "bazel-toolchains-4.1.0",
             urls = [
-                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/1.0.1.tar.gz",
-                "https://github.com/bazelbuild/bazel-toolchains/releases/download/1.0.1/bazel-toolchains-1.0.1.tar.gz",
+                "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/4.1.0.tar.gz",
+                "https://github.com/bazelbuild/bazel-toolchains/releases/download/4.1.0/bazel-toolchains-4.1.0.tar.gz",
             ],
         )
 
@@ -370,22 +370,21 @@ def grpc_deps():
     if "build_bazel_rules_apple" not in native.existing_rules():
         http_archive(
             name = "build_bazel_rules_apple",
-            strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
-            sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
+            sha256 = "0052d452af7742c8f3a4e0929763388a66403de363775db7e90adecb2ba4944b",
             urls = [
-                "https://storage.googleapis.com/grpc-bazel-mirror/github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
-                "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
+                "https://storage.googleapis.com/grpc-bazel-mirror/github.com/bazelbuild/rules_apple/releases/download/0.31.3/rules_apple.0.31.3.tar.gz",
+                "https://github.com/bazelbuild/rules_apple/releases/download/0.31.3/rules_apple.0.31.3.tar.gz",
             ],
         )
 
     if "build_bazel_apple_support" not in native.existing_rules():
         http_archive(
             name = "build_bazel_apple_support",
+            sha256 = "76df040ade90836ff5543888d64616e7ba6c3a7b33b916aa3a4b68f342d1b447",
             urls = [
-                "https://storage.googleapis.com/grpc-bazel-mirror/github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
-                "https://github.com/bazelbuild/apple_support/releases/download/0.7.1/apple_support.0.7.1.tar.gz",
+                "https://storage.googleapis.com/grpc-bazel-mirror/github.com/bazelbuild/apple_support/releases/download/0.11.0/apple_support.0.11.0.tar.gz",
+                "https://github.com/bazelbuild/apple_support/releases/download/0.11.0/apple_support.0.11.0.tar.gz",
             ],
-            sha256 = "122ebf7fe7d1c8e938af6aeaee0efe788a3a2449ece5a8d6a428cb18d6f88033",
         )
 
     if "libuv" not in native.existing_rules():

bazel/update_mirror.sh

@@ -69,6 +69,10 @@ upload github.com/bazelbuild/bazel/releases/download/3.7.1/bazel-3.7.1-linux-x86
 upload github.com/bazelbuild/bazel/releases/download/3.7.1/bazel-3.7.1-darwin-x86_64
 upload github.com/bazelbuild/bazel/releases/download/3.7.1/bazel-3.7.1-windows-x86_64.exe
 
+upload github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-linux-x86_64
+upload github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-darwin-x86_64
+upload github.com/bazelbuild/bazel/releases/download/4.2.1/bazel-4.2.1-windows-x86_64.exe
+
 # Collect the github archives to mirror from grpc_deps.bzl
 grep -o '"https://github.com/[^"]*"' bazel/grpc_deps.bzl | sed 's/^"https:\/\///' | sed 's/"$//' | while read -r line ; do
     echo "Updating mirror for ${line}"

templates/tools/dockerfile/bazel.include

@@ -2,7 +2,7 @@
 # Bazel installation
 
 # Must be in sync with tools/bazel
-ENV BAZEL_VERSION 3.7.1
+ENV BAZEL_VERSION 4.2.1
 
 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper.
 ENV DISABLE_BAZEL_WRAPPER 1

third_party/toolchains/BUILD

@@ -16,7 +16,7 @@ licenses(["notice"])  # Apache v2
 
 package(default_visibility = ["//visibility:public"])
 
-load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_exec_properties_dict", "merge_dicts")
+load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict")
 
 alias(
     name = "rbe_windows",
@@ -30,19 +30,10 @@ platform(
         "@bazel_tools//platforms:x86_64",
         "@bazel_tools//platforms:windows",
     ],
-    exec_properties = merge_dicts(
-        create_exec_properties_dict(
-            # See rbe_win2019/Dockerfile for image details
-            container_image = "docker://gcr.io/grpc-testing/rbe_windows2019_withdbg_toolchain@sha256:7b04ee7e29f942adbf4f70edd2ec4ba20a3e7237e1b54f5cae4b239c6ca41105",
-            os_family = "Windows",
-        ),
-        # TODO(jtattermusch): specifying 'labels = {"abc": "xyz"}' in create_exec_properties_dict
-        # is not possible without https://github.com/bazelbuild/bazel-toolchains/pull/748
-        # and currently the toolchain we're using is too old for that. To be able to select worker
-        # pools through labels, we use a workaround and populate the corresponding label values
-        # manually (see create_exec_properties_dict logic for how labels get transformed)
-        # Remove this workaround once we transition to a new-enough bazel toolchain.
-        # The next line corresponds to 'labels = {"os": "windows_2019", "machine_size": "small"}'
-        {"label:os": "windows_2019", "label:machine_size": "small"}
+    exec_properties = create_rbe_exec_properties_dict(
+        # See rbe_win2019/Dockerfile for image details
+        container_image = "docker://gcr.io/grpc-testing/rbe_windows2019_withdbg_toolchain@sha256:7b04ee7e29f942adbf4f70edd2ec4ba20a3e7237e1b54f5cae4b239c6ca41105",
+        os_family = "Windows",
+        labels = {"os": "windows_2019", "machine_size": "small"},
     ),
 )

tools/bazel

@@ -42,7 +42,7 @@ fi
 
 # IMPORTANT: if you update the version here, other parts of infrastructure might needs updating as well
 # (e.g. win RBE builds, sanity checks, bazel toolchains etc.)
-VERSION=3.7.1
+VERSION=4.2.1
 echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation." >&2
 
 # update tools/update_mirror.sh to populate the mirror with new bazel archives

tools/dockerfile/test/bazel/Dockerfile

@@ -71,7 +71,7 @@ RUN python3.6 -m ensurepip && \
 # Bazel installation
 
 # Must be in sync with tools/bazel
-ENV BAZEL_VERSION 3.7.1
+ENV BAZEL_VERSION 4.2.1
 
 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper.
 ENV DISABLE_BAZEL_WRAPPER 1

tools/dockerfile/test/sanity/Dockerfile

@@ -101,7 +101,7 @@ ENV CLANG_TIDY=clang-tidy-11
 # Bazel installation
 
 # Must be in sync with tools/bazel
-ENV BAZEL_VERSION 3.7.1
+ENV BAZEL_VERSION 4.2.1
 
 # The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper.
 ENV DISABLE_BAZEL_WRAPPER 1

tools/internal_ci/windows/bazel_rbe.bat

@@ -14,7 +14,7 @@
 
 @rem TODO(jtattermusch): make this generate less output
 @rem TODO(jtattermusch): use tools/bazel script to keep the versions in sync
-choco install bazel -y --version 3.7.1 --limit-output
+choco install bazel -y --version 4.2.1 --limit-output
 
 cd github/grpc
 set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH%

tools/remote_build/rbe_common.bazelrc

@@ -87,8 +87,3 @@ build:ubsan --copt=-gmlt
 # use double the default value for "moderate" and "long" timeout as sanitizer
 # tests tend to be slower
 build:ubsan --test_timeout=60,600,1800,3600
-# override the config-agnostic crosstool_top
-# how to update the bazel toolchain for ubsan:
-# - check for the latest released version in https://github.com/bazelbuild/bazel-toolchains/tree/master/configs/experimental/ubuntu16_04_clang
-# - you might need to update the bazel_toolchains dependency in grpc_deps.bzl
-build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.3/bazel_0.29.1/ubsan:toolchain