From 4201956bfa71badcd43a942038ac44c4eb7b226e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kami=C5=84ski?= Date: Mon, 29 Sep 2025 14:13:32 +0000 Subject: [PATCH 1/4] [Benchmarks] Fix SubmitGraph CPU count scenarios grouping Fixes SubmitGraph CPU count scenarios being drawed in the same charts as SubmitGraph time measurement scenarios making them hard to read. --- devops/scripts/benchmarks/benches/compute.py | 34 ++++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index be58255d5fd0d..97eab890ed7ee 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -154,12 +154,18 @@ def additional_metadata(self) -> dict[str, BenchmarkMetadata]: for completion in ["", " with completion"]: for events in ["", " using events"]: for num_kernels in self.submit_graph_num_kernels: - group_name = f"SubmitGraph {order}{completion}{events}, {num_kernels} kernels" - metadata[group_name] = BenchmarkMetadata( - type="group", - tags=base_metadata.tags, - ) - + for host_tasks in ["", " use host tasks"]: + group_name = f"SubmitGraph {order}{completion}{events}{host_tasks}, {num_kernels} kernels" + metadata[group_name] = BenchmarkMetadata( + type="group", + tags=base_metadata.tags, + ) + # CPU count variants + cpu_count_group = f"{group_name}, CPU count" + metadata[cpu_count_group] = BenchmarkMetadata( + type="group", + tags=base_metadata.tags, + ) return metadata def benchmarks(self) -> list[Benchmark]: @@ -1088,6 +1094,22 @@ def bin_args(self, run_trace: TracingType = TracingType.NONE) -> list[str]: bin_args.append(f"--profilerType={self.profiler_type.value}") return bin_args + def get_metadata(self) -> dict[str, BenchmarkMetadata]: + metadata_dict = super().get_metadata() + + # Create CPU count variant with modified display name and explicit_group + cpu_count_name = self.name() + " CPU count" + cpu_count_metadata = copy.deepcopy(metadata_dict[self.name()]) + cpu_count_display_name = self.display_name() + ", CPU count" + cpu_count_explicit_group = ( + self.explicit_group() + ", CPU count" if self.explicit_group() else "" + ) + cpu_count_metadata.display_name = cpu_count_display_name + cpu_count_metadata.explicit_group = cpu_count_explicit_group + metadata_dict[cpu_count_name] = cpu_count_metadata + + return metadata_dict + class UllsEmptyKernel(ComputeBenchmark): def __init__( From ba3078bd69afea3cacf2e8b3c33cfbffbfdb26e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kami=C5=84ski?= Date: Mon, 29 Sep 2025 15:20:37 +0000 Subject: [PATCH 2/4] Refactor compute bench metadata Simplify SubmitGraph and SubmitKernel benchmarks metadata creation. --- devops/scripts/benchmarks/benches/compute.py | 57 ++++++++------------ 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 97eab890ed7ee..2296a8456a80a 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -100,71 +100,56 @@ def setup(self) -> None: def additional_metadata(self) -> dict[str, BenchmarkMetadata]: metadata = { - "SubmitKernel": BenchmarkMetadata( - type="group", - description="Measures CPU time overhead of submitting kernels through different APIs.", - notes="Each layer builds on top of the previous layer, adding functionality and overhead.\n" - "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n" - "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n" - "Work is ongoing to reduce the overhead of the SYCL API\n", - tags=["submit", "micro", "SYCL", "UR", "L0"], - range_min=0.0, - ), "SinKernelGraph": BenchmarkMetadata( type="group", unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.", tags=["submit", "memory", "proxy", "SYCL", "UR", "L0", "graph"], ), - "SubmitGraph": BenchmarkMetadata( - type="group", tags=["submit", "micro", "SYCL", "UR", "L0", "graph"] - ), "FinalizeGraph": BenchmarkMetadata( type="group", tags=["finalize", "micro", "SYCL", "graph"] ), } # Add metadata for all SubmitKernel group variants - base_metadata = metadata["SubmitKernel"] - + submit_kernel_metadata = BenchmarkMetadata( + type="group", + notes="Each layer builds on top of the previous layer, adding functionality and overhead.\n" + "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n" + "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n" + "Work is ongoing to reduce the overhead of the SYCL API\n", + tags=["submit", "micro", "SYCL", "UR", "L0"], + range_min=0.0, + ) for order in ["in order", "out of order"]: for completion in ["", " with completion"]: for events in ["", " using events"]: group_name = f"SubmitKernel {order}{completion}{events} long kernel" - metadata[group_name] = BenchmarkMetadata( - type="group", - description=f"Measures CPU time overhead of submitting {order} kernels with longer execution times through different APIs.", - notes=base_metadata.notes, - tags=base_metadata.tags, - range_min=base_metadata.range_min, + metadata[group_name] = copy.deepcopy(submit_kernel_metadata) + metadata[group_name].description = ( + f"Measures CPU time overhead of submitting {order} kernels with longer execution times through different APIs." ) - # CPU count variants cpu_count_group = f"{group_name}, CPU count" - metadata[cpu_count_group] = BenchmarkMetadata( - type="group", - description=f"Measures CPU time overhead of submitting {order} kernels with longer execution times through different APIs.", - notes=base_metadata.notes, - tags=base_metadata.tags, - range_min=base_metadata.range_min, + metadata[cpu_count_group] = copy.deepcopy(submit_kernel_metadata) + metadata[cpu_count_group].description = ( + f"Measures CPU instructions count overhead of submitting {order} kernels with longer execution times through different APIs." ) # Add metadata for all SubmitGraph group variants - base_metadata = metadata["SubmitGraph"] + submit_graph_metadata = BenchmarkMetadata( + type="group", tags=["submit", "micro", "SYCL", "UR", "L0", "graph"] + ) for order in ["in order", "out of order"]: for completion in ["", " with completion"]: for events in ["", " using events"]: for num_kernels in self.submit_graph_num_kernels: for host_tasks in ["", " use host tasks"]: group_name = f"SubmitGraph {order}{completion}{events}{host_tasks}, {num_kernels} kernels" - metadata[group_name] = BenchmarkMetadata( - type="group", - tags=base_metadata.tags, - ) + metadata[group_name] = copy.deepcopy(submit_graph_metadata) # CPU count variants cpu_count_group = f"{group_name}, CPU count" - metadata[cpu_count_group] = BenchmarkMetadata( - type="group", - tags=base_metadata.tags, + metadata[cpu_count_group] = copy.deepcopy( + submit_graph_metadata ) return metadata From cc9c41e0d0f50ed0ca9d99664e9dc0b2b5fbf948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kami=C5=84ski?= Date: Mon, 29 Sep 2025 15:25:21 +0000 Subject: [PATCH 3/4] Sort imports in compute.py module Imports sorted with isort --- devops/scripts/benchmarks/benches/compute.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 2296a8456a80a..3cc7832bc6f80 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -3,19 +3,19 @@ # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -from itertools import product +import copy import csv import io -import copy import math from enum import Enum +from itertools import product from pathlib import Path -from .base import Benchmark, Suite, TracingType -from utils.result import BenchmarkMetadata, Result -from .base import Benchmark, Suite -from options import options from git_project import GitProject +from options import options +from utils.result import BenchmarkMetadata, Result + +from .base import Benchmark, Suite, TracingType class RUNTIMES(Enum): From a09eb21f6b4c1b47104342697d716eeaa48d8f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20=C5=9Alusarczyk?= Date: Fri, 3 Oct 2025 09:22:52 +0200 Subject: [PATCH 4/4] grammar fix Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- devops/scripts/benchmarks/benches/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 3cc7832bc6f80..7041bad3ed689 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -132,7 +132,7 @@ def additional_metadata(self) -> dict[str, BenchmarkMetadata]: cpu_count_group = f"{group_name}, CPU count" metadata[cpu_count_group] = copy.deepcopy(submit_kernel_metadata) metadata[cpu_count_group].description = ( - f"Measures CPU instructions count overhead of submitting {order} kernels with longer execution times through different APIs." + f"Measures CPU instruction count overhead of submitting {order} kernels with longer execution times through different APIs." ) # Add metadata for all SubmitGraph group variants