Skip to content

[GR-64081] Implement new 'graalos' bench suite #11276

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 29, 2025
Merged

[GR-64081] Implement new 'graalos' bench suite #11276

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion common.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
],

"mx_version": "7.51.2",
"mx_version": "7.54.3",

"COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
"jdks": {
Expand Down
112 changes: 34 additions & 78 deletions sdk/mx.sdk/mx_sdk_benchmark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
import mx_sdk_vm
import mx_sdk_vm_impl
import mx_util
from mx_benchmark import DataPoints, DataPoint, BenchmarkSuite
from mx_benchmark import DataPoints, DataPoint, BenchmarkSuite, Vm, SingleBenchmarkExecutionContext
from mx_sdk_vm_impl import svm_experimental_options

_suite = mx.suite('sdk')
Expand Down Expand Up @@ -2815,17 +2815,8 @@ def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = N
custom_harness_command = BaristaBenchmarkSuite.BaristaCommand()
super().__init__(custom_harness_command)
self._version = None
self._context = None
self._extra_run_options = []

@property
def context(self):
return self._context

@context.setter
def context(self, value):
self._context = value

def readBaristaVersionFromPyproject(self):
# tomllib was included in python standard library with version 3.11
try:
Expand Down Expand Up @@ -2858,6 +2849,9 @@ def group(self):
def subgroup(self):
return "graal-compiler"

def benchmarkName(self):
return self.execution_context.benchmark

def benchmarkList(self, bmSuiteArgs):
exclude = []
# Barista currently does not support running 'micronaut-pegasus' on the JVM - running it results in a crash (GR-59793)
Expand Down Expand Up @@ -2892,15 +2886,18 @@ def baristaHarnessPath(self):
return self.baristaFilePath("barista")

def baristaHarnessBenchmarkName(self):
return _baristaConfig["benchmarks"][self.context.benchmark].get("barista-bench-name", self.context.benchmark)
return _baristaConfig["benchmarks"][self.benchmarkName()].get("barista-bench-name", self.benchmarkName())

def baristaHarnessBenchmarkWorkload(self):
return _baristaConfig["benchmarks"][self.context.benchmark].get("workload")
return _baristaConfig["benchmarks"][self.benchmarkName()].get("workload")

def validateEnvironment(self):
self.baristaProjectConfigurationPath()
self.baristaHarnessPath()

def new_execution_context(self, vm: Vm, benchmarks: List[str], bmSuiteArgs: List[str]) -> SingleBenchmarkExecutionContext:
return SingleBenchmarkExecutionContext(self, vm, benchmarks, bmSuiteArgs)

def register_tracker(self, name, tracker_type):
if tracker_type in _baristaConfig["disable_trackers"]:
mx.log(f"Ignoring the registration of '{name}' tracker as it was disabled for {self.__class__.__name__}.")
Expand Down Expand Up @@ -2934,7 +2931,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):

# Startup
all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
"benchmark": self.context.benchmark,
"benchmark": self.benchmarkName(),
"metric.name": "request-time",
"metric.type": "numeric",
"metric.unit": "ms",
Expand All @@ -2947,7 +2944,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):

# Warmup
all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
"benchmark": self.context.benchmark,
"benchmark": self.benchmarkName(),
"metric.name": "warmup",
"metric.type": "numeric",
"metric.unit": "op/s",
Expand All @@ -2960,7 +2957,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):

# Throughput
all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
"benchmark": self.context.benchmark,
"benchmark": self.benchmarkName(),
"metric.name": "throughput",
"metric.type": "numeric",
"metric.unit": "op/s",
Expand All @@ -2973,7 +2970,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):

# Latency
all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
"benchmark": self.context.benchmark,
"benchmark": self.benchmarkName(),
"metric.name": "latency",
"metric.type": "numeric",
"metric.unit": "ms",
Expand All @@ -2992,7 +2989,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):

# Resource Usage
all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
"benchmark": self.context.benchmark,
"benchmark": self.benchmarkName(),
"metric.name": "rss",
"metric.type": "numeric",
"metric.unit": "MB",
Expand Down Expand Up @@ -3082,17 +3079,6 @@ def extendDatapoints(self, datapoints: DataPoints) -> DataPoints:
del datapoint["load-tester.command"]
return datapoints

def _vmRun(self, vm, workdir, command, benchmarks, bmSuiteArgs):
self.enforce_single_benchmark(benchmarks)
self.context = BaristaBenchmarkSuite.RuntimeContext(self, vm, benchmarks[0], bmSuiteArgs)
return super()._vmRun(vm, workdir, command, benchmarks, bmSuiteArgs)

def enforce_single_benchmark(self, benchmarks):
if not isinstance(benchmarks, list):
raise TypeError(f"{self.__class__.__name__} expects to receive a list of benchmarks to run, instead got an instance of {benchmarks.__class__.__name__}! Please specify a single benchmark!")
if len(benchmarks) != 1:
raise ValueError(f"You have requested {benchmarks} to be run but {self.__class__.__name__} can only run a single benchmark at a time! Please specify a single benchmark!")

class BaristaCommand(mx_benchmark.CustomHarnessCommand):
"""Maps a JVM command into a command tailored for the Barista harness.
"""
Expand Down Expand Up @@ -3151,7 +3137,7 @@ def produceHarnessCommand(self, cmd, suite):
jvm_vm_options = jvm_cmd[index_of_java_exe + 1:]

# Verify that the run arguments don't already contain a "--mode" option
run_args = suite.runArgs(suite.context.bmSuiteArgs) + suite._extra_run_options
run_args = suite.runArgs(suite.execution_context.bmSuiteArgs) + suite._extra_run_options
mode_pattern = r"^(?:-m|--mode)(=.*)?$"
mode_match = self._regexFindInCommand(run_args, mode_pattern)
if mode_match:
Expand All @@ -3175,37 +3161,6 @@ def produceHarnessCommand(self, cmd, suite):
barista_cmd.append(barista_bench_name)
return barista_cmd

class RuntimeContext():
"""Container class for the runtime context of BaristaBenchmarkSuite.
"""
def __init__(self, suite, vm, benchmark, bmSuiteArgs):
if not isinstance(suite, BaristaBenchmarkSuite):
raise TypeError(f"Expected an instance of {BaristaBenchmarkSuite.__name__}, instead got an instance of {suite.__class__.__name__}")
self._suite = suite
self._vm = vm
self._benchmark = benchmark
self._bmSuiteArgs = bmSuiteArgs

@property
def suite(self):
return self._suite

@property
def vm(self):
return self._vm

@property
def benchmark(self):
"""The currently running benchmark.

Corresponds to `benchmarks[0]` in a suite method that has a `benchmarks` argument.
"""
return self._benchmark

@property
def bmSuiteArgs(self):
return self._bmSuiteArgs


mx_benchmark.add_bm_suite(BaristaBenchmarkSuite())

Expand Down Expand Up @@ -3959,24 +3914,25 @@ def run(self, benchmarks, bm_suite_args: List[str]) -> DataPoints:
fallback_reason = self.fallback_mode_reason(bm_suite_args)

vm = self.get_vm_registry().get_vm_from_suite_args(bm_suite_args)
effective_stages, complete_stage_list = vm.prepare_stages(self, bm_suite_args)
self.stages_info = StagesInfo(effective_stages, complete_stage_list, vm, bool(fallback_reason))

if self.stages_info.fallback_mode:
# In fallback mode, all stages are run at once. There is matching code in `NativeImageVM.run_java` for this.
mx.log(f"Running benchmark in fallback mode (reason: {fallback_reason})")
datapoints += super_delegate.run(benchmarks, bm_suite_args)
else:
while self.stages_info.has_next_stage():
stage = self.stages_info.next_stage()
# Start the actual benchmark execution. The stages_info attribute will be used by the NativeImageVM to
# determine which stage to run this time.
stage_dps = super_delegate.run(benchmarks, bm_suite_args)
NativeImageBenchmarkMixin._inject_stage_keys(stage_dps, stage)
datapoints += stage_dps

self.stages_info = None
return datapoints
with self.new_execution_context(vm, benchmarks, bm_suite_args):
effective_stages, complete_stage_list = vm.prepare_stages(self, bm_suite_args)
self.stages_info = StagesInfo(effective_stages, complete_stage_list, vm, bool(fallback_reason))

if self.stages_info.fallback_mode:
# In fallback mode, all stages are run at once. There is matching code in `NativeImageVM.run_java` for this.
mx.log(f"Running benchmark in fallback mode (reason: {fallback_reason})")
datapoints += super_delegate.run(benchmarks, bm_suite_args)
else:
while self.stages_info.has_next_stage():
stage = self.stages_info.next_stage()
# Start the actual benchmark execution. The stages_info attribute will be used by the NativeImageVM to
# determine which stage to run this time.
stage_dps = super_delegate.run(benchmarks, bm_suite_args)
NativeImageBenchmarkMixin._inject_stage_keys(stage_dps, stage)
datapoints += stage_dps

self.stages_info = None
return datapoints

@staticmethod
def _inject_stage_keys(dps: DataPoints, stage: Stage) -> None:
Expand Down
Loading
Loading