diff --git a/.github/workflows/development.yml b/.github/workflows/development.yml index a816b63b..bc53dd75 100644 --- a/.github/workflows/development.yml +++ b/.github/workflows/development.yml @@ -244,6 +244,11 @@ jobs: with: fetch-depth: 0 + - name: Set up Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Check if UI-related files changed id: check-changes run: | @@ -275,7 +280,8 @@ jobs: # Set asset prefix and base path with PR number ASSET_PREFIX=https://neuralmagic.github.io/guidellm/ui/pr/${PR_NUMBER} - USE_MOCK_DATA=true + # temporarily setting to false to test if this build works with guidellm + USE_MOCK_DATA=false BASE_PATH=/ui/pr/${PR_NUMBER} GIT_SHA=${{ github.sha }} export ASSET_PREFIX=${ASSET_PREFIX} diff --git a/benchmarks.html b/benchmarks.html new file mode 100644 index 00000000..3c02dc09 --- /dev/null +++ b/benchmarks.html @@ -0,0 +1,819 @@ +GuideLLM
\ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a78b1fc5..36ab1e8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dependencies = [ "pyyaml>=6.0.0", "rich", "transformers", + "pyhumps>=3.8.0", ] [project.optional-dependencies] diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 7dc06835..8ef06f93 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -206,7 +206,7 @@ def cli(): help=( "The path to save the output to. If it is a directory, " "it will save benchmarks.json under it. " - "Otherwise, json, yaml, or csv files are supported for output types " + "Otherwise, json, yaml, csv, or html files are supported for output types " "which will be read from the extension for the file path." ), ) diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 4847160d..9bd10f1a 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Literal, Optional, Union +import humps import yaml from pydantic import Field from rich.console import Console @@ -25,6 +26,8 @@ StandardBaseModel, StatusDistributionSummary, ) +from guidellm.presentation import UIDataBuilder +from guidellm.presentation.injector import create_report from guidellm.scheduler import strategy_display_str from guidellm.utils import Colors, split_text_list_by_length @@ -68,6 +71,9 @@ def load_file(path: Union[str, Path]) -> "GenerativeBenchmarksReport": if type_ == "csv": raise ValueError(f"CSV file type is not supported for loading: {path}.") + if type_ == "html": + raise ValueError(f"HTML file type is not supported for loading: {path}.") + raise ValueError(f"Unsupported file type: {type_} for {path}.") benchmarks: list[GenerativeBenchmark] = Field( @@ -114,6 +120,9 @@ def save_file(self, path: Union[str, Path]) -> Path: if type_ == "csv": return self.save_csv(path) + if type_ == "html": + return self.save_html(path) + raise ValueError(f"Unsupported file type: {type_} for {path}.") def save_json(self, path: Union[str, Path]) -> Path: @@ -220,11 +229,44 @@ def save_csv(self, path: Union[str, Path]) -> Path: return path + def save_html(self, path: str | Path) -> Path: + """ + Download html, inject report data and save to a file. + If the file is a directory, it will create the report in a file named + benchmarks.html under the directory. + + :param path: The path to create the report at. + :return: The path to the report. + """ + + # json_data = json.dumps(data, indent=2) + # thing = f'window.{variable_name} = {json_data};' + + data_builder = UIDataBuilder(self.benchmarks) + data = data_builder.to_dict() + camel_data = humps.camelize(data) + ui_api_data = { + f"window.{humps.decamelize(k)} = {{}};": f"window.{humps.decamelize(k)} = {json.dumps(v, indent=2)};\n" + for k, v in camel_data.items() + } + print("________") + print("________") + print("________") + print("________") + print("ui_api_data") + print(ui_api_data) + print("________") + print("________") + print("________") + print("________") + create_report(ui_api_data, path) + return path + @staticmethod def _file_setup( path: Union[str, Path], - default_file_type: Literal["json", "yaml", "csv"] = "json", - ) -> tuple[Path, Literal["json", "yaml", "csv"]]: + default_file_type: Literal["json", "yaml", "csv", "html"] = "json", + ) -> tuple[Path, Literal["json", "yaml", "csv", "html"]]: path = Path(path) if not isinstance(path, Path) else path if path.is_dir(): @@ -242,6 +284,9 @@ def _file_setup( if path_suffix in [".csv"]: return path, "csv" + if path_suffix in [".html"]: + return path, "html" + raise ValueError(f"Unsupported file extension: {path_suffix} for {path}.") @staticmethod diff --git a/src/guidellm/config.py b/src/guidellm/config.py index ed7e782b..f8a2eecd 100644 --- a/src/guidellm/config.py +++ b/src/guidellm/config.py @@ -30,10 +30,10 @@ class Environment(str, Enum): ENV_REPORT_MAPPING = { - Environment.PROD: "https://guidellm.neuralmagic.com/local-report/index.html", - Environment.STAGING: "https://staging.guidellm.neuralmagic.com/local-report/index.html", - Environment.DEV: "https://dev.guidellm.neuralmagic.com/local-report/index.html", - Environment.LOCAL: "tests/dummy/report.html", + Environment.PROD: "https://neuralmagic.github.io/guidellm/ui/latest/index.html", + Environment.STAGING: "https://neuralmagic.github.io/guidellm/ui/staging/latest/index.html", + Environment.DEV: "https://neuralmagic.github.io/guidellm/ui/pr/191/index.html", + Environment.LOCAL: "https://neuralmagic.github.io/guidellm/ui/dev/index.html", } @@ -87,6 +87,14 @@ class OpenAISettings(BaseModel): max_output_tokens: int = 16384 +class ReportGenerationSettings(BaseModel): + """ + Report generation settings for the application + """ + + source: str = "" + + class Settings(BaseSettings): """ All the settings are powered by pydantic_settings and could be @@ -109,7 +117,7 @@ class Settings(BaseSettings): ) # general settings - env: Environment = Environment.PROD + env: Environment = Environment.DEV default_async_loop_sleep: float = 10e-5 logging: LoggingSettings = LoggingSettings() default_sweep_number: int = 10 @@ -140,6 +148,9 @@ class Settings(BaseSettings): ) openai: OpenAISettings = OpenAISettings() + # Report settings + report_generation: ReportGenerationSettings = ReportGenerationSettings() + # Output settings table_border_char: str = "=" table_headers_border_char: str = "-" @@ -148,6 +159,8 @@ class Settings(BaseSettings): @model_validator(mode="after") @classmethod def set_default_source(cls, values): + if not values.report_generation.source: + values.report_generation.source = ENV_REPORT_MAPPING.get(values.env) return values def generate_env_file(self) -> str: diff --git a/src/guidellm/objects/statistics.py b/src/guidellm/objects/statistics.py index 552b5c20..7831b2cf 100644 --- a/src/guidellm/objects/statistics.py +++ b/src/guidellm/objects/statistics.py @@ -37,6 +37,9 @@ class Percentiles(StandardBaseModel): p25: float = Field( description="The 25th percentile of the distribution.", ) + p50: float = Field( + description="The 50th percentile of the distribution.", + ) p75: float = Field( description="The 75th percentile of the distribution.", ) @@ -159,6 +162,7 @@ def from_distribution_function( p05=cdf[np.argmax(cdf[:, 1] >= 0.05), 0].item(), # noqa: PLR2004 p10=cdf[np.argmax(cdf[:, 1] >= 0.1), 0].item(), # noqa: PLR2004 p25=cdf[np.argmax(cdf[:, 1] >= 0.25), 0].item(), # noqa: PLR2004 + p50=cdf[np.argmax(cdf[:, 1] >= 0.50), 0].item(), # noqa: PLR2004 p75=cdf[np.argmax(cdf[:, 1] >= 0.75), 0].item(), # noqa: PLR2004 p90=cdf[np.argmax(cdf[:, 1] >= 0.9), 0].item(), # noqa: PLR2004 p95=cdf[np.argmax(cdf[:, 1] >= 0.95), 0].item(), # noqa: PLR2004 @@ -172,6 +176,7 @@ def from_distribution_function( p05=0, p10=0, p25=0, + p50=0, p75=0, p90=0, p95=0, diff --git a/src/guidellm/presentation/__init__.py b/src/guidellm/presentation/__init__.py new file mode 100644 index 00000000..872188db --- /dev/null +++ b/src/guidellm/presentation/__init__.py @@ -0,0 +1,28 @@ +from .builder import UIDataBuilder +from .data_models import ( + BenchmarkDatum, + Bucket, + Dataset, + Distribution, + Model, + RunInfo, + Server, + TokenDetails, + WorkloadDetails, +) +from .injector import create_report, inject_data + +__all__ = [ + "BenchmarkDatum", + "Bucket", + "Dataset", + "Distribution", + "Model", + "RunInfo", + "Server", + "TokenDetails", + "UIDataBuilder", + "WorkloadDetails", + "create_report", + "inject_data", +] diff --git a/src/guidellm/presentation/builder.py b/src/guidellm/presentation/builder.py new file mode 100644 index 00000000..986939a4 --- /dev/null +++ b/src/guidellm/presentation/builder.py @@ -0,0 +1,28 @@ +from typing import Any + +from guidellm.benchmark.benchmark import GenerativeBenchmark + +from .data_models import BenchmarkDatum, RunInfo, WorkloadDetails + +__all__ = ["UIDataBuilder"] + + +class UIDataBuilder: + def __init__(self, benchmarks: list[GenerativeBenchmark]): + self.benchmarks = benchmarks + + def build_run_info(self): + return RunInfo.from_benchmarks(self.benchmarks) + + def build_workload_details(self): + return WorkloadDetails.from_benchmarks(self.benchmarks) + + def build_benchmarks(self): + return [BenchmarkDatum.from_benchmark(b) for b in self.benchmarks] + + def to_dict(self) -> dict[str, Any]: + return { + "run_info": self.build_run_info().model_dump(), + "workload_details": self.build_workload_details().model_dump(), + "benchmarks": [b.model_dump() for b in self.build_benchmarks()], + } diff --git a/src/guidellm/presentation/data_models.py b/src/guidellm/presentation/data_models.py new file mode 100644 index 00000000..d2a5d86c --- /dev/null +++ b/src/guidellm/presentation/data_models.py @@ -0,0 +1,236 @@ +import random +from collections import defaultdict +from math import ceil +from typing import List, Optional, Tuple + +from pydantic import BaseModel, computed_field + +from guidellm.benchmark.benchmark import GenerativeBenchmark +from guidellm.objects.statistics import DistributionSummary + +__all__ = [ + "BenchmarkDatum", + "Bucket", + "Dataset", + "Distribution", + "Model", + "RunInfo", + "Server", + "TokenDetails", + "WorkloadDetails", +] + + +class Bucket(BaseModel): + value: float + count: int + + @staticmethod + def from_data( + data: List[float], + bucket_width: Optional[float] = None, + n_buckets: Optional[int] = None, + ) -> Tuple[List["Bucket"], float]: + if not data: + return [], 1.0 + + min_v = min(data) + max_v = max(data) + range_v = max_v - min_v + + if bucket_width is None: + if n_buckets is None: + n_buckets = 10 + bucket_width = range_v / n_buckets + else: + n_buckets = ceil(range_v / bucket_width) + + bucket_counts = defaultdict(int) + for val in data: + idx = int((val - min_v) // bucket_width) + if idx >= n_buckets: + idx = n_buckets - 1 + bucket_start = min_v + idx * bucket_width + bucket_counts[bucket_start] += 1 + + buckets = [ + Bucket(value=start, count=count) + for start, count in sorted(bucket_counts.items()) + ] + return buckets, bucket_width + + +class Model(BaseModel): + name: str + size: int + + +class Dataset(BaseModel): + name: str + + +class RunInfo(BaseModel): + model: Model + task: str + timestamp: float + dataset: Dataset + + @classmethod + def from_benchmarks(cls, benchmarks: list[GenerativeBenchmark]): + model = benchmarks[0].worker.backend_model or "N/A" + timestamp = max( + bm.run_stats.start_time for bm in benchmarks if bm.start_time is not None + ) + return cls( + model=Model(name=model, size=0), + task="N/A", + timestamp=timestamp, + dataset=Dataset(name="N/A"), + ) + + +class Distribution(BaseModel): + statistics: Optional[DistributionSummary] = None + buckets: list[Bucket] + bucket_width: float + + +class TokenDetails(BaseModel): + samples: list[str] + token_distributions: Distribution + + +class Server(BaseModel): + target: str + + +class RequestOverTime(BaseModel): + num_benchmarks: int + requests_over_time: Distribution + + +class WorkloadDetails(BaseModel): + prompts: TokenDetails + generations: TokenDetails + requests_over_time: RequestOverTime + rate_type: str + server: Server + + @classmethod + def from_benchmarks(cls, benchmarks: list[GenerativeBenchmark]): + target = benchmarks[0].worker.backend_target + rate_type = benchmarks[0].args.profile.type_ + successful_requests = [ + req for bm in benchmarks for req in bm.requests.successful + ] + sample_indices = random.sample( + range(len(successful_requests)), min(5, len(successful_requests)) + ) + sample_prompts = [ + successful_requests[i].prompt.replace("\n", " ").replace('"', "'") + for i in sample_indices + ] + sample_outputs = [ + successful_requests[i].output.replace("\n", " ").replace('"', "'") + for i in sample_indices + ] + + prompt_tokens = [ + req.prompt_tokens for bm in benchmarks for req in bm.requests.successful + ] + output_tokens = [ + req.output_tokens for bm in benchmarks for req in bm.requests.successful + ] + + prompt_token_buckets, _prompt_token_bucket_width = Bucket.from_data( + prompt_tokens, 1 + ) + output_token_buckets, _output_token_bucket_width = Bucket.from_data( + output_tokens, 1 + ) + + prompt_token_stats = DistributionSummary.from_values(prompt_tokens) + output_token_stats = DistributionSummary.from_values(output_tokens) + prompt_token_distributions = Distribution( + statistics=prompt_token_stats, buckets=prompt_token_buckets, bucket_width=1 + ) + output_token_distributions = Distribution( + statistics=output_token_stats, buckets=output_token_buckets, bucket_width=1 + ) + + min_start_time = benchmarks[0].run_stats.start_time + + all_req_times = [ + req.start_time - min_start_time + for bm in benchmarks + for req in bm.requests.successful + if req.start_time is not None + ] + number_of_buckets = len(benchmarks) + request_over_time_buckets, bucket_width = Bucket.from_data( + all_req_times, None, number_of_buckets + ) + request_over_time_distribution = Distribution( + buckets=request_over_time_buckets, bucket_width=bucket_width + ) + return cls( + prompts=TokenDetails( + samples=sample_prompts, token_distributions=prompt_token_distributions + ), + generations=TokenDetails( + samples=sample_outputs, token_distributions=output_token_distributions + ), + requests_over_time=RequestOverTime( + requests_over_time=request_over_time_distribution, + num_benchmarks=number_of_buckets, + ), + rate_type=rate_type, + server=Server(target=target), + ) + + +class TabularDistributionSummary(DistributionSummary): + """ + Same fields as `DistributionSummary`, but adds a ready-to-serialize/iterate + `percentile_rows` helper. + """ + + @computed_field + @property + def percentile_rows(self) -> list[dict[str, float]]: + return [ + {"percentile": name, "value": value} + for name, value in self.percentiles.model_dump().items() + ] + + @classmethod + def from_distribution_summary( + cls, distribution: DistributionSummary + ) -> "TabularDistributionSummary": + return cls(**distribution.model_dump()) + + +class BenchmarkDatum(BaseModel): + requests_per_second: float + tpot: TabularDistributionSummary + ttft: TabularDistributionSummary + throughput: TabularDistributionSummary + time_per_request: TabularDistributionSummary + + @classmethod + def from_benchmark(cls, bm: GenerativeBenchmark): + return cls( + requests_per_second=bm.metrics.requests_per_second.successful.mean, + tpot=TabularDistributionSummary.from_distribution_summary( + bm.metrics.inter_token_latency_ms.successful + ), + ttft=TabularDistributionSummary.from_distribution_summary( + bm.metrics.time_to_first_token_ms.successful + ), + throughput=TabularDistributionSummary.from_distribution_summary( + bm.metrics.output_tokens_per_second.successful + ), + time_per_request=TabularDistributionSummary.from_distribution_summary( + bm.metrics.request_latency.successful + ), + ) diff --git a/src/guidellm/presentation/injector.py b/src/guidellm/presentation/injector.py new file mode 100644 index 00000000..e60a72ed --- /dev/null +++ b/src/guidellm/presentation/injector.py @@ -0,0 +1,65 @@ +from pathlib import Path +import re +from typing import Union + +from guidellm.config import settings +from guidellm.utils.text import load_text + +__all__ = ["create_report", "inject_data"] + + +def create_report(js_data: dict, output_path: Union[str, Path]) -> Path: + """ + Creates a report from the dictionary and saves it to the output path. + + :param js_data: dict with match str and json data to inject + :type js_data: dict + :param output_path: the file to save the report to. + :type output_path: str + :return: the path to the saved report + :rtype: str + """ + + if not isinstance(output_path, Path): + output_path = Path(output_path) + + html_content = load_text(settings.report_generation.source) + report_content = inject_data( + js_data, + html_content, + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report_content) + print(f"Report saved to {output_path}") + return output_path + +def inject_data( + js_data: dict, + html: str, +) -> str: + """ + Injects the json data into the HTML, replacing placeholders only within the section. + + :param js_data: the json data to inject + :type js_data: dict + :param html: the html to inject the data into + :type html: str + :return: the html with the json data injected + :rtype: str + """ + head_match = re.search(r"]*>(.*?)", html, re.DOTALL | re.IGNORECASE) + if not head_match: + return html # or raise error? + + head_content = head_match.group(1) + + # Replace placeholders only inside the content + for placeholder, script in js_data.items(): + head_content = head_content.replace(placeholder, script) + + # Rebuild the HTML + new_head = f"{head_content}" + html = html[:head_match.start()] + new_head + html[head_match.end():] + + return html \ No newline at end of file diff --git a/src/ui/.env.local b/src/ui/.env.local index 44ab168b..b9d5ff2b 100644 --- a/src/ui/.env.local +++ b/src/ui/.env.local @@ -1,4 +1,4 @@ ASSET_PREFIX=http://localhost:3000 BASE_PATH=http://localhost:3000 NEXT_PUBLIC_USE_MOCK_API=true -USE_MOCK_DATA=true +USE_MOCK_DATA=false diff --git a/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx b/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx index ae9a428b..d6bf3725 100644 --- a/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx +++ b/src/ui/lib/components/MetricsSummary/MetricsSummary.component.tsx @@ -95,7 +95,7 @@ export const Component = () => { }, ]; - if ((data?.benchmarks?.length ?? 0) <= 1) { + if ((data?.length ?? 0) <= 1) { return <>; } diff --git a/src/ui/lib/components/WorkloadMetrics/WorkloadMetrics.component.tsx b/src/ui/lib/components/WorkloadMetrics/WorkloadMetrics.component.tsx index b717bb11..7be48983 100644 --- a/src/ui/lib/components/WorkloadMetrics/WorkloadMetrics.component.tsx +++ b/src/ui/lib/components/WorkloadMetrics/WorkloadMetrics.component.tsx @@ -48,10 +48,8 @@ export const Component = () => { throughput: throughputAtRPS, } = useSelector(selectInterpolatedMetrics); - const minX = Math.floor( - Math.min(...(data?.benchmarks?.map((bm) => bm.requestsPerSecond) || [])) - ); - if ((data?.benchmarks?.length ?? 0) <= 1) { + const minX = Math.floor(Math.min(...(data?.map((bm) => bm.requestsPerSecond) || []))); + if ((data?.length ?? 0) <= 1) { return <>; } return ( diff --git a/src/ui/lib/store/benchmarksWindowData.ts b/src/ui/lib/store/benchmarksWindowData.ts index e8a5cc40..7bcb209a 100644 --- a/src/ui/lib/store/benchmarksWindowData.ts +++ b/src/ui/lib/store/benchmarksWindowData.ts @@ -1,17 +1,20 @@ -export const benchmarksScript = `window.benchmarks = { - "benchmarks": [ +export const benchmarksScript = `window.benchmarks = [ { "requestsPerSecond": 0.6668550387660497, "tpot": { - "statistics": { "total": 80, "mean": 23.00635663936911, "median": 22.959455611213805, "min": 22.880917503720237, "max": 24.14080301920573, - "std": 0.18918760384209338 - }, - "percentiles": [ + "std": 0.18918760384209338, + "percentiles": { + "p50": 22.959455611213805, + "p90": 23.01789086962503, + "p95": 23.30297423947242, + "p99": 24.14080301920573, + }, + "percentileRows": [ { "percentile": "p50", "value": 22.959455611213805 @@ -31,15 +34,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 80, "mean": 49.64659512042999, "median": 49.23129081726074, "min": 44.538259506225586, "max": 55.47308921813965, - "std": 1.7735485090634995 - }, - "percentiles": [ + "std": 1.7735485090634995, + "percentiles": { + "p50": 49.23129081726074, + "p90": 50.16160011291504, + "p95": 54.918766021728516, + "p99": 55.47308921813965, + }, + "percentileRows": [ { "percentile": "p50", "value": 49.23129081726074 @@ -59,15 +66,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 210, "mean": 42.58702991319684, "median": 43.536023084668, "min": 0.0, "max": 43.68247620237872, - "std": 4.559764488536857 + "std": 4.559764488536857, + "percentiles": { + "p50": 43.536023084668, + "p90": 43.62613633999709, + "p95": 43.64020767654067, + "p99": 43.68202126662431, }, - "percentiles": [ + "percentileRows": [ { "percentile": "p50", "value": 43.536023084668 @@ -87,15 +98,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 80, "mean": 1496.706646680832, "median": 1496.1087703704834, "min": 1490.584135055542, "max": 1505.8784484863281, - "std": 3.4553340533022667 - }, - "percentiles": [ + "std": 3.4553340533022667, + "percentiles": { + "p50": 1496.1087703704834, + "p90": 1500.9305477142334, + "p95": 1505.3200721740723, + "p99": 1505.8784484863281, + }, + "percentileRows": [ { "percentile": "p50", "value": 1496.1087703704834 @@ -118,15 +133,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 28.075330129628725, "tpot": { - "statistics": { "total": 3416, "mean": 126.08707076148656, "median": 125.30853256346687, "min": 23.034303907364134, "max": 138.08223756693178, - "std": 3.508992115582193 - }, - "percentiles": [ + "std": 3.508992115582193, + "percentiles": { + "p50": 125.30853256346687, + "p90": 129.21135009281218, + "p95": 129.52291770059554, + "p99": 132.21229490686636, + }, + "percentileRows": [ { "percentile": "p50", "value": 125.30853256346687 @@ -146,15 +165,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 3416, "mean": 8585.486161415694, "median": 8965.316534042358, "min": 110.53991317749023, "max": 12575.379610061646, - "std": 1929.5632525234505 - }, - "percentiles": [ + "std": 1929.5632525234505, + "percentiles": { + "p50": 8965.316534042358, + "p90": 9231.79316520691, + "p95": 9485.00108718872, + "p99": 12096.465587615967, + }, + "percentileRows": [ { "percentile": "p50", "value": 8965.316534042358 @@ -174,15 +197,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 15981, "mean": 1795.4403743554367, "median": 670.1236619268253, "min": 0.0, "max": 838860.8, - "std": 5196.545581836957 - }, - "percentiles": [ + "std": 5196.545581836957, + "percentiles": { + "p50": 670.1236619268253, + "p90": 4068.1901066925316, + "p95": 6374.322188449848, + "p99": 16194.223938223939, + }, + "percentileRows": [ { "percentile": "p50", "value": 670.1236619268253 @@ -202,15 +229,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 3416, "mean": 16526.811318389147, "median": 17058.441638946533, "min": 1711.3444805145264, "max": 20646.55351638794, - "std": 2054.9553770234484 - }, - "percentiles": [ + "std": 2054.9553770234484, + "percentiles": { + "p50": 17058.441638946533, + "p90": 17143.84412765503, + "p95": 17248.060703277588, + "p99": 20116.52660369873, + }, + "percentileRows": [ { "percentile": "p50", "value": 17058.441638946533 @@ -233,15 +264,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 4.071681142252993, "tpot": { - "statistics": { "total": 488, "mean": 24.898151556004148, "median": 24.889995181371294, "min": 24.822999560643755, "max": 26.217273871103924, - "std": 0.11227504505081555 - }, - "percentiles": [ + "std": 0.11227504505081555, + "percentiles": { + "p50": 24.889995181371294, + "p90": 24.90483389960395, + "p95": 24.965975019666885, + "p99": 25.306613214554325, + }, + "percentileRows": [ { "percentile": "p50", "value": 24.889995181371294 @@ -261,15 +296,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 488, "mean": 58.341102033364976, "median": 58.38632583618164, "min": 44.857025146484375, "max": 111.23061180114746, - "std": 8.190008649880411 - }, - "percentiles": [ + "std": 8.190008649880411, + "percentiles": { + "p50": 58.38632583618164, + "p90": 67.66843795776367, + "p95": 68.76754760742188, + "p99": 71.46525382995605, + }, + "percentileRows": [ { "percentile": "p50", "value": 58.38632583618164 @@ -289,15 +328,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 11338, "mean": 260.42072092623033, "median": 47.630070406540995, "min": 0.0, "max": 838860.8, - "std": 886.8274389295076 - }, - "percentiles": [ + "std": 886.8274389295076, + "percentiles": { + "p50": 47.630070406540995, + "p90": 604.8895298528987, + "p95": 1621.9273008507348, + "p99": 3054.846321922797, + }, + "percentileRows": [ { "percentile": "p50", "value": 47.630070406540995 @@ -317,15 +360,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 488, "mean": 1626.5668087318297, "median": 1626.236915588379, "min": 1611.9341850280762, "max": 1690.2406215667725, - "std": 8.871477705542668 - }, - "percentiles": [ + "std": 8.871477705542668, + "percentiles": { + "p50": 1626.236915588379, + "p90": 1635.761022567749, + "p95": 1637.390375137329, + "p99": 1643.500804901123, + }, + "percentileRows": [ { "percentile": "p50", "value": 1626.236915588379 @@ -348,15 +395,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 7.466101414346809, "tpot": { - "statistics": { "total": 895, "mean": 27.56459906601014, "median": 27.525402250744047, "min": 26.69054911686824, "max": 29.5785041082473, - "std": 0.18545649185329754 - }, - "percentiles": [ + "std": 0.18545649185329754, + "percentiles": { + "p50": 27.525402250744047, + "p90": 27.62497795952691, + "p95": 27.947206345815506, + "p99": 28.41202157442687, + }, + "percentileRows": [ { "percentile": "p50", "value": 27.525402250744047 @@ -376,15 +427,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 895, "mean": 64.73036744741088, "median": 62.484025955200195, "min": 48.038482666015625, "max": 256.4809322357178, - "std": 21.677914089867077 - }, - "percentiles": [ + "std": 21.677914089867077, + "percentiles": { + "p50": 62.484025955200195, + "p90": 72.04723358154297, + "p95": 72.50738143920898, + "p99": 229.35032844543457, + }, + "percentileRows": [ { "percentile": "p50", "value": 62.484025955200195 @@ -404,15 +459,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 12465, "mean": 477.5134940335642, "median": 49.76925541382379, "min": 0.0, "max": 1677721.6, - "std": 2472.852317203968 - }, - "percentiles": [ + "std": 2472.852317203968, + "percentiles": { + "p50": 49.76925541382379, + "p90": 1191.5636363636363, + "p95": 2501.075730471079, + "p99": 7025.634840871022, + }, + "percentileRows": [ { "percentile": "p50", "value": 49.76925541382379 @@ -432,15 +491,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 895, "mean": 1800.9132816804852, "median": 1797.5835800170898, "min": 1756.2305927276611, "max": 1994.28129196167, - "std": 24.24935353039552 - }, - "percentiles": [ + "std": 24.24935353039552, + "percentiles": { + "p50": 1797.5835800170898, + "p90": 1808.2549571990967, + "p95": 1813.141107559204, + "p99": 1967.8056240081787, + }, + "percentileRows": [ { "percentile": "p50", "value": 1797.5835800170898 @@ -463,15 +526,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 10.83989165148388, "tpot": { - "statistics": { "total": 1300, "mean": 31.6048062981453, "median": 31.577579558841766, "min": 30.171105355927438, "max": 33.10690323511759, - "std": 0.15146862300990216 - }, - "percentiles": [ + "std": 0.15146862300990216, + "percentiles": { + "p50": 31.577579558841766, + "p90": 31.63230986822219, + "p95": 31.682415614052424, + "p99": 32.138043834317116, + }, + "percentileRows": [ { "percentile": "p50", "value": 31.577579558841766 @@ -491,15 +558,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 1300, "mean": 66.61205951984113, "median": 65.78803062438965, "min": 51.81550979614258, "max": 244.69709396362305, - "std": 14.858653160342651 - }, - "percentiles": [ + "std": 14.858653160342651, + "percentiles": { + "p50": 65.78803062438965, + "p90": 76.70044898986816, + "p95": 77.78120040893555, + "p99": 88.29903602600098, + }, + "percentileRows": [ { "percentile": "p50", "value": 65.78803062438965 @@ -519,15 +590,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 12708, "mean": 693.3695002980695, "median": 55.59272071785492, "min": 0.0, "max": 838860.8, - "std": 2454.288991845712 - }, - "percentiles": [ + "std": 2454.288991845712, + "percentiles": { + "p50": 55.59272071785492, + "p90": 1897.875113122172, + "p95": 2931.030048916841, + "p99": 7108.989830508474, + }, + "percentileRows": [ { "percentile": "p50", "value": 55.59272071785492 @@ -547,15 +622,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 1300, "mean": 2057.3723330864545, "median": 2056.5311908721924, "min": 2027.0307064056396, "max": 2233.853578567505, - "std": 16.334707021033957 - }, - "percentiles": [ + "std": 16.334707021033957, + "percentiles": { + "p50": 2056.5311908721924, + "p90": 2065.953254699707, + "p95": 2067.810297012329, + "p99": 2087.8031253814697, + }, + "percentileRows": [ { "percentile": "p50", "value": 2056.5311908721924 @@ -578,15 +657,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 14.211845819540324, "tpot": { - "statistics": { "total": 1704, "mean": 35.695500394825224, "median": 35.60370869106717, "min": 34.798149078611345, "max": 38.94662857055664, - "std": 0.24967658675392423 - }, - "percentiles": [ + "std": 0.24967658675392423, + "percentiles": { + "p50": 35.60370869106717, + "p90": 35.84100708128914, + "p95": 36.09923778041716, + "p99": 36.71476489207784, + }, + "percentileRows": [ { "percentile": "p50", "value": 35.60370869106717 @@ -606,15 +689,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 1704, "mean": 74.19940031750102, "median": 71.50626182556152, "min": 53.643226623535156, "max": 322.6609230041504, - "std": 23.98415146629138 - }, - "percentiles": [ + "std": 23.98415146629138, + "percentiles": { + "p50": 71.50626182556152, + "p90": 83.71734619140625, + "p95": 98.2356071472168, + "p99": 113.44718933105469, + }, + "percentileRows": [ { "percentile": "p50", "value": 71.50626182556152 @@ -634,15 +721,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 15532, "mean": 908.715763654939, "median": 98.84067397195712, "min": 0.0, "max": 838860.8, - "std": 3628.67537220603 - }, - "percentiles": [ + "std": 3628.67537220603, + "percentiles": { + "p50": 98.84067397195712, + "p90": 2205.2071503680336, + "p95": 3775.251125112511, + "p99": 10512.040100250626, + }, + "percentileRows": [ { "percentile": "p50", "value": 98.84067397195712 @@ -662,15 +753,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 1704, "mean": 2321.92987861208, "median": 2313.3785724639893, "min": 2290.93074798584, "max": 2594.4881439208984, - "std": 29.46118583560937 - }, - "percentiles": [ + "std": 29.46118583560937, + "percentiles": { + "p50": 2313.3785724639893, + "p90": 2339.4439220428467, + "p95": 2341.9249057769775, + "p99": 2370.450496673584, + }, + "percentileRows": [ { "percentile": "p50", "value": 2313.3785724639893 @@ -693,15 +788,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 17.5623040970073, "tpot": { - "statistics": { "total": 2106, "mean": 39.546438065771135, "median": 39.47442675393725, "min": 38.74176740646362, "max": 43.32651032341851, - "std": 0.3121106751660994 - }, - "percentiles": [ + "std": 0.3121106751660994, + "percentiles": { + "p50": 39.47442675393725, + "p90": 39.722594003828746, + "p95": 40.083578654697966, + "p99": 40.73049983040231, + }, + "percentileRows": [ { "percentile": "p50", "value": 39.47442675393725 @@ -721,15 +820,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 2106, "mean": 85.68002797259905, "median": 89.88213539123535, "min": 57.360172271728516, "max": 362.8504276275635, - "std": 27.802786177158218 - }, - "percentiles": [ + "std": 27.802786177158218, + "percentiles": { + "p50": 89.88213539123535, + "p90": 101.7305850982666, + "p95": 103.26790809631348, + "p99": 138.88931274414062, + }, + "percentileRows": [ { "percentile": "p50", "value": 89.88213539123535 @@ -749,15 +852,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 15121, "mean": 1123.0284569989917, "median": 99.91909855397003, "min": 0.0, "max": 932067.5555555555, - "std": 4358.833642800455 - }, - "percentiles": [ + "std": 4358.833642800455, + "percentiles": { + "p50": 99.91909855397003, + "p90": 2868.8809849521203, + "p95": 4848.906358381503, + "p99": 12905.55076923077, + }, + "percentileRows": [ { "percentile": "p50", "value": 99.91909855397003 @@ -777,15 +884,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 2106, "mean": 2575.916517267653, "median": 2573.6281871795654, "min": 2533.904790878296, "max": 2894.4458961486816, - "std": 33.18594265783404 - }, - "percentiles": [ + "std": 33.18594265783404, + "percentiles": { + "p50": 2573.6281871795654, + "p90": 2588.9015197753906, + "p95": 2591.136932373047, + "p99": 2700.568437576294, + }, + "percentileRows": [ { "percentile": "p50", "value": 2573.6281871795654 @@ -808,15 +919,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 20.885632360055222, "tpot": { - "statistics": { "total": 2505, "mean": 44.20494748431818, "median": 44.02147020612444, "min": 42.981475591659546, "max": 52.62617986710345, - "std": 1.0422073399474652 - }, - "percentiles": [ + "std": 1.0422073399474652, + "percentiles": { + "p50": 44.02147020612444, + "p90": 44.47330747331892, + "p95": 45.131300316482296, + "p99": 50.400745301019576, + }, + "percentileRows": [ { "percentile": "p50", "value": 44.02147020612444 @@ -836,15 +951,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 2505, "mean": 98.4621736103903, "median": 95.84355354309082, "min": 61.09285354614258, "max": 524.099588394165, - "std": 34.20521833421915 - }, - "percentiles": [ + "std": 34.20521833421915, + "percentiles": { + "p50": 95.84355354309082, + "p90": 109.4822883605957, + "p95": 111.46354675292969, + "p99": 334.31243896484375, + }, + "percentileRows": [ { "percentile": "p50", "value": 95.84355354309082 @@ -864,15 +983,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 14779, "mean": 1335.7133120200747, "median": 104.45284522475407, "min": 0.0, "max": 1677721.6, - "std": 5200.1934248077005 - }, - "percentiles": [ + "std": 5200.1934248077005, + "percentiles": { + "p50": 104.45284522475407, + "p90": 3472.1059602649007, + "p95": 5882.6143057503505, + "p99": 15768.060150375939, + }, + "percentileRows": [ { "percentile": "p50", "value": 104.45284522475407 @@ -892,15 +1015,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 2505, "mean": 2882.6246785070603, "median": 2869.71378326416, "min": 2826.8485069274902, "max": 3324.9876499176025, - "std": 78.07038363701177 - }, - "percentiles": [ + "std": 78.07038363701177, + "percentiles": { + "p50": 2869.71378326416, + "p90": 2888.715982437134, + "p95": 2937.7262592315674, + "p99": 3282.898426055908, + }, + "percentileRows": [ { "percentile": "p50", "value": 2869.71378326416 @@ -923,15 +1050,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 24.179871480414207, "tpot": { - "statistics": { "total": 2900, "mean": 51.023722283946924, "median": 50.24327550615583, "min": 47.58137645143451, "max": 60.63385087935651, - "std": 2.0749227872708285 - }, - "percentiles": [ + "std": 2.0749227872708285, + "percentiles": { + "p50": 50.24327550615583, + "p90": 52.928451507810564, + "p95": 57.28437408568367, + "p99": 58.51330454387362, + }, + "percentileRows": [ { "percentile": "p50", "value": 50.24327550615583 @@ -951,15 +1082,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 2900, "mean": 123.56691516678907, "median": 115.33927917480469, "min": 88.05131912231445, "max": 594.1901206970215, - "std": 44.50765227271787 - }, - "percentiles": [ + "std": 44.50765227271787, + "percentiles": { + "p50": 115.33927917480469, + "p90": 141.8297290802002, + "p95": 144.49095726013184, + "p99": 375.5221366882324, + }, + "percentileRows": [ { "percentile": "p50", "value": 115.33927917480469 @@ -979,15 +1114,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 14925, "mean": 1546.3194569459229, "median": 138.59511614843208, "min": 0.0, "max": 1677721.6, - "std": 5844.302138842639 - }, - "percentiles": [ + "std": 5844.302138842639, + "percentiles": { + "p50": 138.59511614843208, + "p90": 3916.250233426704, + "p95": 6678.828025477707, + "p99": 17924.37606837607, + }, + "percentileRows": [ { "percentile": "p50", "value": 138.59511614843208 @@ -1007,15 +1146,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 2900, "mean": 3336.9750574539444, "median": 3282.672882080078, "min": 3228.010654449463, "max": 3863.8863563537598, - "std": 141.37106520368962 - }, - "percentiles": [ + "std": 141.37106520368962, + "percentiles": { + "p50": 3282.672882080078, + "p90": 3561.7692470550537, + "p95": 3737.921953201294, + "p99": 3811.5434646606445, + }, + "percentileRows": [ { "percentile": "p50", "value": 3282.672882080078 @@ -1038,15 +1181,19 @@ export const benchmarksScript = `window.benchmarks = { { "requestsPerSecond": 27.382251189847466, "tpot": { - "statistics": { "total": 3285, "mean": 62.44881585866599, "median": 60.908238093058266, "min": 58.94644298250713, "max": 72.59870383699061, - "std": 2.9764436606898887 - }, - "percentiles": [ + "std": 2.9764436606898887, + "percentiles": { + "p50": 60.908238093058266, + "p90": 68.3861043718126, + "p95": 69.21934324597555, + "p99": 70.13290269034249, + }, + "percentileRows": [ { "percentile": "p50", "value": 60.908238093058266 @@ -1066,15 +1213,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "ttft": { - "statistics": { "total": 3285, "mean": 142.7834399758953, "median": 129.18686866760254, "min": 92.2248363494873, "max": 802.5562763214111, - "std": 54.896961282893 - }, - "percentiles": [ + "std": 54.896961282893, + "percentiles": { + "p50": 129.18686866760254, + "p90": 158.26964378356934, + "p95": 166.79859161376953, + "p99": 422.8503704071045, + }, + "percentileRows": [ { "percentile": "p50", "value": 129.18686866760254 @@ -1094,15 +1245,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "throughput": { - "statistics": { "total": 15706, "mean": 1751.1720673421933, "median": 318.5950626661603, "min": 0.0, "max": 1677721.6, - "std": 6434.120608249914 - }, - "percentiles": [ + "std": 6434.120608249914, + "percentiles": { + "p50": 318.5950626661603, + "p90": 4165.147964250248, + "p95": 7194.346483704974, + "p99": 19878.218009478675, + }, + "percentileRows": [ { "percentile": "p50", "value": 318.5950626661603 @@ -1122,15 +1277,19 @@ export const benchmarksScript = `window.benchmarks = { ] }, "timePerRequest": { - "statistics": { "total": 3285, "mean": 4076.002237894764, "median": 3972.564697265625, "min": 3890.990972518921, "max": 4623.138666152954, - "std": 197.81266460135544 - }, - "percentiles": [ + "std": 197.81266460135544, + "percentiles": { + "p50": 3972.564697265625, + "p90": 4444.445371627808, + "p95": 4506.659030914307, + "p99": 4553.745985031128, + }, + "percentileRows": [ { "percentile": "p50", "value": 3972.564697265625 @@ -1150,5 +1309,4 @@ export const benchmarksScript = `window.benchmarks = { ] } } - ] -};`; +];`; diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.api.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.api.ts index 67d867d7..838dbc7a 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.api.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.api.ts @@ -1,26 +1,31 @@ import { ThunkDispatch, UnknownAction } from '@reduxjs/toolkit'; import { createApi, fetchBaseQuery } from '@reduxjs/toolkit/query/react'; -import { Benchmarks, MetricData } from './benchmarks.interfaces'; +import { Benchmarks, Statistics } from './benchmarks.interfaces'; import { formatNumber } from '../../../utils/helpers'; import { defaultPercentile } from '../slo/slo.constants'; import { setSloData } from '../slo/slo.slice'; const USE_MOCK_API = process.env.NEXT_PUBLIC_USE_MOCK_API === 'true'; +// currently the injector requires 'window.benchmarks = {};' to be present in the html, but benchmarks is expected to be null or an array const fetchBenchmarks = () => { - return { data: window.benchmarks as Benchmarks }; + let benchmarks = window.benchmarks; + if (!Array.isArray(benchmarks)) { + benchmarks = []; + } + return { data: benchmarks as Benchmarks }; }; const getAverageValueForPercentile = ( - firstMetric: MetricData, - lastMetric: MetricData, + firstMetric: Statistics, + lastMetric: Statistics, percentile: string ) => { - const firstPercentile = firstMetric.percentiles.find( + const firstPercentile = firstMetric?.percentileRows.find( (p) => p.percentile === percentile ); - const lastPercentile = lastMetric.percentiles.find( + const lastPercentile = lastMetric?.percentileRows.find( (p) => p.percentile === percentile ); return ((firstPercentile?.value ?? 0) + (lastPercentile?.value ?? 0)) / 2; @@ -32,33 +37,33 @@ const setDefaultSLOs = ( dispatch: ThunkDispatch ) => { // temporarily set default slo values, long term the backend should set default slos that will not just be the avg at the default percentile - const firstBM = data.benchmarks[0]; - const lastBM = data.benchmarks[data.benchmarks.length - 1]; + const firstBM = data[0]; + const lastBM = data[data.length - 1]; const ttftAvg = getAverageValueForPercentile( - firstBM.ttft, - lastBM.ttft, + firstBM?.ttft, + lastBM?.ttft, defaultPercentile ); const tpotAvg = getAverageValueForPercentile( - firstBM.tpot, - lastBM.tpot, + firstBM?.tpot, + lastBM?.tpot, defaultPercentile ); const timePerRequestAvg = getAverageValueForPercentile( - firstBM.timePerRequest, - lastBM.timePerRequest, + firstBM?.timePerRequest, + lastBM?.timePerRequest, defaultPercentile ); const throughputAvg = getAverageValueForPercentile( - firstBM.throughput, - lastBM.throughput, + firstBM?.throughput, + lastBM?.throughput, defaultPercentile ); dispatch( setSloData({ - currentRequestRate: firstBM.requestsPerSecond, + currentRequestRate: firstBM?.requestsPerSecond, current: { ttft: formatNumber(ttftAvg, 0), tpot: formatNumber(tpotAvg, 0), diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.constants.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.constants.ts index deb444b2..38bddb74 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.constants.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.constants.ts @@ -2,6 +2,4 @@ import { Benchmarks, Name } from './benchmarks.interfaces'; export const name: Readonly = 'benchmarks'; -export const initialState: Benchmarks = { - benchmarks: [], -}; +export const initialState: Benchmarks = []; diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts index 4dc755b2..602ae17e 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.interfaces.ts @@ -1,44 +1,32 @@ export type Name = 'benchmarks'; -interface Statistics { +export interface Statistics { total: number; mean: number; std: number; median: number; min: number; max: number; + percentileRows: Percentile[]; + percentiles: Record; } export type PercentileValues = 'p50' | 'p90' | 'p95' | 'p99'; interface Percentile { - percentile: string; + percentile: PercentileValues; value: number; } -interface Bucket { - value: number; - count: number; -} - -export interface MetricData { - statistics: Statistics; - percentiles: Percentile[]; - buckets: Bucket[]; - bucketWidth: number; -} - export interface BenchmarkMetrics { - ttft: MetricData; - tpot: MetricData; - timePerRequest: MetricData; - throughput: MetricData; + ttft: Statistics; + tpot: Statistics; + timePerRequest: Statistics; + throughput: Statistics; } export interface Benchmark extends BenchmarkMetrics { requestsPerSecond: number; } -export type Benchmarks = { - benchmarks: Benchmark[]; -}; +export type Benchmarks = Benchmark[]; diff --git a/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts b/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts index 9453f772..71366448 100644 --- a/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts +++ b/src/ui/lib/store/slices/benchmarks/benchmarks.selectors.ts @@ -14,7 +14,8 @@ export const selectBenchmarks = (state: RootState) => state.benchmarks.data; export const selectMetricsSummaryLineData = createSelector( [selectBenchmarks, selectSloState], (benchmarks, sloState) => { - const sortedByRPS = benchmarks?.benchmarks + console.log('🚀 ~ benchmarks.selectors.ts:18 ~ benchmarks:', benchmarks); + const sortedByRPS = benchmarks ?.slice() ?.sort((bm1, bm2) => (bm1.requestsPerSecond > bm2.requestsPerSecond ? 1 : -1)); const selectedPercentile = sloState.enforcedPercentile; @@ -34,7 +35,7 @@ export const selectMetricsSummaryLineData = createSelector( metrics.forEach((metric) => { const data: Point[] = []; sortedByRPS?.forEach((benchmark) => { - const percentile = benchmark[metric].percentiles.find( + const percentile = benchmark[metric].percentileRows.find( (p) => p.percentile === selectedPercentile ); data.push({ @@ -58,11 +59,6 @@ const getDefaultMetricValues = () => ({ export const selectInterpolatedMetrics = createSelector( [selectBenchmarks, selectSloState], (benchmarks, sloState) => { - const sortedByRPS = benchmarks?.benchmarks - ?.slice() - ?.sort((bm1, bm2) => (bm1.requestsPerSecond > bm2.requestsPerSecond ? 1 : -1)); - const requestRates = sortedByRPS?.map((bm) => bm.requestsPerSecond) || []; - const { enforcedPercentile, currentRequestRate } = sloState; const metricData: { [K in keyof BenchmarkMetrics | 'mean']: { enforcedPercentileValue: number; @@ -76,6 +72,14 @@ export const selectInterpolatedMetrics = createSelector( throughput: getDefaultMetricValues(), mean: getDefaultMetricValues(), }; + if ((benchmarks?.length || 0) < 2) { + return metricData; + } + const sortedByRPS = benchmarks + ?.slice() + ?.sort((bm1, bm2) => (bm1.requestsPerSecond > bm2.requestsPerSecond ? 1 : -1)); + const requestRates = sortedByRPS?.map((bm) => bm.requestsPerSecond) || []; + const { enforcedPercentile, currentRequestRate } = sloState; const metrics: (keyof BenchmarkMetrics)[] = [ 'ttft', 'tpot', @@ -92,15 +96,13 @@ export const selectInterpolatedMetrics = createSelector( return metricData; } metrics.forEach((metric) => { - const meanValues = sortedByRPS.map((bm) => bm[metric].statistics.mean); + const meanValues = sortedByRPS.map((bm) => bm[metric].mean); const interpolateMeanAt = createMonotoneSpline(requestRates, meanValues); const interpolatedMeanValue: number = interpolateMeanAt(currentRequestRate) || 0; const percentiles: PercentileValues[] = ['p50', 'p90', 'p95', 'p99']; const valuesByPercentile = percentiles.map((p) => { const bmValuesAtP = sortedByRPS.map((bm) => { - const result = - bm[metric].percentiles.find((percentile) => percentile.percentile === p) - ?.value || 0; + const result = bm[metric].percentiles[p] || 0; return result; }); const interpolateValueAtP = createMonotoneSpline(requestRates, bmValuesAtP); @@ -126,7 +128,7 @@ export const selectMetricsDetailsLineData = createSelector( [selectBenchmarks], (benchmarks) => { const sortedByRPS = - benchmarks?.benchmarks + benchmarks ?.slice() ?.sort((bm1, bm2) => bm1.requestsPerSecond > bm2.requestsPerSecond ? 1 : -1 @@ -152,16 +154,16 @@ export const selectMetricsDetailsLineData = createSelector( } const data: { [key: string]: { data: Point[]; id: string; solid?: boolean } } = {}; - sortedByRPS[0].ttft.percentiles.forEach((p) => { + sortedByRPS[0].ttft.percentileRows.forEach((p) => { data[p.percentile] = { data: [], id: p.percentile }; }); data.mean = { data: [], id: 'mean', solid: true }; sortedByRPS?.forEach((benchmark) => { const rps = benchmark.requestsPerSecond; - benchmark[prop].percentiles.forEach((p) => { + benchmark[prop].percentileRows.forEach((p) => { data[p.percentile].data.push({ x: rps, y: p.value }); }); - const mean = benchmark[prop].statistics.mean; + const mean = benchmark[prop].mean; data.mean.data.push({ x: rps, y: mean }); }); lineData[prop] = Object.keys(data).map((key) => { diff --git a/src/ui/lib/store/slices/workloadDetails/workloadDetails.constants.ts b/src/ui/lib/store/slices/workloadDetails/workloadDetails.constants.ts index c45efa76..e6604add 100644 --- a/src/ui/lib/store/slices/workloadDetails/workloadDetails.constants.ts +++ b/src/ui/lib/store/slices/workloadDetails/workloadDetails.constants.ts @@ -1,5 +1,5 @@ import { Name, WorkloadDetails } from './workloadDetails.interfaces'; - +import { PercentileValues } from '../benchmarks/benchmarks.interfaces'; export const name: Readonly = 'workloadDetails'; export const initialState: WorkloadDetails = { @@ -13,8 +13,9 @@ export const initialState: WorkloadDetails = { median: 0, min: 0, max: 0, + percentiles: {} as Record, + percentileRows: [], }, - percentiles: [], buckets: [], bucketWidth: 0, }, @@ -29,8 +30,9 @@ export const initialState: WorkloadDetails = { median: 0, min: 0, max: 0, + percentiles: {} as Record, + percentileRows: [], }, - percentiles: [], buckets: [], bucketWidth: 0, }, @@ -45,8 +47,9 @@ export const initialState: WorkloadDetails = { median: 0, min: 0, max: 0, + percentiles: {} as Record, + percentileRows: [], }, - percentiles: [], buckets: [], bucketWidth: 0, }, diff --git a/src/ui/lib/store/slices/workloadDetails/workloadDetails.interfaces.ts b/src/ui/lib/store/slices/workloadDetails/workloadDetails.interfaces.ts index 2aa7619f..bbe5d7df 100644 --- a/src/ui/lib/store/slices/workloadDetails/workloadDetails.interfaces.ts +++ b/src/ui/lib/store/slices/workloadDetails/workloadDetails.interfaces.ts @@ -1,18 +1,6 @@ -export type Name = 'workloadDetails'; - -interface Statistics { - total: number; - mean: number; - std: number; - median: number; - min: number; - max: number; -} +import { Statistics } from '../benchmarks'; -interface Percentile { - percentile: string; - value: number; -} +export type Name = 'workloadDetails'; interface Bucket { value: number; @@ -21,7 +9,6 @@ interface Bucket { interface Distribution { statistics: Statistics; - percentiles: Percentile[]; buckets: Bucket[]; bucketWidth: number; } diff --git a/tests/ui/integration/page.test.tsx b/tests/ui/integration/page.test.tsx index 85c4bee8..cbd8f324 100644 --- a/tests/ui/integration/page.test.tsx +++ b/tests/ui/integration/page.test.tsx @@ -17,10 +17,7 @@ const route = (input: RequestInfo) => { if (url.endsWith('/run-info')) return jsonResponse({}); if (url.endsWith('/workload-details')) return jsonResponse({}); - if (url.endsWith('/benchmarks')) - return jsonResponse({ - benchmarks: mockBenchmarks, - }); + if (url.endsWith('/benchmarks')) return jsonResponse(mockBenchmarks); /* fall-through → 404 */ return { ok: false, status: 404, json: () => Promise.resolve({}) }; diff --git a/tests/ui/unit/mocks/mockBenchmarks.ts b/tests/ui/unit/mocks/mockBenchmarks.ts index 5acd7d12..884e8b89 100644 --- a/tests/ui/unit/mocks/mockBenchmarks.ts +++ b/tests/ui/unit/mocks/mockBenchmarks.ts @@ -2,15 +2,19 @@ export const mockBenchmarks = [ { requestsPerSecond: 0.6668550387660497, tpot: { - statistics: { - total: 80, - mean: 23.00635663936911, - median: 22.959455611213805, - min: 22.880917503720237, - max: 24.14080301920573, - std: 0.18918760384209338, + total: 80, + mean: 23.00635663936911, + median: 22.959455611213805, + min: 22.880917503720237, + max: 24.14080301920573, + std: 0.18918760384209338, + percentiles: { + p50: 22.959455611213805, + p90: 23.01789086962503, + p95: 23.30297423947242, + p99: 24.14080301920573, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 22.959455611213805, @@ -30,15 +34,19 @@ export const mockBenchmarks = [ ], }, ttft: { - statistics: { - total: 80, - mean: 49.64659512042999, - median: 49.23129081726074, - min: 44.538259506225586, - max: 55.47308921813965, - std: 1.7735485090634995, + total: 80, + mean: 49.64659512042999, + median: 49.23129081726074, + min: 44.538259506225586, + max: 55.47308921813965, + std: 1.7735485090634995, + percentiles: { + p50: 49.23129081726074, + p90: 50.16160011291504, + p95: 54.918766021728516, + p99: 55.47308921813965, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 49.23129081726074, @@ -58,15 +66,19 @@ export const mockBenchmarks = [ ], }, throughput: { - statistics: { - total: 210, - mean: 42.58702991319684, - median: 43.536023084668, - min: 0.0, - max: 43.68247620237872, - std: 4.559764488536857, + total: 210, + mean: 42.58702991319684, + median: 43.536023084668, + min: 0.0, + max: 43.68247620237872, + std: 4.559764488536857, + percentiles: { + p50: 43.536023084668, + p90: 43.62613633999709, + p95: 43.64020767654067, + p99: 43.68202126662431, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 43.536023084668, @@ -86,15 +98,19 @@ export const mockBenchmarks = [ ], }, timePerRequest: { - statistics: { - total: 80, - mean: 1496.706646680832, - median: 1496.1087703704834, - min: 1490.584135055542, - max: 1505.8784484863281, - std: 3.4553340533022667, + total: 80, + mean: 1496.706646680832, + median: 1496.1087703704834, + min: 1490.584135055542, + max: 1505.8784484863281, + std: 3.4553340533022667, + percentiles: { + p50: 1496.1087703704834, + p90: 1500.9305477142334, + p95: 1505.3200721740723, + p99: 1505.8784484863281, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 1496.1087703704834, @@ -117,15 +133,19 @@ export const mockBenchmarks = [ { requestsPerSecond: 28.075330129628725, tpot: { - statistics: { - total: 3416, - mean: 126.08707076148656, - median: 125.30853256346687, - min: 23.034303907364134, - max: 138.08223756693178, - std: 3.508992115582193, + total: 3416, + mean: 126.08707076148656, + median: 125.30853256346687, + min: 23.034303907364134, + max: 138.08223756693178, + std: 3.508992115582193, + percentiles: { + p50: 125.30853256346687, + p90: 129.21135009281218, + p95: 129.52291770059554, + p99: 132.21229490686636, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 125.30853256346687, @@ -145,15 +165,19 @@ export const mockBenchmarks = [ ], }, ttft: { - statistics: { - total: 3416, - mean: 8585.486161415694, - median: 8965.316534042358, - min: 110.53991317749023, - max: 12575.379610061646, - std: 1929.5632525234505, + total: 3416, + mean: 8585.486161415694, + median: 8965.316534042358, + min: 110.53991317749023, + max: 12575.379610061646, + std: 1929.5632525234505, + percentiles: { + p50: 8965.316534042358, + p90: 9231.79316520691, + p95: 9485.00108718872, + p99: 12096.465587615967, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 8965.316534042358, @@ -181,7 +205,13 @@ export const mockBenchmarks = [ max: 838860.8, std: 5196.545581836957, }, - percentiles: [ + percentiles: { + p50: 670.1236619268253, + p90: 4068.1901066925316, + p95: 6374.322188449848, + p99: 16194.223938223939, + }, + percentileRows: [ { percentile: 'p50', value: 670.1236619268253, @@ -201,15 +231,19 @@ export const mockBenchmarks = [ ], }, timePerRequest: { - statistics: { - total: 3416, - mean: 16526.811318389147, - median: 17058.441638946533, - min: 1711.3444805145264, - max: 20646.55351638794, - std: 2054.9553770234484, + total: 3416, + mean: 16526.811318389147, + median: 17058.441638946533, + min: 1711.3444805145264, + max: 20646.55351638794, + std: 2054.9553770234484, + percentiles: { + p50: 17058.441638946533, + p90: 17143.84412765503, + p95: 17248.060703277588, + p99: 20116.52660369873, }, - percentiles: [ + percentileRows: [ { percentile: 'p50', value: 17058.441638946533, diff --git a/tests/unit/objects/test_statistics.py b/tests/unit/objects/test_statistics.py index f3332758..fa8cccd0 100644 --- a/tests/unit/objects/test_statistics.py +++ b/tests/unit/objects/test_statistics.py @@ -21,6 +21,7 @@ def create_default_percentiles() -> Percentiles: p05=5.0, p10=10.0, p25=25.0, + p50=50.0, p75=75.0, p90=90.0, p95=95.0, @@ -52,6 +53,7 @@ def test_percentiles_initialization(): assert percentiles.p05 == 5.0 assert percentiles.p10 == 10.0 assert percentiles.p25 == 25.0 + assert percentiles.p50 == 50.0 assert percentiles.p75 == 75.0 assert percentiles.p90 == 90.0 assert percentiles.p95 == 95.0 @@ -67,6 +69,7 @@ def test_percentiles_invalid_initialization(): "p05": 5.0, "p10": 10.0, "p25": 25.0, + "p50": 50.0, "p75": 75.0, "p90": 90.0, "p95": 95.0, @@ -108,6 +111,7 @@ def test_distribution_summary_initilaization(): assert distribution_summary.percentiles.p05 == 5.0 assert distribution_summary.percentiles.p10 == 10.0 assert distribution_summary.percentiles.p25 == 25.0 + assert distribution_summary.percentiles.p50 == 50.0 assert distribution_summary.percentiles.p75 == 75.0 assert distribution_summary.percentiles.p90 == 90.0 assert distribution_summary.percentiles.p95 == 95.0 @@ -175,6 +179,9 @@ def test_distribution_summary_from_distribution_function(): assert distribution_summary.percentiles.p25 == pytest.approx( np.percentile(values, 25.0) ) + assert distribution_summary.percentiles.p50 == pytest.approx( + np.percentile(values, 50.0) + ) assert distribution_summary.percentiles.p75 == pytest.approx( np.percentile(values, 75.0) ) @@ -226,6 +233,9 @@ def test_distribution_summary_from_values(): assert distribution_summary.percentiles.p25 == pytest.approx( np.percentile(values, 25.0) ) + assert distribution_summary.percentiles.p50 == pytest.approx( + np.percentile(values, 50.0) + ) assert distribution_summary.percentiles.p75 == pytest.approx( np.percentile(values, 75.0) ) @@ -284,6 +294,7 @@ def test_distribution_summary_from_request_times_concurrency(): assert distribution_summary.percentiles.p05 == pytest.approx(10) assert distribution_summary.percentiles.p10 == pytest.approx(10) assert distribution_summary.percentiles.p25 == pytest.approx(10) + assert distribution_summary.percentiles.p50 == pytest.approx(10) assert distribution_summary.percentiles.p75 == pytest.approx(10) assert distribution_summary.percentiles.p90 == pytest.approx(10) assert distribution_summary.percentiles.p95 == pytest.approx(10) @@ -318,6 +329,7 @@ def test_distribution_summary_from_request_times_rate(): assert distribution_summary.percentiles.p05 == pytest.approx(10.0) assert distribution_summary.percentiles.p10 == pytest.approx(10.0) assert distribution_summary.percentiles.p25 == pytest.approx(10.0) + assert distribution_summary.percentiles.p50 == pytest.approx(10.0) assert distribution_summary.percentiles.p75 == pytest.approx(10.0) assert distribution_summary.percentiles.p90 == pytest.approx(10.0) assert distribution_summary.percentiles.p95 == pytest.approx(10.0) @@ -358,6 +370,7 @@ def test_distribution_summary_from_iterable_request_times(): assert distribution_summary.percentiles.p05 == pytest.approx(80.0) assert distribution_summary.percentiles.p10 == pytest.approx(80.0) assert distribution_summary.percentiles.p25 == pytest.approx(80.0) + assert distribution_summary.percentiles.p50 == pytest.approx(80.0) assert distribution_summary.percentiles.p75 == pytest.approx(80.0) assert distribution_summary.percentiles.p90 == pytest.approx(160.0) assert distribution_summary.percentiles.p95 == pytest.approx(160.0) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 316f13e4..ca084ec5 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -5,6 +5,7 @@ Environment, LoggingSettings, OpenAISettings, + ReportGenerationSettings, Settings, print_config, reload_settings, @@ -18,6 +19,10 @@ def test_default_settings(): assert settings.env == Environment.PROD assert settings.logging == LoggingSettings() assert settings.openai == OpenAISettings() + assert ( + settings.report_generation.source + == "https://guidellm.neuralmagic.com/local-report/index.html" + ) @pytest.mark.smoke @@ -29,6 +34,7 @@ def test_settings_from_env_variables(mocker): "GUIDELLM__logging__disabled": "true", "GUIDELLM__OPENAI__API_KEY": "test_key", "GUIDELLM__OPENAI__BASE_URL": "http://test.url", + "GUIDELLM__REPORT_GENERATION__SOURCE": "http://custom.url", }, ) @@ -37,6 +43,34 @@ def test_settings_from_env_variables(mocker): assert settings.logging.disabled is True assert settings.openai.api_key == "test_key" assert settings.openai.base_url == "http://test.url" + assert settings.report_generation.source == "http://custom.url" + + +@pytest.mark.smoke +def test_report_generation_default_source(): + settings = Settings(env=Environment.LOCAL) + assert ( + settings.report_generation.source + == "https://neuralmagic.github.io/ui/dev/index.html" + ) + + settings = Settings(env=Environment.DEV) + assert ( + settings.report_generation.source + == "https://neuralmagic.github.io/ui/dev/index.html" + ) + + settings = Settings(env=Environment.STAGING) + assert ( + settings.report_generation.source + == "https://neuralmagic.github.io/ui/staging/latest/index.html" + ) + + settings = Settings(env=Environment.PROD) + assert ( + settings.report_generation.source + == "https://neuralmagic.github.io/ui/latest/index.html" + ) @pytest.mark.sanity @@ -60,6 +94,11 @@ def test_openai_settings(): assert openai_settings.base_url == "http://test.api" +def test_report_generation_settings(): + report_settings = ReportGenerationSettings(source="http://custom.report") + assert report_settings.source == "http://custom.report" + + @pytest.mark.sanity def test_generate_env_file(): settings = Settings()