diff --git a/devops/scripts/benchmarks/html/config.js b/devops/scripts/benchmarks/html/config.js new file mode 100644 index 0000000000000..019e4e164f917 --- /dev/null +++ b/devops/scripts/benchmarks/html/config.js @@ -0,0 +1,5 @@ +// Specify the following variables to read data remotely from `remoteDataUrl`: + +//remoteDataUrl = 'https://example.com/data.json'; +//defaultCompareNames = ['baseline']; + diff --git a/devops/scripts/benchmarks/html/data.js b/devops/scripts/benchmarks/html/data.js new file mode 100644 index 0000000000000..2f1862fe621b7 --- /dev/null +++ b/devops/scripts/benchmarks/html/data.js @@ -0,0 +1,11 @@ +// This file serves as a placeholder for loading data locally: If +// `remoteDataUrl` (etc.) is not defined in config.js, the dashboard will +// attempt to load data from variables defined here instead. +// +// These variables are empty by default, and are populated by main.py if +// `--output-html local` is specified. + +benchmarkRuns = []; + +defaultCompareNames = []; + diff --git a/devops/scripts/benchmarks/html/index.html b/devops/scripts/benchmarks/html/index.html new file mode 100644 index 0000000000000..81c82c72fe471 --- /dev/null +++ b/devops/scripts/benchmarks/html/index.html @@ -0,0 +1,82 @@ + + + + + + + Benchmark Results + + + + + + + + +
+

Benchmark Results

+ +
+ +
+
+ + +
+
+
+ Options +
+
+

Display Options

+
+ + +
+
+ +
+

Suites

+
+ +
+
+ +
+

Tags

+
+ +
+
+
+
+
+ Historical Results +
+
+
+ Historical Layer Comparisons +
+
+
+ Comparisons +
+
+
+ + diff --git a/devops/scripts/benchmarks/html/scripts.js b/devops/scripts/benchmarks/html/scripts.js new file mode 100644 index 0000000000000..6cd1857387024 --- /dev/null +++ b/devops/scripts/benchmarks/html/scripts.js @@ -0,0 +1,985 @@ +// Copyright (C) 2024-2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// Core state +let activeRuns = new Set(defaultCompareNames); +let chartInstances = new Map(); +let suiteNames = new Set(); +let timeseriesData, barChartsData, allRunNames; +let activeTags = new Set(); +let layerComparisonsData; + +// DOM Elements +let runSelect, selectedRunsDiv, suiteFiltersContainer, tagFiltersContainer; + +const colorPalette = [ + 'rgb(255, 50, 80)', + 'rgb(255, 145, 15)', + 'rgb(255, 220, 0)', + 'rgb(20, 200, 50)', + 'rgb(0, 130, 255)', + 'rgb(180, 60, 255)', + 'rgb(255, 40, 200)', + 'rgb(0, 210, 180)', + 'rgb(255, 90, 0)', + 'rgb(110, 220, 0)', + 'rgb(240, 100, 170)', + 'rgb(30, 175, 255)', + 'rgb(180, 210, 0)', + 'rgb(130, 0, 220)', + 'rgb(255, 170, 0)', + 'rgb(0, 170, 110)', + 'rgb(220, 80, 60)', + 'rgb(80, 115, 230)', + 'rgb(210, 190, 0)', +]; + +// Run selector functions +function updateSelectedRuns(forceUpdate = true) { + selectedRunsDiv.innerHTML = ''; + activeRuns.forEach(name => { + selectedRunsDiv.appendChild(createRunElement(name)); + }); + if (forceUpdate) + updateCharts(); +} + +function createRunElement(name) { + const runElement = document.createElement('span'); + runElement.className = 'selected-run'; + runElement.innerHTML = `${name} `; + return runElement; +} + +function addSelectedRun() { + const selectedRun = runSelect.value; + if (selectedRun && !activeRuns.has(selectedRun)) { + activeRuns.add(selectedRun); + updateSelectedRuns(); + } +} + +function removeRun(name) { + activeRuns.delete(name); + updateSelectedRuns(); +} + +// Chart creation and update +function createChart(data, containerId, type) { + if (chartInstances.has(containerId)) { + chartInstances.get(containerId).destroy(); + } + + const ctx = document.getElementById(containerId).getContext('2d'); + const options = { + responsive: true, + plugins: { + title: { + display: true, + text: data.label + }, + subtitle: { + display: true, + text: data.lower_is_better ? "Lower is better" : "Higher is better" + }, + tooltip: { + callbacks: { + label: (context) => { + if (type === 'time') { + const point = context.raw; + return [ + `${point.seriesName}:`, + `Value: ${point.y.toFixed(2)} ${data.unit}`, + `Stddev: ${point.stddev.toFixed(2)} ${data.unit}`, + `Git Hash: ${point.gitHash}`, + ]; + } else { + return [`${context.dataset.label}:`, + `Value: ${context.parsed.y.toFixed(2)} ${data.unit}`, + ]; + } + } + } + } + }, + scales: { + y: { + title: { + display: true, + text: data.unit + }, + grace: '20%', + } + } + }; + + if (type === 'time') { + options.interaction = { + mode: 'nearest', + intersect: false + }; + options.onClick = (event, elements) => { + if (elements.length > 0) { + const point = elements[0].element.$context.raw; + if (point.gitHash && point.gitRepo) { + window.open(`https://github.com/${point.gitRepo}/commit/${point.gitHash}`, '_blank'); + } + } + }; + options.scales.x = { + type: 'timeseries', + time: { + unit: 'day' + }, + ticks: { + maxRotation: 45, + minRotation: 45, + autoSkip: true, + maxTicksLimit: 10 + } + }; + } + + const chartConfig = { + type: type === 'time' ? 'line' : 'bar', + data: type === 'time' ? { + datasets: createTimeseriesDatasets(data) + } : { + labels: data.labels, + datasets: data.datasets + }, + options: options + }; + + const chart = new Chart(ctx, chartConfig); + chartInstances.set(containerId, chart); + return chart; +} + +function createTimeseriesDatasets(data) { + return Object.entries(data.runs).map(([name, runData], index) => ({ + label: name, + data: runData.points.map(p => ({ + seriesName: name, + x: p.date, + y: p.value, + gitHash: p.git_hash, + gitRepo: p.github_repo, + stddev: p.stddev + })), + borderColor: colorPalette[index % colorPalette.length], + backgroundColor: colorPalette[index % colorPalette.length], + borderWidth: 1, + pointRadius: 3, + pointStyle: 'circle', + pointHoverRadius: 5 + })); +} + +function updateCharts() { + const filterRunData = (chart) => ({ + ...chart, + runs: Object.fromEntries( + Object.entries(chart.runs).filter(([_, data]) => + activeRuns.has(data.runName) + ) + ) + }); + + const filteredTimeseriesData = timeseriesData.map(filterRunData); + const filteredLayerComparisonsData = layerComparisonsData.map(filterRunData); + + const filteredBarChartsData = barChartsData.map(chart => ({ + ...chart, + labels: chart.labels.filter(label => activeRuns.has(label)), + datasets: chart.datasets.map(dataset => ({ + ...dataset, + data: dataset.data.filter((_, i) => activeRuns.has(chart.labels[i])) + })) + })); + + drawCharts(filteredTimeseriesData, filteredBarChartsData, filteredLayerComparisonsData); +} + +function drawCharts(filteredTimeseriesData, filteredBarChartsData, filteredLayerComparisonsData) { + // Clear existing charts + document.querySelectorAll('.charts').forEach(container => container.innerHTML = ''); + chartInstances.forEach(chart => chart.destroy()); + chartInstances.clear(); + + // Create timeseries charts + filteredTimeseriesData.forEach((data, index) => { + const containerId = `timeseries-${index}`; + const container = createChartContainer(data, containerId, 'benchmark'); + document.querySelector('.timeseries .charts').appendChild(container); + createChart(data, containerId, 'time'); + }); + + // Create layer comparison charts + filteredLayerComparisonsData.forEach((data, index) => { + const containerId = `layer-comparison-${index}`; + const container = createChartContainer(data, containerId, 'group'); + document.querySelector('.layer-comparisons .charts').appendChild(container); + createChart(data, containerId, 'time'); + }); + + // Create bar charts + filteredBarChartsData.forEach((data, index) => { + const containerId = `barchart-${index}`; + const container = createChartContainer(data, containerId, 'group'); + document.querySelector('.bar-charts .charts').appendChild(container); + createChart(data, containerId, 'bar'); + }); + + // Apply current filters + filterCharts(); +} + +function createChartContainer(data, canvasId, type) { + const container = document.createElement('div'); + container.className = 'chart-container'; + container.setAttribute('data-label', data.label); + container.setAttribute('data-suite', data.suite); + + // Check if this benchmark is marked as unstable + const metadata = metadataForLabel(data.label, type); + if (metadata && metadata.unstable) { + container.setAttribute('data-unstable', 'true'); + + // Add unstable warning + const unstableWarning = document.createElement('div'); + unstableWarning.className = 'benchmark-unstable'; + unstableWarning.textContent = metadata.unstable; + unstableWarning.style.display = isUnstableEnabled() ? 'block' : 'none'; + container.appendChild(unstableWarning); + } + + // Add description if present in metadata (moved outside of details) + if (metadata && metadata.description) { + const descElement = document.createElement('div'); + descElement.className = 'benchmark-description'; + descElement.textContent = metadata.description; + container.appendChild(descElement); + } + + // Add notes if present + if (metadata && metadata.notes) { + const noteElement = document.createElement('div'); + noteElement.className = 'benchmark-note'; + noteElement.textContent = metadata.notes; + noteElement.style.display = isNotesEnabled() ? 'block' : 'none'; + container.appendChild(noteElement); + } + + // Add tags if present + if (metadata && metadata.tags) { + container.setAttribute('data-tags', metadata.tags.join(',')); + + // Add tags display + const tagsContainer = document.createElement('div'); + tagsContainer.className = 'benchmark-tags'; + + metadata.tags.forEach(tag => { + const tagElement = document.createElement('span'); + tagElement.className = 'tag'; + tagElement.textContent = tag; + tagElement.setAttribute('data-tag', tag); + + // Add tooltip with tag description + if (benchmarkTags[tag]) { + tagElement.setAttribute('title', benchmarkTags[tag].description); + } + + tagsContainer.appendChild(tagElement); + }); + + container.appendChild(tagsContainer); + } + + const canvas = document.createElement('canvas'); + canvas.id = canvasId; + container.appendChild(canvas); + + // Create details section for extra info + const details = document.createElement('details'); + const summary = document.createElement('summary'); + summary.textContent = "Details"; + + // Add subtle download button to the summary + const downloadButton = document.createElement('button'); + downloadButton.className = 'download-button'; + downloadButton.textContent = 'Download'; + downloadButton.onclick = (event) => { + event.stopPropagation(); // Prevent details toggle + downloadChart(canvasId, data.label); + }; + summary.appendChild(downloadButton); + details.appendChild(summary); + + // Create and append extra info + const extraInfo = document.createElement('div'); + extraInfo.className = 'extra-info'; + latestRunsLookup = createLatestRunsLookup(benchmarkRuns); + extraInfo.innerHTML = generateExtraInfo(latestRunsLookup, data, 'benchmark'); + details.appendChild(extraInfo); + + container.appendChild(details); + + return container; +} + +function metadataForLabel(label, type) { + for (const [key, metadata] of Object.entries(benchmarkMetadata)) { + if (metadata.type === type && label.startsWith(key)) { + return metadata; + } + } + + return null; +} + +// Pre-compute a lookup for the latest run per label +function createLatestRunsLookup(benchmarkRuns) { + const latestRunsMap = new Map(); + + benchmarkRuns.forEach(run => { + // Yes, we need to convert the date every time. I checked. + const runDate = new Date(run.date); + run.results.forEach(result => { + const label = result.label; + if (!latestRunsMap.has(label) || runDate > new Date(latestRunsMap.get(label).date)) { + latestRunsMap.set(label, { + run, + result + }); + } + }); + }); + + return latestRunsMap; +} + +function extractLabels(data) { + // For layer comparison charts + if (data.benchmarkLabels) { + return data.benchmarkLabels; + } + + // For bar charts + if (data.datasets) { + return data.datasets.map(dataset => dataset.label); + } + + // For time series charts + return [data.label]; +} + +function generateExtraInfo(latestRunsLookup, data) { + const labels = extractLabels(data); + + return labels.map(label => { + const metadata = metadataForLabel(label, 'benchmark'); + const latestRun = latestRunsLookup.get(label); + + let html = '
'; + + if (metadata && latestRun) { + html += `${label}: ${formatCommand(latestRun.result)}
`; + + if (metadata.description) { + html += `Description: ${metadata.description}`; + } + + if (metadata.notes) { + html += `
Notes: ${metadata.notes}`; + } + + if (metadata.unstable) { + html += `
⚠️ Unstable: ${metadata.unstable}`; + } + } else { + html += `${label}: No data available`; + } + + html += '
'; + return html; + }).join(''); +} + +function formatCommand(run) { + const envVars = Object.entries(run.env || {}).map(([key, value]) => `${key}=${value}`).join(' '); + let command = run.command ? [...run.command] : []; + + return `${envVars} ${command.join(' ')}`.trim(); +} + +function downloadChart(canvasId, label) { + const chart = chartInstances.get(canvasId); + if (chart) { + const link = document.createElement('a'); + link.href = chart.toBase64Image('image/png', 1) + link.download = `${label}.png`; + link.click(); + } +} + +// URL and filtering functions +// +// Information about currently displayed charts, filters, etc. are preserved in +// the URL query string: This allows users to save/share links reproducing exact +// queries, filters, settings, etc. Therefore, for consistency, the URL needs to +// be reconstruted everytime queries, filters, etc. are changed. + +function getQueryParam(param) { + const urlParams = new URLSearchParams(window.location.search); + return urlParams.get(param); +} + +function updateURL() { + const url = new URL(window.location); + const regex = document.getElementById('bench-filter').value; + const activeSuites = getActiveSuites(); + const activeRunsList = Array.from(activeRuns); + const activeTagsList = Array.from(activeTags); + + if (regex) { + url.searchParams.set('regex', regex); + } else { + url.searchParams.delete('regex'); + } + + if (activeSuites.length > 0 && activeSuites.length != suiteNames.size) { + url.searchParams.set('suites', activeSuites.join(',')); + } else { + url.searchParams.delete('suites'); + } + + // Add tags to URL + if (activeTagsList.length > 0) { + url.searchParams.set('tags', activeTagsList.join(',')); + } else { + url.searchParams.delete('tags'); + } + + // Handle the runs parameter + if (activeRunsList.length > 0) { + // Check if the active runs are the same as default runs + const defaultRuns = new Set(defaultCompareNames || []); + const isDefaultRuns = activeRunsList.length === defaultRuns.size && + activeRunsList.every(run => defaultRuns.has(run)); + + if (isDefaultRuns) { + // If it's just the default runs, omit the parameter entirely + url.searchParams.delete('runs'); + } else { + url.searchParams.set('runs', activeRunsList.join(',')); + } + } else { + url.searchParams.delete('runs'); + } + + // Add toggle states to URL + if (isNotesEnabled()) { + url.searchParams.delete('notes'); + } else { + url.searchParams.set('notes', 'false'); + } + + if (!isUnstableEnabled()) { + url.searchParams.delete('unstable'); + } else { + url.searchParams.set('unstable', 'true'); + } + + history.replaceState(null, '', url); +} + +function filterCharts() { + const regexInput = document.getElementById('bench-filter').value; + const regex = new RegExp(regexInput, 'i'); + const activeSuites = getActiveSuites(); + + document.querySelectorAll('.chart-container').forEach(container => { + const label = container.getAttribute('data-label'); + const suite = container.getAttribute('data-suite'); + const isUnstable = container.getAttribute('data-unstable') === 'true'; + const tags = container.getAttribute('data-tags') ? + container.getAttribute('data-tags').split(',') : []; + + // Check if benchmark has all active tags (if any are selected) + const hasAllActiveTags = activeTags.size === 0 || + Array.from(activeTags).every(tag => tags.includes(tag)); + + // Hide unstable benchmarks if showUnstable is false + const shouldShow = regex.test(label) && + activeSuites.includes(suite) && + (isUnstableEnabled() || !isUnstable) && + hasAllActiveTags; + + container.style.display = shouldShow ? '' : 'none'; + }); + + updateURL(); +} + +function getActiveSuites() { + return Array.from(document.querySelectorAll('.suite-checkbox:checked')) + .map(checkbox => checkbox.getAttribute('data-suite')); +} + +// Data processing +function processTimeseriesData(benchmarkRuns) { + const resultsByLabel = {}; + + benchmarkRuns.forEach(run => { + run.results.forEach(result => { + if (!resultsByLabel[result.label]) { + resultsByLabel[result.label] = { + label: result.label, + suite: result.suite, + unit: result.unit, + lower_is_better: result.lower_is_better, + runs: {} + }; + } + + addRunDataPoint(resultsByLabel[result.label], run, result, run.name); + }); + }); + + return Object.values(resultsByLabel); +} + +function processBarChartsData(benchmarkRuns) { + const groupedResults = {}; + + benchmarkRuns.forEach(run => { + run.results.forEach(result => { + if (!result.explicit_group) return; + + if (!groupedResults[result.explicit_group]) { + // Look up group metadata + const groupMetadata = metadataForLabel(result.explicit_group); + + groupedResults[result.explicit_group] = { + label: result.explicit_group, + suite: result.suite, + unit: result.unit, + lower_is_better: result.lower_is_better, + labels: [], + datasets: [], + // Add metadata if available + description: groupMetadata?.description || null, + notes: groupMetadata?.notes || null, + unstable: groupMetadata?.unstable || null + }; + } + + const group = groupedResults[result.explicit_group]; + + if (!group.labels.includes(run.name)) { + group.labels.push(run.name); + } + + let dataset = group.datasets.find(d => d.label === result.label); + if (!dataset) { + const datasetIndex = group.datasets.length; + dataset = { + label: result.label, + data: new Array(group.labels.length).fill(null), + backgroundColor: colorPalette[datasetIndex % colorPalette.length], + borderColor: colorPalette[datasetIndex % colorPalette.length], + borderWidth: 1 + }; + group.datasets.push(dataset); + } + + const runIndex = group.labels.indexOf(run.name); + if (dataset.data[runIndex] == null) + dataset.data[runIndex] = result.value; + }); + }); + + return Object.values(groupedResults); +} + +function getLayerTags(metadata) { + const layerTags = new Set(); + if (metadata?.tags) { + metadata.tags.forEach(tag => { + if (tag.startsWith('SYCL') || tag.startsWith('UR') || tag === 'L0') { + layerTags.add(tag); + } + }); + } + return layerTags; +} + +function processLayerComparisonsData(benchmarkRuns) { + const groupedResults = {}; + + benchmarkRuns.forEach(run => { + run.results.forEach(result => { + if (!result.explicit_group) return; + + // Skip if no metadata available + const metadata = metadataForLabel(result.explicit_group, 'group'); + if (!metadata) return; + + // Get all benchmark labels in this group + const labelsInGroup = new Set( + benchmarkRuns.flatMap(r => + r.results + .filter(res => res.explicit_group === result.explicit_group) + .map(res => res.label) + ) + ); + + // Check if this group compares different layers + const uniqueLayers = new Set(); + labelsInGroup.forEach(label => { + const labelMetadata = metadataForLabel(label, 'benchmark'); + const layerTags = getLayerTags(labelMetadata); + layerTags.forEach(tag => uniqueLayers.add(tag)); + }); + + // Only process groups that compare different layers + if (uniqueLayers.size <= 1) return; + + if (!groupedResults[result.explicit_group]) { + groupedResults[result.explicit_group] = { + label: result.explicit_group, + suite: result.suite, + unit: result.unit, + lower_is_better: result.lower_is_better, + runs: {}, + benchmarkLabels: [], + description: metadata?.description || null, + notes: metadata?.notes || null, + unstable: metadata?.unstable || null + }; + } + + const group = groupedResults[result.explicit_group]; + const name = result.label + ' (' + run.name + ')'; + + // Add the benchmark label if it's not already in the array + if (!group.benchmarkLabels.includes(result.label)) { + group.benchmarkLabels.push(result.label); + } + + addRunDataPoint(group, run, result, name); + }); + }); + + return Object.values(groupedResults); +} + +function createRunDataStructure(run, result, label) { + return { + runName: run.name, + points: [{ + date: new Date(run.date), + value: result.value, + stddev: result.stddev, + git_hash: run.git_hash, + github_repo: run.github_repo, + label: label || result.label + }] + }; +} + +function addRunDataPoint(group, run, result, name = null) { + const runKey = name || result.label + ' (' + run.name + ')'; + + if (!group.runs[runKey]) { + group.runs[runKey] = { + runName: run.name, + points: [] + }; + } + + group.runs[runKey].points.push({ + date: new Date(run.date), + value: result.value, + stddev: result.stddev, + git_hash: run.git_hash, + github_repo: run.github_repo, + }); + + return group; +} + +// Setup functions +function setupRunSelector() { + runSelect = document.getElementById('run-select'); + selectedRunsDiv = document.getElementById('selected-runs'); + + allRunNames.forEach(name => { + const option = document.createElement('option'); + option.value = name; + option.textContent = name; + runSelect.appendChild(option); + }); + + updateSelectedRuns(false); +} + +function setupSuiteFilters() { + suiteFiltersContainer = document.getElementById('suite-filters'); + + benchmarkRuns.forEach(run => { + run.results.forEach(result => { + suiteNames.add(result.suite); + }); + }); + + suiteNames.forEach(suite => { + const label = document.createElement('label'); + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.className = 'suite-checkbox'; + checkbox.dataset.suite = suite; + checkbox.checked = true; + label.appendChild(checkbox); + label.appendChild(document.createTextNode(' ' + suite)); + suiteFiltersContainer.appendChild(label); + suiteFiltersContainer.appendChild(document.createTextNode(' ')); + }); +} + +function isNotesEnabled() { + const notesToggle = document.getElementById('show-notes'); + return notesToggle.checked; +} + +function isUnstableEnabled() { + const unstableToggle = document.getElementById('show-unstable'); + return unstableToggle.checked; +} + +function setupToggles() { + const notesToggle = document.getElementById('show-notes'); + const unstableToggle = document.getElementById('show-unstable'); + + notesToggle.addEventListener('change', function() { + // Update all note elements visibility + document.querySelectorAll('.benchmark-note').forEach(note => { + note.style.display = isNotesEnabled() ? 'block' : 'none'; + }); + updateURL(); + }); + + unstableToggle.addEventListener('change', function() { + // Update all unstable warning elements visibility + document.querySelectorAll('.benchmark-unstable').forEach(warning => { + warning.style.display = isUnstableEnabled() ? 'block' : 'none'; + }); + filterCharts(); + }); + + // Initialize from URL params if present + const notesParam = getQueryParam('notes'); + const unstableParam = getQueryParam('unstable'); + + if (notesParam !== null) { + let showNotes = notesParam === 'true'; + notesToggle.checked = showNotes; + } + + if (unstableParam !== null) { + let showUnstable = unstableParam === 'true'; + unstableToggle.checked = showUnstable; + } +} + +function setupTagFilters() { + tagFiltersContainer = document.getElementById('tag-filters'); + + const allTags = []; + + if (benchmarkTags) { + for (const tag in benchmarkTags) { + if (!allTags.includes(tag)) { + allTags.push(tag); + } + } + } + + // Create tag filter elements + allTags.forEach(tag => { + const tagContainer = document.createElement('div'); + tagContainer.className = 'tag-filter'; + + const checkbox = document.createElement('input'); + checkbox.type = 'checkbox'; + checkbox.id = `tag-${tag}`; + checkbox.className = 'tag-checkbox'; + checkbox.dataset.tag = tag; + + const label = document.createElement('label'); + label.htmlFor = `tag-${tag}`; + label.textContent = tag; + + // Add info icon with tooltip if tag description exists + if (benchmarkTags[tag]) { + const infoIcon = document.createElement('span'); + infoIcon.className = 'tag-info'; + infoIcon.textContent = 'ⓘ'; + infoIcon.title = benchmarkTags[tag].description; + label.appendChild(infoIcon); + } + + checkbox.addEventListener('change', function() { + if (this.checked) { + activeTags.add(tag); + } else { + activeTags.delete(tag); + } + filterCharts(); + }); + + tagContainer.appendChild(checkbox); + tagContainer.appendChild(label); + tagFiltersContainer.appendChild(tagContainer); + }); +} + +function toggleAllTags(select) { + const checkboxes = document.querySelectorAll('.tag-checkbox'); + + checkboxes.forEach(checkbox => { + checkbox.checked = select; + const tag = checkbox.dataset.tag; + + if (select) { + activeTags.add(tag); + } else { + activeTags.delete(tag); + } + }); + + filterCharts(); +} + +function initializeCharts() { + // Process raw data + timeseriesData = processTimeseriesData(benchmarkRuns); + barChartsData = processBarChartsData(benchmarkRuns); + layerComparisonsData = processLayerComparisonsData(benchmarkRuns); + allRunNames = [...new Set(benchmarkRuns.map(run => run.name))]; + + // Set up active runs + const runsParam = getQueryParam('runs'); + if (runsParam) { + const runsFromUrl = runsParam.split(','); + + // Start with an empty set + activeRuns = new Set(); + + // Process each run from URL + runsFromUrl.forEach(run => { + if (run === 'default') { + // Special case: include all default runs + (defaultCompareNames || []).forEach(defaultRun => { + if (allRunNames.includes(defaultRun)) { + activeRuns.add(defaultRun); + } + }); + } else if (allRunNames.includes(run)) { + // Add the specific run if it exists + activeRuns.add(run); + } + }); + } else { + // No runs parameter, use defaults + activeRuns = new Set(defaultCompareNames || []); + } + + // Setup UI components + setupRunSelector(); + setupSuiteFilters(); + setupTagFilters(); + setupToggles(); + + // Apply URL parameters + const regexParam = getQueryParam('regex'); + const suitesParam = getQueryParam('suites'); + const tagsParam = getQueryParam('tags'); + + if (regexParam) { + document.getElementById('bench-filter').value = regexParam; + } + + if (suitesParam) { + const suites = suitesParam.split(','); + document.querySelectorAll('.suite-checkbox').forEach(checkbox => { + checkbox.checked = suites.includes(checkbox.getAttribute('data-suite')); + }); + } + + // Apply tag filters from URL + if (tagsParam) { + const tags = tagsParam.split(','); + tags.forEach(tag => { + const checkbox = document.querySelector(`.tag-checkbox[data-tag="${tag}"]`); + if (checkbox) { + checkbox.checked = true; + activeTags.add(tag); + } + }); + } + + // Setup event listeners + document.querySelectorAll('.suite-checkbox').forEach(checkbox => { + checkbox.addEventListener('change', filterCharts); + }); + document.getElementById('bench-filter').addEventListener('input', filterCharts); + + // Draw initial charts + updateCharts(); +} + +// Make functions available globally for onclick handlers +window.addSelectedRun = addSelectedRun; +window.removeRun = removeRun; +window.toggleAllTags = toggleAllTags; + +// Load data based on configuration +function loadData() { + const loadingIndicator = document.getElementById('loading-indicator'); + loadingIndicator.style.display = 'block'; // Show loading indicator + + if (typeof remoteDataUrl !== 'undefined' && remoteDataUrl !== '') { + // Fetch data from remote URL + fetch(remoteDataUrl) + .then(response => { + if (!response.ok) { throw new Error(`Got response status ${response.status}.`) } + return response.json(); + }) + .then(data => { + benchmarkRuns = data.runs || data; + benchmarkMetadata = data.metadata || benchmarkMetadata || {}; + benchmarkTags = data.tags || benchmarkTags || {}; + initializeCharts(); + }) + .catch(error => { + console.error('Error fetching remote data:', error); + loadingIndicator.textContent = 'Fetching remote data failed.'; + }) + .finally(() => { + loadingIndicator.style.display = 'none'; // Hide loading indicator + }); + } else { + // Use local data (benchmarkRuns and benchmarkMetadata should be defined in data.js) + initializeCharts(); + loadingIndicator.style.display = 'none'; // Hide loading indicator + } +} + +// Initialize when DOM is ready +document.addEventListener('DOMContentLoaded', () => { + loadData(); +}); diff --git a/devops/scripts/benchmarks/html/styles.css b/devops/scripts/benchmarks/html/styles.css new file mode 100644 index 0000000000000..3e9c3bd22fc37 --- /dev/null +++ b/devops/scripts/benchmarks/html/styles.css @@ -0,0 +1,357 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + margin: 0; + padding: 16px; + background: #f8f9fa; +} +.container { + max-width: 1100px; + margin: 0 auto; +} +h1, h2 { + color: #212529; + text-align: center; + margin-bottom: 24px; + font-weight: 500; +} +.chart-container { + background: white; + border-radius: 8px; + padding: 24px; + margin-bottom: 24px; + box-shadow: 0 1px 3px rgba(0,0,0,0.1); +} +@media (max-width: 768px) { + body { + padding: 12px; + } + .chart-container { + padding: 16px; + border-radius: 6px; + } + h1 { + font-size: 24px; + margin-bottom: 16px; + } +} +.filter-container { + text-align: center; + margin-bottom: 24px; +} +.filter-container input { + padding: 8px; + font-size: 16px; + border: 1px solid #ccc; + border-radius: 4px; + width: 400px; + max-width: 100%; +} +.suite-filter-container { + text-align: center; + margin-bottom: 24px; + padding: 16px; + background: #e9ecef; + border-radius: 8px; +} +.suite-checkbox { + margin: 0 8px; +} +details { + margin-bottom: 24px; +} +summary { + display: flex; + justify-content: space-between; + align-items: center; + font-size: 16px; + font-weight: 500; + cursor: pointer; + padding: 12px 16px; + background: #dee2e6; + border-radius: 8px; + user-select: none; +} +summary:hover { + background: #ced4da; +} +summary::marker { + display: none; +} +summary::-webkit-details-marker { + display: none; +} +summary::after { + content: "▼"; + font-size: 12px; + margin-left: 8px; + transition: transform 0.3s; +} +details[open] summary::after { + transform: rotate(180deg); +} +.extra-info { + padding: 8px; + background: #f8f9fa; + border-radius: 8px; + margin-top: 8px; +} +.run-selector { + text-align: center; + margin-bottom: 24px; + padding: 16px; + background: #e9ecef; + border-radius: 8px; +} +.run-selector select { + width: 300px; + padding: 8px; + margin-right: 8px; +} +.run-selector button { + padding: 8px 16px; + background: #0068B5; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; +} +.run-selector button:hover { + background: #00C7FD; +} +.selected-runs { + margin-top: 12px; +} +.selected-run { + display: inline-block; + padding: 4px 8px; + margin: 4px; + background: #e2e6ea; + border-radius: 4px; +} +.selected-run button { + margin-left: 8px; + padding: 0 4px; + background: none; + border: none; + color: #dc3545; + cursor: pointer; +} +.download-button { + background: none; + border: none; + color: #0068B5; + cursor: pointer; + font-size: 16px; + padding: 4px; + margin-left: 8px; +} +.download-button:hover { + color: #00C7FD; +} +.loading-indicator { + text-align: center; + font-size: 18px; + color: #0068B5; + margin-bottom: 20px; +} +.extra-info-entry { + border: 1px solid #ddd; + padding: 10px; + margin-bottom: 10px; + background-color: #f9f9f9; + border-radius: 5px; +} +.extra-info-entry strong { + display: block; + margin-bottom: 5px; +} +.extra-info-entry em { + color: #555; +} +.display-options-container { + text-align: center; + margin-bottom: 24px; + padding: 16px; + background: #e9ecef; + border-radius: 8px; +} +.display-options-container label { + margin: 0 12px; + cursor: pointer; +} +.display-options-container input { + margin-right: 8px; +} +.benchmark-note { + background-color: #cfe2ff; + color: #084298; + padding: 10px; + margin-bottom: 10px; + border-radius: 5px; + border-left: 4px solid #084298; + white-space: pre-line; +} +.benchmark-unstable { + background-color: #f8d7da; + color: #842029; + padding: 10px; + margin-bottom: 10px; + border-radius: 5px; + border-left: 4px solid #842029; + white-space: pre-line; +} +.note-text { + color: #084298; +} +.unstable-warning { + color: #842029; + font-weight: bold; +} +.unstable-text { + color: #842029; +} +.options-container { + margin-bottom: 24px; + background: #e9ecef; + border-radius: 8px; + overflow: hidden; +} +.options-container summary { + padding: 12px 16px; + font-weight: 500; + cursor: pointer; + background: #dee2e6; + user-select: none; +} +.options-container summary:hover { + background: #ced4da; +} +.options-content { + padding: 16px; + display: flex; + flex-wrap: wrap; + gap: 24px; +} +.filter-section { + flex: 1; + min-width: 300px; +} +.filter-section h3 { + margin-top: 0; + margin-bottom: 12px; + font-size: 18px; + font-weight: 500; + text-align: left; + display: flex; + align-items: center; +} +#suite-filters { + display: flex; + flex-wrap: wrap; + max-height: 200px; + overflow-y: auto; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 8px; + background-color: #f8f9fa; +} +.display-options { + display: flex; + flex-direction: column; + gap: 8px; +} +.display-options label { + display: flex; + align-items: center; + cursor: pointer; +} +.display-options input { + margin-right: 8px; +} +.benchmark-description { + background-color: #f2f2f2; + color: #333; + padding: 10px; + margin-bottom: 10px; + border-radius: 5px; + border-left: 4px solid #6c757d; + white-space: pre-line; + font-style: italic; +} +/* Tag styles */ +.benchmark-tags { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-bottom: 10px; +} + +.tag { + display: inline-block; + background-color: #e2e6ea; + color: #495057; + padding: 2px 8px; + border-radius: 12px; + font-size: 12px; + cursor: help; +} + +.tag-filter { + display: inline-flex; + align-items: center; + margin: 4px; +} + +.tag-filter label { + margin-left: 4px; + cursor: pointer; + display: flex; + align-items: center; +} + +.tag-info { + color: #0068B5; + margin-left: 4px; + cursor: help; + font-size: 12px; +} + +#tag-filters { + display: flex; + flex-wrap: wrap; + max-height: 200px; + overflow-y: auto; + border: 1px solid #dee2e6; + border-radius: 4px; + padding: 8px; + background-color: #f8f9fa; +} + +.tag-action-button { + padding: 2px 8px; + background: #e2e6ea; + border: none; + border-radius: 4px; + cursor: pointer; + font-size: 12px; + margin-left: 8px; + vertical-align: middle; +} + +.tag-action-button:hover { + background: #ced4da; +} + +.remove-tag { + background: none; + border: none; + color: white; + margin-left: 4px; + cursor: pointer; + font-size: 16px; + padding: 0 4px; +} + +.remove-tag:hover { + color: #f8d7da; +} diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index 610797687931b..41dd8624210bb 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -168,6 +168,9 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): TestSuite(), ] + # Collect metadata from all benchmarks without setting them up + metadata = collect_metadata(suites) + # If dry run, we're done if options.dry_run: suites = [] @@ -299,14 +302,10 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): compare_names.append(saved_name) if options.output_html: - html_content = generate_html(history.runs, "intel/llvm", compare_names) - - with open("benchmark_results.html", "w") as file: - file.write(html_content) - - print( - f"HTML with benchmark results has been written to {os.getcwd()}/benchmark_results.html" - ) + html_path = options.output_directory + if options.output_directory is None: + html_path = os.path.join(os.path.dirname(__file__), "html") + generate_html(history.runs, compare_names, html_path, metadata) def validate_and_parse_env_args(env_args): @@ -415,13 +414,17 @@ def validate_and_parse_env_args(env_args): help="Specify whether markdown output should fit the content size limit for request validation", ) parser.add_argument( - "--output-html", help="Create HTML output", action="store_true", default=False + "--output-html", + help="Create HTML output. Local output is for direct local viewing of the html file, remote is for server deployment.", + nargs="?", + const=options.output_html, + choices=["local", "remote"], ) parser.add_argument( "--output-dir", type=str, help="Location for output files, if --output-html or --output-markdown was specified.", - default=None, + default=options.output_directory, ) parser.add_argument( "--dry-run", @@ -475,7 +478,7 @@ def validate_and_parse_env_args(env_args): parser.add_argument( "--results-dir", type=str, - help="Specify a custom results directory", + help="Specify a custom directory to load/store (historical) results from", default=options.custom_results_dir, ) parser.add_argument( @@ -529,7 +532,10 @@ def validate_and_parse_env_args(env_args): if not os.path.isdir(args.output_dir): parser.error("Specified --output-dir is not a valid path") options.output_directory = os.path.abspath(args.output_dir) - + if args.results_dir is not None: + if not os.path.isdir(args.results_dir): + parser.error("Specified --results-dir is not a valid path") + options.custom_results_dir = os.path.abspath(args.results_dir) benchmark_filter = re.compile(args.filter) if args.filter else None diff --git a/devops/scripts/benchmarks/options.py b/devops/scripts/benchmarks/options.py index a14e84e786172..c852e50c71372 100644 --- a/devops/scripts/benchmarks/options.py +++ b/devops/scripts/benchmarks/options.py @@ -4,6 +4,7 @@ from presets import presets + class Compare(Enum): LATEST = "latest" AVERAGE = "average" @@ -31,7 +32,7 @@ class Options: compare: Compare = Compare.LATEST compare_max: int = 10 # average/median over how many results output_markdown: MarkdownSize = MarkdownSize.SHORT - output_html: bool = False + output_html: str = "local" output_directory: str = None dry_run: bool = False stddev_threshold: float = 0.02 @@ -39,11 +40,12 @@ class Options: build_compute_runtime: bool = False extra_ld_libraries: list[str] = field(default_factory=list) extra_env_vars: dict = field(default_factory=dict) - compute_runtime_tag: str = "25.05.32567.12" + compute_runtime_tag: str = "25.05.32567.18" build_igc: bool = False current_run_name: str = "This PR" preset: str = "Full" custom_results_dir = None build_jobs: int = multiprocessing.cpu_count() + options = Options() diff --git a/devops/scripts/benchmarks/output_html.py b/devops/scripts/benchmarks/output_html.py index e9c1f135b70cd..54e17043631e4 100644 --- a/devops/scripts/benchmarks/output_html.py +++ b/devops/scripts/benchmarks/output_html.py @@ -1,340 +1,59 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. # See LICENSE.TXT # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -import re +import json import os -from pathlib import Path -import matplotlib.pyplot as plt -import mpld3 -from collections import defaultdict -from dataclasses import dataclass -import matplotlib.dates as mdates -from utils.result import BenchmarkRun, Result -import numpy as np -from string import Template - - -@dataclass -class BenchmarkMetadata: - unit: str - suite: str - lower_is_better: bool - - -@dataclass -class BenchmarkSeries: - label: str - metadata: BenchmarkMetadata - runs: list[BenchmarkRun] - - -@dataclass -class BenchmarkChart: - label: str - suite: str - html: str - - -def tooltip_css() -> str: - return ".mpld3-tooltip{background:white;padding:8px;border:1px solid #ddd;border-radius:4px;font-family:monospace;white-space:pre;}" - - -def create_time_series_chart( - benchmarks: list[BenchmarkSeries], github_repo: str -) -> list[BenchmarkChart]: - plt.close("all") - - num_benchmarks = len(benchmarks) - if num_benchmarks == 0: - return [] - - html_charts = [] - - for _, benchmark in enumerate(benchmarks): - fig, ax = plt.subplots(figsize=(10, 4)) - - all_values = [] - all_stddevs = [] - - for run in benchmark.runs: - sorted_points = sorted(run.results, key=lambda x: x.date) - dates = [point.date for point in sorted_points] - values = [point.value for point in sorted_points] - stddevs = [point.stddev for point in sorted_points] - - all_values.extend(values) - all_stddevs.extend(stddevs) - - ax.errorbar(dates, values, yerr=stddevs, fmt="-", label=run.name, alpha=0.5) - scatter = ax.scatter(dates, values, picker=True) - - tooltip_labels = [ - f"Date: {point.date.strftime('%Y-%m-%d %H:%M:%S')}\n" - f"Value: {point.value:.2f} {benchmark.metadata.unit}\n" - f"Stddev: {point.stddev:.2f} {benchmark.metadata.unit}\n" - f"Git Hash: {point.git_hash}" - for point in sorted_points - ] - - targets = [ - f"https://github.com/{github_repo}/commit/{point.git_hash}" - for point in sorted_points - ] - - tooltip = mpld3.plugins.PointHTMLTooltip( - scatter, tooltip_labels, css=tooltip_css(), targets=targets - ) - mpld3.plugins.connect(fig, tooltip) - - ax.set_title(benchmark.label, pad=20) - performance_indicator = ( - "lower is better" - if benchmark.metadata.lower_is_better - else "higher is better" - ) - ax.text( - 0.5, - 1.05, - f"({performance_indicator})", - ha="center", - transform=ax.transAxes, - style="italic", - fontsize=7, - color="#666666", - ) - - ax.set_xlabel("") - unit = benchmark.metadata.unit - ax.set_ylabel(f"Value ({unit})" if unit else "Value") - ax.grid(True, alpha=0.2) - ax.legend(bbox_to_anchor=(1, 1), loc="upper left") - ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter("%Y-%m-%d %H:%M:%S")) - - plt.tight_layout() - html_charts.append( - BenchmarkChart( - html=mpld3.fig_to_html(fig), - label=benchmark.label, - suite=benchmark.metadata.suite, - ) - ) - plt.close(fig) - - return html_charts - - -@dataclass -class ExplicitGroup: - name: str - nnames: int - metadata: BenchmarkMetadata - runs: dict[str, dict[str, Result]] - - -def create_explicit_groups( - benchmark_runs: list[BenchmarkRun], compare_names: list[str] -) -> list[ExplicitGroup]: - groups = {} - - for run in benchmark_runs: - if run.name in compare_names: - for res in run.results: - if res.explicit_group != "": - if res.explicit_group not in groups: - groups[res.explicit_group] = ExplicitGroup( - name=res.explicit_group, - nnames=len(compare_names), - metadata=BenchmarkMetadata( - unit=res.unit, - lower_is_better=res.lower_is_better, - suite=res.suite, - ), - runs={}, - ) - - group = groups[res.explicit_group] - if res.label not in group.runs: - group.runs[res.label] = {name: None for name in compare_names} - - if group.runs[res.label][run.name] is None: - group.runs[res.label][run.name] = res - - return list(groups.values()) - - -def create_grouped_bar_charts(groups: list[ExplicitGroup]) -> list[BenchmarkChart]: - plt.close("all") - - html_charts = [] - - for group in groups: - fig, ax = plt.subplots(figsize=(10, 6)) - - x = np.arange(group.nnames) - x_labels = [] - width = 0.8 / len(group.runs) - - max_height = 0 - - for i, (run_name, run_results) in enumerate(group.runs.items()): - offset = width * i - - positions = x + offset - x_labels = run_results.keys() - valid_data = [r.value if r is not None else 0 for r in run_results.values()] - rects = ax.bar(positions, valid_data, width, label=run_name) - # This is a hack to disable all bar_label. Setting labels to empty doesn't work. - # We create our own labels below for each bar, this works better in mpld3. - ax.bar_label(rects, fmt="") - - for rect, run, res in zip(rects, run_results.keys(), run_results.values()): - if res is None: - continue - - height = rect.get_height() - if height > max_height: - max_height = height - - ax.text( - rect.get_x() + rect.get_width() / 2.0, - height + 1, - f"{res.value:.1f}", - ha="center", - va="bottom", - fontsize=9, - ) - - tooltip_labels = [ - f"Date: {res.date.strftime('%Y-%m-%d %H:%M:%S')}\n" - f"Run: {run}\n" - f"Label: {res.label}\n" - f"Value: {res.value:.2f} {res.unit}\n" - f"Stddev: {res.stddev:.2f} {res.unit}\n" - ] - tooltip = mpld3.plugins.LineHTMLTooltip( - rect, tooltip_labels, css=tooltip_css() - ) - mpld3.plugins.connect(ax.figure, tooltip) - - # normally we'd just set legend to be outside - # the chart, but this is not supported by mpld3. - # instead, we adjust the y axis to account for - # the height of the bars. - legend_height = len(group.runs) * 0.1 - ax.set_ylim(0, max_height * (1 + legend_height)) - - ax.set_xticks([]) - ax.grid(True, axis="y", alpha=0.2) - ax.set_ylabel(f"Value ({group.metadata.unit})") - ax.legend(loc="upper left") - ax.set_title(group.name, pad=20) - performance_indicator = ( - "lower is better" if group.metadata.lower_is_better else "higher is better" - ) - ax.text( - 0.5, - 1.03, - f"({performance_indicator})", - ha="center", - transform=ax.transAxes, - style="italic", - fontsize=7, - color="#666666", - ) - - for idx, label in enumerate(x_labels): - # this is a hack to get labels to show above the legend - # we normalize the idx to transAxes transform and offset it a little. - x_norm = (idx + 0.3 - ax.get_xlim()[0]) / ( - ax.get_xlim()[1] - ax.get_xlim()[0] - ) - ax.text(x_norm, 1.03, label, transform=ax.transAxes, color="#666666") - - plt.tight_layout() - html_charts.append( - BenchmarkChart( - label=group.name, - html=mpld3.fig_to_html(fig), - suite=group.metadata.suite, - ) - ) - plt.close(fig) - - return html_charts - - -def process_benchmark_data( - benchmark_runs: list[BenchmarkRun], compare_names: list[str] -) -> list[BenchmarkSeries]: - benchmark_metadata: dict[str, BenchmarkMetadata] = {} - run_map: dict[str, dict[str, list[Result]]] = defaultdict(lambda: defaultdict(list)) - - for run in benchmark_runs: - if run.name not in compare_names: - continue - - for result in run.results: - if result.label not in benchmark_metadata: - benchmark_metadata[result.label] = BenchmarkMetadata( - unit=result.unit, - lower_is_better=result.lower_is_better, - suite=result.suite, - ) - - result.date = run.date - result.git_hash = run.git_hash - run_map[result.label][run.name].append(result) - - benchmark_series = [] - for label, metadata in benchmark_metadata.items(): - runs = [ - BenchmarkRun(name=run_name, results=results) - for run_name, results in run_map[label].items() - ] - benchmark_series.append( - BenchmarkSeries(label=label, metadata=metadata, runs=runs) - ) - - return benchmark_series +from options import options +from utils.result import BenchmarkMetadata, BenchmarkOutput +from benches.base import benchmark_tags, benchmark_tags_dict def generate_html( - benchmark_runs: list[BenchmarkRun], github_repo: str, compare_names: list[str] -) -> str: - benchmarks = process_benchmark_data(benchmark_runs, compare_names) - - timeseries = create_time_series_chart(benchmarks, github_repo) - timeseries_charts_html = "\n".join( - f'
{ts.html}
' - for ts in timeseries + benchmark_runs: list, + compare_names: list[str], + html_path: str, + metadata: dict[str, BenchmarkMetadata], +): + benchmark_runs.sort(key=lambda run: run.date, reverse=True) + # Sorted in reverse, such that runs are ordered from newest to oldest + + # Create the comprehensive output object + output = BenchmarkOutput( + runs=benchmark_runs, + metadata=metadata, + tags=benchmark_tags_dict, + default_compare_names=compare_names, ) - explicit_groups = create_explicit_groups(benchmark_runs, compare_names) - - bar_charts = create_grouped_bar_charts(explicit_groups) - bar_charts_html = "\n".join( - f'
{bc.html}
' - for bc in bar_charts - ) - - suite_names = {t.suite for t in timeseries} - suite_checkboxes_html = " ".join( - f'' - for suite in suite_names - ) - - script_path = os.path.dirname(os.path.realpath(__file__)) - results_template_path = Path(script_path, "benchmark_results.html.template") - with open(results_template_path, "r") as file: - html_template = file.read() - - template = Template(html_template) - data = { - "suite_checkboxes_html": suite_checkboxes_html, - "timeseries_charts_html": timeseries_charts_html, - "bar_charts_html": bar_charts_html, - } - - return template.substitute(data) + if options.output_html == "local": + data_path = os.path.join(html_path, "data.js") + with open(data_path, "w") as f: + # For local format, we need to write JavaScript variable assignments + f.write("benchmarkRuns = ") + json.dump(json.loads(output.to_json())["runs"], f, indent=2) + f.write(";\n\n") + + f.write("benchmarkMetadata = ") + json.dump(json.loads(output.to_json())["metadata"], f, indent=2) + f.write(";\n\n") + + f.write("benchmarkTags = ") + json.dump(json.loads(output.to_json())["tags"], f, indent=2) + f.write(";\n\n") + + f.write("defaultCompareNames = ") + json.dump(output.default_compare_names, f, indent=2) + f.write(";\n") + + print(f"See {os.getcwd()}/html/index.html for the results.") + else: + # For remote format, we write a single JSON file + data_path = os.path.join(html_path, "data.json") + with open(data_path, "w") as f: + json.dump(json.loads(output.to_json()), f, indent=2) + + print( + f"Upload {data_path} to a location set in config.js remoteDataUrl argument." + ) diff --git a/devops/scripts/benchmarks/utils/result.py b/devops/scripts/benchmarks/utils/result.py index 14a2ffa905f34..6afcc8d1b627b 100644 --- a/devops/scripts/benchmarks/utils/result.py +++ b/devops/scripts/benchmarks/utils/result.py @@ -29,6 +29,7 @@ class Result: lower_is_better: bool = True suite: str = "Unknown" + @dataclass_json @dataclass class BenchmarkRun: