diff --git a/resources/benchmark-runner.mjs b/resources/benchmark-runner.mjs index 3aeb50526..8a28c2e66 100644 --- a/resources/benchmark-runner.mjs +++ b/resources/benchmark-runner.mjs @@ -362,7 +362,7 @@ export class BenchmarkRunner { const iterationEndLabel = "iteration-end"; for (let i = 0; i < this._iterationCount; i++) { performance.mark(iterationStartLabel); - await this._runAllSuites(); + await this.runAllSuites(); performance.mark(iterationEndLabel); performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel); } @@ -375,7 +375,7 @@ export class BenchmarkRunner { } } - async _appendFrame(src) { + async _appendFrame() { const frame = document.createElement("iframe"); const style = frame.style; style.width = `${params.viewport.width}px`; @@ -396,7 +396,7 @@ export class BenchmarkRunner { return frame; } - async _runAllSuites() { + async _prepareAllSuites() { this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN }; const prepareStartLabel = "runner-prepare-start"; @@ -408,23 +408,32 @@ export class BenchmarkRunner { this._page = new Page(this._frame); let suites = [...this._suites]; - if (this._suiteOrderRandomNumberGenerator) { - // We just do a simple Fisher-Yates shuffle based on the repeated hash of the - // seed. This is not a high quality RNG, but it's plenty good enough. - for (let i = 0; i < suites.length - 1; i++) { - let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i)); - let tmp = suites[i]; - suites[i] = suites[j]; - suites[j] = tmp; - } - } + if (this._suiteOrderRandomNumberGenerator) + this._shuffleSuites(suites); + performance.mark(prepareEndLabel); performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel); + return suites; + } + + _shuffleSuites(suites) { + // We just do a simple Fisher-Yates shuffle based on the repeated hash of the + // seed. This is not a high quality RNG, but it's plenty good enough. + for (let i = 0; i < suites.length - 1; i++) { + const j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i)); + const tmp = suites[i]; + suites[i] = suites[j]; + suites[j] = tmp; + } + } + + async runAllSuites() { + const suites = await this._prepareAllSuites(); try { for (const suite of suites) { if (!suite.disabled) - await this._runSuite(suite); + await this.runSuite(suite); } } finally { @@ -444,23 +453,34 @@ export class BenchmarkRunner { performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel); } - async _runSuite(suite) { + async runSuite(suite) { + await this._prepareSuite(suite); + await this._runSuite(suite); + } + + async _prepareSuite(suite) { const suiteName = suite.name; const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; - const suiteStartLabel = `suite-${suiteName}-start`; - const suiteEndLabel = `suite-${suiteName}-end`; performance.mark(suitePrepareStartLabel); - await this._prepareSuite(suite); + await this._loadFrame(suite); + await suite.prepare(this._page); performance.mark(suitePrepareEndLabel); + performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); + } + + async _runSuite(suite) { + const suiteName = suite.name; + const suiteStartLabel = `suite-${suiteName}-start`; + const suiteEndLabel = `suite-${suiteName}-end`; + performance.mark(suiteStartLabel); for (const test of suite.tests) await this._runTestAndRecordResults(suite, test); performance.mark(suiteEndLabel); - performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); this._validateSuiteTotal(suiteName); } @@ -474,14 +494,12 @@ export class BenchmarkRunner { throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`); } - async _prepareSuite(suite) { - return new Promise((resolve) => { + async _loadFrame(suite) { + return new Promise((resolve, reject) => { const frame = this._page._frame; - frame.onload = async () => { - await suite.prepare(this._page); - resolve(); - }; - frame.src = `${suite.url}`; + frame.onload = () => resolve(); + frame.onerror = () => reject(); + frame.src = suite.url; }); } diff --git a/tests/benchmark-runner-tests.mjs b/tests/benchmark-runner-tests.mjs index b4c092d6c..4e978dc38 100644 --- a/tests/benchmark-runner-tests.mjs +++ b/tests/benchmark-runner-tests.mjs @@ -11,10 +11,12 @@ function TEST_FIXTURE(name) { const SUITES_FIXTURE = [ { name: "Suite 1", + async prepare(page) {}, tests: [TEST_FIXTURE("Test 1"), TEST_FIXTURE("Test 2"), TEST_FIXTURE("Test 3")], }, { name: "Suite 2", + async prepare(page) {}, tests: [TEST_FIXTURE("Test 1")], }, ]; @@ -106,15 +108,28 @@ describe("BenchmarkRunner", () => { }); describe("Suite", () => { - describe("_runAllSuites", () => { - let _runSuiteStub, _finalizeStub, _removeFrameStub; + describe("runAllSuites", () => { + let _runSuiteStub, _finalizeStub, _loadFrameStub, _appendFrameStub, _removeFrameStub; before(async () => { - _runSuiteStub = stub(runner, "_runSuite").callsFake(() => null); - _finalizeStub = stub(runner, "_finalize").callsFake(() => null); + _runSuiteStub = stub(runner, "_runSuite").callsFake(async () => null); + _finalizeStub = stub(runner, "_finalize").callsFake(async () => null); + _loadFrameStub = stub(runner, "_loadFrame").callsFake(async () => null); + _appendFrameStub = stub(runner, "_appendFrame").callsFake(async () => null); _removeFrameStub = stub(runner, "_removeFrame").callsFake(() => null); + for (const suite of runner._suites) + spy(suite, "prepare"); + expect(runner._suites).not.to.have.length(0); + await runner.runAllSuites(); + }); - await runner._runAllSuites(); + it("should call prepare on all suites", () => { + let suitesPrepareCount = 0; + for (const suite of runner._suites) { + suitesPrepareCount += 1; + assert.calledOnce(suite.prepare); + } + expect(suitesPrepareCount).equal(SUITES_FIXTURE.length); }); it("should run all test suites", async () => { @@ -122,6 +137,8 @@ describe("BenchmarkRunner", () => { }); it("should remove the previous frame and then the current frame", () => { + assert.calledTwice(_loadFrameStub); + assert.calledOnce(_appendFrameStub); assert.calledTwice(_removeFrameStub); }); @@ -130,22 +147,25 @@ describe("BenchmarkRunner", () => { }); }); - describe("_runSuite", () => { - let _prepareSuiteStub, _runTestAndRecordResultsStub, performanceMarkSpy; + describe("runSuite", () => { + let _prepareSuiteSpy, _loadFrameStub, _runTestAndRecordResultsStub, _suitePrepareSpy, performanceMarkSpy; const suite = SUITES_FIXTURE[0]; before(async () => { - _prepareSuiteStub = stub(runner, "_prepareSuite").callsFake(() => null); - - _runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(() => null); - + _prepareSuiteSpy = spy(runner, "_prepareSuite"); + _loadFrameStub = stub(runner, "_loadFrame").callsFake(async () => null); + _runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(async () => null); performanceMarkSpy = spy(window.performance, "mark"); - runner._runSuite(suite); + _suitePrepareSpy = spy(suite, "prepare"); + + runner.runSuite(suite); }); it("should prepare the suite first", async () => { - assert.calledOnce(_prepareSuiteStub); + assert.calledOnce(_prepareSuiteSpy); + assert.calledOnce(_suitePrepareSpy); + assert.calledOnce(_loadFrameStub); }); it("should run and record results for every test in suite", async () => {