-
Notifications
You must be signed in to change notification settings - Fork 70
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use more helper runner helper methods #421
base: main
Are you sure you want to change the base?
Changes from all commits
ee5ef73
4a88c3e
8ee7145
b97c8e2
bb6b3eb
f63ec93
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -362,7 +362,7 @@ export class BenchmarkRunner { | |
const iterationEndLabel = "iteration-end"; | ||
for (let i = 0; i < this._iterationCount; i++) { | ||
performance.mark(iterationStartLabel); | ||
await this._runAllSuites(); | ||
await this.runAllSuites(); | ||
performance.mark(iterationEndLabel); | ||
performance.measure(`iteration-${i}`, iterationStartLabel, iterationEndLabel); | ||
} | ||
|
@@ -375,7 +375,7 @@ export class BenchmarkRunner { | |
} | ||
} | ||
|
||
async _appendFrame(src) { | ||
async _appendFrame() { | ||
const frame = document.createElement("iframe"); | ||
const style = frame.style; | ||
style.width = `${params.viewport.width}px`; | ||
|
@@ -396,7 +396,7 @@ export class BenchmarkRunner { | |
return frame; | ||
} | ||
|
||
async _runAllSuites() { | ||
async _prepareAllSuites() { | ||
this._measuredValues = { tests: {}, total: 0, mean: NaN, geomean: NaN, score: NaN }; | ||
|
||
const prepareStartLabel = "runner-prepare-start"; | ||
|
@@ -408,23 +408,32 @@ export class BenchmarkRunner { | |
this._page = new Page(this._frame); | ||
|
||
let suites = [...this._suites]; | ||
if (this._suiteOrderRandomNumberGenerator) { | ||
// We just do a simple Fisher-Yates shuffle based on the repeated hash of the | ||
// seed. This is not a high quality RNG, but it's plenty good enough. | ||
for (let i = 0; i < suites.length - 1; i++) { | ||
let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i)); | ||
let tmp = suites[i]; | ||
suites[i] = suites[j]; | ||
suites[j] = tmp; | ||
} | ||
} | ||
if (this._suiteOrderRandomNumberGenerator) | ||
this._shuffleSuites(suites); | ||
|
||
performance.mark(prepareEndLabel); | ||
performance.measure("runner-prepare", prepareStartLabel, prepareEndLabel); | ||
|
||
return suites; | ||
} | ||
|
||
_shuffleSuites(suites) { | ||
// We just do a simple Fisher-Yates shuffle based on the repeated hash of the | ||
// seed. This is not a high quality RNG, but it's plenty good enough. | ||
for (let i = 0; i < suites.length - 1; i++) { | ||
let j = i + (this._suiteOrderRandomNumberGenerator() % (suites.length - i)); | ||
let tmp = suites[i]; | ||
suites[i] = suites[j]; | ||
suites[j] = tmp; | ||
} | ||
} | ||
|
||
async runAllSuites() { | ||
const suites = await this._prepareAllSuites(); | ||
try { | ||
for (const suite of suites) { | ||
if (!suite.disabled) | ||
await this._runSuite(suite); | ||
await this.runSuite(suite); | ||
} | ||
|
||
} finally { | ||
|
@@ -444,23 +453,34 @@ export class BenchmarkRunner { | |
performance.measure("runner-finalize", finalizeStartLabel, finalizeEndLabel); | ||
} | ||
|
||
async _runSuite(suite) { | ||
async runSuite(suite) { | ||
await this._prepareSuite(suite); | ||
await this._runSuite(suite); | ||
} | ||
|
||
async _prepareSuite(suite) { | ||
const suiteName = suite.name; | ||
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`; | ||
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`; | ||
const suiteStartLabel = `suite-${suiteName}-start`; | ||
const suiteEndLabel = `suite-${suiteName}-end`; | ||
|
||
performance.mark(suitePrepareStartLabel); | ||
await this._prepareSuite(suite); | ||
await this._loadFrame(suite); | ||
await suite.prepare(this._page); | ||
performance.mark(suitePrepareEndLabel); | ||
|
||
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); | ||
} | ||
|
||
async _runSuite(suite) { | ||
const suiteName = suite.name; | ||
const suiteStartLabel = `suite-${suiteName}-start`; | ||
const suiteEndLabel = `suite-${suiteName}-end`; | ||
|
||
performance.mark(suiteStartLabel); | ||
for (const test of suite.tests) | ||
await this._runTestAndRecordResults(suite, test); | ||
performance.mark(suiteEndLabel); | ||
|
||
performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel); | ||
performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel); | ||
this._validateSuiteTotal(suiteName); | ||
} | ||
|
@@ -474,14 +494,12 @@ export class BenchmarkRunner { | |
throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`); | ||
} | ||
|
||
async _prepareSuite(suite) { | ||
return new Promise((resolve) => { | ||
async _loadFrame(suite) { | ||
return new Promise((resolve, reject) => { | ||
const frame = this._page._frame; | ||
frame.onload = async () => { | ||
await suite.prepare(this._page); | ||
resolve(); | ||
}; | ||
frame.src = `${suite.url}`; | ||
frame.onload = () => resolve(); | ||
frame.onerror = () => reject(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 👍 |
||
frame.src = suite.url; | ||
}); | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,10 +11,12 @@ function TEST_FIXTURE(name) { | |
const SUITES_FIXTURE = [ | ||
{ | ||
name: "Suite 1", | ||
async prepare(page) {}, | ||
tests: [TEST_FIXTURE("Test 1"), TEST_FIXTURE("Test 2"), TEST_FIXTURE("Test 3")], | ||
}, | ||
{ | ||
name: "Suite 2", | ||
async prepare(page) {}, | ||
tests: [TEST_FIXTURE("Test 1")], | ||
}, | ||
]; | ||
|
@@ -106,22 +108,31 @@ describe("BenchmarkRunner", () => { | |
}); | ||
|
||
describe("Suite", () => { | ||
describe("_runAllSuites", () => { | ||
let _runSuiteStub, _finalizeStub, _removeFrameStub; | ||
describe("runAllSuites", () => { | ||
let _runSuiteStub, _finalizeStub, _appendFrameStub, _removeFrameStub; | ||
|
||
before(async () => { | ||
_runSuiteStub = stub(runner, "_runSuite").callsFake(() => null); | ||
_finalizeStub = stub(runner, "_finalize").callsFake(() => null); | ||
_runSuiteStub = stub(runner, "runSuite").callsFake(async () => null); | ||
_finalizeStub = stub(runner, "_finalize").callsFake(async () => null); | ||
_appendFrameStub = stub(runner, "_appendFrame").callsFake(async () => null); | ||
_removeFrameStub = stub(runner, "_removeFrame").callsFake(() => null); | ||
for (const suite in runner.suites) | ||
spy(suite, "prepare"); | ||
|
||
await runner._runAllSuites(); | ||
await runner.runAllSuites(); | ||
}); | ||
|
||
it("should call prepare on all suites", () => { | ||
for (const suite in runner.suites) | ||
suite.prepare.calledOnce(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I'm surprised this is working. I think Debugging further it looks like indeed Also it would be good to add something such as |
||
}); | ||
|
||
it("should run all test suites", async () => { | ||
assert.calledTwice(_runSuiteStub); | ||
}); | ||
|
||
it("should remove the previous frame and then the current frame", () => { | ||
assert.calledOnce(_appendFrameStub); | ||
assert.calledTwice(_removeFrameStub); | ||
}); | ||
|
||
|
@@ -130,22 +141,25 @@ describe("BenchmarkRunner", () => { | |
}); | ||
}); | ||
|
||
describe("_runSuite", () => { | ||
let _prepareSuiteStub, _runTestAndRecordResultsStub, performanceMarkSpy; | ||
describe("runSuite", () => { | ||
let _prepareSuiteSpy, _loadFrameStub, _runTestAndRecordResultsStub, _suitePrepareSpy, performanceMarkSpy; | ||
|
||
const suite = SUITES_FIXTURE[0]; | ||
|
||
before(async () => { | ||
_prepareSuiteStub = stub(runner, "_prepareSuite").callsFake(() => null); | ||
|
||
_runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(() => null); | ||
|
||
_prepareSuiteSpy = spy(runner, "_prepareSuite"); | ||
_loadFrameStub = stub(runner, "_loadFrame").callsFake(async () => null); | ||
_runTestAndRecordResultsStub = stub(runner, "_runTestAndRecordResults").callsFake(async () => null); | ||
performanceMarkSpy = spy(window.performance, "mark"); | ||
runner._runSuite(suite); | ||
_suitePrepareSpy = spy(suite, "prepare"); | ||
|
||
runner.runSuite(suite); | ||
}); | ||
|
||
it("should prepare the suite first", async () => { | ||
assert.calledOnce(_prepareSuiteStub); | ||
assert.calledOnce(_prepareSuiteSpy); | ||
assert.calledOnce(_suitePrepareSpy); | ||
assert.calledOnce(_loadFrameStub); | ||
}); | ||
|
||
it("should run and record results for every test in suite", async () => { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Use const maybe?