Core: Create config.testIds to expose all available test ids

What this does

Currently each testId is calculated as each test is executed.
Config.testIds pre-hashes all the loaded tests on demand and returns all
testIds.

How is this helpful

Currently test runners can get access to test modules via config.modules
and partition the modules based on some distribution strategy for running
the tests in parallel. This is fine but it can lead to unbalanced
distribution as some modules have a lot of tests and some modules are
testing integration (longer running) test instead of unit test.

Partitioning the tests by modules caused some instance of test runners
to run way longer than others as some heavier running modules might be
clumped together in the same instance.

To mitigate and minimize the unbalanced test runs, the proposed solution
is to partition based on each individual test with testId. The test
runners can now access testIds beforehand and can distribute those ids
with the testId url param. This will minimize the test execution time
descrepencies between each individual instance of the test runners.

Author: kltan

Gruntfile.js

@@ -165,6 +165,7 @@ module.exports = function( grunt ) {
 						"test/reporter-html/xhtml-single-testid.xhtml",
 						"test/reporter-urlparams.html",
 						"test/moduleId.html",
+						"test/testIds.html",
 						"test/onerror/inside-test.html",
 						"test/onerror/outside-test.html",
 						"test/only.html",

docs/config/QUnit.config.md

@@ -101,6 +101,10 @@ Enabling this option will cause tests to fail, if they don't call `expect()` at
 
 This property allows QUnit to run specific tests identified by the hashed version of their module name and test name. You can specify one or multiple tests to run.
 
+### `QUnit.config.testIds` (array) | default: `array`
+
+This property allows QUnit to return all hashed testIds of all loaded tests.
+
 ### `QUnit.config.testTimeout` (number) | default: `undefined`
 
 Specify a global timeout in milliseconds after which all tests will fail with an appropriate message. Useful when async tests aren't finishing, to prevent the testrunner getting stuck. Set to something high, e.g. 30000 (30 seconds) to avoid slow tests to time out by accident.

package-lock.json

@@ -2,7 +2,7 @@
   "name": "qunit",
   "title": "QUnit",
   "description": "An easy-to-use JavaScript Unit Testing framework.",
-  "version": "2.9.4-pre",
+  "version": "2.9.3-pre",
   "homepage": "https://qunitjs.com",
   "author": {
     "name": "jQuery Foundation and other contributors",

package.json

@@ -1,5 +1,5 @@
 import { window, localSessionStorage } from "../globals";
-import { extend } from "./utilities";
+import { extend, generateHash, uniqueTestName } from "./utilities";
 
 /**
  * Config object: Maintain internal state
@@ -58,7 +58,27 @@ const config = {
 	callbacks: {},
 
 	// The storage module to use for reordering tests
-	storage: localSessionStorage
+	storage: localSessionStorage,
+
+	// Get the testIds for all tests
+	// Simulates how test.js generates testIds by iteratively
+	// pushing testNames to get uniqueTestName
+	get testIds() {
+		var i, j, ml, tl, testIds = [], testNames, testName;
+
+		// Loop through all modules and tests and generate testIds
+		for ( i = 0, ml = config.modules.length; i < ml; i++ ) {
+			testNames = [];
+			for ( j = 0, tl = config.modules[ i ].tests.length; j < tl; j++ ) {
+				testName = uniqueTestName( testNames,
+					config.modules[ i ].tests[ j ].name );
+				testIds.push( generateHash( config.modules[ i ].name, testName ) );
+				testNames.push( testName );
+			}
+		}
+		return testIds;
+	}
+
 };
 
 // take a predefined QUnit.config and extend the defaults

src/core/config.js

@@ -157,3 +157,14 @@ export function generateHash( module, testName ) {
 
 	return hex.slice( -8 );
 }
+
+// Append whitespace to duplicated seen test name
+export function uniqueTestName( testNames, testName ) {
+	var i, l;
+	for ( i = 0, l = testNames; i < l.length; i++ ) {
+		if ( testNames[ i ] === testName ) {
+			testName += " ";
+		}
+	}
+	return testName;
+}

src/core/utilities.js

@@ -15,7 +15,8 @@ import {
 	hasOwn,
 	inArray,
 	now,
-	objectType
+	objectType,
+	uniqueTestName
 } from "./core/utilities";
 import { runLoggingCallbacks } from "./core/logging";
 import { extractStacktrace, sourceFromStacktrace } from "./core/stacktrace";
@@ -26,7 +27,7 @@ import TestReport from "./reports/test";
 let focused = false;
 
 export default function Test( settings ) {
-	var i, l;
+	var testNames;
 
 	++Test.count;
 
@@ -62,12 +63,14 @@ export default function Test( settings ) {
 		valid: this.valid()
 	} );
 
-	// Register unique strings
-	for ( i = 0, l = this.module.tests; i < l.length; i++ ) {
-		if ( this.module.tests[ i ].name === this.testName ) {
-			this.testName += " ";
-		}
-	}
+	// Register unique string for each test inside a module
+	//
+	// Note: this is the authorative way of generating testId
+	// testIds are generated one at a time when individual test is being run
+	// config.testIds also simulates the process here by pushing the tests
+	// one at a time to match the uniqueTestName generation here
+	testNames = this.module.tests.map( test => test.name );
+	this.testName = uniqueTestName( testNames, this.testName );
 
 	this.testId = generateHash( this.module.name, this.testName );
 

src/test.js

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<meta charset="UTF-8">
+		<title>QUnit ModuleId Test Suite</title>
+		<link rel="stylesheet" href="../dist/qunit.css">
+		<script src="../dist/qunit.js"></script>
+		<script src="testIds.js"></script>
+	</head>
+	<body>
+		<div id="qunit"></div>
+	</body>
+</html>

test/testIds.html

@@ -0,0 +1,10 @@
+QUnit.module( "QUnit.config.testIds" );
+
+QUnit.test( "testIds should return all testIds", function( assert ) {
+	assert.deepEqual( QUnit.config.testIds, [ "01e80961", "3b1922df" ], "this is the test with id 01e80961" );
+} );
+
+// This test is to prove identical test name is being hashed differently
+QUnit.test( "testIds should return all testIds", function( assert ) {
+	assert.deepEqual( QUnit.config.testIds, [ "01e80961", "3b1922df" ], "this is the test with id 3b1922df" );
+} );