Skip to content

Commit db71780

Browse files
justin-tayjustin-tay
authored
Add benchmark for performance regression testing (#1195)
1 parent e78c388 commit db71780

24 files changed

+2850
-116
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
name: Benchmarks
2+
on:
3+
push:
4+
branches:
5+
- master
6+
7+
permissions:
8+
contents: write
9+
deployments: write
10+
11+
jobs:
12+
benchmark:
13+
name: Run benchmark
14+
runs-on: ubuntu-latest
15+
steps:
16+
- name: Checkout
17+
uses: actions/checkout@v5
18+
- name: Setup JDK
19+
uses: actions/setup-java@v5
20+
with:
21+
distribution: 'temurin'
22+
java-version: '21'
23+
cache: 'maven'
24+
- name: Build with Maven
25+
run: |
26+
mvn --batch-mode --update-snapshots verify -Dstyle.color=always -Dmaven.javadoc.skip=true -Pbenchmark
27+
- name: Run benchmark
28+
run: |
29+
java -jar target/benchmarks.jar -wi 3 -i 3 -f 1 -rf json
30+
- name: Store raw benchmark result as artifact
31+
uses: actions/upload-artifact@v4
32+
with:
33+
name: benchmark-result
34+
path: jmh-result.json
35+
- name: Store benchmark result
36+
uses: benchmark-action/github-action-benchmark@v1
37+
with:
38+
name: JSON Schema Validator Benchmark
39+
tool: 'jmh'
40+
output-file-path: jmh-result.json
41+
# Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
42+
github-token: ${{ secrets.GITHUB_TOKEN }}
43+
auto-push: true
44+
comment-on-alert: false
45+
max-items-in-chart: 50
46+
summary-always: true

‎pom.xml‎

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,6 @@
8080
<version.graaljs>21.3.10</version.graaljs> <!-- 22.x and above is not Java 8 compatible -->
8181
<version.hamcrest>3.0</version.hamcrest>
8282
<version.junit>5.11.3</version.junit>
83-
<version.undertow>2.2.37.Final</version.undertow> <!-- 2.3.x and above is not Java 8 compatible -->
8483
<version.jacoco-maven-plugin>0.8.12</version.jacoco-maven-plugin>
8584
<version.maven-compiler-plugin>3.13.0</version.maven-compiler-plugin>
8685
<version.maven-bundle-plugin>5.1.9</version.maven-bundle-plugin>
@@ -92,6 +91,7 @@
9291
<version.nexus-staging-maven>1.7.0</version.nexus-staging-maven>
9392
<version.maven-gpg>3.2.7</version.maven-gpg>
9493

94+
<version.jmh>1.37</version.jmh>
9595
<argLine>-Duser.language=en -Duser.region=GB</argLine> <!-- For surefire -->
9696
</properties>
9797

@@ -123,13 +123,6 @@
123123
<version>${version.jackson}</version>
124124
</dependency>
125125

126-
<dependency>
127-
<groupId>io.undertow</groupId>
128-
<artifactId>undertow-core</artifactId>
129-
<version>${version.undertow}</version>
130-
<scope>test</scope>
131-
</dependency>
132-
133126
<dependency>
134127
<groupId>org.hamcrest</groupId>
135128
<artifactId>hamcrest</artifactId>
@@ -184,10 +177,21 @@
184177
<version>${version.slf4j}</version>
185178
</dependency>
186179

180+
<dependency>
181+
<groupId>org.openjdk.jmh</groupId>
182+
<artifactId>jmh-core</artifactId>
183+
<version>${version.jmh}</version>
184+
<scope>test</scope>
185+
</dependency>
186+
<dependency>
187+
<groupId>org.openjdk.jmh</groupId>
188+
<artifactId>jmh-generator-annprocess</artifactId>
189+
<version>${version.jmh}</version>
190+
<scope>test</scope>
191+
</dependency>
187192
</dependencies>
188193

189194
<build>
190-
191195
<resources>
192196
<resource>
193197
<filtering>false</filtering>
@@ -475,7 +479,43 @@
475479
</plugins>
476480
</build>
477481
</profile>
478-
482+
<profile>
483+
<id>benchmark</id>
484+
<properties>
485+
<skipTests>true</skipTests>
486+
</properties>
487+
<build>
488+
<finalName>benchmarks</finalName>
489+
<plugins>
490+
<plugin>
491+
<artifactId>maven-assembly-plugin</artifactId>
492+
<version>3.7.1</version>
493+
<configuration>
494+
<descriptors>
495+
<descriptor>src/test/assembly/assembly.xml</descriptor>
496+
</descriptors>
497+
</configuration>
498+
<executions>
499+
<execution>
500+
<id>make-assembly</id>
501+
<phase>package</phase>
502+
<goals>
503+
<goal>single</goal>
504+
</goals>
505+
<configuration>
506+
<archive>
507+
<manifest>
508+
<mainClass>org.openjdk.jmh.Main</mainClass>
509+
</manifest>
510+
</archive>
511+
<appendAssemblyId>false</appendAssemblyId>
512+
</configuration>
513+
</execution>
514+
</executions>
515+
</plugin>
516+
</plugins>
517+
</build>
518+
</profile>
479519
</profiles>
480520

481521
</project>

‎src/main/resources/draft/2020-12/meta/format-assertion‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "https://json-schema.org/draft/2020-12/meta/format-assertion",
4+
"$dynamicAnchor": "meta",
5+
6+
"title": "Format vocabulary meta-schema for assertion results",
7+
"type": ["object", "boolean"],
8+
"properties": {
9+
"format": { "type": "string" }
10+
}
11+
}

‎src/test/assembly/assembly.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<assembly
2+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
5+
<id>benchmark</id>
6+
<formats>
7+
<format>jar</format>
8+
</formats>
9+
<includeBaseDirectory>false</includeBaseDirectory>
10+
<dependencySets>
11+
<dependencySet>
12+
<outputDirectory>/</outputDirectory>
13+
<useProjectArtifact>true</useProjectArtifact>
14+
<unpack>true</unpack>
15+
<scope>test</scope>
16+
</dependencySet>
17+
</dependencySets>
18+
<fileSets>
19+
<fileSet>
20+
<directory>${project.build.directory}/test-classes</directory>
21+
<outputDirectory>/</outputDirectory>
22+
<includes>
23+
<include>**/*.class</include>
24+
<include>**/*.json</include>
25+
<include>**/*.properties</include>
26+
<include>**/*.xml</include>
27+
<include>**/META-INF/**</include>
28+
</includes>
29+
<useDefaultExcludes>true</useDefaultExcludes>
30+
</fileSet>
31+
</fileSets>
32+
</assembly>

‎src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
import com.networknt.schema.SpecVersion.VersionFlag;
2020
import com.networknt.schema.regex.JDKRegularExpressionFactory;
2121
import com.networknt.schema.regex.JoniRegularExpressionFactory;
22+
import com.networknt.schema.resource.InputStreamSource;
23+
import com.networknt.schema.resource.SchemaLoader;
2224
import com.networknt.schema.suite.TestCase;
2325
import com.networknt.schema.suite.TestSource;
2426
import com.networknt.schema.suite.TestSpec;
@@ -27,6 +29,8 @@
2729
import org.junit.jupiter.api.DynamicNode;
2830
import org.opentest4j.AssertionFailedError;
2931

32+
import java.io.File;
33+
import java.io.FileInputStream;
3034
import java.io.IOException;
3135
import java.io.UncheckedIOException;
3236
import java.nio.file.Files;
@@ -44,17 +48,15 @@
4448
import static org.junit.jupiter.api.DynamicContainer.dynamicContainer;
4549
import static org.junit.jupiter.api.DynamicTest.dynamicTest;
4650

47-
abstract class AbstractJsonSchemaTestSuite extends HTTPServiceSupport {
51+
abstract class AbstractJsonSchemaTestSuite {
4852

4953
private static String toForwardSlashPath(Path file) {
5054
return file.toString().replace('\\', '/');
5155
}
5256

5357
private static void executeTest(JsonSchema schema, TestSpec testSpec) {
5458
Set<ValidationMessage> errors = schema.validate(testSpec.getData(), OutputFormat.DEFAULT, (executionContext, validationContext) -> {
55-
if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")
56-
|| "ecmascript-regex.json"
57-
.equals(testSpec.getTestCase().getSource().getPath().getFileName().toString())) {
59+
if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")) {
5860
executionContext.getExecutionConfig().setFormatAssertionsEnabled(true);
5961
}
6062
});
@@ -180,14 +182,31 @@ private DynamicNode buildContainer(VersionFlag defaultVersion, TestCase testCase
180182

181183
private JsonSchemaFactory buildValidatorFactory(VersionFlag defaultVersion, TestCase testCase) {
182184
if (testCase.isDisabled()) return null;
183-
185+
SchemaLoader schemaLoader = new SchemaLoader() {
186+
@Override
187+
public InputStreamSource getSchema(AbsoluteIri absoluteIri) {
188+
String iri = absoluteIri.toString();
189+
if (iri.startsWith("http://localhost:1234")) {
190+
return () -> {
191+
String path = iri.substring("http://localhost:1234".length());
192+
File file = new File("src/test/resources/remotes" + path);
193+
if (file.exists()) {
194+
return new FileInputStream("src/test/resources/remotes" + path);
195+
}
196+
return new FileInputStream("src/test/suite/remotes" + path);
197+
};
198+
}
199+
return null;
200+
}
201+
};
184202
VersionFlag specVersion = detectVersion(testCase.getSchema(), testCase.getSpecification(), defaultVersion, false);
185203
JsonSchemaFactory base = JsonSchemaFactory.getInstance(specVersion);
186204
return JsonSchemaFactory
187205
.builder(base)
188206
.schemaMappers(schemaMappers -> schemaMappers
189207
.mapPrefix("https://", "http://")
190208
.mapPrefix("http://json-schema.org", "resource:"))
209+
.schemaLoaders(schemaLoaders -> schemaLoaders.add(schemaLoader))
191210
.build();
192211
}
193212

@@ -241,6 +260,9 @@ private void executeAndReset(JsonSchema schema, TestSpec testSpec) {
241260
}
242261
}
243262

263+
protected void cleanup() {
264+
}
265+
244266
private List<Path> findTestCases(String basePath) {
245267
try (Stream<Path> paths = Files.walk(Paths.get(basePath))) {
246268
return paths

‎src/test/java/com/networknt/schema/HTTPServiceSupport.java‎

Lines changed: 0 additions & 87 deletions
This file was deleted.

‎src/test/java/com/networknt/schema/Issue428Test.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
import static org.junit.jupiter.api.Assertions.assertEquals;
1313
import static org.junit.jupiter.api.Assertions.assertFalse;
1414

15-
class Issue428Test extends HTTPServiceSupport {
15+
class Issue428Test {
1616
protected ObjectMapper mapper = new ObjectMapper();
1717
protected JsonSchemaFactory validatorFactory = JsonSchemaFactory
1818
.builder(JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V4)).build();

0 commit comments

Comments
 (0)