diff --git a/build.gradle.kts b/build.gradle.kts index 0fd14d369..e8b24298c 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,11 +5,14 @@ import org.gradle.jvm.tasks.Jar // atomicfu buildscript { val atomicfuVersion: String by project + val serializationPluginVersion: String by project dependencies { classpath("org.jetbrains.kotlinx:atomicfu-gradle-plugin:$atomicfuVersion") + classpath("org.jetbrains.kotlin:kotlin-serialization:$serializationPluginVersion") } } apply(plugin = "kotlinx-atomicfu") +apply(plugin = "kotlinx-serialization") plugins { java @@ -23,6 +26,9 @@ repositories { } kotlin { + // we have to create custom sourceSets in advance before defining corresponding compilation targets + sourceSets.create("jvmBenchmark") + jvm { withJava() @@ -33,6 +39,51 @@ kotlin { val test by compilations.getting { kotlinOptions.jvmTarget = "11" } + + val benchmark by compilations.creating { + kotlinOptions.jvmTarget = "11" + + defaultSourceSet { + dependencies { + implementation(main.compileDependencyFiles + main.output.classesDirs) + } + } + + val benchmarksClassPath = + compileDependencyFiles + + runtimeDependencyFiles + + output.allOutputs + + files("$buildDir/processedResources/jvm/main") + + val benchmarksTestClassesDirs = output.classesDirs + + // task allowing to run benchmarks using JUnit API + val benchmark = tasks.register("jvmBenchmark") { + classpath = benchmarksClassPath + testClassesDirs = benchmarksTestClassesDirs + dependsOn("processResources") + } + + // task aggregating all benchmarks into a single suite and producing custom reports + val benchmarkSuite = tasks.register("jvmBenchmarkSuite") { + classpath = benchmarksClassPath + testClassesDirs = benchmarksTestClassesDirs + filter { + includeTestsMatching("LincheckBenchmarkSuite") + } + // pass the properties + systemProperty("statisticsGranularity", System.getProperty("statisticsGranularity")) + // always re-run test suite + outputs.upToDateWhen { false } + dependsOn("processResources") + } + + // task producing plots given the benchmarks report file + val benchmarkPlots by tasks.register("runBenchmarkPlots") { + classpath = benchmarksClassPath + mainClass.set("org.jetbrains.kotlinx.lincheck_benchmark.PlotsKt") + } + } } sourceSets { @@ -69,6 +120,28 @@ kotlin { implementation("io.mockk:mockk:${mockkVersion}") } } + + val jvmBenchmark by getting { + kotlin.srcDirs("src/jvm/benchmark") + + val junitVersion: String by project + val jctoolsVersion: String by project + val serializationVersion: String by project + val letsPlotVersion: String by project + val letsPlotKotlinVersion: String by project + val cliktVersion: String by project + dependencies { + implementation(project(":bootstrap")) + implementation("junit:junit:$junitVersion") + implementation("org.jctools:jctools-core:$jctoolsVersion") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$serializationVersion") + implementation("org.jetbrains.lets-plot:lets-plot-common:$letsPlotVersion") + implementation("org.jetbrains.lets-plot:lets-plot-kotlin-jvm:$letsPlotKotlinVersion") + implementation("com.github.ajalt.clikt:clikt:$cliktVersion") + } + } + + // jvmBenchmark.dependsOn(jvmMain) } } diff --git a/gradle.properties b/gradle.properties index d46891532..12eb1a237 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,9 +22,15 @@ withEventIdSequentialCheck=false kotlinVersion=1.9.21 kotlinxCoroutinesVersion=1.7.3 + asmVersion=9.6 atomicfuVersion=0.20.2 byteBuddyVersion=1.14.12 +serializationPluginVersion=1.6.21 +serializationVersion=1.3.3 +letsPlotVersion=2.5.0 +letsPlotKotlinVersion=4.0.0 +cliktVersion=3.4.0 junitVersion=4.13.1 jctoolsVersion=3.3.0 diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/AbstractLincheckBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/AbstractLincheckBenchmark.kt new file mode 100644 index 000000000..1aa07ac1e --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/AbstractLincheckBenchmark.kt @@ -0,0 +1,82 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") + +package org.jetbrains.kotlinx.lincheck_benchmark + +import org.jetbrains.kotlinx.lincheck.* +import org.jetbrains.kotlinx.lincheck.strategy.* +import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.ModelCheckingOptions +import org.jetbrains.kotlinx.lincheck.strategy.stress.StressOptions +import kotlin.reflect.KClass +import org.junit.Test + + +abstract class AbstractLincheckBenchmark( + private vararg val expectedFailures: KClass +) { + + @Test(timeout = TIMEOUT) + fun benchmarkWithStressStrategy(): Unit = StressOptions().run { + invocationsPerIteration(5_000) + configure() + runTest() + } + + @Test(timeout = TIMEOUT) + fun benchmarkWithModelCheckingStrategy(): Unit = ModelCheckingOptions().run { + invocationsPerIteration(5_000) + configure() + runTest() + } + + private fun > O.runTest() { + val statisticsTracker = LincheckStatisticsTracker( + granularity = benchmarksReporter.granularity + ) + val klass = this@AbstractLincheckBenchmark::class + val checker = LinChecker(klass.java, this) + val failure = checker.checkImpl(customTracker = statisticsTracker) + if (failure == null) { + assert(expectedFailures.isEmpty()) { + "This test should fail, but no error has been occurred (see the logs for details)" + } + } else { + assert(expectedFailures.contains(failure::class)) { + "This test has failed with an unexpected error: \n $failure" + } + } + val statistics = statisticsTracker.toBenchmarkStatistics( + name = klass.simpleName!!.removeSuffix("Benchmark"), + strategy = when (this) { + is StressOptions -> LincheckStrategy.Stress + is ModelCheckingOptions -> LincheckStrategy.ModelChecking + else -> throw IllegalStateException("Unsupported Lincheck strategy") + } + ) + benchmarksReporter.registerBenchmark(statistics) + } + + private fun > O.configure(): Unit = run { + iterations(30) + threads(3) + actorsPerThread(2) + actorsBefore(2) + actorsAfter(2) + minimizeFailedScenario(false) + customize() + } + + internal open fun > O.customize() {} + +} + +private const val TIMEOUT = 5 * 60 * 1000L // 5 minutes \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/BenchmarkStatistics.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/BenchmarkStatistics.kt new file mode 100644 index 000000000..1709c0a97 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/BenchmarkStatistics.kt @@ -0,0 +1,129 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +@file:Suppress("INVISIBLE_REFERENCE", "INVISIBLE_MEMBER") + +package org.jetbrains.kotlinx.lincheck_benchmark + +import org.jetbrains.kotlinx.lincheck.* +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.encodeToStream +import kotlin.time.Duration.Companion.nanoseconds +import kotlin.time.DurationUnit +import java.io.File + + +typealias BenchmarkID = String + +@Serializable +data class BenchmarksReport( + val data: Map +) + +@Serializable +data class BenchmarkStatistics( + val name: String, + val strategy: LincheckStrategy, + val runningTimeNano: Long, + val iterationsCount: Int, + val invocationsCount: Int, + val scenariosStatistics: List, + val invocationsRunningTimeNano: LongArray, +) + +@Serializable +data class ScenarioStatistics( + val threads: Int, + val operations: Int, + val invocationsCount: Int, + val runningTimeNano: Long, + val invocationAverageTimeNano: Long, + val invocationStandardErrorTimeNano: Long, +) + +val BenchmarksReport.benchmarkIDs: List + get() = data.keys.toList() + +val BenchmarksReport.benchmarkNames: List + get() = data.map { (_, statistics) -> statistics.name }.distinct() + +val BenchmarkStatistics.id: BenchmarkID + get() = "$name-$strategy" + +fun LincheckStatistics.toBenchmarkStatistics(name: String, strategy: LincheckStrategy) = BenchmarkStatistics( + name = name, + strategy = strategy, + runningTimeNano = runningTimeNano, + iterationsCount = iterationsCount, + invocationsCount = invocationsCount, + invocationsRunningTimeNano = iterationsStatistics + .values.map { it.invocationsRunningTimeNano } + .flatten(), + scenariosStatistics = iterationsStatistics + .values.groupBy { (it.scenario.nThreads to it.scenario.parallelExecution[0].size) } + .map { (key, statistics) -> + val (threads, operations) = key + val invocationsRunningTime = statistics + .map { it.invocationsRunningTimeNano } + .flatten() + val invocationsCount = statistics.sumOf { it.invocationsCount } + val runningTimeNano = statistics.sumOf { it.runningTimeNano } + val invocationAverageTimeNano = when { + // handle the case when per-invocation statistics is not gathered + invocationsRunningTime.isEmpty() -> (runningTimeNano.toDouble() / invocationsCount).toLong() + else -> invocationsRunningTime.average().toLong() + } + val invocationStandardErrorTimeNano = when { + // if per-invocation statistics is not gathered we cannot compute standard error + invocationsRunningTime.isEmpty() -> -1L + else -> invocationsRunningTime.standardError().toLong() + } + ScenarioStatistics( + threads = threads, + operations = operations, + invocationsCount = invocationsCount, + runningTimeNano = runningTimeNano, + invocationAverageTimeNano = invocationAverageTimeNano, + invocationStandardErrorTimeNano = invocationStandardErrorTimeNano, + ) + } +) + +fun BenchmarksReport.saveJson(filename: String) { + val file = File("$filename.json") + file.outputStream().use { outputStream -> + Json.encodeToStream(this, outputStream) + } +} + +// saves the report in simple text format for testing integration with ij-perf dashboards +fun BenchmarksReport.saveTxt(filename: String) { + val text = StringBuilder().apply { + appendReportHeader() + for (benchmarkStatistics in data.values) { + // for ij-perf reports, we currently track only benchmarks overall running time + appendBenchmarkRunningTime(benchmarkStatistics) + } + }.toString() + val file = File("$filename.txt") + file.writeText(text, charset = Charsets.US_ASCII) +} + +private fun StringBuilder.appendReportHeader() { + appendLine("Lincheck benchmarks suite") +} + +private fun StringBuilder.appendBenchmarkRunningTime(benchmarkStatistics: BenchmarkStatistics) { + with(benchmarkStatistics) { + val runningTimeMs = runningTimeNano.nanoseconds.toLong(DurationUnit.MILLISECONDS) + appendLine("${strategy}.${name}.runtime.ms $runningTimeMs") + } +} \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/LincheckBenchmarkSuite.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/LincheckBenchmarkSuite.kt new file mode 100644 index 000000000..adc88e0b1 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/LincheckBenchmarkSuite.kt @@ -0,0 +1,81 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark + +import kotlinx.coroutines.InternalCoroutinesApi +import org.jetbrains.kotlinx.lincheck.StatisticsGranularity +import org.jetbrains.kotlinx.lincheck_benchmark.running.* +import org.junit.Before +import org.junit.AfterClass +import org.junit.runner.RunWith +import org.junit.runners.Suite + +@InternalCoroutinesApi +@RunWith(Suite::class) +@Suite.SuiteClasses( + SpinLockBenchmark::class, + ReentrantLockBenchmark::class, + IntrinsicLockBenchmark::class, + ConcurrentLinkedQueueBenchmark::class, + ConcurrentDequeBenchmark::class, + ConcurrentHashMapBenchmark::class, + ConcurrentSkipListMapBenchmark::class, + RendezvousChannelBenchmark::class, + BufferedChannelBenchmark::class, +) +class LincheckBenchmarkSuite { + + @Before + fun setUp() { + System.gc() + } + + companion object { + @AfterClass + @JvmStatic + fun tearDown() { + benchmarksReporter.saveReport() + } + } + +} + +class LincheckBenchmarksReporter { + + private val statistics = mutableMapOf() + + val granularity: StatisticsGranularity = run { + when (val value = System.getProperty("statisticsGranularity").orEmpty()) { + "perInvocation" -> StatisticsGranularity.PER_INVOCATION + "perIteration" -> StatisticsGranularity.PER_ITERATION + "" -> StatisticsGranularity.PER_ITERATION + + else -> throw IllegalStateException(""" + Illegal value "$value" passed for statisticsGranularity parameter. + Allowed values are: perIteration, perInvocation. + """.trimIndent() + ) + } + } + + fun registerBenchmark(benchmarkStatistics: BenchmarkStatistics) { + statistics[benchmarkStatistics.id] = benchmarkStatistics + } + + fun saveReport() { + val report = BenchmarksReport(statistics) + report.saveJson("benchmarks-results") + report.saveTxt("benchmarks-results") + } + +} + +val benchmarksReporter = LincheckBenchmarksReporter() \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Plots.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Plots.kt new file mode 100644 index 000000000..829564e0e --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Plots.kt @@ -0,0 +1,287 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark + +import com.github.ajalt.clikt.core.* +import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.arguments.* +import com.github.ajalt.clikt.parameters.types.* +import org.jetbrains.letsPlot.* +import org.jetbrains.letsPlot.export.* +import org.jetbrains.letsPlot.geom.* +import org.jetbrains.letsPlot.label.* +import org.jetbrains.letsPlot.scale.* +import org.jetbrains.letsPlot.sampling.samplingNone +import org.jetbrains.letsPlot.pos.positionDodge +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import org.jetbrains.kotlinx.lincheck.LincheckStrategy +import kotlin.time.Duration.Companion.nanoseconds +import kotlin.time.DurationUnit +import kotlin.io.path.inputStream + + +fun BenchmarksReport.runningTimeBarPlot(filename: String, path: String? = null) { + val data = runningTimeData() + var plot = letsPlot(data) + plot += ggtitle("Benchmarks running time") + plot += ggsize(600, 800) + plot += labs( + x = "benchmark name", + y = "time (ms)", + ) + plot += geomBar( + stat = Stat.identity, + position = positionDodge(), + ) { + x = "name" + y = "runningTime" + fill = "strategy" + } + ggsave(plot, filename, path = path) +} + +fun BenchmarksReport.runningTimeData( + durationUnit: DurationUnit = DurationUnit.MILLISECONDS +): Map { + val map = mutableMapOf() + val ids = data.keys.toList() + map["name"] = ids.map { id -> + data[id]!!.name + } + map["strategy"] = ids.map { id -> + data[id]!!.strategy.toString() + } + map["runningTime"] = ids.map { id -> + data[id]!!.runningTimeNano.nanoseconds.toLong(durationUnit) + } + return map +} + +fun BenchmarksReport.invocationTimeByScenarioSizeBarPlot( + benchmarkName: String, + filename: String, + path: String? = null) +{ + val data = invocationTimeByScenarioSizeData(benchmarkName) + var plot = letsPlot(data) { + x = "params" + } + plot += ggtitle("Invocation average time by scenario size", subtitle = benchmarkName) + plot += ggsize(600, 800) + plot += labs( + x = "(#threads, #operations)", + y = "time (us)", + ) + plot += geomBar( + stat = Stat.identity, + position = positionDodge(), + ) { + y = "timeAverage" + fill = "strategy" + } + if ("timeErrorLow" in data && "timeErrorHigh" in data) { + plot += geomErrorBar( + width = .1, + position = positionDodge(0.9), + ) { + ymin = "timeErrorLow" + ymax = "timeErrorHigh" + group = "strategy" + } + } + ggsave(plot, filename, path = path) +} + +fun BenchmarksReport.invocationTimeByScenarioSizeData( + benchmarkName: String, + durationUnit: DurationUnit = DurationUnit.MICROSECONDS, +): Map { + val map = mutableMapOf() + val benchmarks = data.values + .filter { it.name == benchmarkName } + .map { (it.strategy to it.scenariosStatistics) } + val isStandardErrorDefined: Boolean = benchmarks.all { (_, scenariosStatistics) -> + scenariosStatistics.all { it.invocationStandardErrorTimeNano >= 0L } + } + map["params"] = benchmarks.flatMap { (_, stats) -> + stats.map { (it.threads to it.operations).toString() } + } + map["strategy"] = benchmarks.flatMap { (strategy, stats) -> + stats.map { strategy.toString() } + } + map["timeAverage"] = benchmarks.flatMap { (_, stats) -> + stats.map { it.invocationAverageTimeNano.nanoseconds.toLong(durationUnit) } + } + if (isStandardErrorDefined) { + map["timeErrorLow"] = benchmarks.flatMap { (_, stats) -> + stats.map { + (it.invocationAverageTimeNano - it.invocationStandardErrorTimeNano) + .nanoseconds.toLong(durationUnit) + } + } + map["timeErrorHigh"] = benchmarks.flatMap { (_, stats) -> + stats.map { + (it.invocationAverageTimeNano + it.invocationStandardErrorTimeNano) + .nanoseconds.toLong(durationUnit) + } + } + } + return map +} + +fun BenchmarksReport.invocationTimeScatterPlot( + benchmarkID: BenchmarkID, + filename: String, + path: String? = null +) { + val data = invocationsTimeData(benchmarkID) + var plot = letsPlot(data) + plot += ggtitle("Invocations time", subtitle = benchmarkID) + plot += labs( + x = "# invocation", + y = "time (us)" + ) + plot += ggsize(1600, 900) + plot += geomPoint( + stat = Stat.identity, + sampling = samplingNone, + ) { + x = "invocationID" + y = "invocationRunningTimeNano" + } + plot += scaleYLog10() + plot += scaleYContinuous( + breaks = listOf(10, 100, 1000, 10_000, 100_000, 1_000_000), + limits = 10 to 5_000_000 + ) + ggsave(plot, filename, path = path) +} + +fun BenchmarksReport.invocationsTimeData(benchmarkID: BenchmarkID, + durationUnit: DurationUnit = DurationUnit.MICROSECONDS, +): Map { + val map = mutableMapOf() + val invocationsRunningTimeNano = data[benchmarkID]!!.invocationsRunningTimeNano + .apply { convertTo(durationUnit) } + map["invocationID"] = invocationsRunningTimeNano.indices.toList() + map["invocationRunningTimeNano"] = invocationsRunningTimeNano + return map +} + +enum class PlotType { + AllPlots, + RunningTimeBarPlot, + InvocationTimeByScenarioSizeBarPlot, + InvocationTimeScatterPlot, +} + +const val defaultPlotExtension = "html" + +fun PlotType.defaultOutputFilename(): String = when (this) { + PlotType.RunningTimeBarPlot -> "running-time-bar-plot.$defaultPlotExtension" + PlotType.InvocationTimeByScenarioSizeBarPlot -> "invocation-time-by-size-bar-plot.$defaultPlotExtension" + PlotType.InvocationTimeScatterPlot -> "invocation-time-scatter-plot.$defaultPlotExtension" + else -> throw IllegalArgumentException() +} + +class PlotCommand : CliktCommand() { + + val plotType by option() + .help("type of the plot to draw") + .enum() + .default(PlotType.AllPlots) + + val benchmarkName by option() + .help("name of the benchmark") + + val strategyName by option() + .help("strategy used in the benchmark") + .enum() + + val report by argument() + .help("path to the benchmarks report file (.json)") + .path(canBeFile = true, canBeDir = false) + + val output by argument() + .help("path to the output directory") + .path(canBeDir = true, canBeFile = false) + .optional() + + override fun run() { + val report = report.inputStream().use { inputStream -> + Json.decodeFromStream(inputStream) + } + + if (plotType == PlotType.RunningTimeBarPlot || plotType == PlotType.AllPlots) { + val plotFilename = PlotType.RunningTimeBarPlot.defaultOutputFilename() + report.runningTimeBarPlot(plotFilename, path = output?.toString()) + logProducedPlot(plotFilename) + } + + if (plotType == PlotType.InvocationTimeByScenarioSizeBarPlot || plotType == PlotType.AllPlots) { + val benchmarkNames = when (plotType) { + PlotType.InvocationTimeByScenarioSizeBarPlot -> { + require(benchmarkName != null) { + "Benchmark name is required for $plotType" + } + listOf(benchmarkName!!) + } + PlotType.AllPlots -> report.benchmarkNames + else -> throw IllegalStateException() + } + for (name in benchmarkNames) { + val plotFilename = "${name}-${PlotType.RunningTimeBarPlot.defaultOutputFilename()}" + report.invocationTimeByScenarioSizeBarPlot(name, plotFilename, path = output?.toString()) + logProducedPlot(plotFilename) + } + } + + if (plotType == PlotType.InvocationTimeScatterPlot || plotType == PlotType.AllPlots) { + val benchmarkIDs = when (plotType) { + PlotType.InvocationTimeScatterPlot -> { + require(benchmarkName != null) { + "Benchmark name is required for $plotType" + } + require(strategyName != null) { + "Strategy name is required for $plotType" + } + val benchmark = report.data.values.find { + it.name == benchmarkName && it.strategy == strategyName + } + require(benchmark != null) { + """ + Benchmark with the given parameters has not been found in the report: + benchmark name = $benchmarkName + strategy name = $strategyName + """.trimIndent() + } + listOf(benchmark.id) + } + PlotType.AllPlots -> report.benchmarkIDs + else -> throw IllegalStateException() + } + for (benchmarkID in benchmarkIDs) { + val benchmark = report.data[benchmarkID]!! + val plotFilename = "${benchmark.name}-${benchmark.strategy}-" + + PlotType.InvocationTimeScatterPlot.defaultOutputFilename() + report.invocationTimeScatterPlot(benchmarkID, plotFilename, path = output?.toString()) + logProducedPlot(plotFilename) + } + } + } + + private fun logProducedPlot(plotFilename: String) { + println("Produced plot $plotFilename") + } +} + +fun main(args: Array) = PlotCommand().main(args) \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/README.md b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/README.md new file mode 100644 index 000000000..c57940843 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/README.md @@ -0,0 +1,111 @@ +# Lincheck Benchmarks + +This document describes various benchmarks used within the Lincheck project +and contains instructions on how to run them to produce reports and plots. + +### Benchmark categories + +Benchmarks are split into several categories with +each category aiming to evaluate particular aspects of the framework. +All benchmarks within each category have similar structure +and collect the same set of metrics. + +#### Category `running` + +Benchmarks in this category aim to evaluate the overall performance of +running Lincheck tests using different strategies. +In each benchmark, specific correctly implemented +data structure or synchronization primitive is tested by Lincheck +(since the implementations are assumed to be correct, the tool should not report any bugs). +For each benchmark, the Lincheck is requested to generate fixed number of scenarios, +and run each scenario fixed number of times with fixed strategy. +The overall running time of the benchmark is measured. + +**Parameters:** +- tested data-structure or synchronization primitive +- number of iterations +- number of invocations per iteration +- Lincheck strategy to run the scenarios + +**Metrics:** +- overall running time + +#### Collected statistics + +Although the main metric for benchmarks from this category is overall running time, +the benchmarking tool is actually capable of collecting more statistical information, +which might be useful when debugging the performance issues of the tool. + +For each benchmark, the following statistics are collected: +- overall running time +- number of iterations +- total number of invocations (across all iterations) +- per-scenario statistics (see below) + +For each scenario within each benchmark, the following statistics are collected: +- overall running time +- number of invocations +- running time of each invocation (optional) + +### Running benchmarks + +Individual benchmarks can be run via the `jvmBenchmark` gradle task +using the standard test task arguments: + +``` +./gradlew :jvmBenchmark --tests ConcurrentHashMapBenchmark +``` + +To run all benchmarks and produce the reports, use the `jvmBenchmarkSuite` gradle task: + +``` +./gradlew :jvmBenchmarkSuite +``` + +By default, this task collects only per-iteration statistics, +without the information about the running time of each invocation. +It is because this information significantly increases the size of the produced reports. +If you need per-invocation statistics, run the `jvmBenchmarkSuite` task +with the `statisticsGranularity` option set to `perInvocation`: + +``` +./gradlew :jvmBenchmarkSuite -DstatisticsGranularity=perInvocation +``` + +The `jvmBenchmarkSuite` task produces the following reports: +- `benchmarks-results.json` - report in JSON format containing all collected statistics +- `benchmarks-results.txt` - report in simple text format containing only the overall running time of each benchmark + +These reports can then be uploaded to an external storage, +or analyzed locally, using the shipped plotting facilities (see below). + +### Drawing plots + +The Lincheck benchmarking project includes a script +to draw various plots based on collected benchmarks statistics. +The script utilizes the [Lets-Plot Kotlin](https://github.com/JetBrains/lets-plot-kotlin) library. + +The following types of plots are currently supported. +- Benchmarks' total running time bar plot. +- Invocation average running time per scenario size bar plot (for each benchmarked data structure). +- Invocation time scatter plot (for each benchmarked data structure and strategy). + +The gradle task `runBenchmarkPlots` can be invoked to produce the plots. +It expects the path to the `.json` report file to be provided as the first argument: + +``` +./gradlew :runBenchmarkPlots --args="benchmarks-results.json" +``` + +By default, this task produces all available plots in `.html` format and stores them into `lets-plot-images` directory. + +To see a description of all available task options, run the following command: + +``` +./gradlew :runBenchmarkPlots --args="--help" +``` + + + + + diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Utils.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Utils.kt new file mode 100644 index 000000000..1a4bf64e7 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/Utils.kt @@ -0,0 +1,48 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark + +import kotlin.math.round +import kotlin.math.sqrt +import kotlin.time.Duration.Companion.nanoseconds +import kotlin.time.DurationUnit + +fun Iterable.flatten(): LongArray { + val size = sumOf { it.size } + val result = LongArray(size) + var i = 0 + for (array in this) { + for (element in array) { + result[i++] = element + } + } + return result +} + +fun LongArray.convertTo(unit: DurationUnit) { + for (i in indices) { + this[i] = this[i].nanoseconds.toLong(unit) + } +} + +fun LongArray.standardDeviation(): Double { + val mean = round(average()).toLong() + var variance = 0L + for (x in this) { + val d = x - mean + variance += d * d + } + return sqrt(variance.toDouble() / (size - 1)) +} + +fun LongArray.standardError(): Double { + return standardDeviation() / sqrt(size.toDouble()) +} \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/BufferedChannelBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/BufferedChannelBenchmark.kt new file mode 100644 index 000000000..97bfad6cb --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/BufferedChannelBenchmark.kt @@ -0,0 +1,45 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.* +import org.jetbrains.kotlinx.lincheck.* +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.specifications.* +import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark + +@InternalCoroutinesApi +@Param(name = "value", gen = IntGen::class, conf = "1:5") +class BufferedChannelBenchmark : AbstractLincheckBenchmark() { + private val c = Channel(2) + + @Operation(cancellableOnSuspension = false) + suspend fun send(@Param(name = "value") value: Int) = c.send(value) + + @Operation(cancellableOnSuspension = false) + suspend fun receive() = c.receive() + + @Operation + fun poll() = c.tryReceive().getOrNull() + + @Operation + fun offer(@Param(name = "value") value: Int) = c.trySend(value).isSuccess + + override fun > O.customize() { + iterations(10) + sequentialSpecification(SequentiaBuffered2IntChannelSpecification::class.java) + } +} + +@InternalCoroutinesApi +class SequentiaBuffered2IntChannelSpecification : SequentialIntChannelSpecification(capacity = 2) \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentDequeBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentDequeBenchmark.kt new file mode 100644 index 000000000..459817149 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentDequeBenchmark.kt @@ -0,0 +1,33 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark +import java.util.concurrent.* + +@Param(name = "value", gen = IntGen::class, conf = "1:5") +class ConcurrentDequeBenchmark : AbstractLincheckBenchmark() { + private val deque = ConcurrentLinkedDeque() + + @Operation + fun addFirst(e: Int) = deque.addFirst(e) + + @Operation + fun addLast(e: Int) = deque.addLast(e) + + @Operation + fun pollFirst() = deque.pollFirst() + + @Operation + fun pollLast() = deque.pollLast() + +} diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentHashMapBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentHashMapBenchmark.kt new file mode 100644 index 000000000..04884acf4 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentHashMapBenchmark.kt @@ -0,0 +1,30 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.paramgen.* +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark +import java.util.concurrent.ConcurrentHashMap + +@Param(name = "key", gen = IntGen::class, conf = "1:5") +class ConcurrentHashMapBenchmark : AbstractLincheckBenchmark() { + private val map = ConcurrentHashMap() + + @Operation + fun put(@Param(name = "key") key: Int, value: Int) = map.put(key, value) + + @Operation + operator fun get(@Param(name = "key") key: Int) = map[key] + + @Operation + fun remove(@Param(name = "key") key: Int) = map.remove(key) +} \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentLinkedQueueBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentLinkedQueueBenchmark.kt new file mode 100644 index 000000000..296f96f5a --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentLinkedQueueBenchmark.kt @@ -0,0 +1,33 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark +import java.util.concurrent.* + +@Param(name = "value", gen = IntGen::class, conf = "1:5") +class ConcurrentLinkedQueueBenchmark : AbstractLincheckBenchmark() { + private val queue = ConcurrentLinkedQueue() + + @Operation + fun add(e: Int) = queue.add(e) + + @Operation + fun offer(e: Int) = queue.offer(e) + + @Operation + fun peek() = queue.peek() + + @Operation + fun poll() = queue.poll() + +} diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentSkipListMapBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentSkipListMapBenchmark.kt new file mode 100644 index 000000000..5f445eb05 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/ConcurrentSkipListMapBenchmark.kt @@ -0,0 +1,33 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.paramgen.* +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark +import java.util.concurrent.* + +@Param(name = "value", gen = IntGen::class, conf = "1:5") +class ConcurrentSkipListMapBenchmark : AbstractLincheckBenchmark() { + private val skiplistMap = ConcurrentSkipListMap() + + @Operation + fun put(key: Int, value: Int) = skiplistMap.put(key, value) + + @Operation + fun get(key: Int) = skiplistMap.get(key) + + @Operation + fun containsKey(key: Int) = skiplistMap.containsKey(key) + + @Operation + fun remove(key: Int) = skiplistMap.remove(key) +} \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/LocksBenchmarks.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/LocksBenchmarks.kt new file mode 100644 index 000000000..7e15f55b5 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/LocksBenchmarks.kt @@ -0,0 +1,60 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import org.jetbrains.kotlinx.lincheck.annotations.Operation +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +class SpinLockBenchmark : AbstractLincheckBenchmark() { + private var counter = 0 + private val locked = AtomicBoolean() + + @Operation + fun increment(): Int = withSpinLock { counter++ } + + @Operation + fun get(): Int = withSpinLock { counter } + + private fun withSpinLock(block: () -> T): T { + while (!locked.compareAndSet(false, true)) { + Thread.yield() + } + try { + return block() + } finally { + locked.set(false) + } + } +} + +class ReentrantLockBenchmark : AbstractLincheckBenchmark() { + private var counter = 0 + private val lock = ReentrantLock() + + @Operation + fun increment(): Int = lock.withLock { counter++ } + + @Operation + fun get(): Int = lock.withLock { counter } +} + +class IntrinsicLockBenchmark : AbstractLincheckBenchmark() { + private var counter = 0 + + @Operation + fun increment(): Int = synchronized(this) { counter++ } + + @Operation + fun get(): Int = synchronized(this) { counter } +} \ No newline at end of file diff --git a/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/RendezvousChannelBenchmark.kt b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/RendezvousChannelBenchmark.kt new file mode 100644 index 000000000..fd4efc0b9 --- /dev/null +++ b/src/jvm/benchmark/org/jetbrains/kotlinx/lincheck_benchmark/running/RendezvousChannelBenchmark.kt @@ -0,0 +1,45 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This Source Code Form is subject to the terms of the + * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package org.jetbrains.kotlinx.lincheck_benchmark.running + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.* +import org.jetbrains.kotlinx.lincheck.* +import org.jetbrains.kotlinx.lincheck.annotations.* +import org.jetbrains.kotlinx.lincheck.specifications.* +import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck_benchmark.AbstractLincheckBenchmark + +@ExperimentalCoroutinesApi +@InternalCoroutinesApi +@Param(name = "value", gen = IntGen::class, conf = "1:5") +class RendezvousChannelBenchmark : AbstractLincheckBenchmark() { + private val ch = Channel() + + @Operation + suspend fun send(@Param(name = "value") value: Int) = ch.send(value) + + @Operation + suspend fun receive() = ch.receive() + + @Operation + suspend fun receiveOrNull() = ch.receiveCatching().getOrNull() + + @Operation + fun close() = ch.close() + + override fun > O.customize() { + iterations(10) + sequentialSpecification(SequentialRendezvousIntChannel::class.java) + } +} + +@InternalCoroutinesApi +class SequentialRendezvousIntChannel : SequentialIntChannelSpecification(capacity = 0) \ No newline at end of file diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/LinChecker.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LinChecker.kt index 6dd4041b4..aac16d153 100644 --- a/src/jvm/main/org/jetbrains/kotlinx/lincheck/LinChecker.kt +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LinChecker.kt @@ -14,7 +14,6 @@ import org.jetbrains.kotlinx.lincheck.annotations.Operation import org.jetbrains.kotlinx.lincheck.execution.* import org.jetbrains.kotlinx.lincheck.strategy.* import org.jetbrains.kotlinx.lincheck.transformation.withLincheckJavaAgent -import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.ModelCheckingCTestConfiguration import org.jetbrains.kotlinx.lincheck.verifier.* import org.jetbrains.kotlinx.lincheck.strategy.managed.modelchecking.* import org.jetbrains.kotlinx.lincheck.strategy.stress.* @@ -52,28 +51,31 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { /** * Runs Lincheck to check the tested class under given configurations. * - * @param cont Optional continuation taking [LincheckFailure] as an argument. + * @param continuation Optional continuation taking [LincheckFailure] as an argument. * The continuation is run in the context when Lincheck java-agent is still attached. * @return [LincheckFailure] if a failure is discovered, null otherwise. */ @Synchronized // never run Lincheck tests in parallel - internal fun checkImpl(cont: LincheckFailureContinuation? = null): LincheckFailure? { + internal fun checkImpl( + customTracker: LincheckRunTracker? = null, + continuation: LincheckFailureContinuation? = null, + ): LincheckFailure? { check(testConfigurations.isNotEmpty()) { "No Lincheck test configuration to run" } lincheckVerificationStarted() for (testCfg in testConfigurations) { withLincheckJavaAgent(testCfg.instrumentationMode) { - val failure = testCfg.checkImpl() + val failure = testCfg.checkImpl(customTracker) if (failure != null) { - if (cont != null) cont(failure) + if (continuation != null) continuation(failure) return failure } } } - if (cont != null) cont(null) + if (continuation != null) continuation(null) return null } - private fun CTestConfiguration.checkImpl(): LincheckFailure? { + private fun CTestConfiguration.checkImpl(customTracker: LincheckRunTracker? = null): LincheckFailure? { var verifier = createVerifier() val generator = createExecutionGenerator(testStructure.randomProvider) // create a sequence that generates scenarios lazily on demand @@ -90,6 +92,11 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { // only after all custom scenarios are checked val scenarios = customScenarios.asSequence() + randomScenarios.take(iterations) val scenariosSize = customScenarios.size + iterations + val tracker = ChainedRunTracker().apply { + if (customTracker != null) + addTracker(customTracker) + } + val statisticsTracker = tracker.addTrackerIfAbsent { LincheckStatisticsTracker() } scenarios.forEachIndexed { i, scenario -> val isCustomScenario = (i < customScenarios.size) // For performance reasons, verifier re-uses LTS from previous iterations. @@ -100,19 +107,20 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { if ((i + 1) % VERIFIER_REFRESH_CYCLE == 0) verifier = createVerifier() scenario.validate() - reporter.logIteration(i + 1, scenariosSize, scenario) - var failure = scenario.run(this, verifier) + reporter.logIteration(i, scenariosSize, scenario) + var failure = scenario.run(i, this, verifier, tracker) + reporter.logIterationStatistics(i, statisticsTracker) if (failure == null) return@forEachIndexed + var j = i + 1 if (minimizeFailedScenario && !isCustomScenario) { - var j = i + 1 reporter.logScenarioMinimization(scenario) failure = failure.minimize { minimizedScenario -> - minimizedScenario.run(this, createVerifier()) + minimizedScenario.run(j++, this, createVerifier(), tracker) } } reporter.logFailedIteration(failure) - runReplayForPlugin(failure, verifier) + runReplayForPlugin(j++, failure, verifier) return failure } return null @@ -123,14 +131,20 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { * We cannot initiate the failed interleaving replaying in the strategy code, * as the failing scenario might need to be minimized first. */ - private fun CTestConfiguration.runReplayForPlugin(failure: LincheckFailure, verifier: Verifier) { + private fun CTestConfiguration.runReplayForPlugin( + iteration: Int, + failure: LincheckFailure, + verifier: Verifier, + tracker: LincheckRunTracker? = null, + ) { if (ideaPluginEnabled() && this is ModelCheckingCTestConfiguration) { reporter.logFailedIteration(failure, loggingLevel = LoggingLevel.WARN) enableReplayModeForIdeaPlugin() val strategy = createStrategy(failure.scenario) check(strategy is ModelCheckingStrategy) + val parameters = createIterationParameters(strategy) strategy.use { - val replayedFailure = it.runIteration(invocationsPerIteration, verifier) + val replayedFailure = it.runIteration(iteration, parameters, verifier, tracker) check(replayedFailure != null) strategy.runReplayIfPluginEnabled(replayedFailure) } @@ -140,15 +154,23 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { } private fun ExecutionScenario.run( + iteration: Int, testCfg: CTestConfiguration, verifier: Verifier, + tracker: LincheckRunTracker? = null, ): LincheckFailure? { val strategy = testCfg.createStrategy(this) + val parameters = testCfg.createIterationParameters(strategy) return strategy.use { - it.runIteration(testCfg.invocationsPerIteration, verifier) + it.runIteration(iteration, parameters, verifier, tracker) } } + private fun Reporter.logIterationStatistics(iteration: Int, statisticsTracker: LincheckStatisticsTracker) { + val statistics = statisticsTracker.iterationsStatistics[iteration]!! + logIterationStatistics(statistics.totalInvocationsCount, statistics.totalRunningTimeNano) + } + private fun CTestConfiguration.createStrategy(scenario: ExecutionScenario) = createStrategy( testClass = testClass, @@ -172,6 +194,17 @@ class LinChecker(private val testClass: Class<*>, options: Options<*, *>?) { return constructor.newInstance(this, testStructure, randomProvider) } + private fun CTestConfiguration.createIterationParameters(strategy: Strategy) = + IterationParameters( + strategy = when (strategy) { + is StressStrategy -> LincheckStrategy.Stress + is ModelCheckingStrategy -> LincheckStrategy.ModelChecking + else -> throw IllegalStateException("Unsupported Lincheck strategy") + }, + invocationsBound = this.invocationsPerIteration, + warmUpInvocationsCount = 0, + ) + private val CTestConfiguration.invocationsPerIteration get() = when (this) { is ModelCheckingCTestConfiguration -> this.invocationsPerIteration is StressCTestConfiguration -> this.invocationsPerIteration @@ -268,12 +301,12 @@ internal fun > O.checkImpl(testClass: Class<*>): LincheckFailu * To overcome this problem, we run the continuation in the context when Lincheck java-agent is still attached. * * @param testClass Tested class. - * @param cont Continuation taking [LincheckFailure] as an argument. + * @param continuation Continuation taking [LincheckFailure] as an argument. * The continuation is run in the context when Lincheck java-agent is still attached. * @return [LincheckFailure] if a failure is discovered, null otherwise. */ -internal fun > O.checkImpl(testClass: Class<*>, cont: LincheckFailureContinuation) { - LinChecker(testClass, this).checkImpl(cont) +internal fun > O.checkImpl(testClass: Class<*>, continuation: LincheckFailureContinuation) { + LinChecker(testClass, this).checkImpl(continuation = continuation) } internal typealias LincheckFailureContinuation = (LincheckFailure?) -> Unit diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckRunTracker.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckRunTracker.kt new file mode 100644 index 000000000..04d6abf9c --- /dev/null +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckRunTracker.kt @@ -0,0 +1,242 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * + */ + +package org.jetbrains.kotlinx.lincheck + +import org.jetbrains.kotlinx.lincheck.execution.ExecutionScenario +import org.jetbrains.kotlinx.lincheck.strategy.LincheckFailure + +/** + * The LincheckRunTracker interface defines methods for tracking the progress and results of a Lincheck test run. + * + * Each Lincheck test run consists of a number of iterations, + * with each iteration corresponding to the specific scenario. + * On each iteration, the same scenario can be invoked multiple times in an attempt to uncover a failure. + * By overriding the methods of LincheckRunTracker interface, + * it is possible to track the beginning and end of each iteration or invocation, as well as their results. + */ +interface LincheckRunTracker { + + /** + * This method is called before the start of each iteration of the Lincheck test. + * + * @param iteration The iteration id. + * @param scenario The execution scenario to be run. + */ + fun iterationStart(iteration: Int, scenario: ExecutionScenario, parameters: IterationParameters) {} + + /** + * This method is called at the end of each iteration of the Lincheck test. + * + * @param iteration The iteration id. + * @param failure The failure that occurred during the iteration or null if no failure occurred. + * @param exception The exception that occurred during the iteration or null if no exception occurred. + */ + fun iterationEnd(iteration: Int, failure: LincheckFailure? = null, exception: Throwable? = null) {} + + /** + * This method is called before the start of each invocation of the specific iteration. + * + * @param iteration The iteration id. + * @param invocation The invocation number. + */ + fun invocationStart(iteration: Int, invocation: Int) {} + + /** + * This method is called at the end of each invocation of the specific iteration. + * + * @param iteration The iteration id. + * @param invocation The invocation number. + * @param failure the failure that occurred during the invocation or null if no failure occurred. + * @param exception the exception that occurred during the invocation or null if no exception occurred. + */ + fun invocationEnd(iteration: Int, invocation: Int, failure: LincheckFailure? = null, exception: Throwable? = null) {} + + /** + * For a composite trackers, encompassing several internal sub-trackers, + * this method should return a list of internal sub-trackers in the order + * they are called by the parent tracker. + * + * @return a list of internal trackers. + */ + fun internalTrackers(): List = listOf() + +} + +/** + * Represents the parameters for controlling the iteration of a Lincheck test run. + * + * @property invocationsBound The maximum number of invocations to be performed on the iteration. + * @property warmUpInvocationsCount The number of warm-up invocations to be performed. + */ +data class IterationParameters( + val strategy: LincheckStrategy, + val invocationsBound: Int, + val warmUpInvocationsCount: Int, +) + +/** + * Represents the testing strategies that can be used in Lincheck. + */ +enum class LincheckStrategy { + Stress, ModelChecking +} + +/** + * Tracks the execution of a given Lincheck test iteration. + * + * @param iteration The iteration id. + * @param scenario The execution scenario for the iteration. + * @param block The code to be executed for the iteration. + * + * @return The failure, if any, that occurred during the execution of the iteration. + */ +inline fun LincheckRunTracker?.trackIteration( + iteration: Int, + scenario: ExecutionScenario, + params: IterationParameters, + block: () -> LincheckFailure? +): LincheckFailure? { + var failure: LincheckFailure? = null + var exception: Throwable? = null + this?.iterationStart(iteration, scenario, params) + try { + return block().also { + failure = it + } + } catch (throwable: Throwable) { + exception = throwable + throw throwable + } finally { + this?.iterationEnd(iteration, failure, exception) + } +} + + +/** + * Tracks the invocation of the specific Lincheck test iteration. + * + * @param iteration The iteration id. + * @param invocation The current invocation within the iteration. + * @param block The block of code to be executed. + * + * @return The failure, if any, that occurred during the execution of the invocation. + */ +inline fun LincheckRunTracker?.trackInvocation( + iteration: Int, + invocation: Int, + block: () -> LincheckFailure? +): LincheckFailure? { + var failure: LincheckFailure? = null + var exception: Throwable? = null + this?.invocationStart(iteration, invocation) + try { + return block().also { + failure = it + } + } catch (throwable: Throwable) { + exception = throwable + throw throwable + } finally { + this?.invocationEnd(iteration, invocation, failure, exception) + } +} + + +/** + * Chains multiple Lincheck run trackers into a single tracker. + * The chained tracker delegates method calls to each tracker in the chain. + * + * @return The chained LincheckRunTracker, or null if the original list is empty. + */ +fun List.chainTrackers(): LincheckRunTracker? = + if (this.isEmpty()) null else ChainedRunTracker(this) + +internal class ChainedRunTracker(trackers: List = listOf()) : LincheckRunTracker { + + private val trackers = mutableListOf() + + init { + this.trackers.addAll(trackers) + } + + fun addTracker(tracker: LincheckRunTracker) { + trackers.add(tracker) + } + + override fun iterationStart(iteration: Int, scenario: ExecutionScenario, parameters: IterationParameters) { + for (tracker in trackers) { + tracker.iterationStart(iteration, scenario, parameters) + } + } + + override fun iterationEnd(iteration: Int, failure: LincheckFailure?, exception: Throwable?) { + for (tracker in trackers) { + tracker.iterationEnd(iteration, failure, exception) + } + } + + override fun invocationStart(iteration: Int, invocation: Int) { + for (tracker in trackers) { + tracker.invocationStart(iteration, invocation) + } + } + + override fun invocationEnd(iteration: Int, invocation: Int, failure: LincheckFailure?, exception: Throwable?) { + for (tracker in trackers) { + tracker.invocationEnd(iteration, invocation, failure, exception) + } + } + + override fun internalTrackers(): List { + return trackers + } +} + +/** + * Searches for a first LincheckRunTracker of the specified type + * among the internal sub-trackers of a given tracker. + * + * @param T The type of LincheckRunTracker to search for. + * + * @return The LincheckRunTracker of the specified type if found, otherwise null. + */ +inline fun LincheckRunTracker.findTracker(): T? { + if (this is T) + return this + val trackers = ArrayDeque() + trackers.addAll(internalTrackers()) + while (trackers.isNotEmpty()) { + val tracker = trackers.removeFirst() + if (tracker is T) + return tracker + trackers.addAll(tracker.internalTrackers()) + } + return null +} + +internal inline fun ChainedRunTracker.addTrackerIfAbsent(createTracker: () -> T): T { + val tracker = findTracker() + if (tracker != null) + return tracker + return createTracker().also { + addTracker(it) + } +} \ No newline at end of file diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckStatistics.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckStatistics.kt new file mode 100644 index 000000000..d2bbd3ddc --- /dev/null +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/LincheckStatistics.kt @@ -0,0 +1,212 @@ +/* + * Lincheck + * + * Copyright (C) 2019 - 2023 JetBrains s.r.o. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * + */ + +package org.jetbrains.kotlinx.lincheck + +import org.jetbrains.kotlinx.lincheck.execution.ExecutionScenario +import org.jetbrains.kotlinx.lincheck.strategy.LincheckFailure + +/** + * Interface representing the statistics collected during the execution of a Lincheck test. + */ +interface LincheckStatistics { + /** + * Total running time in nanoseconds, excluding warm-up time. + */ + val runningTimeNano: Long + + /** + * A mapping from iteration id to the iteration statistics. + */ + val iterationsStatistics: Map + + /** + * Granularity at which running time is captured, either: + * - [StatisticsGranularity.PER_ITERATION] --- per iteration only; + * - [StatisticsGranularity.PER_INVOCATION] --- per each invocation (consumes more memory). + */ + val granularity: StatisticsGranularity +} + +/** + * Represents the statistics of a single Lincheck test iteration. + */ +interface LincheckIterationStatistics { + /** + * Used scenario. + */ + val scenario: ExecutionScenario + + /** + * Running time of this iteration in nanoseconds, excluding warm-up time. + */ + val runningTimeNano: Long + + /** + * Warm-up time of this iteration in nanoseconds. + */ + val warmUpTimeNano: Long + + /** + * Number of invocations performed on this iteration, excluding warm-up invocations. + */ + val invocationsCount: Int + + /** + * Number of warm-up invocations performed on this iteration. + */ + val warmUpInvocationsCount: Int + + /** + * Running time of all invocations in this iteration. + * If per-iteration statistics tracking granularity is specified, + * then running time of invocations is not collected, and this array is left empty. + */ + val invocationsRunningTimeNano: LongArray +} + +/** + * Represents the different granularities at which Lincheck statistics data can be collected. + */ +enum class StatisticsGranularity { + PER_ITERATION, PER_INVOCATION +} + +/** + * Number of performed iterations. + */ +val LincheckStatistics.iterationsCount: Int + get() = iterationsStatistics.size + +/** + * Total running time, including warm-up time. + */ +val LincheckStatistics.totalRunningTimeNano: Long + get() = runningTimeNano + warmUpTimeNano + +/** + * Total warm-up time in nanoseconds, spent on all iterations. + */ +val LincheckStatistics.warmUpTimeNano: Long + get() = iterationsStatistics.values.sumOf { it.warmUpTimeNano } + +/** + * Number of invocations performed on all iterations. + */ +val LincheckStatistics.invocationsCount: Int + get() = iterationsStatistics.values.sumOf { it.invocationsCount } + +/** + * Total number of performed invocations, including warm-up invocations + */ +val LincheckStatistics.totalInvocationsCount: Int + get() = iterationsStatistics.values.sumOf { it.totalInvocationsCount } + +/** + * Average number of invocations performed by iteration. + */ +val LincheckStatistics.averageInvocationsCount: Double + get() = iterationsStatistics.values.map { it.invocationsCount }.average() + +/** + * The average invocation time (across all iterations) in nanoseconds. + */ +val LincheckStatistics.averageInvocationTimeNano + get() = runningTimeNano.toDouble() / invocationsCount + +/** + * Total running time of this iteration in nanoseconds, including warm-up time. + */ +val LincheckIterationStatistics.totalRunningTimeNano: Long + get() = runningTimeNano + warmUpTimeNano + +/** + * Total number of invocations performed on this iteration, including warm-up invocations. + */ +val LincheckIterationStatistics.totalInvocationsCount: Int + get() = invocationsCount + warmUpInvocationsCount + +/** + * Average invocation time on given iteration. + */ +val LincheckIterationStatistics.averageInvocationTimeNano + get() = runningTimeNano.toDouble() / invocationsCount + + +class LincheckStatisticsTracker( + override val granularity: StatisticsGranularity = StatisticsGranularity.PER_ITERATION +) : LincheckStatistics, LincheckRunTracker { + + override var runningTimeNano: Long = 0 + private set + + override val iterationsStatistics: Map + get() = _iterationsStatistics + private val _iterationsStatistics = mutableMapOf() + + private class IterationStatisticsTracker( + override val scenario: ExecutionScenario, + val parameters: IterationParameters, + granularity: StatisticsGranularity, + ) : LincheckIterationStatistics { + override var runningTimeNano: Long = 0 + override var warmUpTimeNano: Long = 0 + override var invocationsCount: Int = 0 + override var warmUpInvocationsCount: Int = 0 + override val invocationsRunningTimeNano: LongArray = when (granularity) { + StatisticsGranularity.PER_ITERATION -> longArrayOf() + StatisticsGranularity.PER_INVOCATION -> LongArray(parameters.invocationsBound) + } + var lastInvocationStartTimeNano = -1L + } + + private val IterationStatisticsTracker.plannedWarmUpInvocationsCount: Int + get() = parameters.warmUpInvocationsCount + + override fun iterationStart(iteration: Int, scenario: ExecutionScenario, parameters: IterationParameters) { + check(iteration !in iterationsStatistics) + _iterationsStatistics[iteration] = IterationStatisticsTracker(scenario, parameters, granularity) + } + + override fun invocationStart(iteration: Int, invocation: Int) { + val statistics = _iterationsStatistics[iteration]!! + statistics.lastInvocationStartTimeNano = System.nanoTime() + } + + override fun invocationEnd(iteration: Int, invocation: Int, failure: LincheckFailure?, exception: Throwable?) { + val statistics = _iterationsStatistics[iteration]!! + val invocationTimeNano = System.nanoTime() - statistics.lastInvocationStartTimeNano + check(invocationTimeNano >= 0) + if (invocation < statistics.plannedWarmUpInvocationsCount) { + statistics.warmUpTimeNano += invocationTimeNano + statistics.warmUpInvocationsCount += 1 + } else { + statistics.runningTimeNano += invocationTimeNano + statistics.invocationsCount += 1 + runningTimeNano += invocationTimeNano + } + if (granularity == StatisticsGranularity.PER_INVOCATION) { + statistics.invocationsRunningTimeNano[invocation] = invocationTimeNano + } + statistics.lastInvocationStartTimeNano = -1L + } + +} + diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/Reporter.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/Reporter.kt index 8fac850d4..cf741e0f3 100644 --- a/src/jvm/main/org/jetbrains/kotlinx/lincheck/Reporter.kt +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/Reporter.kt @@ -11,10 +11,10 @@ package org.jetbrains.kotlinx.lincheck import org.jetbrains.kotlinx.lincheck.LoggingLevel.* -import sun.nio.ch.lincheck.TestThread import org.jetbrains.kotlinx.lincheck.execution.* import org.jetbrains.kotlinx.lincheck.strategy.* import org.jetbrains.kotlinx.lincheck.strategy.managed.* +import sun.nio.ch.lincheck.TestThread import java.io.* import kotlin.math.max @@ -23,10 +23,15 @@ class Reporter(private val logLevel: LoggingLevel) { private val outErr: PrintStream = System.err fun logIteration(iteration: Int, maxIterations: Int, scenario: ExecutionScenario) = log(INFO) { - appendLine("\n= Iteration $iteration / $maxIterations =") + appendLine("\n= Iteration ${iteration + 1} / $maxIterations =") appendExecutionScenario(scenario) } + fun logIterationStatistics(invocations: Int, runningTimeNano: Long) = log(INFO) { + val runningTime = nanoTimeToString(runningTimeNano) + appendLine("= Statistics: #invocations=$invocations, running time ${runningTime}s =") + } + fun logFailedIteration(failure: LincheckFailure, loggingLevel: LoggingLevel = INFO) = log(loggingLevel) { appendFailure(failure) } @@ -714,3 +719,6 @@ private fun StringBuilder.appendException(t: Throwable) { } private const val EXCEPTIONS_TRACES_TITLE = "Exception stack traces:" + +internal fun nanoTimeToString(timeNano: Long, decimalPlaces: Int = 3) = + String.format("%.${decimalPlaces}f", timeNano.toDouble() / 1_000_000_000) \ No newline at end of file diff --git a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/SequentialIntChannel.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/specifications/SequentialIntChannelSpecification.kt similarity index 85% rename from src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/SequentialIntChannel.kt rename to src/jvm/main/org/jetbrains/kotlinx/lincheck/specifications/SequentialIntChannelSpecification.kt index 543412cbd..00f3da271 100644 --- a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/SequentialIntChannel.kt +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/specifications/SequentialIntChannelSpecification.kt @@ -7,14 +7,18 @@ * Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed * with this file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -package org.jetbrains.kotlinx.lincheck_test.verifier.linearizability -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.* -import org.jetbrains.kotlinx.lincheck.verifier.* +package org.jetbrains.kotlinx.lincheck.specifications + +import kotlinx.coroutines.CancellableContinuation +import kotlinx.coroutines.InternalCoroutinesApi +import kotlinx.coroutines.channels.ClosedReceiveChannelException +import kotlinx.coroutines.channels.ClosedSendChannelException +import kotlinx.coroutines.suspendCancellableCoroutine +import org.jetbrains.kotlinx.lincheck.verifier.VerifierState @InternalCoroutinesApi -open class SequentialIntChannel(private val capacity: Int) : VerifierState() { +open class SequentialIntChannelSpecification(private val capacity: Int) : VerifierState() { private val senders = ArrayList, Int>>() private val receivers = ArrayList>() private val buffer = ArrayList() diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/Strategy.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/Strategy.kt index d388e70a5..55a78f593 100644 --- a/src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/Strategy.kt +++ b/src/jvm/main/org/jetbrains/kotlinx/lincheck/strategy/Strategy.kt @@ -9,6 +9,7 @@ */ package org.jetbrains.kotlinx.lincheck.strategy +import org.jetbrains.kotlinx.lincheck.* import org.jetbrains.kotlinx.lincheck.runner.* import org.jetbrains.kotlinx.lincheck.execution.ExecutionScenario import org.jetbrains.kotlinx.lincheck.strategy.managed.Trace @@ -100,12 +101,19 @@ abstract class Strategy protected constructor( * * @return the failure, if detected, null otherwise. */ -fun Strategy.runIteration(invocations: Int, verifier: Verifier): LincheckFailure? { - for (invocation in 0 until invocations) { +fun Strategy.runIteration( + iteration: Int, + params: IterationParameters, + verifier: Verifier, + tracker: LincheckRunTracker? = null +): LincheckFailure? = tracker.trackIteration(iteration, scenario, params) { + for (invocation in 0 until params.invocationsBound) { if (!nextInvocation()) return null - val result = runInvocation() - val failure = verify(result, verifier) + val failure = tracker.trackInvocation(iteration, invocation) { + val result = this.runInvocation() + verify(result, verifier) + } if (failure != null) return failure } diff --git a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/BufferedChannelTest.kt b/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/BufferedChannelTest.kt index ef14336c0..6d2b95e2c 100644 --- a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/BufferedChannelTest.kt +++ b/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/BufferedChannelTest.kt @@ -14,6 +14,7 @@ import kotlinx.coroutines.channels.* import org.jetbrains.kotlinx.lincheck.* import org.jetbrains.kotlinx.lincheck.annotations.* import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck.specifications.* import org.jetbrains.kotlinx.lincheck_test.* @InternalCoroutinesApi @@ -36,4 +37,4 @@ class BufferedChannelTest : AbstractLincheckTest() { } @InternalCoroutinesApi -class SequentialBuffered2IntChannel : SequentialIntChannel(capacity = 2) \ No newline at end of file +class SequentialBuffered2IntChannel : SequentialIntChannelSpecification(capacity = 2) \ No newline at end of file diff --git a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/RendezvousChannelTest.kt b/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/RendezvousChannelTest.kt index 3dd6872d3..be54d47d2 100644 --- a/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/RendezvousChannelTest.kt +++ b/src/jvm/test/org/jetbrains/kotlinx/lincheck_test/verifier/linearizability/RendezvousChannelTest.kt @@ -14,6 +14,7 @@ import kotlinx.coroutines.channels.* import org.jetbrains.kotlinx.lincheck.* import org.jetbrains.kotlinx.lincheck.annotations.* import org.jetbrains.kotlinx.lincheck.paramgen.IntGen +import org.jetbrains.kotlinx.lincheck.specifications.SequentialIntChannelSpecification import org.jetbrains.kotlinx.lincheck_test.* @ExperimentalCoroutinesApi @@ -35,10 +36,10 @@ class RendezvousChannelTest : AbstractLincheckTest() { fun close() = ch.close() override fun > O.customize() { - sequentialSpecification(SequentialRendezvousIntChannel::class.java) + sequentialSpecification(SequentialRendezvousIntChannelSpecification::class.java) iterations(10) } } @InternalCoroutinesApi -class SequentialRendezvousIntChannel : SequentialIntChannel(capacity = 0) \ No newline at end of file +class SequentialRendezvousIntChannelSpecification : SequentialIntChannelSpecification(capacity = 0) \ No newline at end of file