From a85da91e3a612271275856d1321436edfbe10e0f Mon Sep 17 00:00:00 2001 From: Tomasz Godzik Date: Sun, 31 Aug 2025 12:50:28 +0200 Subject: [PATCH] improvement: Add a debugging endpoint --- .../inc/bloop/BloopZincCompiler.scala | 4 +- .../scala/bloop/bsp/BloopBspDefinitions.scala | 69 ++++++++ .../scala/bloop/bsp/BloopBspServices.scala | 148 +++++++++++++++++- .../test/scala/bloop/bsp/BspBaseSuite.scala | 12 ++ .../scala/bloop/bsp/BspMetalsClientSpec.scala | 75 +++++++++ .../test/scala/bloop/bsp/BspTestSpec.scala | 2 +- 6 files changed, 301 insertions(+), 9 deletions(-) diff --git a/backend/src/main/scala/sbt/internal/inc/bloop/BloopZincCompiler.scala b/backend/src/main/scala/sbt/internal/inc/bloop/BloopZincCompiler.scala index 16ed24903d..a615d616dc 100644 --- a/backend/src/main/scala/sbt/internal/inc/bloop/BloopZincCompiler.scala +++ b/backend/src/main/scala/sbt/internal/inc/bloop/BloopZincCompiler.scala @@ -24,7 +24,6 @@ import sbt.internal.inc.bloop.internal.BloopIncremental import sbt.internal.inc.bloop.internal.BloopLookup import sbt.internal.inc.bloop.internal.BloopStamps import sbt.util.InterfaceUtil -import xsbti.AnalysisCallback import xsbti.VirtualFile import xsbti.compile._ @@ -137,8 +136,7 @@ object BloopZincCompiler { val lookup = new BloopLookup(config, previousSetup, logger) val analysis = invalidateAnalysisFromSetup(config.currentSetup, previousSetup, setOfSources, prev, manager, logger) - // Scala needs the explicit type signature to infer the function type arguments - val compile: (Set[VirtualFile], DependencyChanges, AnalysisCallback, ClassFileManager) => Task[Unit] = compiler.compile(_, _, _, _, cancelPromise) + val compile = compiler.compile(_, _, _, _, cancelPromise) BloopIncremental .compile( setOfSources, diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala index e741c74236..efbb68ca59 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspDefinitions.scala @@ -1,5 +1,6 @@ package bloop.bsp +import ch.epfl.scala.bsp.BuildTargetIdentifier import ch.epfl.scala.bsp.Uri import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec @@ -41,4 +42,72 @@ object BloopBspDefinitions { StopClientCachingParams.codec, Endpoint.unitCodec ) + + // Incremental compilation debugging endpoint definitions + final case class DebugIncrementalCompilationParams( + targets: List[BuildTargetIdentifier] + ) + + object DebugIncrementalCompilationParams { + implicit val codec: JsonValueCodec[DebugIncrementalCompilationParams] = + JsonCodecMaker.makeWithRequiredCollectionFields + } + + final case class IncrementalCompilationDebugInfo( + target: BuildTargetIdentifier, + analysisInfo: Option[AnalysisDebugInfo], + allFileHashes: List[FileHashInfo], + lastCompilationInfo: String, + maybeFailedCompilation: String + ) + + object IncrementalCompilationDebugInfo { + implicit val codec: JsonValueCodec[IncrementalCompilationDebugInfo] = + JsonCodecMaker.makeWithRequiredCollectionFields + } + + final case class AnalysisDebugInfo( + lastModified: Long, + sourceFiles: Int, + classFiles: Int, + internalDependencies: Int, + externalDependencies: Int, + location: Uri, + excludedFiles: List[String] + ) + + object AnalysisDebugInfo { + implicit val codec: JsonValueCodec[AnalysisDebugInfo] = + JsonCodecMaker.makeWithRequiredCollectionFields + } + + final case class FileHashInfo( + uri: Uri, + currentHash: Int, + analysisHash: Option[Int], + lastModified: Long, + exists: Boolean + ) + + object FileHashInfo { + implicit val codec: JsonValueCodec[FileHashInfo] = + JsonCodecMaker.makeWithRequiredCollectionFields + } + + final case class DebugIncrementalCompilationResult( + debugInfos: List[IncrementalCompilationDebugInfo] + ) + + object DebugIncrementalCompilationResult { + implicit val codec: JsonValueCodec[DebugIncrementalCompilationResult] = + JsonCodecMaker.makeWithRequiredCollectionFields + } + + object debugIncrementalCompilation + extends Endpoint[DebugIncrementalCompilationParams, DebugIncrementalCompilationResult]( + "bloop/debugIncrementalCompilation" + )( + DebugIncrementalCompilationParams.codec, + JsonCodecMaker.makeWithRequiredCollectionFields + ) } diff --git a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala index a47b207b2e..8b8cb8ab16 100644 --- a/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala +++ b/frontend/src/main/scala/bloop/bsp/BloopBspServices.scala @@ -41,6 +41,9 @@ import bloop.data.ClientInfo import bloop.data.ClientInfo.BspClientInfo import bloop.data.JdkConfig import bloop.data.Platform +import bloop.data.Platform.Js +import bloop.data.Platform.Jvm +import bloop.data.Platform.Native import bloop.data.Project import bloop.data.WorkspaceSettings import bloop.engine.Aggregate @@ -58,6 +61,7 @@ import bloop.engine.tasks.toolchains.ScalaNativeToolchain import bloop.exec.Forker import bloop.internal.build.BuildInfo import bloop.io.AbsolutePath +import bloop.io.ByteHasher import bloop.io.Environment.lineSeparator import bloop.io.RelativePath import bloop.logging.BspServerLogger @@ -70,22 +74,21 @@ import bloop.reporter.ReporterInputs import bloop.task.Task import bloop.testing.LoggingEventHandler import bloop.testing.TestInternals +import bloop.util.JavaCompat._ import bloop.util.JavaRuntime +import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec import com.github.plokhotnyuk.jsoniter_scala.core._ +import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray import com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker -import com.github.plokhotnyuk.jsoniter_scala.core.JsonValueCodec import jsonrpc4s._ -import com.github.plokhotnyuk.jsoniter_scala.core.readFromArray import monix.execution.Cancelable import monix.execution.CancelablePromise import monix.execution.Scheduler import monix.execution.atomic.AtomicBoolean import monix.execution.atomic.AtomicInt import monix.reactive.subjects.BehaviorSubject -import bloop.data.Platform.Js -import bloop.data.Platform.Jvm -import bloop.data.Platform.Native +import sbt.internal.inc.PlainVirtualFileConverter final class BloopBspServices( callSiteState: State, @@ -150,6 +153,9 @@ final class BloopBspServices( .requestAsync(endpoints.BuildTarget.jvmTestEnvironment)(p => schedule(jvmTestEnvironment(p))) .requestAsync(endpoints.BuildTarget.jvmRunEnvironment)(p => schedule(jvmRunEnvironment(p))) .notificationAsync(BloopBspDefinitions.stopClientCaching)(p => stopClientCaching(p)) + .requestAsync(BloopBspDefinitions.debugIncrementalCompilation)(p => + schedule(debugIncrementalCompilation(p)) + ) // Internal state, initial value defaults to @volatile private var currentState: State = callSiteState @@ -424,6 +430,138 @@ final class BloopBspServices( Task.eval { originToCompileStores.remove(params.originId); () }.executeAsync } + def debugIncrementalCompilation( + params: BloopBspDefinitions.DebugIncrementalCompilationParams + ): BspEndpointResponse[BloopBspDefinitions.DebugIncrementalCompilationResult] = { + def debugInfo( + projects: Seq[ProjectMapping], + state: State + ): BspResult[BloopBspDefinitions.DebugIncrementalCompilationResult] = { + val debugInfos = projects.map { + case (target, project) => + collectDebugInfo(target, project, state) + } + Task.sequence(debugInfos).map { debugInfos => + (state, Right(BloopBspDefinitions.DebugIncrementalCompilationResult(debugInfos.toList))) + } + } + + ifInitialized(None) { (state: State, _: BspServerLogger) => + mapToProjects(params.targets, state) match { + case Left(error) => Task.now((state, Left(Response.invalidRequest(error)))) + case Right(mappings) => debugInfo(mappings, state) + } + } + } + + private def collectDebugInfo( + target: bsp.BuildTargetIdentifier, + project: Project, + state: State + ): Task[BloopBspDefinitions.IncrementalCompilationDebugInfo] = { + val allSources = bloop.io.SourceHasher + .findAndHashSourcesInProject( + project, + _ => Task.now(Nil), + 20, + Promise[Unit](), + ioScheduler, + state.logger + ) + .map(res => res.map(_.sortBy(_.source.id()))) + .executeOn(ioScheduler) + allSources.map { allSources => + import bloop.bsp.BloopBspDefinitions._ + import java.nio.file.Files + + val projectAnalysisFile = state.client + .getUniqueClassesDirFor(project, forceGeneration = false) + .resolve(s"../../${project.name}-analysis.bin") + + val converter = PlainVirtualFileConverter.converter + // Extract analysis info from successful compilation results + val analysisInfo = state.results.lastSuccessfulResult(project) match { + case Some(success) => + val maybeAnalysis = success.previous.analysis() + val analysis = maybeAnalysis.toOption match { + case Some(analysis: sbt.internal.inc.Analysis) => analysis + case _ => sbt.internal.inc.Analysis.empty + } + val compilationInfo = analysis + .readCompilations() + .getAllCompilations + .toList + .map { compilation => + s" ${compilation.getStartTime} -> ${compilation.getOutput()}" + } + .mkString("\n") + + val relations = analysis.relations + val lastModifiedA = + if (Files.exists(projectAnalysisFile.underlying)) + Files.getLastModifiedTime(projectAnalysisFile.underlying).toMillis() + else 0L + val changedSource = allSources match { + case Left(_) => Nil + case Right(value) => + value.filterNot { sourceHash => + success.sources.exists(_.source.id() == sourceHash.source.id()) + } + } + val hashes = success.sources.map { sourceHash => + val sourcePath = converter.toPath(sourceHash.source) + val exists = Files.exists(sourcePath) + val lastModified = if (exists) sourcePath.toFile.lastModified() else 0L + val currentHash = + if (exists) ByteHasher.hashFileContents(sourcePath.toFile) + else 0 + + FileHashInfo( + uri = bsp.Uri(sourcePath.toUri()), + currentHash = currentHash, + analysisHash = Some(sourceHash.hash), + lastModified = lastModified, + exists = exists + ) + } + val currentFailedResult = state.results.latestResult(project) match { + case _: Compiler.Result.Success => "" + case otherwise => otherwise.toString() + } + + val analysisInfo = + AnalysisDebugInfo( + lastModified = lastModifiedA, + sourceFiles = analysis.readStamps.getAllSourceStamps.size(), + classFiles = analysis.readStamps.getAllProductStamps.size(), + internalDependencies = relations.allProducts.size, + externalDependencies = relations.allLibraryDeps.size, + location = bsp.Uri(projectAnalysisFile.toBspUri), + excludedFiles = changedSource.map(_.source.toString()) + ) + Some( + IncrementalCompilationDebugInfo( + target = target, + analysisInfo = Some(analysisInfo), + allFileHashes = hashes.toList, + lastCompilationInfo = compilationInfo, + maybeFailedCompilation = currentFailedResult + ) + ) + case _ => None + } + analysisInfo.getOrElse( + IncrementalCompilationDebugInfo( + target = target, + analysisInfo = None, + allFileHashes = Nil, + lastCompilationInfo = "", + maybeFailedCompilation = "" + ) + ) + } + } + def linkProjects( userProjects: Seq[ProjectMapping], state: State, diff --git a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala index f271982c36..cee3dfd37e 100644 --- a/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala +++ b/frontend/src/test/scala/bloop/bsp/BspBaseSuite.scala @@ -518,6 +518,18 @@ abstract class BspBaseSuite extends BaseSuite with BspClientTest { awaitForTask(jvmEnvironmentTask) } + def debugIncrementalCompilation( + project: TestProject + ): (ManagedBspTestState, BloopBspDefinitions.DebugIncrementalCompilationResult) = { + val debugTask = runAfterTargets(project) { target => + rpcRequest( + BloopBspDefinitions.debugIncrementalCompilation, + BloopBspDefinitions.DebugIncrementalCompilationParams(List(target)) + ) + } + await(debugTask) + } + def jvmTestEnvironment( project: TestProject, originId: Option[String] diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 30740071b7..95cd1c9f1e 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -830,6 +830,81 @@ class BspMetalsClientSpec( } } + test("debugIncrementalCompilation-endpoint-succeeds") { + TestUtil.withinWorkspace { workspace => + val sources = List( + """/Foo.scala + |object Foo { + | def main(args: Array[String]): Unit = { + | println("Hello World") + | } + |} + """.stripMargin + ) + + val logger = new RecordingLogger(ansiCodesSupported = false) + val A = TestProject(workspace, "a", sources) + + loadBspState(workspace, List(A), logger) { state => + // First compile the project to generate incremental compilation data + val compiled = state.compile(A) + assertExitStatus(compiled, ExitStatus.Ok) + + val (_, debugInfo) = compiled.debugIncrementalCompilation(A) + + val debugInformation = debugInfo.debugInfos.head + // Verify we get debug information + assert(debugInformation.target == A.bspId) + assert(debugInformation.lastCompilationInfo.nonEmpty) + assert(debugInformation.analysisInfo.isDefined) + assert(debugInformation.analysisInfo.get.classFiles == 2) + assert(debugInformation.analysisInfo.get.sourceFiles == 1) + assert(debugInformation.allFileHashes.size == 1) + } + } + } + + test("debugIncrementalCompilation-after-failure") { + TestUtil.withinWorkspace { workspace => + val fooBefore = + """/Foo.scala + |object Foo { + | def main(args: Array[String]): Unit = { + | println("Hello World") + | } + |} + """.stripMargin + val fooAfter = + """/Foo.scala + |object Foo { + | def main(args: Array[String]): Unit = { + | val a: Int = String + | } + |} + """.stripMargin + + val logger = new RecordingLogger(ansiCodesSupported = false) + val A = TestProject(workspace, "a", List(fooBefore)) + + loadBspState(workspace, List(A), logger) { state => + // First compile the project to generate incremental compilation data + val compiled = state.compile(A) + assertExitStatus(compiled, ExitStatus.Ok) + assertIsFile(writeFile(A.srcFor("Foo.scala"), fooAfter)) + val compiled2 = state.compile(A) + assertExitStatus(compiled2, ExitStatus.CompilationError) + + val (_, debugInfo) = compiled.debugIncrementalCompilation(A) + + val debugInformation = debugInfo.debugInfos.head + // Verify we get debug information + assert(debugInformation.maybeFailedCompilation.startsWith("Failed(")) + assert(debugInformation.analysisInfo.isDefined) + assert(debugInformation.analysisInfo.get.excludedFiles.isEmpty) + } + } + } + private val dummyFooScalaSources = List( """/Foo.scala |class Foo diff --git a/frontend/src/test/scala/bloop/bsp/BspTestSpec.scala b/frontend/src/test/scala/bloop/bsp/BspTestSpec.scala index a6008bf711..11f838f770 100644 --- a/frontend/src/test/scala/bloop/bsp/BspTestSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspTestSpec.scala @@ -16,7 +16,6 @@ import bloop.util.TestUtil import com.github.plokhotnyuk.jsoniter_scala.core.writeToArray import com.github.plokhotnyuk.jsoniter_scala.macros.JsonCodecMaker import jsonrpc4s.RawJson -import scalaz.std.java.time object TcpBspTestSpec extends BspTestSpec(BspProtocol.Tcp) object LocalBspTestSpec extends BspTestSpec(BspProtocol.Local) @@ -394,4 +393,5 @@ class BspTestSpec(override val protocol: BspProtocol) extends BspBaseSuite { } } } + }