Skip to content

Commit 2d4ab23

Browse files
hal-eisen-adfahal-eisen-adfa
authored
ADFA-6381: Build one Kotlin library module per jar (#2108)
* ADFA-6381: Build one Kotlin library module per jar collectKtModules held the boot classpath as a lazy Sequence that called addLibrary, so every source module re-ran it and got a fresh KtLibraryModule for android.jar from each Android module: modules x Android-modules copies. Each copy materializes the jar's full file list for its search scope. GlitchTip breadcrumbs show the same android.jar scope built 55-71 times on large multi-module projects, behind 219 of 336 OOM events. addLibrary now reuses the module for a path, and the boot classpath is a distinct List, so each jar maps to exactly one module. * style: spotless reformat, no functional change * style: spotless reformat, no functional change * ADFA-6381: Add each module dependency once Now that addLibrary returns one shared KtLibraryModule per jar, a jar on both the boot classpath and a module's compile classpath was added to that module's dependencies twice as the same instance. Both module builders now keep dependencies in a LinkedHashSet: O(1) dedupe, order preserved. Tests: compile-classpath jars shared across modules resolve to one instance; a jar on both classpaths is one dependency (fails without the builder change: size 2).
1 parent d2af43b commit 2d4ab23

4 files changed

Lines changed: 254 additions & 118 deletions

File tree

‎lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/WorkspaceExts.kt‎

Lines changed: 67 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -19,75 +19,82 @@ private val logger = LoggerFactory.getLogger("WorkspaceExts")
1919

2020
internal fun Workspace.collectKtModules(
2121
project: Project,
22-
appEnv: CoreApplicationEnvironment
23-
): List<KtModule> = buildList {
24-
fun addModule(module: KtModule) = add(module)
22+
appEnv: CoreApplicationEnvironment,
23+
): List<KtModule> =
24+
buildList {
25+
fun addModule(module: KtModule) = add(module)
2526

26-
val moduleProjects = subProjects
27-
.asSequence()
28-
.filterIsInstance<ModuleProject>()
29-
.filter { it.path != rootProject.path }
30-
31-
val jarToModMap = mutableMapOf<Path, KtLibraryModule>()
32-
33-
fun addLibrary(path: Path): KtLibraryModule {
34-
val module = buildKtLibraryModule(project, appEnv) {
35-
id = path.pathString
36-
addContentRoot(path)
37-
}
38-
jarToModMap[path] = module
39-
return module
40-
}
41-
42-
val bootClassPaths = moduleProjects
43-
.filterIsInstance<AndroidModule>()
44-
.flatMap { project ->
45-
project.bootClassPaths
27+
val moduleProjects =
28+
subProjects
4629
.asSequence()
30+
.filterIsInstance<ModuleProject>()
31+
.filter { it.path != rootProject.path }
32+
33+
val jarToModMap = mutableMapOf<Path, KtLibraryModule>()
34+
35+
// One module per jar: every module shares android.jar, and each KtLibraryModule
36+
// materializes the jar's full file list for its search scope (ADFA-6381).
37+
fun addLibrary(path: Path): KtLibraryModule =
38+
jarToModMap.getOrPut(path) {
39+
buildKtLibraryModule(project, appEnv) {
40+
id = path.pathString
41+
addContentRoot(path)
42+
}
43+
}
44+
45+
// A List, not a Sequence: a Sequence would re-run addLibrary for every source module below.
46+
val bootClassPaths =
47+
moduleProjects
48+
.filterIsInstance<AndroidModule>()
49+
.flatMap { it.bootClassPaths.asSequence() }
4750
.filter { it.exists() }
4851
.map { it.toPath() }
52+
.distinct()
4953
.map(::addLibrary)
50-
}
54+
.toList()
5155

52-
val libraryDependencies = moduleProjects
53-
.flatMap { it.getCompileClasspaths() }
54-
.filter { it.exists() }
55-
.map { it.toPath() }
56-
.associateWith(::addLibrary)
57-
58-
val subprojectsAsModules = mutableMapOf<ModuleProject, KtSourceModule>()
59-
val sourceRootToModuleMap = mutableMapOf<Path, KtSourceModule>()
60-
61-
fun getOrCreateModule(moduleProject: ModuleProject): KtSourceModule {
62-
subprojectsAsModules[moduleProject]?.let { return it }
63-
64-
val module = buildKtSourceModule(project) {
65-
this.module = moduleProject
66-
67-
bootClassPaths.forEach { addDependency(it) }
68-
69-
moduleProject.getCompileClasspaths(excludeSourceGeneratedClassPath = true)
70-
.forEach { classpath ->
71-
val libDep = libraryDependencies[classpath.toPath()]
72-
if (libDep == null) {
73-
logger.error(
74-
"Skipping non-existent classpath classpath: {}",
75-
classpath
76-
)
77-
return@forEach
56+
val libraryDependencies =
57+
moduleProjects
58+
.flatMap { it.getCompileClasspaths() }
59+
.filter { it.exists() }
60+
.map { it.toPath() }
61+
.associateWith(::addLibrary)
62+
63+
val subprojectsAsModules = mutableMapOf<ModuleProject, KtSourceModule>()
64+
val sourceRootToModuleMap = mutableMapOf<Path, KtSourceModule>()
65+
66+
fun getOrCreateModule(moduleProject: ModuleProject): KtSourceModule {
67+
subprojectsAsModules[moduleProject]?.let { return it }
68+
69+
val module =
70+
buildKtSourceModule(project) {
71+
this.module = moduleProject
72+
73+
bootClassPaths.forEach { addDependency(it) }
74+
75+
moduleProject
76+
.getCompileClasspaths(excludeSourceGeneratedClassPath = true)
77+
.forEach { classpath ->
78+
val libDep = libraryDependencies[classpath.toPath()]
79+
if (libDep == null) {
80+
logger.error(
81+
"Skipping non-existent classpath classpath: {}",
82+
classpath,
83+
)
84+
return@forEach
85+
}
86+
addDependency(libDep)
87+
}
88+
89+
moduleProject.getCompileModuleProjects().forEach { dep ->
90+
addDependency(getOrCreateModule(dep))
7891
}
79-
addDependency(libDep)
8092
}
8193

82-
moduleProject.getCompileModuleProjects().forEach { dep ->
83-
addDependency(getOrCreateModule(dep))
84-
}
94+
subprojectsAsModules[moduleProject] = module
95+
module.contentRoots.forEach { root -> sourceRootToModuleMap[root] = module }
96+
return module
8597
}
8698

87-
subprojectsAsModules[moduleProject] = module
88-
module.contentRoots.forEach { root -> sourceRootToModuleMap[root] = module }
89-
return module
99+
moduleProjects.forEach { addModule(getOrCreateModule(it)) }
90100
}
91-
92-
moduleProjects.forEach { addModule(getOrCreateModule(it)) }
93-
}

‎lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/modules/KtLibraryModule.kt‎

Lines changed: 37 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -36,19 +36,20 @@ internal class KtLibraryModule(
3636
override val isSdk: Boolean = false,
3737
private val jvmTarget: JvmTarget = DEFAULT_JVM_TARGET,
3838
override val librarySources: KaLibrarySourceModule? = null,
39-
) : KaLibraryModule,
40-
AbstractKtModule(
39+
) : AbstractKtModule(
4140
project,
42-
dependencies
43-
) {
44-
41+
dependencies,
42+
),
43+
KaLibraryModule {
4544
class Builder(
4645
private val project: Project,
4746
private val applicationEnvironment: CoreApplicationEnvironment,
4847
) {
4948
lateinit var id: String
5049
private val contentRoots = mutableSetOf<Path>()
51-
private val dependencies = mutableListOf<KtModule>()
50+
51+
// A set, like KtSourceModule.Builder: no module is a dependency twice.
52+
private val dependencies = linkedSetOf<KtModule>()
5253
var isSdk: Boolean = false
5354
var jvmTarget: JvmTarget = DEFAULT_JVM_TARGET
5455
var librarySources: KaLibrarySourceModule? = null
@@ -61,31 +62,37 @@ internal class KtLibraryModule(
6162
dependencies.add(dep)
6263
}
6364

64-
fun build(): KtLibraryModule = KtLibraryModule(
65-
project = project,
66-
id = id,
67-
contentRoots = contentRoots.toSet(),
68-
dependencies = dependencies.toList(),
69-
applicationEnvironment = applicationEnvironment,
70-
isSdk = isSdk,
71-
jvmTarget = jvmTarget,
72-
librarySources = librarySources,
73-
)
65+
fun build(): KtLibraryModule =
66+
KtLibraryModule(
67+
project = project,
68+
id = id,
69+
contentRoots = contentRoots.toSet(),
70+
dependencies = dependencies.toList(),
71+
applicationEnvironment = applicationEnvironment,
72+
isSdk = isSdk,
73+
jvmTarget = jvmTarget,
74+
librarySources = librarySources,
75+
)
7476
}
7577

7678
@OptIn(KaImplementationDetail::class)
7779
override fun computeFiles(extended: Boolean): Sequence<VirtualFile> {
78-
val roots = if (isSdk) project.read {
79-
LibraryUtils.findClassesFromJdkHome(
80-
contentRoots.first(),
81-
isJre = false
82-
)
83-
}
84-
else contentRoots
80+
val roots =
81+
if (isSdk) {
82+
project.read {
83+
LibraryUtils.findClassesFromJdkHome(
84+
contentRoots.first(),
85+
isJre = false,
86+
)
87+
}
88+
} else {
89+
contentRoots
90+
}
8591

86-
val notExtendedFiles = roots
87-
.asSequence()
88-
.mapNotNull { getVirtualFileForLibraryRoot(it, applicationEnvironment, project) }
92+
val notExtendedFiles =
93+
roots
94+
.asSequence()
95+
.mapNotNull { getVirtualFileForLibraryRoot(it, applicationEnvironment, project) }
8996

9097
if (!extended) return notExtendedFiles
9198

@@ -97,17 +104,11 @@ internal class KtLibraryModule(
97104
override val baseContentScope: GlobalSearchScope by lazy {
98105
val virtualFileUrls = computeFiles(extended = true).map { it.url }.toSet()
99106
object : GlobalSearchScope(project) {
100-
override fun contains(vf: VirtualFile): Boolean {
101-
return vf.url in virtualFileUrls
102-
}
107+
override fun contains(vf: VirtualFile): Boolean = vf.url in virtualFileUrls
103108

104-
override fun isSearchInModuleContent(module: Module): Boolean {
105-
return false
106-
}
109+
override fun isSearchInModuleContent(module: Module): Boolean = false
107110

108-
override fun isSearchInLibraries(): Boolean {
109-
return true
110-
}
111+
override fun isSearchInLibraries(): Boolean = true
111112
}
112113
}
113114

@@ -136,6 +137,7 @@ internal fun buildKtLibraryModule(
136137
): KtLibraryModule = KtLibraryModule.Builder(project, applicationEnvironment).apply(init).build()
137138

138139
private const val JAR_SEPARATOR = "!/"
140+
139141
private fun getVirtualFileForLibraryRoot(
140142
root: Path,
141143
environment: CoreApplicationEnvironment,

‎lsp/kotlin/src/main/java/com/itsaky/androidide/lsp/kotlin/compiler/modules/KtSourceModule.kt‎

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,19 @@ internal class KtSourceModule(
2121
project: Project,
2222
val module: ModuleProject,
2323
directRegularDependencies: List<KtModule>,
24-
) : KaSourceModule, AbstractSourceModule(project, directRegularDependencies) {
25-
24+
) : AbstractSourceModule(project, directRegularDependencies),
25+
KaSourceModule {
2626
companion object {
2727
private val logger = LoggerFactory.getLogger(KtSourceModule::class.java)
2828
}
2929

30-
class Builder(private val project: Project) {
30+
class Builder(
31+
private val project: Project,
32+
) {
3133
lateinit var module: ModuleProject
32-
private val dependencies = mutableListOf<KtModule>()
34+
35+
// A set: a jar on both the boot and compile classpath is one shared module (ADFA-6381).
36+
private val dependencies = linkedSetOf<KtModule>()
3337

3438
fun addDependency(dep: KtModule) {
3539
dependencies.add(dep)
@@ -42,29 +46,38 @@ internal class KtSourceModule(
4246
get() = module.path
4347

4448
override val contentRoots by lazy {
45-
module.getSourceDirectories()
49+
module
50+
.getSourceDirectories()
4651
.asSequence()
4752
.map { it.toPath() }
4853
.toSet()
4954
}
5055

5156
private val versions by lazy {
52-
val kotlinCompilerSettings = when {
53-
module.hasJavaProject() -> module.javaProject
54-
.kotlinCompilerSettings
55-
56-
module.hasAndroidProject() -> module.androidProject
57-
.kotlinCompilerSettings
58-
59-
else -> null
60-
}
57+
val kotlinCompilerSettings =
58+
when {
59+
module.hasJavaProject() -> {
60+
module.javaProject
61+
.kotlinCompilerSettings
62+
}
63+
64+
module.hasAndroidProject() -> {
65+
module.androidProject
66+
.kotlinCompilerSettings
67+
}
68+
69+
else -> {
70+
null
71+
}
72+
}
6173

6274
if (kotlinCompilerSettings == null) {
6375
return@lazy DEFAULT_LANGUAGE_VERSION to DEFAULT_JVM_TARGET
6476
}
6577

66-
val apiVersion = LanguageVersion.fromVersionString(kotlinCompilerSettings.apiVersion)
67-
?: LanguageVersion.fromFullVersionString(kotlinCompilerSettings.apiVersion)
78+
val apiVersion =
79+
LanguageVersion.fromVersionString(kotlinCompilerSettings.apiVersion)
80+
?: LanguageVersion.fromFullVersionString(kotlinCompilerSettings.apiVersion)
6881

6982
val jvmTarget = JvmTarget.fromString(kotlinCompilerSettings.jvmTarget)
7083

@@ -79,18 +92,17 @@ internal class KtSourceModule(
7992
get() = super<AbstractSourceModule>.moduleDescription
8093

8194
override val languageVersionSettings: LanguageVersionSettings
82-
get() = LanguageVersionSettingsImpl(
83-
languageVersion = versions.first,
84-
apiVersion = ApiVersion.createByLanguageVersion(versions.first),
85-
)
95+
get() =
96+
LanguageVersionSettingsImpl(
97+
languageVersion = versions.first,
98+
apiVersion = ApiVersion.createByLanguageVersion(versions.first),
99+
)
86100

87101
override val targetPlatform: TargetPlatform
88102
get() = JvmPlatforms.jvmPlatformByTargetVersion(versions.second)
89-
90-
91103
}
92104

93105
internal fun buildKtSourceModule(
94106
project: Project,
95107
init: KtSourceModule.Builder.() -> Unit,
96-
): KtSourceModule = KtSourceModule.Builder(project).apply(init).build()
108+
): KtSourceModule = KtSourceModule.Builder(project).apply(init).build()

0 commit comments

Comments
 (0)