Skip to content

Commit 8f1e8d0

Browse files
donoghucdonoghuc
committed
Make deduplicate a separate rake task and prevent gradle errors
Deduplication should happen as a depenedency of installing default gems. In the current workflow we have a top level gradle task for packaging which calls out to rake. Rake then invokes a *separate* gradle process. When we modify the jruby default, when the separate gradle process goes to check of jruby is installed, it sees a modified jruby and tries to re-install. We work around this by changing how gradle detects if jruby is required to be installed.
1 parent e638bd2 commit 8f1e8d0

File tree

3 files changed

+63
-57
lines changed

3 files changed

+63
-57
lines changed

‎rakelib/artifacts.rake‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -82,58 +82,6 @@ namespace "artifact" do
8282
res
8383
end
8484

85-
def duplicated_gems_exclude_paths
86-
shared_gems_path = 'vendor/jruby/lib/ruby/gems/shared/gems'
87-
default_gemspecs_path = 'vendor/jruby/lib/ruby/gems/shared/specifications/default'
88-
bundle_gems_path = 'vendor/bundle/jruby/*/gems'
89-
90-
exclusions = []
91-
92-
# "bundled" gems in jruby
93-
# https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152
94-
shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path|
95-
match = File.basename(path).match(/^(.+?)-\d+/)
96-
match ? match[1] : nil
97-
end.compact
98-
99-
# "default" gems in jruby/ruby
100-
# https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106
101-
default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path|
102-
match = File.basename(path).match(/^(.+?)-\d+/)
103-
match ? match[1] : nil
104-
end.compact
105-
106-
# gems we explicitly manage with bundler (we always want these to take precedence)
107-
bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path|
108-
match = File.basename(path).match(/^(.+?)-\d+/)
109-
match ? match[1] : nil
110-
end.compact
111-
112-
shared_duplicates = shared_gem_names & bundle_gem_names
113-
default_duplicates = default_gem_names & bundle_gem_names
114-
all_duplicates = (shared_duplicates + default_duplicates).uniq
115-
puts "Adding duplicate gems to exclude path: #{all_duplicates.sort.join(', ')}"
116-
117-
# Exclude shared/bundled gems duplicates
118-
shared_duplicates.each do |gem_name|
119-
exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*/**/*"
120-
exclusions << "vendor/jruby/lib/ruby/gems/shared/gems/#{gem_name}-*"
121-
exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/#{gem_name}-*.gemspec"
122-
end
123-
124-
# Exclude default gems duplicates
125-
default_duplicates.each do |gem_name|
126-
# CODEREVIEW: removing the code itself causes issues with gem loading. Remove only the gemspecs
127-
# for duplicated gems. This should help the code scanning case.
128-
exclusions << "vendor/jruby/lib/ruby/gems/shared/specifications/default/#{gem_name}-*.gemspec"
129-
# exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}.rb"
130-
# exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}/**/*"
131-
# exclusions << "vendor/jruby/lib/ruby/stdlib/#{gem_name}"
132-
end
133-
134-
exclusions
135-
end
136-
13785
def default_exclude_paths
13886
return @exclude_paths if @exclude_paths
13987

@@ -153,9 +101,6 @@ namespace "artifact" do
153101
@exclude_paths << 'vendor/**/gems/**/Gemfile.lock'
154102
@exclude_paths << 'vendor/**/gems/**/Gemfile'
155103

156-
@exclude_paths.concat(duplicated_gems_exclude_paths)
157-
puts "Full exclude_paths list:"
158-
@exclude_paths.each { |path| puts " - #{path}" }
159104
@exclude_paths.freeze
160105
end
161106

‎rakelib/plugin.rake‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,65 @@ namespace "plugin" do
9090
task.reenable # Allow this task to be run again
9191
end # task "install"
9292

93+
94+
task "clean-duplicate-gems" do
95+
shared_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME,
96+
'vendor/jruby/lib/ruby/gems/shared/gems')
97+
default_gemspecs_path = File.join(LogStash::Environment::LOGSTASH_HOME,
98+
'vendor/jruby/lib/ruby/gems/shared/specifications/default')
99+
bundle_gems_path = File.join(LogStash::Environment::BUNDLE_DIR,
100+
'jruby/*/gems')
101+
102+
# "bundled" gems in jruby
103+
# https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L108-L152
104+
shared_gem_names = Dir.glob(File.join(shared_gems_path, '*')).map do |path|
105+
match = File.basename(path).match(/^(.+?)-\d+/)
106+
match ? match[1] : nil
107+
end.compact
108+
109+
# "default" gems in jruby/ruby
110+
# https://github.com/jruby/jruby/blob/024123c29d73b672d50730117494f3e4336a0edb/lib/pom.rb#L21-L106
111+
default_gem_names = Dir.glob(File.join(default_gemspecs_path, '*.gemspec')).map do |path|
112+
match = File.basename(path).match(/^(.+?)-\d+/)
113+
match ? match[1] : nil
114+
end.compact
115+
116+
# gems we explicitly manage with bundler (we always want these to take precedence)
117+
bundle_gem_names = Dir.glob(File.join(bundle_gems_path, '*')).map do |path|
118+
match = File.basename(path).match(/^(.+?)-\d+/)
119+
match ? match[1] : nil
120+
end.compact
121+
122+
shared_duplicates = shared_gem_names & bundle_gem_names
123+
default_duplicates = default_gem_names & bundle_gem_names
124+
all_duplicates = (shared_duplicates + default_duplicates).uniq
125+
126+
puts("[plugin:clean-duplicate-gems] Removing duplicate gems: #{all_duplicates.sort.join(', ')}")
127+
128+
# Remove shared/bundled gem duplicates
129+
shared_duplicates.each do |gem_name|
130+
FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/#{gem_name}-*"))
131+
FileUtils.rm_rf(Dir.glob("#{shared_gems_path}/../specifications/#{gem_name}-*.gemspec"))
132+
end
133+
134+
# Remove default gem gemspecs only
135+
default_duplicates.each do |gem_name|
136+
# For stdlib default gems we only remove the gemspecs as removing the source code
137+
# files results in code loading errors and ruby warnings
138+
FileUtils.rm_rf(Dir.glob("#{default_gemspecs_path}/#{gem_name}-*.gemspec"))
139+
end
140+
141+
task.reenable
142+
end
143+
93144
task "install-default" => "bootstrap" do
94145
puts("[plugin:install-default] Installing default plugins")
95146

96147
remove_lockfile # because we want to use the release lockfile
97148
install_plugins("--no-verify", "--preserve", *LogStash::RakeLib::DEFAULT_PLUGINS)
98149

150+
# Clean duplicates after full gem resolution
151+
Rake::Task["plugin:clean-duplicate-gems"].invoke
99152
task.reenable # Allow this task to be run again
100153
end
101154

‎rubyUtils.gradle‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,11 @@ tasks.register("installCustomJRuby", Copy) {
279279
dependsOn buildCustomJRuby
280280
description = "Install custom built JRuby in the vendor directory"
281281
inputs.file(customJRubyTar)
282-
outputs.dir("${projectDir}/vendor/jruby")
282+
// Don't re-extract if core JRuby is already installed. This works around
283+
// gem deduplication when rake calls back in to gradle.
284+
onlyIf {
285+
!file("${projectDir}/vendor/jruby/bin/jruby").exists()
286+
}
283287
from tarTree(customJRubyTar == "" ? jrubyTarPath : customJRubyTar)
284288
eachFile { f ->
285289
f.path = f.path.replaceFirst("^jruby-${customJRubyVersion}", '')
@@ -294,7 +298,11 @@ tasks.register("downloadAndInstallJRuby", Copy) {
294298
dependsOn=[verifyFile, installCustomJRuby]
295299
description = "Install JRuby in the vendor directory"
296300
inputs.file(jrubyTarPath)
297-
outputs.dir("${projectDir}/vendor/jruby")
301+
// Don't re-extract if core JRuby is already installed. This works around
302+
// gem deduplication when rake calls back in to gradle.
303+
onlyIf {
304+
!file("${projectDir}/vendor/jruby/bin/jruby").exists()
305+
}
298306
from tarTree(downloadJRuby.dest)
299307
eachFile { f ->
300308
f.path = f.path.replaceFirst("^jruby-${jRubyVersion}", '')

0 commit comments

Comments
 (0)