[GR-54907] Backport changes in TruffleRuby master branch to the 24.1 release branch. Part 2

PullRequest: truffleruby/4336

Author: andrykonchin

CHANGELOG.md

@@ -17,6 +17,7 @@ Bug fixes:
 * Repair `IO#{wait_readable,wait_writable,wait}` to be interruptible (#3504, @andrykonchin).
 * Fix Hash value omission for constant names (@andrykonchin).
 * Fix `MatchData#[index, length]` when index is larger than number of matched values (@andrykonchin).
+* Fix `#each` for a foreign iterator which is also iterable (#3630, @eregon).
 
 Compatibility:
 
@@ -33,11 +34,23 @@ Compatibility:
 * Fix `Enumerable#reduce` to handle non-Symbol method name parameter (#2931, @andrykonchin).
 * Fix `RangeError` message to match CRuby for `Integer#chr` called with invalid codepoint argument (#2795, @andrykonchin).
 * Joni has been updated from 2.1.44 to 2.2.1 (@andrykonchin).
+* Fix `Hash#to_h` called with a block and pass key and value to the block as separate arguments (#3607, @andrykonchin).
+* Fix `StringIO#initialize` and preserve initial string's encoding when mode is `w` so the initial string is truncated (#3599, @andrykonchin).
+* Fix `IO#{autoclose=,autoclose?}` and raise `IOError` when io is closed (@andrykonchin).
+* Fix `Thread#{thread_variable_get,thread_variable_set,thread_variable?,key?,[],[]=,fetch}` and convert a non-String/Symbol thread-local variable name to String using `#to_str` (@andrykonchin).
+* Fix formatting in `Exception#full_message` when `RuntimeError` is not handled and `highlight` option is specified (@andrykonchin).
+* Fix `String#encode` and convert fallback values into String using `#to_str` (@andrykonchin).
+* Fix `Kernel.warn` and don't call `Warning#warn` if a specified category is disabled (@andrykonchin).
+* Fix `$!` global variable and make it fiber-local (@andrykonchin).
+* Fix `rb_set_errinfo` and `rb_errinfo` and store an error separately from `$!` (#2890, @andrykonchin).
+* Fix `rb_mutex_synchronize` to not wrap/unwrap result value (#3624, @andrykonchin).
+* Add `StringIO#set_encoding_by_bom` method (#3632, @andrykonchin).
 
 Performance:
 
 * Fix inline caching for Regexp creation from Strings (#3492, @andrykonchin, @eregon).
 * Optimize `Integer#pow` method for small modulus values (#3544, @andrykonchin).
+* Avoid repeated copies from native to managed string when matching Regexps on a native string (#2193, @eregon).
 
 Changes:
 

ci.jsonnet

@@ -633,8 +633,8 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-benchmarks-asciidoctor-svm-ce": shared + asciidoctor + svm_configurations["svm-ce"],
       "ruby-benchmarks-asciidoctor-svm-ee": shared + asciidoctor + svm_configurations["svm-ee"],
 
-      local warmup = $.benchmark.runner + $.benchmark.warmup + { timelimit: "01:05:00" },
-      "ruby-benchmarks-warmup-mri": shared + warmup + other_rubies.mri + { timelimit: "01:20:00" },
+      local warmup = $.benchmark.runner + $.benchmark.warmup + { timelimit: "01:15:00" },
+      "ruby-benchmarks-warmup-mri": shared + warmup + other_rubies.mri + { timelimit: "01:30:00" },
       "ruby-benchmarks-warmup-jruby": shared + warmup + other_rubies.jruby,
       "ruby-benchmarks-warmup-jvm-ce": shared + warmup + graal_configurations["jvm-ce"] + $.use.no_multi_tier,
       "ruby-benchmarks-warmup-jvm-ce-3threads": shared + warmup + graal_configurations["jvm-ce"] + $.use.no_multi_tier + $.use.three_threads,

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.25.5",
+  "mx_version": "7.27.5",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {
@@ -45,13 +45,13 @@
     "labsjdk-ee-21-llvm": {"name": "labsjdk",   "version": "ee-21.0.2+13-jvmci-23.1-b33-sulong", "platformspecific": true },
     "graalvm-ee-21":      {"name": "graalvm-java21", "version": "23.1.3", "platformspecific": true },
 
-    "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "23",      "build_id": "jdk-23+26", "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-23+26-jvmci-b01", "platformspecific": true },
-    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-23+26-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-23+26-jvmci-b01-sulong", "platformspecific": true },
-    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-23+26-jvmci-b01", "platformspecific": true },
-    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-23+26-jvmci-b01-debug", "platformspecific": true },
-    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-23+26-jvmci-b01-sulong", "platformspecific": true }
+    "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "23",      "build_id": "jdk-23+32", "platformspecific": true, "extrabundles": ["static-libs"]},
+    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-23+32-jvmci-b01", "platformspecific": true },
+    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-23+32-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-23+32-jvmci-b01-sulong", "platformspecific": true },
+    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-23+32-jvmci-b01", "platformspecific": true },
+    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-23+32-jvmci-b01-debug", "platformspecific": true },
+    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-23+32-jvmci-b01-sulong", "platformspecific": true }
   },
 
   "eclipse": {

doc/contributor/cexts.md

@@ -46,11 +46,10 @@ natively.
 
 ### Compilation
 
-We compile C extensions using the standard `mkmf` tool, and `clang` compilers
-which have flags set to generate bitcode alongside machine code.
+We compile C extensions using the standard `mkmf` tool, and `gcc`/`clang` compilers.
 
 We pipe C source code through a pre-processor `lib/cext/preprocess.rb` before it
-goes to `clang` to workaround some limitations.
+goes to `gcc`/`clang` to work around some limitations.
 
 ### API functions
 

doc/user/known-cves.md

@@ -19,6 +19,7 @@ Cross-reference with the details on [the MRI website](https://www.ruby-lang.org/
 
 Number | Description | Their Mitigation | Test | Our Mitigation
 --- | --- | --- | --- | ---
+CVE-2024-39908 | DoS in REXML | [Fix](https://github.com/ruby/rexml/compare/v3.3.1...v3.3.2) | [Test](https://github.com/ruby/rexml/compare/v3.3.1...v3.3.2) | [same mitigation as CRuby](https://www.ruby-lang.org/en/news/2024/07/16/dos-rexml-cve-2024-39908/), users must update rexml to 3.3.2+ if they use it
 CVE-2024-35176 | DoS in REXML | [Fix](https://github.com/ruby/rexml/pull/126/commits/ba70cfefadf7e79885fe08228d1df4c0549ad0f8) | [Test](https://github.com/ruby/rexml/pull/126/commits/ba70cfefadf7e79885fe08228d1df4c0549ad0f8) | [same mitigation as CRuby](https://www.ruby-lang.org/en/news/2024/05/16/dos-rexml-cve-2024-35176/), users must update rexml to 3.2.7+ if they use it
 CVE-2024-27282 | Arbitrary memory address read vulnerability with Regex search | [Fix](https://github.com/ruby/ruby/commit/989a2355808a63fc45367785c82ffd46d18c900a) | | Not applicable thanks to Java semantics
 CVE-2024-27281 | RCE vulnerability with .rdoc_options in RDoc | [Fix](https://github.com/ruby/rdoc/compare/v6.3.3...v6.3.4.1) | | Not applicable because TruffleRuby does not generate rdoc on gem install

lib/cext/ABI_check.txt

@@ -1 +1 @@
-1
+3

lib/truffle/stringio.rb

@@ -166,6 +166,14 @@ def initialize(string = nil, mode = nil, **options)
       mode_from_string(string.frozen? ? 'r' : 'r+')
     end
 
+    if @writable && @__data__.string.frozen?
+      raise Errno::EACCES, 'Permission denied'
+    end
+
+    if @truncate
+      @__data__.string.replace(''.force_encoding(@__data__.string.encoding))
+    end
+
     self
   end
 
@@ -203,6 +211,22 @@ def set_encoding(external, internal = nil, options = nil)
     self
   end
 
+  def set_encoding_by_bom
+    Primitive.check_frozen self
+    return nil unless @readable
+
+    encoding = Truffle::IOOperations.strip_bom(self)
+    return nil unless encoding
+
+    d = @__data__
+    TruffleRuby.synchronized(d) do
+      d.encoding = encoding
+      d.string.force_encoding(encoding) if @writable
+    end
+
+    encoding
+  end
+
   def external_encoding
     @__data__.encoding
   end
@@ -672,43 +696,37 @@ def yaml_initialize(type, val)
   end
 
   private def mode_from_string(mode)
-    @append = truncate = false
+    @append = @truncate = false
 
     if mode[0] == ?r
       @readable = true
       @writable = mode[-1] == ?+ ? true : false
     end
 
     if mode[0] == ?w
-      @writable = truncate = true
+      @writable = @truncate = true
       @readable = mode[-1] == ?+ ? true : false
     end
 
     if mode[0] == ?a
       @append = @writable = true
       @readable = mode[-1] == ?+ ? true : false
     end
-
-    d = @__data__ # no sync, only called from initialize
-    raise Errno::EACCES, 'Permission denied' if @writable && d.string.frozen?
-    d.string.replace('') if truncate
   end
 
   private def mode_from_integer(mode)
-    @readable = @writable = @append = false
-    d = @__data__ # no sync, only called from initialize
+    @readable = @writable = @append = @truncate = false
 
     if mode == 0 or mode & IO::RDWR != 0
       @readable = true
     end
 
     if mode & (IO::WRONLY | IO::RDWR) != 0
-      raise Errno::EACCES, 'Permission denied' if d.string.frozen?
       @writable = true
     end
 
     @append = true if (mode & IO::APPEND) != 0
-    d.string.replace('') if (mode & IO::TRUNC) != 0
+    @truncate = true if (mode & IO::TRUNC) != 0
   end
 
   private def getline(arg_error, sep, limit, chomp = false)

lib/truffle/truffle/cext.rb

@@ -190,7 +190,15 @@ def self.init_libtrufflerubytrampoline(libtrampoline)
 
     init_functions = libtrampoline[:rb_tr_trampoline_init_functions]
     init_functions = Primitive.interop_eval_nfi('(env,(string):pointer):void').bind(init_functions)
-    init_functions.call(-> name { LIBTRUFFLERUBY[name] })
+    if Truffle::Boot.get_option 'cexts-panama' and Primitive.vm_java_version >= 22 and !TruffleRuby.native?
+      init_functions.call(-> name {
+        closure = LIBTRUFFLERUBY[name].createNativeClosure('panama')
+        keep_alive << closure
+        closure
+      })
+    else
+      init_functions.call(-> name { LIBTRUFFLERUBY[name] })
+    end
 
     init_constants = libtrampoline[:rb_tr_trampoline_init_global_constants]
     init_constants = Primitive.interop_eval_nfi('((string):pointer):void').bind(init_constants)
@@ -713,7 +721,7 @@ def rb_reg_compile(pattern, options)
                nil
              end
     if err
-      Primitive.thread_set_exception(err)
+      Primitive.fiber_set_error_info(err)
       nil
     else
       result
@@ -1231,7 +1239,7 @@ def rb_protect(function, arg, write_status, status)
     unless Primitive.nil?(e)
       store_exception(e)
       pos = extract_tag(e)
-      Primitive.thread_set_exception(extract_ruby_exception(e))
+      Primitive.fiber_set_error_info(extract_ruby_exception(e))
     end
 
     Truffle::Interop.execute_without_conversion(write_status, status, pos)
@@ -1244,7 +1252,7 @@ def rb_jump_tag(pos)
       e = retrieve_exception
       tag = extract_tag(e)
       raise RuntimeError, 'mismatch between jump tag and captured exception' unless pos == tag
-      Primitive.thread_set_exception(nil)
+      Primitive.fiber_set_error_info(nil)
       raise_exception(e)
     end
   end
@@ -1299,7 +1307,7 @@ def rb_exc_raise(exception)
 
   def rb_set_errinfo(error)
     if Primitive.nil?(error) || Primitive.is_a?(error, Exception)
-      Primitive.thread_set_exception(error)
+      Primitive.fiber_set_error_info(error)
     else
       raise TypeError, 'assigning non-exception to ?!'
     end
@@ -1310,7 +1318,7 @@ def rb_make_exception(args)
   end
 
   def rb_errinfo
-    $!
+    Primitive.fiber_get_error_info
   end
 
   def rb_arity_error_string(arg_count, min, max)
@@ -1486,7 +1494,7 @@ def rb_get_alloc_func(ruby_class)
     begin
       allocate_method = ruby_class.method(:__allocate__).owner
     rescue NameError
-      nil
+      nil # it's fine to call this on a class that doesn't have an allocator
     else
       Primitive.object_hidden_var_get(allocate_method, ALLOCATOR_FUNC)
     end
@@ -1611,7 +1619,7 @@ def rb_mutex_sleep(mutex, timeout)
 
   def rb_mutex_synchronize(mutex, func, arg)
     mutex.synchronize do
-      Primitive.cext_unwrap(Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [func, arg]))
+      Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [func, arg])
     end
   end
   Truffle::Graal.always_split instance_method(:rb_mutex_synchronize)
@@ -1784,6 +1792,14 @@ def rb_f_notimplement
     raise NotImplementedError, "#{function}() function is unimplemented on this machine"
   end
 
+  def rb_bug(message)
+    raise Exception, "rb_bug: #{message}"
+  end
+
+  def rb_fatal(message)
+    raise Exception, "rb_fatal: #{message}"
+  end
+
   def test_kwargs(kwargs, raise_error)
     return false if Primitive.nil?(kwargs)
 
@@ -1821,19 +1837,31 @@ def rb_ensure(b_proc, data1, e_proc, data2)
     begin
       Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [b_proc, data1])
     ensure
-      Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [e_proc, data2])
+      errinfo = Primitive.fiber_get_error_info
+      Primitive.fiber_set_error_info($!)
+      begin
+        Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [e_proc, data2])
+      ensure
+        Primitive.fiber_set_error_info(errinfo)
+      end
     end
   end
   Truffle::Graal.always_split instance_method(:rb_ensure)
 
   def rb_rescue(b_proc, data1, r_proc, data2)
     begin
       Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [b_proc, data1])
-    rescue StandardError => e
+    rescue StandardError => exc
       if Truffle::Interop.null?(r_proc)
         Primitive.cext_wrap(nil)
       else
-        Primitive.interop_execute(POINTER2_TO_POINTER_WRAPPER, [r_proc, data2, Primitive.cext_wrap(e)])
+        errinfo = Primitive.fiber_get_error_info
+        Primitive.fiber_set_error_info(exc)
+        begin
+          Primitive.interop_execute(POINTER2_TO_POINTER_WRAPPER, [r_proc, data2, Primitive.cext_wrap(exc)])
+        ensure
+          Primitive.fiber_set_error_info(errinfo)
+        end
       end
     end
   end
@@ -1842,8 +1870,14 @@ def rb_rescue(b_proc, data1, r_proc, data2)
   def rb_rescue2(b_proc, data1, r_proc, data2, rescued)
     begin
       Primitive.interop_execute(POINTER_TO_POINTER_WRAPPER, [b_proc, data1])
-    rescue *rescued => e
-      Primitive.interop_execute(POINTER2_TO_POINTER_WRAPPER, [r_proc, data2, Primitive.cext_wrap(e)])
+    rescue *rescued => exc
+      errinfo = Primitive.fiber_get_error_info
+      Primitive.fiber_set_error_info(exc)
+      begin
+        Primitive.interop_execute(POINTER2_TO_POINTER_WRAPPER, [r_proc, data2, Primitive.cext_wrap(exc)])
+      ensure
+        Primitive.fiber_set_error_info(errinfo)
+      end
     end
   end
   Truffle::Graal.always_split instance_method(:rb_rescue2)

lib/truffle/truffle/cext_preprocessor.rb

@@ -11,7 +11,6 @@
 # rubocop:disable TruffleRuby/*
 
 require_relative 'patches/json_patches'
-require_relative 'patches/nokogiri_patches'
 require_relative 'patches/oci8_patches'
 require_relative 'patches/pg_patches'
 require_relative 'patches/tk_patches'
@@ -42,7 +41,6 @@ def self.add_gem_patches(patch_hash, gem_patches)
       end
 
       add_gem_patches(PATCHED_FILES, ::JsonPatches::PATCHES)
-      add_gem_patches(PATCHED_FILES, ::NokogiriPatches::PATCHES)
       add_gem_patches(PATCHED_FILES, ::OCI8Patches::PATCHES)
       add_gem_patches(PATCHED_FILES, ::PgPatches::PATCHES)
       add_gem_patches(PATCHED_FILES, ::TkPatches::PATCHES)

lib/truffle/truffle/cext_ruby.rb

@@ -33,13 +33,9 @@ def rb_define_method(mod, name, function, argc)
         args = [function, Primitive.cext_wrap(self), *args.map! { |arg| Primitive.cext_wrap(arg) }]
       end
 
-      exc = $!
-      Primitive.thread_set_exception(nil)
       # We must set block argument if given here so that the
       # `rb_block_*` functions will be able to find it by walking the stack.
-      res = Primitive.call_with_c_mutex_and_frame_and_unwrap(wrapper, args, Primitive.caller_special_variables_if_available, block)
-      Primitive.thread_set_exception(exc)
-      res
+      Primitive.call_with_c_mutex_and_frame_and_unwrap(wrapper, args, Primitive.caller_special_variables_if_available, block)
     end
 
     # Even if the argc is -2, the arity number