JBR-7835 macOS: raised NOFILE ulimit breaks interaction with gdb

(cherry picked from commit 41fbb57)
(cherry picked from commit c648556)

src/hotspot/os/bsd/os_bsd.cpp

@@ -1987,25 +1987,11 @@ jint os::init_2(void) {
     if (status != 0) {
       log_info(os)("os::init_2 getrlimit failed: %s", os::strerror(errno));
     } else {
-      rlim_t rlim_original = nbr_files.rlim_cur;
-
-      // On macOS according to setrlimit(2), OPEN_MAX must be used instead
-      // of RLIM_INFINITY, but testing on macOS >= 10.6, reveals that
-      // we can, in fact, use even RLIM_INFINITY, so try the max value
-      // that the system claims can be used first, same as other BSD OSes.
-      // However, some terminals (ksh) will internally use "int" type
-      // to store this value and since RLIM_INFINITY overflows an "int"
-      // we might end up with a negative value, so cap the system limit max
-      // at INT_MAX instead, just in case, for everyone.
-      nbr_files.rlim_cur = MIN(INT_MAX, nbr_files.rlim_max);
-
+      // Limit the number of open files because gdb would
+      // stat() the entire range of [0..rlim_max] and therefore effectively hang
+      // if that limit is sufficiently large like INT_MAX.
+      nbr_files.rlim_cur = MIN(OPEN_MAX, nbr_files.rlim_max);
       status = setrlimit(RLIMIT_NOFILE, &nbr_files);
-      if (status != 0) {
-        // If that fails then try lowering the limit to either OPEN_MAX
-        // (which is safe) or the original limit, whichever was greater.
-        nbr_files.rlim_cur = MAX(OPEN_MAX, rlim_original);
-        status = setrlimit(RLIMIT_NOFILE, &nbr_files);
-      }
       if (status != 0) {
         log_info(os)("os::init_2 setrlimit failed: %s", os::strerror(errno));
       }

test/jdk/jb/hotspot/OpenFilesLimit.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2024 JetBrains s.r.o.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+/**
+ * @test
+ * @summary Verifies that the "soft" limit for the number of open files
+ *          does not exceed 10240, an acceptable number that was
+ *          the default on macOS for a long time.
+ * @library /test/lib
+ * @requires os.family == "mac"
+ * @run main OpenFilesLimit
+ */
+
+public class OpenFilesLimit {
+    private static final String ULIMIT_GET_CMD = "ulimit -n";
+    private static final int MACOS_DEFAULT_LIMIT = 10240;
+
+    public static void main(String[] args) throws Exception {
+        OutputAnalyzer oa = ProcessTools.executeCommand("sh", "-c", ULIMIT_GET_CMD)
+                .shouldHaveExitValue(0);
+        String s = oa.getStdout().strip();
+        System.out.println("ulimit returned :'" + s + "'");
+        int i = Integer.parseInt(s);
+        System.out.println("ulimit -n is " + i);
+        System.out.println("Expecting " + MACOS_DEFAULT_LIMIT + " or less");
+        if (i > MACOS_DEFAULT_LIMIT) {
+            throw new RuntimeException("ulimit -n returned " + i + " that is higher than expected " + MACOS_DEFAULT_LIMIT);
+        }
+    }
+}