8342376: More reliable OOM handling in ExceptionDuringDumpAtObjectsInitPhase test

Reviewed-by: iklam, phh

Author: shipilev

test/hotspot/jtreg/runtime/cds/appcds/javaldr/GCDuringDumpTransformer.java

@@ -34,7 +34,10 @@ public class GCDuringDumpTransformer implements ClassFileTransformer {
     static boolean TEST_WITH_CLEANER = Boolean.getBoolean("test.with.cleaner");
     static boolean TEST_WITH_EXCEPTION = Boolean.getBoolean("test.with.exception");
     static boolean TEST_WITH_OOM = Boolean.getBoolean("test.with.oom");
+
+    static final int WASTE_SIZE = 1024;
     static List<byte[]> waste = new ArrayList();
+    static Object sink;
 
     static Cleaner cleaner;
     static Thread thread;
@@ -59,10 +62,13 @@ public byte[] transform(ClassLoader loader, String name, Class<?> classBeingRede
               return new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
             }
             if (TEST_WITH_OOM) {
-                // fill until OOM
+                // Fill until OOM and fail. This sets up heap for secondary OOM
+                // later on, which should be caught by CDS code. The size of waste
+                // array defines how much max free space would be left for later
+                // code to run with.
                 System.out.println("Fill objects until OOM");
-                for (;;) {
-                    waste.add(new byte[64*1024]);
+                while (true) {
+                    waste.add(new byte[WASTE_SIZE]);
                 }
             }
         }
@@ -104,8 +110,8 @@ public static void agentmain(String args, Instrumentation inst) throws Exception
     }
 
     public static void makeGarbage() {
-        for (int x=0; x<10; x++) {
-            Object[] a = new Object[10000];
+        for (int x = 0; x < 10; x++) {
+            sink = new byte[WASTE_SIZE];
         }
     }
 
@@ -115,7 +121,7 @@ static class MyCleaner implements Runnable {
         public void run() {
             // Allocate something. This will cause G1 to allocate an EDEN region.
             // See JDK-8245925
-            Object o = new Object();
+            sink = new Object();
             System.out.println("cleaning " + i);
         }
     }