Use Vector API in the Java Extension

Rakefile

@@ -15,7 +15,8 @@ end rescue nil
 JAVA_DIR            = "java/src/json/ext"
 JAVA_RAGEL_PATH     = "#{JAVA_DIR}/ParserConfig.rl"
 JAVA_PARSER_SRC     = "#{JAVA_DIR}/ParserConfig.java"
-JAVA_SOURCES        = FileList["#{JAVA_DIR}/*.java"]
+JAVA_SOURCES        = FileList["#{JAVA_DIR}/*.java"].exclude("#{JAVA_DIR}/Vectorized*.java")
+JAVA_VEC_SOURCES    = FileList["#{JAVA_DIR}/Vectorized*.java"]
 JAVA_CLASSES        = []
 JRUBY_PARSER_JAR    = File.expand_path("lib/json/ext/parser.jar")
 JRUBY_GENERATOR_JAR = File.expand_path("lib/json/ext/generator.jar")
@@ -65,11 +66,26 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby'
 
   JRUBY_JAR = File.join(CONFIG["libdir"], "jruby.jar")
   if File.exist?(JRUBY_JAR)
+    classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator
     JAVA_SOURCES.each do |src|
-      classpath = (Dir['java/lib/*.jar'] << 'java/src' << JRUBY_JAR) * path_separator
       obj = src.sub(/\.java\Z/, '.class')
       file obj => src do
-        sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src
+        sh 'javac', '-classpath', classpath, '-source', '1.8', '-target', '1.8', src  
+        # '--enable-preview',
+      end
+      JAVA_CLASSES << obj
+    end
+
+    JAVA_VEC_SOURCES.each do |src|
+      obj = src.sub(/\.java\Z/, '.class')
+      file obj => src do
+        sh 'javac', '--add-modules', 'jdk.incubator.vector', '-classpath', classpath, '--release', '16', src do |success, status| 
+          if success
+            puts "*** 'jdk.incubator.vector' support enabled ***"
+          else
+            puts "*** 'jdk.incubator.vector' support disabled ***"
+          end
+        end
       end
       JAVA_CLASSES << obj
     end
@@ -118,11 +134,15 @@ if defined?(RUBY_ENGINE) and RUBY_ENGINE == 'jruby'
       generator_classes = FileList[
         "json/ext/ByteList*.class",
         "json/ext/OptionsReader*.class",
+        "json/ext/EscapeScanner*.class",
         "json/ext/Generator*.class",
         "json/ext/RuntimeInfo*.class",
         "json/ext/StringEncoder*.class",
-        "json/ext/Utils*.class"
+        "json/ext/Utils*.class",
+        "json/ext/VectorizedEscapeScanner*.class",
+        "json/ext/VectorizedStringEncoder*.class"
       ]
+      puts "Creating generator jar with classes: #{generator_classes.join(', ')}"
       sh 'jar', 'cf', File.basename(JRUBY_GENERATOR_JAR), *generator_classes
       mv File.basename(JRUBY_GENERATOR_JAR), File.dirname(JRUBY_GENERATOR_JAR)
     end

java/src/json/ext/EscapeScanner.java

@@ -0,0 +1,106 @@
+package json.ext;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+interface EscapeScanner {
+    static class State {
+        byte[] ptrBytes;
+        int ptr;
+        int len;
+        int pos;
+        int beg;
+        int ch;
+    }
+
+    static class VectorSupport {
+        private static String VECTORIZED_ESCAPE_SCANNER_CLASS = "json.ext.VectorizedEscapeScanner";
+        private static String VECTORIZED_SCANNER_PROP = "json.enableVectorizedEscapeScanner";
+        private static String VECTORIZED_SCANNER_DEFAULT = "false";
+        static final EscapeScanner VECTORIZED_ESCAPE_SCANNER;
+
+        static {
+            EscapeScanner scanner = null;
+            String enableVectorizedScanner = System.getProperty(VECTORIZED_SCANNER_PROP, VECTORIZED_SCANNER_DEFAULT);
+            if ("true".equalsIgnoreCase(enableVectorizedScanner) || "1".equalsIgnoreCase(enableVectorizedScanner)) {
+                try {
+                    Class<?> vectorEscapeScannerClass = EscapeScanner.class.getClassLoader().loadClass(VECTORIZED_ESCAPE_SCANNER_CLASS);
+                    Constructor<?> vectorizedEscapeScannerConstructor = vectorEscapeScannerClass.getDeclaredConstructor();
+                    scanner = (EscapeScanner) vectorizedEscapeScannerConstructor.newInstance();
+                } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
+                    // Fallback to the ScalarEscapeScanner if we cannot load the VectorizedEscapeScanner.
+                    System.err.println("Failed to load VectorizedEscapeScanner, falling back to ScalarEscapeScanner:");
+                    e.printStackTrace();
+                    scanner = null;
+                }
+            } else {
+                System.err.println("VectorizedEscapeScanner disabled.");
+            }
+            VECTORIZED_ESCAPE_SCANNER = scanner;
+        }
+    }
+
+    default boolean meetsMinimumLengthTreshold(int length) {
+        return true;
+    }
+
+    boolean scan(EscapeScanner.State state) throws java.io.IOException;
+
+    default State createState(byte[] ptrBytes, int ptr, int len, int beg) {
+        State state = new State();
+        state.ptrBytes = ptrBytes;
+        state.ptr = ptr;
+        state.len = len;
+        state.beg = beg;
+        state.pos = 0; // Start scanning from the beginning of the segment
+        return state;
+    }
+
+    public static EscapeScanner basicScanner() {
+        if (VectorSupport.VECTORIZED_ESCAPE_SCANNER != null) {
+            return VectorSupport.VECTORIZED_ESCAPE_SCANNER;
+        }
+
+        return new ScalarEscapeScanner(StringEncoder.ESCAPE_TABLE);
+    }
+
+    public static EscapeScanner create(byte[] escapeTable) {
+        return new ScalarEscapeScanner(escapeTable);
+    }
+
+    public static class BasicScanner implements EscapeScanner {
+        @Override
+        public boolean scan(EscapeScanner.State state) throws java.io.IOException {
+            while (state.pos < state.len) {
+                state.ch = Byte.toUnsignedInt(state.ptrBytes[state.ptr + state.pos]);
+                if (state.ch >= 0 && (state.ch < ' ' || state.ch == '\"' || state.ch == '\\')) {
+                    return true;
+                }
+                state.pos++;
+            }
+            return false;
+        }
+    }
+
+    public static class ScalarEscapeScanner implements EscapeScanner {
+        private final byte[] escapeTable;
+
+        public ScalarEscapeScanner(byte[] escapeTable) {
+            this.escapeTable = escapeTable;
+        }
+
+        @Override
+        public boolean scan(EscapeScanner.State state) throws java.io.IOException {
+            while (state.pos < state.len) {
+                state.ch = Byte.toUnsignedInt(state.ptrBytes[state.ptr + state.pos]);
+                int ch_len = escapeTable[state.ch];
+                if (ch_len > 0) {
+                    return true;
+                }
+                state.pos++;
+            }
+            return false;
+        }
+
+    }
+}

java/src/json/ext/Generator.java

@@ -29,6 +29,8 @@
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.math.BigInteger;
 import java.util.Set;
 
@@ -38,6 +40,32 @@ public final class Generator {
 
     private static final int IO_BUFFER_SIZE = 8192;
 
+    private static String VECTORIZED_ESCAPE_SCANNER_CLASS = "json.ext.VectorizedStringEncoder";
+    private static String VECTORIZED_SCANNER_PROP = "json.enableVectorizedStringEncoder";
+    private static String VECTORIZED_SCANNER_DEFAULT = "false";
+    static final StringEncoder VECTORIZED_STRING_ENCODER;
+
+    static {
+        StringEncoder scanner = null;
+        String enableVectorizedScanner = System.getProperty(VECTORIZED_SCANNER_PROP, VECTORIZED_SCANNER_DEFAULT);
+        if ("true".equalsIgnoreCase(enableVectorizedScanner) || "1".equalsIgnoreCase(enableVectorizedScanner)) {
+            try {
+                Class<?> vectorEscapeScannerClass = StringEncoder.class.getClassLoader().loadClass(VECTORIZED_ESCAPE_SCANNER_CLASS);
+                Constructor<?> vectorizedEscapeScannerConstructor = vectorEscapeScannerClass.getDeclaredConstructor();
+                scanner = (StringEncoder) vectorizedEscapeScannerConstructor.newInstance();
+                System.out.println(scanner.getClass().getName() + " loaded successfully.");
+            } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) {
+                // Fallback to the StringEncoder if we cannot load the VectorizedStringEncoder.
+                System.err.println("Failed to load VectorizedStringEncoder, falling back to StringEncoder:");
+                e.printStackTrace();
+                scanner = null;
+            }
+        } else {
+            System.err.println("VectorizedStringEncoder disabled.");
+        }
+        VECTORIZED_STRING_ENCODER = scanner;
+    }
+
     private Generator() {
         throw new RuntimeException();
     }
@@ -230,9 +258,24 @@ public byte[] getCharBytes() {
         public StringEncoder getStringEncoder(ThreadContext context) {
             if (stringEncoder == null) {
                 GeneratorState state = getState(context);
-                stringEncoder = state.asciiOnly() ?
-                        new StringEncoderAsciiOnly(state.scriptSafe()) :
-                        new StringEncoder(state.scriptSafe());
+
+                if (state.asciiOnly()) {
+                    stringEncoder = new StringEncoderAsciiOnly(state.scriptSafe());
+                } else {
+                    if (state.scriptSafe()) {
+                        stringEncoder = StringEncoder.scriptSafeEncoder();
+                    } else {
+                        if (VECTORIZED_STRING_ENCODER != null) {
+                            stringEncoder = VECTORIZED_STRING_ENCODER;
+                        } else {
+                            // System.err.println("VectorizedStringEncoder not available, using basic StringEncoder.");
+                            stringEncoder = StringEncoder.basicEncoder();
+                        }
+                    }
+                }
+                // stringEncoder = state.asciiOnly() ?
+                //         new StringEncoderAsciiOnly(state.scriptSafe()) :
+                //         state.scriptSafe() ? StringEncoder.scriptSafeEncoder() : StringEncoder.basicEncoder();
             }
             return stringEncoder;
         }