Add yk to ruby.

.buildbot.sh

@@ -1,9 +1,55 @@
 #!/bin/sh
 
-set -eu
+set -e
 
-# First build without the JIT and check the tests still work.
+export CARGO_HOME="`pwd`/.cargo"
+export RUSTUP_HOME="`pwd`/.rustup"
+export RUSTUP_INIT_SKIP_PATH_CHECK="yes"
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh
+sh rustup.sh --default-host x86_64-unknown-linux-gnu \
+    --default-toolchain nightly \
+    --no-modify-path \
+    --profile minimal \
+    -y
+export PATH=`pwd`/.cargo/bin/:$PATH
+
+git clone --depth 1 --recurse-submodules --shallow-submodules https://github.com/ykjit/yk
+cd yk
+echo "yk commit: $(git show -s --format=%H)"
+cat << EOF >> Cargo.toml
+[profile.release-with-asserts]
+inherits = "release"
+debug-assertions = true
+overflow-checks = true
+EOF
+
+cd ykllvm
+ykllvm_hash=$(git rev-parse HEAD)
+if [ -f /opt/ykllvm_cache/ykllvm-release-with-assertions-"${ykllvm_hash}".tgz ]; then
+    mkdir inst
+    cd inst
+    tar xfz /opt/ykllvm_cache/ykllvm-release-with-assertions-"${ykllvm_hash}".tgz
+    cd ..
+    # Minimally check that we can at least run `clang --version`: if we can't,
+    # we assume the cached binary is too old (e.g. linking against old shared
+    # objects) and that we should build our own version.
+    if inst/bin/clang --version > /dev/null; then
+        export YKB_YKLLVM_BIN_DIR="$(pwd)/inst/bin"
+    else
+        echo "Warning: cached ykllvm not runnable; building from scratch" > /dev/stderr
+        rm -rf inst
+    fi
+fi
+cd ..
+
+YKB_YKLLVM_BUILD_ARGS="define:CMAKE_C_COMPILER=/usr/bin/clang,define:CMAKE_CXX_COMPILER=/usr/bin/clang++" \
+    cargo build
+export PATH=`pwd`/bin:${PATH}
+cd ..
+
+# Now build ykruby
 ./autogen.sh
+rm -rf build
 mkdir build && cd build
-../configure
-make btest -j $(nproc)
+../ykconfigure.sh
+make ruby

.buildbot_dockerfile_debian

@@ -4,16 +4,16 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
     --mount=target=/var/cache/apt,type=cache,sharing=locked \
   rm -f /etc/apt/apt.conf.d/docker-clean && \
   apt-get update && \
-  apt-get -y install clang-15 autoconf make curl procps file git cmake python3 ruby-full \
+  apt-get -y install clang-17 autoconf make curl procps file git cmake python3 ruby-full \
     libtinfo-dev libzip-dev mold ninja-build gdb && \
-  update-alternatives --install /usr/bin/cc cc /usr/bin/clang-15 999 && \
-  update-alternatives --set cc /usr/bin/clang-15 && \
-  update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-15 999 && \
-  update-alternatives --set c++ /usr/bin/clang++-15 && \
+  update-alternatives --install /usr/bin/cc cc /usr/bin/clang-17 999 && \
+  update-alternatives --set cc /usr/bin/clang-17 && \
+  update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-17 999 && \
+  update-alternatives --set c++ /usr/bin/clang++-17 && \
   update-alternatives --install /usr/bin/ld ld /usr/bin/mold 999 && \
   update-alternatives --set ld /usr/bin/mold && \
-  ln -sf /usr/bin/clang-15 /usr/bin/clang && \
-  ln -sf /usr/bin/clang++-15 /usr/bin/clang++
+  ln -sf /usr/bin/clang-17 /usr/bin/clang && \
+  ln -sf /usr/bin/clang++-17 /usr/bin/clang++
 ARG CI_UID
 RUN useradd -m -u ${CI_UID} ci && chown ${CI_UID}:${CI_UID} .
 ARG CI_RUNNER

README.md

@@ -52,6 +52,19 @@ if you are a committer.
 
 See [Building Ruby](https://docs.ruby-lang.org/en/master/contributing/building_ruby_md.html)
 
+## Building YKRuby
+
+Create a new folder `build` and `cd` into it:
+
+```
+mkdir build && cd build
+```
+
+Run the `../ykconfigure.sh` which runs Ruby's `configure` script with some
+extra flags and patches the resulting `Makefile`.
+
+Finally, run `make miniruby` to compile `miniruby`.
+
 ## Ruby home page
 
 https://www.ruby-lang.org/

compile.c

@@ -2837,6 +2837,27 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
     body->iseq_encoded = (void *)generated_iseq;
     body->iseq_size = code_index;
     body->stack_max = stack_max;
+    body->yklocs = calloc(code_index, sizeof(YkLocation));
+    int pos = 0;
+    while (pos < code_index) {
+        int opcode = rb_vm_insn_addr2opcode((void *)body->iseq_encoded[pos]);
+        const VALUE instr = body->iseq_encoded[pos];
+        switch(instr) {
+            case BIN(branchif):
+              ; int dst = body->iseq_encoded[pos+1];
+              if (dst < 0) {
+                // Add a new location to backwards branches.
+                body->yklocs[pos] = yk_location_new();
+              } else {
+                body->yklocs[pos] = yk_location_null();
+              }
+              break;
+            default:
+              body->yklocs[pos] = yk_location_null();
+              break;
+        }
+        pos = pos + insn_len(opcode);
+    }
 
     if (ISEQ_COMPILE_DATA(iseq)->is_single_mark_bit) {
         body->mark_bits.single = ISEQ_COMPILE_DATA(iseq)->mark_bits.single;

gc/default/default.c

@@ -19,6 +19,7 @@
 
 #include "ruby/ruby.h"
 #include "ruby/atomic.h"
+#include <yk.h>
 #include "ruby/debug.h"
 #include "ruby/thread.h"
 #include "ruby/util.h"
@@ -2391,6 +2392,9 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size
     return obj;
 }
 
+// FIXME: Inlining this causes guard failures when the cache misses which we
+// can't codegen yet.
+__attribute__((yk_outline))
 static VALUE
 newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t heap_idx, bool vm_locked)
 {

include/ruby/internal/attr/pure.h

@@ -26,7 +26,8 @@
 
 /** Wraps (or simulates) `__attribute__((pure))` */
 #if RBIMPL_HAS_ATTRIBUTE(pure)
-# define RBIMPL_ATTR_PURE() __attribute__((__pure__))
+// FIXME: Compilation fails with pure functions enabled.
+# define RBIMPL_ATTR_PURE()
 #elif RBIMPL_COMPILER_SINCE(SunPro, 5, 10, 0)
 # define RBIMPL_ATTR_PURE() _Pragma("does_not_write_global_data")
 #else

main.c

@@ -20,6 +20,7 @@
 #undef RUBY_EXPORT
 #include "ruby.h"
 #include "vm_debug.h"
+#include <yk.h>
 #include "internal/sanitizers.h"
 #ifdef HAVE_LOCALE_H
 #include <locale.h>
@@ -57,6 +58,7 @@ main(int argc, char **argv)
 #ifdef HAVE_LOCALE_H
     setlocale(LC_CTYPE, "");
 #endif
+    yk_init();
 
     ruby_sysinit(&argc, &argv);
     return ruby_start_main(rb_main, argc, argv);

ruby.c

@@ -49,6 +49,7 @@
 #include "internal/file.h"
 #include "internal/inits.h"
 #include "internal/io.h"
+#include <yk.h>
 #include "internal/load.h"
 #include "internal/loadpath.h"
 #include "internal/missing.h"
@@ -1823,6 +1824,7 @@ ruby_opt_init(ruby_cmdline_options_t *opt)
     memset(ruby_vm_redefined_flag, 0, sizeof(ruby_vm_redefined_flag));
 
     ruby_init_prelude();
+    yk_init();
 
     if (rb_namespace_available())
         rb_initialize_main_namespace();

vm_core.h

@@ -402,11 +402,13 @@ enum rb_builtin_attr {
 
 typedef VALUE (*rb_jit_func_t)(struct rb_execution_context_struct *, struct rb_control_frame_struct *);
 
+#include <yk.h>
 struct rb_iseq_constant_body {
     enum rb_iseq_type type;
 
     unsigned int iseq_size;
     VALUE *iseq_encoded; /* encoded iseq (insn addr and operands) */
+    YkLocation *yklocs;
 
     /**
      * parameter information

vm_exec.c

@@ -10,6 +10,7 @@
 **********************************************************************/
 
 #include <math.h>
+#include <yk.h>
 
 #if USE_YJIT
 // The number of instructions executed on vm_exec_core. --yjit-stats uses this.
@@ -97,6 +98,7 @@ vm_exec_core(rb_execution_context_t *ec)
     reg_cfp = ec->cfp;
     reg_pc = reg_cfp->pc;
 
+  YkMT *mt;
   first:
     INSN_DISPATCH();
 /*****************/

vm_exec.h

@@ -135,7 +135,10 @@ case BIN(insn):
   break;
 
 #define INSN_DISPATCH()         \
+  mt = yk_mt_new(NULL); \
   while (1) {			\
+    int pc = (GET_PC() - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded); \
+    yk_mt_control_point(mt, &ISEQ_BODY(reg_cfp->iseq)->yklocs[pc]); \
     switch (GET_CURRENT_INSN()) {
 
 #define END_INSNS_DISPATCH()    \

vm_insnhelper.c

@@ -6591,6 +6591,9 @@ vm_opt_plus(VALUE recv, VALUE obj)
     }
 }
 
+// FIXME: Needs to be outlined until we've implemented
+// `llvm.ssub.with.overflow.i64`.
+__attribute__((yk_outline))
 static VALUE
 vm_opt_minus(VALUE recv, VALUE obj)
 {

vm_opts.h

@@ -34,7 +34,7 @@
  * 2: call (function call for each insn dispatch)
  */
 #ifndef OPT_THREADED_CODE
-#define OPT_THREADED_CODE 0
+#define OPT_THREADED_CODE -1
 #endif
 
 #define OPT_DIRECT_THREADED_CODE (OPT_THREADED_CODE == 0)

ykconfigure.sh

@@ -0,0 +1,15 @@
+# Run the configure scripts with some yk flags
+CC=`yk-config debug --cc` LDFLAGS=`yk-config debug --ldflags` ../configure --disable-yjit optflags="-O0 -fno-omit-frame-pointer"
+
+# Patch the Makefile with some extras we require
+
+# Add c and cpp flags
+EXTRAARCHFLAGS=`yk-config debug --cflags --cppflags`
+sed -i "s@ARCH_FLAG =@ARCH_FLAG = $EXTRAARCHFLAGS@" Makefile
+# Remove stack-protection, visibility, and pie flags.
+sed -i "s/-fstack-protector-strong//" Makefile
+sed -i "s/-fvisibility=hidden//" Makefile
+sed -i "s/-fPIE//" Makefile
+sed -i "s/-pie//" Makefile
+# Add the ykcapi library.
+sed -i '/^MAINLIBS =/ s/$/ -lykcapi/' Makefile