PGO/LTO optimizations for riscv qemu platforms

docker/Dockerfile

@@ -44,6 +44,7 @@ RUN apt-get update && apt-get install -y \
     wget \
     jq \
     binaryen \
+    parallel \
     && rm -rf /var/lib/apt/lists/*
 
 RUN apt-get update && apt-get install -y \
@@ -88,20 +89,32 @@ WORKDIR /tmp
 
 RUN git clone https://github.com/riscv/riscv-gnu-toolchain riscv-gnu-toolchain && \
     cd riscv-gnu-toolchain && \
-    git checkout 311e70f6c55953a624ca670de7c85af38c2b23c2 && \
+    git checkout 98c8ec4cb520443f16cdd36280b65fba3a5c5f54 && \
     CFLAGS="-O2 -g0" CXXFLAGS="-O2 -g0" ./configure \
         --prefix=/opt/riscv-newlib \
         --with-arch=rv64ima \
         --disable-gdb \
         --with-cmodel=medany \
-        --enable-strip && \
+        --enable-strip \
+        --enable-llvm && \
     make -j$(nproc) && \
     find /opt/riscv-newlib -type f -executable -exec strip --strip-debug {} + 2>/dev/null || true && \
     find /opt/riscv-newlib -name "*.a" -exec strip --strip-debug {} + 2>/dev/null || true && \
     find /opt/riscv-newlib -name "*.so" -exec strip --strip-debug {} + 2>/dev/null || true && \
     cd / && \
     rm -rf /tmp/riscv-gnu-toolchain
 
+# Stage for installing pre-packaged RISC-V GNU LLVM Toolchain
+FROM base AS riscv-gnu-glibc-clang-installer
+
+WORKDIR /tmp
+
+RUN wget https://github.com/riscv-collab/riscv-gnu-toolchain/releases/download/2026.03.13/riscv64-glibc-ubuntu-24.04-llvm.tar.xz && \
+    cd /opt && \
+    tar xf /tmp/riscv64-glibc-ubuntu-24.04-llvm.tar.xz && \
+    mv /opt/riscv /opt/riscv-glibc-llvm && \
+    rm /tmp/riscv64-glibc-ubuntu-24.04-llvm.tar.xz
+
 # Stage for building AARCH64 GCC MUSL Toolchain
 FROM base AS aarch64-musl-builder
 
@@ -296,6 +309,7 @@ WORKDIR /opt
 # Copy all built toolchains and tools
 COPY --from=llvm-builder /opt/llvm-21 /opt/llvm-21
 COPY --from=riscv-gnu-builder /opt/riscv-newlib /opt/riscv-newlib
+COPY --from=riscv-gnu-glibc-clang-installer /opt/riscv-glibc-llvm /opt/riscv-glibc-llvm
 COPY --from=aarch64-musl-builder /opt/aarch64-linux-musl-gcc /opt/aarch64-linux-musl-gcc
 COPY --from=qemu-builder /opt/qemu-install/usr/local /usr/local
 COPY --from=qemu-builder /opt/qemu-install/libinsn.so /libinsn.so

platform/riscv-qemu-user/custom_imports.c

@@ -18,7 +18,7 @@ U32 testmodule__testfunc2(void* instance, U32 a, U32 b) {
     return a * b;
 }
 
-int testmodule__printk(void* instance, U32 val) {
+void testmodule__printk(void* instance, U32 val) {
     char buf[12];
     static const char hex[] = "0123456789abcdef";
 

platform/riscv-qemu-user/scripts/c2riscv-qemu-user.sh

@@ -40,6 +40,12 @@ else
     echo "OPT_LEVEL not set, using default: $OPT_LEVEL"
 fi
 
+if [ -n "$PGO_OPT" ]; then
+    echo "Enabling profile-guided optimization"
+else
+	echo "PGO_OPT not set, not using profile-guided optimization"
+fi
+
 # Change to project root
 cd "$PROJECT_ROOT"
 
@@ -73,21 +79,19 @@ echo "Compiling..."
 
 # Compiler flags (using -O0 for faster compilation of large generated files)
 CFLAGS=(
-    --target=riscv64
-    -march=rv64ima_zicsr
-    -mabi=lp64
     -mcmodel=medany
-    -static
     -include stdbool.h
-    --sysroot=/opt/riscv-newlib/riscv64-unknown-elf
-    --gcc-toolchain=/opt/riscv-newlib
-    $OPT_LEVEL
+    -mllvm -enable-misched=false
 )
 
 LDFLAGS=(
     -fuse-ld=lld
+    -static
 )
 
+# Link libraries
+LIBS=(-lm)
+
 # Include directories
 INCLUDES=(
     -I"$GUEST_DIR"
@@ -101,14 +105,77 @@ SOURCES=(
     "$GUEST_DIR/guest.c"
     $GUEST_DIR/s0*.c
     w2c2/embedded/wasi.c
+    w2c2/embedded/wasip2.c
+)
+
+if [ -n "$PGO_OPT" ]; then
+    echo ""
+    echo "Generate instrumented binary: $OUTPUT.instrumented"
+    echo ""
+
+    # Remove object files
+    rm -f $PROJECT_ROOT/custom_imports.o $PROJECT_ROOT/guest.o $PROJECT_ROOT/main.o $PROJECT_ROOT/memlib.o $PROJECT_ROOT/memops.o $PROJECT_ROOT/s00000*.o $PROJECT_ROOT/startup.o $PROJECT_ROOT/wasi.o $PROJECT_ROOT/wasip2.o $PROJECT_ROOT/zkvm.o
+
+    parallel -j$(nproc) /opt/riscv-glibc-llvm/bin/clang \
+        -c \
+        -fprofile-instr-generate=$OUTPUT.profraw \
+         --target=riscv64-unknown-linux-gnu \
+        "${CFLAGS[@]}" \
+        "${INCLUDES[@]}" \
+        ::: \
+        "${SOURCES[@]}" 2>&1
+
+    /opt/riscv-glibc-llvm/bin/clang \
+        --target=riscv64-unknown-linux-gnu \
+        -fprofile-instr-generate=$OUTPUT.profraw \
+        *.o \
+        "${LDFLAGS[@]}" \
+        "${LIBS[@]}" \
+        -o "$OUTPUT.instrumented" 2>&1
+
+    echo ""
+    echo "Run instrumented binary: $OUTPUT.instrumented"
+    echo ""
+
+    qemu-riscv64 "$OUTPUT.instrumented"
+
+    /opt/riscv-glibc-llvm/bin/llvm-profdata merge -output=$OUTPUT.profdata $OUTPUT.profraw
+fi
+
+echo ""
+echo "Compile binary..."
+echo ""
+
+# Remove object files
+rm -f $PROJECT_ROOT/custom_imports.o $PROJECT_ROOT/guest.o $PROJECT_ROOT/main.o $PROJECT_ROOT/memlib.o $PROJECT_ROOT/memops.o $PROJECT_ROOT/s00000*.o $PROJECT_ROOT/startup.o $PROJECT_ROOT/wasi.o $PROJECT_ROOT/wasip2.o $PROJECT_ROOT/zkvm.o
+
+COMMON_FLAGS=(
+    $OPT_LEVEL
 )
 
-clang \
+if [ -n "$PGO_OPT" ]; then
+    COMMON_FLAGS+=(
+        -fprofile-instr-use=$OUTPUT.profdata
+    )
+fi
+
+parallel -j$(nproc) /opt/riscv-glibc-llvm/bin/clang \
+    -c \
+     --target=riscv64-unknown-linux-gnu \
+    "${COMMON_FLAGS[@]}" \
     "${CFLAGS[@]}" \
     "${LDFLAGS[@]}" \
     "${INCLUDES[@]}" \
-    "${SOURCES[@]}" \
-    -o "$OUTPUT" -lm 2>&1
+    ::: \
+    "${SOURCES[@]}" 2>&1
+
+/opt/riscv-glibc-llvm/bin/clang \
+    --target=riscv64-unknown-linux-gnu \
+    *.o \
+    "${COMMON_FLAGS[@]}" \
+    "${LDFLAGS[@]}" \
+    "${LIBS[@]}" \
+    -o "$OUTPUT" 2>&1
 
 # Check if compilation succeeded
 if [ $? -eq 0 ] && [ -f "$OUTPUT" ]; then

platform/riscv-qemu/custom_imports.c

@@ -35,6 +35,7 @@ int testmodule__printk(void* instance, U32 val) {
 }
 
 void testmodule__shutdown(void* instance) {
+    puts("shutdown()\n");
     exit(0);
 }
 
@@ -45,3 +46,4 @@ U32 testmodule__inputX2DdataX2Dlen(void* instance) {
 U32 testmodule__inputX2Ddata(void* instance, U32 index) {
     return 0;
 }
+

platform/riscv-qemu/main.c

@@ -53,6 +53,10 @@ size_t strlen(const char *s) {
     return strlen_max;
 }
 
+void *fopen(const char *filename, const char *mode) {
+    puts("fopen\n");
+}
+
 /* Trap handler for WASM runtime errors */
 void trap(Trap trap) {
     puts("trap(..)\n");

platform/riscv-qemu/scripts/c2riscv-qemu.sh

@@ -32,6 +32,7 @@ fi
 
 GUEST_DIR="$1"
 OUTPUT="$2"
+OUTPUT_USER_INSTR="$OUTPUT-user-instr"
 
 if [ -n "$OPT_LEVEL" ]; then
     OPT_LEVEL="$OPT_LEVEL"
@@ -42,6 +43,12 @@ else
     echo "OPT_LEVEL not set, using default: $OPT_LEVEL"
 fi
 
+if [ -n "$PGO_OPT" ]; then
+    echo "Enabling profile-guided optimization"
+else
+	echo "PGO_OPT not set, not using profile-guided optimization"
+fi
+
 # Change to project root
 cd "$PROJECT_ROOT"
 
@@ -60,6 +67,81 @@ if [ ! -f "$GUEST_DIR/guest.c" ] || [ ! -f "$GUEST_DIR/guest.h" ]; then
     exit 1
 fi
 
+if [ -n "$PGO_OPT" ]; then
+    echo "=========================================================="
+    echo "C to RISC-V QEMU User Compilation for instrumentation"
+    echo "=========================================================="
+    echo "Guest package: $GUEST_DIR"
+    echo "Output binary: $OUTPUT_USER_INSTR."
+    echo ""
+
+    # Compile everything in one command via Docker
+    echo "Compiling..."
+
+    # Compiler flags (using -O0 for faster compilation of large generated files)
+    CFLAGS=(
+        -mcmodel=medany
+        -include stdbool.h
+        -mllvm -enable-misched=false
+    )
+
+    LDFLAGS=(
+        -fuse-ld=lld
+        -static
+    )
+
+    # Link libraries
+    LIBS=(-lm)
+
+    # Include directories
+    INCLUDES=(
+        -I"$GUEST_DIR"
+        -Iw2c2/embedded
+    )
+
+    # Source files
+    SOURCES=(
+        platform/riscv-qemu-user/main.c
+        platform/riscv-qemu-user/custom_imports.c
+        "$GUEST_DIR/guest.c"
+        $GUEST_DIR/s0*.c
+        w2c2/embedded/wasi.c
+        w2c2/embedded/wasip2.c
+    )
+
+    echo ""
+    echo "Generate instrumented binary: $OUTPUT_USER_INSTR.instrumented"
+    echo ""
+
+    # Remove object files
+    rm -f $PROJECT_ROOT/custom_imports.o $PROJECT_ROOT/guest.o $PROJECT_ROOT/main.o $PROJECT_ROOT/memlib.o $PROJECT_ROOT/memops.o $PROJECT_ROOT/s00000*.o $PROJECT_ROOT/startup.o $PROJECT_ROOT/wasi.o $PROJECT_ROOT/wasip2.o $PROJECT_ROOT/zkvm.o
+
+
+    parallel -j$(nproc) /opt/riscv-glibc-llvm/bin/clang \
+        -c \
+        -fprofile-instr-generate=$OUTPUT_USER_INSTR.profraw \
+        "${CFLAGS[@]}" \
+        "${INCLUDES[@]}" \
+        ::: \
+        "${SOURCES[@]}" 2>&1
+
+    /opt/riscv-glibc-llvm/bin/clang \
+        -fprofile-instr-generate=$OUTPUT_USER_INSTR.profraw \
+        *.o \
+       "${LDFLAGS[@]}" \
+        "${LIBS[@]}" \
+        -o "$OUTPUT.instrumented" 2>&1
+
+    echo ""
+    echo "Run instrumented binary: $OUTPUT_USER_INSTR.instrumented"
+    echo ""
+
+    qemu-riscv64 "$OUTPUT_USER_INSTR.instrumented"
+
+    /opt/riscv-glibc-llvm/bin/llvm-profdata merge -output=$OUTPUT_USER_INSTR.profdata $OUTPUT_USER_INSTR.profraw
+fi
+
+
 # Create output directory
 mkdir -p "$(dirname "$OUTPUT")"
 
@@ -76,16 +158,26 @@ echo "Compiling..."
 # RISC-V toolchain prefix
 PREFIX=/opt/riscv-newlib/bin/riscv64-unknown-elf-
 
+COMMON_FLAGS=(
+    $OPT_LEVEL
+)
+
+if [ -n "$PGO_OPT" ]; then
+    COMMON_FLAGS+=(
+        -fprofile-instr-use=$OUTPUT_USER_INSTR.profdata
+    )
+fi
+
 # Compiler flags (using -O0 for faster compilation of large generated files)
 CFLAGS=(
     --target=riscv64
     -march=rv64ima_zicsr
     -mabi=lp64
     -mcmodel=medany
-    -specs=nosys.specs
     -D__bool_true_false_are_defined
     -include stdbool.h
-    $OPT_LEVEL
+    -Wprofile-instr-out-of-date
+    -Wno-profile-instr-unprofiled
     --sysroot=/opt/riscv-newlib/riscv64-unknown-elf
     --gcc-toolchain=/opt/riscv-newlib
     -mllvm -enable-misched=false
@@ -127,16 +219,27 @@ LDFLAGS=(
     -static
     -Wl,--gc-sections
     -Wl,-Map="${OUTPUT%.elf}.map"
+    -L/opt/riscv-newlib/lib/gcc/riscv64-unknown-elf/15.2.0
 )
 
 # Link libraries
 LIBS=(-lm -lgcc)
 
-clang \
+# Remove object files
+rm -f $PROJECT_ROOT/custom_imports.o $PROJECT_ROOT/guest.o $PROJECT_ROOT/main.o $PROJECT_ROOT/memlib.o $PROJECT_ROOT/memops.o $PROJECT_ROOT/s00000*.o $PROJECT_ROOT/startup.o $PROJECT_ROOT/wasi.o $PROJECT_ROOT/wasip2.o $PROJECT_ROOT/zkvm.o
+
+parallel -j$(nproc) /opt/riscv-newlib/bin/clang \
+    -c \
     "${CFLAGS[@]}" \
+    "${COMMON_FLAGS[@]}" \
     "${INCLUDES[@]}" \
+    ::: \
     "${SOURCES[@]}" \
-    "${ASM_SOURCES[@]}" \
+    "${ASM_SOURCES[@]}" 2>&1
+
+/opt/riscv-newlib/bin/clang \
+    *.o \
+    "${COMMON_FLAGS[@]}" \
     "${LDFLAGS[@]}" \
     "${LIBS[@]}" \
     -o "$OUTPUT" 2>&1