Add AddressSanitizer to GitHub Actions (#3959)

- Make the `qsort` test work under AddressSanitizer. See the comment added to `qsort_.c` for more details.
- Add a GitHub actions build variant for testing the AddressSanitizer integration

Author: Svetlitski

.github/workflows/build.yml

@@ -175,6 +175,11 @@ jobs:
             ocamlparam: '_,w=-46,regalloc=cfg,vectorize=1'
             check_arch: true
 
+          - name: address_sanitizer
+            config: --enable-middle-end=flambda2 --enable-address-sanitizer
+            os: ubuntu-latest
+            cc: clang
+
     env:
       J: "3"
       run_testsuite: "true"
@@ -189,6 +194,10 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: sudo apt-get install afl++
 
+    - name: Install clang
+      if: matrix.os == 'ubuntu-latest' && matrix.cc == 'clang'
+      run: sudo apt-get install clang
+
     - name: Install AFL (for macOS workers)
       # The "afl-fuzz" package is deprecated (2023-10) and can no longer be installed
       if: matrix.os == 'macos-latest'
@@ -280,6 +289,9 @@ jobs:
     - name: Configure Flambda backend
       working-directory: flambda_backend
       run: |
+        if [[ -n "${{matrix.cc}}" ]]; then
+          export CC="${{matrix.cc}}"
+        fi
         autoconf
         ./configure \
           --prefix=$GITHUB_WORKSPACE/_install \

testsuite/tests/frame-pointers/qsort.ml

@@ -1,5 +1,8 @@
 (* TEST
  modules = "qsort_.c";
+ (* [-D_GNU_SOURCE] is needed when AddressSanitizer support is enabled in order
+    for the [dlsym] workaround in [qsort_.c] to work. *)
+ flags = "-ccopt -D_GNU_SOURCE";
  frame_pointers;
  {
    bytecode;

testsuite/tests/frame-pointers/qsort_.c

@@ -1,4 +1,8 @@
 #define CAML_NAME_SPACE
+#include <caml/config.h>
+#ifdef WITH_ADDRESS_SANITIZER
+#include <dlfcn.h>
+#endif
 #include <caml/memory.h>
 #include <caml/mlvalues.h>
 #include <caml/alloc.h>
@@ -71,6 +75,24 @@ static int cmp_callback(const void* p_a, const void* p_b)
   return Long_val(caml_callback2(*cmp_fn, **a, **b));
 }
 
+#ifdef WITH_ADDRESS_SANITIZER
+/* AddressSanitizer provides its own definition of [qsort], which runs the comparator
+   function over the entire array in order to detect memory safety issues [1].
+   That breaks this test, since the output of the test depends on how many times the
+   comparator function is invoked. To work around this, we retrieve the standard [qsort]
+   definition provided by [libc] using [dlsym], and call that directly.
+
+   [1]: https://github.com/llvm/llvm-project/blob/1101b767329dd163d528fa5f667a6c0dbdde0ad5/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc#L10047-L10051
+*/
+void qsort(void *base, size_t nmemb, size_t size, __compar_fn_t compar) {
+  static void (*libc_qsort)(void *, size_t, size_t, __compar_fn_t) = NULL;
+  if (libc_qsort == NULL) {
+    libc_qsort = dlsym(RTLD_NEXT, "qsort");
+  }
+  libc_qsort(base, nmemb, size, compar);
+}
+#endif
+
 value sort2(value cmp_clos, value a, value b)
 {
   CAMLparam3(cmp_clos, a, b);