Enable ccache in CI (#290)

* Remove variant from analysis expressions

* Enable CI on PR creation

* Revert "Enable CI on PR creation"

This reverts commit 335d72f.

* Add more const-correctness

* Use explicit type for conj in to_runtime_data

* Add ccache to CI

* Add clang-tidy-cache

* Enable apt caching in CI again

* Trigger CI

* Put CI packages on one line

* Only install ctcache when analyzing

* Bump CI package version

* Switch to ctcache

* Trigger CI

* Show clang-tidy-cache stats

* Try not recursing submodules

BppTree has a ton of them and I think they're just for testing and
benchmarking.

* Enable ctcache debug

* Use stable version during analysis

* Trigger CI

* Trigger CI

* Trigger CI

* Verbose compile

* Show shell env in job summary

* Disable explicit boost install again

* Show include dir

* Try executing install scripts again

* Trigger CI

* Patch boost files

* Disable boost user config

* Remove boost filesystem dep

* Drop boost filesystem CI dep

* Remove boost filesystem from build docs

* Remove some debug code

* Remove remaining boost usages

* Remove remaining boost references

* Address some analysis issues

Author: jeaye

.github/workflows/build.yml

@@ -58,7 +58,19 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          submodules: recursive
+          submodules: true
+      - uses: awalsh128/[email protected]
+        with:
+          packages: default-jdk software-properties-common lsb-release npm lcov leiningen ccache curl git git-lfs zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build cmake zlib1g-dev libffi-dev libzip-dev libbz2-dev doctest-dev gcc g++ libgc-dev
+          # Increment this when the package list changes.
+          version: 4
+      - name: Cache object files
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ github.workspace }}/compiler+runtime/.ccache
+            ${{ github.workspace }}/compiler+runtime/.ctcache
+          key: ${{ env.JANK_MATRIX_ID }}
       - name: Build and test
         run: |
           curl -sL -o install-bb https://raw.githubusercontent.com/babashka/babashka/master/install

bin/jank/check_everything.clj

@@ -1,7 +1,8 @@
 #!/usr/bin/env bb
 
 (ns jank.check-everything
-  (:require [jank.compiler+runtime.core]
+  (:require [babashka.fs :as b.f]
+            [jank.compiler+runtime.core]
             [jank.clojure-cli.core]
             [jank.lein-jank.core]
             [jank.summary :as summary]
@@ -16,53 +17,52 @@
   (util/log-info "JANK_ANALYZE: " (System/getenv "JANK_ANALYZE"))
   (util/log-info "JANK_SANITIZE: " (System/getenv "JANK_SANITIZE")))
 
-(def os->deps-cmd {"Linux" "sudo apt-get install -y curl git git-lfs zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build cmake zlib1g-dev libffi-dev libzip-dev libbz2-dev doctest-dev libboost-all-dev gcc g++ libgc-dev"
+; Most Linux deps are installed by a Github action. We need to manually install
+; boost for some reason. Otherwise, its headers aren't found by clang.
+(def os->deps-cmd {"Mac OS X" "brew install curl git git-lfs zip entr openssl double-conversion pkg-config ninja python cmake gnupg zlib doctest boost libzip lbzip2 llvm@19"})
 
-                   "Mac OS X" "brew install curl git git-lfs zip entr openssl double-conversion pkg-config ninja python cmake gnupg zlib doctest boost libzip lbzip2 llvm@19"})
-
-; TODO: Cache these deps using https://github.com/actions/cache/
-; Maybe follow this sort of thing: https://github.com/gerlero/apt-install/blob/main/action.yml
 (defmulti install-deps
-  (fn []
+  (fn [_props]
     (System/getProperty "os.name")))
 
-(defmethod install-deps "Linux" []
-  (let [apt? (try
-               (util/quiet-shell {} "which apt-get")
-               true
-               (catch Exception _e
-                 false))]
-    (if-not apt?
-      (util/log-warning "Skipping dependency install, since we don't have apt-get")
-      (do
-        ; Install deps required for running our tests.
-        (util/quiet-shell {} "sudo apt-get update -y")
-        (util/quiet-shell {} "sudo apt-get install -y default-jdk software-properties-common lsb-release npm lcov leiningen")
-        ; TODO: Enable once we're linting Clojure/jank again.
-        ;(util/quiet-shell {} "sudo npm install --global @chrisoakman/standard-clojure-style")
-
-        ; Install jank's build deps.
-        (util/quiet-shell {} (os->deps-cmd "Linux"))
-
-        ; Install Clang/LLVM.
-        (util/quiet-shell {} "curl -L -O https://apt.llvm.org/llvm.sh")
-        (util/quiet-shell {} "chmod +x llvm.sh")
-        (util/quiet-shell {} (str "sudo ./llvm.sh " util/llvm-version " all"))
-        ; The libc++abi headers conflict with the system headers:
-        ; https://github.com/llvm/llvm-project/issues/121300
-        (util/quiet-shell {} (str "sudo apt-get remove -y libc++abi-" util/llvm-version "-dev"))
-
-        ; Install the new Clojure CLI.
-        (util/quiet-shell {} "curl -L -O https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh")
-        (util/quiet-shell {} "chmod +x linux-install.sh")
-        (util/quiet-shell {} "sudo ./linux-install.sh")))))
-
-(defmethod install-deps "Mac OS X" []
+(defmethod install-deps "Linux" [{:keys [validate-formatting?]}]
+  ; TODO: Cache this shit.
+  (when (= "on" (util/get-env "JANK_ANALYZE"))
+    (util/quiet-shell {} "curl -Lo clang-tidy-cache https://raw.githubusercontent.com/matus-chochlik/ctcache/refs/heads/main/src/ctcache/clang_tidy_cache.py")
+    (util/quiet-shell {} "chmod +x clang-tidy-cache")
+    (util/quiet-shell {} "sudo mv clang-tidy-cache /usr/local/bin")
+    (spit "clang-tidy-cache-wrapper"
+          "#!/bin/bash
+           clang-tidy-cache clang-tidy \"${@}\"")
+    (util/quiet-shell {} "chmod +x clang-tidy-cache-wrapper")
+    (util/quiet-shell {} "sudo mv clang-tidy-cache-wrapper /usr/local/bin"))
+
+  ; TODO: Enable once we're linting Clojure/jank again.
+  ;(util/quiet-shell {} "sudo npm install --global @chrisoakman/standard-clojure-style")
+
+  ; Install Clang/LLVM.
+  (util/quiet-shell {} "curl -L -O https://apt.llvm.org/llvm.sh")
+  (util/quiet-shell {} "chmod +x llvm.sh")
+  (util/quiet-shell {} (str "sudo ./llvm.sh " util/llvm-version " all"))
+  ; The libc++abi headers conflict with the system headers:
+  ; https://github.com/llvm/llvm-project/issues/121300
+  (util/quiet-shell {} (str "sudo apt-get remove -y libc++abi-" util/llvm-version "-dev"))
+
+  ; Install the new Clojure CLI.
+  (util/quiet-shell {} "curl -L -O https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh")
+  (util/quiet-shell {} "chmod +x linux-install.sh")
+  (util/quiet-shell {} "sudo ./linux-install.sh"))
+
+(defmethod install-deps "Mac OS X" [_props]
   (util/quiet-shell {:extra-env {"HOMEBREW_NO_AUTO_UPDATE" "1"}}
-                    (os->deps-cmd "Mac OS X")))
+                    (os->deps-cmd "Mac OS X"))
+
+  ; TODO: This is missing some of the other things above.
+  )
 
 (defn -main [{:keys [install-deps? validate-formatting? compiler+runtime
-                     clojure-cli lein-jank]}]
+                     clojure-cli lein-jank]
+              :as props}]
   (summary/initialize)
 
   (util/log-boundary "Show environment")
@@ -72,7 +72,7 @@
   (if-not install-deps?
     (util/log-info "Not enabled")
     (util/with-elapsed-time duration
-      (install-deps)
+      (install-deps props)
       (util/log-info-with-time duration "Dependencies installed")))
 
   (jank.compiler+runtime.core/-main {:validate-formatting? validate-formatting?

bin/jank/summary.clj

@@ -31,9 +31,15 @@
   [s]
   (clojure.string/replace s #"\x1B\[[0-9;]*[mK]" ""))
 
-(defn shell [success? cmd out]
+(defn shell [success? props cmd out]
   (append-line "\n<details>")
   (append-line (str "<summary>" (if success? "✓" "❌") " " cmd "</summary>\n"))
+  (when-some [extra-env (:extra-env props)]
+    (append-line "| Environment variable | Value |")
+    (append-line "| --- | --- |")
+    (doseq [[k v] extra-env]
+      (append-line (str "| " k " | " v " |")))
+    (append-line ""))
   (append-line "```text")
   (append-line (strip-ansi-codes out))
   (append-line "```")

bin/jank/util.clj

@@ -69,10 +69,10 @@
       (do
         (log-error "Failed to run command " cmd)
         (println (:out proc))
-        (summary/shell false cmd (:out proc))
+        (summary/shell false props cmd (:out proc))
         (System/exit 1))
       (do
-        (summary/shell true cmd (:out proc))
+        (summary/shell true props cmd (:out proc))
         proc))))
 
 (defmacro with-elapsed-time

compiler+runtime/CMakeLists.txt

@@ -163,6 +163,7 @@ add_library(
   src/cpp/jank/util/escape.cpp
   src/cpp/jank/util/clang_format.cpp
   src/cpp/jank/util/string_builder.cpp
+  src/cpp/jank/util/string.cpp
   src/cpp/jank/profile/time.cpp
   src/cpp/jank/ui/highlight.cpp
   src/cpp/jank/error.cpp
@@ -286,6 +287,7 @@ target_include_directories(
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/fmt/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/ftxui/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/libzippp/src>"
+  "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/boost-preprocessor/include>"
 )
 
 set_property(TARGET jank_lib PROPERTY OUTPUT_NAME jank)
@@ -302,7 +304,6 @@ include(cmake/dependency/ftxui.cmake)
 include(cmake/dependency/libzippp.cmake)
 
 find_package(OpenSSL REQUIRED COMPONENTS Crypto)
-find_package(Boost CONFIG REQUIRED COMPONENTS filesystem)
 find_package(BDWgc REQUIRED)
 
 target_link_libraries(
@@ -314,7 +315,6 @@ target_link_libraries(
   clang
   clang-cpp
   LLVM
-  Boost::filesystem
   ftxui::screen ftxui::dom
   OpenSSL::Crypto
 )
@@ -453,7 +453,6 @@ target_link_libraries(
   ${jank_link_whole_start} nanobench_lib ${jank_link_whole_end}
   folly_lib
   fmt::fmt
-  Boost::filesystem
 )
 
 jank_hook_llvm(jank_exe)
@@ -503,7 +502,6 @@ if(jank_tests)
     ${jank_link_whole_start} nanobench_lib ${jank_link_whole_end}
     folly_lib
     doctest::doctest
-    Boost::filesystem
   )
 
   jank_hook_llvm(jank_test_exe)

compiler+runtime/bin/jank/compiler+runtime/build+test.clj

@@ -17,15 +17,33 @@
     (util/log-info "Not enabled")
     (let [clang (util/find-llvm-tool "clang")
           clang++ (util/find-llvm-tool "clang++")
-          exports {"CC" clang
-                   "CXX" clang++}
+          exports (merge {"CC" clang
+                          "CXX" clang++
+                          "CCACHE_BASEDIR" compiler+runtime-dir
+                          "CCACHE_DIR" (str compiler+runtime-dir "/.ccache")
+                          "CCACHE_COMPRESS" "true"
+                          "CCACHE_MAXSIZE" "1G"
+                          "CTCACHE_DIR" (str compiler+runtime-dir "/.ctcache")})
           configure-flags ["-GNinja"
                            "-Djank_tests=on"
                            (str "-DCMAKE_BUILD_TYPE=" build-type)
                            (str "-Djank_analyze=" analyze)
                            (str "-Djank_sanitize=" sanitize)
                            (str "-Djank_coverage=" coverage)]
-          configure-cmd (str "./bin/configure " (clojure.string/join " " configure-flags))]
+          configure-flags (cond-> configure-flags
+                            (not= "on" analyze)
+                            (conj "-DCMAKE_C_COMPILER_LAUNCHER=ccache"
+                                  "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache")
+                            (= "on" analyze)
+                            (conj "-DCMAKE_CXX_CLANG_TIDY=clang-tidy-cache-wrapper"))
+          configure-cmd (str "./bin/configure " (clojure.string/join " " configure-flags))
+          stats-cmd (if (= "on" analyze)
+                      "clang-tidy-cache"
+                      "ccache")]
+      (util/quiet-shell {:dir compiler+runtime-dir
+                         :extra-env exports}
+                        (str stats-cmd " --zero-stats"))
+
       (util/quiet-shell {:dir compiler+runtime-dir
                          :extra-env exports}
                         configure-cmd)
@@ -34,9 +52,13 @@
       (util/with-elapsed-time duration
         (util/quiet-shell {:dir compiler+runtime-dir
                            :extra-env exports}
-                          "./bin/compile")
+                          "./bin/compile -v")
         (util/log-info-with-time duration "Compiled"))
 
+      (util/quiet-shell {:dir compiler+runtime-dir
+                         :extra-env exports}
+                        (str stats-cmd " --show-stats"))
+
       (util/with-elapsed-time duration
         (util/quiet-shell {:dir compiler+runtime-dir
                            :extra-env (merge exports

compiler+runtime/cmake/analyze.cmake

@@ -8,4 +8,7 @@ if(jank_analyze)
       set(CMAKE_CXX_CLANG_TIDY clang-tidy --use-color)
     endif()
   endif()
+
+  # Use a stable version name for analysis builds, to enable caching.
+  set(jank_version "jank-${CMAKE_PROJECT_VERSION}-analyze")
 endif()

compiler+runtime/doc/build.md

@@ -6,13 +6,13 @@ scripts to compile your own. See the section on compiling Clang/LLVM below.
 For Debian-based distros, this should be all you need:
 
 ```bash
-sudo apt-get install -y curl git git-lfs zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build cmake zlib1g-dev libffi-dev clang libclang-dev llvm llvm-dev libzip-dev libbz2-dev doctest-dev libboost-all-dev gcc g++ libgc-dev
+sudo apt-get install -y curl git git-lfs zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build cmake zlib1g-dev libffi-dev clang libclang-dev llvm llvm-dev libzip-dev libbz2-dev doctest-dev gcc g++ libgc-dev
 ```
 
 For Arch:
 
 ```bash
-sudo pacman -S git git-lfs clang llvm llvm-libs pkg-config cmake ninja make python3 libffi entr doctest boost libzip lbzip2 gc
+sudo pacman -S git git-lfs clang llvm llvm-libs pkg-config cmake ninja make python3 libffi entr doctest libzip lbzip2 gc
 ```
 
 For Nix:
@@ -27,7 +27,7 @@ nix develop ".#"
 For macOS:
 
 ```bash
-brew install curl git git-lfs zip entr openssl double-conversion pkg-config ninja python cmake gnupg zlib doctest boost libzip lbzip2 llvm@19 bdw-gc
+brew install curl git git-lfs zip entr openssl double-conversion pkg-config ninja python cmake gnupg zlib doctest libzip lbzip2 llvm@19 bdw-gc
 
 # Ensure you have this set up in your shell.
 export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)

compiler+runtime/include/cpp/jank/detail/to_runtime_data.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <boost/filesystem/path.hpp>
+#include <filesystem>
 
 #include <jank/runtime/obj/symbol.hpp>
 #include <jank/runtime/obj/persistent_array_map.hpp>
@@ -45,7 +45,7 @@ namespace jank::detail
     return make_box<obj::symbol>(d);
   }
 
-  inline object_ptr to_runtime_data(boost::filesystem::path const &p)
+  inline object_ptr to_runtime_data(std::filesystem::path const &p)
   {
     return make_box(p.string());
   }

compiler+runtime/include/cpp/jank/jit/processor.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <boost/filesystem/path.hpp>
+#include <filesystem>
 #include <memory>
 
 #include <clang/Interpreter/Interpreter.h>
@@ -57,6 +57,6 @@ namespace jank::jit
 
     std::unique_ptr<clang::Interpreter> interpreter;
     native_integer optimization_level{};
-    native_vector<boost::filesystem::path> library_dirs;
+    native_vector<std::filesystem::path> library_dirs;
   };
 }

compiler+runtime/include/cpp/jank/option.hpp

@@ -2,7 +2,6 @@
 
 #include <cassert>
 #include <utility> // move, forward
-#include <type_traits>
 
 #include <jank/type.hpp>
 
@@ -52,17 +51,16 @@ namespace jank
     }
 
     template <typename D = T>
-    constexpr option(D &&d,
-                     std::enable_if_t<std::is_constructible_v<T, D>
-                                      && !std::is_same_v<std::decay_t<D>, option<T>>> * = nullptr)
+    requires(std::is_constructible_v<T, D> && !std::is_same_v<std::decay_t<D>, option<T>>)
+    constexpr option(D &&d)
       : set{ true }
     {
       new(reinterpret_cast<T *>(data)) T{ std::forward<D>(d) };
     }
 
     template <typename D>
-    constexpr option(option<D> const &o,
-                     std::enable_if_t<std::is_constructible_v<T, D>> * = nullptr)
+    requires(std::is_constructible_v<T, D>)
+    constexpr option(option<D> const &o)
       : set{ o.set }
     {
       if(set)

compiler+runtime/include/cpp/jank/prelude.hpp

@@ -2,7 +2,7 @@
 
 #include <random>
 #include <chrono>
-//#include <cassert>
+#include <unordered_map>
 
 #include <gc/gc.h>
 #include <gc/gc_cpp.h>
@@ -12,8 +12,6 @@
 #include <immer/heap/heap_policy.hpp>
 #include <immer/memory_policy.hpp>
 
-#include <boost/unordered_map.hpp>
-
 #include <nanobench.h>
 
 #include <jank/type.hpp>