diff --git a/bin/common.ml b/bin/common.ml index d2f2b612e4a..d955a2fe069 100644 --- a/bin/common.ml +++ b/bin/common.ml @@ -1377,8 +1377,8 @@ let init_with_root ~(root : Workspace_root.t) (builder : Builder.t) = ]; Log.info [ Pp.textf - "Shared cache location: %s" - (Path.to_string (Lazy.force Dune_cache_storage.Layout.root_dir)) + "Shared build cache location: %s" + (Path.to_string (Lazy.force Dune_cache_storage.Layout.build_cache_dir)) ]; Dune_rules.Main.init ~stats:c.stats @@ -1495,8 +1495,8 @@ let envs = ~doc:"If different than $(b,0), ANSI colors should be enabled no matter what." "CLICOLOR_FORCE" ; info + ~doc:"If set, determines the location of all the different caches used by dune." "DUNE_CACHE_ROOT" - ~doc:"If set, determines the location of the machine-global shared cache." ] ;; diff --git a/doc/changes/fixed/11612.md b/doc/changes/fixed/11612.md new file mode 100644 index 00000000000..da426c89c3d --- /dev/null +++ b/doc/changes/fixed/11612.md @@ -0,0 +1,8 @@ +- Fixed non-build caches not following `$DUNE_CACHE_ROOT` and instead only + relying on `$XDG_CACHE_HOME`. + This means the normal build cache moves: + `$DUNE_CACHE_ROOT -> $DUNE_CACHE_ROOT/db` (no changes if that variable was + unset). Affected users can prevent a full cache invalidation by moving + previous contents: + `cd $DUNE_CACHE_ROOT; mkdir db; mv db`. + (#11612, fixes #11584, @ElectreAAS) diff --git a/src/dune_cache_storage/dune_cache_storage.ml b/src/dune_cache_storage/dune_cache_storage.ml index b30776ac141..ea46e0477f7 100644 --- a/src/dune_cache_storage/dune_cache_storage.ml +++ b/src/dune_cache_storage/dune_cache_storage.ml @@ -376,5 +376,5 @@ let clear () = rm_rf (Lazy.force Layout.temp_dir); (* Do not catch errors when deleting the root directory so that they are reported to the user. *) - Path.rmdir (Lazy.force Layout.root_dir) + Path.rmdir (Lazy.force Layout.build_cache_dir) ;; diff --git a/src/dune_cache_storage/layout.ml b/src/dune_cache_storage/layout.ml index ac695019326..4682504c3ae 100644 --- a/src/dune_cache_storage/layout.ml +++ b/src/dune_cache_storage/layout.ml @@ -1,25 +1,9 @@ open Stdune open Import -let default_root_dir = - lazy - (let cache_dir = Xdg.cache_dir (Lazy.force Dune_util.xdg) in - Path.L.relative (Path.of_filename_relative_to_initial_cwd cache_dir) [ "dune"; "db" ]) -;; - -let root_dir = - lazy - (let var = "DUNE_CACHE_ROOT" in - match Sys.getenv_opt var with - | None -> Lazy.force default_root_dir - | Some path -> - if Filename.is_relative path - then failwith (sprintf "%s should be an absolute path, but is %s" var path); - Path.of_filename_relative_to_initial_cwd path) -;; - let ( / ) = Path.relative -let temp_dir = lazy (Lazy.force root_dir / "temp") +let build_cache_dir = lazy (Lazy.force Dune_util.cache_root_dir / "db") +let temp_dir = lazy (Lazy.force build_cache_dir / "temp") let cache_path ~dir ~hex = let two_first_chars = sprintf "%c%c" hex.[0] hex.[1] in @@ -52,13 +36,15 @@ let list_entries ~storage = module Versioned = struct let metadata_storage_dir t = - lazy (Lazy.force root_dir / "meta" / Version.Metadata.to_string t) + lazy (Lazy.force build_cache_dir / "meta" / Version.Metadata.to_string t) ;; - let file_storage_dir t = lazy (Lazy.force root_dir / "files" / Version.File.to_string t) + let file_storage_dir t = + lazy (Lazy.force build_cache_dir / "files" / Version.File.to_string t) + ;; let value_storage_dir t = - lazy (Lazy.force root_dir / "values" / Version.Value.to_string t) + lazy (Lazy.force build_cache_dir / "values" / Version.Value.to_string t) ;; let metadata_path t ~rule_or_action_digest = diff --git a/src/dune_cache_storage/layout.mli b/src/dune_cache_storage/layout.mli index 73798fc94bf..dc5b89df6d9 100644 --- a/src/dune_cache_storage/layout.mli +++ b/src/dune_cache_storage/layout.mli @@ -8,8 +8,9 @@ open Stdune open Import -(** The path to the root directory of the cache. *) -val root_dir : Path.t Lazy.t +(** The directory containing the build cache. + Set to [Dune_util.cache_root_dir/db]. *) +val build_cache_dir : Path.t Lazy.t (** Create a few subdirectories in [root_dir]. We expose this function because we don't want to modify the file system when the cache is disabled. diff --git a/src/dune_pkg/rev_store.ml b/src/dune_pkg/rev_store.ml index a430cdf7201..a80e890af23 100644 --- a/src/dune_pkg/rev_store.ml +++ b/src/dune_pkg/rev_store.ml @@ -145,32 +145,26 @@ module Cache = struct Dune_config.Config.make_toggle ~name:"rev_store_cache" ~default:`Disabled ;; - let cache_dir = + let revision_store_dir = lazy - (let path = - Path.L.relative - (Lazy.force Dune_util.xdg - |> Xdg.cache_dir - |> Path.Outside_build_dir.of_string - |> Path.outside_build_dir) - [ "dune"; "rev_store" ] - in + (let path = Path.relative (Lazy.force Dune_util.cache_root_dir) "rev_store" in let rev_store_cache = Dune_config.Config.get rev_store_cache in Log.info [ Pp.textf "Revision store cache: %s" (Dune_config.Config.Toggle.to_string rev_store_cache) ]; - match rev_store_cache, Path.mkdir_p path with - | `Enabled, () -> + match rev_store_cache with + | `Enabled -> + Path.mkdir_p path; Log.info [ Pp.textf "Revision store cache location: %s" (Path.to_string path) ]; Some path - | `Disabled, () -> None) + | `Disabled -> None) ;; let db = lazy - (Lazy.force cache_dir + (Lazy.force revision_store_dir |> Option.map ~f:(fun path -> Lmdb.Env.create ~map_size:(Int64.to_int 5_000_000_000L) (* 5 GB *) @@ -1219,13 +1213,14 @@ let content_of_files t files = | None -> Cache.Key.Map.find_exn to_write key) ;; +let git_repo_dir = + lazy + (let dir = Path.relative (Lazy.force Dune_util.cache_root_dir) "git-repo" in + Log.info [ Pp.textf "Git repository cache location: %s" (Path.to_string dir) ]; + dir) +;; + let get = - Fiber.Lazy.create (fun () -> - let dir = - Path.L.relative - (Path.of_string (Xdg.cache_dir (Lazy.force Dune_util.xdg))) - [ "dune"; "git-repo" ] - in - load_or_create ~dir) + Fiber.Lazy.create (fun () -> load_or_create ~dir:(Lazy.force git_repo_dir)) |> Fiber.Lazy.force ;; diff --git a/src/dune_rules/pkg_toolchain.ml b/src/dune_rules/pkg_toolchain.ml index b83ac13c771..e112e81bc07 100644 --- a/src/dune_rules/pkg_toolchain.ml +++ b/src/dune_rules/pkg_toolchain.ml @@ -1,21 +1,21 @@ open Import +let base_dir = + lazy + (let dir = Path.relative (Lazy.force Dune_util.cache_root_dir) "toolchains" in + Log.info [ Pp.textf "Toolchains cache location: %s" (Path.to_string dir) ]; + Path.as_outside_build_dir_exn dir) +;; + let base_dir () = - let cache_dir = - Lazy.force Dune_util.xdg |> Xdg.cache_dir |> Path.Outside_build_dir.of_string - in - let path = - Path.Outside_build_dir.relative - (Path.Outside_build_dir.relative cache_dir "dune") - "toolchains" - in - (let path = Path.outside_build_dir path in - if not (Path.Untracked.exists path) then Path.mkdir_p path; - if not (Path.Untracked.is_directory path) - then - User_error.raise - [ Pp.textf "Expected %s to be a directory but it is not." (Path.to_string path) ]); - path + let base_dir = Lazy.force base_dir in + let path = Path.outside_build_dir base_dir in + if not (Path.Untracked.exists path) then Path.mkdir_p path; + if not (Path.Untracked.is_directory path) + then + User_error.raise + [ Pp.textf "Expected %s to be a directory but it is not." (Path.to_string path) ]; + base_dir ;; let pkg_dir (pkg : Dune_pkg.Lock_dir.Pkg.t) = diff --git a/src/dune_rules/pkg_toolchain.mli b/src/dune_rules/pkg_toolchain.mli index 5e22008f982..d6b91169e5c 100644 --- a/src/dune_rules/pkg_toolchain.mli +++ b/src/dune_rules/pkg_toolchain.mli @@ -1,7 +1,8 @@ open Import (** The path to the directory that will contain all toolchain - versions. Creates the directory if it doesn't already exist. *) + versions. Creates the directory if it doesn't already exist. + Set to [Dune_util.cache_home_dir/toolchains]. *) val base_dir : unit -> Path.Outside_build_dir.t (** Dune will download and build the ocaml-base-compiler and @@ -21,7 +22,7 @@ val is_compiler_and_toolchains_enabled : Package.Name.t -> bool (** Returns the path to the directory containing the given package within the toolchain directory. This will be something like - $XDG_CACHE_HOME/dune/toolchains/ocaml-base-compiler.5.2.1.XXXXXXXX where + [base_dir/ocaml-base-compiler.5.2.1.XXXXXXXX] where XXXXXXXX is a hash of the package's lockfile. *) val installation_prefix : Lock_dir.Pkg.t -> Path.Outside_build_dir.t diff --git a/src/dune_util/dune_util.ml b/src/dune_util/dune_util.ml index 98599afa78d..3a5fdd8ed0b 100644 --- a/src/dune_util/dune_util.ml +++ b/src/dune_util/dune_util.ml @@ -1,14 +1,14 @@ +module Action = Action +module Alias_name = Alias_name +module Build_path_prefix_map = Build_path_prefix_map0 +module Gc = Gc +module Global_lock = Global_lock module Persistent = Persistent module Report_error = Report_error module Stringlike = Stringlike module type Stringlike = Stringlike_intf.S -module Build_path_prefix_map = Build_path_prefix_map0 -module Global_lock = Global_lock -module Action = Action -module Alias_name = Alias_name -module Gc = Gc open Stdune let manual_xdg = ref None @@ -37,6 +37,30 @@ let override_xdg : Xdg.t -> unit = else manual_xdg := Some new_xdg ;; +let ( / ) = Path.relative + +(** The default directory of all caches (build and others), used when + [$DUNE_CACHE_ROOT] is unset. + Set to [$XDG_CACHE_HOME/dune]. *) +let default_cache_dir = + lazy + (let cache_dir = Xdg.cache_dir (Lazy.force xdg) in + Path.of_filename_relative_to_initial_cwd cache_dir / "dune") +;; + +let cache_root_dir = + lazy + (let var = "DUNE_CACHE_ROOT" in + match Sys.getenv_opt var with + | Some path -> + if Filename.is_relative path + then + User_error.raise + [ Pp.paragraphf "$%s should be an absolute path, but is %S" var path ]; + Path.external_ (Path.External.of_string path) + | None -> Lazy.force default_cache_dir) +;; + let frames_per_second () = match Dune_config.Config.(get threaded_console_frames_per_second) with | `Custom fps -> fps diff --git a/src/dune_util/dune_util.mli b/src/dune_util/dune_util.mli new file mode 100644 index 00000000000..8f02438fa41 --- /dev/null +++ b/src/dune_util/dune_util.mli @@ -0,0 +1,22 @@ +module Action = Action +module Alias_name = Alias_name +module Build_path_prefix_map = Build_path_prefix_map0 +module Gc = Gc +module Global_lock = Global_lock +module Persistent = Persistent +module Report_error = Report_error +module Stringlike = Stringlike + +module type Stringlike = Stringlike_intf.S + +open Stdune + +val xdg : Xdg.t Lazy.t +val override_xdg : Xdg.t -> unit + +(** The directory containing all caches (build and others). + Set to [$DUNE_CACHE_ROOT] if it exists, or + [$XDG_CACHE_HOME/dune] otherwise. *) +val cache_root_dir : Path.t Lazy.t + +val frames_per_second : unit -> int diff --git a/test/blackbox-tests/test-cases/dune-cache/clear.t b/test/blackbox-tests/test-cases/dune-cache/clear.t index e4cee9b0084..8454141f2ee 100644 --- a/test/blackbox-tests/test-cases/dune-cache/clear.t +++ b/test/blackbox-tests/test-cases/dune-cache/clear.t @@ -13,7 +13,7 @@ Test for the "dune cache clear" command. $ dune build - $ ls $DUNE_CACHE_ROOT | sort -u + $ ls $DUNE_CACHE_ROOT/db | sort -u files meta temp @@ -21,20 +21,20 @@ Test for the "dune cache clear" command. $ dune cache clear - $ ! test -d $DUNE_CACHE_ROOT + $ ! test -d $DUNE_CACHE_ROOT/db Next let us add some extra directories/files and check that they are not deleted by mistake. $ dune build - $ mkdir -p $DUNE_CACHE_ROOT/extra; touch $DUNE_CACHE_ROOT/extra1 $DUNE_CACHE_ROOT/extra/extra2 + $ mkdir -p $DUNE_CACHE_ROOT/db/extra; touch $DUNE_CACHE_ROOT/db/extra1 $DUNE_CACHE_ROOT/db/extra/extra2 $ dune cache clear Error: - rmdir($TESTCASE_ROOT/dune-cache): Directory not empty + rmdir($TESTCASE_ROOT/dune-cache/db): Directory not empty [1] - $ find $DUNE_CACHE_ROOT -type f | sort -u - $TESTCASE_ROOT/dune-cache/extra/extra2 - $TESTCASE_ROOT/dune-cache/extra1 + $ find $DUNE_CACHE_ROOT/db -type f | sort -u + $TESTCASE_ROOT/dune-cache/db/extra/extra2 + $TESTCASE_ROOT/dune-cache/db/extra1 diff --git a/test/blackbox-tests/test-cases/dune-cache/default-cache.t b/test/blackbox-tests/test-cases/dune-cache/default-cache.t index 7404edfc03a..fe774f6fb7f 100644 --- a/test/blackbox-tests/test-cases/dune-cache/default-cache.t +++ b/test/blackbox-tests/test-cases/dune-cache/default-cache.t @@ -26,7 +26,7 @@ Change source files to force a recompilation > let f x y = x - y > EOF $ dune build - $ ls $DUNE_CACHE_ROOT | sort + $ ls $DUNE_CACHE_ROOT/db | sort files meta temp diff --git a/test/blackbox-tests/test-cases/dune-cache/possible-locations.t b/test/blackbox-tests/test-cases/dune-cache/possible-locations.t new file mode 100644 index 00000000000..ca73050d712 --- /dev/null +++ b/test/blackbox-tests/test-cases/dune-cache/possible-locations.t @@ -0,0 +1,35 @@ +Showcase all possible locations of the cache. + + $ echo "(lang dune 3.17)" > dune-project + + $ cat > dune << EOF + > (library + > (name foo)) + > EOF + + $ cat > foo.ml << EOF + > let f x y = x + y + > EOF + +Populate the different cache locations. +- Without any configuration + $ dune build + $ dune_cmd exists ~/.cache/dune/db + true + +- With XDG standard config + $ XDG_CACHE_HOME=$(pwd)/a dune build --force + $ dune_cmd exists $(pwd)/a/dune/db + true + +- With dune-specific config + $ DUNE_CACHE_ROOT=$(pwd)/b dune build --force + $ dune_cmd exists $(pwd)/b/db + true + +- With both of them, only the latter is used + $ XDG_CACHE_HOME=$(pwd)/c DUNE_CACHE_ROOT=$(pwd)/d dune build --force + $ dune_cmd exists $(pwd)/c/dune/db + false + $ dune_cmd exists $(pwd)/d/db + true diff --git a/test/blackbox-tests/test-cases/dune-cache/readonly-fs.t b/test/blackbox-tests/test-cases/dune-cache/readonly-fs.t index deddcbb01fe..28f836eba3a 100644 --- a/test/blackbox-tests/test-cases/dune-cache/readonly-fs.t +++ b/test/blackbox-tests/test-cases/dune-cache/readonly-fs.t @@ -20,7 +20,7 @@ where Dune is supposed to store the cache: Warning: Cache directories could not be created: Permission denied; disabling cache Hint: Make sure the directory - $TESTCASE_ROOT/readonly/cache-dir/temp + $TESTCASE_ROOT/readonly/cache-dir/db/temp can be created Likewise, this should also happen if the location is set via XDG variables. diff --git a/test/blackbox-tests/test-cases/dune-cache/repro-check.t b/test/blackbox-tests/test-cases/dune-cache/repro-check.t index 3b5c1179717..019eb996a95 100644 --- a/test/blackbox-tests/test-cases/dune-cache/repro-check.t +++ b/test/blackbox-tests/test-cases/dune-cache/repro-check.t @@ -76,10 +76,10 @@ Set 'cache-check-probability' to 1.0, which should trigger the check Check that the reported digests make sense - $ dune_cmd cat $DUNE_CACHE_ROOT/files/v4/73/7378fb2d7d80dc4468d6558d864f0897 + $ dune_cmd cat $DUNE_CACHE_ROOT/db/files/v4/73/7378fb2d7d80dc4468d6558d864f0897 old-content - $ dune_cmd cat $DUNE_CACHE_ROOT/files/v4/074/074ebdc1c3853f27c68566d8d183032c - Fatal error: exception Unix.Unix_error(Unix.ENOENT, "open", "$TESTCASE_ROOT/.cache/files/v4/074/074ebdc1c3853f27c68566d8d183032c") + $ dune_cmd cat $DUNE_CACHE_ROOT/db/files/v4/074/074ebdc1c3853f27c68566d8d183032c + Fatal error: exception Unix.Unix_error(Unix.ENOENT, "open", "$TESTCASE_ROOT/.cache/db/files/v4/074/074ebdc1c3853f27c68566d8d183032c") [2] Check that probability values less than zero and greater than one are rejected diff --git a/test/blackbox-tests/test-cases/dune-cache/symlink.t b/test/blackbox-tests/test-cases/dune-cache/symlink.t index 485b46a31b7..6effe5504a8 100644 --- a/test/blackbox-tests/test-cases/dune-cache/symlink.t +++ b/test/blackbox-tests/test-cases/dune-cache/symlink.t @@ -24,11 +24,11 @@ produced symbolic links work correctly and are appropriately cached. Dune cache contains entries for [source] and [target] but not for [link] - $ (cd "$DUNE_CACHE_ROOT/meta/v5"; grep -rs . -e 'source' | dune_cmd count-lines) + $ (cd "$DUNE_CACHE_ROOT/db/meta/v5"; grep -rs . -e 'source' | dune_cmd count-lines) 1 - $ (cd "$DUNE_CACHE_ROOT/meta/v5"; grep -rs . -e 'target' | dune_cmd count-lines) + $ (cd "$DUNE_CACHE_ROOT/db/meta/v5"; grep -rs . -e 'target' | dune_cmd count-lines) 1 - $ (cd "$DUNE_CACHE_ROOT/meta/v5"; grep -rs . -e 'link' | dune_cmd count-lines) + $ (cd "$DUNE_CACHE_ROOT/db/meta/v5"; grep -rs . -e 'link' | dune_cmd count-lines) 0 The files in the build directory are shared with the cache entries diff --git a/test/blackbox-tests/test-cases/pkg/fetch-cache.t b/test/blackbox-tests/test-cases/pkg/fetch-cache.t index dd5a8239510..6dde35f9162 100644 --- a/test/blackbox-tests/test-cases/pkg/fetch-cache.t +++ b/test/blackbox-tests/test-cases/pkg/fetch-cache.t @@ -37,7 +37,7 @@ disabling the download of the source a second time. Make sure that the file that was fetched is in the cache: - $ find $DUNE_CACHE_ROOT/files -type f -exec md5sum {} \; | grep --quiet $CONTENT_CHECKSUM + $ find $DUNE_CACHE_ROOT/db/files -type f -exec md5sum {} \; | grep --quiet $CONTENT_CHECKSUM Cleaning the project to force rebuilding. If we attempt to build without the cache, it will fail, as the source is 404 now: diff --git a/test/blackbox-tests/test-cases/pkg/toolchain-installation.t b/test/blackbox-tests/test-cases/pkg/toolchain-installation.t index 4978e700cc4..a84aff7394e 100644 --- a/test/blackbox-tests/test-cases/pkg/toolchain-installation.t +++ b/test/blackbox-tests/test-cases/pkg/toolchain-installation.t @@ -76,8 +76,9 @@ name so the output is consistent across test runs. > sed 's/\(ocaml-base-compiler.1-\)[^/]*/\1HASH/' > } -Attempt to build the project. This will fail due to the fake compiler +Attempt to build the project. This will fail due to the fake compiler, but the fake compiler will end up installed as a toolchain package. +Also test that XDG_CACHE_HOME is respected. $ XDG_CACHE_HOME=$PWD/fake-cache DUNE_CONFIG__TOOLCHAINS=enabled build_pkg ocaml-base-compiler 2>&1 | remove_hash Enumerate the contents of the fake toolchains directory: @@ -87,3 +88,14 @@ Enumerate the contents of the fake toolchains directory: fake-cache/dune/toolchains/ocaml-base-compiler.1-HASH/target fake-cache/dune/toolchains/ocaml-base-compiler.1-HASH/target/bin fake-cache/dune/toolchains/ocaml-base-compiler.1-HASH/target/bin/ocamlc + +Also test that DUNE_CACHE_ROOT is respected. + $ DUNE_CACHE_ROOT=$PWD/other-fake-cache DUNE_CONFIG__TOOLCHAINS=enabled build_pkg ocaml-base-compiler 2>&1 | remove_hash + +Enumerate the contents of the fake toolchains directory: + $ find other-fake-cache/toolchains/ | sort | remove_hash + other-fake-cache/toolchains/ + other-fake-cache/toolchains/ocaml-base-compiler.1-HASH + other-fake-cache/toolchains/ocaml-base-compiler.1-HASH/target + other-fake-cache/toolchains/ocaml-base-compiler.1-HASH/target/bin + other-fake-cache/toolchains/ocaml-base-compiler.1-HASH/target/bin/ocamlc