BinaryCacheStore: Allow using NARs from the local NAR cache

src/libstore/binary-cache-store.cc

@@ -13,6 +13,7 @@
 #include "nix/util/callback.hh"
 #include "nix/util/signals.hh"
 #include "nix/util/archive.hh"
+#include "nix/store/nar-cache.hh"
 
 #include <chrono>
 #include <future>
@@ -26,6 +27,7 @@ namespace nix {
 
 BinaryCacheStore::BinaryCacheStore(Config & config)
     : config{config}
+    , narCache{config.localNarCache.get().empty() ? nullptr : std::make_shared<NarCache>(config.localNarCache.get())}
 {
     if (config.secretKeyFile != "")
         signers.push_back(std::make_unique<LocalSigner>(SecretKey{readFile(config.secretKeyFile)}));
@@ -408,10 +410,36 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 {
     auto info = queryPathInfo(storePath).cast<const NarInfo>();
 
-    LengthSink narSize;
-    TeeSink tee{sink, narSize};
+    if (narCache) {
+        if (auto nar = narCache->getNar(info->narHash)) {
+            notice("substituted '%s' from local NAR cache", printStorePath(storePath));
+            sink(*nar);
+            stats.narRead++;
+            stats.narReadBytes += nar->size();
+            return;
+        }
+    }
+
+    std::unique_ptr<Sink> narCacheSink;
+    if (narCache)
+        narCacheSink = sourceToSink([&](Source & source) { narCache->upsertNar(info->narHash, source); });
+
+    uint64_t narSize = 0;
+
+    LambdaSink uncompressedSink{
+        [&](std::string_view data) {
+            narSize += data.size();
+            if (narCacheSink)
+                (*narCacheSink)(data);
+            sink(data);
+        },
+        [&]() {
+            stats.narRead++;
+            // stats.narReadCompressedBytes += nar->size(); // FIXME
+            stats.narReadBytes += narSize;
+        }};
 
-    auto decompressor = makeDecompressionSink(info->compression, tee);
+    auto decompressor = makeDecompressionSink(info->compression, uncompressedSink);
 
     try {
         getFile(info->url, *decompressor);
@@ -421,9 +449,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink)
 
     decompressor->finish();
 
-    stats.narRead++;
-    // stats.narReadCompressedBytes += nar->size(); // FIXME
-    stats.narReadBytes += narSize.length;
+    // Note: don't do anything here because it's never reached if we're called as a coroutine.
 }
 
 void BinaryCacheStore::queryPathInfoUncached(
@@ -541,7 +567,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info)
 
 ref<RemoteFSAccessor> BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath)
 {
-    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, config.localNarCache);
+    return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), requireValidPath, narCache);
 }
 
 ref<SourceAccessor> BinaryCacheStore::getFSAccessor(bool requireValidPath)

src/libstore/include/nix/store/binary-cache-store.hh

@@ -12,6 +12,7 @@
 namespace nix {
 
 struct NarInfo;
+class NarCache;
 class RemoteFSAccessor;
 
 struct BinaryCacheStoreConfig : virtual StoreConfig
@@ -38,7 +39,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
     const Setting<std::string> secretKeyFiles{
         this, "", "secret-keys", "List of comma-separated paths to the secret keys used to sign the binary cache."};
 
-    const Setting<Path> localNarCache{
+    const Setting<std::filesystem::path> localNarCache{
         this,
         "",
         "local-nar-cache",
@@ -85,6 +86,8 @@ protected:
 
     const std::string cacheInfoFile = "nix-cache-info";
 
+    std::shared_ptr<NarCache> narCache;
+
     BinaryCacheStore(Config &);
 
 public:

src/libstore/include/nix/store/meson.build

@@ -56,6 +56,7 @@ headers = [ config_pub_h ] + files(
   'make-content-addressed.hh',
   'names.hh',
   'nar-accessor.hh',
+  'nar-cache.hh',
   'nar-info-disk-cache.hh',
   'nar-info.hh',
   'outputs-spec.hh',

src/libstore/include/nix/store/nar-cache.hh

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "nix/util/hash.hh"
+
+#include <filesystem>
+
+namespace nix {
+
+class NarCache
+{
+
+    const std::filesystem::path cacheDir;
+
+    std::filesystem::path makeCacheFile(const Hash & narHash, const std::string & ext);
+
+public:
+
+    NarCache(std::filesystem::path cacheDir);
+
+    void upsertNar(const Hash & narHash, Source & source);
+
+    void upsertNarListing(const Hash & narHash, std::string_view narListingData);
+
+    // FIXME: use a sink.
+    std::optional<std::string> getNar(const Hash & narHash);
+
+    // FIXME: use a sink.
+    std::string getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length);
+
+    std::optional<std::string> getNarListing(const Hash & narHash);
+};
+
+} // namespace nix

src/libstore/include/nix/store/remote-fs-accessor.hh

@@ -7,6 +7,8 @@
 
 namespace nix {
 
+struct NarCache;
+
 class RemoteFSAccessor : public SourceAccessor
 {
     ref<Store> store;
@@ -15,25 +17,20 @@ class RemoteFSAccessor : public SourceAccessor
 
     bool requireValidPath;
 
-    Path cacheDir;
+    std::shared_ptr<NarCache> narCache;
 
     std::pair<ref<SourceAccessor>, CanonPath> fetch(const CanonPath & path);
 
     friend struct BinaryCacheStore;
 
-    Path makeCacheFile(std::string_view hashPart, const std::string & ext);
-
-    ref<SourceAccessor> addToCache(std::string_view hashPart, std::string && nar);
-
 public:
 
     /**
      * @return nullptr if the store does not contain any object at that path.
      */
     std::shared_ptr<SourceAccessor> accessObject(const StorePath & path);
 
-    RemoteFSAccessor(
-        ref<Store> store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = "");
+    RemoteFSAccessor(ref<Store> store, bool requireValidPath = true, std::shared_ptr<NarCache> narCache = {});
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override;
 

src/libstore/meson.build

@@ -345,6 +345,7 @@ sources = files(
   'misc.cc',
   'names.cc',
   'nar-accessor.cc',
+  'nar-cache.cc',
   'nar-info-disk-cache.cc',
   'nar-info.cc',
   'optimise-store.cc',

src/libstore/nar-cache.cc

@@ -0,0 +1,81 @@
+#include "nix/store/nar-cache.hh"
+#include "nix/util/file-system.hh"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+namespace nix {
+
+NarCache::NarCache(std::filesystem::path cacheDir_)
+    : cacheDir(std::move(cacheDir_))
+{
+    assert(!cacheDir.empty());
+    createDirs(cacheDir);
+}
+
+std::filesystem::path NarCache::makeCacheFile(const Hash & narHash, const std::string & ext)
+{
+    return (cacheDir / narHash.to_string(HashFormat::Nix32, false)) + "." + ext;
+}
+
+void NarCache::upsertNar(const Hash & narHash, Source & source)
+{
+    try {
+        /* FIXME: do this asynchronously. */
+        writeFile(makeCacheFile(narHash, "nar"), source);
+    } catch (SystemError &) {
+        ignoreExceptionExceptInterrupt();
+    }
+}
+
+void NarCache::upsertNarListing(const Hash & narHash, std::string_view narListingData)
+{
+    try {
+        writeFile(makeCacheFile(narHash, "ls"), narListingData);
+    } catch (SystemError &) {
+        ignoreExceptionExceptInterrupt();
+    }
+}
+
+std::optional<std::string> NarCache::getNar(const Hash & narHash)
+{
+    try {
+        return nix::readFile(makeCacheFile(narHash, "nar"));
+    } catch (SystemError &) {
+        return std::nullopt;
+    }
+}
+
+std::string NarCache::getNarBytes(const Hash & narHash, uint64_t offset, uint64_t length)
+{
+    auto cacheFile = makeCacheFile(narHash, "nar");
+
+    AutoCloseFD fd = toDescriptor(open(
+        cacheFile.c_str(),
+        O_RDONLY
+#ifndef _WIN32
+            | O_CLOEXEC
+#endif
+        ));
+    if (!fd)
+        throw SysError("opening NAR cache file %s", cacheFile);
+
+    if (lseek(fromDescriptorReadOnly(fd.get()), offset, SEEK_SET) != (off_t) offset)
+        throw SysError("seeking in %s", cacheFile);
+
+    std::string buf(length, 0);
+    readFull(fd.get(), buf.data(), length);
+    return buf;
+}
+
+std::optional<std::string> NarCache::getNarListing(const Hash & narHash)
+{
+    try {
+        return nix::readFile(makeCacheFile(narHash, "ls"));
+    } catch (SystemError &) {
+        return std::nullopt;
+    }
+}
+
+} // namespace nix